From patchwork Tue Jul 18 07:06:12 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Baolin Wang X-Patchwork-Id: 9847073 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id 754B9602C8 for ; Tue, 18 Jul 2017 07:12:13 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 67DBD28517 for ; Tue, 18 Jul 2017 07:12:13 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 5C87A2854E; Tue, 18 Jul 2017 07:12:13 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=2.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id BE14628517 for ; Tue, 18 Jul 2017 07:12:10 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751353AbdGRHMK (ORCPT ); Tue, 18 Jul 2017 03:12:10 -0400 Received: from sci-ig2.spreadtrum.com ([222.66.158.135]:19694 "EHLO SHSQR01.spreadtrum.com" rhost-flags-OK-FAIL-OK-OK) by vger.kernel.org with ESMTP id S1751026AbdGRHMJ (ORCPT ); Tue, 18 Jul 2017 03:12:09 -0400 Received: from ig2.spreadtrum.com (shcas01.spreadtrum.com [10.0.1.201]) by SHSQR01.spreadtrum.com with ESMTP id v6I79twl099373 (version=TLSv1/SSLv3 cipher=AES256-SHA bits=256 verify=NO); Tue, 18 Jul 2017 15:09:55 +0800 (CST) (envelope-from Orson.Zhai@spreadtrum.com) Received: from SHCAS01.spreadtrum.com (10.0.1.201) by SHMBX01.spreadtrum.com (10.0.1.203) with Microsoft SMTP Server (TLS) id 15.0.847.32; Tue, 18 Jul 2017 15:09:54 +0800 Received: from localhost (10.0.73.143) by SHCAS01.spreadtrum.com (10.0.1.250) with Microsoft SMTP Server (TLS) id 15.0.847.32 via Frontend Transport; Tue, 18 Jul 2017 15:09:53 +0800 From: Baolin Wang To: , , , CC: , , , , , Subject: [PATCH 2/2] dma: Add Spreadtrum DMA controller driver Date: Tue, 18 Jul 2017 15:06:12 +0800 Message-ID: <4331ea088a26f515fe1f1912c568bfd07d539e9d.1500361078.git.baolin.wang@spreadtrum.com> X-Mailer: git-send-email 2.12.2 In-Reply-To: <63f2b93b3e484862f736e37d6d422b5566637163.1500361078.git.baolin.wang@spreadtrum.com> References: <63f2b93b3e484862f736e37d6d422b5566637163.1500361078.git.baolin.wang@spreadtrum.com> MIME-Version: 1.0 X-MAIL: SHSQR01.spreadtrum.com v6I79twl099373 Sender: dmaengine-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: dmaengine@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch adds the DMA controller driver for Spreadtrum SC9860 platform. Signed-off-by: Baolin Wang --- drivers/dma/Kconfig | 7 + drivers/dma/Makefile | 1 + drivers/dma/sprd-dma.c | 1451 ++++++++++++++++++++++++++++++++++++++++++ include/linux/dma/sprd-dma.h | 270 ++++++++ 4 files changed, 1729 insertions(+) create mode 100644 drivers/dma/sprd-dma.c create mode 100644 include/linux/dma/sprd-dma.h diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index fa8f9c0..961f6ea 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -477,6 +477,13 @@ config STM32_DMA If you have a board based on such a MCU and wish to use DMA say Y here. +config SPRD_DMA + bool "Spreadtrum DMA support" + depends on ARCH_SPRD + select DMA_ENGINE + help + Enable support for the on-chip DMA controller on Spreadtrum platform. + config S3C24XX_DMAC bool "Samsung S3C24XX DMA support" depends on ARCH_S3C24XX || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index d12ab29..0fee561 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -58,6 +58,7 @@ obj-$(CONFIG_RENESAS_DMA) += sh/ obj-$(CONFIG_SIRF_DMA) += sirf-dma.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_STM32_DMA) += stm32-dma.o +obj-$(CONFIG_SPRD_DMA) += sprd-dma.o obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c new file mode 100644 index 0000000..64eaef7 --- /dev/null +++ b/drivers/dma/sprd-dma.c @@ -0,0 +1,1451 @@ +/* + * Copyright (C) 2017 Spreadtrum Communications Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" + +#define SPRD_DMA_DESCRIPTORS 16 +#define SPRD_DMA_CFG_COUNT 32 +#define SPRD_DMA_CHN_REG_OFFSET 0x1000 +#define SPRD_DMA_CHN_REG_LENGTH 0x40 +#define SPRD_DMA_MEMCPY_MIN_SIZE 64 + +/* DMA global registers definition */ +#define DMA_GLB_PAUSE 0x0 +#define DMA_GLB_FRAG_WAIT 0x4 +#define DMA_GLB_REQ_PEND0_EN 0x8 +#define DMA_GLB_REQ_PEND1_EN 0xc +#define DMA_GLB_INT_RAW_STS 0x10 +#define DMA_GLB_INT_MSK_STS 0x14 +#define DMA_GLB_REQ_STS 0x18 +#define DMA_GLB_CHN_EN_STS 0x1c +#define DMA_GLB_DEBUG_STS 0x20 +#define DMA_GLB_ARB_SEL_STS 0x24 +#define DMA_GLB_CHN_START_CHN_CFG1 0x28 +#define DMA_GLB_CHN_START_CHN_CFG2 0x2c +#define DMA_CHN_LLIST_OFFSET 0x10 +#define DMA_GLB_REQ_CID(base, uid) \ + ((unsigned long)(base) + 0x2000 + 0x4 * ((uid) - 1)) + +/* DMA_GLB_CHN_START_CHN_CFG register definition */ +#define SRC_CHN_OFFSET 0 +#define DST_CHN_OFFSET 8 +#define START_MODE_OFFSET 16 +#define CHN_START_CHN BIT(24) + +/* DMA channel registers definition */ +#define DMA_CHN_PAUSE 0x0 +#define DMA_CHN_REQ 0x4 +#define DMA_CHN_CFG 0x8 +#define DMA_CHN_INTC 0xc +#define DMA_CHN_SRC_ADDR 0x10 +#define DMA_CHN_DES_ADDR 0x14 +#define DMA_CHN_FRG_LEN 0x18 +#define DMA_CHN_BLK_LEN 0x1c +#define DMA_CHN_TRSC_LEN 0x20 +#define DMA_CHN_TRSF_STEP 0x24 +#define DMA_CHN_WARP_PTR 0x28 +#define DMA_CHN_WARP_TO 0x2c +#define DMA_CHN_LLIST_PTR 0x30 +#define DMA_CHN_FRAG_STEP 0x34 +#define DMA_CHN_SRC_BLK_STEP 0x38 +#define DMA_CHN_DES_BLK_STEP 0x3c + +/* DMA_CHN_INTC register definition */ +#define DMA_CHN_INT_MASK GENMASK(4, 0) +#define DMA_CHN_INT_CLR_OFFSET 24 +#define FRAG_INT_EN BIT(0) +#define BLK_INT_EN BIT(1) +#define TRANS_INT_EN BIT(2) +#define LIST_INT_EN BIT(3) +#define CFG_ERROR_INT_EN BIT(4) + +/* DMA_CHN_CFG register definition */ +#define DMA_CHN_EN BIT(0) +#define DMA_CHN_PRIORITY_OFFSET 12 +#define LLIST_EN_OFFSET 4 +#define CHN_WAIT_BDONE 24 +#define DMA_DONOT_WAIT_BDONE 1 + +/* DMA_CHN_REQ register definition */ +#define DMA_CHN_REQ_EN BIT(0) + +/* DMA_CHN_PAUSE register definition */ +#define DMA_CHN_PAUSE_EN BIT(0) +#define DMA_CHN_PAUSE_STS BIT(2) +#define DMA_CHN_PAUSE_CNT 0x2000 + +/* DMA_CHN_WARP_* register definition */ +#define DMA_CHN_ADDR_MASK GENMASK(31, 28) +#define DMA_CHN_HIGH_ADDR_OFFSET 4 +#define WRAP_DATA_MASK GENMASK(27, 0) + +/* DMA_CHN_INTC register definition */ +#define FRAG_INT_STS BIT(8) +#define BLK_INT_STS BIT(9) +#define TRSC_INT_STS BIT(10) +#define LLIST_INT_STS BIT(11) +#define CFGERR_INT_STS BIT(12) + +/* DMA_CHN_FRG_LEN register definition */ +#define SRC_DATAWIDTH_OFFSET 30 +#define DES_DATAWIDTH_OFFSET 28 +#define SWT_MODE_OFFSET 26 +#define REQ_MODE_OFFSET 24 +#define REQ_MODE_MASK 0x3 +#define ADDR_WRAP_SEL_OFFSET 23 +#define ADDR_WRAP_EN_OFFSET 22 +#define ADDR_FIX_SEL_OFFSET 21 +#define ADDR_FIX_SEL_EN 20 +#define LLIST_END_OFFSET 19 +#define BLK_LEN_REC_H_OFFSET 17 +#define FRG_LEN_OFFSET 0 +#define FRG_LEN_MASK GENMASK(16, 0) + +/* DMA_CHN_BLK_LEN register definition */ +#define BLK_LEN_MASK GENMASK(16, 0) + +/* DMA_CHN_TRSC_LEN register definition */ +#define TRSC_LEN_MASK GENMASK(27, 0) + +/* DMA_CHN_TRSF_STEP register definition */ +#define DEST_TRSF_STEP_OFFSET 16 +#define SRC_TRSF_STEP_OFFSET 0 +#define TRSF_STEP_MASK GENMASK(15, 0) + +/* DMA_CHN_FRAG_STEP register definition */ +#define DEST_FRAG_STEP_OFFSET 16 +#define SRC_FRAG_STEP_OFFSET 0 +#define FRAG_STEP_MASK GENMASK(15, 0) + +/* DMA_CHN_SRC_BLK_STEP register definition */ +#define PTR_HIGH_ADDR_MASK GENMASK(31, 28) +#define PTR_HIGH_ADDR_OFFSET 4 + +enum dma_config_type { + SINGLE_CONFIG, + LINKLIST_CONFIG, +}; + +/* dma channel configuration */ +struct sprd_dma_chn_config { + u32 pause; + u32 req; + u32 cfg; + u32 intc; + u32 src_addr; + u32 des_addr; + u32 frg_len; + u32 blk_len; + u32 trsc_len; + u32 trsf_step; + u32 wrap_ptr; + u32 wrap_to; + u32 llist_ptr; + u32 frg_step; + u32 src_blk_step; + u32 des_blk_step; +}; + +/* dma request description */ +struct sprd_dma_desc { + struct dma_async_tx_descriptor desc; + struct sprd_dma_chn_config *chn_config; + struct list_head node; + enum dma_flags dma_flags; + int cycle; +}; + +/* dma channel description */ +struct sprd_dma_chn { + struct dma_chan chan; + struct list_head free; + struct list_head prepared; + struct list_head queued; + struct list_head active; + struct list_head completed; + void __iomem *chn_base; + spinlock_t chn_lock; + int chn_num; + u32 dev_id; + void *dma_desc_configs; + struct sprd_dma_cfg dma_cfg[SPRD_DMA_CFG_COUNT]; + int cfg_count; +}; + +/* SPRD dma device */ +struct sprd_dma_dev { + struct dma_device dma_dev; + void __iomem *glb_base; + struct clk *clk; + struct clk *ashb_clk; + int irq; + struct tasklet_struct tasklet; + struct kmem_cache *dma_desc_node_cachep; + u32 total_chns; + struct sprd_dma_chn channels[0]; +}; + +static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param); +static struct of_dma_filter_info sprd_dma_info = { + .filter_fn = sprd_dma_filter_fn, +}; + +static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c) +{ + return container_of(c, struct sprd_dma_chn, chan); +} + +static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(c); + + return container_of(mchan, struct sprd_dma_dev, channels[c->chan_id]); +} + +static inline struct sprd_dma_desc * +to_sprd_dma_desc(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct sprd_dma_desc, desc); +} + +static int sprd_dma_enable(struct sprd_dma_dev *sdev) +{ + int ret; + + ret = clk_prepare_enable(sdev->clk); + if (ret) + return ret; + + if (!IS_ERR(sdev->ashb_clk)) + ret = clk_prepare_enable(sdev->ashb_clk); + + return ret; +} + +static void sprd_dma_disable(struct sprd_dma_dev *sdev) +{ + clk_disable_unprepare(sdev->clk); + + if (!IS_ERR(sdev->ashb_clk)) + clk_disable_unprepare(sdev->ashb_clk); +} + +static void sprd_dma_set_uid(struct sprd_dma_chn *mchan) +{ + struct sprd_dma_dev *sdev = to_sprd_dma_dev(&mchan->chan); + u32 dev_id = mchan->dev_id; + + if (dev_id != DMA_SOFTWARE_UID) + writel(mchan->chn_num + 1, (void __iomem *)DMA_GLB_REQ_CID( + sdev->glb_base, dev_id)); +} + +static void sprd_dma_unset_uid(struct sprd_dma_chn *mchan) +{ + struct sprd_dma_dev *sdev = to_sprd_dma_dev(&mchan->chan); + u32 dev_id = mchan->dev_id; + + if (dev_id != DMA_SOFTWARE_UID) + writel(0, (void __iomem *)DMA_GLB_REQ_CID(sdev->glb_base, + dev_id)); +} + +static void sprd_dma_clear_int(struct sprd_dma_chn *mchan) +{ + u32 intc = readl(mchan->chn_base + DMA_CHN_INTC); + + intc |= DMA_CHN_INT_MASK << DMA_CHN_INT_CLR_OFFSET; + writel(intc, mchan->chn_base + DMA_CHN_INTC); +} + +static void sprd_dma_enable_chn(struct sprd_dma_chn *mchan) +{ + u32 cfg = readl(mchan->chn_base + DMA_CHN_CFG); + + cfg |= DMA_CHN_EN; + writel(cfg, mchan->chn_base + DMA_CHN_CFG); +} + +static void sprd_dma_disable_chn(struct sprd_dma_chn *mchan) +{ + u32 cfg = readl(mchan->chn_base + DMA_CHN_CFG); + + cfg &= ~DMA_CHN_EN; + writel(cfg, mchan->chn_base + DMA_CHN_CFG); +} + +static void sprd_dma_soft_request(struct sprd_dma_chn *mchan) +{ + u32 req = readl(mchan->chn_base + DMA_CHN_REQ); + + req |= DMA_CHN_REQ_EN; + writel(req, mchan->chn_base + DMA_CHN_REQ); +} + +static void sprd_dma_stop_and_disable(struct sprd_dma_chn *mchan) +{ + u32 cfg = readl(mchan->chn_base + DMA_CHN_CFG); + u32 pause, timeout = DMA_CHN_PAUSE_CNT; + + if (!(cfg & DMA_CHN_EN)) + return; + + pause = readl(mchan->chn_base + DMA_CHN_PAUSE); + pause |= DMA_CHN_PAUSE_EN; + writel(pause, mchan->chn_base + DMA_CHN_PAUSE); + + do { + pause = readl(mchan->chn_base + DMA_CHN_PAUSE); + if (pause & DMA_CHN_PAUSE_STS) + break; + + cpu_relax(); + } while (--timeout > 0); + + sprd_dma_disable_chn(mchan); + writel(0, mchan->chn_base + DMA_CHN_PAUSE); +} + +static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *mchan) +{ + unsigned long addr; + + addr = readl(mchan->chn_base + DMA_CHN_DES_ADDR); + addr |= (readl(mchan->chn_base + DMA_CHN_WARP_TO) & + DMA_CHN_ADDR_MASK) << DMA_CHN_HIGH_ADDR_OFFSET; + return addr; +} + +static enum dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *mchan) +{ + u32 intc_reg = readl(mchan->chn_base + DMA_CHN_INTC); + + if (intc_reg & CFGERR_INT_STS) + return CONFIG_ERR; + else if (intc_reg & LLIST_INT_STS) + return LIST_DONE; + else if (intc_reg & TRSC_INT_STS) + return TRANS_DONE; + else if (intc_reg & BLK_INT_STS) + return BLK_DONE; + else if (intc_reg & FRAG_INT_STS) + return FRAG_DONE; + else + return NO_INT; +} + +static enum dma_request_mode sprd_dma_get_req_type(struct sprd_dma_chn *mchan) +{ + u32 frag_reg = readl(mchan->chn_base + DMA_CHN_FRG_LEN); + + switch ((frag_reg >> REQ_MODE_OFFSET) & REQ_MODE_MASK) { + case 0: + return FRAG_REQ_MODE; + case 1: + return BLOCK_REQ_MODE; + case 2: + return TRANS_REQ_MODE; + case 3: + return LIST_REQ_MODE; + default: + return FRAG_REQ_MODE; + } +} + +static int sprd_dma_chn_start_chn(struct sprd_dma_chn *mchan, + struct sprd_dma_desc *mdesc) +{ + struct sprd_dma_dev *sdev = to_sprd_dma_dev(&mchan->chan); + enum dma_flags flag = mdesc->dma_flags; + int chn = mchan->chn_num + 1; + unsigned int cfg_group1, cfg_group2, start_mode = 0; + + if (!(flag & (DMA_GROUP1_SRC | DMA_GROUP2_SRC | DMA_GROUP1_DST | + DMA_GROUP2_DST))) + return 0; + + if (flag & (DMA_GROUP1_SRC | DMA_GROUP2_SRC)) { + switch (flag & (DMA_MUTL_FRAG_DONE | DMA_MUTL_BLK_DONE | + DMA_MUTL_TRANS_DONE | DMA_MUTL_LIST_DONE)) { + case DMA_MUTL_FRAG_DONE: + start_mode = 0; + break; + case DMA_MUTL_BLK_DONE: + start_mode = 1; + break; + case DMA_MUTL_TRANS_DONE: + start_mode = 2; + break; + case DMA_MUTL_LIST_DONE: + start_mode = 3; + break; + default: + dev_err(sdev->dma_dev.dev, + "chn stat chn mode incorrect\n"); + return -EINVAL; + } + } + + cfg_group1 = readl(sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG1); + cfg_group2 = readl(sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG2); + + switch (flag & (DMA_GROUP1_SRC | DMA_GROUP2_SRC | + DMA_GROUP1_DST | DMA_GROUP2_DST)) { + case DMA_GROUP1_SRC: + cfg_group1 |= chn << SRC_CHN_OFFSET; + cfg_group1 |= (1 << start_mode) << START_MODE_OFFSET; + cfg_group1 |= CHN_START_CHN; + writel(cfg_group1, sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG1); + break; + case DMA_GROUP2_SRC: + cfg_group2 |= chn << SRC_CHN_OFFSET; + cfg_group2 |= (1 << start_mode) << START_MODE_OFFSET; + cfg_group2 |= CHN_START_CHN; + writel(cfg_group2, sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG2); + break; + case DMA_GROUP1_DST: + cfg_group1 |= chn << DST_CHN_OFFSET; + cfg_group1 |= CHN_START_CHN; + writel(cfg_group1, sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG1); + break; + case DMA_GROUP2_DST: + cfg_group2 |= chn << DST_CHN_OFFSET; + cfg_group2 |= CHN_START_CHN; + writel(cfg_group2, sdev->glb_base + DMA_GLB_CHN_START_CHN_CFG2); + break; + default: + break; + } + + return 0; +} + +static int sprd_dma_set_chn_config(struct sprd_dma_chn *mchan, + struct sprd_dma_desc *mdesc) +{ + struct sprd_dma_chn_config *cfg = mdesc->chn_config; + int ret; + + ret = sprd_dma_chn_start_chn(mchan, mdesc); + if (ret) + return ret; + + writel(cfg->pause, mchan->chn_base + DMA_CHN_PAUSE); + writel(cfg->cfg, mchan->chn_base + DMA_CHN_CFG); + writel(cfg->intc, mchan->chn_base + DMA_CHN_INTC); + writel(cfg->src_addr, mchan->chn_base + DMA_CHN_SRC_ADDR); + writel(cfg->des_addr, mchan->chn_base + DMA_CHN_DES_ADDR); + writel(cfg->frg_len, mchan->chn_base + DMA_CHN_FRG_LEN); + writel(cfg->blk_len, mchan->chn_base + DMA_CHN_BLK_LEN); + writel(cfg->trsc_len, mchan->chn_base + DMA_CHN_TRSC_LEN); + writel(cfg->trsf_step, mchan->chn_base + DMA_CHN_TRSF_STEP); + writel(cfg->wrap_ptr, mchan->chn_base + DMA_CHN_WARP_PTR); + writel(cfg->wrap_to, mchan->chn_base + DMA_CHN_WARP_TO); + writel(cfg->llist_ptr, mchan->chn_base + DMA_CHN_LLIST_PTR); + writel(cfg->frg_step, mchan->chn_base + DMA_CHN_FRAG_STEP); + writel(cfg->src_blk_step, mchan->chn_base + DMA_CHN_SRC_BLK_STEP); + writel(cfg->des_blk_step, mchan->chn_base + DMA_CHN_DES_BLK_STEP); + writel(cfg->req, mchan->chn_base + DMA_CHN_REQ); + + return 0; +} + +static int sprd_dma_start(struct sprd_dma_chn *mchan) +{ + struct sprd_dma_desc *first; + int ret; + + if (list_empty(&mchan->active)) + return 0; + + /* + * Get the first DMA descriptor from active list, and copy the DMA + * configuration from DMA descriptor to this DMA channel. + */ + first = list_first_entry(&mchan->active, struct sprd_dma_desc, node); + ret = sprd_dma_set_chn_config(mchan, first); + if (ret) + return ret; + + sprd_dma_set_uid(mchan); + sprd_dma_enable_chn(mchan); + + if (mchan->dev_id == DMA_SOFTWARE_UID && + !(first->dma_flags & (DMA_GROUP1_DST | DMA_GROUP2_DST))) + sprd_dma_soft_request(mchan); + + return 0; +} + +static void sprd_dma_stop(struct sprd_dma_chn *mchan) +{ + sprd_dma_unset_uid(mchan); + sprd_dma_stop_and_disable(mchan); + sprd_dma_clear_int(mchan); +} + +static int sprd_dma_process_completed(struct sprd_dma_dev *sdev) +{ + struct sprd_dma_chn *mchan; + struct sprd_dma_desc *mdesc; + struct sprd_dma_desc *first; + struct dma_async_tx_descriptor *desc; + u32 dma_total_chns = sdev->total_chns; + dma_cookie_t last_cookie = 0; + unsigned long flags; + LIST_HEAD(list); + int i; + + for (i = 0; i < dma_total_chns; i++) { + mchan = &sdev->channels[i]; + + spin_lock_irqsave(&mchan->chn_lock, flags); + if (!list_empty(&mchan->completed)) + list_splice_tail_init(&mchan->completed, &list); + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + if (list_empty(&list)) + continue; + + list_for_each_entry(mdesc, &list, node) { + desc = &mdesc->desc; + + if (desc->callback) + desc->callback(desc->callback_param); + + dma_run_dependencies(desc); + last_cookie = desc->cookie; + } + + spin_lock_irqsave(&mchan->chn_lock, flags); + list_splice_tail_init(&list, &mchan->free); + + /* + * Check if there are pending DMA descriptor in queued list + * need to be queued. + */ + if (!list_empty(&mchan->queued)) { + list_for_each_entry(mdesc, &mchan->queued, node) { + /* + * If the DMA descriptor flag is set + * DMA_GROUP1_DST or DMA_GROUP1_DST flag, which + * means this DMA descriptor do not need to be + * activated and it will be invoked by another + * DMA descriptor which set DMA_GROUP1_SRC or + * DMA_GROUP2_SRC flag. + */ + if (!(mdesc->dma_flags & + (DMA_GROUP1_DST | DMA_GROUP2_DST)) && + list_empty(&mchan->active)) { + first = list_first_entry(&mchan->queued, + struct sprd_dma_desc, node); + list_move_tail(&first->node, + &mchan->active); + sprd_dma_start(mchan); + break; + } + } + } else { + mchan->chan.completed_cookie = last_cookie; + } + + spin_unlock_irqrestore(&mchan->chn_lock, flags); + } + + return 0; +} + +static void sprd_dma_tasklet(unsigned long data) +{ + struct sprd_dma_dev *sdev = (void *)data; + + sprd_dma_process_completed(sdev); +} + +static bool sprd_dma_check_trans_done(struct sprd_dma_desc *mdesc, + enum dma_int_type int_type, + enum dma_request_mode req_mode) +{ + if (mdesc->cycle == 1) + return false; + + if ((unsigned int)int_type >= ((unsigned int)req_mode + 1)) + return true; + else + return false; +} + +static irqreturn_t dma_irq_handle(int irq, void *dev_id) +{ + struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id; + u32 irq_status = readl(sdev->glb_base + DMA_GLB_INT_MSK_STS); + struct sprd_dma_chn *mchan; + struct sprd_dma_desc *mdesc; + struct dma_async_tx_descriptor *desc; + enum dma_request_mode req_type; + enum dma_int_type int_type; + bool trans_done = false; + u32 i; + + while (irq_status) { + i = __ffs(irq_status); + irq_status &= (irq_status - 1); + mchan = &sdev->channels[i]; + + spin_lock(&mchan->chn_lock); + int_type = sprd_dma_get_int_type(mchan); + req_type = sprd_dma_get_req_type(mchan); + sprd_dma_clear_int(mchan); + + if (!list_empty(&mchan->active)) { + mdesc = list_first_entry(&mchan->active, + struct sprd_dma_desc, node); + /* + * Check if the dma request descriptor is done, if it + * is done we should move this descriptor into + * complete list to handle. + */ + trans_done = sprd_dma_check_trans_done(mdesc, int_type, + req_type); + if (trans_done == true) + list_splice_tail_init(&mchan->active, + &mchan->completed); + /* + * If the cycle is 1, which means this DMA descriptor + * will always in active state until user free this DMA + * channel's resources. But user may need to be notified + * when transfer interrupt is generated. + */ + if (mdesc->cycle == 1) { + desc = &mdesc->desc; + spin_unlock(&mchan->chn_lock); + if (desc->callback) + desc->callback(desc->callback_param); + continue; + } + } + spin_unlock(&mchan->chn_lock); + } + + tasklet_schedule(&sdev->tasklet); + return IRQ_HANDLED; +} + +static dma_cookie_t sprd_desc_submit(struct dma_async_tx_descriptor *tx) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(tx->chan); + struct sprd_dma_desc *mdesc = to_sprd_dma_desc(tx); + struct sprd_dma_desc *first; + unsigned long flags; + dma_cookie_t cookie; + int ret; + + spin_lock_irqsave(&mchan->chn_lock, flags); + cookie = dma_cookie_assign(tx); + list_move_tail(&mdesc->node, &mchan->queued); + + /* + * If we set DMA_GROUP1_DST or DMA_GROUP2_DST flags, which means we do + * not need start this DMA transfer and it will be invoked by other DMA + * transfer setting DMA_GROUP1_SRC or DMA_GROUP2_SRC flag. + */ + if (mdesc->dma_flags & (DMA_GROUP1_DST | DMA_GROUP2_DST)) { + ret = sprd_dma_set_chn_config(mchan, mdesc); + if (ret) { + spin_unlock_irqrestore(&mchan->chn_lock, flags); + return ret; + } + + sprd_dma_enable_chn(mchan); + } else if (list_empty(&mchan->active)) { + first = list_first_entry(&mchan->queued, struct sprd_dma_desc, + node); + list_move_tail(&first->node, &mchan->active); + ret = sprd_dma_start(mchan); + if (ret) + cookie = ret; + } + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + return cookie; +} + +static int sprd_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan); + struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan); + struct sprd_dma_desc *mdesc, *tmp; + struct sprd_dma_chn_config *chn_configs; + LIST_HEAD(descs); + unsigned long flags; + int ret, i; + + ret = pm_runtime_get_sync(chan->device->dev); + if (ret < 0) { + dev_err(sdev->dma_dev.dev, "pm runtime get failed: %d\n", ret); + return ret; + } + + chn_configs = devm_kzalloc(sdev->dma_dev.dev, + SPRD_DMA_DESCRIPTORS * + sizeof(struct sprd_dma_chn_config), + GFP_KERNEL); + if (!chn_configs) { + ret = -ENOMEM; + goto err_alloc; + } + + for (i = 0; i < SPRD_DMA_DESCRIPTORS; i++) { + mdesc = (struct sprd_dma_desc *)kmem_cache_zalloc( + sdev->dma_desc_node_cachep, + GFP_ATOMIC); + if (!mdesc) { + ret = -ENOMEM; + goto err_cache_alloc; + } + + dma_async_tx_descriptor_init(&mdesc->desc, chan); + mdesc->desc.flags = DMA_CTRL_ACK; + mdesc->desc.tx_submit = sprd_desc_submit; + mdesc->chn_config = &chn_configs[i]; + mdesc->cycle = 0; + mdesc->dma_flags = 0; + INIT_LIST_HEAD(&mdesc->node); + list_add_tail(&mdesc->node, &descs); + } + + spin_lock_irqsave(&mchan->chn_lock, flags); + list_splice_tail_init(&descs, &mchan->free); + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + mchan->dma_desc_configs = chn_configs; + mchan->dev_id = 0; + mchan->cfg_count = 0; + memset(mchan->dma_cfg, 0, sizeof(struct sprd_dma_cfg) * + SPRD_DMA_CFG_COUNT); + + return 0; + +err_cache_alloc: + list_for_each_entry_safe(mdesc, tmp, &descs, node) + kmem_cache_free(sdev->dma_desc_node_cachep, mdesc); + devm_kfree(sdev->dma_dev.dev, chn_configs); +err_alloc: + pm_runtime_put_sync(chan->device->dev); + return ret; +} + +static void sprd_dma_free_chan_resources(struct dma_chan *chan) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan); + struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan); + struct sprd_dma_desc *mdesc, *tmp; + unsigned long flags; + LIST_HEAD(descs); + + spin_lock_irqsave(&mchan->chn_lock, flags); + list_splice_tail_init(&mchan->prepared, &mchan->free); + list_splice_tail_init(&mchan->queued, &mchan->free); + list_splice_tail_init(&mchan->active, &mchan->free); + list_splice_tail_init(&mchan->completed, &mchan->free); + list_splice_tail_init(&mchan->free, &descs); + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + list_for_each_entry_safe(mdesc, tmp, &descs, node) + kmem_cache_free(sdev->dma_desc_node_cachep, mdesc); + devm_kfree(sdev->dma_dev.dev, mchan->dma_desc_configs); + + spin_lock_irqsave(&mchan->chn_lock, flags); + sprd_dma_stop(mchan); + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + pm_runtime_put_sync(chan->device->dev); +} + +static enum dma_status sprd_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan); + unsigned long flags; + enum dma_status ret; + + ret = dma_cookie_status(chan, cookie, txstate); + + spin_lock_irqsave(&mchan->chn_lock, flags); + txstate->residue = sprd_dma_get_dst_addr(mchan); + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + return ret; +} + +static void sprd_dma_issue_pending(struct dma_chan *chan) +{ + /* + * We are posting descriptors to the hardware as soon as + * they are ready, so this function does nothing. + */ +} + +static int sprd_dma_config(struct dma_chan *chan, struct sprd_dma_desc *mdesc, + struct sprd_dma_cfg *cfg_list, + struct sprd_dma_chn_config *chn_config, + enum dma_config_type type) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan); + struct sprd_dma_dev *sdev = to_sprd_dma_dev(&mchan->chan); + struct sprd_dma_cfg *cfg = cfg_list; + struct sprd_dma_chn_config *dma_chn_config; + static unsigned long first_link_p; + unsigned long linklist_ptr = 0; + u32 fix_mode = 0, fix_en = 0; + u32 wrap_en = 0, wrap_mode = 0; + u32 llist_en = 0, list_end = 0; + + if (!IS_ALIGNED(cfg->src_step, 1 << cfg->datawidth)) { + dev_err(sdev->dma_dev.dev, "source step is not aligned\n"); + return -EINVAL; + } + + if (!IS_ALIGNED(cfg->des_step, 1 << cfg->datawidth)) { + dev_err(sdev->dma_dev.dev, "destination step is not aligned\n"); + return -EINVAL; + } + + if (type == SINGLE_CONFIG) + dma_chn_config = mdesc->chn_config; + else if (type == LINKLIST_CONFIG) + dma_chn_config = chn_config; + else { + dev_err(sdev->dma_dev.dev, + "check configuration type failed %d\n", type); + return -EINVAL; + } + + if (!mchan->dev_id) + mchan->dev_id = cfg->dev_id; + + dma_chn_config->pause = 0; + dma_chn_config->req = 0; + + if (cfg->link_cfg_p) + llist_en = 1; + + dma_chn_config->cfg = cfg->chn_pri << DMA_CHN_PRIORITY_OFFSET | + llist_en << LLIST_EN_OFFSET | + DMA_DONOT_WAIT_BDONE << CHN_WAIT_BDONE; + + if (cfg->wrap_ptr && cfg->wrap_to) { + wrap_en = 1; + dma_chn_config->wrap_ptr = cfg->wrap_ptr & WRAP_DATA_MASK; + dma_chn_config->wrap_to = cfg->wrap_to & WRAP_DATA_MASK; + } + + dma_chn_config->wrap_ptr |= + (u32)((cfg->src_addr >> DMA_CHN_HIGH_ADDR_OFFSET) & + DMA_CHN_ADDR_MASK); + dma_chn_config->wrap_to |= + (u32)((cfg->des_addr >> DMA_CHN_HIGH_ADDR_OFFSET) & + DMA_CHN_ADDR_MASK); + + dma_chn_config->src_addr = (u32)cfg->src_addr; + dma_chn_config->des_addr = (u32)cfg->des_addr; + + if ((cfg->src_step != 0 && cfg->des_step != 0) || + ((cfg->src_step | cfg->des_step) == 0)) { + fix_en = 0; + } else { + fix_en = 1; + if (cfg->src_step) + fix_mode = 1; + else + fix_mode = 0; + } + + if (wrap_en == 1) { + if (cfg->wrap_to == cfg->src_addr) { + wrap_mode = 0; + } else if (cfg->wrap_to == cfg->des_addr) { + wrap_mode = 1; + } else { + dev_err(sdev->dma_dev.dev, + "check wrap config failed\n"); + return -EINVAL; + } + } + + if (llist_en == 1 && cfg->is_end == DMA_END) + list_end = 1; + + dma_chn_config->frg_len = (cfg->datawidth << SRC_DATAWIDTH_OFFSET) | + (cfg->datawidth << DES_DATAWIDTH_OFFSET) | + (cfg->swt_mode << SWT_MODE_OFFSET) | + (cfg->req_mode << REQ_MODE_OFFSET) | + (wrap_mode << ADDR_WRAP_SEL_OFFSET) | + (wrap_en << ADDR_WRAP_EN_OFFSET) | + (fix_mode << ADDR_FIX_SEL_OFFSET) | + (fix_en << ADDR_FIX_SEL_EN) | + (list_end << LLIST_END_OFFSET) | + (cfg->fragmens_len & FRG_LEN_MASK); + + dma_chn_config->blk_len = cfg->block_len & BLK_LEN_MASK; + + if (type == SINGLE_CONFIG) { + dma_chn_config->intc &= ~DMA_CHN_INT_MASK; + dma_chn_config->intc |= CFG_ERROR_INT_EN; + + switch (cfg->irq_mode) { + case NO_INT: + break; + case FRAG_DONE: + dma_chn_config->intc |= FRAG_INT_EN; + break; + case BLK_DONE: + dma_chn_config->intc |= BLK_INT_EN; + break; + case BLOCK_FRAG_DONE: + dma_chn_config->intc |= BLK_INT_EN | FRAG_INT_EN; + break; + case TRANS_DONE: + dma_chn_config->intc |= TRANS_INT_EN; + break; + case TRANS_FRAG_DONE: + dma_chn_config->intc |= TRANS_INT_EN | FRAG_INT_EN; + break; + case TRANS_BLOCK_DONE: + dma_chn_config->intc |= TRANS_INT_EN | BLK_INT_EN; + break; + case LIST_DONE: + dma_chn_config->intc |= LIST_INT_EN; + break; + case CONFIG_ERR: + dma_chn_config->intc |= CFG_ERROR_INT_EN; + break; + default: + dev_err(sdev->dma_dev.dev, "irq mode failed\n"); + return -EINVAL; + } + } else { + dma_chn_config->intc = 0; + } + + if (cfg->transcation_len == 0) + dma_chn_config->trsc_len = cfg->block_len & TRSC_LEN_MASK; + else + dma_chn_config->trsc_len = cfg->transcation_len & + TRSC_LEN_MASK; + + dma_chn_config->trsf_step = (cfg->des_step & TRSF_STEP_MASK) << + DEST_TRSF_STEP_OFFSET | + (cfg->src_step & TRSF_STEP_MASK) << SRC_TRSF_STEP_OFFSET; + + dma_chn_config->frg_step = (cfg->dst_frag_step & FRAG_STEP_MASK) + << DEST_FRAG_STEP_OFFSET | + (cfg->src_frag_step & FRAG_STEP_MASK) + << SRC_FRAG_STEP_OFFSET; + + dma_chn_config->src_blk_step = cfg->src_blk_step; + dma_chn_config->des_blk_step = cfg->dst_blk_step; + + if (first_link_p == 0) + first_link_p = cfg->link_cfg_p; + + if (type == SINGLE_CONFIG) { + linklist_ptr = first_link_p + DMA_CHN_LLIST_OFFSET; + first_link_p = 0; + } else if (type == LINKLIST_CONFIG) { + if (cfg->is_end == DMA_LINK) + linklist_ptr = first_link_p + DMA_CHN_LLIST_OFFSET; + else + linklist_ptr = cfg->link_cfg_p + + sizeof(struct sprd_dma_chn_config) + + DMA_CHN_LLIST_OFFSET; + } + + dma_chn_config->src_blk_step |= + (u32)((linklist_ptr >> PTR_HIGH_ADDR_OFFSET) & + PTR_HIGH_ADDR_MASK); + dma_chn_config->llist_ptr = (u32)linklist_ptr; + return 0; +} + +static int sprd_dma_config_linklist(struct dma_chan *chan, + struct sprd_dma_desc *mdesc, + struct sprd_dma_cfg *cfg_list, + u32 node_size) +{ + struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan); + struct sprd_dma_chn_config *chn_config_list_v; + struct sprd_dma_cfg start_list; + int ret, i; + + chn_config_list_v = (struct sprd_dma_chn_config *)cfg_list[0].link_cfg_v; + + for (i = 0; i < node_size; i++) { + if (cfg_list[i].link_cfg_v == 0 || cfg_list[i].link_cfg_p == 0) { + dev_err(sdev->dma_dev.dev, + "no allocated memory for list node\n"); + return -EINVAL; + } + + ret = sprd_dma_config(chan, NULL, cfg_list + i, + chn_config_list_v + i, + LINKLIST_CONFIG); + if (ret < 0) { + dev_err(sdev->dma_dev.dev, + "linklist configuration failed\n"); + return ret; + } + } + + memset((void *)&start_list, 0, sizeof(struct sprd_dma_cfg)); + start_list.link_cfg_p = cfg_list[0].link_cfg_p; + start_list.irq_mode = cfg_list[0].irq_mode; + start_list.src_addr = cfg_list[0].src_addr; + start_list.des_addr = cfg_list[0].des_addr; + + if (cfg_list[node_size - 1].is_end == DMA_LINK) + mdesc->cycle = 1; + + return sprd_dma_config(chan, mdesc, &start_list, NULL, SINGLE_CONFIG); +} + +struct dma_async_tx_descriptor *sprd_dma_prep_dma_memcpy(struct dma_chan *chan, + dma_addr_t dest, + dma_addr_t src, + size_t len, + unsigned long flags) +{ + struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan); + struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan); + struct sprd_dma_desc *mdesc = NULL; + struct sprd_dma_cfg dma_cfg; + u32 datawidth, src_step, des_step; + int cfg_count = mchan->cfg_count; + unsigned long irq_flags; + int ret; + + /* + * If user did not have configuration for DMA transfer, then we should + * configure the DMA controller as default configuration. + */ + if (cfg_count == 0) { + if (IS_ALIGNED(len, 4)) { + datawidth = 2; + src_step = 4; + des_step = 4; + } else if (IS_ALIGNED(len, 2)) { + datawidth = 1; + src_step = 2; + des_step = 2; + } else { + datawidth = 0; + src_step = 1; + des_step = 1; + } + + memset(&dma_cfg, 0, sizeof(struct sprd_dma_cfg)); + dma_cfg.src_addr = src; + dma_cfg.des_addr = dest; + dma_cfg.datawidth = datawidth; + dma_cfg.src_step = src_step; + dma_cfg.des_step = src_step; + dma_cfg.fragmens_len = SPRD_DMA_MEMCPY_MIN_SIZE; + if (len <= BLK_LEN_MASK) { + dma_cfg.block_len = len; + dma_cfg.req_mode = BLOCK_REQ_MODE; + dma_cfg.irq_mode = BLK_DONE; + } else { + dma_cfg.block_len = SPRD_DMA_MEMCPY_MIN_SIZE; + dma_cfg.transcation_len = len; + dma_cfg.req_mode = TRANS_REQ_MODE; + dma_cfg.irq_mode = TRANS_DONE; + } + cfg_count = 1; + } else if (cfg_count == 1) { + memcpy(&dma_cfg, mchan->dma_cfg, sizeof(struct sprd_dma_cfg)); + } + + /* get one free DMA descriptor for this DMA transfer. */ + spin_lock_irqsave(&mchan->chn_lock, irq_flags); + if (!list_empty(&mchan->free)) { + mdesc = list_first_entry(&mchan->free, + struct sprd_dma_desc, node); + list_del(&mdesc->node); + } + spin_unlock_irqrestore(&mchan->chn_lock, irq_flags); + + if (!mdesc) { + dev_err(sdev->dma_dev.dev, "get free descriptor failed\n"); + sprd_dma_process_completed(sdev); + goto out; + } + + mdesc->dma_flags = flags; + + /* + * Set the DMA configuration into channel register of this DMA + * descriptor. If we have multiple configuration, which means + * need enable the link-list function, then we should set these + * configuration into link-list address. + */ + if (cfg_count == 1) + ret = sprd_dma_config(chan, mdesc, &dma_cfg, NULL, + SINGLE_CONFIG); + else + ret = sprd_dma_config_linklist(chan, mdesc, mchan->dma_cfg, + cfg_count); + + if (ret < 0) { + spin_lock_irqsave(&mchan->chn_lock, irq_flags); + list_add_tail(&mdesc->node, &mchan->free); + spin_unlock_irqrestore(&mchan->chn_lock, irq_flags); + dev_err(sdev->dma_dev.dev, "DMA configuration is failed\n"); + goto out; + } + + if (!(flags & DMA_HARDWARE_REQ)) + mchan->dev_id = DMA_SOFTWARE_UID; + + /* add the free descriptor to the prepared list. */ + spin_lock_irqsave(&mchan->chn_lock, irq_flags); + list_add_tail(&mdesc->node, &mchan->prepared); + spin_unlock_irqrestore(&mchan->chn_lock, irq_flags); + + mchan->cfg_count = 0; + memset(mchan->dma_cfg, 0, sizeof(struct sprd_dma_cfg) * + SPRD_DMA_CFG_COUNT); + return &mdesc->desc; + +out: + mchan->cfg_count = 0; + memset(mchan->dma_cfg, 0, sizeof(struct sprd_dma_cfg) * + SPRD_DMA_CFG_COUNT); + return NULL; +} + +static int sprd_dma_pause(struct dma_chan *chan) +{ + struct sprd_dma_chn *mchan = + container_of(chan, struct sprd_dma_chn, chan); + unsigned long flags; + + spin_lock_irqsave(&mchan->chn_lock, flags); + sprd_dma_stop(mchan); + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + return 0; +} + +static int sprd_dma_resume(struct dma_chan *chan) +{ + struct sprd_dma_chn *mchan = + container_of(chan, struct sprd_dma_chn, chan); + unsigned long flags; + int ret; + + spin_lock_irqsave(&mchan->chn_lock, flags); + ret = sprd_dma_start(mchan); + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + return ret; +} + +static int sprd_terminate_all(struct dma_chan *chan) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&mchan->chn_lock, flags); + list_splice_tail_init(&mchan->prepared, &mchan->free); + list_splice_tail_init(&mchan->queued, &mchan->free); + list_splice_tail_init(&mchan->active, &mchan->free); + list_splice_tail_init(&mchan->completed, &mchan->free); + + sprd_dma_stop(mchan); + spin_unlock_irqrestore(&mchan->chn_lock, flags); + + return 0; +} + +static int sprd_dma_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan); + struct sprd_dma_cfg *sprd_cfg = container_of(config, + struct sprd_dma_cfg, + config); + int i = 0; + + do { + memcpy(&mchan->dma_cfg[i], sprd_cfg, sizeof(*sprd_cfg)); + sprd_cfg++; + } while (mchan->dma_cfg[i++].is_end == DMA_NOT_END + && i < (SPRD_DMA_CFG_COUNT - 1)); + + mchan->cfg_count = i; + return 0; +} + +static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param) +{ + struct sprd_dma_chn *mchan = to_sprd_dma_chan(chan); + struct of_phandle_args *dma_spec = (struct of_phandle_args *)param; + unsigned int req = (unsigned int)dma_spec->args[0]; + + if (chan->device->dev->of_node == dma_spec->np) + return req == (mchan->chn_num + 1); + else + return false; +} + +static struct dma_chan *sprd_dma_simple_xlate(struct of_phandle_args *dma_spec, + struct of_dma *of_dma) +{ + struct of_dma_filter_info *info = of_dma->of_dma_data; + int count = dma_spec->args_count; + + if (!info || !info->filter_fn) + return NULL; + + if (count != 1) + return NULL; + + return dma_request_channel(info->dma_cap, info->filter_fn, dma_spec); +} + +static int sprd_dma_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct sprd_dma_dev *sdev; + struct sprd_dma_chn *dma_chn; + struct resource *res; + u32 chn_count; + int ret, i; + + if (of_property_read_u32(np, "#dma-channels", &chn_count)) { + dev_err(&pdev->dev, "get dma channels count failed\n"); + return -ENODEV; + } + + sdev = devm_kzalloc(&pdev->dev, (sizeof(*sdev) + + (sizeof(struct sprd_dma_chn) * chn_count)), + GFP_KERNEL); + if (!sdev) + return -ENOMEM; + + sdev->clk = devm_clk_get(&pdev->dev, "enable"); + if (IS_ERR(sdev->clk)) { + dev_err(&pdev->dev, "get enable clock failed\n"); + return PTR_ERR(sdev->clk); + } + + /* ashb clock is optional for AGCP DMA */ + sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb"); + if (IS_ERR(sdev->ashb_clk)) + dev_warn(&pdev->dev, "get ashb eb clock failed\n"); + + sdev->irq = platform_get_irq(pdev, 0); + if (sdev->irq > 0) { + ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle, + 0, "sprd_dma", (void *)sdev); + if (ret < 0) { + dev_err(&pdev->dev, "request dma irq failed\n"); + return ret; + } + + tasklet_init(&sdev->tasklet, sprd_dma_tasklet, + (unsigned long)sdev); + } else { + dev_warn(&pdev->dev, "no interrupts for the DMA controller\n"); + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + sdev->glb_base = devm_ioremap_nocache(&pdev->dev, res->start, + resource_size(res)); + if (!sdev->glb_base) + return -ENOMEM; + + dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask); + sdev->total_chns = chn_count; + sdev->dma_dev.chancnt = chn_count; + INIT_LIST_HEAD(&sdev->dma_dev.channels); + INIT_LIST_HEAD(&sdev->dma_dev.global_node); + sdev->dma_dev.dev = &pdev->dev; + sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources; + sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources; + sdev->dma_dev.device_tx_status = sprd_dma_tx_status; + sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending; + sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy; + sdev->dma_dev.device_config = sprd_dma_slave_config; + sdev->dma_dev.device_pause = sprd_dma_pause; + sdev->dma_dev.device_resume = sprd_dma_resume; + sdev->dma_dev.device_terminate_all = sprd_terminate_all; + + for (i = 0; i < chn_count; i++) { + dma_chn = &sdev->channels[i]; + dma_chn->chan.device = &sdev->dma_dev; + dma_cookie_init(&dma_chn->chan); + list_add_tail(&dma_chn->chan.device_node, + &sdev->dma_dev.channels); + + dma_chn->chn_num = i; + /* get each channel's registers base address. */ + dma_chn->chn_base = (void __iomem *) + ((unsigned long)sdev->glb_base + + SPRD_DMA_CHN_REG_OFFSET + + SPRD_DMA_CHN_REG_LENGTH * i); + + spin_lock_init(&dma_chn->chn_lock); + INIT_LIST_HEAD(&dma_chn->free); + INIT_LIST_HEAD(&dma_chn->prepared); + INIT_LIST_HEAD(&dma_chn->queued); + INIT_LIST_HEAD(&dma_chn->active); + INIT_LIST_HEAD(&dma_chn->completed); + } + + sdev->dma_desc_node_cachep = kmem_cache_create("dma_desc_node", + sizeof(struct sprd_dma_desc), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!sdev->dma_desc_node_cachep) { + dev_err(&pdev->dev, "allocate memory cache failed\n"); + return -ENOMEM; + } + + platform_set_drvdata(pdev, sdev); + ret = sprd_dma_enable(sdev); + if (ret) + goto err_enable; + + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) + goto err_rpm; + + ret = dma_async_device_register(&sdev->dma_dev); + if (ret < 0) { + dev_err(&pdev->dev, "register dma device failed:%d\n", ret); + goto err_register; + } + + sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask; + ret = of_dma_controller_register(np, sprd_dma_simple_xlate, + &sprd_dma_info); + if (ret) { + dev_err(&pdev->dev, "failed to register of DMA controller\n"); + goto err_of_register; + } + + pm_runtime_put_sync(&pdev->dev); + return 0; + +err_of_register: + dma_async_device_unregister(&sdev->dma_dev); +err_register: + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); +err_rpm: + sprd_dma_disable(sdev); +err_enable: + kmem_cache_destroy(sdev->dma_desc_node_cachep); + return ret; +} + +static int sprd_dma_remove(struct platform_device *pdev) +{ + struct sprd_dma_dev *sdev = platform_get_drvdata(pdev); + int ret; + + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) + return ret; + + kmem_cache_destroy(sdev->dma_desc_node_cachep); + dma_async_device_unregister(&sdev->dma_dev); + sprd_dma_disable(sdev); + + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); + return 0; +} + +static const struct of_device_id sprd_dma_match[] = { + { .compatible = "sprd,sc9860-dma", }, + {}, +}; + +static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev) +{ + struct sprd_dma_dev *sdev = dev_get_drvdata(dev); + + sprd_dma_disable(sdev); + return 0; +} + +static int __maybe_unused sprd_dma_runtime_resume(struct device *dev) +{ + struct sprd_dma_dev *sdev = dev_get_drvdata(dev); + + return sprd_dma_enable(sdev); +} + +static const struct dev_pm_ops sprd_dma_pm_ops = { + SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend, + sprd_dma_runtime_resume, + NULL) +}; + +static struct platform_driver sprd_dma_driver = { + .probe = sprd_dma_probe, + .remove = sprd_dma_remove, + .driver = { + .name = "sprd-dma", + .owner = THIS_MODULE, + .of_match_table = sprd_dma_match, + .pm = &sprd_dma_pm_ops, + }, +}; + +static int __init sprd_dma_init(void) +{ + return platform_driver_register(&sprd_dma_driver); +} +arch_initcall_sync(sprd_dma_init); + +static void __exit sprd_dma_exit(void) +{ + platform_driver_unregister(&sprd_dma_driver); +} +module_exit(sprd_dma_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("DMA driver for Spreadtrum"); +MODULE_AUTHOR("Baolin Wang "); +MODULE_AUTHOR("Eric Long "); diff --git a/include/linux/dma/sprd-dma.h b/include/linux/dma/sprd-dma.h new file mode 100644 index 0000000..40a8b1c --- /dev/null +++ b/include/linux/dma/sprd-dma.h @@ -0,0 +1,270 @@ +/* + * Copyright (C) 2017 Spreadtrum Communications Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __SPRD_DMA_H +#define __SPRD_DMA_H +#include + +/* DMA request ID definition */ +#define DMA_SOFTWARE_UID 0 +#define DMA_SIM_RX 1 +#define DMA_SIM_TX 2 +#define DMA_IIS0_RX 3 +#define DMA_IIS0_TX 4 +#define DMA_IIS1_RX 5 +#define DMA_IIS1_TX 6 +#define DMA_IIS2_RX 7 +#define DMA_IIS2_TX 8 +#define DMA_IIS3_RX 9 +#define DMA_IIS3_TX 10 +#define DMA_SPI0_RX 11 +#define DMA_SPI0_TX 12 +#define DMA_SPI1_RX 13 +#define DMA_SPI1_TX 14 +#define DMA_SPI2_RX 15 +#define DMA_SPI2_TX 16 +#define DMA_UART0_RX 17 +#define DMA_UART0_TX 18 +#define DMA_UART1_RX 19 +#define DMA_UART1_TX 20 +#define DMA_UART2_RX 21 +#define DMA_UART2_TX 22 +#define DMA_UART3_RX 23 +#define DMA_UART3_TX 24 +#define DMA_UART4_RX 25 +#define DMA_UART4_TX 26 +#define DMA_DRM_CPT 27 +#define DMA_DRM_RAW 28 +#define DMA_VB_DA0 29 +#define DMA_VB_DA1 30 +#define DMA_VB_AD0 31 +#define DMA_VB_AD1 32 +#define DMA_VB_AD2 33 +#define DMA_VB_AD3 34 +#define DMA_GPS 35 +#define DMA_SDIO0_RD 36 +#define DMA_SDIO0_WR 37 +#define DMA_SDIO1_RD 38 +#define DMA_SDIO1_WR 39 +#define DMA_SDIO2_RD 40 +#define DMA_SDIO2_WR 41 +#define DMA_EMMC_RD 42 +#define DMA_EMMC_WR 43 + +/* + * enum dma_datawidth: define the DMA transfer data width + * @BYTE_WIDTH: 1 byte width + * @SHORT_WIDTH: 2 bytes width + * @WORD_WIDTH: 4 bytes width + * @DWORD_WIDTH: 8 bytes width + */ +enum dma_datawidth { + BYTE_WIDTH, + SHORT_WIDTH, + WORD_WIDTH, + DWORD_WIDTH, +}; + +/* + * enum dma_request_mode: define the DMA request mode + * @FRAG_REQ_MODE: fragment request mode + * @BLOCK_REQ_MODE: block request mode + * @TRANS_REQ_MODE: transaction request mode + * @LIST_REQ_MODE: link-list request mode + * + * We have 4 types request mode: fragment mode, block mode, transaction mode + * and linklist mode. One transaction can contain several blocks, one block can + * contain several fragments. Link-list mode means we can save several DMA + * configuration into one reserved memory, then DMA can fetch each DMA + * configuration automatically to start transfer. + */ +enum dma_request_mode { + FRAG_REQ_MODE, + BLOCK_REQ_MODE, + TRANS_REQ_MODE, + LIST_REQ_MODE, +}; + +/* + * enum dma_int_type: define the DMA interrupt type + * @NO_INT: do not need generate DMA interrupt. + * @FRAG_DONE: fragment done interrupt when one fragment request is done. + * @BLK_DONE: block done interrupt when one block request is done. + * @TRANS_DONE: tansaction done interrupt when one transaction request is done. + * @LIST_DONE: link-list done interrupt when one link-list request is done. + * @CONFIG_ERR: configure error interrupt when configuration is incorrect + * @BLOCK_FRAG_DONE: block and fragment interrupt when one fragment or block + * request is done. + * @TRANS_FRAG_DONE: transaction and fragment interrupt when one transaction + * request or fragment request is done. + * @TRANS_BLOCK_DONE: transaction and block interrupt when one transaction + * request or block request is done. + */ +enum dma_int_type { + NO_INT, + FRAG_DONE, + BLK_DONE, + TRANS_DONE, + LIST_DONE, + CONFIG_ERR, + BLOCK_FRAG_DONE, + TRANS_FRAG_DONE, + TRANS_BLOCK_DONE, +}; + +/* + * enum dma_pri_level: define the DMA channel priority level + * @DMA_PRI_0: level 0 + * @DMA_PRI_1: level 1 + * @DMA_PRI_2: level 2 + * @DMA_PRI_3: level 3 + * + * When there are several DMA channels need to start, the DMA controller's + * arbitration will choose the high priority channel to start firstly. + */ +enum dma_pri_level { + DMA_PRI_0, + DMA_PRI_1, + DMA_PRI_2, + DMA_PRI_3, +}; + +/* + * enum dma_switch_mode: define the DMA transfer format + * @DATA_ABCD: ABCD to ABCD + * @DATA_DCBA: ABCD to DCBA + * @DATA_BADC: ABCD to BADC + * @DATA_CDAB: ABCD to CDAB + */ +enum dma_switch_mode { + DATA_ABCD, + DATA_DCBA, + DATA_BADC, + DATA_CDAB, +}; + +/* + * enum dma_end_type: define the DMA configuration end type + * @DMA_NOT_END: DMA configuration is not end + * @DMA_END: DMA configuration is end but not one link-list cycle configuration + * @DMA_LINK: DMA configuration is end but it is one link-list cycle + * configuration + * + * Since DMA contrller can support link-list transfer mode, that means user can + * supply several DMA configuration and each configuration can be pointed by + * previous link pointer register, then DMA controller will start to transfer + * for each DMA configuration automatically. DMA_END and DMA_LINK flag can + * indicate these several configuration is end, but DMA_LINK can also indicate + * these listed configuration is one cycle. For example if we have 4 group DMA + * configuration and we set DMA_LINK, which means it will start to transfer from + * cfg0--->cfg1--->cfg2--->cfg3, then back to cfg0 as one cycle. + */ +enum dma_end_type { + DMA_NOT_END, + DMA_END, + DMA_LINK, +}; + +/* + * enum dma_flags: define the DMA flags + * @DMA_HARDWARE_REQ: hardware request channel to start transfer by hardware id. + * @DMA_SOFTWARE_REQ: software request channel to start transfer. + * @DMA_GROUP1_SRC: indicate this is source channel of group 1 which can be + * start another channel. + * @DMA_GROUP1_DST: indicate this is destination channel of group 1 which will + * be started by source channel. + * @DMA_GROUP2_SRC: indicate this is source channel of group 2 which can be + * start another channel. + * @DMA_GROUP2_DST: indicate this is destination channel of group 2 which will + * be started by source channel. + * @DMA_MUTL_FRAG_DONE: when fragment is done of source channel which will + * start another destination channel. + * @DMA_MUTL_BLK_DONE: when block is done of source channel which will start + * another destination channel. + * @DMA_MUTL_TRANS_DONE: when transaction is done of source channel which will + * start another destination channel. + * @DMA_MUTL_LIST_DONE: when link-list is done of source channel which will + * start another destination channel. + * + * Since DMA controller can support 2 stage transfer which means when one + * channel transfer is done, then it can start another channel's transfer + * automatically by interrupt type. + */ +enum dma_flags { + DMA_HARDWARE_REQ = BIT(0), + DMA_SOFTWARE_REQ = BIT(1), + DMA_GROUP1_SRC = BIT(2), + DMA_GROUP1_DST = BIT(3), + DMA_GROUP2_SRC = BIT(4), + DMA_GROUP2_DST = BIT(5), + DMA_MUTL_FRAG_DONE = BIT(6), + DMA_MUTL_BLK_DONE = BIT(7), + DMA_MUTL_TRANS_DONE = BIT(8), + DMA_MUTL_LIST_DONE = BIT(9), +}; + +/* + * struct sprd_dma_cfg: DMA configuration for users + * @config: salve config structure + * @chn_pri: the channel priority + * @datawidth: the data width + * @req_mode: the request mode + * @irq_mode: the interrupt mode + * @swt_mode: the switch mode + * @link_cfg_v: point to the virtual memory address to save link-list DMA + * configuration + * @link_cfg_p: point to the physical memory address to save link-list DMA + * configuration + * @src_addr: the source address + * @des_addr: the destination address + * @fragmens_len: one fragment request length + * @block_len; one block request length + * @transcation_len: one transaction request length + * @src_step: source side transfer step + * @des_step: destination side transfer step + * @src_frag_step: source fragment transfer step + * @dst_frag_step: destination fragment transfer step + * @src_blk_step: source block transfer step + * @dst_blk_step: destination block transfer step + * @wrap_ptr: wrap jump pointer address + * @wrap_to: wrap jump to address + * @dev_id: hardware device id to start DMA transfer + * @is_end: DMA configuration end type + */ +struct sprd_dma_cfg { + struct dma_slave_config config; + enum dma_pri_level chn_pri; + enum dma_datawidth datawidth; + enum dma_request_mode req_mode; + enum dma_int_type irq_mode; + enum dma_switch_mode swt_mode; + unsigned long link_cfg_v; + unsigned long link_cfg_p; + unsigned long src_addr; + unsigned long des_addr; + u32 fragmens_len; + u32 block_len; + u32 transcation_len; + u32 src_step; + u32 des_step; + u32 src_frag_step; + u32 dst_frag_step; + u32 src_blk_step; + u32 dst_blk_step; + u32 wrap_ptr; + u32 wrap_to; + u32 dev_id; + enum dma_end_type is_end; +}; + +#endif