[v8,1/3] dmaengine: fsl-edma: extract common fsl-edma code (no changes in behavior intended)

Message ID	20180803193244.12084-1-angelo@sysam.it (mailing list archive)
State	New, archived
Headers	show Return-Path: <linux-arm-kernel-bounces+patchwork-linux-arm=patchwork.kernel.org@lists.infradead.org> From: Angelo Dureghello <angelo@sysam.it> To: vinod.koul@linaro.org Subject: [PATCH v8 1/3] dmaengine: fsl-edma: extract common fsl-edma code (no changes in behavior intended) Date: Fri, 3 Aug 2018 21:32:42 +0200 Message-Id: <20180803193244.12084-1-angelo@sysam.it> Precedence: list Cc: linux-m68k@vger.kernel.org, Angelo Dureghello <angelo@sysam.it>, krzk@kernel.org, stefan@agner.ch, dmaengine@vger.kernel.org, linux-arm-kernel@lists.infradead.org MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Sender: "linux-arm-kernel" <linux-arm-kernel-bounces@lists.infradead.org> Errors-To: linux-arm-kernel-bounces+patchwork-linux-arm=patchwork.kernel.org@lists.infradead.org
Series	[v8,1/3] dmaengine: fsl-edma: extract common fsl-edma code (no changes in behavior intended) \| expand [v8,1/3] dmaengine: fsl-edma: extract common fsl-edma code (no changes in behavior intended) [v8,2/3] dmaengine: fsl-edma: add edma version and configurable registers [v8,3/3] dmaengine: fsl-edma: add ColdFire mcf5441x edma support

On 03.08.2018 21:32, Angelo Dureghello wrote: > This patch adds a new fsl-edma-common module to allow new > mcf-edma module code to use most of the fsl-edma code. > > Signed-off-by: Angelo Dureghello <angelo@sysam.it> > --- > Changes for v2: > - patch splitted into 4 > - add mcf-edma as minimal different parts from fsl-edma > > Changes for v3: > none > > Changes for v4: > - patch simplified from 4/4 into 2/2 > - collecting all the mcf-edma-related changes > > Changes for v5: > none > > Changes for v6: > - adjusted comment header > - fixed bit shift with BIT() > - we need to free the interrupts at remove(), so removed all devm_ > interrupt related calls > > Changes for v7: > none > > Changes for v8: > - patch rewritten from scratch, splitted into 3, common code isolated, > minimal changes from the original Freescale code have been done. > The patch has been tested with both Iris + Colibri Vybrid VF50 and > stmark2/mcf54415 Coldfire boards. > --- > drivers/dma/Makefile | 2 +- > drivers/dma/fsl-edma-common.c | 576 ++++++++++++++++++++++++++++ > drivers/dma/fsl-edma-common.h | 196 ++++++++++ > drivers/dma/fsl-edma.c | 697 +--------------------------------- > 4 files changed, 774 insertions(+), 697 deletions(-) > create mode 100644 drivers/dma/fsl-edma-common.c > create mode 100644 drivers/dma/fsl-edma-common.h > > diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile > index 203a99d68315..66022f59fca4 100644 > --- a/drivers/dma/Makefile > +++ b/drivers/dma/Makefile > @@ -31,7 +31,7 @@ obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/ > obj-$(CONFIG_DW_DMAC_CORE) += dw/ > obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o > obj-$(CONFIG_FSL_DMA) += fsldma.o > -obj-$(CONFIG_FSL_EDMA) += fsl-edma.o > +obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o > obj-$(CONFIG_FSL_RAID) += fsl_raid.o > obj-$(CONFIG_HSU_DMA) += hsu/ > obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o > diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c > new file mode 100644 > index 000000000000..0ae7094f477a > --- /dev/null > +++ b/drivers/dma/fsl-edma-common.c > @@ -0,0 +1,576 @@ > +// SPDX-License-Identifier: GPL-2.0 > +// > +// Copyright (c) 2013-2014 Freescale Semiconductor, Inc > +// Copyright (c) 2017 Sysam, Angelo Dureghello <angelo@sysam.it> > + > +#include <linux/dmapool.h> > +#include <linux/module.h> > +#include <linux/slab.h> > + > +#include "fsl-edma-common.h" > + > +/* > + * R/W functions for big- or little-endian registers: > + * The eDMA controller's endian is independent of the CPU core's endian. > + * For the big-endian IP module, the offset for 8-bit or 16-bit registers > + * should also be swapped opposite to that in little-endian IP. > + */ > +u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) > +{ > + if (edma->big_endian) > + return ioread32be(addr); > + else > + return ioread32(addr); > +} > +EXPORT_SYMBOL_GPL(edma_readl); In 3/3 you link the common object into the two modules individually: obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o obj-$(CONFIG_MCF_EDMA) += mcf-edma.o fsl-edma-common.o Therefor you do not access those functions from another module (they are within the module). No exporting should be necessary. Drop all those exports. If possible I would prefer if you start with cleanup/conversions, then split-up and finally add functionality. So ideally: 1. Use macros for preprocessor defines (where you move to BIT/GENMASK) 2. Split 3. Add EDMA macros etc. 4. Add ColdFire mcf5441x edma support -- Stefan > + > +void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr) > +{ > + /* swap the reg offset for these in big-endian mode */ > + if (edma->big_endian) > + iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); > + else > + iowrite8(val, addr); > +} > +EXPORT_SYMBOL_GPL(edma_writeb); > + > +void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr) > +{ > + /* swap the reg offset for these in big-endian mode */ > + if (edma->big_endian) > + iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); > + else > + iowrite16(val, addr); > +} > +EXPORT_SYMBOL_GPL(edma_writew); > + > +void edma_writel(struct fsl_edma_engine *edma, u32 val, void __iomem *addr) > +{ > + if (edma->big_endian) > + iowrite32be(val, addr); > + else > + iowrite32(val, addr); > +} > +EXPORT_SYMBOL_GPL(edma_writel); > + > +struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) > +{ > + return container_of(chan, struct fsl_edma_chan, vchan.chan); > +} > +EXPORT_SYMBOL_GPL(to_fsl_edma_chan); > + > +struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) > +{ > + return container_of(vd, struct fsl_edma_desc, vdesc); > +} > +EXPORT_SYMBOL_GPL(to_fsl_edma_desc); > + > +static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) > +{ > + void __iomem *addr = fsl_chan->edma->membase; > + u32 ch = fsl_chan->vchan.chan.chan_id; > + > + edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI); > + edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ); > +} > + > +void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) > +{ > + void __iomem *addr = fsl_chan->edma->membase; > + u32 ch = fsl_chan->vchan.chan.chan_id; > + > + edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ); > + edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI); > +} > +EXPORT_SYMBOL_GPL(fsl_edma_disable_request); > + > +void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, > + unsigned int slot, bool enable) > +{ > + u32 ch = fsl_chan->vchan.chan.chan_id; > + void __iomem *muxaddr; > + unsigned int chans_per_mux, ch_off; > + > + chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; > + ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; > + muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; > + slot = EDMAMUX_CHCFG_SOURCE(slot); > + > + if (enable) > + iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); > + else > + iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); > +} > +EXPORT_SYMBOL_GPL(fsl_edma_chan_mux); > + > +static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) > +{ > + switch (addr_width) { > + case 1: > + return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; > + case 2: > + return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; > + case 4: > + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; > + case 8: > + return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; > + default: > + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; > + } > +} > + > +void fsl_edma_free_desc(struct virt_dma_desc *vdesc) > +{ > + struct fsl_edma_desc *fsl_desc; > + int i; > + > + fsl_desc = to_fsl_edma_desc(vdesc); > + for (i = 0; i < fsl_desc->n_tcds; i++) > + dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, > + fsl_desc->tcd[i].ptcd); > + kfree(fsl_desc); > +} > +EXPORT_SYMBOL_GPL(fsl_edma_free_desc); > + > +int fsl_edma_terminate_all(struct dma_chan *chan) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + unsigned long flags; > + LIST_HEAD(head); > + > + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > + fsl_edma_disable_request(fsl_chan); > + fsl_chan->edesc = NULL; > + fsl_chan->idle = true; > + vchan_get_all_descriptors(&fsl_chan->vchan, &head); > + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); > + return 0; > +} > +EXPORT_SYMBOL_GPL(fsl_edma_terminate_all); > + > +int fsl_edma_pause(struct dma_chan *chan) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + unsigned long flags; > + > + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > + if (fsl_chan->edesc) { > + fsl_edma_disable_request(fsl_chan); > + fsl_chan->status = DMA_PAUSED; > + fsl_chan->idle = true; > + } > + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > + return 0; > +} > +EXPORT_SYMBOL_GPL(fsl_edma_pause); > + > +int fsl_edma_resume(struct dma_chan *chan) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + unsigned long flags; > + > + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > + if (fsl_chan->edesc) { > + fsl_edma_enable_request(fsl_chan); > + fsl_chan->status = DMA_IN_PROGRESS; > + fsl_chan->idle = false; > + } > + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > + return 0; > +} > +EXPORT_SYMBOL_GPL(fsl_edma_resume); > + > +int fsl_edma_slave_config(struct dma_chan *chan, > + struct dma_slave_config *cfg) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + > + fsl_chan->fsc.dir = cfg->direction; > + if (cfg->direction == DMA_DEV_TO_MEM) { > + fsl_chan->fsc.dev_addr = cfg->src_addr; > + fsl_chan->fsc.addr_width = cfg->src_addr_width; > + fsl_chan->fsc.burst = cfg->src_maxburst; > + fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width); > + } else if (cfg->direction == DMA_MEM_TO_DEV) { > + fsl_chan->fsc.dev_addr = cfg->dst_addr; > + fsl_chan->fsc.addr_width = cfg->dst_addr_width; > + fsl_chan->fsc.burst = cfg->dst_maxburst; > + fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width); > + } else > + return -EINVAL; > + > + return 0; > +} > +EXPORT_SYMBOL_GPL(fsl_edma_slave_config); > + > +static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, > + struct virt_dma_desc *vdesc, bool in_progress) > +{ > + struct fsl_edma_desc *edesc = fsl_chan->edesc; > + void __iomem *addr = fsl_chan->edma->membase; > + u32 ch = fsl_chan->vchan.chan.chan_id; > + enum dma_transfer_direction dir = fsl_chan->fsc.dir; > + dma_addr_t cur_addr, dma_addr; > + size_t len, size; > + int i; > + > + /* calculate the total size in this desc */ > + for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) > + len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) > + * le16_to_cpu(edesc->tcd[i].vtcd->biter); > + > + if (!in_progress) > + return len; > + > + if (dir == DMA_MEM_TO_DEV) > + cur_addr = edma_readl( > + fsl_chan->edma, addr + EDMA_TCD_SADDR(ch)); > + else > + cur_addr = edma_readl( > + fsl_chan->edma, addr + EDMA_TCD_DADDR(ch)); > + > + /* figure out the finished and calculate the residue */ > + for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { > + size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) > + * le16_to_cpu(edesc->tcd[i].vtcd->biter); > + if (dir == DMA_MEM_TO_DEV) > + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); > + else > + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); > + > + len -= size; > + if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { > + len += dma_addr + size - cur_addr; > + break; > + } > + } > + > + return len; > +} > + > +enum dma_status fsl_edma_tx_status(struct dma_chan *chan, > + dma_cookie_t cookie, struct dma_tx_state *txstate) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + struct virt_dma_desc *vdesc; > + enum dma_status status; > + unsigned long flags; > + > + status = dma_cookie_status(chan, cookie, txstate); > + if (status == DMA_COMPLETE) > + return status; > + > + if (!txstate) > + return fsl_chan->status; > + > + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > + vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); > + if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) > + txstate->residue = > + fsl_edma_desc_residue(fsl_chan, vdesc, true); > + else if (vdesc) > + txstate->residue = > + fsl_edma_desc_residue(fsl_chan, vdesc, false); > + else > + txstate->residue = 0; > + > + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > + > + return fsl_chan->status; > +} > +EXPORT_SYMBOL_GPL(fsl_edma_tx_status); > + > +static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, > + struct fsl_edma_hw_tcd *tcd) > +{ > + struct fsl_edma_engine *edma = fsl_chan->edma; > + void __iomem *addr = fsl_chan->edma->membase; > + u32 ch = fsl_chan->vchan.chan.chan_id; > + > + /* > + * TCD parameters are stored in struct fsl_edma_hw_tcd in little > + * endian format. However, we need to load the TCD registers in > + * big- or little-endian obeying the eDMA engine model endian. > + */ > + edma_writew(edma, 0, addr + EDMA_TCD_CSR(ch)); > + edma_writel(edma, le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR(ch)); > + edma_writel(edma, le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR(ch)); > + > + edma_writew(edma, le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR(ch)); > + edma_writew(edma, le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF(ch)); > + > + edma_writel(edma, le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES(ch)); > + edma_writel(edma, le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST(ch)); > + > + edma_writew(edma, le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER(ch)); > + edma_writew(edma, le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER(ch)); > + edma_writew(edma, le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF(ch)); > + > + edma_writel(edma, > + le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA(ch)); > + > + edma_writew(edma, le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR(ch)); > +} > + > +static inline > +void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, > + u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, > + u16 biter, u16 doff, u32 dlast_sga, bool major_int, > + bool disable_req, bool enable_sg) > +{ > + u16 csr = 0; > + > + /* > + * eDMA hardware SGs require the TCDs to be stored in little > + * endian format irrespective of the register endian model. > + * So we put the value in little endian in memory, waiting > + * for fsl_edma_set_tcd_regs doing the swap. > + */ > + tcd->saddr = cpu_to_le32(src); > + tcd->daddr = cpu_to_le32(dst); > + > + tcd->attr = cpu_to_le16(attr); > + > + tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); > + > + tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); > + tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); > + > + tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); > + tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); > + > + tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); > + > + tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); > + if (major_int) > + csr |= EDMA_TCD_CSR_INT_MAJOR; > + > + if (disable_req) > + csr |= EDMA_TCD_CSR_D_REQ; > + > + if (enable_sg) > + csr |= EDMA_TCD_CSR_E_SG; > + > + tcd->csr = cpu_to_le16(csr); > +} > + > +static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan > *fsl_chan, > + int sg_len) > +{ > + struct fsl_edma_desc *fsl_desc; > + int i; > + > + fsl_desc = kzalloc(sizeof(*fsl_desc) + > + sizeof(struct fsl_edma_sw_tcd) * > + sg_len, GFP_NOWAIT); > + if (!fsl_desc) > + return NULL; > + > + fsl_desc->echan = fsl_chan; > + fsl_desc->n_tcds = sg_len; > + for (i = 0; i < sg_len; i++) { > + fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, > + GFP_NOWAIT, &fsl_desc->tcd[i].ptcd); > + if (!fsl_desc->tcd[i].vtcd) > + goto err; > + } > + return fsl_desc; > + > +err: > + while (--i >= 0) > + dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, > + fsl_desc->tcd[i].ptcd); > + kfree(fsl_desc); > + return NULL; > +} > + > +struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( > + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, > + size_t period_len, enum dma_transfer_direction direction, > + unsigned long flags) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + struct fsl_edma_desc *fsl_desc; > + dma_addr_t dma_buf_next; > + int sg_len, i; > + u32 src_addr, dst_addr, last_sg, nbytes; > + u16 soff, doff, iter; > + > + if (!is_slave_direction(fsl_chan->fsc.dir)) > + return NULL; > + > + sg_len = buf_len / period_len; > + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); > + if (!fsl_desc) > + return NULL; > + fsl_desc->iscyclic = true; > + > + dma_buf_next = dma_addr; > + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; > + iter = period_len / nbytes; > + > + for (i = 0; i < sg_len; i++) { > + if (dma_buf_next >= dma_addr + buf_len) > + dma_buf_next = dma_addr; > + > + /* get next sg's physical address */ > + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; > + > + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { > + src_addr = dma_buf_next; > + dst_addr = fsl_chan->fsc.dev_addr; > + soff = fsl_chan->fsc.addr_width; > + doff = 0; > + } else { > + src_addr = fsl_chan->fsc.dev_addr; > + dst_addr = dma_buf_next; > + soff = 0; > + doff = fsl_chan->fsc.addr_width; > + } > + > + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, > + fsl_chan->fsc.attr, soff, nbytes, 0, iter, > + iter, doff, last_sg, true, false, true); > + dma_buf_next += period_len; > + } > + > + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); > +} > +EXPORT_SYMBOL_GPL(fsl_edma_prep_dma_cyclic); > + > +struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( > + struct dma_chan *chan, struct scatterlist *sgl, > + unsigned int sg_len, enum dma_transfer_direction direction, > + unsigned long flags, void *context) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + struct fsl_edma_desc *fsl_desc; > + struct scatterlist *sg; > + u32 src_addr, dst_addr, last_sg, nbytes; > + u16 soff, doff, iter; > + int i; > + > + if (!is_slave_direction(fsl_chan->fsc.dir)) > + return NULL; > + > + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); > + if (!fsl_desc) > + return NULL; > + fsl_desc->iscyclic = false; > + > + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; > + for_each_sg(sgl, sg, sg_len, i) { > + /* get next sg's physical address */ > + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; > + > + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { > + src_addr = sg_dma_address(sg); > + dst_addr = fsl_chan->fsc.dev_addr; > + soff = fsl_chan->fsc.addr_width; > + doff = 0; > + } else { > + src_addr = fsl_chan->fsc.dev_addr; > + dst_addr = sg_dma_address(sg); > + soff = 0; > + doff = fsl_chan->fsc.addr_width; > + } > + > + iter = sg_dma_len(sg) / nbytes; > + if (i < sg_len - 1) { > + last_sg = fsl_desc->tcd[(i + 1)].ptcd; > + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, > + dst_addr, fsl_chan->fsc.attr, soff, > + nbytes, 0, iter, iter, doff, last_sg, > + false, false, true); > + } else { > + last_sg = 0; > + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, > + dst_addr, fsl_chan->fsc.attr, soff, > + nbytes, 0, iter, iter, doff, last_sg, > + true, true, false); > + } > + } > + > + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); > +} > +EXPORT_SYMBOL_GPL(fsl_edma_prep_slave_sg); > + > +void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) > +{ > + struct virt_dma_desc *vdesc; > + > + vdesc = vchan_next_desc(&fsl_chan->vchan); > + if (!vdesc) > + return; > + fsl_chan->edesc = to_fsl_edma_desc(vdesc); > + fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); > + fsl_edma_enable_request(fsl_chan); > + fsl_chan->status = DMA_IN_PROGRESS; > + fsl_chan->idle = false; > +} > +EXPORT_SYMBOL_GPL(fsl_edma_xfer_desc); > + > +void fsl_edma_issue_pending(struct dma_chan *chan) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + unsigned long flags; > + > + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > + > + if (unlikely(fsl_chan->pm_state != RUNNING)) { > + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > + /* cannot submit due to suspend */ > + return; > + } > + > + if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) > + fsl_edma_xfer_desc(fsl_chan); > + > + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > +} > +EXPORT_SYMBOL_GPL(fsl_edma_issue_pending); > + > +int fsl_edma_alloc_chan_resources(struct dma_chan *chan) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + > + fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, > + sizeof(struct fsl_edma_hw_tcd), > + 32, 0); > + return 0; > +} > +EXPORT_SYMBOL_GPL(fsl_edma_alloc_chan_resources); > + > +void fsl_edma_free_chan_resources(struct dma_chan *chan) > +{ > + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > + unsigned long flags; > + LIST_HEAD(head); > + > + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > + fsl_edma_disable_request(fsl_chan); > + fsl_edma_chan_mux(fsl_chan, 0, false); > + fsl_chan->edesc = NULL; > + vchan_get_all_descriptors(&fsl_chan->vchan, &head); > + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > + > + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); > + dma_pool_destroy(fsl_chan->tcd_pool); > + fsl_chan->tcd_pool = NULL; > +} > +EXPORT_SYMBOL_GPL(fsl_edma_free_chan_resources); > + > +void fsl_edma_cleanup_vchan(struct dma_device *dmadev) > +{ > + struct fsl_edma_chan *chan, *_chan; > + > + list_for_each_entry_safe(chan, _chan, > + &dmadev->channels, vchan.chan.device_node) { > + list_del(&chan->vchan.chan.device_node); > + tasklet_kill(&chan->vchan.task); > + } > +} > +EXPORT_SYMBOL_GPL(fsl_edma_cleanup_vchan); > + > +MODULE_LICENSE("GPL v2"); > diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h > new file mode 100644 > index 000000000000..f3ea68e15b23 > --- /dev/null > +++ b/drivers/dma/fsl-edma-common.h > @@ -0,0 +1,196 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +/* > + * Copyright 2013-2014 Freescale Semiconductor, Inc. > + * Copyright 2018 Angelo Dureghello <angelo@sysam.it> > + */ > +#ifndef _FSL_EDMA_COMMON_H_ > +#define _FSL_EDMA_COMMON_H_ > + > +#include "virt-dma.h" > + > +#define EDMA_CR 0x00 > +#define EDMA_ES 0x04 > +#define EDMA_ERQ 0x0C > +#define EDMA_EEI 0x14 > +#define EDMA_SERQ 0x1B > +#define EDMA_CERQ 0x1A > +#define EDMA_SEEI 0x19 > +#define EDMA_CEEI 0x18 > +#define EDMA_CINT 0x1F > +#define EDMA_CERR 0x1E > +#define EDMA_SSRT 0x1D > +#define EDMA_CDNE 0x1C > +#define EDMA_INTR 0x24 > +#define EDMA_ERR 0x2C > + > +#define EDMA_TCD_SADDR(x) (0x1000 + 32 * (x)) > +#define EDMA_TCD_SOFF(x) (0x1004 + 32 * (x)) > +#define EDMA_TCD_ATTR(x) (0x1006 + 32 * (x)) > +#define EDMA_TCD_NBYTES(x) (0x1008 + 32 * (x)) > +#define EDMA_TCD_SLAST(x) (0x100C + 32 * (x)) > +#define EDMA_TCD_DADDR(x) (0x1010 + 32 * (x)) > +#define EDMA_TCD_DOFF(x) (0x1014 + 32 * (x)) > +#define EDMA_TCD_CITER_ELINK(x) (0x1016 + 32 * (x)) > +#define EDMA_TCD_CITER(x) (0x1016 + 32 * (x)) > +#define EDMA_TCD_DLAST_SGA(x) (0x1018 + 32 * (x)) > +#define EDMA_TCD_CSR(x) (0x101C + 32 * (x)) > +#define EDMA_TCD_BITER_ELINK(x) (0x101E + 32 * (x)) > +#define EDMA_TCD_BITER(x) (0x101E + 32 * (x)) > + > +#define EDMA_CR_EDBG BIT(1) > +#define EDMA_CR_ERCA BIT(2) > +#define EDMA_CR_ERGA BIT(3) > +#define EDMA_CR_HOE BIT(4) > +#define EDMA_CR_HALT BIT(5) > +#define EDMA_CR_CLM BIT(6) > +#define EDMA_CR_EMLM BIT(7) > +#define EDMA_CR_ECX BIT(16) > +#define EDMA_CR_CX BIT(17) > + > +#define EDMA_SEEI_SEEI(x) ((x) & 0x1F) > +#define EDMA_CEEI_CEEI(x) ((x) & 0x1F) > +#define EDMA_CINT_CINT(x) ((x) & 0x1F) > +#define EDMA_CERR_CERR(x) ((x) & 0x1F) > + > +#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) > +#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) > +#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) > +#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) > +#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) > +#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) > +#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) > +#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) > +#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) > +#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) > +#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) > +#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) > +#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) > +#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) > + > +#define EDMA_TCD_SOFF_SOFF(x) (x) > +#define EDMA_TCD_NBYTES_NBYTES(x) (x) > +#define EDMA_TCD_SLAST_SLAST(x) (x) > +#define EDMA_TCD_DADDR_DADDR(x) (x) > +#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) > +#define EDMA_TCD_DOFF_DOFF(x) (x) > +#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) > +#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) > + > +#define EDMA_TCD_CSR_START BIT(0) > +#define EDMA_TCD_CSR_INT_MAJOR BIT(1) > +#define EDMA_TCD_CSR_INT_HALF BIT(2) > +#define EDMA_TCD_CSR_D_REQ BIT(3) > +#define EDMA_TCD_CSR_E_SG BIT(4) > +#define EDMA_TCD_CSR_E_LINK BIT(5) > +#define EDMA_TCD_CSR_ACTIVE BIT(6) > +#define EDMA_TCD_CSR_DONE BIT(7) > + > +#define EDMAMUX_CHCFG_DIS 0x0 > +#define EDMAMUX_CHCFG_ENBL 0x80 > +#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) > + > +#define DMAMUX_NR 2 > + > +#define FSL_EDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ > + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ > + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ > + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) > +enum fsl_edma_pm_state { > + RUNNING = 0, > + SUSPENDED, > +}; > + > +struct fsl_edma_hw_tcd { > + __le32 saddr; > + __le16 soff; > + __le16 attr; > + __le32 nbytes; > + __le32 slast; > + __le32 daddr; > + __le16 doff; > + __le16 citer; > + __le32 dlast_sga; > + __le16 csr; > + __le16 biter; > +}; > + > +struct fsl_edma_sw_tcd { > + dma_addr_t ptcd; > + struct fsl_edma_hw_tcd *vtcd; > +}; > + > +struct fsl_edma_slave_config { > + enum dma_transfer_direction dir; > + enum dma_slave_buswidth addr_width; > + u32 dev_addr; > + u32 burst; > + u32 attr; > +}; > + > +struct fsl_edma_chan { > + struct virt_dma_chan vchan; > + enum dma_status status; > + enum fsl_edma_pm_state pm_state; > + bool idle; > + u32 slave_id; > + struct fsl_edma_engine *edma; > + struct fsl_edma_desc *edesc; > + struct fsl_edma_slave_config fsc; > + struct dma_pool *tcd_pool; > +}; > + > +struct fsl_edma_desc { > + struct virt_dma_desc vdesc; > + struct fsl_edma_chan *echan; > + bool iscyclic; > + unsigned int n_tcds; > + struct fsl_edma_sw_tcd tcd[]; > +}; > + > +struct fsl_edma_engine { > + struct dma_device dma_dev; > + void __iomem *membase; > + void __iomem *muxbase[DMAMUX_NR]; > + struct clk *muxclk[DMAMUX_NR]; > + struct mutex fsl_edma_mutex; > + u32 n_chans; > + int txirq; > + int errirq; > + bool big_endian; > + struct fsl_edma_chan chans[]; > +}; > + > +u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr); > +void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr); > +void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr); > +void edma_writel(struct fsl_edma_engine *edma, u32 val, void __iomem *addr); > + > +struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan); > +struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd); > + > +void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan); > +void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, > + unsigned int slot, bool enable); > +void fsl_edma_free_desc(struct virt_dma_desc *vdesc); > +int fsl_edma_terminate_all(struct dma_chan *chan); > +int fsl_edma_pause(struct dma_chan *chan); > +int fsl_edma_resume(struct dma_chan *chan); > +int fsl_edma_slave_config(struct dma_chan *chan, > + struct dma_slave_config *cfg); > +enum dma_status fsl_edma_tx_status(struct dma_chan *chan, > + dma_cookie_t cookie, struct dma_tx_state *txstate); > +struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( > + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, > + size_t period_len, enum dma_transfer_direction direction, > + unsigned long flags); > +struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( > + struct dma_chan *chan, struct scatterlist *sgl, > + unsigned int sg_len, enum dma_transfer_direction direction, > + unsigned long flags, void *context); > +void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan); > +void fsl_edma_issue_pending(struct dma_chan *chan); > +int fsl_edma_alloc_chan_resources(struct dma_chan *chan); > +void fsl_edma_free_chan_resources(struct dma_chan *chan); > +void fsl_edma_cleanup_vchan(struct dma_device *dmadev); > + > +#endif /* _FSL_EDMA_COMMON_H_ */ > diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c > index c7568869284e..337ad0c0ea78 100644 > --- a/drivers/dma/fsl-edma.c > +++ b/drivers/dma/fsl-edma.c > @@ -13,653 +13,16 @@ > * option) any later version. > */ > > -#include <linux/init.h> > #include <linux/module.h> > #include <linux/interrupt.h> > #include <linux/clk.h> > -#include <linux/dma-mapping.h> > -#include <linux/dmapool.h> > -#include <linux/slab.h> > -#include <linux/spinlock.h> > #include <linux/of.h> > #include <linux/of_device.h> > #include <linux/of_address.h> > #include <linux/of_irq.h> > #include <linux/of_dma.h> > > -#include "virt-dma.h" > - > -#define EDMA_CR 0x00 > -#define EDMA_ES 0x04 > -#define EDMA_ERQ 0x0C > -#define EDMA_EEI 0x14 > -#define EDMA_SERQ 0x1B > -#define EDMA_CERQ 0x1A > -#define EDMA_SEEI 0x19 > -#define EDMA_CEEI 0x18 > -#define EDMA_CINT 0x1F > -#define EDMA_CERR 0x1E > -#define EDMA_SSRT 0x1D > -#define EDMA_CDNE 0x1C > -#define EDMA_INTR 0x24 > -#define EDMA_ERR 0x2C > - > -#define EDMA_TCD_SADDR(x) (0x1000 + 32 * (x)) > -#define EDMA_TCD_SOFF(x) (0x1004 + 32 * (x)) > -#define EDMA_TCD_ATTR(x) (0x1006 + 32 * (x)) > -#define EDMA_TCD_NBYTES(x) (0x1008 + 32 * (x)) > -#define EDMA_TCD_SLAST(x) (0x100C + 32 * (x)) > -#define EDMA_TCD_DADDR(x) (0x1010 + 32 * (x)) > -#define EDMA_TCD_DOFF(x) (0x1014 + 32 * (x)) > -#define EDMA_TCD_CITER_ELINK(x) (0x1016 + 32 * (x)) > -#define EDMA_TCD_CITER(x) (0x1016 + 32 * (x)) > -#define EDMA_TCD_DLAST_SGA(x) (0x1018 + 32 * (x)) > -#define EDMA_TCD_CSR(x) (0x101C + 32 * (x)) > -#define EDMA_TCD_BITER_ELINK(x) (0x101E + 32 * (x)) > -#define EDMA_TCD_BITER(x) (0x101E + 32 * (x)) > - > -#define EDMA_CR_EDBG BIT(1) > -#define EDMA_CR_ERCA BIT(2) > -#define EDMA_CR_ERGA BIT(3) > -#define EDMA_CR_HOE BIT(4) > -#define EDMA_CR_HALT BIT(5) > -#define EDMA_CR_CLM BIT(6) > -#define EDMA_CR_EMLM BIT(7) > -#define EDMA_CR_ECX BIT(16) > -#define EDMA_CR_CX BIT(17) > - > -#define EDMA_SEEI_SEEI(x) ((x) & 0x1F) > -#define EDMA_CEEI_CEEI(x) ((x) & 0x1F) > -#define EDMA_CINT_CINT(x) ((x) & 0x1F) > -#define EDMA_CERR_CERR(x) ((x) & 0x1F) > - > -#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) > -#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) > -#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) > -#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) > -#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) > -#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) > -#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) > -#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) > -#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) > -#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) > -#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) > -#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) > -#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) > -#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) > - > -#define EDMA_TCD_SOFF_SOFF(x) (x) > -#define EDMA_TCD_NBYTES_NBYTES(x) (x) > -#define EDMA_TCD_SLAST_SLAST(x) (x) > -#define EDMA_TCD_DADDR_DADDR(x) (x) > -#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) > -#define EDMA_TCD_DOFF_DOFF(x) (x) > -#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) > -#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) > - > -#define EDMA_TCD_CSR_START BIT(0) > -#define EDMA_TCD_CSR_INT_MAJOR BIT(1) > -#define EDMA_TCD_CSR_INT_HALF BIT(2) > -#define EDMA_TCD_CSR_D_REQ BIT(3) > -#define EDMA_TCD_CSR_E_SG BIT(4) > -#define EDMA_TCD_CSR_E_LINK BIT(5) > -#define EDMA_TCD_CSR_ACTIVE BIT(6) > -#define EDMA_TCD_CSR_DONE BIT(7) > - > -#define EDMAMUX_CHCFG_DIS 0x0 > -#define EDMAMUX_CHCFG_ENBL 0x80 > -#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) > - > -#define DMAMUX_NR 2 > - > -#define FSL_EDMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ > - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ > - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ > - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) > -enum fsl_edma_pm_state { > - RUNNING = 0, > - SUSPENDED, > -}; > - > -struct fsl_edma_hw_tcd { > - __le32 saddr; > - __le16 soff; > - __le16 attr; > - __le32 nbytes; > - __le32 slast; > - __le32 daddr; > - __le16 doff; > - __le16 citer; > - __le32 dlast_sga; > - __le16 csr; > - __le16 biter; > -}; > - > -struct fsl_edma_sw_tcd { > - dma_addr_t ptcd; > - struct fsl_edma_hw_tcd *vtcd; > -}; > - > -struct fsl_edma_slave_config { > - enum dma_transfer_direction dir; > - enum dma_slave_buswidth addr_width; > - u32 dev_addr; > - u32 burst; > - u32 attr; > -}; > - > -struct fsl_edma_chan { > - struct virt_dma_chan vchan; > - enum dma_status status; > - enum fsl_edma_pm_state pm_state; > - bool idle; > - u32 slave_id; > - struct fsl_edma_engine *edma; > - struct fsl_edma_desc *edesc; > - struct fsl_edma_slave_config fsc; > - struct dma_pool *tcd_pool; > -}; > - > -struct fsl_edma_desc { > - struct virt_dma_desc vdesc; > - struct fsl_edma_chan *echan; > - bool iscyclic; > - unsigned int n_tcds; > - struct fsl_edma_sw_tcd tcd[]; > -}; > - > -struct fsl_edma_engine { > - struct dma_device dma_dev; > - void __iomem *membase; > - void __iomem *muxbase[DMAMUX_NR]; > - struct clk *muxclk[DMAMUX_NR]; > - struct mutex fsl_edma_mutex; > - u32 n_chans; > - int txirq; > - int errirq; > - bool big_endian; > - struct fsl_edma_chan chans[]; > -}; > - > -/* > - * R/W functions for big- or little-endian registers: > - * The eDMA controller's endian is independent of the CPU core's endian. > - * For the big-endian IP module, the offset for 8-bit or 16-bit registers > - * should also be swapped opposite to that in little-endian IP. > - */ > - > -static u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) > -{ > - if (edma->big_endian) > - return ioread32be(addr); > - else > - return ioread32(addr); > -} > - > -static void edma_writeb(struct fsl_edma_engine *edma, u8 val, void > __iomem *addr) > -{ > - /* swap the reg offset for these in big-endian mode */ > - if (edma->big_endian) > - iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); > - else > - iowrite8(val, addr); > -} > - > -static void edma_writew(struct fsl_edma_engine *edma, u16 val, void > __iomem *addr) > -{ > - /* swap the reg offset for these in big-endian mode */ > - if (edma->big_endian) > - iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); > - else > - iowrite16(val, addr); > -} > - > -static void edma_writel(struct fsl_edma_engine *edma, u32 val, void > __iomem *addr) > -{ > - if (edma->big_endian) > - iowrite32be(val, addr); > - else > - iowrite32(val, addr); > -} > - > -static struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) > -{ > - return container_of(chan, struct fsl_edma_chan, vchan.chan); > -} > - > -static struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) > -{ > - return container_of(vd, struct fsl_edma_desc, vdesc); > -} > - > -static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) > -{ > - void __iomem *addr = fsl_chan->edma->membase; > - u32 ch = fsl_chan->vchan.chan.chan_id; > - > - edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI); > - edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ); > -} > - > -static void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) > -{ > - void __iomem *addr = fsl_chan->edma->membase; > - u32 ch = fsl_chan->vchan.chan.chan_id; > - > - edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ); > - edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI); > -} > - > -static void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, > - unsigned int slot, bool enable) > -{ > - u32 ch = fsl_chan->vchan.chan.chan_id; > - void __iomem *muxaddr; > - unsigned chans_per_mux, ch_off; > - > - chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; > - ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; > - muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; > - slot = EDMAMUX_CHCFG_SOURCE(slot); > - > - if (enable) > - iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); > - else > - iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); > -} > - > -static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) > -{ > - switch (addr_width) { > - case 1: > - return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; > - case 2: > - return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; > - case 4: > - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; > - case 8: > - return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; > - default: > - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; > - } > -} > - > -static void fsl_edma_free_desc(struct virt_dma_desc *vdesc) > -{ > - struct fsl_edma_desc *fsl_desc; > - int i; > - > - fsl_desc = to_fsl_edma_desc(vdesc); > - for (i = 0; i < fsl_desc->n_tcds; i++) > - dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, > - fsl_desc->tcd[i].ptcd); > - kfree(fsl_desc); > -} > - > -static int fsl_edma_terminate_all(struct dma_chan *chan) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - unsigned long flags; > - LIST_HEAD(head); > - > - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > - fsl_edma_disable_request(fsl_chan); > - fsl_chan->edesc = NULL; > - fsl_chan->idle = true; > - vchan_get_all_descriptors(&fsl_chan->vchan, &head); > - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > - vchan_dma_desc_free_list(&fsl_chan->vchan, &head); > - return 0; > -} > - > -static int fsl_edma_pause(struct dma_chan *chan) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - unsigned long flags; > - > - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > - if (fsl_chan->edesc) { > - fsl_edma_disable_request(fsl_chan); > - fsl_chan->status = DMA_PAUSED; > - fsl_chan->idle = true; > - } > - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > - return 0; > -} > - > -static int fsl_edma_resume(struct dma_chan *chan) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - unsigned long flags; > - > - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > - if (fsl_chan->edesc) { > - fsl_edma_enable_request(fsl_chan); > - fsl_chan->status = DMA_IN_PROGRESS; > - fsl_chan->idle = false; > - } > - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > - return 0; > -} > - > -static int fsl_edma_slave_config(struct dma_chan *chan, > - struct dma_slave_config *cfg) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - > - fsl_chan->fsc.dir = cfg->direction; > - if (cfg->direction == DMA_DEV_TO_MEM) { > - fsl_chan->fsc.dev_addr = cfg->src_addr; > - fsl_chan->fsc.addr_width = cfg->src_addr_width; > - fsl_chan->fsc.burst = cfg->src_maxburst; > - fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width); > - } else if (cfg->direction == DMA_MEM_TO_DEV) { > - fsl_chan->fsc.dev_addr = cfg->dst_addr; > - fsl_chan->fsc.addr_width = cfg->dst_addr_width; > - fsl_chan->fsc.burst = cfg->dst_maxburst; > - fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width); > - } else { > - return -EINVAL; > - } > - return 0; > -} > - > -static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, > - struct virt_dma_desc *vdesc, bool in_progress) > -{ > - struct fsl_edma_desc *edesc = fsl_chan->edesc; > - void __iomem *addr = fsl_chan->edma->membase; > - u32 ch = fsl_chan->vchan.chan.chan_id; > - enum dma_transfer_direction dir = fsl_chan->fsc.dir; > - dma_addr_t cur_addr, dma_addr; > - size_t len, size; > - int i; > - > - /* calculate the total size in this desc */ > - for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) > - len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) > - * le16_to_cpu(edesc->tcd[i].vtcd->biter); > - > - if (!in_progress) > - return len; > - > - if (dir == DMA_MEM_TO_DEV) > - cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_SADDR(ch)); > - else > - cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_DADDR(ch)); > - > - /* figure out the finished and calculate the residue */ > - for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { > - size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) > - * le16_to_cpu(edesc->tcd[i].vtcd->biter); > - if (dir == DMA_MEM_TO_DEV) > - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); > - else > - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); > - > - len -= size; > - if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { > - len += dma_addr + size - cur_addr; > - break; > - } > - } > - > - return len; > -} > - > -static enum dma_status fsl_edma_tx_status(struct dma_chan *chan, > - dma_cookie_t cookie, struct dma_tx_state *txstate) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - struct virt_dma_desc *vdesc; > - enum dma_status status; > - unsigned long flags; > - > - status = dma_cookie_status(chan, cookie, txstate); > - if (status == DMA_COMPLETE) > - return status; > - > - if (!txstate) > - return fsl_chan->status; > - > - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > - vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); > - if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) > - txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, true); > - else if (vdesc) > - txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, false); > - else > - txstate->residue = 0; > - > - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > - > - return fsl_chan->status; > -} > - > -static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, > - struct fsl_edma_hw_tcd *tcd) > -{ > - struct fsl_edma_engine *edma = fsl_chan->edma; > - void __iomem *addr = fsl_chan->edma->membase; > - u32 ch = fsl_chan->vchan.chan.chan_id; > - > - /* > - * TCD parameters are stored in struct fsl_edma_hw_tcd in little > - * endian format. However, we need to load the TCD registers in > - * big- or little-endian obeying the eDMA engine model endian. > - */ > - edma_writew(edma, 0, addr + EDMA_TCD_CSR(ch)); > - edma_writel(edma, le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR(ch)); > - edma_writel(edma, le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR(ch)); > - > - edma_writew(edma, le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR(ch)); > - edma_writew(edma, le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF(ch)); > - > - edma_writel(edma, le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES(ch)); > - edma_writel(edma, le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST(ch)); > - > - edma_writew(edma, le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER(ch)); > - edma_writew(edma, le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER(ch)); > - edma_writew(edma, le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF(ch)); > - > - edma_writel(edma, le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA(ch)); > - > - edma_writew(edma, le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR(ch)); > -} > - > -static inline > -void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, > - u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, > - u16 biter, u16 doff, u32 dlast_sga, bool major_int, > - bool disable_req, bool enable_sg) > -{ > - u16 csr = 0; > - > - /* > - * eDMA hardware SGs require the TCDs to be stored in little > - * endian format irrespective of the register endian model. > - * So we put the value in little endian in memory, waiting > - * for fsl_edma_set_tcd_regs doing the swap. > - */ > - tcd->saddr = cpu_to_le32(src); > - tcd->daddr = cpu_to_le32(dst); > - > - tcd->attr = cpu_to_le16(attr); > - > - tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); > - > - tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); > - tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); > - > - tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); > - tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); > - > - tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); > - > - tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); > - if (major_int) > - csr |= EDMA_TCD_CSR_INT_MAJOR; > - > - if (disable_req) > - csr |= EDMA_TCD_CSR_D_REQ; > - > - if (enable_sg) > - csr |= EDMA_TCD_CSR_E_SG; > - > - tcd->csr = cpu_to_le16(csr); > -} > - > -static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan > *fsl_chan, > - int sg_len) > -{ > - struct fsl_edma_desc *fsl_desc; > - int i; > - > - fsl_desc = kzalloc(sizeof(*fsl_desc) + sizeof(struct > fsl_edma_sw_tcd) * sg_len, > - GFP_NOWAIT); > - if (!fsl_desc) > - return NULL; > - > - fsl_desc->echan = fsl_chan; > - fsl_desc->n_tcds = sg_len; > - for (i = 0; i < sg_len; i++) { > - fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, > - GFP_NOWAIT, &fsl_desc->tcd[i].ptcd); > - if (!fsl_desc->tcd[i].vtcd) > - goto err; > - } > - return fsl_desc; > - > -err: > - while (--i >= 0) > - dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, > - fsl_desc->tcd[i].ptcd); > - kfree(fsl_desc); > - return NULL; > -} > - > -static struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( > - struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, > - size_t period_len, enum dma_transfer_direction direction, > - unsigned long flags) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - struct fsl_edma_desc *fsl_desc; > - dma_addr_t dma_buf_next; > - int sg_len, i; > - u32 src_addr, dst_addr, last_sg, nbytes; > - u16 soff, doff, iter; > - > - if (!is_slave_direction(fsl_chan->fsc.dir)) > - return NULL; > - > - sg_len = buf_len / period_len; > - fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); > - if (!fsl_desc) > - return NULL; > - fsl_desc->iscyclic = true; > - > - dma_buf_next = dma_addr; > - nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; > - iter = period_len / nbytes; > - > - for (i = 0; i < sg_len; i++) { > - if (dma_buf_next >= dma_addr + buf_len) > - dma_buf_next = dma_addr; > - > - /* get next sg's physical address */ > - last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; > - > - if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { > - src_addr = dma_buf_next; > - dst_addr = fsl_chan->fsc.dev_addr; > - soff = fsl_chan->fsc.addr_width; > - doff = 0; > - } else { > - src_addr = fsl_chan->fsc.dev_addr; > - dst_addr = dma_buf_next; > - soff = 0; > - doff = fsl_chan->fsc.addr_width; > - } > - > - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, > - fsl_chan->fsc.attr, soff, nbytes, 0, iter, > - iter, doff, last_sg, true, false, true); > - dma_buf_next += period_len; > - } > - > - return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); > -} > - > -static struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( > - struct dma_chan *chan, struct scatterlist *sgl, > - unsigned int sg_len, enum dma_transfer_direction direction, > - unsigned long flags, void *context) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - struct fsl_edma_desc *fsl_desc; > - struct scatterlist *sg; > - u32 src_addr, dst_addr, last_sg, nbytes; > - u16 soff, doff, iter; > - int i; > - > - if (!is_slave_direction(fsl_chan->fsc.dir)) > - return NULL; > - > - fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); > - if (!fsl_desc) > - return NULL; > - fsl_desc->iscyclic = false; > - > - nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; > - for_each_sg(sgl, sg, sg_len, i) { > - /* get next sg's physical address */ > - last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; > - > - if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { > - src_addr = sg_dma_address(sg); > - dst_addr = fsl_chan->fsc.dev_addr; > - soff = fsl_chan->fsc.addr_width; > - doff = 0; > - } else { > - src_addr = fsl_chan->fsc.dev_addr; > - dst_addr = sg_dma_address(sg); > - soff = 0; > - doff = fsl_chan->fsc.addr_width; > - } > - > - iter = sg_dma_len(sg) / nbytes; > - if (i < sg_len - 1) { > - last_sg = fsl_desc->tcd[(i + 1)].ptcd; > - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, > - dst_addr, fsl_chan->fsc.attr, soff, > - nbytes, 0, iter, iter, doff, last_sg, > - false, false, true); > - } else { > - last_sg = 0; > - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, > - dst_addr, fsl_chan->fsc.attr, soff, > - nbytes, 0, iter, iter, doff, last_sg, > - true, true, false); > - } > - } > - > - return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); > -} > - > -static void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) > -{ > - struct virt_dma_desc *vdesc; > - > - vdesc = vchan_next_desc(&fsl_chan->vchan); > - if (!vdesc) > - return; > - fsl_chan->edesc = to_fsl_edma_desc(vdesc); > - fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); > - fsl_edma_enable_request(fsl_chan); > - fsl_chan->status = DMA_IN_PROGRESS; > - fsl_chan->idle = false; > -} > +#include "fsl-edma-common.h" > > static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) > { > @@ -730,25 +93,6 @@ static irqreturn_t fsl_edma_irq_handler(int irq, > void *dev_id) > return fsl_edma_err_handler(irq, dev_id); > } > > -static void fsl_edma_issue_pending(struct dma_chan *chan) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - unsigned long flags; > - > - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > - > - if (unlikely(fsl_chan->pm_state != RUNNING)) { > - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > - /* cannot submit due to suspend */ > - return; > - } > - > - if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) > - fsl_edma_xfer_desc(fsl_chan); > - > - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > -} > - > static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, > struct of_dma *ofdma) > { > @@ -781,34 +125,6 @@ static struct dma_chan *fsl_edma_xlate(struct > of_phandle_args *dma_spec, > return NULL; > } > > -static int fsl_edma_alloc_chan_resources(struct dma_chan *chan) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - > - fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, > - sizeof(struct fsl_edma_hw_tcd), > - 32, 0); > - return 0; > -} > - > -static void fsl_edma_free_chan_resources(struct dma_chan *chan) > -{ > - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); > - unsigned long flags; > - LIST_HEAD(head); > - > - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); > - fsl_edma_disable_request(fsl_chan); > - fsl_edma_chan_mux(fsl_chan, 0, false); > - fsl_chan->edesc = NULL; > - vchan_get_all_descriptors(&fsl_chan->vchan, &head); > - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); > - > - vchan_dma_desc_free_list(&fsl_chan->vchan, &head); > - dma_pool_destroy(fsl_chan->tcd_pool); > - fsl_chan->tcd_pool = NULL; > -} > - > static int > fsl_edma_irq_init(struct platform_device *pdev, struct > fsl_edma_engine *fsl_edma) > { > @@ -995,17 +311,6 @@ static int fsl_edma_probe(struct platform_device *pdev) > return 0; > } > > -static void fsl_edma_cleanup_vchan(struct dma_device *dmadev) > -{ > - struct fsl_edma_chan *chan, *_chan; > - > - list_for_each_entry_safe(chan, _chan, > - &dmadev->channels, vchan.chan.device_node) { > - list_del(&chan->vchan.chan.device_node); > - tasklet_kill(&chan->vchan.task); > - } > -} > - > static int fsl_edma_remove(struct platform_device *pdev) > { > struct device_node *np = pdev->dev.of_node;

diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 203a99d68315..66022f59fca4 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -31,7 +31,7 @@ obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/ obj-$(CONFIG_DW_DMAC_CORE) += dw/ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o obj-$(CONFIG_FSL_DMA) += fsldma.o -obj-$(CONFIG_FSL_EDMA) += fsl-edma.o +obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o obj-$(CONFIG_FSL_RAID) += fsl_raid.o obj-$(CONFIG_HSU_DMA) += hsu/ obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c new file mode 100644 index 000000000000..0ae7094f477a --- /dev/null +++ b/drivers/dma/fsl-edma-common.c @@ -0,0 +1,576 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2013-2014 Freescale Semiconductor, Inc +// Copyright (c) 2017 Sysam, Angelo Dureghello <angelo@sysam.it> + +#include <linux/dmapool.h> +#include <linux/module.h> +#include <linux/slab.h> + +#include "fsl-edma-common.h" + +/* + * R/W functions for big- or little-endian registers: + * The eDMA controller's endian is independent of the CPU core's endian. + * For the big-endian IP module, the offset for 8-bit or 16-bit registers + * should also be swapped opposite to that in little-endian IP. + */ +u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) +{ + if (edma->big_endian) + return ioread32be(addr); + else + return ioread32(addr); +} +EXPORT_SYMBOL_GPL(edma_readl); + +void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr) +{ + /* swap the reg offset for these in big-endian mode */ + if (edma->big_endian) + iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); + else + iowrite8(val, addr); +} +EXPORT_SYMBOL_GPL(edma_writeb); + +void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr) +{ + /* swap the reg offset for these in big-endian mode */ + if (edma->big_endian) + iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); + else + iowrite16(val, addr); +} +EXPORT_SYMBOL_GPL(edma_writew); + +void edma_writel(struct fsl_edma_engine *edma, u32 val, void __iomem *addr) +{ + if (edma->big_endian) + iowrite32be(val, addr); + else + iowrite32(val, addr); +} +EXPORT_SYMBOL_GPL(edma_writel); + +struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct fsl_edma_chan, vchan.chan); +} +EXPORT_SYMBOL_GPL(to_fsl_edma_chan); + +struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct fsl_edma_desc, vdesc); +} +EXPORT_SYMBOL_GPL(to_fsl_edma_desc); + +static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) +{ + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI); + edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ); +} + +void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) +{ + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ); + edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI); +} +EXPORT_SYMBOL_GPL(fsl_edma_disable_request); + +void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, + unsigned int slot, bool enable) +{ + u32 ch = fsl_chan->vchan.chan.chan_id; + void __iomem *muxaddr; + unsigned int chans_per_mux, ch_off; + + chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; + ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; + muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; + slot = EDMAMUX_CHCFG_SOURCE(slot); + + if (enable) + iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); + else + iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); +} +EXPORT_SYMBOL_GPL(fsl_edma_chan_mux); + +static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) +{ + switch (addr_width) { + case 1: + return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; + case 2: + return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; + case 4: + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; + case 8: + return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; + default: + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; + } +} + +void fsl_edma_free_desc(struct virt_dma_desc *vdesc) +{ + struct fsl_edma_desc *fsl_desc; + int i; + + fsl_desc = to_fsl_edma_desc(vdesc); + for (i = 0; i < fsl_desc->n_tcds; i++) + dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); + kfree(fsl_desc); +} +EXPORT_SYMBOL_GPL(fsl_edma_free_desc); + +int fsl_edma_terminate_all(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma_disable_request(fsl_chan); + fsl_chan->edesc = NULL; + fsl_chan->idle = true; + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_terminate_all); + +int fsl_edma_pause(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (fsl_chan->edesc) { + fsl_edma_disable_request(fsl_chan); + fsl_chan->status = DMA_PAUSED; + fsl_chan->idle = true; + } + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_pause); + +int fsl_edma_resume(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (fsl_chan->edesc) { + fsl_edma_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; + fsl_chan->idle = false; + } + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_resume); + +int fsl_edma_slave_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + + fsl_chan->fsc.dir = cfg->direction; + if (cfg->direction == DMA_DEV_TO_MEM) { + fsl_chan->fsc.dev_addr = cfg->src_addr; + fsl_chan->fsc.addr_width = cfg->src_addr_width; + fsl_chan->fsc.burst = cfg->src_maxburst; + fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width); + } else if (cfg->direction == DMA_MEM_TO_DEV) { + fsl_chan->fsc.dev_addr = cfg->dst_addr; + fsl_chan->fsc.addr_width = cfg->dst_addr_width; + fsl_chan->fsc.burst = cfg->dst_maxburst; + fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width); + } else + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_slave_config); + +static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, + struct virt_dma_desc *vdesc, bool in_progress) +{ + struct fsl_edma_desc *edesc = fsl_chan->edesc; + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + enum dma_transfer_direction dir = fsl_chan->fsc.dir; + dma_addr_t cur_addr, dma_addr; + size_t len, size; + int i; + + /* calculate the total size in this desc */ + for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) + len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) + * le16_to_cpu(edesc->tcd[i].vtcd->biter); + + if (!in_progress) + return len; + + if (dir == DMA_MEM_TO_DEV) + cur_addr = edma_readl( + fsl_chan->edma, addr + EDMA_TCD_SADDR(ch)); + else + cur_addr = edma_readl( + fsl_chan->edma, addr + EDMA_TCD_DADDR(ch)); + + /* figure out the finished and calculate the residue */ + for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { + size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) + * le16_to_cpu(edesc->tcd[i].vtcd->biter); + if (dir == DMA_MEM_TO_DEV) + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); + else + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); + + len -= size; + if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { + len += dma_addr + size - cur_addr; + break; + } + } + + return len; +} + +enum dma_status fsl_edma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct virt_dma_desc *vdesc; + enum dma_status status; + unsigned long flags; + + status = dma_cookie_status(chan, cookie, txstate); + if (status == DMA_COMPLETE) + return status; + + if (!txstate) + return fsl_chan->status; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); + if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) + txstate->residue = + fsl_edma_desc_residue(fsl_chan, vdesc, true); + else if (vdesc) + txstate->residue = + fsl_edma_desc_residue(fsl_chan, vdesc, false); + else + txstate->residue = 0; + + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + return fsl_chan->status; +} +EXPORT_SYMBOL_GPL(fsl_edma_tx_status); + +static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, + struct fsl_edma_hw_tcd *tcd) +{ + struct fsl_edma_engine *edma = fsl_chan->edma; + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + /* + * TCD parameters are stored in struct fsl_edma_hw_tcd in little + * endian format. However, we need to load the TCD registers in + * big- or little-endian obeying the eDMA engine model endian. + */ + edma_writew(edma, 0, addr + EDMA_TCD_CSR(ch)); + edma_writel(edma, le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR(ch)); + edma_writel(edma, le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR(ch)); + + edma_writew(edma, le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR(ch)); + edma_writew(edma, le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF(ch)); + + edma_writel(edma, le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES(ch)); + edma_writel(edma, le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST(ch)); + + edma_writew(edma, le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER(ch)); + edma_writew(edma, le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER(ch)); + edma_writew(edma, le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF(ch)); + + edma_writel(edma, + le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA(ch)); + + edma_writew(edma, le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR(ch)); +} + +static inline +void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, + u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, + u16 biter, u16 doff, u32 dlast_sga, bool major_int, + bool disable_req, bool enable_sg) +{ + u16 csr = 0; + + /* + * eDMA hardware SGs require the TCDs to be stored in little + * endian format irrespective of the register endian model. + * So we put the value in little endian in memory, waiting + * for fsl_edma_set_tcd_regs doing the swap. + */ + tcd->saddr = cpu_to_le32(src); + tcd->daddr = cpu_to_le32(dst); + + tcd->attr = cpu_to_le16(attr); + + tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); + + tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); + tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); + + tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); + tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); + + tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); + + tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); + if (major_int) + csr |= EDMA_TCD_CSR_INT_MAJOR; + + if (disable_req) + csr |= EDMA_TCD_CSR_D_REQ; + + if (enable_sg) + csr |= EDMA_TCD_CSR_E_SG; + + tcd->csr = cpu_to_le16(csr); +} + +static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, + int sg_len) +{ + struct fsl_edma_desc *fsl_desc; + int i; + + fsl_desc = kzalloc(sizeof(*fsl_desc) + + sizeof(struct fsl_edma_sw_tcd) * + sg_len, GFP_NOWAIT); + if (!fsl_desc) + return NULL; + + fsl_desc->echan = fsl_chan; + fsl_desc->n_tcds = sg_len; + for (i = 0; i < sg_len; i++) { + fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, + GFP_NOWAIT, &fsl_desc->tcd[i].ptcd); + if (!fsl_desc->tcd[i].vtcd) + goto err; + } + return fsl_desc; + +err: + while (--i >= 0) + dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); + kfree(fsl_desc); + return NULL; +} + +struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct fsl_edma_desc *fsl_desc; + dma_addr_t dma_buf_next; + int sg_len, i; + u32 src_addr, dst_addr, last_sg, nbytes; + u16 soff, doff, iter; + + if (!is_slave_direction(fsl_chan->fsc.dir)) + return NULL; + + sg_len = buf_len / period_len; + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = true; + + dma_buf_next = dma_addr; + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; + iter = period_len / nbytes; + + for (i = 0; i < sg_len; i++) { + if (dma_buf_next >= dma_addr + buf_len) + dma_buf_next = dma_addr; + + /* get next sg's physical address */ + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; + + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + src_addr = dma_buf_next; + dst_addr = fsl_chan->fsc.dev_addr; + soff = fsl_chan->fsc.addr_width; + doff = 0; + } else { + src_addr = fsl_chan->fsc.dev_addr; + dst_addr = dma_buf_next; + soff = 0; + doff = fsl_chan->fsc.addr_width; + } + + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, + fsl_chan->fsc.attr, soff, nbytes, 0, iter, + iter, doff, last_sg, true, false, true); + dma_buf_next += period_len; + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} +EXPORT_SYMBOL_GPL(fsl_edma_prep_dma_cyclic); + +struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct fsl_edma_desc *fsl_desc; + struct scatterlist *sg; + u32 src_addr, dst_addr, last_sg, nbytes; + u16 soff, doff, iter; + int i; + + if (!is_slave_direction(fsl_chan->fsc.dir)) + return NULL; + + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = false; + + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; + for_each_sg(sgl, sg, sg_len, i) { + /* get next sg's physical address */ + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; + + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + src_addr = sg_dma_address(sg); + dst_addr = fsl_chan->fsc.dev_addr; + soff = fsl_chan->fsc.addr_width; + doff = 0; + } else { + src_addr = fsl_chan->fsc.dev_addr; + dst_addr = sg_dma_address(sg); + soff = 0; + doff = fsl_chan->fsc.addr_width; + } + + iter = sg_dma_len(sg) / nbytes; + if (i < sg_len - 1) { + last_sg = fsl_desc->tcd[(i + 1)].ptcd; + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->fsc.attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + false, false, true); + } else { + last_sg = 0; + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->fsc.attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + true, true, false); + } + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} +EXPORT_SYMBOL_GPL(fsl_edma_prep_slave_sg); + +void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) +{ + struct virt_dma_desc *vdesc; + + vdesc = vchan_next_desc(&fsl_chan->vchan); + if (!vdesc) + return; + fsl_chan->edesc = to_fsl_edma_desc(vdesc); + fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); + fsl_edma_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; + fsl_chan->idle = false; +} +EXPORT_SYMBOL_GPL(fsl_edma_xfer_desc); + +void fsl_edma_issue_pending(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + + if (unlikely(fsl_chan->pm_state != RUNNING)) { + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + /* cannot submit due to suspend */ + return; + } + + if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) + fsl_edma_xfer_desc(fsl_chan); + + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); +} +EXPORT_SYMBOL_GPL(fsl_edma_issue_pending); + +int fsl_edma_alloc_chan_resources(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + + fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, + sizeof(struct fsl_edma_hw_tcd), + 32, 0); + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_alloc_chan_resources); + +void fsl_edma_free_chan_resources(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma_disable_request(fsl_chan); + fsl_edma_chan_mux(fsl_chan, 0, false); + fsl_chan->edesc = NULL; + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + dma_pool_destroy(fsl_chan->tcd_pool); + fsl_chan->tcd_pool = NULL; +} +EXPORT_SYMBOL_GPL(fsl_edma_free_chan_resources); + +void fsl_edma_cleanup_vchan(struct dma_device *dmadev) +{ + struct fsl_edma_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, + &dmadev->channels, vchan.chan.device_node) { + list_del(&chan->vchan.chan.device_node); + tasklet_kill(&chan->vchan.task); + } +} +EXPORT_SYMBOL_GPL(fsl_edma_cleanup_vchan); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h new file mode 100644 index 000000000000..f3ea68e15b23 --- /dev/null +++ b/drivers/dma/fsl-edma-common.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2013-2014 Freescale Semiconductor, Inc. + * Copyright 2018 Angelo Dureghello <angelo@sysam.it> + */ +#ifndef _FSL_EDMA_COMMON_H_ +#define _FSL_EDMA_COMMON_H_ + +#include "virt-dma.h" + +#define EDMA_CR 0x00 +#define EDMA_ES 0x04 +#define EDMA_ERQ 0x0C +#define EDMA_EEI 0x14 +#define EDMA_SERQ 0x1B +#define EDMA_CERQ 0x1A +#define EDMA_SEEI 0x19 +#define EDMA_CEEI 0x18 +#define EDMA_CINT 0x1F +#define EDMA_CERR 0x1E +#define EDMA_SSRT 0x1D +#define EDMA_CDNE 0x1C +#define EDMA_INTR 0x24 +#define EDMA_ERR 0x2C + +#define EDMA_TCD_SADDR(x) (0x1000 + 32 * (x)) +#define EDMA_TCD_SOFF(x) (0x1004 + 32 * (x)) +#define EDMA_TCD_ATTR(x) (0x1006 + 32 * (x)) +#define EDMA_TCD_NBYTES(x) (0x1008 + 32 * (x)) +#define EDMA_TCD_SLAST(x) (0x100C + 32 * (x)) +#define EDMA_TCD_DADDR(x) (0x1010 + 32 * (x)) +#define EDMA_TCD_DOFF(x) (0x1014 + 32 * (x)) +#define EDMA_TCD_CITER_ELINK(x) (0x1016 + 32 * (x)) +#define EDMA_TCD_CITER(x) (0x1016 + 32 * (x)) +#define EDMA_TCD_DLAST_SGA(x) (0x1018 + 32 * (x)) +#define EDMA_TCD_CSR(x) (0x101C + 32 * (x)) +#define EDMA_TCD_BITER_ELINK(x) (0x101E + 32 * (x)) +#define EDMA_TCD_BITER(x) (0x101E + 32 * (x)) + +#define EDMA_CR_EDBG BIT(1) +#define EDMA_CR_ERCA BIT(2) +#define EDMA_CR_ERGA BIT(3) +#define EDMA_CR_HOE BIT(4) +#define EDMA_CR_HALT BIT(5) +#define EDMA_CR_CLM BIT(6) +#define EDMA_CR_EMLM BIT(7) +#define EDMA_CR_ECX BIT(16) +#define EDMA_CR_CX BIT(17) + +#define EDMA_SEEI_SEEI(x) ((x) & 0x1F) +#define EDMA_CEEI_CEEI(x) ((x) & 0x1F) +#define EDMA_CINT_CINT(x) ((x) & 0x1F) +#define EDMA_CERR_CERR(x) ((x) & 0x1F) + +#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) +#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) +#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) +#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) +#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) +#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) +#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) +#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) +#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) +#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) +#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) +#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) +#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) +#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) + +#define EDMA_TCD_SOFF_SOFF(x) (x) +#define EDMA_TCD_NBYTES_NBYTES(x) (x) +#define EDMA_TCD_SLAST_SLAST(x) (x) +#define EDMA_TCD_DADDR_DADDR(x) (x) +#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) +#define EDMA_TCD_DOFF_DOFF(x) (x) +#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) +#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) + +#define EDMA_TCD_CSR_START BIT(0) +#define EDMA_TCD_CSR_INT_MAJOR BIT(1) +#define EDMA_TCD_CSR_INT_HALF BIT(2) +#define EDMA_TCD_CSR_D_REQ BIT(3) +#define EDMA_TCD_CSR_E_SG BIT(4) +#define EDMA_TCD_CSR_E_LINK BIT(5) +#define EDMA_TCD_CSR_ACTIVE BIT(6) +#define EDMA_TCD_CSR_DONE BIT(7) + +#define EDMAMUX_CHCFG_DIS 0x0 +#define EDMAMUX_CHCFG_ENBL 0x80 +#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) + +#define DMAMUX_NR 2 + +#define FSL_EDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) +enum fsl_edma_pm_state { + RUNNING = 0, + SUSPENDED, +}; + +struct fsl_edma_hw_tcd { + __le32 saddr; + __le16 soff; + __le16 attr; + __le32 nbytes; + __le32 slast; + __le32 daddr; + __le16 doff; + __le16 citer; + __le32 dlast_sga; + __le16 csr; + __le16 biter; +}; + +struct fsl_edma_sw_tcd { + dma_addr_t ptcd; + struct fsl_edma_hw_tcd *vtcd; +}; + +struct fsl_edma_slave_config { + enum dma_transfer_direction dir; + enum dma_slave_buswidth addr_width; + u32 dev_addr; + u32 burst; + u32 attr; +}; + +struct fsl_edma_chan { + struct virt_dma_chan vchan; + enum dma_status status; + enum fsl_edma_pm_state pm_state; + bool idle; + u32 slave_id; + struct fsl_edma_engine *edma; + struct fsl_edma_desc *edesc; + struct fsl_edma_slave_config fsc; + struct dma_pool *tcd_pool; +}; + +struct fsl_edma_desc { + struct virt_dma_desc vdesc; + struct fsl_edma_chan *echan; + bool iscyclic; + unsigned int n_tcds; + struct fsl_edma_sw_tcd tcd[]; +}; + +struct fsl_edma_engine { + struct dma_device dma_dev; + void __iomem *membase; + void __iomem *muxbase[DMAMUX_NR]; + struct clk *muxclk[DMAMUX_NR]; + struct mutex fsl_edma_mutex; + u32 n_chans; + int txirq; + int errirq; + bool big_endian; + struct fsl_edma_chan chans[]; +}; + +u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr); +void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr); +void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr); +void edma_writel(struct fsl_edma_engine *edma, u32 val, void __iomem *addr); + +struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan); +struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd); + +void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan); +void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, + unsigned int slot, bool enable); +void fsl_edma_free_desc(struct virt_dma_desc *vdesc); +int fsl_edma_terminate_all(struct dma_chan *chan); +int fsl_edma_pause(struct dma_chan *chan); +int fsl_edma_resume(struct dma_chan *chan); +int fsl_edma_slave_config(struct dma_chan *chan, + struct dma_slave_config *cfg); +enum dma_status fsl_edma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate); +struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags); +struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context); +void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan); +void fsl_edma_issue_pending(struct dma_chan *chan); +int fsl_edma_alloc_chan_resources(struct dma_chan *chan); +void fsl_edma_free_chan_resources(struct dma_chan *chan); +void fsl_edma_cleanup_vchan(struct dma_device *dmadev); + +#endif /* _FSL_EDMA_COMMON_H_ */ diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c index c7568869284e..337ad0c0ea78 100644 --- a/drivers/dma/fsl-edma.c +++ b/drivers/dma/fsl-edma.c @@ -13,653 +13,16 @@ * option) any later version. */ -#include <linux/init.h> #include <linux/module.h> #include <linux/interrupt.h> #include <linux/clk.h> -#include <linux/dma-mapping.h> -#include <linux/dmapool.h> -#include <linux/slab.h> -#include <linux/spinlock.h> #include <linux/of.h> #include <linux/of_device.h> #include <linux/of_address.h> #include <linux/of_irq.h> #include <linux/of_dma.h> -#include "virt-dma.h" - -#define EDMA_CR 0x00 -#define EDMA_ES 0x04 -#define EDMA_ERQ 0x0C -#define EDMA_EEI 0x14 -#define EDMA_SERQ 0x1B -#define EDMA_CERQ 0x1A -#define EDMA_SEEI 0x19 -#define EDMA_CEEI 0x18 -#define EDMA_CINT 0x1F -#define EDMA_CERR 0x1E -#define EDMA_SSRT 0x1D -#define EDMA_CDNE 0x1C -#define EDMA_INTR 0x24 -#define EDMA_ERR 0x2C - -#define EDMA_TCD_SADDR(x) (0x1000 + 32 * (x)) -#define EDMA_TCD_SOFF(x) (0x1004 + 32 * (x)) -#define EDMA_TCD_ATTR(x) (0x1006 + 32 * (x)) -#define EDMA_TCD_NBYTES(x) (0x1008 + 32 * (x)) -#define EDMA_TCD_SLAST(x) (0x100C + 32 * (x)) -#define EDMA_TCD_DADDR(x) (0x1010 + 32 * (x)) -#define EDMA_TCD_DOFF(x) (0x1014 + 32 * (x)) -#define EDMA_TCD_CITER_ELINK(x) (0x1016 + 32 * (x)) -#define EDMA_TCD_CITER(x) (0x1016 + 32 * (x)) -#define EDMA_TCD_DLAST_SGA(x) (0x1018 + 32 * (x)) -#define EDMA_TCD_CSR(x) (0x101C + 32 * (x)) -#define EDMA_TCD_BITER_ELINK(x) (0x101E + 32 * (x)) -#define EDMA_TCD_BITER(x) (0x101E + 32 * (x)) - -#define EDMA_CR_EDBG BIT(1) -#define EDMA_CR_ERCA BIT(2) -#define EDMA_CR_ERGA BIT(3) -#define EDMA_CR_HOE BIT(4) -#define EDMA_CR_HALT BIT(5) -#define EDMA_CR_CLM BIT(6) -#define EDMA_CR_EMLM BIT(7) -#define EDMA_CR_ECX BIT(16) -#define EDMA_CR_CX BIT(17) - -#define EDMA_SEEI_SEEI(x) ((x) & 0x1F) -#define EDMA_CEEI_CEEI(x) ((x) & 0x1F) -#define EDMA_CINT_CINT(x) ((x) & 0x1F) -#define EDMA_CERR_CERR(x) ((x) & 0x1F) - -#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) -#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) -#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) -#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) -#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) -#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) -#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) -#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) -#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) -#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) -#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) -#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) -#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) -#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) - -#define EDMA_TCD_SOFF_SOFF(x) (x) -#define EDMA_TCD_NBYTES_NBYTES(x) (x) -#define EDMA_TCD_SLAST_SLAST(x) (x) -#define EDMA_TCD_DADDR_DADDR(x) (x) -#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) -#define EDMA_TCD_DOFF_DOFF(x) (x) -#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) -#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) - -#define EDMA_TCD_CSR_START BIT(0) -#define EDMA_TCD_CSR_INT_MAJOR BIT(1) -#define EDMA_TCD_CSR_INT_HALF BIT(2) -#define EDMA_TCD_CSR_D_REQ BIT(3) -#define EDMA_TCD_CSR_E_SG BIT(4) -#define EDMA_TCD_CSR_E_LINK BIT(5) -#define EDMA_TCD_CSR_ACTIVE BIT(6) -#define EDMA_TCD_CSR_DONE BIT(7) - -#define EDMAMUX_CHCFG_DIS 0x0 -#define EDMAMUX_CHCFG_ENBL 0x80 -#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) - -#define DMAMUX_NR 2 - -#define FSL_EDMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) -enum fsl_edma_pm_state { - RUNNING = 0, - SUSPENDED, -}; - -struct fsl_edma_hw_tcd { - __le32 saddr; - __le16 soff; - __le16 attr; - __le32 nbytes; - __le32 slast; - __le32 daddr; - __le16 doff; - __le16 citer; - __le32 dlast_sga; - __le16 csr; - __le16 biter; -}; - -struct fsl_edma_sw_tcd { - dma_addr_t ptcd; - struct fsl_edma_hw_tcd *vtcd; -}; - -struct fsl_edma_slave_config { - enum dma_transfer_direction dir; - enum dma_slave_buswidth addr_width; - u32 dev_addr; - u32 burst; - u32 attr; -}; - -struct fsl_edma_chan { - struct virt_dma_chan vchan; - enum dma_status status; - enum fsl_edma_pm_state pm_state; - bool idle; - u32 slave_id; - struct fsl_edma_engine *edma; - struct fsl_edma_desc *edesc; - struct fsl_edma_slave_config fsc; - struct dma_pool *tcd_pool; -}; - -struct fsl_edma_desc { - struct virt_dma_desc vdesc; - struct fsl_edma_chan *echan; - bool iscyclic; - unsigned int n_tcds; - struct fsl_edma_sw_tcd tcd[]; -}; - -struct fsl_edma_engine { - struct dma_device dma_dev; - void __iomem *membase; - void __iomem *muxbase[DMAMUX_NR]; - struct clk *muxclk[DMAMUX_NR]; - struct mutex fsl_edma_mutex; - u32 n_chans; - int txirq; - int errirq; - bool big_endian; - struct fsl_edma_chan chans[]; -}; - -/* - * R/W functions for big- or little-endian registers: - * The eDMA controller's endian is independent of the CPU core's endian. - * For the big-endian IP module, the offset for 8-bit or 16-bit registers - * should also be swapped opposite to that in little-endian IP. - */ - -static u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) -{ - if (edma->big_endian) - return ioread32be(addr); - else - return ioread32(addr); -} - -static void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr) -{ - /* swap the reg offset for these in big-endian mode */ - if (edma->big_endian) - iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); - else - iowrite8(val, addr); -} - -static void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr) -{ - /* swap the reg offset for these in big-endian mode */ - if (edma->big_endian) - iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); - else - iowrite16(val, addr); -} - -static void edma_writel(struct fsl_edma_engine *edma, u32 val, void __iomem *addr) -{ - if (edma->big_endian) - iowrite32be(val, addr); - else - iowrite32(val, addr); -} - -static struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) -{ - return container_of(chan, struct fsl_edma_chan, vchan.chan); -} - -static struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) -{ - return container_of(vd, struct fsl_edma_desc, vdesc); -} - -static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) -{ - void __iomem *addr = fsl_chan->edma->membase; - u32 ch = fsl_chan->vchan.chan.chan_id; - - edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI); - edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ); -} - -static void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) -{ - void __iomem *addr = fsl_chan->edma->membase; - u32 ch = fsl_chan->vchan.chan.chan_id; - - edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ); - edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI); -} - -static void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, - unsigned int slot, bool enable) -{ - u32 ch = fsl_chan->vchan.chan.chan_id; - void __iomem *muxaddr; - unsigned chans_per_mux, ch_off; - - chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; - ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; - muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; - slot = EDMAMUX_CHCFG_SOURCE(slot); - - if (enable) - iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); - else - iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); -} - -static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) -{ - switch (addr_width) { - case 1: - return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; - case 2: - return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; - case 4: - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; - case 8: - return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; - default: - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; - } -} - -static void fsl_edma_free_desc(struct virt_dma_desc *vdesc) -{ - struct fsl_edma_desc *fsl_desc; - int i; - - fsl_desc = to_fsl_edma_desc(vdesc); - for (i = 0; i < fsl_desc->n_tcds; i++) - dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, - fsl_desc->tcd[i].ptcd); - kfree(fsl_desc); -} - -static int fsl_edma_terminate_all(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - LIST_HEAD(head); - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - fsl_edma_disable_request(fsl_chan); - fsl_chan->edesc = NULL; - fsl_chan->idle = true; - vchan_get_all_descriptors(&fsl_chan->vchan, &head); - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - vchan_dma_desc_free_list(&fsl_chan->vchan, &head); - return 0; -} - -static int fsl_edma_pause(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - if (fsl_chan->edesc) { - fsl_edma_disable_request(fsl_chan); - fsl_chan->status = DMA_PAUSED; - fsl_chan->idle = true; - } - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - return 0; -} - -static int fsl_edma_resume(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - if (fsl_chan->edesc) { - fsl_edma_enable_request(fsl_chan); - fsl_chan->status = DMA_IN_PROGRESS; - fsl_chan->idle = false; - } - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - return 0; -} - -static int fsl_edma_slave_config(struct dma_chan *chan, - struct dma_slave_config *cfg) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - - fsl_chan->fsc.dir = cfg->direction; - if (cfg->direction == DMA_DEV_TO_MEM) { - fsl_chan->fsc.dev_addr = cfg->src_addr; - fsl_chan->fsc.addr_width = cfg->src_addr_width; - fsl_chan->fsc.burst = cfg->src_maxburst; - fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width); - } else if (cfg->direction == DMA_MEM_TO_DEV) { - fsl_chan->fsc.dev_addr = cfg->dst_addr; - fsl_chan->fsc.addr_width = cfg->dst_addr_width; - fsl_chan->fsc.burst = cfg->dst_maxburst; - fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width); - } else { - return -EINVAL; - } - return 0; -} - -static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, - struct virt_dma_desc *vdesc, bool in_progress) -{ - struct fsl_edma_desc *edesc = fsl_chan->edesc; - void __iomem *addr = fsl_chan->edma->membase; - u32 ch = fsl_chan->vchan.chan.chan_id; - enum dma_transfer_direction dir = fsl_chan->fsc.dir; - dma_addr_t cur_addr, dma_addr; - size_t len, size; - int i; - - /* calculate the total size in this desc */ - for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) - len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) - * le16_to_cpu(edesc->tcd[i].vtcd->biter); - - if (!in_progress) - return len; - - if (dir == DMA_MEM_TO_DEV) - cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_SADDR(ch)); - else - cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_DADDR(ch)); - - /* figure out the finished and calculate the residue */ - for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { - size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) - * le16_to_cpu(edesc->tcd[i].vtcd->biter); - if (dir == DMA_MEM_TO_DEV) - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); - else - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); - - len -= size; - if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { - len += dma_addr + size - cur_addr; - break; - } - } - - return len; -} - -static enum dma_status fsl_edma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, struct dma_tx_state *txstate) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - struct virt_dma_desc *vdesc; - enum dma_status status; - unsigned long flags; - - status = dma_cookie_status(chan, cookie, txstate); - if (status == DMA_COMPLETE) - return status; - - if (!txstate) - return fsl_chan->status; - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); - if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) - txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, true); - else if (vdesc) - txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, false); - else - txstate->residue = 0; - - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - - return fsl_chan->status; -} - -static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, - struct fsl_edma_hw_tcd *tcd) -{ - struct fsl_edma_engine *edma = fsl_chan->edma; - void __iomem *addr = fsl_chan->edma->membase; - u32 ch = fsl_chan->vchan.chan.chan_id; - - /* - * TCD parameters are stored in struct fsl_edma_hw_tcd in little - * endian format. However, we need to load the TCD registers in - * big- or little-endian obeying the eDMA engine model endian. - */ - edma_writew(edma, 0, addr + EDMA_TCD_CSR(ch)); - edma_writel(edma, le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR(ch)); - edma_writel(edma, le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR(ch)); - - edma_writew(edma, le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR(ch)); - edma_writew(edma, le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF(ch)); - - edma_writel(edma, le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES(ch)); - edma_writel(edma, le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST(ch)); - - edma_writew(edma, le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER(ch)); - edma_writew(edma, le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER(ch)); - edma_writew(edma, le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF(ch)); - - edma_writel(edma, le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA(ch)); - - edma_writew(edma, le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR(ch)); -} - -static inline -void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, - u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, - u16 biter, u16 doff, u32 dlast_sga, bool major_int, - bool disable_req, bool enable_sg) -{ - u16 csr = 0; - - /* - * eDMA hardware SGs require the TCDs to be stored in little - * endian format irrespective of the register endian model. - * So we put the value in little endian in memory, waiting - * for fsl_edma_set_tcd_regs doing the swap. - */ - tcd->saddr = cpu_to_le32(src); - tcd->daddr = cpu_to_le32(dst); - - tcd->attr = cpu_to_le16(attr); - - tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); - - tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); - tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); - - tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); - tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); - - tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); - - tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); - if (major_int) - csr |= EDMA_TCD_CSR_INT_MAJOR; - - if (disable_req) - csr |= EDMA_TCD_CSR_D_REQ; - - if (enable_sg) - csr |= EDMA_TCD_CSR_E_SG; - - tcd->csr = cpu_to_le16(csr); -} - -static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, - int sg_len) -{ - struct fsl_edma_desc *fsl_desc; - int i; - - fsl_desc = kzalloc(sizeof(*fsl_desc) + sizeof(struct fsl_edma_sw_tcd) * sg_len, - GFP_NOWAIT); - if (!fsl_desc) - return NULL; - - fsl_desc->echan = fsl_chan; - fsl_desc->n_tcds = sg_len; - for (i = 0; i < sg_len; i++) { - fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, - GFP_NOWAIT, &fsl_desc->tcd[i].ptcd); - if (!fsl_desc->tcd[i].vtcd) - goto err; - } - return fsl_desc; - -err: - while (--i >= 0) - dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, - fsl_desc->tcd[i].ptcd); - kfree(fsl_desc); - return NULL; -} - -static struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction direction, - unsigned long flags) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - struct fsl_edma_desc *fsl_desc; - dma_addr_t dma_buf_next; - int sg_len, i; - u32 src_addr, dst_addr, last_sg, nbytes; - u16 soff, doff, iter; - - if (!is_slave_direction(fsl_chan->fsc.dir)) - return NULL; - - sg_len = buf_len / period_len; - fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); - if (!fsl_desc) - return NULL; - fsl_desc->iscyclic = true; - - dma_buf_next = dma_addr; - nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; - iter = period_len / nbytes; - - for (i = 0; i < sg_len; i++) { - if (dma_buf_next >= dma_addr + buf_len) - dma_buf_next = dma_addr; - - /* get next sg's physical address */ - last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; - - if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { - src_addr = dma_buf_next; - dst_addr = fsl_chan->fsc.dev_addr; - soff = fsl_chan->fsc.addr_width; - doff = 0; - } else { - src_addr = fsl_chan->fsc.dev_addr; - dst_addr = dma_buf_next; - soff = 0; - doff = fsl_chan->fsc.addr_width; - } - - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, - fsl_chan->fsc.attr, soff, nbytes, 0, iter, - iter, doff, last_sg, true, false, true); - dma_buf_next += period_len; - } - - return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); -} - -static struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - struct fsl_edma_desc *fsl_desc; - struct scatterlist *sg; - u32 src_addr, dst_addr, last_sg, nbytes; - u16 soff, doff, iter; - int i; - - if (!is_slave_direction(fsl_chan->fsc.dir)) - return NULL; - - fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); - if (!fsl_desc) - return NULL; - fsl_desc->iscyclic = false; - - nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; - for_each_sg(sgl, sg, sg_len, i) { - /* get next sg's physical address */ - last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; - - if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { - src_addr = sg_dma_address(sg); - dst_addr = fsl_chan->fsc.dev_addr; - soff = fsl_chan->fsc.addr_width; - doff = 0; - } else { - src_addr = fsl_chan->fsc.dev_addr; - dst_addr = sg_dma_address(sg); - soff = 0; - doff = fsl_chan->fsc.addr_width; - } - - iter = sg_dma_len(sg) / nbytes; - if (i < sg_len - 1) { - last_sg = fsl_desc->tcd[(i + 1)].ptcd; - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, - dst_addr, fsl_chan->fsc.attr, soff, - nbytes, 0, iter, iter, doff, last_sg, - false, false, true); - } else { - last_sg = 0; - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, - dst_addr, fsl_chan->fsc.attr, soff, - nbytes, 0, iter, iter, doff, last_sg, - true, true, false); - } - } - - return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); -} - -static void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) -{ - struct virt_dma_desc *vdesc; - - vdesc = vchan_next_desc(&fsl_chan->vchan); - if (!vdesc) - return; - fsl_chan->edesc = to_fsl_edma_desc(vdesc); - fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); - fsl_edma_enable_request(fsl_chan); - fsl_chan->status = DMA_IN_PROGRESS; - fsl_chan->idle = false; -} +#include "fsl-edma-common.h" static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) { @@ -730,25 +93,6 @@ static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) return fsl_edma_err_handler(irq, dev_id); } -static void fsl_edma_issue_pending(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - - if (unlikely(fsl_chan->pm_state != RUNNING)) { - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - /* cannot submit due to suspend */ - return; - } - - if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) - fsl_edma_xfer_desc(fsl_chan); - - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); -} - static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { @@ -781,34 +125,6 @@ static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, return NULL; } -static int fsl_edma_alloc_chan_resources(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - - fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, - sizeof(struct fsl_edma_hw_tcd), - 32, 0); - return 0; -} - -static void fsl_edma_free_chan_resources(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - LIST_HEAD(head); - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - fsl_edma_disable_request(fsl_chan); - fsl_edma_chan_mux(fsl_chan, 0, false); - fsl_chan->edesc = NULL; - vchan_get_all_descriptors(&fsl_chan->vchan, &head); - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - - vchan_dma_desc_free_list(&fsl_chan->vchan, &head); - dma_pool_destroy(fsl_chan->tcd_pool); - fsl_chan->tcd_pool = NULL; -} - static int fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) { @@ -995,17 +311,6 @@ static int fsl_edma_probe(struct platform_device *pdev) return 0; } -static void fsl_edma_cleanup_vchan(struct dma_device *dmadev) -{ - struct fsl_edma_chan *chan, *_chan; - - list_for_each_entry_safe(chan, _chan, - &dmadev->channels, vchan.chan.device_node) { - list_del(&chan->vchan.chan.device_node); - tasklet_kill(&chan->vchan.task); - } -} - static int fsl_edma_remove(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node;

[v8,1/3] dmaengine: fsl-edma: extract common fsl-edma code (no changes in behavior intended)

Commit Message

Comments

Patch