[V2,XDMA,1/1] dmaengine: xilinx: xdma: add xilinx xdma driver

Commit Message

Lizhi Hou Sept. 16, 2022, 3:26 p.m. UTC

Add driver to enable PCIe board which uses XDMA (the DMA/Bridge Subsystem
for PCI Express). For example, Xilinx Alveo PCIe devices.
    https://www.xilinx.com/products/boards-and-kits/alveo.html

The XDMA engine support up to 4 Host to Card (H2C) and 4 Card to Host (C2H)
channels. Memory transfers are specified on a per-channel basis in
descriptor linked lists, which the DMA fetches from host memory and
processes. Events such as descriptor completion and errors are signaled
using interrupts. The hardware detail is provided by
    https://docs.xilinx.com/r/en-US/pg195-pcie-dma/Introduction

This driver implements dmaengine APIs.
    - probe the available DMA channels
    - use dma_slave_map for channel lookup
    - use virtual channel to manage dmaengine tx descriptors
    - implement device_prep_slave_sg callback to handle host scatter gather
      list
    - implement device_config to config device address for DMA transfer

Signed-off-by: Lizhi Hou <[e-mail]>
Signed-off-by: Sonal Santan <[e-mail]>
Signed-off-by: Max Zhen <[e-mail]>
Signed-off-by: Brian Xu <[e-mail]>
---
 MAINTAINERS                            |  10 +
 drivers/dma/Kconfig                    |  13 +
 drivers/dma/xilinx/Makefile            |   1 +
 drivers/dma/xilinx/xdma-regs.h         | 171 +++++
 drivers/dma/xilinx/xdma.c              | 982 +++++++++++++++++++++++++
 include/linux/platform_data/amd_xdma.h |  34 +
 6 files changed, 1211 insertions(+)
 create mode 100644 drivers/dma/xilinx/xdma-regs.h
 create mode 100644 drivers/dma/xilinx/xdma.c
 create mode 100644 include/linux/platform_data/amd_xdma.h

Comments

Dave Jiang Sept. 16, 2022, 3:39 p.m. UTC | #1

On 9/16/2022 8:26 AM, Lizhi Hou wrote:
> Add driver to enable PCIe board which uses XDMA (the DMA/Bridge Subsystem
> for PCI Express). For example, Xilinx Alveo PCIe devices.
>      https://www.xilinx.com/products/boards-and-kits/alveo.html
>
> The XDMA engine support up to 4 Host to Card (H2C) and 4 Card to Host (C2H)
> channels. Memory transfers are specified on a per-channel basis in
> descriptor linked lists, which the DMA fetches from host memory and
> processes. Events such as descriptor completion and errors are signaled
> using interrupts. The hardware detail is provided by
>      https://docs.xilinx.com/r/en-US/pg195-pcie-dma/Introduction
>
> This driver implements dmaengine APIs.
>      - probe the available DMA channels
>      - use dma_slave_map for channel lookup
>      - use virtual channel to manage dmaengine tx descriptors
>      - implement device_prep_slave_sg callback to handle host scatter gather
>        list
>      - implement device_config to config device address for DMA transfer
>
> Signed-off-by: Lizhi Hou <[e-mail]>
> Signed-off-by: Sonal Santan <[e-mail]>
> Signed-off-by: Max Zhen <[e-mail]>
> Signed-off-by: Brian Xu <[e-mail]>
> ---
>   MAINTAINERS                            |  10 +
>   drivers/dma/Kconfig                    |  13 +
>   drivers/dma/xilinx/Makefile            |   1 +
>   drivers/dma/xilinx/xdma-regs.h         | 171 +++++
>   drivers/dma/xilinx/xdma.c              | 982 +++++++++++++++++++++++++
>   include/linux/platform_data/amd_xdma.h |  34 +
>   6 files changed, 1211 insertions(+)
>   create mode 100644 drivers/dma/xilinx/xdma-regs.h
>   create mode 100644 drivers/dma/xilinx/xdma.c
>   create mode 100644 include/linux/platform_data/amd_xdma.h
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index e8c52d0192a6..c1be0b2e378a 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -21683,6 +21683,16 @@ F:	Documentation/devicetree/bindings/media/xilinx/
>   F:	drivers/media/platform/xilinx/
>   F:	include/uapi/linux/xilinx-v4l2-controls.h
>   
> +XILINX XDMA DRIVER
> +M:	Lizhi Hou <[e-mail]>
> +M:	Brian Xu <[e-mail]>
> +M:	Raj Kumar Rampelli <[e-mail]>
> +L:	[e-mail]
> +S:	Supported
> +F:	drivers/dma/xilinx/xdma-regs.h
> +F:	drivers/dma/xilinx/xdma.c
> +F:	include/linux/platform_data/amd_xdma.h
> +
>   XILINX ZYNQMP DPDMA DRIVER
>   M:	Hyun Kwon <[e-mail]>
>   M:	Laurent Pinchart <[e-mail]>
> diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
> index d5de3f77d3aa..4d90f2f51655 100644
> --- a/drivers/dma/Kconfig
> +++ b/drivers/dma/Kconfig
> @@ -733,6 +733,19 @@ config XILINX_ZYNQMP_DPDMA
>   	  driver provides the dmaengine required by the DisplayPort subsystem
>   	  display driver.
>   
> +config XILINX_XDMA
> +	tristate "Xilinx DMA/Bridge Subsystem DMA Engine"
> +	select DMA_ENGINE
> +	select DMA_VIRTUAL_CHANNELS
> +	select REGMAP_MMIO
> +	help
> +	  Enable support for Xilinx DMA/Bridge Subsystem DMA engine. The DMA
> +	  provides high performance block data movement between Host memory
> +	  and the DMA subsystem. These direct memory transfers can be both in
> +	  the Host to Card (H2C) and Card to Host (C2H) transfers.
> +	  The core also provides up to 16 user interrupt wires that generate
> +	  interrupts to the host.
> +
>   # driver files
>   source "drivers/dma/bestcomm/Kconfig"
>   
> diff --git a/drivers/dma/xilinx/Makefile b/drivers/dma/xilinx/Makefile
> index 767bb45f641f..c7a538a56643 100644
> --- a/drivers/dma/xilinx/Makefile
> +++ b/drivers/dma/xilinx/Makefile
> @@ -2,3 +2,4 @@
>   obj-$(CONFIG_XILINX_DMA) += xilinx_dma.o
>   obj-$(CONFIG_XILINX_ZYNQMP_DMA) += zynqmp_dma.o
>   obj-$(CONFIG_XILINX_ZYNQMP_DPDMA) += xilinx_dpdma.o
> +obj-$(CONFIG_XILINX_XDMA) += xdma.o
> diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h
> new file mode 100644
> index 000000000000..c42df0ed4aca
> --- /dev/null
> +++ b/drivers/dma/xilinx/xdma-regs.h
> @@ -0,0 +1,171 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
> + */
> +
> +#ifndef __DMA_XDMA_REGS_H
> +#define __DMA_XDMA_REGS_H
> +
> +/* The length of register space exposed to host */
> +#define XDMA_REG_SPACE_LEN	65536
> +
> +/*
> + * maximum number of DMA channels for each direction:
> + * Host to Card (H2C) or Card to Host (C2H)
> + */
> +#define XDMA_MAX_CHANNELS	4
> +
> +/* macros to get higher and lower 32-bit address */
> +#define XDMA_HI_ADDR_MASK	GENMASK_ULL(63, 32)
> +#define XDMA_LO_ADDR_MASK	GENMASK_ULL(31, 0)
> +
> +/*
> + * macros to define the number of descriptor blocks can be used in one
> + * DMA transfer request.
> + * the DMA engine uses a linked list of descriptor blocks that specify the
> + * source, destination, and length of the DMA transfers.
> + */
> +#define XDMA_DESC_BLOCK_NUM		BIT(7)
> +#define XDMA_DESC_BLOCK_MASK		(XDMA_DESC_BLOCK_NUM - 1)
> +
> +/* descriptor definitions */
> +#define XDMA_DESC_ADJACENT		BIT(5)
> +#define XDMA_DESC_ADJACENT_MASK		(XDMA_DESC_ADJACENT - 1)
> +#define XDMA_DESC_MAGIC			0xad4bUL
> +#define XDMA_DESC_MAGIC_SHIFT		16
> +#define XDMA_DESC_ADJACENT_SHIFT	8
> +#define XDMA_DESC_STOPPED		BIT(0)
> +#define XDMA_DESC_COMPLETED		BIT(1)
> +#define XDMA_DESC_BLEN_BITS		28
> +#define XDMA_DESC_BLEN_MAX		(BIT(XDMA_DESC_BLEN_BITS) - PAGE_SIZE)
> +
> +/* macros to construct the descriptor control word */
> +#define XDMA_DESC_CONTROL(adjacent, flag)				\
> +	((XDMA_DESC_MAGIC << XDMA_DESC_MAGIC_SHIFT) |			\
> +	 (((adjacent) - 1) << XDMA_DESC_ADJACENT_SHIFT) | (flag))
> +#define XDMA_DESC_CONTROL_LAST						\
> +	XDMA_DESC_CONTROL(1, XDMA_DESC_STOPPED | XDMA_DESC_COMPLETED)
> +
> +/*
> + * Descriptor for a single contiguous memory block transfer.
> + *
> + * Multiple descriptors are linked by means of the next pointer. An additional
> + * extra adjacent number gives the amount of extra contiguous descriptors.
> + *
> + * The descriptors are in root complex memory, and the bytes in the 32-bit
> + * words must be in little-endian byte ordering.
> + */
> +struct xdma_hw_desc {
> +	__le32		control;
> +	__le32		bytes;
> +	__le64		src_addr;
> +	__le64		dst_addr;
> +	__le64		next_desc;
> +};
> +
> +#define XDMA_DESC_SIZE		sizeof(struct xdma_hw_desc)
> +#define XDMA_DESC_BLOCK_SIZE	(XDMA_DESC_SIZE * XDMA_DESC_ADJACENT)
> +#define XDMA_DESC_BLOCK_ALIGN	4096
> +
> +/*
> + * Channel registers
> + */
> +#define XDMA_CHAN_IDENTIFIER		0x0
> +#define XDMA_CHAN_CONTROL		0x4
> +#define XDMA_CHAN_CONTROL_W1S		0x8
> +#define XDMA_CHAN_CONTROL_W1C		0xc
> +#define XDMA_CHAN_STATUS		0x40
> +#define XDMA_CHAN_COMPLETED_DESC	0x48
> +#define XDMA_CHAN_ALIGNMENTS		0x4c
> +#define XDMA_CHAN_INTR_ENABLE		0x90
> +#define XDMA_CHAN_INTR_ENABLE_W1S	0x94
> +#define XDMA_CHAN_INTR_ENABLE_W1C	0x9c
> +
> +#define XDMA_CHAN_STRIDE	0x100
> +#define XDMA_CHAN_H2C_OFFSET	0x0
> +#define XDMA_CHAN_C2H_OFFSET	0x1000
> +#define XDMA_CHAN_H2C_TARGET	0x0
> +#define XDMA_CHAN_C2H_TARGET	0x1
> +
> +/* macro to check if channel is available */
> +#define XDMA_CHAN_MAGIC		0x1fc0
> +#define XDMA_CHAN_CHECK_TARGET(id, target)		\
> +	(((u32)(id) >> 16) == XDMA_CHAN_MAGIC + (target))
> +
> +/* bits of the channel control register */
> +#define CHAN_CTRL_RUN_STOP			BIT(0)
> +#define CHAN_CTRL_IE_DESC_STOPPED		BIT(1)
> +#define CHAN_CTRL_IE_DESC_COMPLETED		BIT(2)
> +#define CHAN_CTRL_IE_DESC_ALIGN_MISMATCH	BIT(3)
> +#define CHAN_CTRL_IE_MAGIC_STOPPED		BIT(4)
> +#define CHAN_CTRL_IE_IDLE_STOPPED		BIT(6)
> +#define CHAN_CTRL_IE_READ_ERROR			(0x1FUL << 9)
> +#define CHAN_CTRL_IE_DESC_ERROR			(0x1FUL << 19)
> +#define CHAN_CTRL_NON_INCR_ADDR			BIT(25)
> +#define CHAN_CTRL_POLL_MODE_WB			BIT(26)
> +
> +#define CHAN_CTRL_START	(CHAN_CTRL_RUN_STOP |				\
> +			 CHAN_CTRL_IE_DESC_STOPPED |			\
> +			 CHAN_CTRL_IE_DESC_COMPLETED |			\
> +			 CHAN_CTRL_IE_DESC_ALIGN_MISMATCH |		\
> +			 CHAN_CTRL_IE_MAGIC_STOPPED |			\
> +			 CHAN_CTRL_IE_READ_ERROR |			\
> +			 CHAN_CTRL_IE_DESC_ERROR)
> +
> +/* bits of the channel interrupt enable mask */
> +#define CHAN_IM_DESC_ERROR			BIT(19)
> +#define CHAN_IM_READ_ERROR			BIT(9)
> +#define CHAN_IM_IDLE_STOPPED			BIT(6)
> +#define CHAN_IM_MAGIC_STOPPED			BIT(4)
> +#define CHAN_IM_DESC_COMPLETED			BIT(2)
> +#define CHAN_IM_DESC_STOPPED			BIT(1)
> +
> +#define CHAN_IM_ALL	(CHAN_IM_DESC_ERROR | CHAN_IM_READ_ERROR |	\
> +			 CHAN_IM_IDLE_STOPPED | CHAN_IM_MAGIC_STOPPED | \
> +			 CHAN_IM_DESC_COMPLETED | CHAN_IM_DESC_STOPPED)
> +
> +/*
> + * Channel SGDMA registers
> + */
> +#define XDMA_SGDMA_IDENTIFIER	0x0
> +#define XDMA_SGDMA_DESC_LO	0x80
> +#define XDMA_SGDMA_DESC_HI	0x84
> +#define XDMA_SGDMA_DESC_ADJ	0x88
> +#define XDMA_SGDMA_DESC_CREDIT	0x8c
> +
> +#define XDMA_SGDMA_BASE(chan_base)	((chan_base) + 0x4000)
> +
> +/* bits of the SG DMA control register */
> +#define XDMA_CTRL_RUN_STOP			BIT(0)
> +#define XDMA_CTRL_IE_DESC_STOPPED		BIT(1)
> +#define XDMA_CTRL_IE_DESC_COMPLETED		BIT(2)
> +#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH	BIT(3)
> +#define XDMA_CTRL_IE_MAGIC_STOPPED		BIT(4)
> +#define XDMA_CTRL_IE_IDLE_STOPPED		BIT(6)
> +#define XDMA_CTRL_IE_READ_ERROR			(0x1FUL << 9)
> +#define XDMA_CTRL_IE_DESC_ERROR			(0x1FUL << 19)
> +#define XDMA_CTRL_NON_INCR_ADDR			BIT(25)
> +#define XDMA_CTRL_POLL_MODE_WB			BIT(26)
> +
> +/*
> + * interrupt registers
> + */
> +#define XDMA_IRQ_IDENTIFIER		0x0
> +#define XDMA_IRQ_USER_INT_EN		0x04
> +#define XDMA_IRQ_USER_INT_EN_W1S	0x08
> +#define XDMA_IRQ_USER_INT_EN_W1C	0x0c
> +#define XDMA_IRQ_CHAN_INT_EN		0x10
> +#define XDMA_IRQ_CHAN_INT_EN_W1S	0x14
> +#define XDMA_IRQ_CHAN_INT_EN_W1C	0x18
> +#define XDMA_IRQ_USER_INT_REQ		0x40
> +#define XDMA_IRQ_CHAN_INT_REQ		0x44
> +#define XDMA_IRQ_USER_INT_PEND		0x48
> +#define XDMA_IRQ_CHAN_INT_PEND		0x4c
> +#define XDMA_IRQ_USER_VEC_NUM		0x80
> +#define XDMA_IRQ_CHAN_VEC_NUM		0xa0
> +
> +#define XDMA_IRQ_BASE			0x2000
> +#define XDMA_IRQ_VEC_SHIFT		8
> +
> +#endif /* __DMA_XDMA_REGS_H */
> diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c
> new file mode 100644
> index 000000000000..59797ae5251c
> --- /dev/null
> +++ b/drivers/dma/xilinx/xdma.c
> @@ -0,0 +1,982 @@
> +// SPDX-License-Identifier: GPL-2.0-or-later
> +/*
> + * DMA driver for Xilinx DMA/Bridge Subsystem
> + *
> + * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
> + */
> +
> +/*
> + * The DMA/Bridge Subsystem for PCI Express allows for the movement of data
> + * between Host memory and the DMA subsystem. It does this by operating on
> + * 'descriptors' that contain information about the source, destination and
> + * amount of data to transfer. These direct memory transfers can be both in
> + * the Host to Card (H2C) and Card to Host (C2H) transfers. The DMA can be
> + * configured to have a single AXI4 Master interface shared by all channels
> + * or one AXI4-Stream interface for each channel enabled. Memory transfers are
> + * specified on a per-channel basis in descriptor linked lists, which the DMA
> + * fetches from host memory and processes. Events such as descriptor completion
> + * and errors are signaled using interrupts. The core also provides up to 16
> + * user interrupt wires that generate interrupts to the host.
> + */
> +
> +#include <linux/mod_devicetable.h>
> +#include <linux/bitfield.h>
> +#include <linux/dmapool.h>
> +#include <linux/regmap.h>
> +#include <linux/dmaengine.h>
> +#include <linux/platform_device.h>
> +#include <linux/platform_data/amd_xdma.h>
> +#include <linux/dma-mapping.h>
> +#include "../virt-dma.h"
> +#include "xdma-regs.h"
> +
> +/* mmio regmap config for all XDMA registers */
> +static const struct regmap_config xdma_regmap_config = {
> +	.reg_bits = 32,
> +	.val_bits = 32,
> +	.reg_stride = 4,
> +	.max_register = XDMA_REG_SPACE_LEN,
> +};
> +
> +/**
> + * struct xdma_desc_block - Descriptor block
> + * @virt_addr: Virtual address of block start
> + * @dma_addr: DMA address of block start
> + */
> +struct xdma_desc_block {
> +	void		*virt_addr;
> +	dma_addr_t	dma_addr;
> +};
> +
> +/**
> + * struct xdma_chan - Driver specific DMA channel structure
> + * @vchan: Virtual channel
> + * @xdev_hdl: Pointer to DMA device structure
> + * @base: Offset of channel registers
> + * @desc_pool: Descriptor pool
> + * @busy: Busy flag of the channel
> + * @dir: Transferring direction of the channel
> + * @cfg: Transferring config of the channel
> + * @irq: IRQ assigned to the channel
> + * @tasklet: Channel tasklet
> + */
> +struct xdma_chan {
> +	struct virt_dma_chan		vchan;
> +	void				*xdev_hdl;
> +	u32				base;
> +	struct dma_pool			*desc_pool;
> +	bool				busy;
> +	enum dma_transfer_direction	dir;
> +	struct dma_slave_config		cfg;
> +	u32				irq;
> +	struct tasklet_struct		tasklet;
> +};
> +
> +/**
> + * struct xdma_desc - DMA desc structure
> + * @vdesc: Virtual DMA descriptor
> + * @dir: Transferring direction of the request
> + * @dev_addr: Physical address on DMA device side
> + * @desc_blocks: Hardware descriptor blocks
> + * @dblk_num: Number of hardware descriptor blocks
> + * @desc_num: Number of hardware descriptors
> + * @completed_desc_num: Completed hardware descriptors
> + */
> +struct xdma_desc {
> +	struct virt_dma_desc		vdesc;
> +	struct xdma_chan		*chan;
> +	enum dma_transfer_direction	dir;
> +	u64				dev_addr;
> +	struct xdma_desc_block		*desc_blocks;
> +	u32				dblk_num;
> +	u32				desc_num;
> +	u32				completed_desc_num;
> +};
> +
> +#define XDMA_DEV_STATUS_REG_DMA		BIT(0)
> +#define XDMA_DEV_STATUS_INIT_MSIX	BIT(1)
> +
> +/**
> + * struct xdma_device - DMA device structure
> + * @pdev: Platform device pointer
> + * @dma_dev: DMA device structure
> + * @regmap: MMIO regmap for DMA registers
> + * @h2c_chans: Host to Card channels
> + * @c2h_chans: Card to Host channels
> + * @h2c_chan_num: Number of H2C channels
> + * @c2h_chan_num: Number of C2H channels
> + * @irq_start: Start IRQ assigned to device
> + * @irq_num: Number of IRQ assigned to device
> + * @status: Initialization status
> + */
> +struct xdma_device {
> +	struct platform_device	*pdev;
> +	struct dma_device	dma_dev;
> +	struct regmap		*regmap;
> +	struct xdma_chan	*h2c_chans;
> +	struct xdma_chan	*c2h_chans;
> +	u32			h2c_chan_num;
> +	u32			c2h_chan_num;
> +	u32			irq_start;
> +	u32			irq_num;
> +	u32			status;
> +};
> +
> +#define xdma_err(xdev, fmt, args...)					\
> +	dev_err(&(xdev)->pdev->dev, fmt, ##args)
> +
> +/* Read and Write DMA registers */
> +static inline int xdma_read_reg(struct xdma_device *xdev, u32 base, u32 reg,
> +				u32 *val)
> +{
> +	return regmap_read(xdev->regmap, base + reg, val);
> +}
> +
> +static inline int xdma_write_reg(struct xdma_device *xdev, u32 base, u32 reg,
> +				 u32 val)
> +{
> +	return regmap_write(xdev->regmap, base + reg, val);
> +}
> +
> +/* Get the last desc in a desc block */
> +static inline void *xdma_blk_last_desc(struct xdma_desc_block *block)
> +{
> +	return block->virt_addr + (XDMA_DESC_ADJACENT - 1) * XDMA_DESC_SIZE;
> +}
> +
> +/**
> + * xdma_link_desc_blocks - Link descriptor blocks for DMA transfer
> + * @sw_desc: Tx descriptor pointer
> + */
> +static void xdma_link_desc_blocks(struct xdma_desc *sw_desc)
> +{
> +	struct xdma_desc_block *block;
> +	u32 last_blk_desc_num, desc_control;
> +	struct xdma_hw_desc *desc;
> +	int i;
> +
> +	desc_control = XDMA_DESC_CONTROL(XDMA_DESC_ADJACENT, 0);
> +	for (i = 1; i < sw_desc->dblk_num; i++) {
> +		block = &sw_desc->desc_blocks[i - 1];
> +		desc = xdma_blk_last_desc(block);
> +
> +		if (!(i & XDMA_DESC_BLOCK_MASK)) {
> +			desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
> +			continue;
> +		}
> +		desc->control = cpu_to_le32(desc_control);
> +		desc->next_desc = cpu_to_le64(block[1].dma_addr);
> +	}
> +
> +	/* update the last block */
> +	last_blk_desc_num = sw_desc->desc_num & XDMA_DESC_ADJACENT_MASK;
> +	if ((sw_desc->dblk_num & XDMA_DESC_BLOCK_MASK) > 1 &&
> +	    last_blk_desc_num) {
> +		block = &sw_desc->desc_blocks[sw_desc->dblk_num - 2];
> +		desc = xdma_blk_last_desc(block);
> +		desc_control = XDMA_DESC_CONTROL(last_blk_desc_num, 0);
> +		desc->control = cpu_to_le32(desc_control);
> +	}
> +
> +	block = &sw_desc->desc_blocks[sw_desc->dblk_num - 1];
> +	desc = block->virt_addr + (last_blk_desc_num - 1) * XDMA_DESC_SIZE;
> +	desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
> +}
> +
> +static inline struct xdma_chan *to_xdma_chan(struct dma_chan *chan)
> +{
> +	return container_of(chan, struct xdma_chan, vchan.chan);
> +}
> +
> +static inline struct xdma_desc *to_xdma_desc(struct virt_dma_desc *vdesc)
> +{
> +	return container_of(vdesc, struct xdma_desc, vdesc);
> +}
> +
> +static int xdma_enable_intr(struct xdma_device *xdev)
> +{
> +	int ret;
> +
> +	ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, XDMA_IRQ_CHAN_INT_EN_W1S, ~0);
> +	if (ret)
> +		xdma_err(xdev, "enable channel intr failed: %d", ret);
> +
> +	return ret;
> +}
> +
> +static int xdma_disable_intr(struct xdma_device *xdev)
> +{
> +	int ret;
> +
> +	ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, XDMA_IRQ_CHAN_INT_EN_W1C, ~0);
> +	if (ret)
> +		xdma_err(xdev, "disable channel intr failed: %d", ret);
> +
> +	return ret;
> +}
> +
> +/**
> + * xdma_channel_init - Initialize DMA channel registers
> + * @chan: DMA channel pointer
> + */
> +static int xdma_channel_init(struct xdma_chan *chan)
> +{
> +	struct xdma_device *xdev = chan->xdev_hdl;
> +	int ret;
> +
> +	ret = xdma_write_reg(xdev, chan->base, XDMA_CHAN_CONTROL_W1C,
> +			     CHAN_CTRL_NON_INCR_ADDR);
> +	if (ret) {
> +		xdma_err(xdev, "clear non incr addr failed: %d", ret);
> +		return ret;
> +	}
> +
> +	ret = xdma_write_reg(xdev, chan->base, XDMA_CHAN_INTR_ENABLE,
> +			     CHAN_IM_ALL);
> +	if (ret) {
> +		xdma_err(xdev, "failed to set interrupt mask: %d", ret);
> +		return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * xdma_free_desc - Free descriptor
> + * @vdesc: Virtual DMA descriptor
> + */
> +static void xdma_free_desc(struct virt_dma_desc *vdesc)
> +{
> +	struct xdma_desc *sw_desc;
> +	int i;
> +
> +	sw_desc = to_xdma_desc(vdesc);
> +	for (i = 0; i < sw_desc->dblk_num; i++) {
> +		if (!sw_desc->desc_blocks[i].virt_addr)
> +			break;
> +		dma_pool_free(sw_desc->chan->desc_pool,
> +			      sw_desc->desc_blocks[i].virt_addr,
> +			      sw_desc->desc_blocks[i].dma_addr);
> +	}
> +	kfree(sw_desc->desc_blocks);
> +	kfree(sw_desc);
> +}
> +
> +/**
> + * xdma_alloc_desc - Allocate descriptor
> + * @chan: DMA channel pointer
> + * @desc_num: Number of hardware descriptors
> + */
> +static struct xdma_desc *
> +xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num)
> +{
> +	struct xdma_desc *sw_desc;
> +	struct xdma_hw_desc *desc;
> +	dma_addr_t dma_addr;
> +	u32 dblk_num;
> +	void *addr;
> +	int i, j;
> +
> +	sw_desc = kzalloc(sizeof(*sw_desc), GFP_NOWAIT);
> +	if (!sw_desc)
> +		return NULL;
> +
> +	sw_desc->chan = chan;
> +	sw_desc->desc_num = desc_num;
> +	dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT);
> +	sw_desc->desc_blocks = kcalloc(dblk_num, sizeof(*sw_desc->desc_blocks),
> +				       GFP_NOWAIT);
> +	if (!sw_desc->desc_blocks)
> +		goto failed;
> +
> +	sw_desc->dblk_num = dblk_num;
> +	for (i = 0; i < sw_desc->dblk_num; i++) {
> +		addr = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &dma_addr);
> +		if (!addr)
> +			goto failed;
> +
> +		sw_desc->desc_blocks[i].virt_addr = addr;
> +		sw_desc->desc_blocks[i].dma_addr = dma_addr;
> +		for (j = 0, desc = addr; j < XDMA_DESC_ADJACENT; j++)
> +			desc[j].control = cpu_to_le32(XDMA_DESC_CONTROL(1, 0));
> +	}
> +
> +	xdma_link_desc_blocks(sw_desc);
> +
> +	return sw_desc;
> +
> +failed:
> +	xdma_free_desc(&sw_desc->vdesc);
> +	return NULL;
> +}
> +
> +/**
> + * xdma_xfer_start - Start DMA transfer
> + * @xdma_chan: DMA channel pointer
> + */
> +static int xdma_xfer_start(struct xdma_chan *xdma_chan)
> +{
> +	struct virt_dma_desc *vd = vchan_next_desc(&xdma_chan->vchan);
> +	struct xdma_device *xdev = xdma_chan->xdev_hdl;
> +	struct xdma_desc_block *block;
> +	u32 val, completed_blocks;
> +	struct xdma_desc *desc;
> +	int ret;
> +
> +	/*
> +	 * check if there is not any submitted descriptor or channel is busy.
> +	 * vchan lock should be held where this function is called.
> +	 */
> +	if (!vd || xdma_chan->busy)
> +		return -EINVAL;
> +
> +	/* clear run stop bit to get ready for transfer */
> +	ret = xdma_write_reg(xdev, xdma_chan->base, XDMA_CHAN_CONTROL_W1C,
> +			     CHAN_CTRL_RUN_STOP);
> +	if (ret) {
> +		dev_err(&xdev->pdev->dev, "write control failed: %d", ret);
> +		return ret;
> +	}
> +
> +	desc = to_xdma_desc(vd);
> +	if (desc->dir != xdma_chan->dir) {
> +		dev_err(&xdev->pdev->dev, "incorrect request direction");
> +		return -EINVAL;
> +	}
> +
> +	/* set DMA engine to the first descriptor block */
> +	completed_blocks = desc->completed_desc_num / XDMA_DESC_ADJACENT;
> +	block = &desc->desc_blocks[completed_blocks];
> +	val = FIELD_GET(XDMA_LO_ADDR_MASK, block->dma_addr);
> +	ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
> +			     XDMA_SGDMA_DESC_LO, val);
> +	if (ret) {
> +		dev_err(&xdev->pdev->dev, "write hi addr failed: %d", ret);
> +		return ret;
> +	}
> +
> +	val = FIELD_GET(XDMA_HI_ADDR_MASK, block->dma_addr);
> +	ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
> +			     XDMA_SGDMA_DESC_HI, val);
> +	if (ret) {
> +		dev_err(&xdev->pdev->dev, "write lo addr failed: %d", ret);
> +		return ret;
> +	}
> +
> +	if (completed_blocks + 1 == desc->dblk_num)
> +		val = (desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK;
> +	else
> +		val = XDMA_DESC_ADJACENT - 1;
> +	ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
> +			     XDMA_SGDMA_DESC_ADJ, val);
> +	if (ret) {
> +		dev_err(&xdev->pdev->dev, "write adjacent failed: %d", ret);
> +		return ret;
> +	}
> +
> +	/* kick off DMA transfer */
> +	ret = xdma_write_reg(xdev, xdma_chan->base, XDMA_CHAN_CONTROL,
> +			     CHAN_CTRL_START);
> +	if (ret) {
> +		dev_err(&xdev->pdev->dev, "write control failed: %d", ret);
> +		return ret;
> +	}
> +
> +	xdma_chan->busy = true;
> +	return 0;
> +}
> +
> +static void xdma_channel_tasklet(struct tasklet_struct *t)
> +{
> +	struct xdma_chan *xdma_chan = from_tasklet(xdma_chan, t, tasklet);
> +
> +	spin_lock(&xdma_chan->vchan.lock);
> +	xdma_xfer_start(xdma_chan);
> +	spin_unlock(&xdma_chan->vchan.lock);
> +}
> +
> +/**
> + * xdma_config_channels - Detect and config DMA channels
> + * @xdev: DMA device pointer
> + * @dir: Channel direction
> + */
> +static int xdma_config_channels(struct xdma_device *xdev,
> +				enum dma_transfer_direction dir)
> +{
> +	struct xdma_platdata *pdata = dev_get_platdata(&xdev->pdev->dev);
> +	u32 base, identifier, target;
> +	struct xdma_chan **chans;
> +	u32 *chan_num;
> +	int i, j, ret;
> +
> +	if (dir == DMA_MEM_TO_DEV) {
> +		base = XDMA_CHAN_H2C_OFFSET;
> +		target = XDMA_CHAN_H2C_TARGET;
> +		chans = &xdev->h2c_chans;
> +		chan_num = &xdev->h2c_chan_num;
> +	} else if (dir == DMA_DEV_TO_MEM) {
> +		base = XDMA_CHAN_C2H_OFFSET;
> +		target = XDMA_CHAN_C2H_TARGET;
> +		chans = &xdev->c2h_chans;
> +		chan_num = &xdev->c2h_chan_num;
> +	} else {
> +		dev_err(&xdev->pdev->dev, "invalid direction specified");
> +		return -EINVAL;
> +	}
> +
> +	/* detect number of available DMA channels */
> +	for (i = 0, *chan_num = 0; i < pdata->max_dma_channels; i++) {
> +		ret = xdma_read_reg(xdev, base + i * XDMA_CHAN_STRIDE,
> +				    XDMA_CHAN_IDENTIFIER, &identifier);
> +		if (ret) {
> +			dev_err(&xdev->pdev->dev,
> +				"failed to read channel id: %d", ret);
> +			return ret;
> +		}
> +
> +		/* check if it is available DMA channel */
> +		if (XDMA_CHAN_CHECK_TARGET(identifier, target))
> +			(*chan_num)++;
> +	}
> +
> +	if (!*chan_num) {
> +		dev_err(&xdev->pdev->dev, "does not probe any channel");
> +		return -EINVAL;
> +	}
> +
> +	*chans = devm_kzalloc(&xdev->pdev->dev, sizeof(**chans) * (*chan_num),
> +			      GFP_KERNEL);
> +	if (!*chans)
> +		return -ENOMEM;
> +
> +	for (i = 0, j = 0; i < pdata->max_dma_channels; i++) {
> +		ret = xdma_read_reg(xdev, base + i * XDMA_CHAN_STRIDE,
> +				    XDMA_CHAN_IDENTIFIER, &identifier);
> +		if (ret) {
> +			dev_err(&xdev->pdev->dev,
> +				"failed to read channel id: %d", ret);
> +			return ret;
> +		}
> +
> +		if (!XDMA_CHAN_CHECK_TARGET(identifier, target))
> +			continue;
> +
> +		if (j == *chan_num) {
> +			dev_err(&xdev->pdev->dev, "invalid channel number");
> +			return -EIO;
> +		}
> +
> +		/* init channel structure and hardware */
> +		(*chans)[j].xdev_hdl = xdev;
> +		(*chans)[j].base = base + i * XDMA_CHAN_STRIDE;
> +		(*chans)[j].dir = dir;
> +
> +		ret = xdma_channel_init(&(*chans)[j]);
> +		if (ret)
> +			return ret;
> +		(*chans)[j].vchan.desc_free = xdma_free_desc;
> +		vchan_init(&(*chans)[j].vchan, &xdev->dma_dev);
> +
> +		j++;
> +	}
> +
> +	dev_info(&xdev->pdev->dev, "configured %d %s channels", j,
> +		 (dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H");
> +
> +	return 0;
> +}
> +
> +/**
> + * xdma_issue_pending - Issue pending transactions
> + * @chan: DMA channel pointer
> + */
> +static void xdma_issue_pending(struct dma_chan *chan)
> +{
> +	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&xdma_chan->vchan.lock, flags);
> +	if (vchan_issue_pending(&xdma_chan->vchan))
> +		xdma_xfer_start(xdma_chan);
> +	spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);
> +}
> +
> +/**
> + * xdma_prep_device_sg - prepare a descriptor for a
> + *	DMA transaction
> + * @chan: DMA channel pointer
> + * @sgl: Transfer scatter gather list
> + * @sg_len: Length of scatter gather list
> + * @dir: Transfer direction
> + * @flags: transfer ack flags
> + * @context: APP words of the descriptor
> + */
> +static struct dma_async_tx_descriptor *
> +xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,
> +		    unsigned int sg_len, enum dma_transfer_direction dir,
> +		    unsigned long flags, void *context)
> +{
> +	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
> +	struct dma_async_tx_descriptor *tx_desc;
> +	u32 desc_num = 0, i, len, rest;
> +	struct xdma_desc_block *dblk;
> +	struct xdma_desc *sw_desc;
> +	struct scatterlist *sg;
> +	dma_addr_t addr;
> +	u64 dev_addr, *src, *dst;
> +	struct xdma_hw_desc *desc;
> +
> +	for_each_sg(sgl, sg, sg_len, i)
> +		desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX);
> +
> +	sw_desc = xdma_alloc_desc(xdma_chan, desc_num);
> +	if (!sw_desc)
> +		return NULL;
> +	sw_desc->dir = dir;
> +
> +	if (dir == DMA_MEM_TO_DEV) {
> +		dev_addr = xdma_chan->cfg.dst_addr;
> +		src = &addr;
> +		dst = &dev_addr;
> +	} else {
> +		dev_addr = xdma_chan->cfg.src_addr;
> +		src = &dev_addr;
> +		dst = &addr;
> +	}
> +
> +	dblk = sw_desc->desc_blocks;
> +	desc = dblk->virt_addr;
> +	desc_num = 1;
> +	for_each_sg(sgl, sg, sg_len, i) {
> +		addr = sg_dma_address(sg);
> +		rest = sg_dma_len(sg);
> +
> +		do {
> +			len = min_t(u32, rest, XDMA_DESC_BLEN_MAX);
> +			/* set hardware descriptor */
> +			desc->bytes = cpu_to_le32(len);
> +			desc->src_addr = cpu_to_le64(*src);
> +			desc->dst_addr = cpu_to_le64(*dst);
> +
> +			if (!(desc_num & XDMA_DESC_ADJACENT_MASK)) {
> +				dblk++;
> +				desc = dblk->virt_addr;
> +			} else {
> +				desc++;
> +			}
> +
> +			desc_num++;
> +			dev_addr += len;
> +			addr += len;
> +			rest -= len;
> +		} while (rest);
> +	}
> +
> +	tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags);
> +	if (!tx_desc)
> +		goto failed;
> +
> +	return tx_desc;
> +
> +failed:
> +	xdma_free_desc(&sw_desc->vdesc);
> +
> +	return NULL;
> +}
> +
> +/**
> + * xdma_device_config - Configure the DMA channel
> + * @chan: DMA channel
> + * @cfg: channel configuration
> + */
> +static int xdma_device_config(struct dma_chan *chan,
> +			      struct dma_slave_config *cfg)
> +{
> +	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
> +
> +	memcpy(&xdma_chan->cfg, cfg, sizeof(*cfg));
> +
> +	return 0;
> +}
> +
> +/**
> + * xdma_free_chan_resources - Free channel resources
> + * @chan: DMA channel
> + */
> +static void xdma_free_chan_resources(struct dma_chan *chan)
> +{
> +	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
> +
> +	vchan_free_chan_resources(&xdma_chan->vchan);
> +	dma_pool_destroy(xdma_chan->desc_pool);
> +	xdma_chan->desc_pool = NULL;
> +}
> +
> +/**
> + * xdma_alloc_chan_resources - Allocate channel resources
> + * @chan: DMA channel
> + */
> +static int xdma_alloc_chan_resources(struct dma_chan *chan)
> +{
> +	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
> +	struct xdma_device *xdev = xdma_chan->xdev_hdl;
> +
> +	xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan),
> +					       xdev->dma_dev.dev,
> +					       XDMA_DESC_BLOCK_SIZE,
> +					       XDMA_DESC_BLOCK_ALIGN,
> +					       0);
> +	if (!xdma_chan->desc_pool) {
> +		dev_err(&chan->dev->device,
> +			"unable to allocate descriptor pool");
> +		return -ENOMEM;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * xdma_channel_isr - XDMA channel interrupt handler
> + * @irq: IRQ number
> + * @dev_id: Pointer to the DMA channel structure
> + */
> +static irqreturn_t xdma_channel_isr(int irq, void *dev_id)
> +{
> +	struct xdma_chan *xdma_chan = dev_id;
> +	u32 complete_desc_num = 0;
> +	struct virt_dma_desc *vd;
> +	struct xdma_desc *desc;
> +	int ret;
> +
> +	spin_lock(&xdma_chan->vchan.lock);
> +
> +	/* get submitted request */
> +	vd = vchan_next_desc(&xdma_chan->vchan);
> +	if (!vd)
> +		goto out;
> +
> +	xdma_chan->busy = false;
> +	desc = to_xdma_desc(vd);
> +
> +	ret = xdma_read_reg(xdma_chan->xdev_hdl, xdma_chan->base,
> +			    XDMA_CHAN_COMPLETED_DESC, &complete_desc_num);
> +	if (ret)
> +		goto out;
> +
> +	desc->completed_desc_num += complete_desc_num;
> +	/*
> +	 * if all data blocks are transferred, remove and complete the request
> +	 */
> +	if (desc->completed_desc_num == desc->desc_num) {
> +		list_del(&vd->node);
> +		vchan_cookie_complete(vd);
> +		goto out;
> +	}
> +
> +	if (desc->completed_desc_num > desc->desc_num ||
> +	    complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT)
> +		goto out;
> +
> +	/* transfer the rest of data */
> +	tasklet_schedule(&xdma_chan->tasklet);

There's an effort upstream to deprecate the usage tasklets [1]. Any 
thought on using threaded irq instead especially with a new driver?

[1]: https://lwn.net/Articles/830964/

DJ

> +
> +out:
> +	spin_unlock(&xdma_chan->vchan.lock);
> +	return IRQ_HANDLED;
> +}
> +
> +/**
> + * xdma_irq_fini - Uninitialize IRQ
> + * @xdev: DMA device pointer
> + */
> +static void xdma_irq_fini(struct xdma_device *xdev)
> +{
> +	int ret, i;
> +
> +	/* disable interrupt */
> +	ret = xdma_disable_intr(xdev);
> +	if (ret) {
> +		dev_err(&xdev->pdev->dev, "failed to disable interrupts: %d",
> +			ret);
> +	}
> +
> +	/* free irq handler */
> +	for (i = 0; i < xdev->h2c_chan_num; i++) {
> +		free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
> +		tasklet_kill(&xdev->h2c_chans[i].tasklet);
> +	}
> +	for (i = 0; i < xdev->c2h_chan_num; i++) {
> +		free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
> +		tasklet_kill(&xdev->c2h_chans[i].tasklet);
> +	}
> +}
> +
> +/**
> + * xdma_set_vector_reg - configure hardware IRQ registers
> + * @xdev: DMA device pointer
> + * @vec_tbl_start: Start of IRQ registers
> + * @irq_start: Start of IRQ
> + * @irq_num: Number of IRQ
> + */
> +static int xdma_set_vector_reg(struct xdma_device *xdev, u32 vec_tbl_start,
> +			       u32 irq_start, u32 irq_num)
> +{
> +	u32 shift, i, val = 0;
> +	int ret;
> +
> +	/* Each IRQ register is 32 bit and contains 4 IRQs */
> +	while (irq_num > 0) {
> +		for (i = 0; i < 4; i++) {
> +			shift = XDMA_IRQ_VEC_SHIFT * i;
> +			val |= irq_start << shift;
> +			irq_start++;
> +			irq_num--;
> +		}
> +
> +		/* write IRQ register */
> +		ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, vec_tbl_start, val);
> +		if (ret) {
> +			dev_err(&xdev->pdev->dev, "failed to set vector: %d",
> +				ret);
> +			return ret;
> +		}
> +		vec_tbl_start += sizeof(u32);
> +		val = 0;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * xdma_irq_init - initialize IRQs
> + * @xdev: DMA device pointer
> + */
> +static int xdma_irq_init(struct xdma_device *xdev)
> +{
> +	u32 irq = xdev->irq_start;
> +	int i, j, ret;
> +
> +	/* return failure if there are not enough IRQs */
> +	if (xdev->irq_num < xdev->h2c_chan_num + xdev->c2h_chan_num) {
> +		dev_err(&xdev->pdev->dev, "not enough irq");
> +		return -EINVAL;
> +	}
> +
> +	/* setup H2C interrupt handler */
> +	for (i = 0; i < xdev->h2c_chan_num; i++) {
> +		ret = request_irq(irq, xdma_channel_isr, 0,
> +				  "xdma-h2c-channel", &xdev->h2c_chans[i]);
> +		if (ret) {
> +			dev_err(&xdev->pdev->dev,
> +				"H2C channel%d request irq%d failed: %d",
> +				i, irq, ret);
> +			for (j = 0; j < i; j++) {
> +				free_irq(xdev->h2c_chans[j].irq,
> +					 &xdev->h2c_chans[j]);
> +			}
> +			return ret;
> +		}
> +		xdev->h2c_chans[i].irq = irq;
> +		tasklet_setup(&xdev->h2c_chans[i].tasklet,
> +			      xdma_channel_tasklet);
> +		irq++;
> +	}
> +
> +	/* setup C2H interrupt handler */
> +	for (i = 0; i < xdev->c2h_chan_num; i++) {
> +		ret = request_irq(irq, xdma_channel_isr, 0,
> +				  "xdma-c2h-channel", &xdev->c2h_chans[i]);
> +		if (ret) {
> +			dev_err(&xdev->pdev->dev,
> +				"H2C channel%d request irq%d failed: %d",
> +				i, irq, ret);
> +			for (j = 0; j < i; j++) {
> +				free_irq(xdev->c2h_chans[j].irq,
> +					 &xdev->c2h_chans[j]);
> +			}
> +			goto failed_init_c2h;
> +		}
> +		xdev->c2h_chans[i].irq = irq;
> +		tasklet_setup(&xdev->c2h_chans[i].tasklet,
> +			      xdma_channel_tasklet);
> +		irq++;
> +	}
> +
> +	/* config hardware IRQ registers */
> +	ret = xdma_set_vector_reg(xdev, XDMA_IRQ_CHAN_VEC_NUM, 0,
> +				  xdev->h2c_chan_num + xdev->c2h_chan_num);
> +	if (ret) {
> +		dev_err(&xdev->pdev->dev, "failed to set channel vectors: %d",
> +			ret);
> +		goto failed;
> +	}
> +
> +	/* enable interrupt */
> +	ret = xdma_enable_intr(xdev);
> +	if (ret) {
> +		dev_err(&xdev->pdev->dev, "failed to enable interrupts: %d",
> +			ret);
> +		goto failed;
> +	}
> +
> +	return 0;
> +
> +failed:
> +	for (i = 0; i < xdev->c2h_chan_num; i++)
> +		free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
> +failed_init_c2h:
> +	for (i = 0; i < xdev->h2c_chan_num; i++)
> +		free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
> +
> +	return ret;
> +}
> +
> +static bool xdma_filter_fn(struct dma_chan *chan, void *param)
> +{
> +	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
> +	struct xdma_chan_info *chan_info = param;
> +
> +	if (chan_info->dir != xdma_chan->dir)
> +		return false;
> +
> +	return true;
> +}
> +
> +/**
> + * xdma_remove - Driver remove function
> + * @pdev: Pointer to the platform_device structure
> + */
> +static int xdma_remove(struct platform_device *pdev)
> +{
> +	struct xdma_device *xdev = platform_get_drvdata(pdev);
> +
> +	if (xdev->status & XDMA_DEV_STATUS_INIT_MSIX)
> +		xdma_irq_fini(xdev);
> +
> +	if (xdev->status & XDMA_DEV_STATUS_REG_DMA)
> +		dma_async_device_unregister(&xdev->dma_dev);
> +
> +	return 0;
> +}
> +
> +/**
> + * xdma_probe - Driver probe function
> + * @pdev: Pointer to the platform_device structure
> + */
> +static int xdma_probe(struct platform_device *pdev)
> +{
> +	struct xdma_platdata *pdata = dev_get_platdata(&pdev->dev);
> +	struct xdma_device *xdev;
> +	void __iomem *reg_base;
> +	struct resource *res;
> +	int ret = -ENODEV;
> +
> +	if (pdata->max_dma_channels > XDMA_MAX_CHANNELS) {
> +		dev_err(&pdev->dev, "invalid max dma channels %d",
> +			pdata->max_dma_channels);
> +		return -EINVAL;
> +	}
> +
> +	xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);
> +	if (!xdev)
> +		return -ENOMEM;
> +
> +	platform_set_drvdata(pdev, xdev);
> +	xdev->pdev = pdev;
> +
> +	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
> +	if (!res) {
> +		dev_err(&pdev->dev, "failed to get irq resource");
> +		goto failed;
> +	}
> +	xdev->irq_start = res->start;
> +	xdev->irq_num = res->end - res->start + 1;
> +
> +	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
> +	if (!res) {
> +		dev_err(&pdev->dev, "failed to get io resource");
> +		goto failed;
> +	}
> +
> +	reg_base = devm_ioremap_resource(&pdev->dev, res);
> +	if (!reg_base) {
> +		dev_err(&pdev->dev, "ioremap failed");
> +		goto failed;
> +	}
> +
> +	xdev->regmap = devm_regmap_init_mmio(&pdev->dev, reg_base,
> +					     &xdma_regmap_config);
> +	if (!xdev->regmap) {
> +		dev_err(&pdev->dev, "config regmap failed: %d", ret);
> +		goto failed;
> +	}
> +	INIT_LIST_HEAD(&xdev->dma_dev.channels);
> +
> +	ret = xdma_config_channels(xdev, DMA_MEM_TO_DEV);
> +	if (ret) {
> +		dev_err(&pdev->dev, "config H2C channels failed: %d", ret);
> +		goto failed;
> +	}
> +
> +	ret = xdma_config_channels(xdev, DMA_DEV_TO_MEM);
> +	if (ret) {
> +		dev_err(&pdev->dev, "config C2H channels failed: %d", ret);
> +		goto failed;
> +	}
> +
> +	dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask);
> +	dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask);
> +
> +	xdev->dma_dev.dev = &pdev->dev;
> +	xdev->dma_dev.device_free_chan_resources = xdma_free_chan_resources;
> +	xdev->dma_dev.device_alloc_chan_resources = xdma_alloc_chan_resources;
> +	xdev->dma_dev.device_tx_status = dma_cookie_status;
> +	xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg;
> +	xdev->dma_dev.device_config = xdma_device_config;
> +	xdev->dma_dev.device_issue_pending = xdma_issue_pending;
> +	xdev->dma_dev.filter.map = pdata->device_map;
> +	xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt;
> +	xdev->dma_dev.filter.fn = xdma_filter_fn;
> +
> +	ret = dma_async_device_register(&xdev->dma_dev);
> +	if (ret) {
> +		dev_err(&pdev->dev, "failed to register Xilinx XDMA: %d", ret);
> +		goto failed;
> +	}
> +	xdev->status |= XDMA_DEV_STATUS_REG_DMA;
> +
> +	ret = xdma_irq_init(xdev);
> +	if (ret) {
> +		dev_err(&pdev->dev, "failed to init msix: %d", ret);
> +		goto failed;
> +	}
> +	xdev->status |= XDMA_DEV_STATUS_INIT_MSIX;
> +
> +	return 0;
> +
> +failed:
> +	xdma_remove(pdev);
> +
> +	return ret;
> +}
> +
> +static const struct platform_device_id xdma_id_table[] = {
> +	{ "xdma", 0},
> +	{ },
> +};
> +
> +static struct platform_driver xdma_driver = {
> +	.driver		= {
> +		.name = "xdma",
> +	},
> +	.id_table	= xdma_id_table,
> +	.probe		= xdma_probe,
> +	.remove		= xdma_remove,
> +};
> +
> +module_platform_driver(xdma_driver);
> +
> +MODULE_DESCRIPTION("AMD XDMA driver");
> +MODULE_AUTHOR("XRT Team <[e-mail]>");
> +MODULE_LICENSE("GPL");
> diff --git a/include/linux/platform_data/amd_xdma.h b/include/linux/platform_data/amd_xdma.h
> new file mode 100644
> index 000000000000..04bcfc74ab36
> --- /dev/null
> +++ b/include/linux/platform_data/amd_xdma.h
> @@ -0,0 +1,34 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
> + */
> +
> +#ifndef _PLATDATA_AMD_XDMA_H
> +#define _PLATDATA_AMD_XDMA_H
> +
> +#include <linux/dmaengine.h>
> +
> +/**
> + * struct xdma_chan_info - DMA channel information
> + *	This information is used to match channel when request dma channel
> + * @dir: Channel transfer direction
> + */
> +struct xdma_chan_info {
> +	enum dma_transfer_direction dir;
> +};
> +
> +#define XDMA_FILTER_PARAM(chan_info)	((void *)(chan_info))
> +
> +struct dma_slave_map;
> +
> +/**
> + * struct xdma_platdata - platform specific data for XDMA engine
> + * @max_dma_channels: Maximum dma channels in each direction
> + */
> +struct xdma_platdata {
> +	u32 max_dma_channels;
> +	u32 device_map_cnt;
> +	struct dma_slave_map *device_map;
> +};
> +
> +#endif /* _PLATDATA_AMD_XDMA_H */

Lizhi Hou Sept. 16, 2022, 8:10 p.m. UTC | #2

On 9/16/22 08:39, Dave Jiang wrote:
>
> On 9/16/2022 8:26 AM, Lizhi Hou wrote:
>> Add driver to enable PCIe board which uses XDMA (the DMA/Bridge 
>> Subsystem
>> for PCI Express). For example, Xilinx Alveo PCIe devices.
>>      https://www.xilinx.com/products/boards-and-kits/alveo.html
>>
>> The XDMA engine support up to 4 Host to Card (H2C) and 4 Card to Host 
>> (C2H)
>> channels. Memory transfers are specified on a per-channel basis in
>> descriptor linked lists, which the DMA fetches from host memory and
>> processes. Events such as descriptor completion and errors are signaled
>> using interrupts. The hardware detail is provided by
>>      https://docs.xilinx.com/r/en-US/pg195-pcie-dma/Introduction
>>
>> This driver implements dmaengine APIs.
>>      - probe the available DMA channels
>>      - use dma_slave_map for channel lookup
>>      - use virtual channel to manage dmaengine tx descriptors
>>      - implement device_prep_slave_sg callback to handle host scatter 
>> gather
>>        list
>>      - implement device_config to config device address for DMA transfer
>>
>> Signed-off-by: Lizhi Hou <[e-mail]>
>> Signed-off-by: Sonal Santan <[e-mail]>
>> Signed-off-by: Max Zhen <[e-mail]>
>> Signed-off-by: Brian Xu <[e-mail]>
>> ---
>>   MAINTAINERS                            |  10 +
>>   drivers/dma/Kconfig                    |  13 +
>>   drivers/dma/xilinx/Makefile            |   1 +
>>   drivers/dma/xilinx/xdma-regs.h         | 171 +++++
>>   drivers/dma/xilinx/xdma.c              | 982 +++++++++++++++++++++++++
>>   include/linux/platform_data/amd_xdma.h |  34 +
>>   6 files changed, 1211 insertions(+)
>>   create mode 100644 drivers/dma/xilinx/xdma-regs.h
>>   create mode 100644 drivers/dma/xilinx/xdma.c
>>   create mode 100644 include/linux/platform_data/amd_xdma.h
>>
>> diff --git a/MAINTAINERS b/MAINTAINERS
>> index e8c52d0192a6..c1be0b2e378a 100644
>> --- a/MAINTAINERS
>> +++ b/MAINTAINERS
>> @@ -21683,6 +21683,16 @@ F: 
>> Documentation/devicetree/bindings/media/xilinx/
>>   F:    drivers/media/platform/xilinx/
>>   F:    include/uapi/linux/xilinx-v4l2-controls.h
>>   +XILINX XDMA DRIVER
>> +M:    Lizhi Hou <[e-mail]>
>> +M:    Brian Xu <[e-mail]>
>> +M:    Raj Kumar Rampelli <[e-mail]>
>> +L:    [e-mail]
>> +S:    Supported
>> +F:    drivers/dma/xilinx/xdma-regs.h
>> +F:    drivers/dma/xilinx/xdma.c
>> +F:    include/linux/platform_data/amd_xdma.h
>> +
>>   XILINX ZYNQMP DPDMA DRIVER
>>   M:    Hyun Kwon <[e-mail]>
>>   M:    Laurent Pinchart <[e-mail]>
>> diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
>> index d5de3f77d3aa..4d90f2f51655 100644
>> --- a/drivers/dma/Kconfig
>> +++ b/drivers/dma/Kconfig
>> @@ -733,6 +733,19 @@ config XILINX_ZYNQMP_DPDMA
>>         driver provides the dmaengine required by the DisplayPort 
>> subsystem
>>         display driver.
>>   +config XILINX_XDMA
>> +    tristate "Xilinx DMA/Bridge Subsystem DMA Engine"
>> +    select DMA_ENGINE
>> +    select DMA_VIRTUAL_CHANNELS
>> +    select REGMAP_MMIO
>> +    help
>> +      Enable support for Xilinx DMA/Bridge Subsystem DMA engine. The 
>> DMA
>> +      provides high performance block data movement between Host memory
>> +      and the DMA subsystem. These direct memory transfers can be 
>> both in
>> +      the Host to Card (H2C) and Card to Host (C2H) transfers.
>> +      The core also provides up to 16 user interrupt wires that 
>> generate
>> +      interrupts to the host.
>> +
>>   # driver files
>>   source "drivers/dma/bestcomm/Kconfig"
>>   diff --git a/drivers/dma/xilinx/Makefile b/drivers/dma/xilinx/Makefile
>> index 767bb45f641f..c7a538a56643 100644
>> --- a/drivers/dma/xilinx/Makefile
>> +++ b/drivers/dma/xilinx/Makefile
>> @@ -2,3 +2,4 @@
>>   obj-$(CONFIG_XILINX_DMA) += xilinx_dma.o
>>   obj-$(CONFIG_XILINX_ZYNQMP_DMA) += zynqmp_dma.o
>>   obj-$(CONFIG_XILINX_ZYNQMP_DPDMA) += xilinx_dpdma.o
>> +obj-$(CONFIG_XILINX_XDMA) += xdma.o
>> diff --git a/drivers/dma/xilinx/xdma-regs.h 
>> b/drivers/dma/xilinx/xdma-regs.h
>> new file mode 100644
>> index 000000000000..c42df0ed4aca
>> --- /dev/null
>> +++ b/drivers/dma/xilinx/xdma-regs.h
>> @@ -0,0 +1,171 @@
>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>> +/*
>> + * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
>> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
>> + */
>> +
>> +#ifndef __DMA_XDMA_REGS_H
>> +#define __DMA_XDMA_REGS_H
>> +
>> +/* The length of register space exposed to host */
>> +#define XDMA_REG_SPACE_LEN    65536
>> +
>> +/*
>> + * maximum number of DMA channels for each direction:
>> + * Host to Card (H2C) or Card to Host (C2H)
>> + */
>> +#define XDMA_MAX_CHANNELS    4
>> +
>> +/* macros to get higher and lower 32-bit address */
>> +#define XDMA_HI_ADDR_MASK    GENMASK_ULL(63, 32)
>> +#define XDMA_LO_ADDR_MASK    GENMASK_ULL(31, 0)
>> +
>> +/*
>> + * macros to define the number of descriptor blocks can be used in one
>> + * DMA transfer request.
>> + * the DMA engine uses a linked list of descriptor blocks that 
>> specify the
>> + * source, destination, and length of the DMA transfers.
>> + */
>> +#define XDMA_DESC_BLOCK_NUM        BIT(7)
>> +#define XDMA_DESC_BLOCK_MASK        (XDMA_DESC_BLOCK_NUM - 1)
>> +
>> +/* descriptor definitions */
>> +#define XDMA_DESC_ADJACENT        BIT(5)
>> +#define XDMA_DESC_ADJACENT_MASK        (XDMA_DESC_ADJACENT - 1)
>> +#define XDMA_DESC_MAGIC            0xad4bUL
>> +#define XDMA_DESC_MAGIC_SHIFT        16
>> +#define XDMA_DESC_ADJACENT_SHIFT    8
>> +#define XDMA_DESC_STOPPED        BIT(0)
>> +#define XDMA_DESC_COMPLETED        BIT(1)
>> +#define XDMA_DESC_BLEN_BITS        28
>> +#define XDMA_DESC_BLEN_MAX        (BIT(XDMA_DESC_BLEN_BITS) - 
>> PAGE_SIZE)
>> +
>> +/* macros to construct the descriptor control word */
>> +#define XDMA_DESC_CONTROL(adjacent, flag)                \
>> +    ((XDMA_DESC_MAGIC << XDMA_DESC_MAGIC_SHIFT) |            \
>> +     (((adjacent) - 1) << XDMA_DESC_ADJACENT_SHIFT) | (flag))
>> +#define XDMA_DESC_CONTROL_LAST                        \
>> +    XDMA_DESC_CONTROL(1, XDMA_DESC_STOPPED | XDMA_DESC_COMPLETED)
>> +
>> +/*
>> + * Descriptor for a single contiguous memory block transfer.
>> + *
>> + * Multiple descriptors are linked by means of the next pointer. An 
>> additional
>> + * extra adjacent number gives the amount of extra contiguous 
>> descriptors.
>> + *
>> + * The descriptors are in root complex memory, and the bytes in the 
>> 32-bit
>> + * words must be in little-endian byte ordering.
>> + */
>> +struct xdma_hw_desc {
>> +    __le32        control;
>> +    __le32        bytes;
>> +    __le64        src_addr;
>> +    __le64        dst_addr;
>> +    __le64        next_desc;
>> +};
>> +
>> +#define XDMA_DESC_SIZE        sizeof(struct xdma_hw_desc)
>> +#define XDMA_DESC_BLOCK_SIZE    (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT)
>> +#define XDMA_DESC_BLOCK_ALIGN    4096
>> +
>> +/*
>> + * Channel registers
>> + */
>> +#define XDMA_CHAN_IDENTIFIER        0x0
>> +#define XDMA_CHAN_CONTROL        0x4
>> +#define XDMA_CHAN_CONTROL_W1S        0x8
>> +#define XDMA_CHAN_CONTROL_W1C        0xc
>> +#define XDMA_CHAN_STATUS        0x40
>> +#define XDMA_CHAN_COMPLETED_DESC    0x48
>> +#define XDMA_CHAN_ALIGNMENTS        0x4c
>> +#define XDMA_CHAN_INTR_ENABLE        0x90
>> +#define XDMA_CHAN_INTR_ENABLE_W1S    0x94
>> +#define XDMA_CHAN_INTR_ENABLE_W1C    0x9c
>> +
>> +#define XDMA_CHAN_STRIDE    0x100
>> +#define XDMA_CHAN_H2C_OFFSET    0x0
>> +#define XDMA_CHAN_C2H_OFFSET    0x1000
>> +#define XDMA_CHAN_H2C_TARGET    0x0
>> +#define XDMA_CHAN_C2H_TARGET    0x1
>> +
>> +/* macro to check if channel is available */
>> +#define XDMA_CHAN_MAGIC        0x1fc0
>> +#define XDMA_CHAN_CHECK_TARGET(id, target)        \
>> +    (((u32)(id) >> 16) == XDMA_CHAN_MAGIC + (target))
>> +
>> +/* bits of the channel control register */
>> +#define CHAN_CTRL_RUN_STOP            BIT(0)
>> +#define CHAN_CTRL_IE_DESC_STOPPED        BIT(1)
>> +#define CHAN_CTRL_IE_DESC_COMPLETED        BIT(2)
>> +#define CHAN_CTRL_IE_DESC_ALIGN_MISMATCH    BIT(3)
>> +#define CHAN_CTRL_IE_MAGIC_STOPPED        BIT(4)
>> +#define CHAN_CTRL_IE_IDLE_STOPPED        BIT(6)
>> +#define CHAN_CTRL_IE_READ_ERROR            (0x1FUL << 9)
>> +#define CHAN_CTRL_IE_DESC_ERROR            (0x1FUL << 19)
>> +#define CHAN_CTRL_NON_INCR_ADDR            BIT(25)
>> +#define CHAN_CTRL_POLL_MODE_WB            BIT(26)
>> +
>> +#define CHAN_CTRL_START    (CHAN_CTRL_RUN_STOP | \
>> +             CHAN_CTRL_IE_DESC_STOPPED |            \
>> +             CHAN_CTRL_IE_DESC_COMPLETED |            \
>> +             CHAN_CTRL_IE_DESC_ALIGN_MISMATCH |        \
>> +             CHAN_CTRL_IE_MAGIC_STOPPED |            \
>> +             CHAN_CTRL_IE_READ_ERROR |            \
>> +             CHAN_CTRL_IE_DESC_ERROR)
>> +
>> +/* bits of the channel interrupt enable mask */
>> +#define CHAN_IM_DESC_ERROR            BIT(19)
>> +#define CHAN_IM_READ_ERROR            BIT(9)
>> +#define CHAN_IM_IDLE_STOPPED            BIT(6)
>> +#define CHAN_IM_MAGIC_STOPPED            BIT(4)
>> +#define CHAN_IM_DESC_COMPLETED            BIT(2)
>> +#define CHAN_IM_DESC_STOPPED            BIT(1)
>> +
>> +#define CHAN_IM_ALL    (CHAN_IM_DESC_ERROR | CHAN_IM_READ_ERROR |    \
>> +             CHAN_IM_IDLE_STOPPED | CHAN_IM_MAGIC_STOPPED | \
>> +             CHAN_IM_DESC_COMPLETED | CHAN_IM_DESC_STOPPED)
>> +
>> +/*
>> + * Channel SGDMA registers
>> + */
>> +#define XDMA_SGDMA_IDENTIFIER    0x0
>> +#define XDMA_SGDMA_DESC_LO    0x80
>> +#define XDMA_SGDMA_DESC_HI    0x84
>> +#define XDMA_SGDMA_DESC_ADJ    0x88
>> +#define XDMA_SGDMA_DESC_CREDIT    0x8c
>> +
>> +#define XDMA_SGDMA_BASE(chan_base)    ((chan_base) + 0x4000)
>> +
>> +/* bits of the SG DMA control register */
>> +#define XDMA_CTRL_RUN_STOP            BIT(0)
>> +#define XDMA_CTRL_IE_DESC_STOPPED        BIT(1)
>> +#define XDMA_CTRL_IE_DESC_COMPLETED        BIT(2)
>> +#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH    BIT(3)
>> +#define XDMA_CTRL_IE_MAGIC_STOPPED        BIT(4)
>> +#define XDMA_CTRL_IE_IDLE_STOPPED        BIT(6)
>> +#define XDMA_CTRL_IE_READ_ERROR            (0x1FUL << 9)
>> +#define XDMA_CTRL_IE_DESC_ERROR            (0x1FUL << 19)
>> +#define XDMA_CTRL_NON_INCR_ADDR            BIT(25)
>> +#define XDMA_CTRL_POLL_MODE_WB            BIT(26)
>> +
>> +/*
>> + * interrupt registers
>> + */
>> +#define XDMA_IRQ_IDENTIFIER        0x0
>> +#define XDMA_IRQ_USER_INT_EN        0x04
>> +#define XDMA_IRQ_USER_INT_EN_W1S    0x08
>> +#define XDMA_IRQ_USER_INT_EN_W1C    0x0c
>> +#define XDMA_IRQ_CHAN_INT_EN        0x10
>> +#define XDMA_IRQ_CHAN_INT_EN_W1S    0x14
>> +#define XDMA_IRQ_CHAN_INT_EN_W1C    0x18
>> +#define XDMA_IRQ_USER_INT_REQ        0x40
>> +#define XDMA_IRQ_CHAN_INT_REQ        0x44
>> +#define XDMA_IRQ_USER_INT_PEND        0x48
>> +#define XDMA_IRQ_CHAN_INT_PEND        0x4c
>> +#define XDMA_IRQ_USER_VEC_NUM        0x80
>> +#define XDMA_IRQ_CHAN_VEC_NUM        0xa0
>> +
>> +#define XDMA_IRQ_BASE            0x2000
>> +#define XDMA_IRQ_VEC_SHIFT        8
>> +
>> +#endif /* __DMA_XDMA_REGS_H */
>> diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c
>> new file mode 100644
>> index 000000000000..59797ae5251c
>> --- /dev/null
>> +++ b/drivers/dma/xilinx/xdma.c
>> @@ -0,0 +1,982 @@
>> +// SPDX-License-Identifier: GPL-2.0-or-later
>> +/*
>> + * DMA driver for Xilinx DMA/Bridge Subsystem
>> + *
>> + * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
>> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
>> + */
>> +
>> +/*
>> + * The DMA/Bridge Subsystem for PCI Express allows for the movement 
>> of data
>> + * between Host memory and the DMA subsystem. It does this by 
>> operating on
>> + * 'descriptors' that contain information about the source, 
>> destination and
>> + * amount of data to transfer. These direct memory transfers can be 
>> both in
>> + * the Host to Card (H2C) and Card to Host (C2H) transfers. The DMA 
>> can be
>> + * configured to have a single AXI4 Master interface shared by all 
>> channels
>> + * or one AXI4-Stream interface for each channel enabled. Memory 
>> transfers are
>> + * specified on a per-channel basis in descriptor linked lists, 
>> which the DMA
>> + * fetches from host memory and processes. Events such as descriptor 
>> completion
>> + * and errors are signaled using interrupts. The core also provides 
>> up to 16
>> + * user interrupt wires that generate interrupts to the host.
>> + */
>> +
>> +#include <linux/mod_devicetable.h>
>> +#include <linux/bitfield.h>
>> +#include <linux/dmapool.h>
>> +#include <linux/regmap.h>
>> +#include <linux/dmaengine.h>
>> +#include <linux/platform_device.h>
>> +#include <linux/platform_data/amd_xdma.h>
>> +#include <linux/dma-mapping.h>
>> +#include "../virt-dma.h"
>> +#include "xdma-regs.h"
>> +
>> +/* mmio regmap config for all XDMA registers */
>> +static const struct regmap_config xdma_regmap_config = {
>> +    .reg_bits = 32,
>> +    .val_bits = 32,
>> +    .reg_stride = 4,
>> +    .max_register = XDMA_REG_SPACE_LEN,
>> +};
>> +
>> +/**
>> + * struct xdma_desc_block - Descriptor block
>> + * @virt_addr: Virtual address of block start
>> + * @dma_addr: DMA address of block start
>> + */
>> +struct xdma_desc_block {
>> +    void        *virt_addr;
>> +    dma_addr_t    dma_addr;
>> +};
>> +
>> +/**
>> + * struct xdma_chan - Driver specific DMA channel structure
>> + * @vchan: Virtual channel
>> + * @xdev_hdl: Pointer to DMA device structure
>> + * @base: Offset of channel registers
>> + * @desc_pool: Descriptor pool
>> + * @busy: Busy flag of the channel
>> + * @dir: Transferring direction of the channel
>> + * @cfg: Transferring config of the channel
>> + * @irq: IRQ assigned to the channel
>> + * @tasklet: Channel tasklet
>> + */
>> +struct xdma_chan {
>> +    struct virt_dma_chan        vchan;
>> +    void                *xdev_hdl;
>> +    u32                base;
>> +    struct dma_pool            *desc_pool;
>> +    bool                busy;
>> +    enum dma_transfer_direction    dir;
>> +    struct dma_slave_config        cfg;
>> +    u32                irq;
>> +    struct tasklet_struct        tasklet;
>> +};
>> +
>> +/**
>> + * struct xdma_desc - DMA desc structure
>> + * @vdesc: Virtual DMA descriptor
>> + * @dir: Transferring direction of the request
>> + * @dev_addr: Physical address on DMA device side
>> + * @desc_blocks: Hardware descriptor blocks
>> + * @dblk_num: Number of hardware descriptor blocks
>> + * @desc_num: Number of hardware descriptors
>> + * @completed_desc_num: Completed hardware descriptors
>> + */
>> +struct xdma_desc {
>> +    struct virt_dma_desc        vdesc;
>> +    struct xdma_chan        *chan;
>> +    enum dma_transfer_direction    dir;
>> +    u64                dev_addr;
>> +    struct xdma_desc_block        *desc_blocks;
>> +    u32                dblk_num;
>> +    u32                desc_num;
>> +    u32                completed_desc_num;
>> +};
>> +
>> +#define XDMA_DEV_STATUS_REG_DMA        BIT(0)
>> +#define XDMA_DEV_STATUS_INIT_MSIX    BIT(1)
>> +
>> +/**
>> + * struct xdma_device - DMA device structure
>> + * @pdev: Platform device pointer
>> + * @dma_dev: DMA device structure
>> + * @regmap: MMIO regmap for DMA registers
>> + * @h2c_chans: Host to Card channels
>> + * @c2h_chans: Card to Host channels
>> + * @h2c_chan_num: Number of H2C channels
>> + * @c2h_chan_num: Number of C2H channels
>> + * @irq_start: Start IRQ assigned to device
>> + * @irq_num: Number of IRQ assigned to device
>> + * @status: Initialization status
>> + */
>> +struct xdma_device {
>> +    struct platform_device    *pdev;
>> +    struct dma_device    dma_dev;
>> +    struct regmap        *regmap;
>> +    struct xdma_chan    *h2c_chans;
>> +    struct xdma_chan    *c2h_chans;
>> +    u32            h2c_chan_num;
>> +    u32            c2h_chan_num;
>> +    u32            irq_start;
>> +    u32            irq_num;
>> +    u32            status;
>> +};
>> +
>> +#define xdma_err(xdev, fmt, args...)                    \
>> +    dev_err(&(xdev)->pdev->dev, fmt, ##args)
>> +
>> +/* Read and Write DMA registers */
>> +static inline int xdma_read_reg(struct xdma_device *xdev, u32 base, 
>> u32 reg,
>> +                u32 *val)
>> +{
>> +    return regmap_read(xdev->regmap, base + reg, val);
>> +}
>> +
>> +static inline int xdma_write_reg(struct xdma_device *xdev, u32 base, 
>> u32 reg,
>> +                 u32 val)
>> +{
>> +    return regmap_write(xdev->regmap, base + reg, val);
>> +}
>> +
>> +/* Get the last desc in a desc block */
>> +static inline void *xdma_blk_last_desc(struct xdma_desc_block *block)
>> +{
>> +    return block->virt_addr + (XDMA_DESC_ADJACENT - 1) * 
>> XDMA_DESC_SIZE;
>> +}
>> +
>> +/**
>> + * xdma_link_desc_blocks - Link descriptor blocks for DMA transfer
>> + * @sw_desc: Tx descriptor pointer
>> + */
>> +static void xdma_link_desc_blocks(struct xdma_desc *sw_desc)
>> +{
>> +    struct xdma_desc_block *block;
>> +    u32 last_blk_desc_num, desc_control;
>> +    struct xdma_hw_desc *desc;
>> +    int i;
>> +
>> +    desc_control = XDMA_DESC_CONTROL(XDMA_DESC_ADJACENT, 0);
>> +    for (i = 1; i < sw_desc->dblk_num; i++) {
>> +        block = &sw_desc->desc_blocks[i - 1];
>> +        desc = xdma_blk_last_desc(block);
>> +
>> +        if (!(i & XDMA_DESC_BLOCK_MASK)) {
>> +            desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
>> +            continue;
>> +        }
>> +        desc->control = cpu_to_le32(desc_control);
>> +        desc->next_desc = cpu_to_le64(block[1].dma_addr);
>> +    }
>> +
>> +    /* update the last block */
>> +    last_blk_desc_num = sw_desc->desc_num & XDMA_DESC_ADJACENT_MASK;
>> +    if ((sw_desc->dblk_num & XDMA_DESC_BLOCK_MASK) > 1 &&
>> +        last_blk_desc_num) {
>> +        block = &sw_desc->desc_blocks[sw_desc->dblk_num - 2];
>> +        desc = xdma_blk_last_desc(block);
>> +        desc_control = XDMA_DESC_CONTROL(last_blk_desc_num, 0);
>> +        desc->control = cpu_to_le32(desc_control);
>> +    }
>> +
>> +    block = &sw_desc->desc_blocks[sw_desc->dblk_num - 1];
>> +    desc = block->virt_addr + (last_blk_desc_num - 1) * XDMA_DESC_SIZE;
>> +    desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
>> +}
>> +
>> +static inline struct xdma_chan *to_xdma_chan(struct dma_chan *chan)
>> +{
>> +    return container_of(chan, struct xdma_chan, vchan.chan);
>> +}
>> +
>> +static inline struct xdma_desc *to_xdma_desc(struct virt_dma_desc 
>> *vdesc)
>> +{
>> +    return container_of(vdesc, struct xdma_desc, vdesc);
>> +}
>> +
>> +static int xdma_enable_intr(struct xdma_device *xdev)
>> +{
>> +    int ret;
>> +
>> +    ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, 
>> XDMA_IRQ_CHAN_INT_EN_W1S, ~0);
>> +    if (ret)
>> +        xdma_err(xdev, "enable channel intr failed: %d", ret);
>> +
>> +    return ret;
>> +}
>> +
>> +static int xdma_disable_intr(struct xdma_device *xdev)
>> +{
>> +    int ret;
>> +
>> +    ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, 
>> XDMA_IRQ_CHAN_INT_EN_W1C, ~0);
>> +    if (ret)
>> +        xdma_err(xdev, "disable channel intr failed: %d", ret);
>> +
>> +    return ret;
>> +}
>> +
>> +/**
>> + * xdma_channel_init - Initialize DMA channel registers
>> + * @chan: DMA channel pointer
>> + */
>> +static int xdma_channel_init(struct xdma_chan *chan)
>> +{
>> +    struct xdma_device *xdev = chan->xdev_hdl;
>> +    int ret;
>> +
>> +    ret = xdma_write_reg(xdev, chan->base, XDMA_CHAN_CONTROL_W1C,
>> +                 CHAN_CTRL_NON_INCR_ADDR);
>> +    if (ret) {
>> +        xdma_err(xdev, "clear non incr addr failed: %d", ret);
>> +        return ret;
>> +    }
>> +
>> +    ret = xdma_write_reg(xdev, chan->base, XDMA_CHAN_INTR_ENABLE,
>> +                 CHAN_IM_ALL);
>> +    if (ret) {
>> +        xdma_err(xdev, "failed to set interrupt mask: %d", ret);
>> +        return ret;
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +/**
>> + * xdma_free_desc - Free descriptor
>> + * @vdesc: Virtual DMA descriptor
>> + */
>> +static void xdma_free_desc(struct virt_dma_desc *vdesc)
>> +{
>> +    struct xdma_desc *sw_desc;
>> +    int i;
>> +
>> +    sw_desc = to_xdma_desc(vdesc);
>> +    for (i = 0; i < sw_desc->dblk_num; i++) {
>> +        if (!sw_desc->desc_blocks[i].virt_addr)
>> +            break;
>> +        dma_pool_free(sw_desc->chan->desc_pool,
>> +                  sw_desc->desc_blocks[i].virt_addr,
>> +                  sw_desc->desc_blocks[i].dma_addr);
>> +    }
>> +    kfree(sw_desc->desc_blocks);
>> +    kfree(sw_desc);
>> +}
>> +
>> +/**
>> + * xdma_alloc_desc - Allocate descriptor
>> + * @chan: DMA channel pointer
>> + * @desc_num: Number of hardware descriptors
>> + */
>> +static struct xdma_desc *
>> +xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num)
>> +{
>> +    struct xdma_desc *sw_desc;
>> +    struct xdma_hw_desc *desc;
>> +    dma_addr_t dma_addr;
>> +    u32 dblk_num;
>> +    void *addr;
>> +    int i, j;
>> +
>> +    sw_desc = kzalloc(sizeof(*sw_desc), GFP_NOWAIT);
>> +    if (!sw_desc)
>> +        return NULL;
>> +
>> +    sw_desc->chan = chan;
>> +    sw_desc->desc_num = desc_num;
>> +    dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT);
>> +    sw_desc->desc_blocks = kcalloc(dblk_num, 
>> sizeof(*sw_desc->desc_blocks),
>> +                       GFP_NOWAIT);
>> +    if (!sw_desc->desc_blocks)
>> +        goto failed;
>> +
>> +    sw_desc->dblk_num = dblk_num;
>> +    for (i = 0; i < sw_desc->dblk_num; i++) {
>> +        addr = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &dma_addr);
>> +        if (!addr)
>> +            goto failed;
>> +
>> +        sw_desc->desc_blocks[i].virt_addr = addr;
>> +        sw_desc->desc_blocks[i].dma_addr = dma_addr;
>> +        for (j = 0, desc = addr; j < XDMA_DESC_ADJACENT; j++)
>> +            desc[j].control = cpu_to_le32(XDMA_DESC_CONTROL(1, 0));
>> +    }
>> +
>> +    xdma_link_desc_blocks(sw_desc);
>> +
>> +    return sw_desc;
>> +
>> +failed:
>> +    xdma_free_desc(&sw_desc->vdesc);
>> +    return NULL;
>> +}
>> +
>> +/**
>> + * xdma_xfer_start - Start DMA transfer
>> + * @xdma_chan: DMA channel pointer
>> + */
>> +static int xdma_xfer_start(struct xdma_chan *xdma_chan)
>> +{
>> +    struct virt_dma_desc *vd = vchan_next_desc(&xdma_chan->vchan);
>> +    struct xdma_device *xdev = xdma_chan->xdev_hdl;
>> +    struct xdma_desc_block *block;
>> +    u32 val, completed_blocks;
>> +    struct xdma_desc *desc;
>> +    int ret;
>> +
>> +    /*
>> +     * check if there is not any submitted descriptor or channel is 
>> busy.
>> +     * vchan lock should be held where this function is called.
>> +     */
>> +    if (!vd || xdma_chan->busy)
>> +        return -EINVAL;
>> +
>> +    /* clear run stop bit to get ready for transfer */
>> +    ret = xdma_write_reg(xdev, xdma_chan->base, XDMA_CHAN_CONTROL_W1C,
>> +                 CHAN_CTRL_RUN_STOP);
>> +    if (ret) {
>> +        dev_err(&xdev->pdev->dev, "write control failed: %d", ret);
>> +        return ret;
>> +    }
>> +
>> +    desc = to_xdma_desc(vd);
>> +    if (desc->dir != xdma_chan->dir) {
>> +        dev_err(&xdev->pdev->dev, "incorrect request direction");
>> +        return -EINVAL;
>> +    }
>> +
>> +    /* set DMA engine to the first descriptor block */
>> +    completed_blocks = desc->completed_desc_num / XDMA_DESC_ADJACENT;
>> +    block = &desc->desc_blocks[completed_blocks];
>> +    val = FIELD_GET(XDMA_LO_ADDR_MASK, block->dma_addr);
>> +    ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
>> +                 XDMA_SGDMA_DESC_LO, val);
>> +    if (ret) {
>> +        dev_err(&xdev->pdev->dev, "write hi addr failed: %d", ret);
>> +        return ret;
>> +    }
>> +
>> +    val = FIELD_GET(XDMA_HI_ADDR_MASK, block->dma_addr);
>> +    ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
>> +                 XDMA_SGDMA_DESC_HI, val);
>> +    if (ret) {
>> +        dev_err(&xdev->pdev->dev, "write lo addr failed: %d", ret);
>> +        return ret;
>> +    }
>> +
>> +    if (completed_blocks + 1 == desc->dblk_num)
>> +        val = (desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK;
>> +    else
>> +        val = XDMA_DESC_ADJACENT - 1;
>> +    ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
>> +                 XDMA_SGDMA_DESC_ADJ, val);
>> +    if (ret) {
>> +        dev_err(&xdev->pdev->dev, "write adjacent failed: %d", ret);
>> +        return ret;
>> +    }
>> +
>> +    /* kick off DMA transfer */
>> +    ret = xdma_write_reg(xdev, xdma_chan->base, XDMA_CHAN_CONTROL,
>> +                 CHAN_CTRL_START);
>> +    if (ret) {
>> +        dev_err(&xdev->pdev->dev, "write control failed: %d", ret);
>> +        return ret;
>> +    }
>> +
>> +    xdma_chan->busy = true;
>> +    return 0;
>> +}
>> +
>> +static void xdma_channel_tasklet(struct tasklet_struct *t)
>> +{
>> +    struct xdma_chan *xdma_chan = from_tasklet(xdma_chan, t, tasklet);
>> +
>> +    spin_lock(&xdma_chan->vchan.lock);
>> +    xdma_xfer_start(xdma_chan);
>> +    spin_unlock(&xdma_chan->vchan.lock);
>> +}
>> +
>> +/**
>> + * xdma_config_channels - Detect and config DMA channels
>> + * @xdev: DMA device pointer
>> + * @dir: Channel direction
>> + */
>> +static int xdma_config_channels(struct xdma_device *xdev,
>> +                enum dma_transfer_direction dir)
>> +{
>> +    struct xdma_platdata *pdata = dev_get_platdata(&xdev->pdev->dev);
>> +    u32 base, identifier, target;
>> +    struct xdma_chan **chans;
>> +    u32 *chan_num;
>> +    int i, j, ret;
>> +
>> +    if (dir == DMA_MEM_TO_DEV) {
>> +        base = XDMA_CHAN_H2C_OFFSET;
>> +        target = XDMA_CHAN_H2C_TARGET;
>> +        chans = &xdev->h2c_chans;
>> +        chan_num = &xdev->h2c_chan_num;
>> +    } else if (dir == DMA_DEV_TO_MEM) {
>> +        base = XDMA_CHAN_C2H_OFFSET;
>> +        target = XDMA_CHAN_C2H_TARGET;
>> +        chans = &xdev->c2h_chans;
>> +        chan_num = &xdev->c2h_chan_num;
>> +    } else {
>> +        dev_err(&xdev->pdev->dev, "invalid direction specified");
>> +        return -EINVAL;
>> +    }
>> +
>> +    /* detect number of available DMA channels */
>> +    for (i = 0, *chan_num = 0; i < pdata->max_dma_channels; i++) {
>> +        ret = xdma_read_reg(xdev, base + i * XDMA_CHAN_STRIDE,
>> +                    XDMA_CHAN_IDENTIFIER, &identifier);
>> +        if (ret) {
>> +            dev_err(&xdev->pdev->dev,
>> +                "failed to read channel id: %d", ret);
>> +            return ret;
>> +        }
>> +
>> +        /* check if it is available DMA channel */
>> +        if (XDMA_CHAN_CHECK_TARGET(identifier, target))
>> +            (*chan_num)++;
>> +    }
>> +
>> +    if (!*chan_num) {
>> +        dev_err(&xdev->pdev->dev, "does not probe any channel");
>> +        return -EINVAL;
>> +    }
>> +
>> +    *chans = devm_kzalloc(&xdev->pdev->dev, sizeof(**chans) * 
>> (*chan_num),
>> +                  GFP_KERNEL);
>> +    if (!*chans)
>> +        return -ENOMEM;
>> +
>> +    for (i = 0, j = 0; i < pdata->max_dma_channels; i++) {
>> +        ret = xdma_read_reg(xdev, base + i * XDMA_CHAN_STRIDE,
>> +                    XDMA_CHAN_IDENTIFIER, &identifier);
>> +        if (ret) {
>> +            dev_err(&xdev->pdev->dev,
>> +                "failed to read channel id: %d", ret);
>> +            return ret;
>> +        }
>> +
>> +        if (!XDMA_CHAN_CHECK_TARGET(identifier, target))
>> +            continue;
>> +
>> +        if (j == *chan_num) {
>> +            dev_err(&xdev->pdev->dev, "invalid channel number");
>> +            return -EIO;
>> +        }
>> +
>> +        /* init channel structure and hardware */
>> +        (*chans)[j].xdev_hdl = xdev;
>> +        (*chans)[j].base = base + i * XDMA_CHAN_STRIDE;
>> +        (*chans)[j].dir = dir;
>> +
>> +        ret = xdma_channel_init(&(*chans)[j]);
>> +        if (ret)
>> +            return ret;
>> +        (*chans)[j].vchan.desc_free = xdma_free_desc;
>> +        vchan_init(&(*chans)[j].vchan, &xdev->dma_dev);
>> +
>> +        j++;
>> +    }
>> +
>> +    dev_info(&xdev->pdev->dev, "configured %d %s channels", j,
>> +         (dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H");
>> +
>> +    return 0;
>> +}
>> +
>> +/**
>> + * xdma_issue_pending - Issue pending transactions
>> + * @chan: DMA channel pointer
>> + */
>> +static void xdma_issue_pending(struct dma_chan *chan)
>> +{
>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>> +    unsigned long flags;
>> +
>> +    spin_lock_irqsave(&xdma_chan->vchan.lock, flags);
>> +    if (vchan_issue_pending(&xdma_chan->vchan))
>> +        xdma_xfer_start(xdma_chan);
>> +    spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);
>> +}
>> +
>> +/**
>> + * xdma_prep_device_sg - prepare a descriptor for a
>> + *    DMA transaction
>> + * @chan: DMA channel pointer
>> + * @sgl: Transfer scatter gather list
>> + * @sg_len: Length of scatter gather list
>> + * @dir: Transfer direction
>> + * @flags: transfer ack flags
>> + * @context: APP words of the descriptor
>> + */
>> +static struct dma_async_tx_descriptor *
>> +xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,
>> +            unsigned int sg_len, enum dma_transfer_direction dir,
>> +            unsigned long flags, void *context)
>> +{
>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>> +    struct dma_async_tx_descriptor *tx_desc;
>> +    u32 desc_num = 0, i, len, rest;
>> +    struct xdma_desc_block *dblk;
>> +    struct xdma_desc *sw_desc;
>> +    struct scatterlist *sg;
>> +    dma_addr_t addr;
>> +    u64 dev_addr, *src, *dst;
>> +    struct xdma_hw_desc *desc;
>> +
>> +    for_each_sg(sgl, sg, sg_len, i)
>> +        desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX);
>> +
>> +    sw_desc = xdma_alloc_desc(xdma_chan, desc_num);
>> +    if (!sw_desc)
>> +        return NULL;
>> +    sw_desc->dir = dir;
>> +
>> +    if (dir == DMA_MEM_TO_DEV) {
>> +        dev_addr = xdma_chan->cfg.dst_addr;
>> +        src = &addr;
>> +        dst = &dev_addr;
>> +    } else {
>> +        dev_addr = xdma_chan->cfg.src_addr;
>> +        src = &dev_addr;
>> +        dst = &addr;
>> +    }
>> +
>> +    dblk = sw_desc->desc_blocks;
>> +    desc = dblk->virt_addr;
>> +    desc_num = 1;
>> +    for_each_sg(sgl, sg, sg_len, i) {
>> +        addr = sg_dma_address(sg);
>> +        rest = sg_dma_len(sg);
>> +
>> +        do {
>> +            len = min_t(u32, rest, XDMA_DESC_BLEN_MAX);
>> +            /* set hardware descriptor */
>> +            desc->bytes = cpu_to_le32(len);
>> +            desc->src_addr = cpu_to_le64(*src);
>> +            desc->dst_addr = cpu_to_le64(*dst);
>> +
>> +            if (!(desc_num & XDMA_DESC_ADJACENT_MASK)) {
>> +                dblk++;
>> +                desc = dblk->virt_addr;
>> +            } else {
>> +                desc++;
>> +            }
>> +
>> +            desc_num++;
>> +            dev_addr += len;
>> +            addr += len;
>> +            rest -= len;
>> +        } while (rest);
>> +    }
>> +
>> +    tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags);
>> +    if (!tx_desc)
>> +        goto failed;
>> +
>> +    return tx_desc;
>> +
>> +failed:
>> +    xdma_free_desc(&sw_desc->vdesc);
>> +
>> +    return NULL;
>> +}
>> +
>> +/**
>> + * xdma_device_config - Configure the DMA channel
>> + * @chan: DMA channel
>> + * @cfg: channel configuration
>> + */
>> +static int xdma_device_config(struct dma_chan *chan,
>> +                  struct dma_slave_config *cfg)
>> +{
>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>> +
>> +    memcpy(&xdma_chan->cfg, cfg, sizeof(*cfg));
>> +
>> +    return 0;
>> +}
>> +
>> +/**
>> + * xdma_free_chan_resources - Free channel resources
>> + * @chan: DMA channel
>> + */
>> +static void xdma_free_chan_resources(struct dma_chan *chan)
>> +{
>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>> +
>> +    vchan_free_chan_resources(&xdma_chan->vchan);
>> +    dma_pool_destroy(xdma_chan->desc_pool);
>> +    xdma_chan->desc_pool = NULL;
>> +}
>> +
>> +/**
>> + * xdma_alloc_chan_resources - Allocate channel resources
>> + * @chan: DMA channel
>> + */
>> +static int xdma_alloc_chan_resources(struct dma_chan *chan)
>> +{
>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>> +    struct xdma_device *xdev = xdma_chan->xdev_hdl;
>> +
>> +    xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan),
>> +                           xdev->dma_dev.dev,
>> +                           XDMA_DESC_BLOCK_SIZE,
>> +                           XDMA_DESC_BLOCK_ALIGN,
>> +                           0);
>> +    if (!xdma_chan->desc_pool) {
>> +        dev_err(&chan->dev->device,
>> +            "unable to allocate descriptor pool");
>> +        return -ENOMEM;
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +/**
>> + * xdma_channel_isr - XDMA channel interrupt handler
>> + * @irq: IRQ number
>> + * @dev_id: Pointer to the DMA channel structure
>> + */
>> +static irqreturn_t xdma_channel_isr(int irq, void *dev_id)
>> +{
>> +    struct xdma_chan *xdma_chan = dev_id;
>> +    u32 complete_desc_num = 0;
>> +    struct virt_dma_desc *vd;
>> +    struct xdma_desc *desc;
>> +    int ret;
>> +
>> +    spin_lock(&xdma_chan->vchan.lock);
>> +
>> +    /* get submitted request */
>> +    vd = vchan_next_desc(&xdma_chan->vchan);
>> +    if (!vd)
>> +        goto out;
>> +
>> +    xdma_chan->busy = false;
>> +    desc = to_xdma_desc(vd);
>> +
>> +    ret = xdma_read_reg(xdma_chan->xdev_hdl, xdma_chan->base,
>> +                XDMA_CHAN_COMPLETED_DESC, &complete_desc_num);
>> +    if (ret)
>> +        goto out;
>> +
>> +    desc->completed_desc_num += complete_desc_num;
>> +    /*
>> +     * if all data blocks are transferred, remove and complete the 
>> request
>> +     */
>> +    if (desc->completed_desc_num == desc->desc_num) {
>> +        list_del(&vd->node);
>> +        vchan_cookie_complete(vd);
>> +        goto out;
>> +    }
>> +
>> +    if (desc->completed_desc_num > desc->desc_num ||
>> +        complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT)
>> +        goto out;
>> +
>> +    /* transfer the rest of data */
>> +    tasklet_schedule(&xdma_chan->tasklet);
>
> There's an effort upstream to deprecate the usage tasklets [1]. Any 
> thought on using threaded irq instead especially with a new driver?
>
> [1]: https://lwn.net/Articles/830964/
>
> DJ

Thanks a lot for pointing this out. In this V2 patch series, we have 
moved most of code from tasklet handler to prep_* (only few register 
accesses left). Because the handler is pretty short and quick now, I 
will remove the tasklet and call xdma_xfer_start() from interrupt 
handler directly.


Thanks,

Lizhi

>
>> +
>> +out:
>> +    spin_unlock(&xdma_chan->vchan.lock);
>> +    return IRQ_HANDLED;
>> +}
>> +
>> +/**
>> + * xdma_irq_fini - Uninitialize IRQ
>> + * @xdev: DMA device pointer
>> + */
>> +static void xdma_irq_fini(struct xdma_device *xdev)
>> +{
>> +    int ret, i;
>> +
>> +    /* disable interrupt */
>> +    ret = xdma_disable_intr(xdev);
>> +    if (ret) {
>> +        dev_err(&xdev->pdev->dev, "failed to disable interrupts: %d",
>> +            ret);
>> +    }
>> +
>> +    /* free irq handler */
>> +    for (i = 0; i < xdev->h2c_chan_num; i++) {
>> +        free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
>> +        tasklet_kill(&xdev->h2c_chans[i].tasklet);
>> +    }
>> +    for (i = 0; i < xdev->c2h_chan_num; i++) {
>> +        free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
>> +        tasklet_kill(&xdev->c2h_chans[i].tasklet);
>> +    }
>> +}
>> +
>> +/**
>> + * xdma_set_vector_reg - configure hardware IRQ registers
>> + * @xdev: DMA device pointer
>> + * @vec_tbl_start: Start of IRQ registers
>> + * @irq_start: Start of IRQ
>> + * @irq_num: Number of IRQ
>> + */
>> +static int xdma_set_vector_reg(struct xdma_device *xdev, u32 
>> vec_tbl_start,
>> +                   u32 irq_start, u32 irq_num)
>> +{
>> +    u32 shift, i, val = 0;
>> +    int ret;
>> +
>> +    /* Each IRQ register is 32 bit and contains 4 IRQs */
>> +    while (irq_num > 0) {
>> +        for (i = 0; i < 4; i++) {
>> +            shift = XDMA_IRQ_VEC_SHIFT * i;
>> +            val |= irq_start << shift;
>> +            irq_start++;
>> +            irq_num--;
>> +        }
>> +
>> +        /* write IRQ register */
>> +        ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, vec_tbl_start, val);
>> +        if (ret) {
>> +            dev_err(&xdev->pdev->dev, "failed to set vector: %d",
>> +                ret);
>> +            return ret;
>> +        }
>> +        vec_tbl_start += sizeof(u32);
>> +        val = 0;
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +/**
>> + * xdma_irq_init - initialize IRQs
>> + * @xdev: DMA device pointer
>> + */
>> +static int xdma_irq_init(struct xdma_device *xdev)
>> +{
>> +    u32 irq = xdev->irq_start;
>> +    int i, j, ret;
>> +
>> +    /* return failure if there are not enough IRQs */
>> +    if (xdev->irq_num < xdev->h2c_chan_num + xdev->c2h_chan_num) {
>> +        dev_err(&xdev->pdev->dev, "not enough irq");
>> +        return -EINVAL;
>> +    }
>> +
>> +    /* setup H2C interrupt handler */
>> +    for (i = 0; i < xdev->h2c_chan_num; i++) {
>> +        ret = request_irq(irq, xdma_channel_isr, 0,
>> +                  "xdma-h2c-channel", &xdev->h2c_chans[i]);
>> +        if (ret) {
>> +            dev_err(&xdev->pdev->dev,
>> +                "H2C channel%d request irq%d failed: %d",
>> +                i, irq, ret);
>> +            for (j = 0; j < i; j++) {
>> +                free_irq(xdev->h2c_chans[j].irq,
>> +                     &xdev->h2c_chans[j]);
>> +            }
>> +            return ret;
>> +        }
>> +        xdev->h2c_chans[i].irq = irq;
>> +        tasklet_setup(&xdev->h2c_chans[i].tasklet,
>> +                  xdma_channel_tasklet);
>> +        irq++;
>> +    }
>> +
>> +    /* setup C2H interrupt handler */
>> +    for (i = 0; i < xdev->c2h_chan_num; i++) {
>> +        ret = request_irq(irq, xdma_channel_isr, 0,
>> +                  "xdma-c2h-channel", &xdev->c2h_chans[i]);
>> +        if (ret) {
>> +            dev_err(&xdev->pdev->dev,
>> +                "H2C channel%d request irq%d failed: %d",
>> +                i, irq, ret);
>> +            for (j = 0; j < i; j++) {
>> +                free_irq(xdev->c2h_chans[j].irq,
>> +                     &xdev->c2h_chans[j]);
>> +            }
>> +            goto failed_init_c2h;
>> +        }
>> +        xdev->c2h_chans[i].irq = irq;
>> +        tasklet_setup(&xdev->c2h_chans[i].tasklet,
>> +                  xdma_channel_tasklet);
>> +        irq++;
>> +    }
>> +
>> +    /* config hardware IRQ registers */
>> +    ret = xdma_set_vector_reg(xdev, XDMA_IRQ_CHAN_VEC_NUM, 0,
>> +                  xdev->h2c_chan_num + xdev->c2h_chan_num);
>> +    if (ret) {
>> +        dev_err(&xdev->pdev->dev, "failed to set channel vectors: %d",
>> +            ret);
>> +        goto failed;
>> +    }
>> +
>> +    /* enable interrupt */
>> +    ret = xdma_enable_intr(xdev);
>> +    if (ret) {
>> +        dev_err(&xdev->pdev->dev, "failed to enable interrupts: %d",
>> +            ret);
>> +        goto failed;
>> +    }
>> +
>> +    return 0;
>> +
>> +failed:
>> +    for (i = 0; i < xdev->c2h_chan_num; i++)
>> +        free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
>> +failed_init_c2h:
>> +    for (i = 0; i < xdev->h2c_chan_num; i++)
>> +        free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
>> +
>> +    return ret;
>> +}
>> +
>> +static bool xdma_filter_fn(struct dma_chan *chan, void *param)
>> +{
>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>> +    struct xdma_chan_info *chan_info = param;
>> +
>> +    if (chan_info->dir != xdma_chan->dir)
>> +        return false;
>> +
>> +    return true;
>> +}
>> +
>> +/**
>> + * xdma_remove - Driver remove function
>> + * @pdev: Pointer to the platform_device structure
>> + */
>> +static int xdma_remove(struct platform_device *pdev)
>> +{
>> +    struct xdma_device *xdev = platform_get_drvdata(pdev);
>> +
>> +    if (xdev->status & XDMA_DEV_STATUS_INIT_MSIX)
>> +        xdma_irq_fini(xdev);
>> +
>> +    if (xdev->status & XDMA_DEV_STATUS_REG_DMA)
>> +        dma_async_device_unregister(&xdev->dma_dev);
>> +
>> +    return 0;
>> +}
>> +
>> +/**
>> + * xdma_probe - Driver probe function
>> + * @pdev: Pointer to the platform_device structure
>> + */
>> +static int xdma_probe(struct platform_device *pdev)
>> +{
>> +    struct xdma_platdata *pdata = dev_get_platdata(&pdev->dev);
>> +    struct xdma_device *xdev;
>> +    void __iomem *reg_base;
>> +    struct resource *res;
>> +    int ret = -ENODEV;
>> +
>> +    if (pdata->max_dma_channels > XDMA_MAX_CHANNELS) {
>> +        dev_err(&pdev->dev, "invalid max dma channels %d",
>> +            pdata->max_dma_channels);
>> +        return -EINVAL;
>> +    }
>> +
>> +    xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);
>> +    if (!xdev)
>> +        return -ENOMEM;
>> +
>> +    platform_set_drvdata(pdev, xdev);
>> +    xdev->pdev = pdev;
>> +
>> +    res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
>> +    if (!res) {
>> +        dev_err(&pdev->dev, "failed to get irq resource");
>> +        goto failed;
>> +    }
>> +    xdev->irq_start = res->start;
>> +    xdev->irq_num = res->end - res->start + 1;
>> +
>> +    res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
>> +    if (!res) {
>> +        dev_err(&pdev->dev, "failed to get io resource");
>> +        goto failed;
>> +    }
>> +
>> +    reg_base = devm_ioremap_resource(&pdev->dev, res);
>> +    if (!reg_base) {
>> +        dev_err(&pdev->dev, "ioremap failed");
>> +        goto failed;
>> +    }
>> +
>> +    xdev->regmap = devm_regmap_init_mmio(&pdev->dev, reg_base,
>> +                         &xdma_regmap_config);
>> +    if (!xdev->regmap) {
>> +        dev_err(&pdev->dev, "config regmap failed: %d", ret);
>> +        goto failed;
>> +    }
>> +    INIT_LIST_HEAD(&xdev->dma_dev.channels);
>> +
>> +    ret = xdma_config_channels(xdev, DMA_MEM_TO_DEV);
>> +    if (ret) {
>> +        dev_err(&pdev->dev, "config H2C channels failed: %d", ret);
>> +        goto failed;
>> +    }
>> +
>> +    ret = xdma_config_channels(xdev, DMA_DEV_TO_MEM);
>> +    if (ret) {
>> +        dev_err(&pdev->dev, "config C2H channels failed: %d", ret);
>> +        goto failed;
>> +    }
>> +
>> +    dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask);
>> +    dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask);
>> +
>> +    xdev->dma_dev.dev = &pdev->dev;
>> +    xdev->dma_dev.device_free_chan_resources = 
>> xdma_free_chan_resources;
>> +    xdev->dma_dev.device_alloc_chan_resources = 
>> xdma_alloc_chan_resources;
>> +    xdev->dma_dev.device_tx_status = dma_cookie_status;
>> +    xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg;
>> +    xdev->dma_dev.device_config = xdma_device_config;
>> +    xdev->dma_dev.device_issue_pending = xdma_issue_pending;
>> +    xdev->dma_dev.filter.map = pdata->device_map;
>> +    xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt;
>> +    xdev->dma_dev.filter.fn = xdma_filter_fn;
>> +
>> +    ret = dma_async_device_register(&xdev->dma_dev);
>> +    if (ret) {
>> +        dev_err(&pdev->dev, "failed to register Xilinx XDMA: %d", ret);
>> +        goto failed;
>> +    }
>> +    xdev->status |= XDMA_DEV_STATUS_REG_DMA;
>> +
>> +    ret = xdma_irq_init(xdev);
>> +    if (ret) {
>> +        dev_err(&pdev->dev, "failed to init msix: %d", ret);
>> +        goto failed;
>> +    }
>> +    xdev->status |= XDMA_DEV_STATUS_INIT_MSIX;
>> +
>> +    return 0;
>> +
>> +failed:
>> +    xdma_remove(pdev);
>> +
>> +    return ret;
>> +}
>> +
>> +static const struct platform_device_id xdma_id_table[] = {
>> +    { "xdma", 0},
>> +    { },
>> +};
>> +
>> +static struct platform_driver xdma_driver = {
>> +    .driver        = {
>> +        .name = "xdma",
>> +    },
>> +    .id_table    = xdma_id_table,
>> +    .probe        = xdma_probe,
>> +    .remove        = xdma_remove,
>> +};
>> +
>> +module_platform_driver(xdma_driver);
>> +
>> +MODULE_DESCRIPTION("AMD XDMA driver");
>> +MODULE_AUTHOR("XRT Team <[e-mail]>");
>> +MODULE_LICENSE("GPL");
>> diff --git a/include/linux/platform_data/amd_xdma.h 
>> b/include/linux/platform_data/amd_xdma.h
>> new file mode 100644
>> index 000000000000..04bcfc74ab36
>> --- /dev/null
>> +++ b/include/linux/platform_data/amd_xdma.h
>> @@ -0,0 +1,34 @@
>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>> +/*
>> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
>> + */
>> +
>> +#ifndef _PLATDATA_AMD_XDMA_H
>> +#define _PLATDATA_AMD_XDMA_H
>> +
>> +#include <linux/dmaengine.h>
>> +
>> +/**
>> + * struct xdma_chan_info - DMA channel information
>> + *    This information is used to match channel when request dma 
>> channel
>> + * @dir: Channel transfer direction
>> + */
>> +struct xdma_chan_info {
>> +    enum dma_transfer_direction dir;
>> +};
>> +
>> +#define XDMA_FILTER_PARAM(chan_info)    ((void *)(chan_info))
>> +
>> +struct dma_slave_map;
>> +
>> +/**
>> + * struct xdma_platdata - platform specific data for XDMA engine
>> + * @max_dma_channels: Maximum dma channels in each direction
>> + */
>> +struct xdma_platdata {
>> +    u32 max_dma_channels;
>> +    u32 device_map_cnt;
>> +    struct dma_slave_map *device_map;
>> +};
>> +
>> +#endif /* _PLATDATA_AMD_XDMA_H */

Dave Jiang Sept. 16, 2022, 8:52 p.m. UTC | #3

On 9/16/2022 1:10 PM, Lizhi Hou wrote:
>
> On 9/16/22 08:39, Dave Jiang wrote:
>>
>> On 9/16/2022 8:26 AM, Lizhi Hou wrote:
>>> Add driver to enable PCIe board which uses XDMA (the DMA/Bridge 
>>> Subsystem
>>> for PCI Express). For example, Xilinx Alveo PCIe devices.
>>> https://www.xilinx.com/products/boards-and-kits/alveo.html
>>>
>>> The XDMA engine support up to 4 Host to Card (H2C) and 4 Card to 
>>> Host (C2H)
>>> channels. Memory transfers are specified on a per-channel basis in
>>> descriptor linked lists, which the DMA fetches from host memory and
>>> processes. Events such as descriptor completion and errors are signaled
>>> using interrupts. The hardware detail is provided by
>>> https://docs.xilinx.com/r/en-US/pg195-pcie-dma/Introduction
>>>
>>> This driver implements dmaengine APIs.
>>>      - probe the available DMA channels
>>>      - use dma_slave_map for channel lookup
>>>      - use virtual channel to manage dmaengine tx descriptors
>>>      - implement device_prep_slave_sg callback to handle host 
>>> scatter gather
>>>        list
>>>      - implement device_config to config device address for DMA 
>>> transfer
>>>
>>> Signed-off-by: Lizhi Hou <[e-mail]>
>>> Signed-off-by: Sonal Santan <[e-mail]>
>>> Signed-off-by: Max Zhen <[e-mail]>
>>> Signed-off-by: Brian Xu <[e-mail]>
>>> ---
>>>   MAINTAINERS                            |  10 +
>>>   drivers/dma/Kconfig                    |  13 +
>>>   drivers/dma/xilinx/Makefile            |   1 +
>>>   drivers/dma/xilinx/xdma-regs.h         | 171 +++++
>>>   drivers/dma/xilinx/xdma.c              | 982 
>>> +++++++++++++++++++++++++
>>>   include/linux/platform_data/amd_xdma.h |  34 +
>>>   6 files changed, 1211 insertions(+)
>>>   create mode 100644 drivers/dma/xilinx/xdma-regs.h
>>>   create mode 100644 drivers/dma/xilinx/xdma.c
>>>   create mode 100644 include/linux/platform_data/amd_xdma.h
>>>
>>> diff --git a/MAINTAINERS b/MAINTAINERS
>>> index e8c52d0192a6..c1be0b2e378a 100644
>>> --- a/MAINTAINERS
>>> +++ b/MAINTAINERS
>>> @@ -21683,6 +21683,16 @@ F: 
>>> Documentation/devicetree/bindings/media/xilinx/
>>>   F:    drivers/media/platform/xilinx/
>>>   F:    include/uapi/linux/xilinx-v4l2-controls.h
>>>   +XILINX XDMA DRIVER
>>> +M:    Lizhi Hou <[e-mail]>
>>> +M:    Brian Xu <[e-mail]>
>>> +M:    Raj Kumar Rampelli <[e-mail]>
>>> +L:    [e-mail]
>>> +S:    Supported
>>> +F:    drivers/dma/xilinx/xdma-regs.h
>>> +F:    drivers/dma/xilinx/xdma.c
>>> +F:    include/linux/platform_data/amd_xdma.h
>>> +
>>>   XILINX ZYNQMP DPDMA DRIVER
>>>   M:    Hyun Kwon <[e-mail]>
>>>   M:    Laurent Pinchart <[e-mail]>
>>> diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
>>> index d5de3f77d3aa..4d90f2f51655 100644
>>> --- a/drivers/dma/Kconfig
>>> +++ b/drivers/dma/Kconfig
>>> @@ -733,6 +733,19 @@ config XILINX_ZYNQMP_DPDMA
>>>         driver provides the dmaengine required by the DisplayPort 
>>> subsystem
>>>         display driver.
>>>   +config XILINX_XDMA
>>> +    tristate "Xilinx DMA/Bridge Subsystem DMA Engine"
>>> +    select DMA_ENGINE
>>> +    select DMA_VIRTUAL_CHANNELS
>>> +    select REGMAP_MMIO
>>> +    help
>>> +      Enable support for Xilinx DMA/Bridge Subsystem DMA engine. 
>>> The DMA
>>> +      provides high performance block data movement between Host 
>>> memory
>>> +      and the DMA subsystem. These direct memory transfers can be 
>>> both in
>>> +      the Host to Card (H2C) and Card to Host (C2H) transfers.
>>> +      The core also provides up to 16 user interrupt wires that 
>>> generate
>>> +      interrupts to the host.
>>> +
>>>   # driver files
>>>   source "drivers/dma/bestcomm/Kconfig"
>>>   diff --git a/drivers/dma/xilinx/Makefile 
>>> b/drivers/dma/xilinx/Makefile
>>> index 767bb45f641f..c7a538a56643 100644
>>> --- a/drivers/dma/xilinx/Makefile
>>> +++ b/drivers/dma/xilinx/Makefile
>>> @@ -2,3 +2,4 @@
>>>   obj-$(CONFIG_XILINX_DMA) += xilinx_dma.o
>>>   obj-$(CONFIG_XILINX_ZYNQMP_DMA) += zynqmp_dma.o
>>>   obj-$(CONFIG_XILINX_ZYNQMP_DPDMA) += xilinx_dpdma.o
>>> +obj-$(CONFIG_XILINX_XDMA) += xdma.o
>>> diff --git a/drivers/dma/xilinx/xdma-regs.h 
>>> b/drivers/dma/xilinx/xdma-regs.h
>>> new file mode 100644
>>> index 000000000000..c42df0ed4aca
>>> --- /dev/null
>>> +++ b/drivers/dma/xilinx/xdma-regs.h
>>> @@ -0,0 +1,171 @@
>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>>> +/*
>>> + * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights 
>>> reserved.
>>> + */
>>> +
>>> +#ifndef __DMA_XDMA_REGS_H
>>> +#define __DMA_XDMA_REGS_H
>>> +
>>> +/* The length of register space exposed to host */
>>> +#define XDMA_REG_SPACE_LEN    65536
>>> +
>>> +/*
>>> + * maximum number of DMA channels for each direction:
>>> + * Host to Card (H2C) or Card to Host (C2H)
>>> + */
>>> +#define XDMA_MAX_CHANNELS    4
>>> +
>>> +/* macros to get higher and lower 32-bit address */
>>> +#define XDMA_HI_ADDR_MASK    GENMASK_ULL(63, 32)
>>> +#define XDMA_LO_ADDR_MASK    GENMASK_ULL(31, 0)
>>> +
>>> +/*
>>> + * macros to define the number of descriptor blocks can be used in one
>>> + * DMA transfer request.
>>> + * the DMA engine uses a linked list of descriptor blocks that 
>>> specify the
>>> + * source, destination, and length of the DMA transfers.
>>> + */
>>> +#define XDMA_DESC_BLOCK_NUM        BIT(7)
>>> +#define XDMA_DESC_BLOCK_MASK        (XDMA_DESC_BLOCK_NUM - 1)
>>> +
>>> +/* descriptor definitions */
>>> +#define XDMA_DESC_ADJACENT        BIT(5)
>>> +#define XDMA_DESC_ADJACENT_MASK        (XDMA_DESC_ADJACENT - 1)
>>> +#define XDMA_DESC_MAGIC            0xad4bUL
>>> +#define XDMA_DESC_MAGIC_SHIFT        16
>>> +#define XDMA_DESC_ADJACENT_SHIFT    8
>>> +#define XDMA_DESC_STOPPED        BIT(0)
>>> +#define XDMA_DESC_COMPLETED        BIT(1)
>>> +#define XDMA_DESC_BLEN_BITS        28
>>> +#define XDMA_DESC_BLEN_MAX        (BIT(XDMA_DESC_BLEN_BITS) - 
>>> PAGE_SIZE)
>>> +
>>> +/* macros to construct the descriptor control word */
>>> +#define XDMA_DESC_CONTROL(adjacent, flag)                \
>>> +    ((XDMA_DESC_MAGIC << XDMA_DESC_MAGIC_SHIFT) |            \
>>> +     (((adjacent) - 1) << XDMA_DESC_ADJACENT_SHIFT) | (flag))
>>> +#define XDMA_DESC_CONTROL_LAST                        \
>>> +    XDMA_DESC_CONTROL(1, XDMA_DESC_STOPPED | XDMA_DESC_COMPLETED)
>>> +
>>> +/*
>>> + * Descriptor for a single contiguous memory block transfer.
>>> + *
>>> + * Multiple descriptors are linked by means of the next pointer. An 
>>> additional
>>> + * extra adjacent number gives the amount of extra contiguous 
>>> descriptors.
>>> + *
>>> + * The descriptors are in root complex memory, and the bytes in the 
>>> 32-bit
>>> + * words must be in little-endian byte ordering.
>>> + */
>>> +struct xdma_hw_desc {
>>> +    __le32        control;
>>> +    __le32        bytes;
>>> +    __le64        src_addr;
>>> +    __le64        dst_addr;
>>> +    __le64        next_desc;
>>> +};
>>> +
>>> +#define XDMA_DESC_SIZE        sizeof(struct xdma_hw_desc)
>>> +#define XDMA_DESC_BLOCK_SIZE    (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT)
>>> +#define XDMA_DESC_BLOCK_ALIGN    4096
>>> +
>>> +/*
>>> + * Channel registers
>>> + */
>>> +#define XDMA_CHAN_IDENTIFIER        0x0
>>> +#define XDMA_CHAN_CONTROL        0x4
>>> +#define XDMA_CHAN_CONTROL_W1S        0x8
>>> +#define XDMA_CHAN_CONTROL_W1C        0xc
>>> +#define XDMA_CHAN_STATUS        0x40
>>> +#define XDMA_CHAN_COMPLETED_DESC    0x48
>>> +#define XDMA_CHAN_ALIGNMENTS        0x4c
>>> +#define XDMA_CHAN_INTR_ENABLE        0x90
>>> +#define XDMA_CHAN_INTR_ENABLE_W1S    0x94
>>> +#define XDMA_CHAN_INTR_ENABLE_W1C    0x9c
>>> +
>>> +#define XDMA_CHAN_STRIDE    0x100
>>> +#define XDMA_CHAN_H2C_OFFSET    0x0
>>> +#define XDMA_CHAN_C2H_OFFSET    0x1000
>>> +#define XDMA_CHAN_H2C_TARGET    0x0
>>> +#define XDMA_CHAN_C2H_TARGET    0x1
>>> +
>>> +/* macro to check if channel is available */
>>> +#define XDMA_CHAN_MAGIC        0x1fc0
>>> +#define XDMA_CHAN_CHECK_TARGET(id, target)        \
>>> +    (((u32)(id) >> 16) == XDMA_CHAN_MAGIC + (target))
>>> +
>>> +/* bits of the channel control register */
>>> +#define CHAN_CTRL_RUN_STOP            BIT(0)
>>> +#define CHAN_CTRL_IE_DESC_STOPPED        BIT(1)
>>> +#define CHAN_CTRL_IE_DESC_COMPLETED        BIT(2)
>>> +#define CHAN_CTRL_IE_DESC_ALIGN_MISMATCH    BIT(3)
>>> +#define CHAN_CTRL_IE_MAGIC_STOPPED        BIT(4)
>>> +#define CHAN_CTRL_IE_IDLE_STOPPED        BIT(6)
>>> +#define CHAN_CTRL_IE_READ_ERROR            (0x1FUL << 9)
>>> +#define CHAN_CTRL_IE_DESC_ERROR            (0x1FUL << 19)
>>> +#define CHAN_CTRL_NON_INCR_ADDR            BIT(25)
>>> +#define CHAN_CTRL_POLL_MODE_WB            BIT(26)
>>> +
>>> +#define CHAN_CTRL_START    (CHAN_CTRL_RUN_STOP | \
>>> +             CHAN_CTRL_IE_DESC_STOPPED |            \
>>> +             CHAN_CTRL_IE_DESC_COMPLETED |            \
>>> +             CHAN_CTRL_IE_DESC_ALIGN_MISMATCH |        \
>>> +             CHAN_CTRL_IE_MAGIC_STOPPED |            \
>>> +             CHAN_CTRL_IE_READ_ERROR |            \
>>> +             CHAN_CTRL_IE_DESC_ERROR)
>>> +
>>> +/* bits of the channel interrupt enable mask */
>>> +#define CHAN_IM_DESC_ERROR            BIT(19)
>>> +#define CHAN_IM_READ_ERROR            BIT(9)
>>> +#define CHAN_IM_IDLE_STOPPED            BIT(6)
>>> +#define CHAN_IM_MAGIC_STOPPED            BIT(4)
>>> +#define CHAN_IM_DESC_COMPLETED            BIT(2)
>>> +#define CHAN_IM_DESC_STOPPED            BIT(1)
>>> +
>>> +#define CHAN_IM_ALL    (CHAN_IM_DESC_ERROR | CHAN_IM_READ_ERROR |    \
>>> +             CHAN_IM_IDLE_STOPPED | CHAN_IM_MAGIC_STOPPED | \
>>> +             CHAN_IM_DESC_COMPLETED | CHAN_IM_DESC_STOPPED)
>>> +
>>> +/*
>>> + * Channel SGDMA registers
>>> + */
>>> +#define XDMA_SGDMA_IDENTIFIER    0x0
>>> +#define XDMA_SGDMA_DESC_LO    0x80
>>> +#define XDMA_SGDMA_DESC_HI    0x84
>>> +#define XDMA_SGDMA_DESC_ADJ    0x88
>>> +#define XDMA_SGDMA_DESC_CREDIT    0x8c
>>> +
>>> +#define XDMA_SGDMA_BASE(chan_base)    ((chan_base) + 0x4000)
>>> +
>>> +/* bits of the SG DMA control register */
>>> +#define XDMA_CTRL_RUN_STOP            BIT(0)
>>> +#define XDMA_CTRL_IE_DESC_STOPPED        BIT(1)
>>> +#define XDMA_CTRL_IE_DESC_COMPLETED        BIT(2)
>>> +#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH    BIT(3)
>>> +#define XDMA_CTRL_IE_MAGIC_STOPPED        BIT(4)
>>> +#define XDMA_CTRL_IE_IDLE_STOPPED        BIT(6)
>>> +#define XDMA_CTRL_IE_READ_ERROR            (0x1FUL << 9)
>>> +#define XDMA_CTRL_IE_DESC_ERROR            (0x1FUL << 19)
>>> +#define XDMA_CTRL_NON_INCR_ADDR            BIT(25)
>>> +#define XDMA_CTRL_POLL_MODE_WB            BIT(26)
>>> +
>>> +/*
>>> + * interrupt registers
>>> + */
>>> +#define XDMA_IRQ_IDENTIFIER        0x0
>>> +#define XDMA_IRQ_USER_INT_EN        0x04
>>> +#define XDMA_IRQ_USER_INT_EN_W1S    0x08
>>> +#define XDMA_IRQ_USER_INT_EN_W1C    0x0c
>>> +#define XDMA_IRQ_CHAN_INT_EN        0x10
>>> +#define XDMA_IRQ_CHAN_INT_EN_W1S    0x14
>>> +#define XDMA_IRQ_CHAN_INT_EN_W1C    0x18
>>> +#define XDMA_IRQ_USER_INT_REQ        0x40
>>> +#define XDMA_IRQ_CHAN_INT_REQ        0x44
>>> +#define XDMA_IRQ_USER_INT_PEND        0x48
>>> +#define XDMA_IRQ_CHAN_INT_PEND        0x4c
>>> +#define XDMA_IRQ_USER_VEC_NUM        0x80
>>> +#define XDMA_IRQ_CHAN_VEC_NUM        0xa0
>>> +
>>> +#define XDMA_IRQ_BASE            0x2000
>>> +#define XDMA_IRQ_VEC_SHIFT        8
>>> +
>>> +#endif /* __DMA_XDMA_REGS_H */
>>> diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c
>>> new file mode 100644
>>> index 000000000000..59797ae5251c
>>> --- /dev/null
>>> +++ b/drivers/dma/xilinx/xdma.c
>>> @@ -0,0 +1,982 @@
>>> +// SPDX-License-Identifier: GPL-2.0-or-later
>>> +/*
>>> + * DMA driver for Xilinx DMA/Bridge Subsystem
>>> + *
>>> + * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights 
>>> reserved.
>>> + */
>>> +
>>> +/*
>>> + * The DMA/Bridge Subsystem for PCI Express allows for the movement 
>>> of data
>>> + * between Host memory and the DMA subsystem. It does this by 
>>> operating on
>>> + * 'descriptors' that contain information about the source, 
>>> destination and
>>> + * amount of data to transfer. These direct memory transfers can be 
>>> both in
>>> + * the Host to Card (H2C) and Card to Host (C2H) transfers. The DMA 
>>> can be
>>> + * configured to have a single AXI4 Master interface shared by all 
>>> channels
>>> + * or one AXI4-Stream interface for each channel enabled. Memory 
>>> transfers are
>>> + * specified on a per-channel basis in descriptor linked lists, 
>>> which the DMA
>>> + * fetches from host memory and processes. Events such as 
>>> descriptor completion
>>> + * and errors are signaled using interrupts. The core also provides 
>>> up to 16
>>> + * user interrupt wires that generate interrupts to the host.
>>> + */
>>> +
>>> +#include <linux/mod_devicetable.h>
>>> +#include <linux/bitfield.h>
>>> +#include <linux/dmapool.h>
>>> +#include <linux/regmap.h>
>>> +#include <linux/dmaengine.h>
>>> +#include <linux/platform_device.h>
>>> +#include <linux/platform_data/amd_xdma.h>
>>> +#include <linux/dma-mapping.h>
>>> +#include "../virt-dma.h"
>>> +#include "xdma-regs.h"
>>> +
>>> +/* mmio regmap config for all XDMA registers */
>>> +static const struct regmap_config xdma_regmap_config = {
>>> +    .reg_bits = 32,
>>> +    .val_bits = 32,
>>> +    .reg_stride = 4,
>>> +    .max_register = XDMA_REG_SPACE_LEN,
>>> +};
>>> +
>>> +/**
>>> + * struct xdma_desc_block - Descriptor block
>>> + * @virt_addr: Virtual address of block start
>>> + * @dma_addr: DMA address of block start
>>> + */
>>> +struct xdma_desc_block {
>>> +    void        *virt_addr;
>>> +    dma_addr_t    dma_addr;
>>> +};
>>> +
>>> +/**
>>> + * struct xdma_chan - Driver specific DMA channel structure
>>> + * @vchan: Virtual channel
>>> + * @xdev_hdl: Pointer to DMA device structure
>>> + * @base: Offset of channel registers
>>> + * @desc_pool: Descriptor pool
>>> + * @busy: Busy flag of the channel
>>> + * @dir: Transferring direction of the channel
>>> + * @cfg: Transferring config of the channel
>>> + * @irq: IRQ assigned to the channel
>>> + * @tasklet: Channel tasklet
>>> + */
>>> +struct xdma_chan {
>>> +    struct virt_dma_chan        vchan;
>>> +    void                *xdev_hdl;
>>> +    u32                base;
>>> +    struct dma_pool            *desc_pool;
>>> +    bool                busy;
>>> +    enum dma_transfer_direction    dir;
>>> +    struct dma_slave_config        cfg;
>>> +    u32                irq;
>>> +    struct tasklet_struct        tasklet;
>>> +};
>>> +
>>> +/**
>>> + * struct xdma_desc - DMA desc structure
>>> + * @vdesc: Virtual DMA descriptor
>>> + * @dir: Transferring direction of the request
>>> + * @dev_addr: Physical address on DMA device side
>>> + * @desc_blocks: Hardware descriptor blocks
>>> + * @dblk_num: Number of hardware descriptor blocks
>>> + * @desc_num: Number of hardware descriptors
>>> + * @completed_desc_num: Completed hardware descriptors
>>> + */
>>> +struct xdma_desc {
>>> +    struct virt_dma_desc        vdesc;
>>> +    struct xdma_chan        *chan;
>>> +    enum dma_transfer_direction    dir;
>>> +    u64                dev_addr;
>>> +    struct xdma_desc_block        *desc_blocks;
>>> +    u32                dblk_num;
>>> +    u32                desc_num;
>>> +    u32                completed_desc_num;
>>> +};
>>> +
>>> +#define XDMA_DEV_STATUS_REG_DMA        BIT(0)
>>> +#define XDMA_DEV_STATUS_INIT_MSIX    BIT(1)
>>> +
>>> +/**
>>> + * struct xdma_device - DMA device structure
>>> + * @pdev: Platform device pointer
>>> + * @dma_dev: DMA device structure
>>> + * @regmap: MMIO regmap for DMA registers
>>> + * @h2c_chans: Host to Card channels
>>> + * @c2h_chans: Card to Host channels
>>> + * @h2c_chan_num: Number of H2C channels
>>> + * @c2h_chan_num: Number of C2H channels
>>> + * @irq_start: Start IRQ assigned to device
>>> + * @irq_num: Number of IRQ assigned to device
>>> + * @status: Initialization status
>>> + */
>>> +struct xdma_device {
>>> +    struct platform_device    *pdev;
>>> +    struct dma_device    dma_dev;
>>> +    struct regmap        *regmap;
>>> +    struct xdma_chan    *h2c_chans;
>>> +    struct xdma_chan    *c2h_chans;
>>> +    u32            h2c_chan_num;
>>> +    u32            c2h_chan_num;
>>> +    u32            irq_start;
>>> +    u32            irq_num;
>>> +    u32            status;
>>> +};
>>> +
>>> +#define xdma_err(xdev, fmt, args...)                    \
>>> +    dev_err(&(xdev)->pdev->dev, fmt, ##args)
>>> +
>>> +/* Read and Write DMA registers */
>>> +static inline int xdma_read_reg(struct xdma_device *xdev, u32 base, 
>>> u32 reg,
>>> +                u32 *val)
>>> +{
>>> +    return regmap_read(xdev->regmap, base + reg, val);
>>> +}
>>> +
>>> +static inline int xdma_write_reg(struct xdma_device *xdev, u32 
>>> base, u32 reg,
>>> +                 u32 val)
>>> +{
>>> +    return regmap_write(xdev->regmap, base + reg, val);
>>> +}
>>> +
>>> +/* Get the last desc in a desc block */
>>> +static inline void *xdma_blk_last_desc(struct xdma_desc_block *block)
>>> +{
>>> +    return block->virt_addr + (XDMA_DESC_ADJACENT - 1) * 
>>> XDMA_DESC_SIZE;
>>> +}
>>> +
>>> +/**
>>> + * xdma_link_desc_blocks - Link descriptor blocks for DMA transfer
>>> + * @sw_desc: Tx descriptor pointer
>>> + */
>>> +static void xdma_link_desc_blocks(struct xdma_desc *sw_desc)
>>> +{
>>> +    struct xdma_desc_block *block;
>>> +    u32 last_blk_desc_num, desc_control;
>>> +    struct xdma_hw_desc *desc;
>>> +    int i;
>>> +
>>> +    desc_control = XDMA_DESC_CONTROL(XDMA_DESC_ADJACENT, 0);
>>> +    for (i = 1; i < sw_desc->dblk_num; i++) {
>>> +        block = &sw_desc->desc_blocks[i - 1];
>>> +        desc = xdma_blk_last_desc(block);
>>> +
>>> +        if (!(i & XDMA_DESC_BLOCK_MASK)) {
>>> +            desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
>>> +            continue;
>>> +        }
>>> +        desc->control = cpu_to_le32(desc_control);
>>> +        desc->next_desc = cpu_to_le64(block[1].dma_addr);
>>> +    }
>>> +
>>> +    /* update the last block */
>>> +    last_blk_desc_num = sw_desc->desc_num & XDMA_DESC_ADJACENT_MASK;
>>> +    if ((sw_desc->dblk_num & XDMA_DESC_BLOCK_MASK) > 1 &&
>>> +        last_blk_desc_num) {
>>> +        block = &sw_desc->desc_blocks[sw_desc->dblk_num - 2];
>>> +        desc = xdma_blk_last_desc(block);
>>> +        desc_control = XDMA_DESC_CONTROL(last_blk_desc_num, 0);
>>> +        desc->control = cpu_to_le32(desc_control);
>>> +    }
>>> +
>>> +    block = &sw_desc->desc_blocks[sw_desc->dblk_num - 1];
>>> +    desc = block->virt_addr + (last_blk_desc_num - 1) * 
>>> XDMA_DESC_SIZE;
>>> +    desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
>>> +}
>>> +
>>> +static inline struct xdma_chan *to_xdma_chan(struct dma_chan *chan)
>>> +{
>>> +    return container_of(chan, struct xdma_chan, vchan.chan);
>>> +}
>>> +
>>> +static inline struct xdma_desc *to_xdma_desc(struct virt_dma_desc 
>>> *vdesc)
>>> +{
>>> +    return container_of(vdesc, struct xdma_desc, vdesc);
>>> +}
>>> +
>>> +static int xdma_enable_intr(struct xdma_device *xdev)
>>> +{
>>> +    int ret;
>>> +
>>> +    ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, 
>>> XDMA_IRQ_CHAN_INT_EN_W1S, ~0);
>>> +    if (ret)
>>> +        xdma_err(xdev, "enable channel intr failed: %d", ret);
>>> +
>>> +    return ret;
>>> +}
>>> +
>>> +static int xdma_disable_intr(struct xdma_device *xdev)
>>> +{
>>> +    int ret;
>>> +
>>> +    ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, 
>>> XDMA_IRQ_CHAN_INT_EN_W1C, ~0);
>>> +    if (ret)
>>> +        xdma_err(xdev, "disable channel intr failed: %d", ret);
>>> +
>>> +    return ret;
>>> +}
>>> +
>>> +/**
>>> + * xdma_channel_init - Initialize DMA channel registers
>>> + * @chan: DMA channel pointer
>>> + */
>>> +static int xdma_channel_init(struct xdma_chan *chan)
>>> +{
>>> +    struct xdma_device *xdev = chan->xdev_hdl;
>>> +    int ret;
>>> +
>>> +    ret = xdma_write_reg(xdev, chan->base, XDMA_CHAN_CONTROL_W1C,
>>> +                 CHAN_CTRL_NON_INCR_ADDR);
>>> +    if (ret) {
>>> +        xdma_err(xdev, "clear non incr addr failed: %d", ret);
>>> +        return ret;
>>> +    }
>>> +
>>> +    ret = xdma_write_reg(xdev, chan->base, XDMA_CHAN_INTR_ENABLE,
>>> +                 CHAN_IM_ALL);
>>> +    if (ret) {
>>> +        xdma_err(xdev, "failed to set interrupt mask: %d", ret);
>>> +        return ret;
>>> +    }
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +/**
>>> + * xdma_free_desc - Free descriptor
>>> + * @vdesc: Virtual DMA descriptor
>>> + */
>>> +static void xdma_free_desc(struct virt_dma_desc *vdesc)
>>> +{
>>> +    struct xdma_desc *sw_desc;
>>> +    int i;
>>> +
>>> +    sw_desc = to_xdma_desc(vdesc);
>>> +    for (i = 0; i < sw_desc->dblk_num; i++) {
>>> +        if (!sw_desc->desc_blocks[i].virt_addr)
>>> +            break;
>>> +        dma_pool_free(sw_desc->chan->desc_pool,
>>> +                  sw_desc->desc_blocks[i].virt_addr,
>>> +                  sw_desc->desc_blocks[i].dma_addr);
>>> +    }
>>> +    kfree(sw_desc->desc_blocks);
>>> +    kfree(sw_desc);
>>> +}
>>> +
>>> +/**
>>> + * xdma_alloc_desc - Allocate descriptor
>>> + * @chan: DMA channel pointer
>>> + * @desc_num: Number of hardware descriptors
>>> + */
>>> +static struct xdma_desc *
>>> +xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num)
>>> +{
>>> +    struct xdma_desc *sw_desc;
>>> +    struct xdma_hw_desc *desc;
>>> +    dma_addr_t dma_addr;
>>> +    u32 dblk_num;
>>> +    void *addr;
>>> +    int i, j;
>>> +
>>> +    sw_desc = kzalloc(sizeof(*sw_desc), GFP_NOWAIT);
>>> +    if (!sw_desc)
>>> +        return NULL;
>>> +
>>> +    sw_desc->chan = chan;
>>> +    sw_desc->desc_num = desc_num;
>>> +    dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT);
>>> +    sw_desc->desc_blocks = kcalloc(dblk_num, 
>>> sizeof(*sw_desc->desc_blocks),
>>> +                       GFP_NOWAIT);
>>> +    if (!sw_desc->desc_blocks)
>>> +        goto failed;
>>> +
>>> +    sw_desc->dblk_num = dblk_num;
>>> +    for (i = 0; i < sw_desc->dblk_num; i++) {
>>> +        addr = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &dma_addr);
>>> +        if (!addr)
>>> +            goto failed;
>>> +
>>> +        sw_desc->desc_blocks[i].virt_addr = addr;
>>> +        sw_desc->desc_blocks[i].dma_addr = dma_addr;
>>> +        for (j = 0, desc = addr; j < XDMA_DESC_ADJACENT; j++)
>>> +            desc[j].control = cpu_to_le32(XDMA_DESC_CONTROL(1, 0));
>>> +    }
>>> +
>>> +    xdma_link_desc_blocks(sw_desc);
>>> +
>>> +    return sw_desc;
>>> +
>>> +failed:
>>> +    xdma_free_desc(&sw_desc->vdesc);
>>> +    return NULL;
>>> +}
>>> +
>>> +/**
>>> + * xdma_xfer_start - Start DMA transfer
>>> + * @xdma_chan: DMA channel pointer
>>> + */
>>> +static int xdma_xfer_start(struct xdma_chan *xdma_chan)
>>> +{
>>> +    struct virt_dma_desc *vd = vchan_next_desc(&xdma_chan->vchan);
>>> +    struct xdma_device *xdev = xdma_chan->xdev_hdl;
>>> +    struct xdma_desc_block *block;
>>> +    u32 val, completed_blocks;
>>> +    struct xdma_desc *desc;
>>> +    int ret;
>>> +
>>> +    /*
>>> +     * check if there is not any submitted descriptor or channel is 
>>> busy.
>>> +     * vchan lock should be held where this function is called.
>>> +     */
>>> +    if (!vd || xdma_chan->busy)
>>> +        return -EINVAL;
>>> +
>>> +    /* clear run stop bit to get ready for transfer */
>>> +    ret = xdma_write_reg(xdev, xdma_chan->base, XDMA_CHAN_CONTROL_W1C,
>>> +                 CHAN_CTRL_RUN_STOP);
>>> +    if (ret) {
>>> +        dev_err(&xdev->pdev->dev, "write control failed: %d", ret);
>>> +        return ret;
>>> +    }
>>> +
>>> +    desc = to_xdma_desc(vd);
>>> +    if (desc->dir != xdma_chan->dir) {
>>> +        dev_err(&xdev->pdev->dev, "incorrect request direction");
>>> +        return -EINVAL;
>>> +    }
>>> +
>>> +    /* set DMA engine to the first descriptor block */
>>> +    completed_blocks = desc->completed_desc_num / XDMA_DESC_ADJACENT;
>>> +    block = &desc->desc_blocks[completed_blocks];
>>> +    val = FIELD_GET(XDMA_LO_ADDR_MASK, block->dma_addr);
>>> +    ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
>>> +                 XDMA_SGDMA_DESC_LO, val);
>>> +    if (ret) {
>>> +        dev_err(&xdev->pdev->dev, "write hi addr failed: %d", ret);
>>> +        return ret;
>>> +    }
>>> +
>>> +    val = FIELD_GET(XDMA_HI_ADDR_MASK, block->dma_addr);
>>> +    ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
>>> +                 XDMA_SGDMA_DESC_HI, val);
>>> +    if (ret) {
>>> +        dev_err(&xdev->pdev->dev, "write lo addr failed: %d", ret);
>>> +        return ret;
>>> +    }
>>> +
>>> +    if (completed_blocks + 1 == desc->dblk_num)
>>> +        val = (desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK;
>>> +    else
>>> +        val = XDMA_DESC_ADJACENT - 1;
>>> +    ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
>>> +                 XDMA_SGDMA_DESC_ADJ, val);
>>> +    if (ret) {
>>> +        dev_err(&xdev->pdev->dev, "write adjacent failed: %d", ret);
>>> +        return ret;
>>> +    }
>>> +
>>> +    /* kick off DMA transfer */
>>> +    ret = xdma_write_reg(xdev, xdma_chan->base, XDMA_CHAN_CONTROL,
>>> +                 CHAN_CTRL_START);
>>> +    if (ret) {
>>> +        dev_err(&xdev->pdev->dev, "write control failed: %d", ret);
>>> +        return ret;
>>> +    }
>>> +
>>> +    xdma_chan->busy = true;
>>> +    return 0;
>>> +}
>>> +
>>> +static void xdma_channel_tasklet(struct tasklet_struct *t)
>>> +{
>>> +    struct xdma_chan *xdma_chan = from_tasklet(xdma_chan, t, tasklet);
>>> +
>>> +    spin_lock(&xdma_chan->vchan.lock);
>>> +    xdma_xfer_start(xdma_chan);
>>> +    spin_unlock(&xdma_chan->vchan.lock);
>>> +}
>>> +
>>> +/**
>>> + * xdma_config_channels - Detect and config DMA channels
>>> + * @xdev: DMA device pointer
>>> + * @dir: Channel direction
>>> + */
>>> +static int xdma_config_channels(struct xdma_device *xdev,
>>> +                enum dma_transfer_direction dir)
>>> +{
>>> +    struct xdma_platdata *pdata = dev_get_platdata(&xdev->pdev->dev);
>>> +    u32 base, identifier, target;
>>> +    struct xdma_chan **chans;
>>> +    u32 *chan_num;
>>> +    int i, j, ret;
>>> +
>>> +    if (dir == DMA_MEM_TO_DEV) {
>>> +        base = XDMA_CHAN_H2C_OFFSET;
>>> +        target = XDMA_CHAN_H2C_TARGET;
>>> +        chans = &xdev->h2c_chans;
>>> +        chan_num = &xdev->h2c_chan_num;
>>> +    } else if (dir == DMA_DEV_TO_MEM) {
>>> +        base = XDMA_CHAN_C2H_OFFSET;
>>> +        target = XDMA_CHAN_C2H_TARGET;
>>> +        chans = &xdev->c2h_chans;
>>> +        chan_num = &xdev->c2h_chan_num;
>>> +    } else {
>>> +        dev_err(&xdev->pdev->dev, "invalid direction specified");
>>> +        return -EINVAL;
>>> +    }
>>> +
>>> +    /* detect number of available DMA channels */
>>> +    for (i = 0, *chan_num = 0; i < pdata->max_dma_channels; i++) {
>>> +        ret = xdma_read_reg(xdev, base + i * XDMA_CHAN_STRIDE,
>>> +                    XDMA_CHAN_IDENTIFIER, &identifier);
>>> +        if (ret) {
>>> +            dev_err(&xdev->pdev->dev,
>>> +                "failed to read channel id: %d", ret);
>>> +            return ret;
>>> +        }
>>> +
>>> +        /* check if it is available DMA channel */
>>> +        if (XDMA_CHAN_CHECK_TARGET(identifier, target))
>>> +            (*chan_num)++;
>>> +    }
>>> +
>>> +    if (!*chan_num) {
>>> +        dev_err(&xdev->pdev->dev, "does not probe any channel");
>>> +        return -EINVAL;
>>> +    }
>>> +
>>> +    *chans = devm_kzalloc(&xdev->pdev->dev, sizeof(**chans) * 
>>> (*chan_num),
>>> +                  GFP_KERNEL);
>>> +    if (!*chans)
>>> +        return -ENOMEM;
>>> +
>>> +    for (i = 0, j = 0; i < pdata->max_dma_channels; i++) {
>>> +        ret = xdma_read_reg(xdev, base + i * XDMA_CHAN_STRIDE,
>>> +                    XDMA_CHAN_IDENTIFIER, &identifier);
>>> +        if (ret) {
>>> +            dev_err(&xdev->pdev->dev,
>>> +                "failed to read channel id: %d", ret);
>>> +            return ret;
>>> +        }
>>> +
>>> +        if (!XDMA_CHAN_CHECK_TARGET(identifier, target))
>>> +            continue;
>>> +
>>> +        if (j == *chan_num) {
>>> +            dev_err(&xdev->pdev->dev, "invalid channel number");
>>> +            return -EIO;
>>> +        }
>>> +
>>> +        /* init channel structure and hardware */
>>> +        (*chans)[j].xdev_hdl = xdev;
>>> +        (*chans)[j].base = base + i * XDMA_CHAN_STRIDE;
>>> +        (*chans)[j].dir = dir;
>>> +
>>> +        ret = xdma_channel_init(&(*chans)[j]);
>>> +        if (ret)
>>> +            return ret;
>>> +        (*chans)[j].vchan.desc_free = xdma_free_desc;
>>> +        vchan_init(&(*chans)[j].vchan, &xdev->dma_dev);
>>> +
>>> +        j++;
>>> +    }
>>> +
>>> +    dev_info(&xdev->pdev->dev, "configured %d %s channels", j,
>>> +         (dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H");
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +/**
>>> + * xdma_issue_pending - Issue pending transactions
>>> + * @chan: DMA channel pointer
>>> + */
>>> +static void xdma_issue_pending(struct dma_chan *chan)
>>> +{
>>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>>> +    unsigned long flags;
>>> +
>>> +    spin_lock_irqsave(&xdma_chan->vchan.lock, flags);
>>> +    if (vchan_issue_pending(&xdma_chan->vchan))
>>> +        xdma_xfer_start(xdma_chan);
>>> +    spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);
>>> +}
>>> +
>>> +/**
>>> + * xdma_prep_device_sg - prepare a descriptor for a
>>> + *    DMA transaction
>>> + * @chan: DMA channel pointer
>>> + * @sgl: Transfer scatter gather list
>>> + * @sg_len: Length of scatter gather list
>>> + * @dir: Transfer direction
>>> + * @flags: transfer ack flags
>>> + * @context: APP words of the descriptor
>>> + */
>>> +static struct dma_async_tx_descriptor *
>>> +xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,
>>> +            unsigned int sg_len, enum dma_transfer_direction dir,
>>> +            unsigned long flags, void *context)
>>> +{
>>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>>> +    struct dma_async_tx_descriptor *tx_desc;
>>> +    u32 desc_num = 0, i, len, rest;
>>> +    struct xdma_desc_block *dblk;
>>> +    struct xdma_desc *sw_desc;
>>> +    struct scatterlist *sg;
>>> +    dma_addr_t addr;
>>> +    u64 dev_addr, *src, *dst;
>>> +    struct xdma_hw_desc *desc;
>>> +
>>> +    for_each_sg(sgl, sg, sg_len, i)
>>> +        desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX);
>>> +
>>> +    sw_desc = xdma_alloc_desc(xdma_chan, desc_num);
>>> +    if (!sw_desc)
>>> +        return NULL;
>>> +    sw_desc->dir = dir;
>>> +
>>> +    if (dir == DMA_MEM_TO_DEV) {
>>> +        dev_addr = xdma_chan->cfg.dst_addr;
>>> +        src = &addr;
>>> +        dst = &dev_addr;
>>> +    } else {
>>> +        dev_addr = xdma_chan->cfg.src_addr;
>>> +        src = &dev_addr;
>>> +        dst = &addr;
>>> +    }
>>> +
>>> +    dblk = sw_desc->desc_blocks;
>>> +    desc = dblk->virt_addr;
>>> +    desc_num = 1;
>>> +    for_each_sg(sgl, sg, sg_len, i) {
>>> +        addr = sg_dma_address(sg);
>>> +        rest = sg_dma_len(sg);
>>> +
>>> +        do {
>>> +            len = min_t(u32, rest, XDMA_DESC_BLEN_MAX);
>>> +            /* set hardware descriptor */
>>> +            desc->bytes = cpu_to_le32(len);
>>> +            desc->src_addr = cpu_to_le64(*src);
>>> +            desc->dst_addr = cpu_to_le64(*dst);
>>> +
>>> +            if (!(desc_num & XDMA_DESC_ADJACENT_MASK)) {
>>> +                dblk++;
>>> +                desc = dblk->virt_addr;
>>> +            } else {
>>> +                desc++;
>>> +            }
>>> +
>>> +            desc_num++;
>>> +            dev_addr += len;
>>> +            addr += len;
>>> +            rest -= len;
>>> +        } while (rest);
>>> +    }
>>> +
>>> +    tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, 
>>> flags);
>>> +    if (!tx_desc)
>>> +        goto failed;
>>> +
>>> +    return tx_desc;
>>> +
>>> +failed:
>>> +    xdma_free_desc(&sw_desc->vdesc);
>>> +
>>> +    return NULL;
>>> +}
>>> +
>>> +/**
>>> + * xdma_device_config - Configure the DMA channel
>>> + * @chan: DMA channel
>>> + * @cfg: channel configuration
>>> + */
>>> +static int xdma_device_config(struct dma_chan *chan,
>>> +                  struct dma_slave_config *cfg)
>>> +{
>>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>>> +
>>> +    memcpy(&xdma_chan->cfg, cfg, sizeof(*cfg));
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +/**
>>> + * xdma_free_chan_resources - Free channel resources
>>> + * @chan: DMA channel
>>> + */
>>> +static void xdma_free_chan_resources(struct dma_chan *chan)
>>> +{
>>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>>> +
>>> +    vchan_free_chan_resources(&xdma_chan->vchan);
>>> +    dma_pool_destroy(xdma_chan->desc_pool);
>>> +    xdma_chan->desc_pool = NULL;
>>> +}
>>> +
>>> +/**
>>> + * xdma_alloc_chan_resources - Allocate channel resources
>>> + * @chan: DMA channel
>>> + */
>>> +static int xdma_alloc_chan_resources(struct dma_chan *chan)
>>> +{
>>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>>> +    struct xdma_device *xdev = xdma_chan->xdev_hdl;
>>> +
>>> +    xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan),
>>> +                           xdev->dma_dev.dev,
>>> +                           XDMA_DESC_BLOCK_SIZE,
>>> +                           XDMA_DESC_BLOCK_ALIGN,
>>> +                           0);
>>> +    if (!xdma_chan->desc_pool) {
>>> +        dev_err(&chan->dev->device,
>>> +            "unable to allocate descriptor pool");
>>> +        return -ENOMEM;
>>> +    }
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +/**
>>> + * xdma_channel_isr - XDMA channel interrupt handler
>>> + * @irq: IRQ number
>>> + * @dev_id: Pointer to the DMA channel structure
>>> + */
>>> +static irqreturn_t xdma_channel_isr(int irq, void *dev_id)
>>> +{
>>> +    struct xdma_chan *xdma_chan = dev_id;
>>> +    u32 complete_desc_num = 0;
>>> +    struct virt_dma_desc *vd;
>>> +    struct xdma_desc *desc;
>>> +    int ret;
>>> +
>>> +    spin_lock(&xdma_chan->vchan.lock);
>>> +
>>> +    /* get submitted request */
>>> +    vd = vchan_next_desc(&xdma_chan->vchan);
>>> +    if (!vd)
>>> +        goto out;
>>> +
>>> +    xdma_chan->busy = false;
>>> +    desc = to_xdma_desc(vd);
>>> +
>>> +    ret = xdma_read_reg(xdma_chan->xdev_hdl, xdma_chan->base,
>>> +                XDMA_CHAN_COMPLETED_DESC, &complete_desc_num);
>>> +    if (ret)
>>> +        goto out;
>>> +
>>> +    desc->completed_desc_num += complete_desc_num;
>>> +    /*
>>> +     * if all data blocks are transferred, remove and complete the 
>>> request
>>> +     */
>>> +    if (desc->completed_desc_num == desc->desc_num) {
>>> +        list_del(&vd->node);
>>> +        vchan_cookie_complete(vd);
>>> +        goto out;
>>> +    }
>>> +
>>> +    if (desc->completed_desc_num > desc->desc_num ||
>>> +        complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT)
>>> +        goto out;
>>> +
>>> +    /* transfer the rest of data */
>>> +    tasklet_schedule(&xdma_chan->tasklet);
>>
>> There's an effort upstream to deprecate the usage tasklets [1]. Any 
>> thought on using threaded irq instead especially with a new driver?
>>
>> [1]: https://lwn.net/Articles/830964/
>>
>> DJ
>
> Thanks a lot for pointing this out. In this V2 patch series, we have 
> moved most of code from tasklet handler to prep_* (only few register 
> accesses left). Because the handler is pretty short and quick now, I 
> will remove the tasklet and call xdma_xfer_start() from interrupt 
> handler directly.

That works too.


>
>
> Thanks,
>
> Lizhi
>
>>
>>> +
>>> +out:
>>> +    spin_unlock(&xdma_chan->vchan.lock);
>>> +    return IRQ_HANDLED;
>>> +}
>>> +
>>> +/**
>>> + * xdma_irq_fini - Uninitialize IRQ
>>> + * @xdev: DMA device pointer
>>> + */
>>> +static void xdma_irq_fini(struct xdma_device *xdev)
>>> +{
>>> +    int ret, i;
>>> +
>>> +    /* disable interrupt */
>>> +    ret = xdma_disable_intr(xdev);
>>> +    if (ret) {
>>> +        dev_err(&xdev->pdev->dev, "failed to disable interrupts: %d",
>>> +            ret);
>>> +    }
>>> +
>>> +    /* free irq handler */
>>> +    for (i = 0; i < xdev->h2c_chan_num; i++) {
>>> +        free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
>>> +        tasklet_kill(&xdev->h2c_chans[i].tasklet);
>>> +    }
>>> +    for (i = 0; i < xdev->c2h_chan_num; i++) {
>>> +        free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
>>> +        tasklet_kill(&xdev->c2h_chans[i].tasklet);
>>> +    }
>>> +}
>>> +
>>> +/**
>>> + * xdma_set_vector_reg - configure hardware IRQ registers
>>> + * @xdev: DMA device pointer
>>> + * @vec_tbl_start: Start of IRQ registers
>>> + * @irq_start: Start of IRQ
>>> + * @irq_num: Number of IRQ
>>> + */
>>> +static int xdma_set_vector_reg(struct xdma_device *xdev, u32 
>>> vec_tbl_start,
>>> +                   u32 irq_start, u32 irq_num)
>>> +{
>>> +    u32 shift, i, val = 0;
>>> +    int ret;
>>> +
>>> +    /* Each IRQ register is 32 bit and contains 4 IRQs */
>>> +    while (irq_num > 0) {
>>> +        for (i = 0; i < 4; i++) {
>>> +            shift = XDMA_IRQ_VEC_SHIFT * i;
>>> +            val |= irq_start << shift;
>>> +            irq_start++;
>>> +            irq_num--;
>>> +        }
>>> +
>>> +        /* write IRQ register */
>>> +        ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, vec_tbl_start, val);
>>> +        if (ret) {
>>> +            dev_err(&xdev->pdev->dev, "failed to set vector: %d",
>>> +                ret);
>>> +            return ret;
>>> +        }
>>> +        vec_tbl_start += sizeof(u32);
>>> +        val = 0;
>>> +    }
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +/**
>>> + * xdma_irq_init - initialize IRQs
>>> + * @xdev: DMA device pointer
>>> + */
>>> +static int xdma_irq_init(struct xdma_device *xdev)
>>> +{
>>> +    u32 irq = xdev->irq_start;
>>> +    int i, j, ret;
>>> +
>>> +    /* return failure if there are not enough IRQs */
>>> +    if (xdev->irq_num < xdev->h2c_chan_num + xdev->c2h_chan_num) {
>>> +        dev_err(&xdev->pdev->dev, "not enough irq");
>>> +        return -EINVAL;
>>> +    }
>>> +
>>> +    /* setup H2C interrupt handler */
>>> +    for (i = 0; i < xdev->h2c_chan_num; i++) {
>>> +        ret = request_irq(irq, xdma_channel_isr, 0,
>>> +                  "xdma-h2c-channel", &xdev->h2c_chans[i]);
>>> +        if (ret) {
>>> +            dev_err(&xdev->pdev->dev,
>>> +                "H2C channel%d request irq%d failed: %d",
>>> +                i, irq, ret);
>>> +            for (j = 0; j < i; j++) {
>>> +                free_irq(xdev->h2c_chans[j].irq,
>>> +                     &xdev->h2c_chans[j]);
>>> +            }
>>> +            return ret;
>>> +        }
>>> +        xdev->h2c_chans[i].irq = irq;
>>> +        tasklet_setup(&xdev->h2c_chans[i].tasklet,
>>> +                  xdma_channel_tasklet);
>>> +        irq++;
>>> +    }
>>> +
>>> +    /* setup C2H interrupt handler */
>>> +    for (i = 0; i < xdev->c2h_chan_num; i++) {
>>> +        ret = request_irq(irq, xdma_channel_isr, 0,
>>> +                  "xdma-c2h-channel", &xdev->c2h_chans[i]);
>>> +        if (ret) {
>>> +            dev_err(&xdev->pdev->dev,
>>> +                "H2C channel%d request irq%d failed: %d",
>>> +                i, irq, ret);
>>> +            for (j = 0; j < i; j++) {
>>> +                free_irq(xdev->c2h_chans[j].irq,
>>> +                     &xdev->c2h_chans[j]);
>>> +            }
>>> +            goto failed_init_c2h;
>>> +        }
>>> +        xdev->c2h_chans[i].irq = irq;
>>> +        tasklet_setup(&xdev->c2h_chans[i].tasklet,
>>> +                  xdma_channel_tasklet);
>>> +        irq++;
>>> +    }
>>> +
>>> +    /* config hardware IRQ registers */
>>> +    ret = xdma_set_vector_reg(xdev, XDMA_IRQ_CHAN_VEC_NUM, 0,
>>> +                  xdev->h2c_chan_num + xdev->c2h_chan_num);
>>> +    if (ret) {
>>> +        dev_err(&xdev->pdev->dev, "failed to set channel vectors: %d",
>>> +            ret);
>>> +        goto failed;
>>> +    }
>>> +
>>> +    /* enable interrupt */
>>> +    ret = xdma_enable_intr(xdev);
>>> +    if (ret) {
>>> +        dev_err(&xdev->pdev->dev, "failed to enable interrupts: %d",
>>> +            ret);
>>> +        goto failed;
>>> +    }
>>> +
>>> +    return 0;
>>> +
>>> +failed:
>>> +    for (i = 0; i < xdev->c2h_chan_num; i++)
>>> +        free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
>>> +failed_init_c2h:
>>> +    for (i = 0; i < xdev->h2c_chan_num; i++)
>>> +        free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
>>> +
>>> +    return ret;
>>> +}
>>> +
>>> +static bool xdma_filter_fn(struct dma_chan *chan, void *param)
>>> +{
>>> +    struct xdma_chan *xdma_chan = to_xdma_chan(chan);
>>> +    struct xdma_chan_info *chan_info = param;
>>> +
>>> +    if (chan_info->dir != xdma_chan->dir)
>>> +        return false;
>>> +
>>> +    return true;
>>> +}
>>> +
>>> +/**
>>> + * xdma_remove - Driver remove function
>>> + * @pdev: Pointer to the platform_device structure
>>> + */
>>> +static int xdma_remove(struct platform_device *pdev)
>>> +{
>>> +    struct xdma_device *xdev = platform_get_drvdata(pdev);
>>> +
>>> +    if (xdev->status & XDMA_DEV_STATUS_INIT_MSIX)
>>> +        xdma_irq_fini(xdev);
>>> +
>>> +    if (xdev->status & XDMA_DEV_STATUS_REG_DMA)
>>> +        dma_async_device_unregister(&xdev->dma_dev);
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +/**
>>> + * xdma_probe - Driver probe function
>>> + * @pdev: Pointer to the platform_device structure
>>> + */
>>> +static int xdma_probe(struct platform_device *pdev)
>>> +{
>>> +    struct xdma_platdata *pdata = dev_get_platdata(&pdev->dev);
>>> +    struct xdma_device *xdev;
>>> +    void __iomem *reg_base;
>>> +    struct resource *res;
>>> +    int ret = -ENODEV;
>>> +
>>> +    if (pdata->max_dma_channels > XDMA_MAX_CHANNELS) {
>>> +        dev_err(&pdev->dev, "invalid max dma channels %d",
>>> +            pdata->max_dma_channels);
>>> +        return -EINVAL;
>>> +    }
>>> +
>>> +    xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);
>>> +    if (!xdev)
>>> +        return -ENOMEM;
>>> +
>>> +    platform_set_drvdata(pdev, xdev);
>>> +    xdev->pdev = pdev;
>>> +
>>> +    res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
>>> +    if (!res) {
>>> +        dev_err(&pdev->dev, "failed to get irq resource");
>>> +        goto failed;
>>> +    }
>>> +    xdev->irq_start = res->start;
>>> +    xdev->irq_num = res->end - res->start + 1;
>>> +
>>> +    res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
>>> +    if (!res) {
>>> +        dev_err(&pdev->dev, "failed to get io resource");
>>> +        goto failed;
>>> +    }
>>> +
>>> +    reg_base = devm_ioremap_resource(&pdev->dev, res);
>>> +    if (!reg_base) {
>>> +        dev_err(&pdev->dev, "ioremap failed");
>>> +        goto failed;
>>> +    }
>>> +
>>> +    xdev->regmap = devm_regmap_init_mmio(&pdev->dev, reg_base,
>>> +                         &xdma_regmap_config);
>>> +    if (!xdev->regmap) {
>>> +        dev_err(&pdev->dev, "config regmap failed: %d", ret);
>>> +        goto failed;
>>> +    }
>>> +    INIT_LIST_HEAD(&xdev->dma_dev.channels);
>>> +
>>> +    ret = xdma_config_channels(xdev, DMA_MEM_TO_DEV);
>>> +    if (ret) {
>>> +        dev_err(&pdev->dev, "config H2C channels failed: %d", ret);
>>> +        goto failed;
>>> +    }
>>> +
>>> +    ret = xdma_config_channels(xdev, DMA_DEV_TO_MEM);
>>> +    if (ret) {
>>> +        dev_err(&pdev->dev, "config C2H channels failed: %d", ret);
>>> +        goto failed;
>>> +    }
>>> +
>>> +    dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask);
>>> +    dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask);
>>> +
>>> +    xdev->dma_dev.dev = &pdev->dev;
>>> +    xdev->dma_dev.device_free_chan_resources = 
>>> xdma_free_chan_resources;
>>> +    xdev->dma_dev.device_alloc_chan_resources = 
>>> xdma_alloc_chan_resources;
>>> +    xdev->dma_dev.device_tx_status = dma_cookie_status;
>>> +    xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg;
>>> +    xdev->dma_dev.device_config = xdma_device_config;
>>> +    xdev->dma_dev.device_issue_pending = xdma_issue_pending;
>>> +    xdev->dma_dev.filter.map = pdata->device_map;
>>> +    xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt;
>>> +    xdev->dma_dev.filter.fn = xdma_filter_fn;
>>> +
>>> +    ret = dma_async_device_register(&xdev->dma_dev);
>>> +    if (ret) {
>>> +        dev_err(&pdev->dev, "failed to register Xilinx XDMA: %d", 
>>> ret);
>>> +        goto failed;
>>> +    }
>>> +    xdev->status |= XDMA_DEV_STATUS_REG_DMA;
>>> +
>>> +    ret = xdma_irq_init(xdev);
>>> +    if (ret) {
>>> +        dev_err(&pdev->dev, "failed to init msix: %d", ret);
>>> +        goto failed;
>>> +    }
>>> +    xdev->status |= XDMA_DEV_STATUS_INIT_MSIX;
>>> +
>>> +    return 0;
>>> +
>>> +failed:
>>> +    xdma_remove(pdev);
>>> +
>>> +    return ret;
>>> +}
>>> +
>>> +static const struct platform_device_id xdma_id_table[] = {
>>> +    { "xdma", 0},
>>> +    { },
>>> +};
>>> +
>>> +static struct platform_driver xdma_driver = {
>>> +    .driver        = {
>>> +        .name = "xdma",
>>> +    },
>>> +    .id_table    = xdma_id_table,
>>> +    .probe        = xdma_probe,
>>> +    .remove        = xdma_remove,
>>> +};
>>> +
>>> +module_platform_driver(xdma_driver);
>>> +
>>> +MODULE_DESCRIPTION("AMD XDMA driver");
>>> +MODULE_AUTHOR("XRT Team <[e-mail]>");
>>> +MODULE_LICENSE("GPL");
>>> diff --git a/include/linux/platform_data/amd_xdma.h 
>>> b/include/linux/platform_data/amd_xdma.h
>>> new file mode 100644
>>> index 000000000000..04bcfc74ab36
>>> --- /dev/null
>>> +++ b/include/linux/platform_data/amd_xdma.h
>>> @@ -0,0 +1,34 @@
>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>>> +/*
>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights 
>>> reserved.
>>> + */
>>> +
>>> +#ifndef _PLATDATA_AMD_XDMA_H
>>> +#define _PLATDATA_AMD_XDMA_H
>>> +
>>> +#include <linux/dmaengine.h>
>>> +
>>> +/**
>>> + * struct xdma_chan_info - DMA channel information
>>> + *    This information is used to match channel when request dma 
>>> channel
>>> + * @dir: Channel transfer direction
>>> + */
>>> +struct xdma_chan_info {
>>> +    enum dma_transfer_direction dir;
>>> +};
>>> +
>>> +#define XDMA_FILTER_PARAM(chan_info)    ((void *)(chan_info))
>>> +
>>> +struct dma_slave_map;
>>> +
>>> +/**
>>> + * struct xdma_platdata - platform specific data for XDMA engine
>>> + * @max_dma_channels: Maximum dma channels in each direction
>>> + */
>>> +struct xdma_platdata {
>>> +    u32 max_dma_channels;
>>> +    u32 device_map_cnt;
>>> +    struct dma_slave_map *device_map;
>>> +};
>>> +
>>> +#endif /* _PLATDATA_AMD_XDMA_H */

kernel test robot Sept. 17, 2022, 2:08 p.m. UTC | #4

Hi Lizhi,

I love your patch! Yet something to improve:

[auto build test ERROR on vkoul-dmaengine/next]
[also build test ERROR on linus/master v6.0-rc5 next-20220916]
[cannot apply to xilinx-xlnx/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Lizhi-Hou/xilinx-XDMA-driver/20220916-232740
base:   https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git next
config: arm-randconfig-r003-20220918 (https://download.01.org/0day-ci/archive/20220917/[e-mail]/config)
compiler: clang version 16.0.0 (https://github.com/llvm/llvm-project 791a7ae1ba3efd6bca96338e10ffde557ba83920)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # install arm cross compiling tool for clang build
        # apt-get install binutils-arm-linux-gnueabi
        # https://github.com/intel-lab-lkp/linux/commit/f624c2f1706209a308a17cc8799995dc478c3d00
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Lizhi-Hou/xilinx-XDMA-driver/20220916-232740
        git checkout f624c2f1706209a308a17cc8799995dc478c3d00
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=arm SHELL=/bin/bash drivers/dma/xilinx/

If you fix the issue, kindly add following tag where applicable
Reported-by: kernel test robot <[e-mail]>

All errors (new ones prefixed by >>):

>> drivers/dma/xilinx/xdma.c:540:7: error: incompatible pointer types assigning to 'u64 *' (aka 'unsigned long long *') from 'dma_addr_t *' (aka 'unsigned int *') [-Werror,-Wincompatible-pointer-types]
                   src = &addr;
                       ^ ~~~~~
   drivers/dma/xilinx/xdma.c:545:7: error: incompatible pointer types assigning to 'u64 *' (aka 'unsigned long long *') from 'dma_addr_t *' (aka 'unsigned int *') [-Werror,-Wincompatible-pointer-types]
                   dst = &addr;
                       ^ ~~~~~
   2 errors generated.


vim +540 drivers/dma/xilinx/xdma.c

   504	
   505	/**
   506	 * xdma_prep_device_sg - prepare a descriptor for a
   507	 *	DMA transaction
   508	 * @chan: DMA channel pointer
   509	 * @sgl: Transfer scatter gather list
   510	 * @sg_len: Length of scatter gather list
   511	 * @dir: Transfer direction
   512	 * @flags: transfer ack flags
   513	 * @context: APP words of the descriptor
   514	 */
   515	static struct dma_async_tx_descriptor *
   516	xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,
   517			    unsigned int sg_len, enum dma_transfer_direction dir,
   518			    unsigned long flags, void *context)
   519	{
   520		struct xdma_chan *xdma_chan = to_xdma_chan(chan);
   521		struct dma_async_tx_descriptor *tx_desc;
   522		u32 desc_num = 0, i, len, rest;
   523		struct xdma_desc_block *dblk;
   524		struct xdma_desc *sw_desc;
   525		struct scatterlist *sg;
   526		dma_addr_t addr;
   527		u64 dev_addr, *src, *dst;
   528		struct xdma_hw_desc *desc;
   529	
   530		for_each_sg(sgl, sg, sg_len, i)
   531			desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX);
   532	
   533		sw_desc = xdma_alloc_desc(xdma_chan, desc_num);
   534		if (!sw_desc)
   535			return NULL;
   536		sw_desc->dir = dir;
   537	
   538		if (dir == DMA_MEM_TO_DEV) {
   539			dev_addr = xdma_chan->cfg.dst_addr;
 > 540			src = &addr;
   541			dst = &dev_addr;
   542		} else {
   543			dev_addr = xdma_chan->cfg.src_addr;
   544			src = &dev_addr;
   545			dst = &addr;
   546		}
   547	
   548		dblk = sw_desc->desc_blocks;
   549		desc = dblk->virt_addr;
   550		desc_num = 1;
   551		for_each_sg(sgl, sg, sg_len, i) {
   552			addr = sg_dma_address(sg);
   553			rest = sg_dma_len(sg);
   554	
   555			do {
   556				len = min_t(u32, rest, XDMA_DESC_BLEN_MAX);
   557				/* set hardware descriptor */
   558				desc->bytes = cpu_to_le32(len);
   559				desc->src_addr = cpu_to_le64(*src);
   560				desc->dst_addr = cpu_to_le64(*dst);
   561	
   562				if (!(desc_num & XDMA_DESC_ADJACENT_MASK)) {
   563					dblk++;
   564					desc = dblk->virt_addr;
   565				} else {
   566					desc++;
   567				}
   568	
   569				desc_num++;
   570				dev_addr += len;
   571				addr += len;
   572				rest -= len;
   573			} while (rest);
   574		}
   575	
   576		tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags);
   577		if (!tx_desc)
   578			goto failed;
   579	
   580		return tx_desc;
   581	
   582	failed:
   583		xdma_free_desc(&sw_desc->vdesc);
   584	
   585		return NULL;
   586	}
   587

Patch

diff --git a/MAINTAINERS b/MAINTAINERS
index e8c52d0192a6..c1be0b2e378a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -21683,6 +21683,16 @@  F:	Documentation/devicetree/bindings/media/xilinx/
 F:	drivers/media/platform/xilinx/
 F:	include/uapi/linux/xilinx-v4l2-controls.h
 
+XILINX XDMA DRIVER
+M:	Lizhi Hou <[e-mail]>
+M:	Brian Xu <[e-mail]>
+M:	Raj Kumar Rampelli <[e-mail]>
+L:	[e-mail]
+S:	Supported
+F:	drivers/dma/xilinx/xdma-regs.h
+F:	drivers/dma/xilinx/xdma.c
+F:	include/linux/platform_data/amd_xdma.h
+
 XILINX ZYNQMP DPDMA DRIVER
 M:	Hyun Kwon <[e-mail]>
 M:	Laurent Pinchart <[e-mail]>
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index d5de3f77d3aa..4d90f2f51655 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -733,6 +733,19 @@  config XILINX_ZYNQMP_DPDMA
 	  driver provides the dmaengine required by the DisplayPort subsystem
 	  display driver.
 
+config XILINX_XDMA
+	tristate "Xilinx DMA/Bridge Subsystem DMA Engine"
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	select REGMAP_MMIO
+	help
+	  Enable support for Xilinx DMA/Bridge Subsystem DMA engine. The DMA
+	  provides high performance block data movement between Host memory
+	  and the DMA subsystem. These direct memory transfers can be both in
+	  the Host to Card (H2C) and Card to Host (C2H) transfers.
+	  The core also provides up to 16 user interrupt wires that generate
+	  interrupts to the host.
+
 # driver files
 source "drivers/dma/bestcomm/Kconfig"
 
diff --git a/drivers/dma/xilinx/Makefile b/drivers/dma/xilinx/Makefile
index 767bb45f641f..c7a538a56643 100644
--- a/drivers/dma/xilinx/Makefile
+++ b/drivers/dma/xilinx/Makefile
@@ -2,3 +2,4 @@ 
 obj-$(CONFIG_XILINX_DMA) += xilinx_dma.o
 obj-$(CONFIG_XILINX_ZYNQMP_DMA) += zynqmp_dma.o
 obj-$(CONFIG_XILINX_ZYNQMP_DPDMA) += xilinx_dpdma.o
+obj-$(CONFIG_XILINX_XDMA) += xdma.o
diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h
new file mode 100644
index 000000000000..c42df0ed4aca
--- /dev/null
+++ b/drivers/dma/xilinx/xdma-regs.h
@@ -0,0 +1,171 @@ 
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
+ * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
+ */
+
+#ifndef __DMA_XDMA_REGS_H
+#define __DMA_XDMA_REGS_H
+
+/* The length of register space exposed to host */
+#define XDMA_REG_SPACE_LEN	65536
+
+/*
+ * maximum number of DMA channels for each direction:
+ * Host to Card (H2C) or Card to Host (C2H)
+ */
+#define XDMA_MAX_CHANNELS	4
+
+/* macros to get higher and lower 32-bit address */
+#define XDMA_HI_ADDR_MASK	GENMASK_ULL(63, 32)
+#define XDMA_LO_ADDR_MASK	GENMASK_ULL(31, 0)
+
+/*
+ * macros to define the number of descriptor blocks can be used in one
+ * DMA transfer request.
+ * the DMA engine uses a linked list of descriptor blocks that specify the
+ * source, destination, and length of the DMA transfers.
+ */
+#define XDMA_DESC_BLOCK_NUM		BIT(7)
+#define XDMA_DESC_BLOCK_MASK		(XDMA_DESC_BLOCK_NUM - 1)
+
+/* descriptor definitions */
+#define XDMA_DESC_ADJACENT		BIT(5)
+#define XDMA_DESC_ADJACENT_MASK		(XDMA_DESC_ADJACENT - 1)
+#define XDMA_DESC_MAGIC			0xad4bUL
+#define XDMA_DESC_MAGIC_SHIFT		16
+#define XDMA_DESC_ADJACENT_SHIFT	8
+#define XDMA_DESC_STOPPED		BIT(0)
+#define XDMA_DESC_COMPLETED		BIT(1)
+#define XDMA_DESC_BLEN_BITS		28
+#define XDMA_DESC_BLEN_MAX		(BIT(XDMA_DESC_BLEN_BITS) - PAGE_SIZE)
+
+/* macros to construct the descriptor control word */
+#define XDMA_DESC_CONTROL(adjacent, flag)				\
+	((XDMA_DESC_MAGIC << XDMA_DESC_MAGIC_SHIFT) |			\
+	 (((adjacent) - 1) << XDMA_DESC_ADJACENT_SHIFT) | (flag))
+#define XDMA_DESC_CONTROL_LAST						\
+	XDMA_DESC_CONTROL(1, XDMA_DESC_STOPPED | XDMA_DESC_COMPLETED)
+
+/*
+ * Descriptor for a single contiguous memory block transfer.
+ *
+ * Multiple descriptors are linked by means of the next pointer. An additional
+ * extra adjacent number gives the amount of extra contiguous descriptors.
+ *
+ * The descriptors are in root complex memory, and the bytes in the 32-bit
+ * words must be in little-endian byte ordering.
+ */
+struct xdma_hw_desc {
+	__le32		control;
+	__le32		bytes;
+	__le64		src_addr;
+	__le64		dst_addr;
+	__le64		next_desc;
+};
+
+#define XDMA_DESC_SIZE		sizeof(struct xdma_hw_desc)
+#define XDMA_DESC_BLOCK_SIZE	(XDMA_DESC_SIZE * XDMA_DESC_ADJACENT)
+#define XDMA_DESC_BLOCK_ALIGN	4096
+
+/*
+ * Channel registers
+ */
+#define XDMA_CHAN_IDENTIFIER		0x0
+#define XDMA_CHAN_CONTROL		0x4
+#define XDMA_CHAN_CONTROL_W1S		0x8
+#define XDMA_CHAN_CONTROL_W1C		0xc
+#define XDMA_CHAN_STATUS		0x40
+#define XDMA_CHAN_COMPLETED_DESC	0x48
+#define XDMA_CHAN_ALIGNMENTS		0x4c
+#define XDMA_CHAN_INTR_ENABLE		0x90
+#define XDMA_CHAN_INTR_ENABLE_W1S	0x94
+#define XDMA_CHAN_INTR_ENABLE_W1C	0x9c
+
+#define XDMA_CHAN_STRIDE	0x100
+#define XDMA_CHAN_H2C_OFFSET	0x0
+#define XDMA_CHAN_C2H_OFFSET	0x1000
+#define XDMA_CHAN_H2C_TARGET	0x0
+#define XDMA_CHAN_C2H_TARGET	0x1
+
+/* macro to check if channel is available */
+#define XDMA_CHAN_MAGIC		0x1fc0
+#define XDMA_CHAN_CHECK_TARGET(id, target)		\
+	(((u32)(id) >> 16) == XDMA_CHAN_MAGIC + (target))
+
+/* bits of the channel control register */
+#define CHAN_CTRL_RUN_STOP			BIT(0)
+#define CHAN_CTRL_IE_DESC_STOPPED		BIT(1)
+#define CHAN_CTRL_IE_DESC_COMPLETED		BIT(2)
+#define CHAN_CTRL_IE_DESC_ALIGN_MISMATCH	BIT(3)
+#define CHAN_CTRL_IE_MAGIC_STOPPED		BIT(4)
+#define CHAN_CTRL_IE_IDLE_STOPPED		BIT(6)
+#define CHAN_CTRL_IE_READ_ERROR			(0x1FUL << 9)
+#define CHAN_CTRL_IE_DESC_ERROR			(0x1FUL << 19)
+#define CHAN_CTRL_NON_INCR_ADDR			BIT(25)
+#define CHAN_CTRL_POLL_MODE_WB			BIT(26)
+
+#define CHAN_CTRL_START	(CHAN_CTRL_RUN_STOP |				\
+			 CHAN_CTRL_IE_DESC_STOPPED |			\
+			 CHAN_CTRL_IE_DESC_COMPLETED |			\
+			 CHAN_CTRL_IE_DESC_ALIGN_MISMATCH |		\
+			 CHAN_CTRL_IE_MAGIC_STOPPED |			\
+			 CHAN_CTRL_IE_READ_ERROR |			\
+			 CHAN_CTRL_IE_DESC_ERROR)
+
+/* bits of the channel interrupt enable mask */
+#define CHAN_IM_DESC_ERROR			BIT(19)
+#define CHAN_IM_READ_ERROR			BIT(9)
+#define CHAN_IM_IDLE_STOPPED			BIT(6)
+#define CHAN_IM_MAGIC_STOPPED			BIT(4)
+#define CHAN_IM_DESC_COMPLETED			BIT(2)
+#define CHAN_IM_DESC_STOPPED			BIT(1)
+
+#define CHAN_IM_ALL	(CHAN_IM_DESC_ERROR | CHAN_IM_READ_ERROR |	\
+			 CHAN_IM_IDLE_STOPPED | CHAN_IM_MAGIC_STOPPED | \
+			 CHAN_IM_DESC_COMPLETED | CHAN_IM_DESC_STOPPED)
+
+/*
+ * Channel SGDMA registers
+ */
+#define XDMA_SGDMA_IDENTIFIER	0x0
+#define XDMA_SGDMA_DESC_LO	0x80
+#define XDMA_SGDMA_DESC_HI	0x84
+#define XDMA_SGDMA_DESC_ADJ	0x88
+#define XDMA_SGDMA_DESC_CREDIT	0x8c
+
+#define XDMA_SGDMA_BASE(chan_base)	((chan_base) + 0x4000)
+
+/* bits of the SG DMA control register */
+#define XDMA_CTRL_RUN_STOP			BIT(0)
+#define XDMA_CTRL_IE_DESC_STOPPED		BIT(1)
+#define XDMA_CTRL_IE_DESC_COMPLETED		BIT(2)
+#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH	BIT(3)
+#define XDMA_CTRL_IE_MAGIC_STOPPED		BIT(4)
+#define XDMA_CTRL_IE_IDLE_STOPPED		BIT(6)
+#define XDMA_CTRL_IE_READ_ERROR			(0x1FUL << 9)
+#define XDMA_CTRL_IE_DESC_ERROR			(0x1FUL << 19)
+#define XDMA_CTRL_NON_INCR_ADDR			BIT(25)
+#define XDMA_CTRL_POLL_MODE_WB			BIT(26)
+
+/*
+ * interrupt registers
+ */
+#define XDMA_IRQ_IDENTIFIER		0x0
+#define XDMA_IRQ_USER_INT_EN		0x04
+#define XDMA_IRQ_USER_INT_EN_W1S	0x08
+#define XDMA_IRQ_USER_INT_EN_W1C	0x0c
+#define XDMA_IRQ_CHAN_INT_EN		0x10
+#define XDMA_IRQ_CHAN_INT_EN_W1S	0x14
+#define XDMA_IRQ_CHAN_INT_EN_W1C	0x18
+#define XDMA_IRQ_USER_INT_REQ		0x40
+#define XDMA_IRQ_CHAN_INT_REQ		0x44
+#define XDMA_IRQ_USER_INT_PEND		0x48
+#define XDMA_IRQ_CHAN_INT_PEND		0x4c
+#define XDMA_IRQ_USER_VEC_NUM		0x80
+#define XDMA_IRQ_CHAN_VEC_NUM		0xa0
+
+#define XDMA_IRQ_BASE			0x2000
+#define XDMA_IRQ_VEC_SHIFT		8
+
+#endif /* __DMA_XDMA_REGS_H */
diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c
new file mode 100644
index 000000000000..59797ae5251c
--- /dev/null
+++ b/drivers/dma/xilinx/xdma.c
@@ -0,0 +1,982 @@ 
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * DMA driver for Xilinx DMA/Bridge Subsystem
+ *
+ * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
+ * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
+ */
+
+/*
+ * The DMA/Bridge Subsystem for PCI Express allows for the movement of data
+ * between Host memory and the DMA subsystem. It does this by operating on
+ * 'descriptors' that contain information about the source, destination and
+ * amount of data to transfer. These direct memory transfers can be both in
+ * the Host to Card (H2C) and Card to Host (C2H) transfers. The DMA can be
+ * configured to have a single AXI4 Master interface shared by all channels
+ * or one AXI4-Stream interface for each channel enabled. Memory transfers are
+ * specified on a per-channel basis in descriptor linked lists, which the DMA
+ * fetches from host memory and processes. Events such as descriptor completion
+ * and errors are signaled using interrupts. The core also provides up to 16
+ * user interrupt wires that generate interrupts to the host.
+ */
+
+#include <linux/mod_devicetable.h>
+#include <linux/bitfield.h>
+#include <linux/dmapool.h>
+#include <linux/regmap.h>
+#include <linux/dmaengine.h>
+#include <linux/platform_device.h>
+#include <linux/platform_data/amd_xdma.h>
+#include <linux/dma-mapping.h>
+#include "../virt-dma.h"
+#include "xdma-regs.h"
+
+/* mmio regmap config for all XDMA registers */
+static const struct regmap_config xdma_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = XDMA_REG_SPACE_LEN,
+};
+
+/**
+ * struct xdma_desc_block - Descriptor block
+ * @virt_addr: Virtual address of block start
+ * @dma_addr: DMA address of block start
+ */
+struct xdma_desc_block {
+	void		*virt_addr;
+	dma_addr_t	dma_addr;
+};
+
+/**
+ * struct xdma_chan - Driver specific DMA channel structure
+ * @vchan: Virtual channel
+ * @xdev_hdl: Pointer to DMA device structure
+ * @base: Offset of channel registers
+ * @desc_pool: Descriptor pool
+ * @busy: Busy flag of the channel
+ * @dir: Transferring direction of the channel
+ * @cfg: Transferring config of the channel
+ * @irq: IRQ assigned to the channel
+ * @tasklet: Channel tasklet
+ */
+struct xdma_chan {
+	struct virt_dma_chan		vchan;
+	void				*xdev_hdl;
+	u32				base;
+	struct dma_pool			*desc_pool;
+	bool				busy;
+	enum dma_transfer_direction	dir;
+	struct dma_slave_config		cfg;
+	u32				irq;
+	struct tasklet_struct		tasklet;
+};
+
+/**
+ * struct xdma_desc - DMA desc structure
+ * @vdesc: Virtual DMA descriptor
+ * @dir: Transferring direction of the request
+ * @dev_addr: Physical address on DMA device side
+ * @desc_blocks: Hardware descriptor blocks
+ * @dblk_num: Number of hardware descriptor blocks
+ * @desc_num: Number of hardware descriptors
+ * @completed_desc_num: Completed hardware descriptors
+ */
+struct xdma_desc {
+	struct virt_dma_desc		vdesc;
+	struct xdma_chan		*chan;
+	enum dma_transfer_direction	dir;
+	u64				dev_addr;
+	struct xdma_desc_block		*desc_blocks;
+	u32				dblk_num;
+	u32				desc_num;
+	u32				completed_desc_num;
+};
+
+#define XDMA_DEV_STATUS_REG_DMA		BIT(0)
+#define XDMA_DEV_STATUS_INIT_MSIX	BIT(1)
+
+/**
+ * struct xdma_device - DMA device structure
+ * @pdev: Platform device pointer
+ * @dma_dev: DMA device structure
+ * @regmap: MMIO regmap for DMA registers
+ * @h2c_chans: Host to Card channels
+ * @c2h_chans: Card to Host channels
+ * @h2c_chan_num: Number of H2C channels
+ * @c2h_chan_num: Number of C2H channels
+ * @irq_start: Start IRQ assigned to device
+ * @irq_num: Number of IRQ assigned to device
+ * @status: Initialization status
+ */
+struct xdma_device {
+	struct platform_device	*pdev;
+	struct dma_device	dma_dev;
+	struct regmap		*regmap;
+	struct xdma_chan	*h2c_chans;
+	struct xdma_chan	*c2h_chans;
+	u32			h2c_chan_num;
+	u32			c2h_chan_num;
+	u32			irq_start;
+	u32			irq_num;
+	u32			status;
+};
+
+#define xdma_err(xdev, fmt, args...)					\
+	dev_err(&(xdev)->pdev->dev, fmt, ##args)
+
+/* Read and Write DMA registers */
+static inline int xdma_read_reg(struct xdma_device *xdev, u32 base, u32 reg,
+				u32 *val)
+{
+	return regmap_read(xdev->regmap, base + reg, val);
+}
+
+static inline int xdma_write_reg(struct xdma_device *xdev, u32 base, u32 reg,
+				 u32 val)
+{
+	return regmap_write(xdev->regmap, base + reg, val);
+}
+
+/* Get the last desc in a desc block */
+static inline void *xdma_blk_last_desc(struct xdma_desc_block *block)
+{
+	return block->virt_addr + (XDMA_DESC_ADJACENT - 1) * XDMA_DESC_SIZE;
+}
+
+/**
+ * xdma_link_desc_blocks - Link descriptor blocks for DMA transfer
+ * @sw_desc: Tx descriptor pointer
+ */
+static void xdma_link_desc_blocks(struct xdma_desc *sw_desc)
+{
+	struct xdma_desc_block *block;
+	u32 last_blk_desc_num, desc_control;
+	struct xdma_hw_desc *desc;
+	int i;
+
+	desc_control = XDMA_DESC_CONTROL(XDMA_DESC_ADJACENT, 0);
+	for (i = 1; i < sw_desc->dblk_num; i++) {
+		block = &sw_desc->desc_blocks[i - 1];
+		desc = xdma_blk_last_desc(block);
+
+		if (!(i & XDMA_DESC_BLOCK_MASK)) {
+			desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
+			continue;
+		}
+		desc->control = cpu_to_le32(desc_control);
+		desc->next_desc = cpu_to_le64(block[1].dma_addr);
+	}
+
+	/* update the last block */
+	last_blk_desc_num = sw_desc->desc_num & XDMA_DESC_ADJACENT_MASK;
+	if ((sw_desc->dblk_num & XDMA_DESC_BLOCK_MASK) > 1 &&
+	    last_blk_desc_num) {
+		block = &sw_desc->desc_blocks[sw_desc->dblk_num - 2];
+		desc = xdma_blk_last_desc(block);
+		desc_control = XDMA_DESC_CONTROL(last_blk_desc_num, 0);
+		desc->control = cpu_to_le32(desc_control);
+	}
+
+	block = &sw_desc->desc_blocks[sw_desc->dblk_num - 1];
+	desc = block->virt_addr + (last_blk_desc_num - 1) * XDMA_DESC_SIZE;
+	desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
+}
+
+static inline struct xdma_chan *to_xdma_chan(struct dma_chan *chan)
+{
+	return container_of(chan, struct xdma_chan, vchan.chan);
+}
+
+static inline struct xdma_desc *to_xdma_desc(struct virt_dma_desc *vdesc)
+{
+	return container_of(vdesc, struct xdma_desc, vdesc);
+}
+
+static int xdma_enable_intr(struct xdma_device *xdev)
+{
+	int ret;
+
+	ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, XDMA_IRQ_CHAN_INT_EN_W1S, ~0);
+	if (ret)
+		xdma_err(xdev, "enable channel intr failed: %d", ret);
+
+	return ret;
+}
+
+static int xdma_disable_intr(struct xdma_device *xdev)
+{
+	int ret;
+
+	ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, XDMA_IRQ_CHAN_INT_EN_W1C, ~0);
+	if (ret)
+		xdma_err(xdev, "disable channel intr failed: %d", ret);
+
+	return ret;
+}
+
+/**
+ * xdma_channel_init - Initialize DMA channel registers
+ * @chan: DMA channel pointer
+ */
+static int xdma_channel_init(struct xdma_chan *chan)
+{
+	struct xdma_device *xdev = chan->xdev_hdl;
+	int ret;
+
+	ret = xdma_write_reg(xdev, chan->base, XDMA_CHAN_CONTROL_W1C,
+			     CHAN_CTRL_NON_INCR_ADDR);
+	if (ret) {
+		xdma_err(xdev, "clear non incr addr failed: %d", ret);
+		return ret;
+	}
+
+	ret = xdma_write_reg(xdev, chan->base, XDMA_CHAN_INTR_ENABLE,
+			     CHAN_IM_ALL);
+	if (ret) {
+		xdma_err(xdev, "failed to set interrupt mask: %d", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * xdma_free_desc - Free descriptor
+ * @vdesc: Virtual DMA descriptor
+ */
+static void xdma_free_desc(struct virt_dma_desc *vdesc)
+{
+	struct xdma_desc *sw_desc;
+	int i;
+
+	sw_desc = to_xdma_desc(vdesc);
+	for (i = 0; i < sw_desc->dblk_num; i++) {
+		if (!sw_desc->desc_blocks[i].virt_addr)
+			break;
+		dma_pool_free(sw_desc->chan->desc_pool,
+			      sw_desc->desc_blocks[i].virt_addr,
+			      sw_desc->desc_blocks[i].dma_addr);
+	}
+	kfree(sw_desc->desc_blocks);
+	kfree(sw_desc);
+}
+
+/**
+ * xdma_alloc_desc - Allocate descriptor
+ * @chan: DMA channel pointer
+ * @desc_num: Number of hardware descriptors
+ */
+static struct xdma_desc *
+xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num)
+{
+	struct xdma_desc *sw_desc;
+	struct xdma_hw_desc *desc;
+	dma_addr_t dma_addr;
+	u32 dblk_num;
+	void *addr;
+	int i, j;
+
+	sw_desc = kzalloc(sizeof(*sw_desc), GFP_NOWAIT);
+	if (!sw_desc)
+		return NULL;
+
+	sw_desc->chan = chan;
+	sw_desc->desc_num = desc_num;
+	dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT);
+	sw_desc->desc_blocks = kcalloc(dblk_num, sizeof(*sw_desc->desc_blocks),
+				       GFP_NOWAIT);
+	if (!sw_desc->desc_blocks)
+		goto failed;
+
+	sw_desc->dblk_num = dblk_num;
+	for (i = 0; i < sw_desc->dblk_num; i++) {
+		addr = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &dma_addr);
+		if (!addr)
+			goto failed;
+
+		sw_desc->desc_blocks[i].virt_addr = addr;
+		sw_desc->desc_blocks[i].dma_addr = dma_addr;
+		for (j = 0, desc = addr; j < XDMA_DESC_ADJACENT; j++)
+			desc[j].control = cpu_to_le32(XDMA_DESC_CONTROL(1, 0));
+	}
+
+	xdma_link_desc_blocks(sw_desc);
+
+	return sw_desc;
+
+failed:
+	xdma_free_desc(&sw_desc->vdesc);
+	return NULL;
+}
+
+/**
+ * xdma_xfer_start - Start DMA transfer
+ * @xdma_chan: DMA channel pointer
+ */
+static int xdma_xfer_start(struct xdma_chan *xdma_chan)
+{
+	struct virt_dma_desc *vd = vchan_next_desc(&xdma_chan->vchan);
+	struct xdma_device *xdev = xdma_chan->xdev_hdl;
+	struct xdma_desc_block *block;
+	u32 val, completed_blocks;
+	struct xdma_desc *desc;
+	int ret;
+
+	/*
+	 * check if there is not any submitted descriptor or channel is busy.
+	 * vchan lock should be held where this function is called.
+	 */
+	if (!vd || xdma_chan->busy)
+		return -EINVAL;
+
+	/* clear run stop bit to get ready for transfer */
+	ret = xdma_write_reg(xdev, xdma_chan->base, XDMA_CHAN_CONTROL_W1C,
+			     CHAN_CTRL_RUN_STOP);
+	if (ret) {
+		dev_err(&xdev->pdev->dev, "write control failed: %d", ret);
+		return ret;
+	}
+
+	desc = to_xdma_desc(vd);
+	if (desc->dir != xdma_chan->dir) {
+		dev_err(&xdev->pdev->dev, "incorrect request direction");
+		return -EINVAL;
+	}
+
+	/* set DMA engine to the first descriptor block */
+	completed_blocks = desc->completed_desc_num / XDMA_DESC_ADJACENT;
+	block = &desc->desc_blocks[completed_blocks];
+	val = FIELD_GET(XDMA_LO_ADDR_MASK, block->dma_addr);
+	ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
+			     XDMA_SGDMA_DESC_LO, val);
+	if (ret) {
+		dev_err(&xdev->pdev->dev, "write hi addr failed: %d", ret);
+		return ret;
+	}
+
+	val = FIELD_GET(XDMA_HI_ADDR_MASK, block->dma_addr);
+	ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
+			     XDMA_SGDMA_DESC_HI, val);
+	if (ret) {
+		dev_err(&xdev->pdev->dev, "write lo addr failed: %d", ret);
+		return ret;
+	}
+
+	if (completed_blocks + 1 == desc->dblk_num)
+		val = (desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK;
+	else
+		val = XDMA_DESC_ADJACENT - 1;
+	ret = xdma_write_reg(xdev, XDMA_SGDMA_BASE(xdma_chan->base),
+			     XDMA_SGDMA_DESC_ADJ, val);
+	if (ret) {
+		dev_err(&xdev->pdev->dev, "write adjacent failed: %d", ret);
+		return ret;
+	}
+
+	/* kick off DMA transfer */
+	ret = xdma_write_reg(xdev, xdma_chan->base, XDMA_CHAN_CONTROL,
+			     CHAN_CTRL_START);
+	if (ret) {
+		dev_err(&xdev->pdev->dev, "write control failed: %d", ret);
+		return ret;
+	}
+
+	xdma_chan->busy = true;
+	return 0;
+}
+
+static void xdma_channel_tasklet(struct tasklet_struct *t)
+{
+	struct xdma_chan *xdma_chan = from_tasklet(xdma_chan, t, tasklet);
+
+	spin_lock(&xdma_chan->vchan.lock);
+	xdma_xfer_start(xdma_chan);
+	spin_unlock(&xdma_chan->vchan.lock);
+}
+
+/**
+ * xdma_config_channels - Detect and config DMA channels
+ * @xdev: DMA device pointer
+ * @dir: Channel direction
+ */
+static int xdma_config_channels(struct xdma_device *xdev,
+				enum dma_transfer_direction dir)
+{
+	struct xdma_platdata *pdata = dev_get_platdata(&xdev->pdev->dev);
+	u32 base, identifier, target;
+	struct xdma_chan **chans;
+	u32 *chan_num;
+	int i, j, ret;
+
+	if (dir == DMA_MEM_TO_DEV) {
+		base = XDMA_CHAN_H2C_OFFSET;
+		target = XDMA_CHAN_H2C_TARGET;
+		chans = &xdev->h2c_chans;
+		chan_num = &xdev->h2c_chan_num;
+	} else if (dir == DMA_DEV_TO_MEM) {
+		base = XDMA_CHAN_C2H_OFFSET;
+		target = XDMA_CHAN_C2H_TARGET;
+		chans = &xdev->c2h_chans;
+		chan_num = &xdev->c2h_chan_num;
+	} else {
+		dev_err(&xdev->pdev->dev, "invalid direction specified");
+		return -EINVAL;
+	}
+
+	/* detect number of available DMA channels */
+	for (i = 0, *chan_num = 0; i < pdata->max_dma_channels; i++) {
+		ret = xdma_read_reg(xdev, base + i * XDMA_CHAN_STRIDE,
+				    XDMA_CHAN_IDENTIFIER, &identifier);
+		if (ret) {
+			dev_err(&xdev->pdev->dev,
+				"failed to read channel id: %d", ret);
+			return ret;
+		}
+
+		/* check if it is available DMA channel */
+		if (XDMA_CHAN_CHECK_TARGET(identifier, target))
+			(*chan_num)++;
+	}
+
+	if (!*chan_num) {
+		dev_err(&xdev->pdev->dev, "does not probe any channel");
+		return -EINVAL;
+	}
+
+	*chans = devm_kzalloc(&xdev->pdev->dev, sizeof(**chans) * (*chan_num),
+			      GFP_KERNEL);
+	if (!*chans)
+		return -ENOMEM;
+
+	for (i = 0, j = 0; i < pdata->max_dma_channels; i++) {
+		ret = xdma_read_reg(xdev, base + i * XDMA_CHAN_STRIDE,
+				    XDMA_CHAN_IDENTIFIER, &identifier);
+		if (ret) {
+			dev_err(&xdev->pdev->dev,
+				"failed to read channel id: %d", ret);
+			return ret;
+		}
+
+		if (!XDMA_CHAN_CHECK_TARGET(identifier, target))
+			continue;
+
+		if (j == *chan_num) {
+			dev_err(&xdev->pdev->dev, "invalid channel number");
+			return -EIO;
+		}
+
+		/* init channel structure and hardware */
+		(*chans)[j].xdev_hdl = xdev;
+		(*chans)[j].base = base + i * XDMA_CHAN_STRIDE;
+		(*chans)[j].dir = dir;
+
+		ret = xdma_channel_init(&(*chans)[j]);
+		if (ret)
+			return ret;
+		(*chans)[j].vchan.desc_free = xdma_free_desc;
+		vchan_init(&(*chans)[j].vchan, &xdev->dma_dev);
+
+		j++;
+	}
+
+	dev_info(&xdev->pdev->dev, "configured %d %s channels", j,
+		 (dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H");
+
+	return 0;
+}
+
+/**
+ * xdma_issue_pending - Issue pending transactions
+ * @chan: DMA channel pointer
+ */
+static void xdma_issue_pending(struct dma_chan *chan)
+{
+	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&xdma_chan->vchan.lock, flags);
+	if (vchan_issue_pending(&xdma_chan->vchan))
+		xdma_xfer_start(xdma_chan);
+	spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);
+}
+
+/**
+ * xdma_prep_device_sg - prepare a descriptor for a
+ *	DMA transaction
+ * @chan: DMA channel pointer
+ * @sgl: Transfer scatter gather list
+ * @sg_len: Length of scatter gather list
+ * @dir: Transfer direction
+ * @flags: transfer ack flags
+ * @context: APP words of the descriptor
+ */
+static struct dma_async_tx_descriptor *
+xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,
+		    unsigned int sg_len, enum dma_transfer_direction dir,
+		    unsigned long flags, void *context)
+{
+	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
+	struct dma_async_tx_descriptor *tx_desc;
+	u32 desc_num = 0, i, len, rest;
+	struct xdma_desc_block *dblk;
+	struct xdma_desc *sw_desc;
+	struct scatterlist *sg;
+	dma_addr_t addr;
+	u64 dev_addr, *src, *dst;
+	struct xdma_hw_desc *desc;
+
+	for_each_sg(sgl, sg, sg_len, i)
+		desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX);
+
+	sw_desc = xdma_alloc_desc(xdma_chan, desc_num);
+	if (!sw_desc)
+		return NULL;
+	sw_desc->dir = dir;
+
+	if (dir == DMA_MEM_TO_DEV) {
+		dev_addr = xdma_chan->cfg.dst_addr;
+		src = &addr;
+		dst = &dev_addr;
+	} else {
+		dev_addr = xdma_chan->cfg.src_addr;
+		src = &dev_addr;
+		dst = &addr;
+	}
+
+	dblk = sw_desc->desc_blocks;
+	desc = dblk->virt_addr;
+	desc_num = 1;
+	for_each_sg(sgl, sg, sg_len, i) {
+		addr = sg_dma_address(sg);
+		rest = sg_dma_len(sg);
+
+		do {
+			len = min_t(u32, rest, XDMA_DESC_BLEN_MAX);
+			/* set hardware descriptor */
+			desc->bytes = cpu_to_le32(len);
+			desc->src_addr = cpu_to_le64(*src);
+			desc->dst_addr = cpu_to_le64(*dst);
+
+			if (!(desc_num & XDMA_DESC_ADJACENT_MASK)) {
+				dblk++;
+				desc = dblk->virt_addr;
+			} else {
+				desc++;
+			}
+
+			desc_num++;
+			dev_addr += len;
+			addr += len;
+			rest -= len;
+		} while (rest);
+	}
+
+	tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags);
+	if (!tx_desc)
+		goto failed;
+
+	return tx_desc;
+
+failed:
+	xdma_free_desc(&sw_desc->vdesc);
+
+	return NULL;
+}
+
+/**
+ * xdma_device_config - Configure the DMA channel
+ * @chan: DMA channel
+ * @cfg: channel configuration
+ */
+static int xdma_device_config(struct dma_chan *chan,
+			      struct dma_slave_config *cfg)
+{
+	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
+
+	memcpy(&xdma_chan->cfg, cfg, sizeof(*cfg));
+
+	return 0;
+}
+
+/**
+ * xdma_free_chan_resources - Free channel resources
+ * @chan: DMA channel
+ */
+static void xdma_free_chan_resources(struct dma_chan *chan)
+{
+	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
+
+	vchan_free_chan_resources(&xdma_chan->vchan);
+	dma_pool_destroy(xdma_chan->desc_pool);
+	xdma_chan->desc_pool = NULL;
+}
+
+/**
+ * xdma_alloc_chan_resources - Allocate channel resources
+ * @chan: DMA channel
+ */
+static int xdma_alloc_chan_resources(struct dma_chan *chan)
+{
+	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
+	struct xdma_device *xdev = xdma_chan->xdev_hdl;
+
+	xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan),
+					       xdev->dma_dev.dev,
+					       XDMA_DESC_BLOCK_SIZE,
+					       XDMA_DESC_BLOCK_ALIGN,
+					       0);
+	if (!xdma_chan->desc_pool) {
+		dev_err(&chan->dev->device,
+			"unable to allocate descriptor pool");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * xdma_channel_isr - XDMA channel interrupt handler
+ * @irq: IRQ number
+ * @dev_id: Pointer to the DMA channel structure
+ */
+static irqreturn_t xdma_channel_isr(int irq, void *dev_id)
+{
+	struct xdma_chan *xdma_chan = dev_id;
+	u32 complete_desc_num = 0;
+	struct virt_dma_desc *vd;
+	struct xdma_desc *desc;
+	int ret;
+
+	spin_lock(&xdma_chan->vchan.lock);
+
+	/* get submitted request */
+	vd = vchan_next_desc(&xdma_chan->vchan);
+	if (!vd)
+		goto out;
+
+	xdma_chan->busy = false;
+	desc = to_xdma_desc(vd);
+
+	ret = xdma_read_reg(xdma_chan->xdev_hdl, xdma_chan->base,
+			    XDMA_CHAN_COMPLETED_DESC, &complete_desc_num);
+	if (ret)
+		goto out;
+
+	desc->completed_desc_num += complete_desc_num;
+	/*
+	 * if all data blocks are transferred, remove and complete the request
+	 */
+	if (desc->completed_desc_num == desc->desc_num) {
+		list_del(&vd->node);
+		vchan_cookie_complete(vd);
+		goto out;
+	}
+
+	if (desc->completed_desc_num > desc->desc_num ||
+	    complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT)
+		goto out;
+
+	/* transfer the rest of data */
+	tasklet_schedule(&xdma_chan->tasklet);
+
+out:
+	spin_unlock(&xdma_chan->vchan.lock);
+	return IRQ_HANDLED;
+}
+
+/**
+ * xdma_irq_fini - Uninitialize IRQ
+ * @xdev: DMA device pointer
+ */
+static void xdma_irq_fini(struct xdma_device *xdev)
+{
+	int ret, i;
+
+	/* disable interrupt */
+	ret = xdma_disable_intr(xdev);
+	if (ret) {
+		dev_err(&xdev->pdev->dev, "failed to disable interrupts: %d",
+			ret);
+	}
+
+	/* free irq handler */
+	for (i = 0; i < xdev->h2c_chan_num; i++) {
+		free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
+		tasklet_kill(&xdev->h2c_chans[i].tasklet);
+	}
+	for (i = 0; i < xdev->c2h_chan_num; i++) {
+		free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
+		tasklet_kill(&xdev->c2h_chans[i].tasklet);
+	}
+}
+
+/**
+ * xdma_set_vector_reg - configure hardware IRQ registers
+ * @xdev: DMA device pointer
+ * @vec_tbl_start: Start of IRQ registers
+ * @irq_start: Start of IRQ
+ * @irq_num: Number of IRQ
+ */
+static int xdma_set_vector_reg(struct xdma_device *xdev, u32 vec_tbl_start,
+			       u32 irq_start, u32 irq_num)
+{
+	u32 shift, i, val = 0;
+	int ret;
+
+	/* Each IRQ register is 32 bit and contains 4 IRQs */
+	while (irq_num > 0) {
+		for (i = 0; i < 4; i++) {
+			shift = XDMA_IRQ_VEC_SHIFT * i;
+			val |= irq_start << shift;
+			irq_start++;
+			irq_num--;
+		}
+
+		/* write IRQ register */
+		ret = xdma_write_reg(xdev, XDMA_IRQ_BASE, vec_tbl_start, val);
+		if (ret) {
+			dev_err(&xdev->pdev->dev, "failed to set vector: %d",
+				ret);
+			return ret;
+		}
+		vec_tbl_start += sizeof(u32);
+		val = 0;
+	}
+
+	return 0;
+}
+
+/**
+ * xdma_irq_init - initialize IRQs
+ * @xdev: DMA device pointer
+ */
+static int xdma_irq_init(struct xdma_device *xdev)
+{
+	u32 irq = xdev->irq_start;
+	int i, j, ret;
+
+	/* return failure if there are not enough IRQs */
+	if (xdev->irq_num < xdev->h2c_chan_num + xdev->c2h_chan_num) {
+		dev_err(&xdev->pdev->dev, "not enough irq");
+		return -EINVAL;
+	}
+
+	/* setup H2C interrupt handler */
+	for (i = 0; i < xdev->h2c_chan_num; i++) {
+		ret = request_irq(irq, xdma_channel_isr, 0,
+				  "xdma-h2c-channel", &xdev->h2c_chans[i]);
+		if (ret) {
+			dev_err(&xdev->pdev->dev,
+				"H2C channel%d request irq%d failed: %d",
+				i, irq, ret);
+			for (j = 0; j < i; j++) {
+				free_irq(xdev->h2c_chans[j].irq,
+					 &xdev->h2c_chans[j]);
+			}
+			return ret;
+		}
+		xdev->h2c_chans[i].irq = irq;
+		tasklet_setup(&xdev->h2c_chans[i].tasklet,
+			      xdma_channel_tasklet);
+		irq++;
+	}
+
+	/* setup C2H interrupt handler */
+	for (i = 0; i < xdev->c2h_chan_num; i++) {
+		ret = request_irq(irq, xdma_channel_isr, 0,
+				  "xdma-c2h-channel", &xdev->c2h_chans[i]);
+		if (ret) {
+			dev_err(&xdev->pdev->dev,
+				"H2C channel%d request irq%d failed: %d",
+				i, irq, ret);
+			for (j = 0; j < i; j++) {
+				free_irq(xdev->c2h_chans[j].irq,
+					 &xdev->c2h_chans[j]);
+			}
+			goto failed_init_c2h;
+		}
+		xdev->c2h_chans[i].irq = irq;
+		tasklet_setup(&xdev->c2h_chans[i].tasklet,
+			      xdma_channel_tasklet);
+		irq++;
+	}
+
+	/* config hardware IRQ registers */
+	ret = xdma_set_vector_reg(xdev, XDMA_IRQ_CHAN_VEC_NUM, 0,
+				  xdev->h2c_chan_num + xdev->c2h_chan_num);
+	if (ret) {
+		dev_err(&xdev->pdev->dev, "failed to set channel vectors: %d",
+			ret);
+		goto failed;
+	}
+
+	/* enable interrupt */
+	ret = xdma_enable_intr(xdev);
+	if (ret) {
+		dev_err(&xdev->pdev->dev, "failed to enable interrupts: %d",
+			ret);
+		goto failed;
+	}
+
+	return 0;
+
+failed:
+	for (i = 0; i < xdev->c2h_chan_num; i++)
+		free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
+failed_init_c2h:
+	for (i = 0; i < xdev->h2c_chan_num; i++)
+		free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
+
+	return ret;
+}
+
+static bool xdma_filter_fn(struct dma_chan *chan, void *param)
+{
+	struct xdma_chan *xdma_chan = to_xdma_chan(chan);
+	struct xdma_chan_info *chan_info = param;
+
+	if (chan_info->dir != xdma_chan->dir)
+		return false;
+
+	return true;
+}
+
+/**
+ * xdma_remove - Driver remove function
+ * @pdev: Pointer to the platform_device structure
+ */
+static int xdma_remove(struct platform_device *pdev)
+{
+	struct xdma_device *xdev = platform_get_drvdata(pdev);
+
+	if (xdev->status & XDMA_DEV_STATUS_INIT_MSIX)
+		xdma_irq_fini(xdev);
+
+	if (xdev->status & XDMA_DEV_STATUS_REG_DMA)
+		dma_async_device_unregister(&xdev->dma_dev);
+
+	return 0;
+}
+
+/**
+ * xdma_probe - Driver probe function
+ * @pdev: Pointer to the platform_device structure
+ */
+static int xdma_probe(struct platform_device *pdev)
+{
+	struct xdma_platdata *pdata = dev_get_platdata(&pdev->dev);
+	struct xdma_device *xdev;
+	void __iomem *reg_base;
+	struct resource *res;
+	int ret = -ENODEV;
+
+	if (pdata->max_dma_channels > XDMA_MAX_CHANNELS) {
+		dev_err(&pdev->dev, "invalid max dma channels %d",
+			pdata->max_dma_channels);
+		return -EINVAL;
+	}
+
+	xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);
+	if (!xdev)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, xdev);
+	xdev->pdev = pdev;
+
+	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "failed to get irq resource");
+		goto failed;
+	}
+	xdev->irq_start = res->start;
+	xdev->irq_num = res->end - res->start + 1;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "failed to get io resource");
+		goto failed;
+	}
+
+	reg_base = devm_ioremap_resource(&pdev->dev, res);
+	if (!reg_base) {
+		dev_err(&pdev->dev, "ioremap failed");
+		goto failed;
+	}
+
+	xdev->regmap = devm_regmap_init_mmio(&pdev->dev, reg_base,
+					     &xdma_regmap_config);
+	if (!xdev->regmap) {
+		dev_err(&pdev->dev, "config regmap failed: %d", ret);
+		goto failed;
+	}
+	INIT_LIST_HEAD(&xdev->dma_dev.channels);
+
+	ret = xdma_config_channels(xdev, DMA_MEM_TO_DEV);
+	if (ret) {
+		dev_err(&pdev->dev, "config H2C channels failed: %d", ret);
+		goto failed;
+	}
+
+	ret = xdma_config_channels(xdev, DMA_DEV_TO_MEM);
+	if (ret) {
+		dev_err(&pdev->dev, "config C2H channels failed: %d", ret);
+		goto failed;
+	}
+
+	dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask);
+	dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask);
+
+	xdev->dma_dev.dev = &pdev->dev;
+	xdev->dma_dev.device_free_chan_resources = xdma_free_chan_resources;
+	xdev->dma_dev.device_alloc_chan_resources = xdma_alloc_chan_resources;
+	xdev->dma_dev.device_tx_status = dma_cookie_status;
+	xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg;
+	xdev->dma_dev.device_config = xdma_device_config;
+	xdev->dma_dev.device_issue_pending = xdma_issue_pending;
+	xdev->dma_dev.filter.map = pdata->device_map;
+	xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt;
+	xdev->dma_dev.filter.fn = xdma_filter_fn;
+
+	ret = dma_async_device_register(&xdev->dma_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to register Xilinx XDMA: %d", ret);
+		goto failed;
+	}
+	xdev->status |= XDMA_DEV_STATUS_REG_DMA;
+
+	ret = xdma_irq_init(xdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to init msix: %d", ret);
+		goto failed;
+	}
+	xdev->status |= XDMA_DEV_STATUS_INIT_MSIX;
+
+	return 0;
+
+failed:
+	xdma_remove(pdev);
+
+	return ret;
+}
+
+static const struct platform_device_id xdma_id_table[] = {
+	{ "xdma", 0},
+	{ },
+};
+
+static struct platform_driver xdma_driver = {
+	.driver		= {
+		.name = "xdma",
+	},
+	.id_table	= xdma_id_table,
+	.probe		= xdma_probe,
+	.remove		= xdma_remove,
+};
+
+module_platform_driver(xdma_driver);
+
+MODULE_DESCRIPTION("AMD XDMA driver");
+MODULE_AUTHOR("XRT Team <[e-mail]>");
+MODULE_LICENSE("GPL");
diff --git a/include/linux/platform_data/amd_xdma.h b/include/linux/platform_data/amd_xdma.h
new file mode 100644
index 000000000000..04bcfc74ab36
--- /dev/null
+++ b/include/linux/platform_data/amd_xdma.h
@@ -0,0 +1,34 @@ 
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
+ */
+
+#ifndef _PLATDATA_AMD_XDMA_H
+#define _PLATDATA_AMD_XDMA_H
+
+#include <linux/dmaengine.h>
+
+/**
+ * struct xdma_chan_info - DMA channel information
+ *	This information is used to match channel when request dma channel
+ * @dir: Channel transfer direction
+ */
+struct xdma_chan_info {
+	enum dma_transfer_direction dir;
+};
+
+#define XDMA_FILTER_PARAM(chan_info)	((void *)(chan_info))
+
+struct dma_slave_map;
+
+/**
+ * struct xdma_platdata - platform specific data for XDMA engine
+ * @max_dma_channels: Maximum dma channels in each direction
+ */
+struct xdma_platdata {
+	u32 max_dma_channels;
+	u32 device_map_cnt;
+	struct dma_slave_map *device_map;
+};
+
+#endif /* _PLATDATA_AMD_XDMA_H */