@@ -4,4 +4,4 @@ obj-$(CONFIG_INTEL_FPGA_AFU) += intel-fpga-afu.o
intel-fpga-pci-objs := pcie.o feature-dev.o
intel-fpga-fme-objs := fme-main.o fme-pr.o
-intel-fpga-afu-objs := afu-main.o afu-region.o
+intel-fpga-afu-objs := afu-main.o afu-region.o afu-dma-region.o
new file mode 100644
@@ -0,0 +1,373 @@
+/*
+ * Driver for Intel FPGA Accelerated Function Unit (AFU) DMA Region Management
+ *
+ * Copyright (C) 2017 Intel Corporation, Inc.
+ *
+ * Authors:
+ * Wu Hao <hao.wu@intel.com>
+ * Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ *
+ * This work is licensed under a dual BSD/GPLv2 license. When using or
+ * redistributing this file, you may do so under either license. See the
+ * LICENSE.BSD file under this directory for the BSD license and see
+ * the COPYING file in the top-level directory for the GPLv2 license.
+ */
+
+#include <linux/sched/signal.h>
+#include <linux/uaccess.h>
+
+#include "afu.h"
+
+static void put_all_pages(struct page **pages, int npages)
+{
+ int i;
+
+ for (i = 0; i < npages; i++)
+ if (pages[i] != NULL)
+ put_page(pages[i]);
+}
+
+void afu_dma_region_init(struct feature_platform_data *pdata)
+{
+ struct fpga_afu *afu = fpga_pdata_get_private(pdata);
+
+ afu->dma_regions = RB_ROOT;
+}
+
+static long afu_dma_adjust_locked_vm(struct device *dev, long npages, bool incr)
+{
+ unsigned long locked, lock_limit;
+ int ret = 0;
+
+ /* the task is exiting. */
+ if (!current->mm)
+ return 0;
+
+ down_write(¤t->mm->mmap_sem);
+
+ if (incr) {
+ locked = current->mm->locked_vm + npages;
+ lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+
+ if (locked > lock_limit && !capable(CAP_IPC_LOCK))
+ ret = -ENOMEM;
+ else
+ current->mm->locked_vm += npages;
+ } else {
+
+ if (WARN_ON_ONCE(npages > current->mm->locked_vm))
+ npages = current->mm->locked_vm;
+ current->mm->locked_vm -= npages;
+ }
+
+ dev_dbg(dev, "[%d] RLIMIT_MEMLOCK %c%ld %ld/%ld%s\n", current->pid,
+ incr ? '+' : '-',
+ npages << PAGE_SHIFT,
+ current->mm->locked_vm << PAGE_SHIFT,
+ rlimit(RLIMIT_MEMLOCK),
+ ret ? "- execeeded" : "");
+
+ up_write(¤t->mm->mmap_sem);
+
+ return ret;
+}
+
+static long afu_dma_pin_pages(struct feature_platform_data *pdata,
+ struct fpga_afu_dma_region *region)
+{
+ long npages = region->length >> PAGE_SHIFT;
+ struct device *dev = &pdata->dev->dev;
+ long ret, pinned;
+
+ ret = afu_dma_adjust_locked_vm(dev, npages, true);
+ if (ret)
+ return ret;
+
+ region->pages = kcalloc(npages, sizeof(struct page *), GFP_KERNEL);
+ if (!region->pages) {
+ afu_dma_adjust_locked_vm(dev, npages, false);
+ return -ENOMEM;
+ }
+
+ pinned = get_user_pages_fast(region->user_addr, npages, 1,
+ region->pages);
+ if (pinned < 0) {
+ ret = pinned;
+ goto err_put_pages;
+ } else if (pinned != npages) {
+ ret = -EFAULT;
+ goto err;
+ }
+
+ dev_dbg(dev, "%ld pages pinned\n", pinned);
+
+ return 0;
+
+err_put_pages:
+ put_all_pages(region->pages, pinned);
+err:
+ kfree(region->pages);
+ afu_dma_adjust_locked_vm(dev, npages, false);
+ return ret;
+}
+
+static void afu_dma_unpin_pages(struct feature_platform_data *pdata,
+ struct fpga_afu_dma_region *region)
+{
+ long npages = region->length >> PAGE_SHIFT;
+ struct device *dev = &pdata->dev->dev;
+
+ put_all_pages(region->pages, npages);
+ kfree(region->pages);
+ afu_dma_adjust_locked_vm(dev, npages, false);
+
+ dev_dbg(dev, "%ld pages unpinned\n", npages);
+}
+
+static bool afu_dma_check_continuous_pages(struct fpga_afu_dma_region *region)
+{
+ int npages = region->length >> PAGE_SHIFT;
+ int i;
+
+ for (i = 0; i < npages - 1; i++)
+ if (page_to_pfn(region->pages[i]) + 1 !=
+ page_to_pfn(region->pages[i+1]))
+ return false;
+
+ return true;
+}
+
+static bool dma_region_check_iova(struct fpga_afu_dma_region *region,
+ u64 iova, u64 size)
+{
+ if (!size && region->iova != iova)
+ return false;
+
+ return (region->iova <= iova) &&
+ (region->length + region->iova >= iova + size);
+}
+
+/* Need to be called with pdata->lock held */
+static int afu_dma_region_add(struct feature_platform_data *pdata,
+ struct fpga_afu_dma_region *region)
+{
+ struct fpga_afu *afu = fpga_pdata_get_private(pdata);
+ struct rb_node **new, *parent = NULL;
+
+ dev_dbg(&pdata->dev->dev, "add region (iova = %llx)\n",
+ (unsigned long long)region->iova);
+
+ new = &(afu->dma_regions.rb_node);
+
+ while (*new) {
+ struct fpga_afu_dma_region *this;
+
+ this = container_of(*new, struct fpga_afu_dma_region, node);
+
+ parent = *new;
+
+ if (dma_region_check_iova(this, region->iova, region->length))
+ return -EEXIST;
+
+ if (region->iova < this->iova)
+ new = &((*new)->rb_left);
+ else if (region->iova > this->iova)
+ new = &((*new)->rb_right);
+ else
+ return -EEXIST;
+ }
+
+ rb_link_node(®ion->node, parent, new);
+ rb_insert_color(®ion->node, &afu->dma_regions);
+
+ return 0;
+}
+
+/* Need to be called with pdata->lock held */
+static void afu_dma_region_remove(struct feature_platform_data *pdata,
+ struct fpga_afu_dma_region *region)
+{
+ struct fpga_afu *afu;
+
+ dev_dbg(&pdata->dev->dev, "del region (iova = %llx)\n",
+ (unsigned long long)region->iova);
+
+ afu = fpga_pdata_get_private(pdata);
+ rb_erase(®ion->node, &afu->dma_regions);
+}
+
+/* Need to be called with pdata->lock held */
+void afu_dma_region_destroy(struct feature_platform_data *pdata)
+{
+ struct fpga_afu *afu = fpga_pdata_get_private(pdata);
+ struct rb_node *node = rb_first(&afu->dma_regions);
+ struct fpga_afu_dma_region *region;
+
+ while (node) {
+ region = container_of(node, struct fpga_afu_dma_region, node);
+
+ dev_dbg(&pdata->dev->dev, "del region (iova = %llx)\n",
+ (unsigned long long)region->iova);
+
+ rb_erase(node, &afu->dma_regions);
+
+ if (region->iova)
+ dma_unmap_page(fpga_pdata_to_pcidev(pdata),
+ region->iova, region->length,
+ DMA_BIDIRECTIONAL);
+
+ if (region->pages)
+ afu_dma_unpin_pages(pdata, region);
+
+ node = rb_next(node);
+ kfree(region);
+ }
+}
+
+/*
+ * It finds the dma region from the rbtree based on @iova and @size:
+ * - if @size == 0, it finds the dma region which starts from @iova
+ * - otherwise, it finds the dma region which fully contains
+ * [@iova, @iova+size)
+ * If nothing is matched returns NULL.
+ *
+ * Need to be called with pdata->lock held.
+ */
+struct fpga_afu_dma_region *
+afu_dma_region_find(struct feature_platform_data *pdata, u64 iova, u64 size)
+{
+ struct fpga_afu *afu = fpga_pdata_get_private(pdata);
+ struct rb_node *node = afu->dma_regions.rb_node;
+ struct device *dev = &pdata->dev->dev;
+
+ while (node) {
+ struct fpga_afu_dma_region *region;
+
+ region = container_of(node, struct fpga_afu_dma_region, node);
+
+ if (dma_region_check_iova(region, iova, size)) {
+ dev_dbg(dev, "find region (iova = %llx)\n",
+ (unsigned long long)region->iova);
+ return region;
+ }
+
+ if (iova < region->iova)
+ node = node->rb_left;
+ else if (iova > region->iova)
+ node = node->rb_right;
+ else
+ /* the iova region is not fully covered. */
+ break;
+ }
+
+ dev_dbg(dev, "region with iova %llx and size %llx is not found\n",
+ (unsigned long long)iova, (unsigned long long)size);
+ return NULL;
+}
+
+static struct fpga_afu_dma_region *
+afu_dma_region_find_iova(struct feature_platform_data *pdata, u64 iova)
+{
+ return afu_dma_region_find(pdata, iova, 0);
+}
+
+long afu_dma_map_region(struct feature_platform_data *pdata,
+ u64 user_addr, u64 length, u64 *iova)
+{
+ struct fpga_afu_dma_region *region;
+ int ret;
+
+ /*
+ * Check Inputs, only accept page-aligned user memory region with
+ * valid length.
+ */
+ if (!PAGE_ALIGNED(user_addr) || !PAGE_ALIGNED(length) || !length)
+ return -EINVAL;
+
+ /* Check overflow */
+ if (user_addr + length < user_addr)
+ return -EINVAL;
+
+ if (!access_ok(VERIFY_WRITE, user_addr, length))
+ return -EINVAL;
+
+ region = kzalloc(sizeof(*region), GFP_KERNEL);
+ if (!region)
+ return -ENOMEM;
+
+ region->user_addr = user_addr;
+ region->length = length;
+
+ /* Pin the user memory region */
+ ret = afu_dma_pin_pages(pdata, region);
+ if (ret) {
+ dev_err(&pdata->dev->dev, "fail to pin memory region\n");
+ goto free_region;
+ }
+
+ /* Only accept continuous pages, return error if no */
+ if (!afu_dma_check_continuous_pages(region)) {
+ dev_err(&pdata->dev->dev, "pages are not continuous\n");
+ ret = -EINVAL;
+ goto unpin_pages;
+ }
+
+ /* As pages are continuous then start to do DMA mapping */
+ region->iova = dma_map_page(fpga_pdata_to_pcidev(pdata),
+ region->pages[0], 0,
+ region->length,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(&pdata->dev->dev, region->iova)) {
+ dev_err(&pdata->dev->dev, "fail to map dma mapping\n");
+ ret = -EFAULT;
+ goto unpin_pages;
+ }
+
+ *iova = region->iova;
+
+ mutex_lock(&pdata->lock);
+ ret = afu_dma_region_add(pdata, region);
+ mutex_unlock(&pdata->lock);
+ if (ret) {
+ dev_err(&pdata->dev->dev, "fail to add dma region\n");
+ goto unmap_dma;
+ }
+
+ return 0;
+
+unmap_dma:
+ dma_unmap_page(fpga_pdata_to_pcidev(pdata),
+ region->iova, region->length, DMA_BIDIRECTIONAL);
+unpin_pages:
+ afu_dma_unpin_pages(pdata, region);
+free_region:
+ kfree(region);
+ return ret;
+}
+
+long afu_dma_unmap_region(struct feature_platform_data *pdata, u64 iova)
+{
+ struct fpga_afu_dma_region *region;
+
+ mutex_lock(&pdata->lock);
+ region = afu_dma_region_find_iova(pdata, iova);
+ if (!region) {
+ mutex_unlock(&pdata->lock);
+ return -EINVAL;
+ }
+
+ if (region->in_use) {
+ mutex_unlock(&pdata->lock);
+ return -EBUSY;
+ }
+
+ afu_dma_region_remove(pdata, region);
+ mutex_unlock(&pdata->lock);
+
+ dma_unmap_page(fpga_pdata_to_pcidev(pdata),
+ region->iova, region->length, DMA_BIDIRECTIONAL);
+ afu_dma_unpin_pages(pdata, region);
+ kfree(region);
+
+ return 0;
+}
@@ -177,7 +177,11 @@ static int afu_release(struct inode *inode, struct file *filp)
dev_dbg(&pdev->dev, "Device File Release\n");
- fpga_port_reset(pdev);
+ mutex_lock(&pdata->lock);
+ __fpga_port_reset(pdev);
+ afu_dma_region_destroy(pdata);
+ mutex_unlock(&pdata->lock);
+
feature_dev_use_end(pdata);
return 0;
}
@@ -247,6 +251,55 @@ afu_ioctl_get_region_info(struct feature_platform_data *pdata, void __user *arg)
return 0;
}
+static long
+afu_ioctl_dma_map(struct feature_platform_data *pdata, void __user *arg)
+{
+ struct fpga_port_dma_map map;
+ unsigned long minsz;
+ long ret;
+
+ minsz = offsetofend(struct fpga_port_dma_map, iova);
+
+ if (copy_from_user(&map, arg, minsz))
+ return -EFAULT;
+
+ if (map.argsz < minsz || map.flags)
+ return -EINVAL;
+
+ ret = afu_dma_map_region(pdata, map.user_addr, map.length, &map.iova);
+ if (ret)
+ return ret;
+
+ if (copy_to_user(arg, &map, sizeof(map))) {
+ afu_dma_unmap_region(pdata, map.iova);
+ return -EFAULT;
+ }
+
+ dev_dbg(&pdata->dev->dev, "dma map: ua=%llx, len=%llx, iova=%llx\n",
+ (unsigned long long)map.user_addr,
+ (unsigned long long)map.length,
+ (unsigned long long)map.iova);
+
+ return 0;
+}
+
+static long
+afu_ioctl_dma_unmap(struct feature_platform_data *pdata, void __user *arg)
+{
+ struct fpga_port_dma_unmap unmap;
+ unsigned long minsz;
+
+ minsz = offsetofend(struct fpga_port_dma_unmap, iova);
+
+ if (copy_from_user(&unmap, arg, minsz))
+ return -EFAULT;
+
+ if (unmap.argsz < minsz || unmap.flags)
+ return -EINVAL;
+
+ return afu_dma_unmap_region(pdata, unmap.iova);
+}
+
static long afu_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
struct platform_device *pdev = filp->private_data;
@@ -265,6 +318,10 @@ static long afu_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
return afu_ioctl_get_info(pdata, (void __user *)arg);
case FPGA_PORT_GET_REGION_INFO:
return afu_ioctl_get_region_info(pdata, (void __user *)arg);
+ case FPGA_PORT_DMA_MAP:
+ return afu_ioctl_dma_map(pdata, (void __user *)arg);
+ case FPGA_PORT_DMA_UNMAP:
+ return afu_ioctl_dma_unmap(pdata, (void __user *)arg);
default:
/*
* Let sub-feature's ioctl function to handle the cmd
@@ -339,6 +396,7 @@ static int afu_dev_init(struct platform_device *pdev)
mutex_lock(&pdata->lock);
fpga_pdata_set_private(pdata, afu);
afu_region_init(pdata);
+ afu_dma_region_init(pdata);
mutex_unlock(&pdata->lock);
return 0;
}
@@ -351,6 +409,7 @@ static int afu_dev_destroy(struct platform_device *pdev)
mutex_lock(&pdata->lock);
afu = fpga_pdata_get_private(pdata);
afu_region_destroy(pdata);
+ afu_dma_region_destroy(pdata);
fpga_pdata_set_private(pdata, NULL);
mutex_unlock(&pdata->lock);
@@ -32,11 +32,21 @@ struct fpga_afu_region {
struct list_head node;
};
+struct fpga_afu_dma_region {
+ u64 user_addr;
+ u64 length;
+ u64 iova;
+ struct page **pages;
+ struct rb_node node;
+ bool in_use;
+};
+
struct fpga_afu {
u64 region_cur_offset;
int num_regions;
u8 num_umsgs;
struct list_head regions;
+ struct rb_root dma_regions;
struct feature_platform_data *pdata;
};
@@ -51,4 +61,12 @@ int afu_get_region_by_offset(struct feature_platform_data *pdata,
u64 offset, u64 size,
struct fpga_afu_region *pregion);
+void afu_dma_region_init(struct feature_platform_data *pdata);
+void afu_dma_region_destroy(struct feature_platform_data *pdata);
+long afu_dma_map_region(struct feature_platform_data *pdata,
+ u64 user_addr, u64 length, u64 *iova);
+long afu_dma_unmap_region(struct feature_platform_data *pdata, u64 iova);
+struct fpga_afu_dma_region *afu_dma_region_find(
+ struct feature_platform_data *pdata, u64 iova, u64 size);
+
#endif
@@ -113,6 +113,43 @@ struct fpga_port_region_info {
#define FPGA_PORT_GET_REGION_INFO _IO(FPGA_MAGIC, PORT_BASE + 2)
+/**
+ * FPGA_PORT_DMA_MAP - _IOWR(FPGA_MAGIC, PORT_BASE + 3,
+ * struct fpga_port_dma_map)
+ *
+ * Map the dma memory per user_addr and length which are provided by caller.
+ * Driver fills the iova in provided struct afu_port_dma_map.
+ * This interface only accepts page-size aligned user memory for dma mapping.
+ * Return: 0 on success, -errno on failure.
+ */
+struct fpga_port_dma_map {
+ /* Input */
+ __u32 argsz; /* Structure length */
+ __u32 flags; /* Zero for now */
+ __u64 user_addr; /* Process virtual address */
+ __u64 length; /* Length of mapping (bytes)*/
+ /* Output */
+ __u64 iova; /* IO virtual address */
+};
+
+#define FPGA_PORT_DMA_MAP _IO(FPGA_MAGIC, PORT_BASE + 3)
+
+/**
+ * FPGA_PORT_DMA_UNMAP - _IOW(FPGA_MAGIC, PORT_BASE + 4,
+ * struct fpga_port_dma_unmap)
+ *
+ * Unmap the dma memory per iova provided by caller.
+ * Return: 0 on success, -errno on failure.
+ */
+struct fpga_port_dma_unmap {
+ /* Input */
+ __u32 argsz; /* Structure length */
+ __u32 flags; /* Zero for now */
+ __u64 iova; /* IO virtual address */
+};
+
+#define FPGA_PORT_DMA_UNMAP _IO(FPGA_MAGIC, PORT_BASE + 4)
+
/* IOCTLs for FME file descriptor */
/**