@@ -10,6 +10,7 @@
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/shmem_fs.h>
+#include <linux/hugetlb.h>
#include <linux/slab.h>
#include <linux/udmabuf.h>
#include <linux/vmalloc.h>
@@ -28,6 +29,7 @@ struct udmabuf {
struct page **pages;
struct sg_table *sg;
struct miscdevice *device;
+ pgoff_t *subpgoff;
};
static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
@@ -90,23 +92,31 @@ static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf,
{
struct udmabuf *ubuf = buf->priv;
struct sg_table *sg;
+ struct scatterlist *sgl;
+ pgoff_t offset;
+ unsigned long i = 0;
int ret;
sg = kzalloc(sizeof(*sg), GFP_KERNEL);
if (!sg)
return ERR_PTR(-ENOMEM);
- ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount,
- 0, ubuf->pagecount << PAGE_SHIFT,
- GFP_KERNEL);
+
+ ret = sg_alloc_table(sg, ubuf->pagecount, GFP_KERNEL);
if (ret < 0)
- goto err;
+ goto err_alloc;
+
+ for_each_sg(sg->sgl, sgl, ubuf->pagecount, i) {
+ offset = ubuf->subpgoff ? ubuf->subpgoff[i] : 0;
+ sg_set_page(sgl, ubuf->pages[i], PAGE_SIZE, offset);
+ }
ret = dma_map_sgtable(dev, sg, direction, 0);
if (ret < 0)
- goto err;
+ goto err_map;
return sg;
-err:
+err_map:
sg_free_table(sg);
+err_alloc:
kfree(sg);
return ERR_PTR(ret);
}
@@ -142,7 +152,9 @@ static void release_udmabuf(struct dma_buf *buf)
put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL);
for (pg = 0; pg < ubuf->pagecount; pg++)
- put_page(ubuf->pages[pg]);
+ unpin_user_page(ubuf->pages[pg]);
+
+ kfree(ubuf->subpgoff);
kfree(ubuf->pages);
kfree(ubuf);
}
@@ -202,12 +214,13 @@ static long udmabuf_create(struct miscdevice *device,
{
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
struct file *memfd = NULL;
- struct address_space *mapping = NULL;
struct udmabuf *ubuf;
struct dma_buf *buf;
- pgoff_t pgoff, pgcnt, pgidx, pgbuf = 0, pglimit;
- struct page *page;
- int seals, ret = -EINVAL;
+ pgoff_t pgoff, pgcnt, pgbuf = 0, pglimit, nr_pages;
+ pgoff_t subpgoff, maxsubpgs;
+ struct hstate *hpstate;
+ long ret = -EINVAL;
+ int seals;
u32 i, flags;
ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL);
@@ -241,8 +254,7 @@ static long udmabuf_create(struct miscdevice *device,
memfd = fget(list[i].memfd);
if (!memfd)
goto err;
- mapping = memfd->f_mapping;
- if (!shmem_mapping(mapping))
+ if (!shmem_file(memfd) && !is_file_hugepages(memfd))
goto err;
seals = memfd_fcntl(memfd, F_GET_SEALS, 0);
if (seals == -EINVAL)
@@ -253,14 +265,41 @@ static long udmabuf_create(struct miscdevice *device,
goto err;
pgoff = list[i].offset >> PAGE_SHIFT;
pgcnt = list[i].size >> PAGE_SHIFT;
- for (pgidx = 0; pgidx < pgcnt; pgidx++) {
- page = shmem_read_mapping_page(mapping, pgoff + pgidx);
- if (IS_ERR(page)) {
- ret = PTR_ERR(page);
+ if (is_file_hugepages(memfd)) {
+ if (!ubuf->subpgoff) {
+ ubuf->subpgoff = kmalloc_array(ubuf->pagecount,
+ sizeof(*ubuf->subpgoff),
+ GFP_KERNEL);
+ if (!ubuf->subpgoff) {
+ ret = -ENOMEM;
+ goto err;
+ }
+ }
+ hpstate = hstate_file(memfd);
+ pgoff = list[i].offset >> huge_page_shift(hpstate);
+ subpgoff = (list[i].offset &
+ ~huge_page_mask(hpstate)) >> PAGE_SHIFT;
+ maxsubpgs = huge_page_size(hpstate) >> PAGE_SHIFT;
+ }
+
+ do {
+ nr_pages = shmem_file(memfd) ? pgcnt : 1;
+ ret = pin_user_pages_fd(list[i].memfd, pgoff,
+ nr_pages, FOLL_LONGTERM,
+ ubuf->pages + pgbuf);
+ if (ret < 0)
goto err;
+
+ if (is_file_hugepages(memfd)) {
+ ubuf->subpgoff[pgbuf] = subpgoff << PAGE_SHIFT;
+ if (++subpgoff == maxsubpgs) {
+ subpgoff = 0;
+ pgoff++;
+ }
}
- ubuf->pages[pgbuf++] = page;
- }
+ pgbuf += nr_pages;
+ pgcnt -= nr_pages;
+ } while (pgcnt > 0);
fput(memfd);
memfd = NULL;
}
@@ -283,10 +322,11 @@ static long udmabuf_create(struct miscdevice *device,
return dma_buf_fd(buf, flags);
err:
- while (pgbuf > 0)
- put_page(ubuf->pages[--pgbuf]);
+ while (pgbuf > 0 && ubuf->pages[--pgbuf])
+ unpin_user_page(ubuf->pages[pgbuf]);
if (memfd)
fput(memfd);
+ kfree(ubuf->subpgoff);
kfree(ubuf->pages);
kfree(ubuf);
return ret;
Using pin_user_pages_fd() will ensure that the pages are pinned correctly using FOLL_PIN. And, this also ensures that we don't accidentally break features such as memory hotunplug as it would not allow pinning pages in the movable zone. This patch also adds back support for mapping hugetlbfs pages by noting the subpage offsets within the huge pages and uses this information while populating the scatterlist. Cc: David Hildenbrand <david@redhat.com> Cc: Daniel Vetter <daniel.vetter@ffwll.ch> Cc: Mike Kravetz <mike.kravetz@oracle.com> Cc: Hugh Dickins <hughd@google.com> Cc: Peter Xu <peterx@redhat.com> Cc: Jason Gunthorpe <jgg@nvidia.com> Cc: Gerd Hoffmann <kraxel@redhat.com> Cc: Dongwon Kim <dongwon.kim@intel.com> Cc: Junxiao Chang <junxiao.chang@intel.com> Signed-off-by: Vivek Kasireddy <vivek.kasireddy@intel.com> --- drivers/dma-buf/udmabuf.c | 82 +++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 21 deletions(-)