@@ -1,9 +1,11 @@
# SPDX-License-Identifier: GPL-2.0
vfio_virqfd-y := virqfd.o
-vfio-y += vfio_main.o
-
obj-$(CONFIG_VFIO) += vfio.o
+
+vfio-y := vfio_main.o \
+ iova_bitmap.o \
+
obj-$(CONFIG_VFIO_VIRQFD) += vfio_virqfd.o
obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
new file mode 100644
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates.
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
+ */
+
+#include <linux/iova_bitmap.h>
+
+static unsigned long iova_bitmap_array_length(unsigned long iova_length,
+ unsigned long page_shift)
+{
+ return DIV_ROUND_UP(iova_length, BITS_PER_TYPE(u64) * (1 << page_shift));
+}
+
+static unsigned long iova_bitmap_index_to_length(struct iova_bitmap_iter *iter,
+ unsigned long index)
+{
+ return ((index * sizeof(u64) * BITS_PER_BYTE) << iter->dirty.pgshift);
+}
+
+static unsigned long iova_bitmap_iter_left(struct iova_bitmap_iter *iter)
+{
+ unsigned long left = iter->count - iter->offset;
+
+ left = min_t(unsigned long, left,
+ (iter->dirty.npages << PAGE_SHIFT) / sizeof(u64));
+
+ return left;
+}
+
+/*
+ * Input argument of number of bits to bitmap_set() is unsigned integer, which
+ * further casts to signed integer for unaligned multi-bit operation,
+ * __bitmap_set().
+ * Then maximum bitmap size supported is 2^31 bits divided by 2^3 bits/byte,
+ * that is 2^28 (256 MB) which maps to 2^31 * 2^12 = 2^43 (8TB) on 4K page
+ * system.
+ */
+int iova_bitmap_iter_init(struct iova_bitmap_iter *iter,
+ unsigned long iova, unsigned long length,
+ unsigned long __user *data)
+{
+ struct iova_bitmap *dirty = &iter->dirty;
+
+ iter->data = data;
+ iter->offset = 0;
+ iter->count = iova_bitmap_array_length(length, dirty->pgshift);
+ iter->iova = iova;
+ iter->length = length;
+ dirty->pages = (struct page **)__get_free_page(GFP_KERNEL);
+
+ return !dirty->pages ? -ENOMEM : 0;
+}
+
+void iova_bitmap_iter_free(struct iova_bitmap_iter *iter)
+{
+ struct iova_bitmap *dirty = &iter->dirty;
+
+ if (dirty->pages) {
+ free_page((unsigned long)dirty->pages);
+ dirty->pages = NULL;
+ }
+}
+
+bool iova_bitmap_iter_done(struct iova_bitmap_iter *iter)
+{
+ return (iter->count - iter->offset) > 0;
+}
+
+static unsigned long iova_bitmap_iter_length(struct iova_bitmap_iter *iter)
+{
+ return iova_bitmap_index_to_length(iter, iter->count);
+}
+
+unsigned long iova_bitmap_length(struct iova_bitmap_iter *iter)
+{
+ unsigned long left = iova_bitmap_iter_left(iter);
+ unsigned long iova = iova_bitmap_iova(iter);
+
+ left = iova_bitmap_index_to_length(iter, left);
+ if (iova_bitmap_iter_length(iter) > iter->length &&
+ iova + left > iter->iova + iter->length - 1)
+ left -= ((iova + left) - (iova + iter->length));
+ return left;
+}
+
+unsigned long iova_bitmap_iova(struct iova_bitmap_iter *iter)
+{
+ unsigned long skip = iter->offset;
+
+ return iter->iova + iova_bitmap_index_to_length(iter, skip);
+}
+
+void iova_bitmap_iter_advance(struct iova_bitmap_iter *iter)
+{
+ unsigned long length = iova_bitmap_length(iter);
+
+ iter->offset += iova_bitmap_array_length(length, iter->dirty.pgshift);
+}
+
+void iova_bitmap_iter_put(struct iova_bitmap_iter *iter)
+{
+ struct iova_bitmap *dirty = &iter->dirty;
+
+ if (dirty->npages)
+ unpin_user_pages(dirty->pages, dirty->npages);
+}
+
+int iova_bitmap_iter_get(struct iova_bitmap_iter *iter)
+{
+ struct iova_bitmap *dirty = &iter->dirty;
+ unsigned long npages;
+ void __user *addr;
+ long ret;
+
+ npages = DIV_ROUND_UP((iter->count - iter->offset) * sizeof(u64),
+ PAGE_SIZE);
+ npages = min(npages, PAGE_SIZE / sizeof(struct page *));
+ addr = iter->data + (iter->offset * sizeof(u64));
+ ret = pin_user_pages_fast((unsigned long)addr, npages,
+ FOLL_WRITE, dirty->pages);
+ if (ret <= 0)
+ return ret;
+
+ dirty->npages = (unsigned long)ret;
+ dirty->iova = iova_bitmap_iova(iter);
+ dirty->start_offset = offset_in_page(addr);
+ return 0;
+}
+
+void iova_bitmap_init(struct iova_bitmap *bitmap,
+ unsigned long base, unsigned long pgshift)
+{
+ memset(bitmap, 0, sizeof(*bitmap));
+ bitmap->iova = base;
+ bitmap->pgshift = pgshift;
+}
+
+unsigned int iova_bitmap_set(struct iova_bitmap *dirty,
+ unsigned long iova,
+ unsigned long length)
+{
+ unsigned long nbits, offset, start_offset, idx, size, *kaddr;
+
+ nbits = max(1UL, length >> dirty->pgshift);
+ offset = (iova - dirty->iova) >> dirty->pgshift;
+ idx = offset / (PAGE_SIZE * BITS_PER_BYTE);
+ offset = offset % (PAGE_SIZE * BITS_PER_BYTE);
+ start_offset = dirty->start_offset;
+
+ while (nbits > 0) {
+ kaddr = kmap_local_page(dirty->pages[idx]) + start_offset;
+ size = min(PAGE_SIZE * BITS_PER_BYTE - offset, nbits);
+ bitmap_set(kaddr, offset, size);
+ kunmap_local(kaddr - start_offset);
+ start_offset = offset = 0;
+ nbits -= size;
+ idx++;
+ }
+
+ return nbits;
+}
+EXPORT_SYMBOL_GPL(iova_bitmap_set);
+
new file mode 100644
@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates.
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
+ */
+
+#ifndef _IOVA_BITMAP_H_
+#define _IOVA_BITMAP_H_
+
+#include <linux/highmem.h>
+#include <linux/mm.h>
+#include <linux/uio.h>
+
+struct iova_bitmap {
+ unsigned long iova;
+ unsigned long pgshift;
+ unsigned long start_offset;
+ unsigned long npages;
+ struct page **pages;
+};
+
+struct iova_bitmap_iter {
+ struct iova_bitmap dirty;
+ void __user *data;
+ size_t offset;
+ size_t count;
+ unsigned long iova;
+ unsigned long length;
+};
+
+int iova_bitmap_iter_init(struct iova_bitmap_iter *iter, unsigned long iova,
+ unsigned long length, unsigned long __user *data);
+void iova_bitmap_iter_free(struct iova_bitmap_iter *iter);
+bool iova_bitmap_iter_done(struct iova_bitmap_iter *iter);
+unsigned long iova_bitmap_length(struct iova_bitmap_iter *iter);
+unsigned long iova_bitmap_iova(struct iova_bitmap_iter *iter);
+void iova_bitmap_iter_advance(struct iova_bitmap_iter *iter);
+int iova_bitmap_iter_get(struct iova_bitmap_iter *iter);
+void iova_bitmap_iter_put(struct iova_bitmap_iter *iter);
+void iova_bitmap_init(struct iova_bitmap *bitmap,
+ unsigned long base, unsigned long pgshift);
+unsigned int iova_bitmap_set(struct iova_bitmap *dirty,
+ unsigned long iova,
+ unsigned long length);
+
+#endif