[v2,rdma-next,2/5] RDMA/verbs: Add a DMA iterator to return aligned contiguous memory blocks

Commit Message

Saleem, Shiraz April 19, 2019, 1:43 p.m. UTC

This helper iterates over a DMA-mapped SGL and returns contiguous
memory blocks aligned to a HW supported page size.
The implementation is intended to work for HW that support
single page sizes or mixed page sizes.

Suggested-by: Jason Gunthorpe <[e-mail]>
Cc: Gal Pressman <[e-mail]>
Signed-off-by: Shiraz Saleem <[e-mail]>
---
 drivers/infiniband/core/verbs.c | 68 +++++++++++++++++++++++++++++++++++++++++
 include/rdma/ib_verbs.h         | 57 ++++++++++++++++++++++++++++++++++
 2 files changed, 125 insertions(+)

Patch

diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 7313edc..fa9725d 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -2711,3 +2711,71 @@  int rdma_init_netdev(struct ib_device *device, u8 port_num,
 					     netdev, params.param);
 }
 EXPORT_SYMBOL(rdma_init_netdev);
+
+static unsigned int rdma_find_mixed_pg_bit(struct ib_block_iter *biter)
+{
+	if (biter->__sg == biter->__sgl_head) {
+		return rdma_find_pg_bit(sg_dma_address(biter->__sg) +
+					sg_dma_len(biter->__sg),
+					biter->pgsz_bitmap);
+	} else if (sg_is_last(biter->__sg)) {
+		return rdma_find_pg_bit(sg_dma_address(biter->__sg),
+					biter->pgsz_bitmap);
+	} else {
+		unsigned int remaining =
+			sg_dma_address(biter->__sg) + sg_dma_len(biter->__sg) -
+			biter->__dma_addr;
+		unsigned int pg_bit = rdma_find_pg_bit(biter->__dma_addr,
+						       biter->pgsz_bitmap);
+		if (remaining < BIT_ULL(biter->__pg_bit))
+			pg_bit = rdma_find_pg_bit(remaining,
+						  biter->pgsz_bitmap);
+
+		return pg_bit;
+	}
+}
+
+void __rdma_block_iter_start(struct ib_block_iter *biter,
+			     struct scatterlist *sglist, unsigned int nents,
+			     unsigned long pgsz_bitmap)
+{
+	memset(biter, 0, sizeof(struct ib_block_iter));
+	biter->__sg = sglist;
+	biter->pgsz_bitmap = pgsz_bitmap;
+	biter->__sg_nents = nents;
+
+	/* Driver provides best block size to use */
+	if (hweight_long(pgsz_bitmap) == 1) {
+		biter->__pg_bit = __fls(pgsz_bitmap);
+	} else {
+		/* mixed block size support. compute best block size to use */
+		WARN_ON(!(pgsz_bitmap & GENMASK(PAGE_SHIFT, 0)));
+		biter->__sgl_head = &sglist[0];
+		biter->__mixed = true;
+	}
+}
+EXPORT_SYMBOL(__rdma_block_iter_start);
+
+bool __rdma_block_iter_next(struct ib_block_iter *biter)
+{
+	unsigned int block_offset;
+
+	if (!biter->__sg_nents || !biter->__sg)
+		return false;
+
+	biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
+	if (biter->__mixed)
+		biter->__pg_bit = rdma_find_mixed_pg_bit(biter);
+
+	block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
+	biter->__sg_advance += BIT_ULL(biter->__pg_bit) - block_offset;
+
+	if (biter->__sg_advance >= sg_dma_len(biter->__sg)) {
+		biter->__sg_advance = 0;
+		biter->__sg = sg_next(biter->__sg);
+		biter->__sg_nents--;
+	}
+
+	return true;
+}
+EXPORT_SYMBOL(__rdma_block_iter_next);
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 720ce23..3398348 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -2681,6 +2681,31 @@  struct ib_client {
 	u8 no_kverbs_req:1;
 };
 
+/*
+ * IB block DMA iterator
+ *
+ * Iterates the DMA-mapped SGL in contiguous memory blocks aligned
+ * to a HW supported page size.
+ */
+struct ib_block_iter {
+	unsigned long pgsz_bitmap;	/* bitmap of supported HW page sizes.
+					 * HW that can handle only blocks of a
+					 * single page size must just provide
+					 * the best page size to use in pgsz_bitmap
+					 */
+
+	/* internal states */
+	struct scatterlist *__sg;	/* sg holding the current aligned block */
+	struct scatterlist *__sgl_head;	/* scatterlist head */
+	dma_addr_t __dma_addr;		/* unaligned DMA address of this block */
+	unsigned int __sg_nents;	/* number of SG entries */
+	unsigned int __sg_advance;	/* number of bytes to advance in sg in next step */
+	unsigned int __pg_bit;		/* alignment of current block */
+	u8 __mixed;			/* HW supports single block size or mixed
+					 * block sizes
+					 */
+};
+
 struct ib_device *_ib_alloc_device(size_t size);
 #define ib_alloc_device(drv_struct, member)                                    \
 	container_of(_ib_alloc_device(sizeof(struct drv_struct) +              \
@@ -2701,6 +2726,38 @@  struct ib_client {
 int ib_register_client   (struct ib_client *client);
 void ib_unregister_client(struct ib_client *client);
 
+void __rdma_block_iter_start(struct ib_block_iter *biter,
+			     struct scatterlist *sglist,
+			     unsigned int nents,
+			     unsigned long pgsz_bitmap);
+bool __rdma_block_iter_next(struct ib_block_iter *biter);
+
+/**
+ * rdma_block_iter_dma_address - get the aligned dma address of the current
+ * block held by the block iterator.
+ * @biter: block iterator holding the memory block
+ */
+static inline dma_addr_t
+rdma_block_iter_dma_address(struct ib_block_iter *biter)
+{
+	return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
+}
+
+/**
+ * rdma_for_each_block - iterate over contiguous memory blocks of the sg list
+ * @sglist: sglist to iterate over
+ * @biter: block iterator holding the memory block
+ * @nents: maximum number of sg entries to iterate over
+ * @pgsz_bitmap: bitmap of HW supported page sizes
+ *
+ * Callers may use rdma_block_iter_dma_address() to get each
+ * blocks aligned DMA address.
+ */
+#define rdma_for_each_block(sglist, biter, nents, pgsz_bitmap)	\
+	for (__rdma_block_iter_start(biter, sglist, nents,	\
+				     pgsz_bitmap);		\
+	     __rdma_block_iter_next(biter);)
+
 /**
  * ib_get_client_data - Get IB client context
  * @device:Device to get context for