diff mbox series

[for-next,1/2] RDMA/bnxt_re: Code refactor while populating user MRs

Message ID 1610001559-13193-2-git-send-email-selvin.xavier@broadcom.com (mailing list archive)
State Superseded
Headers show
Series RDMA/bnxt_re: Allow bigger user MRs | expand

Commit Message

Selvin Xavier Jan. 7, 2021, 6:39 a.m. UTC 
Refactor code that populates MR page buffer list. Instead of allocating
a pbl_tbl to hold the buffer list, pass the struct ib_umem directly
to bnxt_qplib_alloc_init_hwq as done for other user space memories.
Fix the PBL level to handle the above mentioned change.

Also, remove an unwanted flag from the input to bnxt_qplib_reg_mr
function.

Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
---
 drivers/infiniband/hw/bnxt_re/ib_verbs.c | 39 +++++---------------------------
 drivers/infiniband/hw/bnxt_re/qplib_sp.c | 17 +++++---------
 drivers/infiniband/hw/bnxt_re/qplib_sp.h |  2 +-
 3 files changed, 13 insertions(+), 45 deletions(-)

Comments

kernel test robot Jan. 7, 2021, 8:03 a.m. UTC | #1 
Hi Selvin,

I love your patch! Perhaps something to improve:

[auto build test WARNING on rdma/for-next]
[also build test WARNING on v5.11-rc2 next-20210104]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Selvin-Xavier/RDMA-bnxt_re-Allow-bigger-user-MRs/20210107-145124
base:   https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git for-next
config: ia64-allmodconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/0day-ci/linux/commit/a37827df6f226ed7b3fd1a3ccc4c47f5b893adf9
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Selvin-Xavier/RDMA-bnxt_re-Allow-bigger-user-MRs/20210107-145124
        git checkout a37827df6f226ed7b3fd1a3ccc4c47f5b893adf9
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=ia64 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

   drivers/infiniband/hw/bnxt_re/ib_verbs.c: In function 'bnxt_re_create_fence_mr':
>> drivers/infiniband/hw/bnxt_re/ib_verbs.c:472:6: warning: variable 'pbl_tbl' set but not used [-Wunused-but-set-variable]
     472 |  u64 pbl_tbl;
         |      ^~~~~~~

Kconfig warnings: (for reference only)
   WARNING: unmet direct dependencies detected for FRAME_POINTER
   Depends on DEBUG_KERNEL && (M68K || UML || SUPERH) || ARCH_WANT_FRAME_POINTERS
   Selected by
   - FAULT_INJECTION_STACKTRACE_FILTER && FAULT_INJECTION_DEBUG_FS && STACKTRACE_SUPPORT && !X86_64 && !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM && !ARC && !X86


vim +/pbl_tbl +472 drivers/infiniband/hw/bnxt_re/ib_verbs.c

9152e0b722b290 Eddie Wai     2017-06-14  462  
9152e0b722b290 Eddie Wai     2017-06-14  463  static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
9152e0b722b290 Eddie Wai     2017-06-14  464  {
9152e0b722b290 Eddie Wai     2017-06-14  465  	int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
9152e0b722b290 Eddie Wai     2017-06-14  466  	struct bnxt_re_fence_data *fence = &pd->fence;
9152e0b722b290 Eddie Wai     2017-06-14  467  	struct bnxt_re_dev *rdev = pd->rdev;
9152e0b722b290 Eddie Wai     2017-06-14  468  	struct device *dev = &rdev->en_dev->pdev->dev;
9152e0b722b290 Eddie Wai     2017-06-14  469  	struct bnxt_re_mr *mr = NULL;
9152e0b722b290 Eddie Wai     2017-06-14  470  	dma_addr_t dma_addr = 0;
9152e0b722b290 Eddie Wai     2017-06-14  471  	struct ib_mw *mw;
9152e0b722b290 Eddie Wai     2017-06-14 @472  	u64 pbl_tbl;
9152e0b722b290 Eddie Wai     2017-06-14  473  	int rc;
9152e0b722b290 Eddie Wai     2017-06-14  474  
9152e0b722b290 Eddie Wai     2017-06-14  475  	dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
9152e0b722b290 Eddie Wai     2017-06-14  476  				  DMA_BIDIRECTIONAL);
9152e0b722b290 Eddie Wai     2017-06-14  477  	rc = dma_mapping_error(dev, dma_addr);
9152e0b722b290 Eddie Wai     2017-06-14  478  	if (rc) {
6ccad8483b2866 Devesh Sharma 2020-02-15  479  		ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
9152e0b722b290 Eddie Wai     2017-06-14  480  		rc = -EIO;
9152e0b722b290 Eddie Wai     2017-06-14  481  		fence->dma_addr = 0;
9152e0b722b290 Eddie Wai     2017-06-14  482  		goto fail;
9152e0b722b290 Eddie Wai     2017-06-14  483  	}
9152e0b722b290 Eddie Wai     2017-06-14  484  	fence->dma_addr = dma_addr;
9152e0b722b290 Eddie Wai     2017-06-14  485  
9152e0b722b290 Eddie Wai     2017-06-14  486  	/* Allocate a MR */
9152e0b722b290 Eddie Wai     2017-06-14  487  	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
9152e0b722b290 Eddie Wai     2017-06-14  488  	if (!mr) {
9152e0b722b290 Eddie Wai     2017-06-14  489  		rc = -ENOMEM;
9152e0b722b290 Eddie Wai     2017-06-14  490  		goto fail;
9152e0b722b290 Eddie Wai     2017-06-14  491  	}
9152e0b722b290 Eddie Wai     2017-06-14  492  	fence->mr = mr;
9152e0b722b290 Eddie Wai     2017-06-14  493  	mr->rdev = rdev;
9152e0b722b290 Eddie Wai     2017-06-14  494  	mr->qplib_mr.pd = &pd->qplib_pd;
9152e0b722b290 Eddie Wai     2017-06-14  495  	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
9152e0b722b290 Eddie Wai     2017-06-14  496  	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
9152e0b722b290 Eddie Wai     2017-06-14  497  	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
9152e0b722b290 Eddie Wai     2017-06-14  498  	if (rc) {
6ccad8483b2866 Devesh Sharma 2020-02-15  499  		ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
9152e0b722b290 Eddie Wai     2017-06-14  500  		goto fail;
9152e0b722b290 Eddie Wai     2017-06-14  501  	}
9152e0b722b290 Eddie Wai     2017-06-14  502  
9152e0b722b290 Eddie Wai     2017-06-14  503  	/* Register MR */
9152e0b722b290 Eddie Wai     2017-06-14  504  	mr->ib_mr.lkey = mr->qplib_mr.lkey;
9152e0b722b290 Eddie Wai     2017-06-14  505  	mr->qplib_mr.va = (u64)(unsigned long)fence->va;
9152e0b722b290 Eddie Wai     2017-06-14  506  	mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
9152e0b722b290 Eddie Wai     2017-06-14  507  	pbl_tbl = dma_addr;
a37827df6f226e Selvin Xavier 2021-01-06  508  	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
a37827df6f226e Selvin Xavier 2021-01-06  509  			       BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
9152e0b722b290 Eddie Wai     2017-06-14  510  	if (rc) {
6ccad8483b2866 Devesh Sharma 2020-02-15  511  		ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
9152e0b722b290 Eddie Wai     2017-06-14  512  		goto fail;
9152e0b722b290 Eddie Wai     2017-06-14  513  	}
9152e0b722b290 Eddie Wai     2017-06-14  514  	mr->ib_mr.rkey = mr->qplib_mr.rkey;
9152e0b722b290 Eddie Wai     2017-06-14  515  
9152e0b722b290 Eddie Wai     2017-06-14  516  	/* Create a fence MW only for kernel consumers */
9152e0b722b290 Eddie Wai     2017-06-14  517  	mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
653f0a71daf1a7 Dan Carpenter 2017-07-10  518  	if (IS_ERR(mw)) {
6ccad8483b2866 Devesh Sharma 2020-02-15  519  		ibdev_err(&rdev->ibdev,
9152e0b722b290 Eddie Wai     2017-06-14  520  			  "Failed to create fence-MW for PD: %p\n", pd);
653f0a71daf1a7 Dan Carpenter 2017-07-10  521  		rc = PTR_ERR(mw);
9152e0b722b290 Eddie Wai     2017-06-14  522  		goto fail;
9152e0b722b290 Eddie Wai     2017-06-14  523  	}
9152e0b722b290 Eddie Wai     2017-06-14  524  	fence->mw = mw;
9152e0b722b290 Eddie Wai     2017-06-14  525  
9152e0b722b290 Eddie Wai     2017-06-14  526  	bnxt_re_create_fence_wqe(pd);
9152e0b722b290 Eddie Wai     2017-06-14  527  	return 0;
9152e0b722b290 Eddie Wai     2017-06-14  528  
9152e0b722b290 Eddie Wai     2017-06-14  529  fail:
9152e0b722b290 Eddie Wai     2017-06-14  530  	bnxt_re_destroy_fence_mr(pd);
9152e0b722b290 Eddie Wai     2017-06-14  531  	return rc;
9152e0b722b290 Eddie Wai     2017-06-14  532  }
9152e0b722b290 Eddie Wai     2017-06-14  533  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
diff mbox series

Patch

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 401bdc9..6b5b8d1 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -505,8 +505,8 @@  static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
 	mr->qplib_mr.va = (u64)(unsigned long)fence->va;
 	mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
 	pbl_tbl = dma_addr;
-	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl_tbl,
-			       BNXT_RE_FENCE_PBL_SIZE, false, PAGE_SIZE);
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
+			       BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
 	if (rc) {
 		ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
 		goto fail;
@@ -3589,7 +3589,6 @@  struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 	struct bnxt_re_dev *rdev = pd->rdev;
 	struct bnxt_re_mr *mr;
-	u64 pbl = 0;
 	int rc;
 
 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
@@ -3608,7 +3607,7 @@  struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
 
 	mr->qplib_mr.hwq.level = PBL_LVL_MAX;
 	mr->qplib_mr.total_size = -1; /* Infinte length */
-	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false,
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0,
 			       PAGE_SIZE);
 	if (rc)
 		goto fail_mr;
@@ -3779,19 +3778,6 @@  int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
 	return rc;
 }
 
-static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig,
-			     int page_shift)
-{
-	u64 *pbl_tbl = pbl_tbl_orig;
-	u64 page_size =  BIT_ULL(page_shift);
-	struct ib_block_iter biter;
-
-	rdma_umem_for_each_dma_block(umem, &biter, page_size)
-		*pbl_tbl++ = rdma_block_iter_dma_address(&biter);
-
-	return pbl_tbl - pbl_tbl_orig;
-}
-
 /* uverbs */
 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 				  u64 virt_addr, int mr_access_flags,
@@ -3801,7 +3787,6 @@  struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 	struct bnxt_re_dev *rdev = pd->rdev;
 	struct bnxt_re_mr *mr;
 	struct ib_umem *umem;
-	u64 *pbl_tbl = NULL;
 	unsigned long page_size;
 	int umem_pgs, rc;
 
@@ -3855,30 +3840,18 @@  struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 	}
 
 	umem_pgs = ib_umem_num_dma_blocks(umem, page_size);
-	pbl_tbl = kcalloc(umem_pgs, sizeof(*pbl_tbl), GFP_KERNEL);
-	if (!pbl_tbl) {
-		rc = -ENOMEM;
-		goto free_umem;
-	}
-
-	/* Map umem buf ptrs to the PBL */
-	umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, order_base_2(page_size));
-	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl,
-			       umem_pgs, false, page_size);
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem,
+			       umem_pgs, page_size);
 	if (rc) {
 		ibdev_err(&rdev->ibdev, "Failed to register user MR");
-		goto fail;
+		goto free_umem;
 	}
 
-	kfree(pbl_tbl);
-
 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
 	mr->ib_mr.rkey = mr->qplib_mr.lkey;
 	atomic_inc(&rdev->mr_count);
 
 	return &mr->ib_mr;
-fail:
-	kfree(pbl_tbl);
 free_umem:
 	ib_umem_release(umem);
 free_mrw:
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index 6316179..22cb46a 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -650,16 +650,15 @@  int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
 }
 
 int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
-		      u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size)
+		      struct ib_umem *umem, int num_pbls, u32 buf_pg_size)
 {
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 	struct bnxt_qplib_hwq_attr hwq_attr = {};
 	struct bnxt_qplib_sg_info sginfo = {};
 	struct creq_register_mr_resp resp;
 	struct cmdq_register_mr req;
-	int pg_ptrs, pages, i, rc;
 	u16 cmd_flags = 0, level;
-	dma_addr_t **pbl_ptr;
+	int pages, rc, pg_ptrs;
 	u32 pg_size;
 
 	if (num_pbls) {
@@ -683,9 +682,10 @@  int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
 		/* Use system PAGE_SIZE */
 		hwq_attr.res = res;
 		hwq_attr.depth = pages;
-		hwq_attr.stride = PAGE_SIZE;
+		hwq_attr.stride = buf_pg_size;
 		hwq_attr.type = HWQ_TYPE_MR;
 		hwq_attr.sginfo = &sginfo;
+		hwq_attr.sginfo->umem = umem;
 		hwq_attr.sginfo->npages = pages;
 		hwq_attr.sginfo->pgsize = PAGE_SIZE;
 		hwq_attr.sginfo->pgshft = PAGE_SHIFT;
@@ -695,11 +695,6 @@  int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
 				"SP: Reg MR memory allocation failed\n");
 			return -ENOMEM;
 		}
-		/* Write to the hwq */
-		pbl_ptr = (dma_addr_t **)mr->hwq.pbl_ptr;
-		for (i = 0; i < num_pbls; i++)
-			pbl_ptr[PTR_PG(i)][PTR_IDX(i)] =
-				(pbl_tbl[i] & PAGE_MASK) | PTU_PTE_VALID;
 	}
 
 	RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags);
@@ -711,7 +706,7 @@  int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
 		req.pbl = 0;
 		pg_size = PAGE_SIZE;
 	} else {
-		level = mr->hwq.level + 1;
+		level = mr->hwq.level;
 		req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
 	}
 	pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE;
@@ -728,7 +723,7 @@  int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
 	req.mr_size = cpu_to_le64(mr->total_size);
 
 	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-					  (void *)&resp, NULL, block);
+					  (void *)&resp, NULL, false);
 	if (rc)
 		goto fail;
 
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
index 967890c..bc22834 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
@@ -254,7 +254,7 @@  int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res,
 int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
 			 bool block);
 int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
-		      u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size);
+		      struct ib_umem *umem, int num_pbls, u32 buf_pg_size);
 int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr);
 int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res,
 				 struct bnxt_qplib_mrw *mr, int max);