Message ID | 1610001559-13193-2-git-send-email-selvin.xavier@broadcom.com (mailing list archive) |
---|---|
State | Superseded |
Headers | show |
Series | RDMA/bnxt_re: Allow bigger user MRs | expand |
Hi Selvin,
I love your patch! Perhaps something to improve:
[auto build test WARNING on rdma/for-next]
[also build test WARNING on v5.11-rc2 next-20210104]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Selvin-Xavier/RDMA-bnxt_re-Allow-bigger-user-MRs/20210107-145124
base: https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git for-next
config: ia64-allmodconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/a37827df6f226ed7b3fd1a3ccc4c47f5b893adf9
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Selvin-Xavier/RDMA-bnxt_re-Allow-bigger-user-MRs/20210107-145124
git checkout a37827df6f226ed7b3fd1a3ccc4c47f5b893adf9
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=ia64
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All warnings (new ones prefixed by >>):
drivers/infiniband/hw/bnxt_re/ib_verbs.c: In function 'bnxt_re_create_fence_mr':
>> drivers/infiniband/hw/bnxt_re/ib_verbs.c:472:6: warning: variable 'pbl_tbl' set but not used [-Wunused-but-set-variable]
472 | u64 pbl_tbl;
| ^~~~~~~
Kconfig warnings: (for reference only)
WARNING: unmet direct dependencies detected for FRAME_POINTER
Depends on DEBUG_KERNEL && (M68K || UML || SUPERH) || ARCH_WANT_FRAME_POINTERS
Selected by
- FAULT_INJECTION_STACKTRACE_FILTER && FAULT_INJECTION_DEBUG_FS && STACKTRACE_SUPPORT && !X86_64 && !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM && !ARC && !X86
vim +/pbl_tbl +472 drivers/infiniband/hw/bnxt_re/ib_verbs.c
9152e0b722b290 Eddie Wai 2017-06-14 462
9152e0b722b290 Eddie Wai 2017-06-14 463 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
9152e0b722b290 Eddie Wai 2017-06-14 464 {
9152e0b722b290 Eddie Wai 2017-06-14 465 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
9152e0b722b290 Eddie Wai 2017-06-14 466 struct bnxt_re_fence_data *fence = &pd->fence;
9152e0b722b290 Eddie Wai 2017-06-14 467 struct bnxt_re_dev *rdev = pd->rdev;
9152e0b722b290 Eddie Wai 2017-06-14 468 struct device *dev = &rdev->en_dev->pdev->dev;
9152e0b722b290 Eddie Wai 2017-06-14 469 struct bnxt_re_mr *mr = NULL;
9152e0b722b290 Eddie Wai 2017-06-14 470 dma_addr_t dma_addr = 0;
9152e0b722b290 Eddie Wai 2017-06-14 471 struct ib_mw *mw;
9152e0b722b290 Eddie Wai 2017-06-14 @472 u64 pbl_tbl;
9152e0b722b290 Eddie Wai 2017-06-14 473 int rc;
9152e0b722b290 Eddie Wai 2017-06-14 474
9152e0b722b290 Eddie Wai 2017-06-14 475 dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
9152e0b722b290 Eddie Wai 2017-06-14 476 DMA_BIDIRECTIONAL);
9152e0b722b290 Eddie Wai 2017-06-14 477 rc = dma_mapping_error(dev, dma_addr);
9152e0b722b290 Eddie Wai 2017-06-14 478 if (rc) {
6ccad8483b2866 Devesh Sharma 2020-02-15 479 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
9152e0b722b290 Eddie Wai 2017-06-14 480 rc = -EIO;
9152e0b722b290 Eddie Wai 2017-06-14 481 fence->dma_addr = 0;
9152e0b722b290 Eddie Wai 2017-06-14 482 goto fail;
9152e0b722b290 Eddie Wai 2017-06-14 483 }
9152e0b722b290 Eddie Wai 2017-06-14 484 fence->dma_addr = dma_addr;
9152e0b722b290 Eddie Wai 2017-06-14 485
9152e0b722b290 Eddie Wai 2017-06-14 486 /* Allocate a MR */
9152e0b722b290 Eddie Wai 2017-06-14 487 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
9152e0b722b290 Eddie Wai 2017-06-14 488 if (!mr) {
9152e0b722b290 Eddie Wai 2017-06-14 489 rc = -ENOMEM;
9152e0b722b290 Eddie Wai 2017-06-14 490 goto fail;
9152e0b722b290 Eddie Wai 2017-06-14 491 }
9152e0b722b290 Eddie Wai 2017-06-14 492 fence->mr = mr;
9152e0b722b290 Eddie Wai 2017-06-14 493 mr->rdev = rdev;
9152e0b722b290 Eddie Wai 2017-06-14 494 mr->qplib_mr.pd = &pd->qplib_pd;
9152e0b722b290 Eddie Wai 2017-06-14 495 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
9152e0b722b290 Eddie Wai 2017-06-14 496 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
9152e0b722b290 Eddie Wai 2017-06-14 497 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
9152e0b722b290 Eddie Wai 2017-06-14 498 if (rc) {
6ccad8483b2866 Devesh Sharma 2020-02-15 499 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
9152e0b722b290 Eddie Wai 2017-06-14 500 goto fail;
9152e0b722b290 Eddie Wai 2017-06-14 501 }
9152e0b722b290 Eddie Wai 2017-06-14 502
9152e0b722b290 Eddie Wai 2017-06-14 503 /* Register MR */
9152e0b722b290 Eddie Wai 2017-06-14 504 mr->ib_mr.lkey = mr->qplib_mr.lkey;
9152e0b722b290 Eddie Wai 2017-06-14 505 mr->qplib_mr.va = (u64)(unsigned long)fence->va;
9152e0b722b290 Eddie Wai 2017-06-14 506 mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
9152e0b722b290 Eddie Wai 2017-06-14 507 pbl_tbl = dma_addr;
a37827df6f226e Selvin Xavier 2021-01-06 508 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
a37827df6f226e Selvin Xavier 2021-01-06 509 BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
9152e0b722b290 Eddie Wai 2017-06-14 510 if (rc) {
6ccad8483b2866 Devesh Sharma 2020-02-15 511 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
9152e0b722b290 Eddie Wai 2017-06-14 512 goto fail;
9152e0b722b290 Eddie Wai 2017-06-14 513 }
9152e0b722b290 Eddie Wai 2017-06-14 514 mr->ib_mr.rkey = mr->qplib_mr.rkey;
9152e0b722b290 Eddie Wai 2017-06-14 515
9152e0b722b290 Eddie Wai 2017-06-14 516 /* Create a fence MW only for kernel consumers */
9152e0b722b290 Eddie Wai 2017-06-14 517 mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
653f0a71daf1a7 Dan Carpenter 2017-07-10 518 if (IS_ERR(mw)) {
6ccad8483b2866 Devesh Sharma 2020-02-15 519 ibdev_err(&rdev->ibdev,
9152e0b722b290 Eddie Wai 2017-06-14 520 "Failed to create fence-MW for PD: %p\n", pd);
653f0a71daf1a7 Dan Carpenter 2017-07-10 521 rc = PTR_ERR(mw);
9152e0b722b290 Eddie Wai 2017-06-14 522 goto fail;
9152e0b722b290 Eddie Wai 2017-06-14 523 }
9152e0b722b290 Eddie Wai 2017-06-14 524 fence->mw = mw;
9152e0b722b290 Eddie Wai 2017-06-14 525
9152e0b722b290 Eddie Wai 2017-06-14 526 bnxt_re_create_fence_wqe(pd);
9152e0b722b290 Eddie Wai 2017-06-14 527 return 0;
9152e0b722b290 Eddie Wai 2017-06-14 528
9152e0b722b290 Eddie Wai 2017-06-14 529 fail:
9152e0b722b290 Eddie Wai 2017-06-14 530 bnxt_re_destroy_fence_mr(pd);
9152e0b722b290 Eddie Wai 2017-06-14 531 return rc;
9152e0b722b290 Eddie Wai 2017-06-14 532 }
9152e0b722b290 Eddie Wai 2017-06-14 533
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 401bdc9..6b5b8d1 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -505,8 +505,8 @@ static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) mr->qplib_mr.va = (u64)(unsigned long)fence->va; mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES; pbl_tbl = dma_addr; - rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl_tbl, - BNXT_RE_FENCE_PBL_SIZE, false, PAGE_SIZE); + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, + BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE); if (rc) { ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n"); goto fail; @@ -3589,7 +3589,6 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_mr *mr; - u64 pbl = 0; int rc; mr = kzalloc(sizeof(*mr), GFP_KERNEL); @@ -3608,7 +3607,7 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) mr->qplib_mr.hwq.level = PBL_LVL_MAX; mr->qplib_mr.total_size = -1; /* Infinte length */ - rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false, + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0, PAGE_SIZE); if (rc) goto fail_mr; @@ -3779,19 +3778,6 @@ int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) return rc; } -static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig, - int page_shift) -{ - u64 *pbl_tbl = pbl_tbl_orig; - u64 page_size = BIT_ULL(page_shift); - struct ib_block_iter biter; - - rdma_umem_for_each_dma_block(umem, &biter, page_size) - *pbl_tbl++ = rdma_block_iter_dma_address(&biter); - - return pbl_tbl - pbl_tbl_orig; -} - /* uverbs */ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, u64 virt_addr, int mr_access_flags, @@ -3801,7 +3787,6 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_mr *mr; struct ib_umem *umem; - u64 *pbl_tbl = NULL; unsigned long page_size; int umem_pgs, rc; @@ -3855,30 +3840,18 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, } umem_pgs = ib_umem_num_dma_blocks(umem, page_size); - pbl_tbl = kcalloc(umem_pgs, sizeof(*pbl_tbl), GFP_KERNEL); - if (!pbl_tbl) { - rc = -ENOMEM; - goto free_umem; - } - - /* Map umem buf ptrs to the PBL */ - umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, order_base_2(page_size)); - rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl, - umem_pgs, false, page_size); + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem, + umem_pgs, page_size); if (rc) { ibdev_err(&rdev->ibdev, "Failed to register user MR"); - goto fail; + goto free_umem; } - kfree(pbl_tbl); - mr->ib_mr.lkey = mr->qplib_mr.lkey; mr->ib_mr.rkey = mr->qplib_mr.lkey; atomic_inc(&rdev->mr_count); return &mr->ib_mr; -fail: - kfree(pbl_tbl); free_umem: ib_umem_release(umem); free_mrw: diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c index 6316179..22cb46a 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c @@ -650,16 +650,15 @@ int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, } int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, - u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size) + struct ib_umem *umem, int num_pbls, u32 buf_pg_size) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_sg_info sginfo = {}; struct creq_register_mr_resp resp; struct cmdq_register_mr req; - int pg_ptrs, pages, i, rc; u16 cmd_flags = 0, level; - dma_addr_t **pbl_ptr; + int pages, rc, pg_ptrs; u32 pg_size; if (num_pbls) { @@ -683,9 +682,10 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, /* Use system PAGE_SIZE */ hwq_attr.res = res; hwq_attr.depth = pages; - hwq_attr.stride = PAGE_SIZE; + hwq_attr.stride = buf_pg_size; hwq_attr.type = HWQ_TYPE_MR; hwq_attr.sginfo = &sginfo; + hwq_attr.sginfo->umem = umem; hwq_attr.sginfo->npages = pages; hwq_attr.sginfo->pgsize = PAGE_SIZE; hwq_attr.sginfo->pgshft = PAGE_SHIFT; @@ -695,11 +695,6 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, "SP: Reg MR memory allocation failed\n"); return -ENOMEM; } - /* Write to the hwq */ - pbl_ptr = (dma_addr_t **)mr->hwq.pbl_ptr; - for (i = 0; i < num_pbls; i++) - pbl_ptr[PTR_PG(i)][PTR_IDX(i)] = - (pbl_tbl[i] & PAGE_MASK) | PTU_PTE_VALID; } RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags); @@ -711,7 +706,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, req.pbl = 0; pg_size = PAGE_SIZE; } else { - level = mr->hwq.level + 1; + level = mr->hwq.level; req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); } pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE; @@ -728,7 +723,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, req.mr_size = cpu_to_le64(mr->total_size); rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, - (void *)&resp, NULL, block); + (void *)&resp, NULL, false); if (rc) goto fail; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index 967890c..bc22834 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -254,7 +254,7 @@ int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, bool block); int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, - u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size); + struct ib_umem *umem, int num_pbls, u32 buf_pg_size); int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr); int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, int max);