diff mbox series

RDMA/rxe: Bump up default maximum values used via uverbs

Message ID 20210713083647.393983-1-Rao.Shoaib@oracle.com (mailing list archive)
State Superseded
Headers show
Series RDMA/rxe: Bump up default maximum values used via uverbs | expand

Commit Message

Shoaib Rao July 13, 2021, 8:36 a.m. UTC 
From: Rao Shoaib <rao.shoaib@oracle.com>

In our internal testing we have found that the
current maximum are too smalls. Ideally there should
be no limits but currently maximum values are reported
via ibv_query_device, so we have to keep maximum values
but they have been made suffiently large.

Signed-off-by: Rao Shoaib <rao.shoaib@oracle.com>
---
 drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

Comments

Zhu Yanjun July 13, 2021, 8:40 a.m. UTC | #1 
On Tue, Jul 13, 2021 at 4:37 PM Rao Shoaib <Rao.Shoaib@oracle.com> wrote:
>
> From: Rao Shoaib <rao.shoaib@oracle.com>
>
> In our internal testing we have found that the
> current maximum are too smalls. Ideally there should
> be no limits but currently maximum values are reported
> via ibv_query_device, so we have to keep maximum values
> but they have been made suffiently large.
>
> Signed-off-by: Rao Shoaib <rao.shoaib@oracle.com>
> ---
>  drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------
>  1 file changed, 14 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
> index 742e6ec93686..66a948adb1e1 100644
> --- a/drivers/infiniband/sw/rxe/rxe_param.h
> +++ b/drivers/infiniband/sw/rxe/rxe_param.h
> @@ -9,6 +9,8 @@
>
>  #include <uapi/rdma/rdma_user_rxe.h>
>
> +#define DEFAULT_MAX_VALUE (1 << 20)

How do you get this DEFAULT_MAX_VALUE? From spec? or others?

Zhu Yanjun

> +
>  static inline enum ib_mtu rxe_mtu_int_to_enum(int mtu)
>  {
>         if (mtu < 256)
> @@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
>  enum rxe_device_param {
>         RXE_MAX_MR_SIZE                 = -1ull,
>         RXE_PAGE_SIZE_CAP               = 0xfffff000,
> -       RXE_MAX_QP_WR                   = 0x4000,
> +       RXE_MAX_QP_WR                   = DEFAULT_MAX_VALUE,
>         RXE_DEVICE_CAP_FLAGS            = IB_DEVICE_BAD_PKEY_CNTR
>                                         | IB_DEVICE_BAD_QKEY_CNTR
>                                         | IB_DEVICE_AUTO_PATH_MIG
> @@ -58,40 +60,40 @@ enum rxe_device_param {
>         RXE_MAX_INLINE_DATA             = RXE_MAX_WQE_SIZE -
>                                           sizeof(struct rxe_send_wqe),
>         RXE_MAX_SGE_RD                  = 32,
> -       RXE_MAX_CQ                      = 16384,
> +       RXE_MAX_CQ                      = DEFAULT_MAX_VALUE,
>         RXE_MAX_LOG_CQE                 = 15,
> -       RXE_MAX_PD                      = 0x7ffc,
> +       RXE_MAX_PD                      = DEFAULT_MAX_VALUE,
>         RXE_MAX_QP_RD_ATOM              = 128,
>         RXE_MAX_RES_RD_ATOM             = 0x3f000,
>         RXE_MAX_QP_INIT_RD_ATOM         = 128,
>         RXE_MAX_MCAST_GRP               = 8192,
>         RXE_MAX_MCAST_QP_ATTACH         = 56,
>         RXE_MAX_TOT_MCAST_QP_ATTACH     = 0x70000,
> -       RXE_MAX_AH                      = 100,
> -       RXE_MAX_SRQ_WR                  = 0x4000,
> +       RXE_MAX_AH                      = DEFAULT_MAX_VALUE,
> +       RXE_MAX_SRQ_WR                  = DEFAULT_MAX_VALUE,
>         RXE_MIN_SRQ_WR                  = 1,
>         RXE_MAX_SRQ_SGE                 = 27,
>         RXE_MIN_SRQ_SGE                 = 1,
>         RXE_MAX_FMR_PAGE_LIST_LEN       = 512,
> -       RXE_MAX_PKEYS                   = 1,
> +       RXE_MAX_PKEYS                   = DEFAULT_MAX_VALUE,
>         RXE_LOCAL_CA_ACK_DELAY          = 15,
>
> -       RXE_MAX_UCONTEXT                = 512,
> +       RXE_MAX_UCONTEXT                = DEFAULT_MAX_VALUE,
>
>         RXE_NUM_PORT                    = 1,
>
> -       RXE_MAX_QP                      = 0x10000,
> +       RXE_MAX_QP                      = DEFAULT_MAX_VALUE,
>         RXE_MIN_QP_INDEX                = 16,
> -       RXE_MAX_QP_INDEX                = 0x00020000,
> +       RXE_MAX_QP_INDEX                = 0x00040000,
>
> -       RXE_MAX_SRQ                     = 0x00001000,
> +       RXE_MAX_SRQ                     = DEFAULT_MAX_VALUE,
>         RXE_MIN_SRQ_INDEX               = 0x00020001,
>         RXE_MAX_SRQ_INDEX               = 0x00040000,
>
> -       RXE_MAX_MR                      = 0x00001000,
> +       RXE_MAX_MR                      = DEFAULT_MAX_VALUE,
>         RXE_MAX_MW                      = 0x00001000,
>         RXE_MIN_MR_INDEX                = 0x00000001,
> -       RXE_MAX_MR_INDEX                = 0x00010000,
> +       RXE_MAX_MR_INDEX                = 0x00040000,
>         RXE_MIN_MW_INDEX                = 0x00010001,
>         RXE_MAX_MW_INDEX                = 0x00020000,
>
> --
> 2.27.0
>
Shoaib Rao July 13, 2021, 4:38 p.m. UTC | #2 
On 7/13/21 1:40 AM, Zhu Yanjun wrote:
> On Tue, Jul 13, 2021 at 4:37 PM Rao Shoaib <Rao.Shoaib@oracle.com> wrote:
>> From: Rao Shoaib <rao.shoaib@oracle.com>
>>
>> In our internal testing we have found that the
>> current maximum are too smalls. Ideally there should
>> be no limits but currently maximum values are reported
>> via ibv_query_device, so we have to keep maximum values
>> but they have been made suffiently large.
>>
>> Signed-off-by: Rao Shoaib <rao.shoaib@oracle.com>
>> ---
>>   drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------
>>   1 file changed, 14 insertions(+), 12 deletions(-)
>>
>> diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
>> index 742e6ec93686..66a948adb1e1 100644
>> --- a/drivers/infiniband/sw/rxe/rxe_param.h
>> +++ b/drivers/infiniband/sw/rxe/rxe_param.h
>> @@ -9,6 +9,8 @@
>>
>>   #include <uapi/rdma/rdma_user_rxe.h>
>>
>> +#define DEFAULT_MAX_VALUE (1 << 20)
> How do you get this DEFAULT_MAX_VALUE? From spec? or others?
>
> Zhu Yanjun

The spec does not specify any limits. The value was proposed as part of 
discussion on my previous patch.

Shoaib


>
>> +
>>   static inline enum ib_mtu rxe_mtu_int_to_enum(int mtu)
>>   {
>>          if (mtu < 256)
>> @@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
>>   enum rxe_device_param {
>>          RXE_MAX_MR_SIZE                 = -1ull,
>>          RXE_PAGE_SIZE_CAP               = 0xfffff000,
>> -       RXE_MAX_QP_WR                   = 0x4000,
>> +       RXE_MAX_QP_WR                   = DEFAULT_MAX_VALUE,
>>          RXE_DEVICE_CAP_FLAGS            = IB_DEVICE_BAD_PKEY_CNTR
>>                                          | IB_DEVICE_BAD_QKEY_CNTR
>>                                          | IB_DEVICE_AUTO_PATH_MIG
>> @@ -58,40 +60,40 @@ enum rxe_device_param {
>>          RXE_MAX_INLINE_DATA             = RXE_MAX_WQE_SIZE -
>>                                            sizeof(struct rxe_send_wqe),
>>          RXE_MAX_SGE_RD                  = 32,
>> -       RXE_MAX_CQ                      = 16384,
>> +       RXE_MAX_CQ                      = DEFAULT_MAX_VALUE,
>>          RXE_MAX_LOG_CQE                 = 15,
>> -       RXE_MAX_PD                      = 0x7ffc,
>> +       RXE_MAX_PD                      = DEFAULT_MAX_VALUE,
>>          RXE_MAX_QP_RD_ATOM              = 128,
>>          RXE_MAX_RES_RD_ATOM             = 0x3f000,
>>          RXE_MAX_QP_INIT_RD_ATOM         = 128,
>>          RXE_MAX_MCAST_GRP               = 8192,
>>          RXE_MAX_MCAST_QP_ATTACH         = 56,
>>          RXE_MAX_TOT_MCAST_QP_ATTACH     = 0x70000,
>> -       RXE_MAX_AH                      = 100,
>> -       RXE_MAX_SRQ_WR                  = 0x4000,
>> +       RXE_MAX_AH                      = DEFAULT_MAX_VALUE,
>> +       RXE_MAX_SRQ_WR                  = DEFAULT_MAX_VALUE,
>>          RXE_MIN_SRQ_WR                  = 1,
>>          RXE_MAX_SRQ_SGE                 = 27,
>>          RXE_MIN_SRQ_SGE                 = 1,
>>          RXE_MAX_FMR_PAGE_LIST_LEN       = 512,
>> -       RXE_MAX_PKEYS                   = 1,
>> +       RXE_MAX_PKEYS                   = DEFAULT_MAX_VALUE,
>>          RXE_LOCAL_CA_ACK_DELAY          = 15,
>>
>> -       RXE_MAX_UCONTEXT                = 512,
>> +       RXE_MAX_UCONTEXT                = DEFAULT_MAX_VALUE,
>>
>>          RXE_NUM_PORT                    = 1,
>>
>> -       RXE_MAX_QP                      = 0x10000,
>> +       RXE_MAX_QP                      = DEFAULT_MAX_VALUE,
>>          RXE_MIN_QP_INDEX                = 16,
>> -       RXE_MAX_QP_INDEX                = 0x00020000,
>> +       RXE_MAX_QP_INDEX                = 0x00040000,
>>
>> -       RXE_MAX_SRQ                     = 0x00001000,
>> +       RXE_MAX_SRQ                     = DEFAULT_MAX_VALUE,
>>          RXE_MIN_SRQ_INDEX               = 0x00020001,
>>          RXE_MAX_SRQ_INDEX               = 0x00040000,
>>
>> -       RXE_MAX_MR                      = 0x00001000,
>> +       RXE_MAX_MR                      = DEFAULT_MAX_VALUE,
>>          RXE_MAX_MW                      = 0x00001000,
>>          RXE_MIN_MR_INDEX                = 0x00000001,
>> -       RXE_MAX_MR_INDEX                = 0x00010000,
>> +       RXE_MAX_MR_INDEX                = 0x00040000,
>>          RXE_MIN_MW_INDEX                = 0x00010001,
>>          RXE_MAX_MW_INDEX                = 0x00020000,
>>
>> --
>> 2.27.0
>>
kernel test robot July 13, 2021, 4:54 p.m. UTC | #3 
Hi Rao,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on rdma/for-next]
[also build test WARNING on v5.14-rc1 next-20210713]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Rao-Shoaib/RDMA-rxe-Bump-up-default-maximum-values-used-via-uverbs/20210713-163821
base:   https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git for-next
config: x86_64-allyesconfig (attached as .config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce (this is a W=1 build):
        # https://github.com/0day-ci/linux/commit/823f0c9f1aff85ad2abeed713bf89fad22254f24
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Rao-Shoaib/RDMA-rxe-Bump-up-default-maximum-values-used-via-uverbs/20210713-163821
        git checkout 823f0c9f1aff85ad2abeed713bf89fad22254f24
        # save the attached .config to linux build tree
        make W=1 ARCH=x86_64 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

   drivers/infiniband/sw/rxe/rxe.c: In function 'rxe_init_device_param':
>> drivers/infiniband/sw/rxe/rxe.c:71:26: warning: unsigned conversion from 'int' to 'u16' {aka 'short unsigned int'} changes value from '1048576' to '0' [-Woverflow]
      71 |  rxe->attr.max_pkeys   = RXE_MAX_PKEYS;
         |                          ^~~~~~~~~~~~~

Kconfig warnings: (for reference only)
   WARNING: unmet direct dependencies detected for PHY_SPARX5_SERDES
   Depends on (ARCH_SPARX5 || COMPILE_TEST && OF && HAS_IOMEM
   Selected by
   - SPARX5_SWITCH && NETDEVICES && ETHERNET && NET_VENDOR_MICROCHIP && NET_SWITCHDEV && HAS_IOMEM


vim +71 drivers/infiniband/sw/rxe/rxe.c

8700e3e7c4857d Moni Shoua        2016-06-16  39  
8700e3e7c4857d Moni Shoua        2016-06-16  40  /* initialize rxe device parameters */
befd8d98f230d9 Zhu Yanjun        2018-03-07  41  static void rxe_init_device_param(struct rxe_dev *rxe)
8700e3e7c4857d Moni Shoua        2016-06-16  42  {
8700e3e7c4857d Moni Shoua        2016-06-16  43  	rxe->max_inline_data			= RXE_MAX_INLINE_DATA;
8700e3e7c4857d Moni Shoua        2016-06-16  44  
0184afd15a141d Zhu Yanjun        2020-04-06  45  	rxe->attr.vendor_id			= RXE_VENDOR_ID;
8700e3e7c4857d Moni Shoua        2016-06-16  46  	rxe->attr.max_mr_size			= RXE_MAX_MR_SIZE;
8700e3e7c4857d Moni Shoua        2016-06-16  47  	rxe->attr.page_size_cap			= RXE_PAGE_SIZE_CAP;
8700e3e7c4857d Moni Shoua        2016-06-16  48  	rxe->attr.max_qp			= RXE_MAX_QP;
8700e3e7c4857d Moni Shoua        2016-06-16  49  	rxe->attr.max_qp_wr			= RXE_MAX_QP_WR;
8700e3e7c4857d Moni Shoua        2016-06-16  50  	rxe->attr.device_cap_flags		= RXE_DEVICE_CAP_FLAGS;
33023fb85a42b5 Steve Wise        2018-06-18  51  	rxe->attr.max_send_sge			= RXE_MAX_SGE;
33023fb85a42b5 Steve Wise        2018-06-18  52  	rxe->attr.max_recv_sge			= RXE_MAX_SGE;
8700e3e7c4857d Moni Shoua        2016-06-16  53  	rxe->attr.max_sge_rd			= RXE_MAX_SGE_RD;
8700e3e7c4857d Moni Shoua        2016-06-16  54  	rxe->attr.max_cq			= RXE_MAX_CQ;
8700e3e7c4857d Moni Shoua        2016-06-16  55  	rxe->attr.max_cqe			= (1 << RXE_MAX_LOG_CQE) - 1;
8700e3e7c4857d Moni Shoua        2016-06-16  56  	rxe->attr.max_mr			= RXE_MAX_MR;
af732adfacb2c6 Bob Pearson       2021-06-07  57  	rxe->attr.max_mw			= RXE_MAX_MW;
8700e3e7c4857d Moni Shoua        2016-06-16  58  	rxe->attr.max_pd			= RXE_MAX_PD;
8700e3e7c4857d Moni Shoua        2016-06-16  59  	rxe->attr.max_qp_rd_atom		= RXE_MAX_QP_RD_ATOM;
8700e3e7c4857d Moni Shoua        2016-06-16  60  	rxe->attr.max_res_rd_atom		= RXE_MAX_RES_RD_ATOM;
8700e3e7c4857d Moni Shoua        2016-06-16  61  	rxe->attr.max_qp_init_rd_atom		= RXE_MAX_QP_INIT_RD_ATOM;
0797e6f1a85872 Nathan Chancellor 2018-09-26  62  	rxe->attr.atomic_cap			= IB_ATOMIC_HCA;
8700e3e7c4857d Moni Shoua        2016-06-16  63  	rxe->attr.max_mcast_grp			= RXE_MAX_MCAST_GRP;
8700e3e7c4857d Moni Shoua        2016-06-16  64  	rxe->attr.max_mcast_qp_attach		= RXE_MAX_MCAST_QP_ATTACH;
8700e3e7c4857d Moni Shoua        2016-06-16  65  	rxe->attr.max_total_mcast_qp_attach	= RXE_MAX_TOT_MCAST_QP_ATTACH;
8700e3e7c4857d Moni Shoua        2016-06-16  66  	rxe->attr.max_ah			= RXE_MAX_AH;
8700e3e7c4857d Moni Shoua        2016-06-16  67  	rxe->attr.max_srq			= RXE_MAX_SRQ;
8700e3e7c4857d Moni Shoua        2016-06-16  68  	rxe->attr.max_srq_wr			= RXE_MAX_SRQ_WR;
8700e3e7c4857d Moni Shoua        2016-06-16  69  	rxe->attr.max_srq_sge			= RXE_MAX_SRQ_SGE;
8700e3e7c4857d Moni Shoua        2016-06-16  70  	rxe->attr.max_fast_reg_page_list_len	= RXE_MAX_FMR_PAGE_LIST_LEN;
8700e3e7c4857d Moni Shoua        2016-06-16 @71  	rxe->attr.max_pkeys			= RXE_MAX_PKEYS;
8700e3e7c4857d Moni Shoua        2016-06-16  72  	rxe->attr.local_ca_ack_delay		= RXE_LOCAL_CA_ACK_DELAY;
d0ca2c35dd15a3 Zhu Yanjun        2020-03-23  73  	addrconf_addr_eui48((unsigned char *)&rxe->attr.sys_image_guid,
d0ca2c35dd15a3 Zhu Yanjun        2020-03-23  74  			rxe->ndev->dev_addr);
8700e3e7c4857d Moni Shoua        2016-06-16  75  
8700e3e7c4857d Moni Shoua        2016-06-16  76  	rxe->max_ucontext			= RXE_MAX_UCONTEXT;
8700e3e7c4857d Moni Shoua        2016-06-16  77  }
8700e3e7c4857d Moni Shoua        2016-06-16  78  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
diff mbox series

Patch

diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
index 742e6ec93686..66a948adb1e1 100644
--- a/drivers/infiniband/sw/rxe/rxe_param.h
+++ b/drivers/infiniband/sw/rxe/rxe_param.h
@@ -9,6 +9,8 @@ 
 
 #include <uapi/rdma/rdma_user_rxe.h>
 
+#define DEFAULT_MAX_VALUE (1 << 20)
+
 static inline enum ib_mtu rxe_mtu_int_to_enum(int mtu)
 {
 	if (mtu < 256)
@@ -37,7 +39,7 @@  static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
 enum rxe_device_param {
 	RXE_MAX_MR_SIZE			= -1ull,
 	RXE_PAGE_SIZE_CAP		= 0xfffff000,
-	RXE_MAX_QP_WR			= 0x4000,
+	RXE_MAX_QP_WR			= DEFAULT_MAX_VALUE,
 	RXE_DEVICE_CAP_FLAGS		= IB_DEVICE_BAD_PKEY_CNTR
 					| IB_DEVICE_BAD_QKEY_CNTR
 					| IB_DEVICE_AUTO_PATH_MIG
@@ -58,40 +60,40 @@  enum rxe_device_param {
 	RXE_MAX_INLINE_DATA		= RXE_MAX_WQE_SIZE -
 					  sizeof(struct rxe_send_wqe),
 	RXE_MAX_SGE_RD			= 32,
-	RXE_MAX_CQ			= 16384,
+	RXE_MAX_CQ			= DEFAULT_MAX_VALUE,
 	RXE_MAX_LOG_CQE			= 15,
-	RXE_MAX_PD			= 0x7ffc,
+	RXE_MAX_PD			= DEFAULT_MAX_VALUE,
 	RXE_MAX_QP_RD_ATOM		= 128,
 	RXE_MAX_RES_RD_ATOM		= 0x3f000,
 	RXE_MAX_QP_INIT_RD_ATOM		= 128,
 	RXE_MAX_MCAST_GRP		= 8192,
 	RXE_MAX_MCAST_QP_ATTACH		= 56,
 	RXE_MAX_TOT_MCAST_QP_ATTACH	= 0x70000,
-	RXE_MAX_AH			= 100,
-	RXE_MAX_SRQ_WR			= 0x4000,
+	RXE_MAX_AH			= DEFAULT_MAX_VALUE,
+	RXE_MAX_SRQ_WR			= DEFAULT_MAX_VALUE,
 	RXE_MIN_SRQ_WR			= 1,
 	RXE_MAX_SRQ_SGE			= 27,
 	RXE_MIN_SRQ_SGE			= 1,
 	RXE_MAX_FMR_PAGE_LIST_LEN	= 512,
-	RXE_MAX_PKEYS			= 1,
+	RXE_MAX_PKEYS			= DEFAULT_MAX_VALUE,
 	RXE_LOCAL_CA_ACK_DELAY		= 15,
 
-	RXE_MAX_UCONTEXT		= 512,
+	RXE_MAX_UCONTEXT		= DEFAULT_MAX_VALUE,
 
 	RXE_NUM_PORT			= 1,
 
-	RXE_MAX_QP			= 0x10000,
+	RXE_MAX_QP			= DEFAULT_MAX_VALUE,
 	RXE_MIN_QP_INDEX		= 16,
-	RXE_MAX_QP_INDEX		= 0x00020000,
+	RXE_MAX_QP_INDEX		= 0x00040000,
 
-	RXE_MAX_SRQ			= 0x00001000,
+	RXE_MAX_SRQ			= DEFAULT_MAX_VALUE,
 	RXE_MIN_SRQ_INDEX		= 0x00020001,
 	RXE_MAX_SRQ_INDEX		= 0x00040000,
 
-	RXE_MAX_MR			= 0x00001000,
+	RXE_MAX_MR			= DEFAULT_MAX_VALUE,
 	RXE_MAX_MW			= 0x00001000,
 	RXE_MIN_MR_INDEX		= 0x00000001,
-	RXE_MAX_MR_INDEX		= 0x00010000,
+	RXE_MAX_MR_INDEX		= 0x00040000,
 	RXE_MIN_MW_INDEX		= 0x00010001,
 	RXE_MAX_MW_INDEX		= 0x00020000,