diff mbox

[v2,1/8] IB/SRP: Avoid using IB_MR_TYPE_SG_GAPS

Message ID 20170214185636.29250-2-bart.vanassche@sandisk.com (mailing list archive)
State Accepted
Headers show

Commit Message

Bart Van Assche Feb. 14, 2017, 6:56 p.m. UTC 
Tests have shown that the following error message is reported when
using SG-GAPS registration with an mlx5 adapter:

scsi host1: ib_srp: failed RECV status WR flushed (5) for CQE ffff880bd4270eb0
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000000 0f007806 2500002a ad9fafd1
scsi host1: ib_srp: reconnect succeeded
mlx5_0:dump_cqe:262:(pid 7369): dump error cqe
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000000 0f007806 25000032 00105dd0
scsi host1: ib_srp: failed FAST REG status memory management operation error (6) for CQE ffff880b92860138

Hence avoid using SG-GAPS memory registrations. Additionally,
always configure the blk_queue_virt_boundary() to avoid to trigger
a mapping failure when using adapters that support SG-GAPS (e.g.
mlx5).

Fixes: commit ad8e66b4a801 ("IB/srp: fix mr allocation when the device supports sg gaps")
Fixes: commit 509c5f33f4f6 ("IB/srp: Prevent mapping failures")
Reported-by: Laurence Oberman <loberman@redhat.com>
Signed-off-by: Bart Van Assche <bart.vanassche@sandisk.com>
Cc: Israel Rukshin <israelr@mellanox.com>
Cc: Max Gurtovoy <maxg@mellanox.com>
Cc: Leon Romanovsky <leonro@mellanox.com>
Cc: Mark Bloch <markb@mellanox.com>
Cc: Yuval Shaia <yuval.shaia@oracle.com>
Cc: <stable@vger.kernel.org> # 4.7+
---
 drivers/infiniband/ulp/srp/ib_srp.c | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

Comments

Bart Van Assche Feb. 15, 2017, 12:22 a.m. UTC | #1 
On Tue, 2017-02-14 at 10:56 -0800, Bart Van Assche wrote:
> @@ -2666,7 +2661,7 @@ static int srp_slave_alloc(struct scsi_device *sdev)
>  	struct srp_device *srp_dev = target->srp_host->srp_dev;
>  	struct ib_device *ibdev = srp_dev->dev;
>  
> -	if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG))
> +	if (true)
>  		blk_queue_virt_boundary(sdev->request_queue,
>  					~srp_dev->mr_page_mask);

Hello Doug,

This patch introduces a new warning when building with W=1, namely that
variable ibdev is not used. The build bot figured this out. Do you want me
to address this?

Thanks,

Bart.--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Leon Romanovsky Feb. 15, 2017, 7:14 a.m. UTC | #2 
On Tue, Feb 14, 2017 at 10:56:29AM -0800, Bart Van Assche wrote:
> Tests have shown that the following error message is reported when
> using SG-GAPS registration with an mlx5 adapter:
>
> scsi host1: ib_srp: failed RECV status WR flushed (5) for CQE ffff880bd4270eb0
> 00000000 00000000 00000000 00000000
> 00000000 00000000 00000000 00000000
> 00000000 00000000 00000000 00000000
> 00000000 0f007806 2500002a ad9fafd1
> scsi host1: ib_srp: reconnect succeeded
> mlx5_0:dump_cqe:262:(pid 7369): dump error cqe
> 00000000 00000000 00000000 00000000
> 00000000 00000000 00000000 00000000
> 00000000 00000000 00000000 00000000
> 00000000 0f007806 25000032 00105dd0
> scsi host1: ib_srp: failed FAST REG status memory management operation error (6) for CQE ffff880b92860138
>
> Hence avoid using SG-GAPS memory registrations. Additionally,
> always configure the blk_queue_virt_boundary() to avoid to trigger
> a mapping failure when using adapters that support SG-GAPS (e.g.
> mlx5).

According to the error dump, we have an issue with max_page_list_len supplied and/or
internal calculations from that value to the UMR byte count.

I assume that iser works with SG_GAPS.

>
> Fixes: commit ad8e66b4a801 ("IB/srp: fix mr allocation when the device supports sg gaps")
> Fixes: commit 509c5f33f4f6 ("IB/srp: Prevent mapping failures")
> Reported-by: Laurence Oberman <loberman@redhat.com>
> Signed-off-by: Bart Van Assche <bart.vanassche@sandisk.com>
> Cc: Israel Rukshin <israelr@mellanox.com>
> Cc: Max Gurtovoy <maxg@mellanox.com>
> Cc: Leon Romanovsky <leonro@mellanox.com>
> Cc: Mark Bloch <markb@mellanox.com>
> Cc: Yuval Shaia <yuval.shaia@oracle.com>
> Cc: <stable@vger.kernel.org> # 4.7+
> ---
>  drivers/infiniband/ulp/srp/ib_srp.c | 11 +++--------
>  1 file changed, 3 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
> index 79bf48477ddb..07e3613e4798 100644
> --- a/drivers/infiniband/ulp/srp/ib_srp.c
> +++ b/drivers/infiniband/ulp/srp/ib_srp.c
> @@ -371,7 +371,6 @@ static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
>  	struct srp_fr_desc *d;
>  	struct ib_mr *mr;
>  	int i, ret = -EINVAL;
> -	enum ib_mr_type mr_type;
>
>  	if (pool_size <= 0)
>  		goto err;
> @@ -385,13 +384,9 @@ static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
>  	spin_lock_init(&pool->lock);
>  	INIT_LIST_HEAD(&pool->free_list);
>
> -	if (device->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)
> -		mr_type = IB_MR_TYPE_SG_GAPS;
> -	else
> -		mr_type = IB_MR_TYPE_MEM_REG;
> -
>  	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
> -		mr = ib_alloc_mr(pd, mr_type, max_page_list_len);
> +		mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
> +				 max_page_list_len);
>  		if (IS_ERR(mr)) {
>  			ret = PTR_ERR(mr);
>  			if (ret == -ENOMEM)
> @@ -2666,7 +2661,7 @@ static int srp_slave_alloc(struct scsi_device *sdev)
>  	struct srp_device *srp_dev = target->srp_host->srp_dev;
>  	struct ib_device *ibdev = srp_dev->dev;
>
> -	if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG))
> +	if (true)
>  		blk_queue_virt_boundary(sdev->request_queue,
>  					~srp_dev->mr_page_mask);
>
> --
> 2.11.0
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
diff mbox

Patch

diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 79bf48477ddb..07e3613e4798 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -371,7 +371,6 @@  static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 	struct srp_fr_desc *d;
 	struct ib_mr *mr;
 	int i, ret = -EINVAL;
-	enum ib_mr_type mr_type;
 
 	if (pool_size <= 0)
 		goto err;
@@ -385,13 +384,9 @@  static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 	spin_lock_init(&pool->lock);
 	INIT_LIST_HEAD(&pool->free_list);
 
-	if (device->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)
-		mr_type = IB_MR_TYPE_SG_GAPS;
-	else
-		mr_type = IB_MR_TYPE_MEM_REG;
-
 	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
-		mr = ib_alloc_mr(pd, mr_type, max_page_list_len);
+		mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
+				 max_page_list_len);
 		if (IS_ERR(mr)) {
 			ret = PTR_ERR(mr);
 			if (ret == -ENOMEM)
@@ -2666,7 +2661,7 @@  static int srp_slave_alloc(struct scsi_device *sdev)
 	struct srp_device *srp_dev = target->srp_host->srp_dev;
 	struct ib_device *ibdev = srp_dev->dev;
 
-	if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG))
+	if (true)
 		blk_queue_virt_boundary(sdev->request_queue,
 					~srp_dev->mr_page_mask);