[rdma-core,v2,2/2] mlx5: Disable locking on a QP if it is assigned to a thread domain

Commit Message

Rohit Zambre May 1, 2018, 11:17 p.m. UTC

By creating a QP within a thread domain the application is guaranteeing
that the QP will not be accessed concurrently from multiple user threads.
Hence, a lock is not needed for QP that is assigned to a thread domain.

This patch disables locking on the QP if a thread domain is passed during
QP-creation. To do so, it uses a new mlx5_spinlock_init_pd function which
will check to see if the ibv_pd is a Parent Domain. If the Parent Domain
contains a Thread Domain, it will eliminate the need of a lock on that
object. All objects associated with a Protection Domain will use this
function to initialize their locks.

Signed-off-by: Rohit Zambre <[e-mail]>
---
 providers/mlx5/mlx5.h  | 14 ++++++++++++++
 providers/mlx5/verbs.c | 10 +++++-----
 2 files changed, 19 insertions(+), 5 deletions(-)

Patch

diff --git a/providers/mlx5/mlx5.h b/providers/mlx5/mlx5.h
index 08ab610..f0f376c 100644
--- a/providers/mlx5/mlx5.h
+++ b/providers/mlx5/mlx5.h
@@ -882,6 +882,20 @@  static inline int mlx5_spinlock_init(struct mlx5_spinlock *lock, int need_lock)
 	return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE);
 }
 
+static inline int mlx5_spinlock_init_pd(struct mlx5_spinlock *lock, struct ibv_pd *pd)
+{
+	struct mlx5_parent_domain *mparent_domain;
+	int thread_safe;
+	
+	mparent_domain = to_mparent_domain(pd);
+	if (mparent_domain && mparent_domain->mtd)
+		thread_safe = 1;
+	else
+		thread_safe = mlx5_single_threaded;
+
+	return mlx5_spinlock_init(lock, !thread_safe);
+}
+
 static inline int mlx5_spinlock_destroy(struct mlx5_spinlock *lock)
 {
 	return pthread_spin_destroy(&lock->lock);
diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c
index f05b2f3..71728c8 100644
--- a/providers/mlx5/verbs.c
+++ b/providers/mlx5/verbs.c
@@ -907,7 +907,7 @@  struct ibv_srq *mlx5_create_srq(struct ibv_pd *pd,
 	ibsrq = &srq->vsrq.srq;
 
 	memset(&cmd, 0, sizeof cmd);
-	if (mlx5_spinlock_init(&srq->lock, !mlx5_single_threaded)) {
+	if (mlx5_spinlock_init_pd(&srq->lock, pd)) {
 		fprintf(stderr, "%s-%d:\n", __func__, __LINE__);
 		goto err;
 	}
@@ -1751,8 +1751,8 @@  static struct ibv_qp *create_qp(struct ibv_context *context,
 
 	mlx5_init_qp_indices(qp);
 
-	if (mlx5_spinlock_init(&qp->sq.lock, !mlx5_single_threaded) ||
-			mlx5_spinlock_init(&qp->rq.lock, !mlx5_single_threaded))
+	if (mlx5_spinlock_init_pd(&qp->sq.lock, attr->pd) ||
+			mlx5_spinlock_init_pd(&qp->rq.lock, attr->pd))
 		goto err_free_qp_buf;
 
 	qp->db = mlx5_alloc_dbrec(ctx);
@@ -2495,7 +2495,7 @@  struct ibv_srq *mlx5_create_srq_ex(struct ibv_context *context,
 	memset(&cmd, 0, sizeof(cmd));
 	memset(&resp, 0, sizeof(resp));
 
-	if (mlx5_spinlock_init(&msrq->lock, !mlx5_single_threaded)) {
+	if (mlx5_spinlock_init_pd(&msrq->lock, attr->pd)) {
 		fprintf(stderr, "%s-%d:\n", __func__, __LINE__);
 		goto err;
 	}
@@ -2799,7 +2799,7 @@  static struct ibv_wq *create_wq(struct ibv_context *context,
 
 	mlx5_init_rwq_indices(rwq);
 
-	if (mlx5_spinlock_init(&rwq->rq.lock, !mlx5_single_threaded))
+	if (mlx5_spinlock_init_pd(&rwq->rq.lock, attr->pd))
 		goto err_free_rwq_buf;
 
 	rwq->db = mlx5_alloc_dbrec(ctx);