ibv_post_send/recv kernel path optimizations

Commit Message

Walukiewicz, Miroslaw Dec. 27, 2010, 3:18 p.m. UTC

None

Patch

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 3520182..234324f 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -269,6 +269,54 @@  static void put_xrcd_read(struct ib_uobject *uobj)
 	put_uobj_read(uobj);
 }
 
+#define RAW_QP_HASH(h) ((((h) >> 24) ^ ((h) >> 16) ^ \
+			((h) >> 8) ^ (h)) & (MAX_RAW_QP_HASH - 1))
+
+static void init_raw_qp_hash(struct ib_ucontext *ucontext) {
+	int idx;
+
+	for (idx = 0; idx < MAX_RAW_QP_HASH; idx++)
+		INIT_LIST_HEAD(&ucontext->raw_qp_hash[idx]);
+}
+
+static void raw_qp_hash_add(struct ib_ucontext *ucontext,
+			u32 qp_handle,
+			struct ib_qp *qp)
+{
+	int key;
+
+	if (qp->qp_type != IB_QPT_RAW_ETH)
+		return;
+
+	key = RAW_QP_HASH(qp_handle);
+	list_add(&qp->uobject->raw_qp_list, &ucontext->raw_qp_hash[key]); }
+
+static void raw_qp_hash_del(struct ib_qp *qp) {
+	if (qp->qp_type != IB_QPT_RAW_ETH)
+		return;
+
+	list_del(&qp->uobject->raw_qp_list);
+}
+
+static struct ib_qp *raw_qp_lookup(u32 qp_handle, struct ib_ucontext 
+*ucontext) {
+	int key;
+	struct ib_qp *qp;
+	struct ib_uobject *uobj, *tmp;
+
+	key = RAW_QP_HASH(qp_handle);
+	list_for_each_entry_safe(uobj, tmp,
+				&ucontext->raw_qp_hash[key], raw_qp_list) {
+		qp = uobj->object;
+		if (qp->uobject->id == qp_handle)
+			return qp;
+	}
+	return NULL;
+}
+
 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
 			      const char __user *buf,
 			      int in_len, int out_len)
@@ -313,6 +361,7 @@  ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
 	INIT_LIST_HEAD(&ucontext->srq_list);
 	INIT_LIST_HEAD(&ucontext->ah_list);
 	INIT_LIST_HEAD(&ucontext->xrc_domain_list);
+	init_raw_qp_hash(ucontext);
 	ucontext->closing = 0;
 
 	resp.num_comp_vectors = file->device->num_comp_vectors; @@ -1155,6 +1204,7 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
 
 	mutex_lock(&file->mutex);
 	list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
+	raw_qp_hash_add(file->ucontext, obj->uevent.uobject.id, qp);
 	mutex_unlock(&file->mutex);
 
 	obj->uevent.uobject.live = 1;
@@ -1412,6 +1462,7 @@  ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
 
 	mutex_lock(&file->mutex);
 	list_del(&uobj->list);
+	raw_qp_hash_del(qp);
 	mutex_unlock(&file->mutex);
 
 	ib_uverbs_release_uevent(file, &obj->uevent); @@ -1450,6 +1501,25 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
 	if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr))
 		return -EINVAL;
 
+	mutex_lock(&file->mutex);
+	qp = raw_qp_lookup(cmd.qp_handle, file->ucontext);
+	mutex_unlock(&file->mutex);
+	if (qp) {
+		if (qp->qp_type == IB_QPT_RAW_ETH) {
+			resp.bad_wr = 0;
+			ret = qp->device->post_send(qp, NULL, &bad_wr);
+			if (ret)
+				resp.bad_wr = cmd.wr_count;
+
+			if (copy_to_user((void __user *) (unsigned long)
+					cmd.response,
+					&resp,
+					sizeof resp))
+					ret = -EFAULT;
+					goto out_raw_qp;
+		}
+	}
+
 	user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
 	if (!user_wr)
 		return -ENOMEM;
@@ -1579,7 +1649,7 @@  out_put:
 
 out:
 	kfree(user_wr);
-
+out_raw_qp:
 	return ret ? ret : in_len;
 }
 
@@ -1664,7 +1734,6 @@  err:
 		kfree(wr);
 		wr = next;
 	}
-
 	return ERR_PTR(ret);
 }
 
@@ -1681,6 +1750,25 @@  ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
+	mutex_lock(&file->mutex);
+	qp = raw_qp_lookup(cmd.qp_handle, file->ucontext);
+	mutex_unlock(&file->mutex);
+	if (qp) {
+		if (qp->qp_type == IB_QPT_RAW_ETH) {
+			resp.bad_wr = 0;
+			ret = qp->device->post_recv(qp, NULL, &bad_wr);
+			if (ret)
+				resp.bad_wr = cmd.wr_count;
+
+			if (copy_to_user((void __user *) (unsigned long)
+					cmd.response,
+					&resp,
+					sizeof resp))
+				ret = -EFAULT;
+			goto out_raw_qp;
+		}
+	}
+
 	wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
 				       in_len - sizeof cmd, cmd.wr_count,
 				       cmd.sge_count, cmd.wqe_size); @@ -1713,7 +1801,7 @@ out:
 		kfree(wr);
 		wr = next;
 	}
-
+out_raw_qp:
 	return ret ? ret : in_len;
 }
 
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index f5b054a..adf1dd8 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -838,6 +838,8 @@  struct ib_fmr_attr {
 	u8	page_shift;
 };
 
+#define MAX_RAW_QP_HASH 8
+
 struct ib_ucontext {
 	struct ib_device       *device;
 	struct list_head	pd_list;
@@ -848,6 +850,7 @@  struct ib_ucontext {
 	struct list_head	srq_list;
 	struct list_head	ah_list;
 	struct list_head	xrc_domain_list;
+	struct list_head	raw_qp_hash[MAX_RAW_QP_HASH];
 	int			closing;
 };
 
@@ -859,6 +862,7 @@  struct ib_uobject {
 	int			id;		/* index into kernel idr */
 	struct kref		ref;
 	struct rw_semaphore	mutex;		/* protects .live */
+	struct list_head        raw_qp_list;    /* raw qp hash */
 	int			live;
 };