diff mbox

[libmlx4,V3,2/2] Add ibv_query_port_ex support

Message ID 1400405952-14388-3-git-send-email-ogerlitz@r-vnc04.mtr.labs.mlnx (mailing list archive)
State Rejected
Headers show

Commit Message

Or Gerlitz May 18, 2014, 9:39 a.m. UTC 
From: Matan Barak <matanb@mellanox.com>

This patch adds the new extended support for query_port.
The purpose of this is:
1. Request fields that aren't availible by today's ibv_query_port
2. Don't fetch fields that the user doesn't need. Hence, there is
   more chance to optimize.
3. Cache link layer's type in mlx4_context.
   Caching will allow us to avoid ibv_query_port calls and save time
   in ibv_create_ah.

Signed-off-by: Matan Barak <matanb@mellanox.com>
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
---
 src/mlx4.c  |    3 ++
 src/mlx4.h  |    9 ++++++++
 src/verbs.c |   66 +++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 72 insertions(+), 6 deletions(-)
diff mbox

Patch

diff --git a/src/mlx4.c b/src/mlx4.c
index a9f3a98..1d42270 100644
--- a/src/mlx4.c
+++ b/src/mlx4.c
@@ -157,6 +157,8 @@  static int mlx4_init_context(struct verbs_device *v_device,
 
 	context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS;
 	context->qp_table_mask	= (1 << context->qp_table_shift) - 1;
+	for (i = 0; i < MLX4_PORTS_NUM; ++i)
+		context->port_query_cache[i].valid = 0;
 
 	pthread_mutex_init(&context->qp_table_mutex, NULL);
 	for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i)
@@ -206,6 +208,7 @@  static int mlx4_init_context(struct verbs_device *v_device,
 	verbs_set_ctx_op(verbs_ctx, create_flow, ibv_cmd_create_flow);
 	verbs_set_ctx_op(verbs_ctx, destroy_flow, ibv_cmd_destroy_flow);
 	verbs_set_ctx_op(verbs_ctx, create_ah_ex, mlx4_create_ah_ex);
+	verbs_set_ctx_op(verbs_ctx, query_port_ex, mlx4_query_port_ex);
 
 	return 0;
 
diff --git a/src/mlx4.h b/src/mlx4.h
index 3015357..06fd2ba 100644
--- a/src/mlx4.h
+++ b/src/mlx4.h
@@ -40,6 +40,8 @@ 
 #include <infiniband/arch.h>
 #include <infiniband/verbs.h>
 
+#define MLX4_PORTS_NUM 2
+
 #ifdef HAVE_VALGRIND_MEMCHECK_H
 
 #  include <valgrind/memcheck.h>
@@ -189,6 +191,11 @@  struct mlx4_context {
 	pthread_mutex_t			db_list_mutex;
 	int				cqe_size;
 	struct mlx4_xsrq_table		xsrq_table;
+	struct {
+		uint8_t			valid;
+		uint8_t			link_layer;
+		enum ibv_port_cap_flags	caps;
+	} port_query_cache[MLX4_PORTS_NUM];
 };
 
 struct mlx4_buf {
@@ -354,6 +361,8 @@  int mlx4_query_device(struct ibv_context *context,
 		       struct ibv_device_attr *attr);
 int mlx4_query_port(struct ibv_context *context, uint8_t port,
 		     struct ibv_port_attr *attr);
+int mlx4_query_port_ex(struct ibv_context *context, uint8_t port_num,
+		       struct ibv_port_attr_ex *port_attr);
 
 struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context);
 int mlx4_free_pd(struct ibv_pd *pd);
diff --git a/src/verbs.c b/src/verbs.c
index ccfd678..b2c71e7 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -72,8 +72,58 @@  int mlx4_query_port(struct ibv_context *context, uint8_t port,
 		     struct ibv_port_attr *attr)
 {
 	struct ibv_query_port cmd;
+	int err;
+
+	err = ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
+	if (!err && port <= MLX4_PORTS_NUM && port > 0) {
+		struct mlx4_context *mctx = to_mctx(context);
+		if (!mctx->port_query_cache[port - 1].valid) {
+			mctx->port_query_cache[port - 1].link_layer =
+				attr->link_layer;
+			mctx->port_query_cache[port - 1].caps =
+				attr->port_cap_flags;
+			mctx->port_query_cache[port - 1].valid = 1;
+		}
+	}
+
+	return err;
+}
+
+int mlx4_query_port_ex(struct ibv_context *context, uint8_t port_num,
+		       struct ibv_port_attr_ex *port_attr)
+{
+	/* Check that only valid flags were given */
+	if (port_attr->comp_mask & ~IBV_QUERY_PORT_EX_MASK)
+		return -EINVAL;
+
+	/* Optimize the link type query */
+	if (!(port_attr->comp_mask & ~(IBV_QUERY_PORT_EX_LINK_LAYER |
+				       IBV_QUERY_PORT_EX_CAP_FLAGS))) {
+		struct mlx4_context *mctx = to_mctx(context);
+		if (port_num <= 0 || port_num > MLX4_PORTS_NUM)
+			return -EINVAL;
+		if (mctx->port_query_cache[port_num - 1].valid) {
+			if (port_attr->comp_mask &
+			    IBV_QUERY_PORT_EX_LINK_LAYER)
+				port_attr->link_layer =
+					mctx->
+					port_query_cache[port_num - 1].
+					link_layer;
+			if (port_attr->comp_mask &
+			    IBV_QUERY_PORT_EX_CAP_FLAGS)
+				port_attr->port_cap_flags =
+					mctx->
+					port_query_cache[port_num - 1].
+					caps;
+			return 0;
+		}
+	}
+	if (port_attr->comp_mask & IBV_QUERY_PORT_EX_STD_MASK) {
+		return mlx4_query_port(context, port_num,
+				       (struct ibv_port_attr *)port_attr);
+	}
 
-	return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
+	return -EOPNOTSUPP;
 }
 
 struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context)
@@ -826,15 +876,17 @@  static struct ibv_ah *mlx4_create_ah_common(struct ibv_pd *pd,
 struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
 {
 	struct ibv_ah *ah;
-	struct ibv_port_attr port_attr;
+	struct ibv_port_attr_ex port_attr;
+
+	port_attr.comp_mask = IBV_QUERY_PORT_EX_LINK_LAYER;
 
-	if (ibv_query_port(pd->context, attr->port_num, &port_attr))
+	if (ibv_query_port_ex(pd->context, attr->port_num, &port_attr))
 		return NULL;
 
 	ah = mlx4_create_ah_common(pd, attr, port_attr.link_layer);
 	if (NULL != ah &&
 	    (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET ||
-	    !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr)))
+	     !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr)))
 		return ah;
 
 	if (ah)
@@ -845,11 +897,13 @@  struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
 struct ibv_ah *mlx4_create_ah_ex(struct ibv_pd *pd,
 				 struct ibv_ah_attr_ex *attr_ex)
 {
-	struct ibv_port_attr port_attr;
+	struct ibv_port_attr_ex port_attr;
 	struct ibv_ah *ah;
 	struct mlx4_ah *mah;
 
-	if (ibv_query_port(pd->context, attr_ex->port_num, &port_attr))
+	port_attr.comp_mask = IBV_QUERY_PORT_EX_LINK_LAYER;
+
+	if (ibv_query_port_ex(pd->context, attr_ex->port_num, &port_attr))
 		return NULL;
 
 	ah = mlx4_create_ah_common(pd, (struct ibv_ah_attr *)attr_ex,