diff mbox

[libmlx4,2/2] Add ibv_query_port_ex support

Message ID 1391356501-4270-3-git-send-email-ogerlitz@mellanox.com (mailing list archive)
State Rejected
Headers show

Commit Message

Or Gerlitz Feb. 2, 2014, 3:55 p.m. UTC 
From: Matan Barak <matanb@mellanox.com>

This patch adds the extended support for ibv_query_port in libmlx4.

This allows to request fields that aren't availible by the current
ibv_query_port API and avoid fetching from vendor library fields that
the user doesn't need, which gives more room for optimizations.

Also, it adds caching of link layer's type and the port caps.

Signed-off-by: Matan Barak <matanb@mellanox.com>
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
---
 src/mlx4.c  |    4 +++
 src/mlx4.h  |    9 +++++++
 src/verbs.c |   78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 3 files changed, 85 insertions(+), 6 deletions(-)
diff mbox

Patch

diff --git a/src/mlx4.c b/src/mlx4.c
index ca91df4..58f5435 100644
--- a/src/mlx4.c
+++ b/src/mlx4.c
@@ -157,6 +157,8 @@  static int mlx4_init_context(struct verbs_device *v_device,
 
 	context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS;
 	context->qp_table_mask	= (1 << context->qp_table_shift) - 1;
+	for (i = 0; i < MLX4_PORTS_NUM; ++i)
+		context->port_query_cache[i].valid = 0;
 
 	pthread_mutex_init(&context->qp_table_mutex, NULL);
 	for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i)
@@ -205,6 +207,8 @@  static int mlx4_init_context(struct verbs_device *v_device,
 	verbs_set_ctx_op(verbs_ctx, open_qp, mlx4_open_qp);
 
 	verbs_set_ctx_op(verbs_ctx, drv_ibv_create_ah_ex, mlx4_create_ah_ex);
+	verbs_set_ctx_op(verbs_ctx, drv_query_port_ex,
+			 mlx4_query_port_ex);
 	verbs_ctx->has_comp_mask = VERBS_CONTEXT_CREATE_AH;
 
 	return 0;
diff --git a/src/mlx4.h b/src/mlx4.h
index b7afa31..446e3b5 100644
--- a/src/mlx4.h
+++ b/src/mlx4.h
@@ -40,6 +40,8 @@ 
 #include <infiniband/arch.h>
 #include <infiniband/verbs.h>
 
+#define MLX4_PORTS_NUM 2
+
 #ifdef HAVE_VALGRIND_MEMCHECK_H
 
 #  include <valgrind/memcheck.h>
@@ -189,6 +191,11 @@  struct mlx4_context {
 	pthread_mutex_t			db_list_mutex;
 	int				cqe_size;
 	struct mlx4_xsrq_table		xsrq_table;
+	struct {
+		uint8_t			valid;
+		uint8_t			link_layer;
+		enum ibv_port_cap_flags	caps;
+	} port_query_cache[MLX4_PORTS_NUM];
 };
 
 struct mlx4_buf {
@@ -354,6 +361,8 @@  int mlx4_query_device(struct ibv_context *context,
 		       struct ibv_device_attr *attr);
 int mlx4_query_port(struct ibv_context *context, uint8_t port,
 		     struct ibv_port_attr *attr);
+int mlx4_query_port_ex(struct ibv_context *context, uint8_t port_num,
+		       struct ibv_port_attr_ex *port_attr);
 
 struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context);
 int mlx4_free_pd(struct ibv_pd *pd);
diff --git a/src/verbs.c b/src/verbs.c
index 38fa3d3..a46e26c 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -70,8 +70,63 @@  int mlx4_query_port(struct ibv_context *context, uint8_t port,
 		     struct ibv_port_attr *attr)
 {
 	struct ibv_query_port cmd;
+	int err;
+
+	err = ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
+	if (!err) {
+		struct mlx4_context *mctx = to_mctx(context);
+		if (!mctx->port_query_cache[port - 1].valid) {
+			mctx->port_query_cache[port - 1].link_layer =
+				attr->link_layer;
+			mctx->port_query_cache[port - 1].caps =
+				attr->port_cap_flags;
+			mctx->port_query_cache[port - 1].valid = 1;
+		}
+	}
+
+	return err;
+}
+
+int mlx4_query_port_ex(struct ibv_context *context, uint8_t port_num,
+		       struct ibv_port_attr_ex *port_attr)
+{
+	/* Check that only valid flags were given */
+	if ((!port_attr->comp_mask & IBV_QUERY_PORT_EX_ATTR_MASK1) ||
+	    (port_attr->comp_mask & ~IBV_QUERY_PORT_EX_ATTR_MASKS) ||
+	    (port_attr->mask1 & ~IBV_QUERY_PORT_EX_MASK)) {
+		return -EINVAL;
+	}
+
+	/* Optimize the link type query */
+	if (port_attr->comp_mask == IBV_QUERY_PORT_EX_ATTR_MASK1) {
+		if (!(port_attr->mask1 & ~(IBV_QUERY_PORT_EX_LINK_LAYER |
+					   IBV_QUERY_PORT_EX_CAP_FLAGS))) {
+			struct mlx4_context *mctx = to_mctx(context);
+			if (port_num > MLX4_PORTS_NUM)
+				return -EINVAL;
+			if (mctx->port_query_cache[port_num - 1].valid) {
+				if (port_attr->mask1 &
+				    IBV_QUERY_PORT_EX_LINK_LAYER)
+					port_attr->link_layer =
+						mctx->
+						port_query_cache[port_num - 1].
+						link_layer;
+				if (port_attr->mask1 &
+				    IBV_QUERY_PORT_EX_CAP_FLAGS)
+					port_attr->port_cap_flags =
+						mctx->
+						port_query_cache[port_num - 1].
+						caps;
+				return 0;
+			}
+		}
+		if (port_attr->mask1 & IBV_QUERY_PORT_EX_STD_MASK) {
+			return mlx4_query_port(context, port_num,
+					       &port_attr->port_attr);
+		}
+	}
 
-	return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
+	return -EOPNOTSUPP;
 }
 
 struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context)
@@ -787,9 +842,17 @@  static struct ibv_ah *mlx4_create_ah_common(struct ibv_pd *pd,
 					    struct ibv_ah_attr *attr)
 {
 	struct mlx4_ah *ah;
-	struct ibv_port_attr port_attr;
+	struct mlx4_context *mctx;
+	struct ibv_port_attr_ex port_attr;
+
+	mctx = to_mctx(pd->context);
 
-	if (ibv_query_port(pd->context, attr->port_num, &port_attr))
+	if (attr->port_num < 1 || attr->port_num > MLX4_PORTS_NUM)
+		return NULL;
+	port_attr.comp_mask = IBV_QUERY_PORT_EX_ATTR_MASK1;
+	port_attr.mask1 = IBV_QUERY_PORT_EX_LINK_LAYER;
+
+	if (ibv_query_port_ex(pd->context, attr->port_num, &port_attr))
 		return NULL;
 
 	ah = malloc(sizeof *ah);
@@ -827,12 +890,15 @@  static struct ibv_ah *mlx4_create_ah_common(struct ibv_pd *pd,
 struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
 {
 	struct ibv_ah *ah = mlx4_create_ah_common(pd, attr);
-	struct ibv_port_attr port_attr;
+	struct ibv_port_attr_ex port_attr;
+
+	port_attr.comp_mask = IBV_QUERY_PORT_EX_ATTR_MASK1;
+	port_attr.mask1 = IBV_QUERY_PORT_EX_LINK_LAYER;
 
 	if (NULL != ah &&
-	    !ibv_query_port(pd->context, attr->port_num, &port_attr) &&
+	    !ibv_query_port_ex(pd->context, attr->port_num, &port_attr) &&
 	    (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET ||
-	    !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr)))
+	     !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr)))
 		return ah;
 
 	if (ah)