[rdma-core,3/3] mlx5: Report PCI atomic capabilities

Commit Message

Yishai Hadas Jan. 27, 2019, 12:01 p.m. UTC

From: Michael Guralnik <[e-mail]>

Query the PCI atomic capabilities using the DEVX infrastructure and
report it through device query.

Signed-off-by: Michael Guralnik <[e-mail]>
Signed-off-by: Yishai Hadas <[e-mail]>
---
 providers/mlx5/mlx5_ifc.h | 100 ++++++++++++++++++++++++++++++++++++++++++++++
 providers/mlx5/verbs.c    |  31 ++++++++++++++
 2 files changed, 131 insertions(+)
 create mode 100644 providers/mlx5/mlx5_ifc.h

Patch

diff --git a/providers/mlx5/mlx5_ifc.h b/providers/mlx5/mlx5_ifc.h
new file mode 100644
index 0000000..5cf89d6
--- /dev/null
+++ b/providers/mlx5/mlx5_ifc.h
@@ -0,0 +1,100 @@ 
+/*
+ * Copyright (c) 2019 Mellanox Technologies, Inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define u8 uint8_t
+
+enum mlx5_cap_mode {
+	HCA_CAP_OPMOD_GET_CUR	= 1,
+};
+
+enum {
+	MLX5_CMD_OP_QUERY_HCA_CAP = 0x100,
+};
+
+struct mlx5_ifc_atomic_caps_bits {
+	u8         reserved_at_0[0x40];
+
+	u8         atomic_req_8B_endianness_mode[0x2];
+	u8         reserved_at_42[0x4];
+	u8         supported_atomic_req_8B_endianness_mode_1[0x1];
+
+	u8         reserved_at_47[0x19];
+
+	u8         reserved_at_60[0x20];
+
+	u8         reserved_at_80[0x10];
+	u8         atomic_operations[0x10];
+
+	u8         reserved_at_a0[0x10];
+	u8         atomic_size_qp[0x10];
+
+	u8         reserved_at_c0[0x10];
+	u8         atomic_size_dc[0x10];
+
+	u8         reserved_at_e0[0x1a0];
+
+	u8         fetch_add_pci_atomic[0x10];
+	u8         swap_pci_atomic[0x10];
+	u8         compare_swap_pci_atomic[0x10];
+
+	u8         reserved_at_2b0[0x550];
+};
+
+union mlx5_ifc_hca_cap_union_bits {
+	struct mlx5_ifc_atomic_caps_bits atomic_caps;
+	u8         reserved_at_0[0x8000];
+};
+
+struct mlx5_ifc_query_hca_cap_out_bits {
+	u8         status[0x8];
+	u8         reserved_at_8[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_at_40[0x40];
+
+	union mlx5_ifc_hca_cap_union_bits capability;
+};
+
+struct mlx5_ifc_query_hca_cap_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_at_10[0x10];
+
+	u8         reserved_at_20[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_at_40[0x40];
+};
+
+enum mlx5_cap_type {
+	MLX5_CAP_ATOMIC = 3,
+};
diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c
index bab675f..4bc3eac 100644
--- a/providers/mlx5/verbs.c
+++ b/providers/mlx5/verbs.c
@@ -55,6 +55,7 @@ 
 #include "mlx5.h"
 #include "mlx5-abi.h"
 #include "wqe.h"
+#include "mlx5_ifc.h"
 
 int mlx5_single_threaded = 0;
 
@@ -2816,6 +2817,32 @@  err:
 	return NULL;
 }
 
+static void get_pci_atomic_caps(struct ibv_context *context,
+				struct ibv_device_attr_ex *attr)
+{
+	uint32_t in[DEVX_ST_SZ_DW(query_hca_cap_in)] = {};
+	uint32_t out[DEVX_ST_SZ_DW(query_hca_cap_out)] = {};
+	uint16_t opmod = (MLX5_CAP_ATOMIC << 1) | HCA_CAP_OPMOD_GET_CUR;
+	int ret;
+
+	DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
+	DEVX_SET(query_hca_cap_in, in, op_mod, opmod);
+
+	ret = mlx5dv_devx_general_cmd(context, in, sizeof(in), out,
+				      sizeof(out));
+	if (!ret) {
+		attr->pci_atomic_caps.fetch_add =
+			DEVX_GET(query_hca_cap_out, out,
+				 capability.atomic_caps.fetch_add_pci_atomic);
+		attr->pci_atomic_caps.swap =
+			DEVX_GET(query_hca_cap_out, out,
+				 capability.atomic_caps.swap_pci_atomic);
+		attr->pci_atomic_caps.compare_swap =
+			DEVX_GET(query_hca_cap_out, out,
+			capability.atomic_caps.compare_swap_pci_atomic);
+	}
+}
+
 int mlx5_query_device_ex(struct ibv_context *context,
 			 const struct ibv_query_device_ex_input *input,
 			 struct ibv_device_attr_ex *attr,
@@ -2894,6 +2921,10 @@  int mlx5_query_device_ex(struct ibv_context *context,
 	snprintf(a->fw_ver, sizeof(a->fw_ver), "%d.%d.%04d",
 		 major, minor, sub_minor);
 
+	if (attr_size >= offsetof(struct ibv_device_attr_ex, pci_atomic_caps) +
+			sizeof(attr->pci_atomic_caps))
+		get_pci_atomic_caps(context, attr);
+
 	return 0;
 }