From patchwork Thu Mar 13 17:04:57 2014 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Arlin Davis X-Patchwork-Id: 3827371 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.19.201]) by patchwork1.web.kernel.org (Postfix) with ESMTP id 897C49F369 for ; Thu, 13 Mar 2014 17:06:55 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id CBF40201E4 for ; Thu, 13 Mar 2014 17:06:54 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id DF97F20172 for ; Thu, 13 Mar 2014 17:06:53 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754517AbaCMRGs (ORCPT ); Thu, 13 Mar 2014 13:06:48 -0400 Received: from mga02.intel.com ([134.134.136.20]:2299 "EHLO mga02.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754510AbaCMRGr (ORCPT ); Thu, 13 Mar 2014 13:06:47 -0400 Received: from orsmga001.jf.intel.com ([10.7.209.18]) by orsmga101.jf.intel.com with ESMTP; 13 Mar 2014 10:04:59 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.97,648,1389772800"; d="scan'208";a="471772439" Received: from orsmsx107.amr.corp.intel.com ([10.22.240.5]) by orsmga001.jf.intel.com with ESMTP; 13 Mar 2014 10:04:58 -0700 Received: from orsmsx113.amr.corp.intel.com (10.22.240.9) by ORSMSX107.amr.corp.intel.com (10.22.240.5) with Microsoft SMTP Server (TLS) id 14.3.123.3; Thu, 13 Mar 2014 10:04:58 -0700 Received: from orsmsx101.amr.corp.intel.com ([169.254.8.45]) by ORSMSX113.amr.corp.intel.com ([10.22.240.9]) with mapi id 14.03.0123.003; Thu, 13 Mar 2014 10:04:58 -0700 From: "Davis, Arlin R" To: "linux-rdma@vger.kernel.org" CC: "ofw@lists.openfabrics.org" , "ewg@lists.openfabrics.org" Subject: [PATCH 2/5] openib: add new provider specific attributes Thread-Topic: [PATCH 2/5] openib: add new provider specific attributes Thread-Index: Ac8+3dAJC5qn5HymTUOwOhrlafvrvg== Date: Thu, 13 Mar 2014 17:04:57 +0000 Message-ID: <54347E5A035A054EAE9D05927FB467F972E0C8F1@ORSMSX101.amr.corp.intel.com> Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.22.254.139] MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, T_RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP DAT_IB_PROVIDER_NAME = UCM/CMA/SCM DAT_IB_DEVICE_NAME = ibv_get_device_name DAT_IB_CONNECTIVITY_MODE = DIRECT/PROXY DAT_IB_RDMA_READ = TRUE/FALSE DAT_IB_NODE_GUID = xxxx:xxxx:xxxx:xxxx DAT_IB_PORT_STATE = ibv_port_state_str Signed-off-by: Arlin Davis --- dapl/openib_cma/dapl_ib_util.h | 4 ++- dapl/openib_common/dapl_ib_common.h | 13 ++++++++- dapl/openib_common/dapl_ib_dto.h | 10 +++++++ dapl/openib_common/util.c | 52 ++++++++++++++++++++++++++++------- dapl/openib_scm/dapl_ib_util.h | 4 ++- dapl/openib_ucm/dapl_ib_util.h | 4 ++- test/dtest/dtest.c | 14 +++++++++- 7 files changed, 86 insertions(+), 15 deletions(-) -- 1.7.3 diff --git a/dapl/openib_cma/dapl_ib_util.h b/dapl/openib_cma/dapl_ib_util.h index 454f7e1..de95485 100755 --- a/dapl/openib_cma/dapl_ib_util.h +++ b/dapl/openib_cma/dapl_ib_util.h @@ -120,10 +120,12 @@ typedef struct _ib_hca_transport uint8_t hop_limit; uint8_t tclass; uint8_t mtu; - DAT_NAMED_ATTR named_attr; uint8_t sl; uint16_t pkey; int pkey_idx; + uint64_t guid; + char guid_str[32]; + ib_named_attr_t na; #ifdef DAT_IB_COLLECTIVES /* Collective member device and address information */ ib_thread_state_t coll_thread_state; diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h index ba805d0..dfc80a9 100644 --- a/dapl/openib_common/dapl_ib_common.h +++ b/dapl/openib_common/dapl_ib_common.h @@ -109,6 +109,17 @@ typedef struct _ib_cm_msg } ib_cm_msg_t; +typedef struct _ib_named_attr +{ + const char *dev; + const char *mode; + const char *read; + const char *guid; + const char *mtu; + const char *port; + +} ib_named_attr_t; + /* CM events */ typedef enum { IB_CME_CONNECTED, @@ -304,7 +315,7 @@ int32_t dapls_ib_release(void); /* util.c */ enum ibv_mtu dapl_ib_mtu(int mtu); -char *dapl_ib_mtu_str(enum ibv_mtu mtu); +const char *dapl_ib_mtu_str(enum ibv_mtu mtu); int getipaddr_netdev(char *name, char *addr, int addr_len); DAT_RETURN getlocalipaddr(char *addr, int addr_len); diff --git a/dapl/openib_common/dapl_ib_dto.h b/dapl/openib_common/dapl_ib_dto.h index b93565c..2bd6e7e 100644 --- a/dapl/openib_common/dapl_ib_dto.h +++ b/dapl/openib_common/dapl_ib_dto.h @@ -35,6 +35,16 @@ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p); +#if defined(_OPENIB_CMA_) +#define PROVIDER_NAME "CMA" +#elif defined(_OPENIB_UCM_) +#define PROVIDER_NAME "UCM" +#elif defined(_OPENIB_SCM_) +#define PROVIDER_NAME "SCM" +#else +#define PROVIDER_NAME "" +#endif + #define CQE_WR_TYPE_UD(id) \ (((DAPL_COOKIE *)(uintptr_t)id)->ep->qp_handle->qp_type == IBV_QPT_UD) diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c index 20fb8b2..258d172 100644 --- a/dapl/openib_common/util.c +++ b/dapl/openib_common/util.c @@ -246,7 +246,7 @@ enum ibv_mtu dapl_ib_mtu(int mtu) } } -char *dapl_ib_mtu_str(enum ibv_mtu mtu) +const char *dapl_ib_mtu_str(enum ibv_mtu mtu) { switch (mtu) { case IBV_MTU_256: @@ -264,8 +264,6 @@ char *dapl_ib_mtu_str(enum ibv_mtu mtu) } } - - /* * dapls_ib_query_hca * @@ -377,10 +375,19 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr, DAPL_MAX(dev_attr.local_ca_ack_delay, hca_ptr->ib_trans.ack_timer); - /* set MTU in transport specific named attribute */ - hca_ptr->ib_trans.named_attr.name = "DAT_IB_TRANSPORT_MTU"; - hca_ptr->ib_trans.named_attr.value = - dapl_ib_mtu_str(hca_ptr->ib_trans.mtu); + /* set provider/transport specific named attributes */ + hca_ptr->ib_trans.na.dev = ia_attr->adapter_name; + hca_ptr->ib_trans.na.mtu = dapl_ib_mtu_str(hca_ptr->ib_trans.mtu); + hca_ptr->ib_trans.na.port = ibv_port_state_str(port_attr.state); + hca_ptr->ib_trans.guid = ntohll(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev)); + sprintf(hca_ptr->ib_trans.guid_str, "%04x:%04x:%04x:%04x", + (unsigned) (hca_ptr->ib_trans.guid >> 48) & 0xffff, + (unsigned) (hca_ptr->ib_trans.guid >> 32) & 0xffff, + (unsigned) (hca_ptr->ib_trans.guid >> 16) & 0xffff, + (unsigned) (hca_ptr->ib_trans.guid >> 0) & 0xffff); + hca_ptr->ib_trans.na.guid = hca_ptr->ib_trans.guid_str; + hca_ptr->ib_trans.na.mode = "DIRECT"; + hca_ptr->ib_trans.na.read = "TRUE"; if (hca_ptr->ib_hca_handle->device->transport_type != IBV_TRANSPORT_IB) goto skip_ib; @@ -635,7 +642,9 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca) } /* - * dapls_set_provider_specific_attr + * dapls_query_provider_specific_attrs + * + * Common for openib providers: cma, ucm, scm, mcm * * Input: * attr_ptr Pointer provider specific attributes @@ -648,8 +657,26 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca) */ DAT_NAMED_ATTR ib_attrs[] = { { + "DAT_IB_PROVIDER_NAME", PROVIDER_NAME} + , + { + "DAT_IB_DEVICE_NAME", "OFA_HCA_0000"} + , + { + "DAT_IB_CONNECTIVITY_MODE", "DIRECT"} + , + { + "DAT_IB_RDMA_READ", "TRUE"} + , + { + "DAT_IB_NODE_GUID", "xxxx:xxxx:xxxx:xxxx"} + , + { "DAT_IB_TRANSPORT_MTU", "2048"} , + { + "DAT_IB_PORT_STATUS", "UNKNOWN"} + , #ifdef DAT_EXTENSIONS { "DAT_EXTENSION_INTERFACE", "TRUE"} @@ -704,8 +731,13 @@ void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr, attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs); attr_ptr->provider_specific_attr = ib_attrs; - /* set MTU to actual settings */ - ib_attrs[0].value = ia_ptr->hca_ptr->ib_trans.named_attr.value; + /* update common attributes from providers */ + ib_attrs[1].value = ia_ptr->hca_ptr->ib_trans.na.dev; + ib_attrs[2].value = ia_ptr->hca_ptr->ib_trans.na.mode; + ib_attrs[3].value = ia_ptr->hca_ptr->ib_trans.na.read; + ib_attrs[4].value = ia_ptr->hca_ptr->ib_trans.na.guid; + ib_attrs[5].value = ia_ptr->hca_ptr->ib_trans.na.mtu; + ib_attrs[6].value = ia_ptr->hca_ptr->ib_trans.na.port; } /* diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h index 2050c2c..d382226 100644 --- a/dapl/openib_scm/dapl_ib_util.h +++ b/dapl/openib_scm/dapl_ib_util.h @@ -106,11 +106,13 @@ typedef struct _ib_hca_transport uint8_t hop_limit; uint8_t tclass; uint8_t mtu; - DAT_NAMED_ATTR named_attr; DAPL_SOCKET scm[2]; uint8_t sl; uint16_t pkey; int pkey_idx; + uint64_t guid; + char guid_str[32]; + ib_named_attr_t na; #ifdef DAT_IB_COLLECTIVES /* Collective member device and address information */ ib_thread_state_t coll_thread_state; diff --git a/dapl/openib_ucm/dapl_ib_util.h b/dapl/openib_ucm/dapl_ib_util.h index 469560e..91c8466 100644 --- a/dapl/openib_ucm/dapl_ib_util.h +++ b/dapl/openib_ucm/dapl_ib_util.h @@ -93,7 +93,6 @@ typedef struct _ib_hca_transport uint8_t hop_limit; uint8_t tclass; uint8_t mtu; - DAT_NAMED_ATTR named_attr; struct dapl_thread_signal signal; int cqe; int qpe; @@ -121,6 +120,9 @@ typedef struct _ib_hca_transport uint8_t sl; uint16_t pkey; int pkey_idx; + uint64_t guid; + char guid_str[32]; + ib_named_attr_t na; #ifdef DAT_IB_COLLECTIVES /* Collective member device and address information */ ib_thread_state_t coll_thread_state; diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c index d43a09b..8f4c80f 100755 --- a/test/dtest/dtest.c +++ b/test/dtest/dtest.c @@ -338,6 +338,7 @@ int main(int argc, char **argv) DAT_RETURN ret; DAT_EP_PARAM ep_param; DAT_IA_ATTR ia_attr; + DAT_PROVIDER_ATTR pr_attr; /* parse arguments */ while ((c = getopt(argc, argv, "tscvpq:l:b:d:B:h:P:")) != -1) { @@ -445,7 +446,10 @@ int main(int argc, char **argv) } else LOGPRINTF("%d Opened Interface Adaptor\n", getpid()); - ret = dat_ia_query(h_ia, 0, DAT_IA_FIELD_ALL, &ia_attr, 0, 0); + ret = dat_ia_query(h_ia, 0, + DAT_IA_FIELD_ALL, &ia_attr, + DAT_PROVIDER_FIELD_PROVIDER_SPECIFIC_ATTR, + &pr_attr); if (ret != DAT_SUCCESS) { fprintf(stderr, "%d: Error Adaptor query: %s\n", getpid(), DT_RetToStr(ret)); @@ -453,6 +457,14 @@ int main(int argc, char **argv) } print_ia_address(ia_attr.ia_address_ptr); + /* Provider specific attributes */ + for (i=0; i