From patchwork Tue May 19 18:12:49 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Wan, Kaike" X-Patchwork-Id: 6440251 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork1.web.kernel.org (Postfix) with ESMTP id 9B8CC9F1CC for ; Tue, 19 May 2015 18:13:06 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id 5364F203A5 for ; Tue, 19 May 2015 18:13:05 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 7655E201F2 for ; Tue, 19 May 2015 18:13:03 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754482AbbESSNB (ORCPT ); Tue, 19 May 2015 14:13:01 -0400 Received: from mga09.intel.com ([134.134.136.24]:1771 "EHLO mga09.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754452AbbESSNA (ORCPT ); Tue, 19 May 2015 14:13:00 -0400 Received: from orsmga002.jf.intel.com ([10.7.209.21]) by orsmga102.jf.intel.com with ESMTP; 19 May 2015 11:13:00 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.13,459,1427785200"; d="scan'208";a="731915033" Received: from phlsvsds.ph.intel.com ([10.228.195.38]) by orsmga002.jf.intel.com with ESMTP; 19 May 2015 11:12:59 -0700 Received: from phlsvsds.ph.intel.com (localhost.localdomain [127.0.0.1]) by phlsvsds.ph.intel.com (8.13.8/8.13.8) with ESMTP id t4JICwQc007043; Tue, 19 May 2015 14:12:58 -0400 Received: (from kaikewan@localhost) by phlsvsds.ph.intel.com (8.13.8/8.13.8/Submit) id t4JICwPw007036; Tue, 19 May 2015 14:12:58 -0400 X-Authentication-Warning: phlsvsds.ph.intel.com: kaikewan set sender to kaike.wan@intel.com using -f From: kaike.wan@intel.com To: linux-rdma@vger.kernel.org Cc: Kaike Wan , John Fleck , Ira Weiny Subject: [PATCH v2 1/1] ibacm: Add support for pathrecord query through netlink Date: Tue, 19 May 2015 14:12:49 -0400 Message-Id: <1432059169-6840-1-git-send-email-kaike.wan@intel.com> X-Mailer: git-send-email 1.7.1 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, T_RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Kaike Wan This patch enables ibacm to process pathrecord queries through netlink. Since ibacm can cache pathrecords, this implementation provides an easy pathrecord cache for kernel components and therefore offers great performance advantage on large fabric systems. At this time, only non-RMPP SA pathrecord queries are supported. All other queries will be returned with errors. Signed-off-by: Kaike Wan Signed-off-by: John Fleck Signed-off-by: Ira Weiny Reviewed-by: Sean Hefty --- Changes since v1: - Check rmpp_flags for rmpp MAD instead of rmpp_version. src/acm.c | 219 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 216 insertions(+), 3 deletions(-) diff --git a/src/acm.c b/src/acm.c index 7649725..796e906 100644 --- a/src/acm.c +++ b/src/acm.c @@ -46,6 +46,8 @@ #include #include #include +#include +#include #include #include #include @@ -55,6 +57,7 @@ #include #include #include +#include #include #include "acm_mad.h" #include "acm_util.h" @@ -66,6 +69,14 @@ #define MAX_EP_ADDR 4 #define NL_MSG_BUF_SIZE 4096 #define ACM_PROV_NAME_SIZE 64 +#define NL_CLIENT_INDEX 0 +#ifndef RDMA_NL_GROUP_MAD + #define RDMA_NL_GROUP_MAD 3 + #define RDMA_NL_MAD 4 + #define RDMA_NL_MAD_REQUEST 0 +#endif +#define RDMA_NL_TYPE_MAD_REQUEST \ + RDMA_NL_GET_TYPE(RDMA_NL_MAD, RDMA_NL_MAD_REQUEST) struct acmc_subnet { DLIST_ENTRY entry; @@ -151,6 +162,14 @@ struct acmc_sa_req { struct acm_sa_mad mad; }; +struct acm_nl_resolve_msg { + struct nlmsghdr nlmsg_header; + union { + struct acm_mad mad; + struct ib_sa_mad sa_mad; + }; +}; + static char def_prov_name[ACM_PROV_NAME_SIZE] = "ibacmp"; static DLIST_ENTRY provider_list; static struct acmc_prov *def_provider = NULL; @@ -172,6 +191,7 @@ static struct acmc_ep *acm_find_ep(struct acmc_port *port, uint16_t pkey); static int acm_ep_insert_addr(struct acmc_ep *ep, const char *name, uint8_t *addr, size_t addr_len, uint8_t addr_type); static void acm_event_handler(struct acmc_device *dev); +static int acm_nl_send(SOCKET sock, struct acm_msg *msg); static struct sa_data { int timeout; @@ -466,7 +486,11 @@ int acm_resolve_response(uint64_t id, struct acm_msg *msg) goto release; } - ret = send(client->sock, (char *) msg, msg->hdr.length, 0); + if (id == NL_CLIENT_INDEX) + ret = acm_nl_send(client->sock, msg); + else + ret = send(client->sock, (char *) msg, msg->hdr.length, 0); + if (ret != msg->hdr.length) acm_log(0, "ERROR - failed to send response\n"); else @@ -597,6 +621,8 @@ static void acm_svr_accept(void) } for (i = 0; i < FD_SETSIZE - 1; i++) { + if (i == NL_CLIENT_INDEX) + continue; if (!atomic_get(&client_array[i].refcnt)) break; } @@ -1346,6 +1372,188 @@ static void acm_ipnl_handler(void) } } +static int acm_nl_send(SOCKET sock, struct acm_msg *msg) +{ + struct sockaddr_nl dst_addr; + struct acm_nl_resolve_msg acmnlmsg; + struct acm_nl_resolve_msg *orig; + struct ib_sa_mad *sa_mad = &acmnlmsg.sa_mad; + int ret; + int datalen; + + orig = (struct acm_nl_resolve_msg *) msg->hdr.tid; + + memset(&dst_addr, 0, sizeof(dst_addr)); + dst_addr.nl_family = AF_NETLINK; + dst_addr.nl_groups = (1 << (RDMA_NL_GROUP_MAD - 1)); + + memset(&acmnlmsg, 0, sizeof(acmnlmsg)); + datalen = sizeof(acmnlmsg); + acmnlmsg.nlmsg_header.nlmsg_len = datalen; + acmnlmsg.nlmsg_header.nlmsg_pid = getpid(); + acmnlmsg.nlmsg_header.nlmsg_type = RDMA_NL_TYPE_MAD_REQUEST; + acmnlmsg.nlmsg_header.nlmsg_flags = NLM_F_REQUEST; + acmnlmsg.nlmsg_header.nlmsg_seq = orig->nlmsg_header.nlmsg_seq; + + if (msg->hdr.status != ACM_STATUS_SUCCESS) { + acm_log(2, "acm status no success = %d\n", msg->hdr.status); + memcpy(&acmnlmsg.mad, &orig->mad, + sizeof(acmnlmsg.mad)); + acmnlmsg.mad.method |= IB_METHOD_RESP; + sa_mad->status = htons(UMAD_SA_STATUS_NO_RECORDS); + } else { + acm_log(2, "acm status success\n"); + memcpy(sa_mad, &orig->sa_mad, + sizeof(*sa_mad) - sizeof(sa_mad->data)); + sa_mad->method |= IB_METHOD_RESP; + sa_mad->status = UMAD_SA_STATUS_SUCCESS; + memcpy(sa_mad->data, &msg->resolve_data[0].info.path, + ACM_MSG_EP_LENGTH); + } + + ret = sendto(sock, &acmnlmsg, datalen, 0, + (const struct sockaddr *)&dst_addr, + (socklen_t)sizeof(dst_addr)); + if (ret != datalen) { + acm_log(0, "ERROR - sendto = %d errno = %d\n", ret, errno); + ret = -1; + } else { + ret = msg->hdr.length; + } + + free(orig); + + return ret; +} + +static inline int acm_nl_supported_sa_mad(struct ib_sa_mad *sa_mad) +{ + return (!(sa_mad->rmpp_flags & UMAD_RMPP_FLAG_ACTIVE) && + sa_mad->method == UMAD_METHOD_GET && + sa_mad->attr_id == htons(UMAD_SA_ATTR_PATH_REC)); +} + +static void acm_nl_process_sa_mad(struct acmc_client *client, + struct acm_nl_resolve_msg *acmnlmsg) +{ + struct acm_msg msg; + struct ib_sa_mad *sa_mad = &acmnlmsg->sa_mad; + + acm_format_name(2, log_data, sizeof(log_data), ACM_ADDRESS_GID, + &sa_mad->data[8], sizeof(union ibv_gid)); + acm_log(2, "path dgid %s\n", log_data); + acm_format_name(2, log_data, sizeof(log_data), ACM_ADDRESS_GID, + &sa_mad->data[24], sizeof(union ibv_gid)); + acm_log(2, "path sgid %s\n", log_data); + atomic_inc(&counter[ACM_CNTR_RESOLVE]); + + memset(&msg, 0, sizeof(msg)); + msg.hdr.opcode = ACM_OP_RESOLVE; + msg.hdr.version = ACM_VERSION; + msg.hdr.length = ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH; + msg.hdr.status = ACM_STATUS_SUCCESS; + msg.hdr.tid = (uint64_t) acmnlmsg; + msg.resolve_data[0].type = ACM_EP_INFO_PATH; + msg.resolve_data[0].flags = (ACM_EP_FLAG_SOURCE | ACM_EP_FLAG_DEST); + memcpy(&msg.resolve_data[0].info.path, &sa_mad->data, + sizeof(struct ibv_path_record)); + + acm_svr_resolve(client, &msg); +} + +static void acm_nl_process_invalid_mad(struct acmc_client *client, + struct acm_nl_resolve_msg *acmnlmsg) +{ + struct acm_msg msg; + + memset(&msg, 0, sizeof(msg)); + msg.hdr.opcode = ACM_OP_RESOLVE; + msg.hdr.version = ACM_VERSION; + msg.hdr.length = ACM_MSG_HDR_LENGTH; + msg.hdr.status = ACM_STATUS_EINVAL; + msg.hdr.tid = (uint64_t) acmnlmsg; + + acm_nl_send(client->sock, &msg); +} + +static void acm_nl_receive(struct acmc_client *client) +{ + struct acm_nl_resolve_msg *acmnlmsg; + int datalen = sizeof(*acmnlmsg); + int ret; + + acmnlmsg = calloc(1, sizeof(*acmnlmsg)); + if (!acmnlmsg) { + acm_log(0, "Out of memory for recving nl msg.\n"); + return; + } + ret = recv(client->sock, acmnlmsg, datalen, 0); + if (ret < 0 && errno == ENOBUFS) { + acm_log(0, "ENOBUFS returned from netlink receive.\n"); + free(acmnlmsg); + return; + } + + acm_log(2, "nlmsg: len %d type 0x%x flags 0x%x seq %d pid %d\n", + acmnlmsg->nlmsg_header.nlmsg_len, + acmnlmsg->nlmsg_header.nlmsg_type, + acmnlmsg->nlmsg_header.nlmsg_flags, + acmnlmsg->nlmsg_header.nlmsg_seq, + acmnlmsg->nlmsg_header.nlmsg_pid); + if (acmnlmsg->nlmsg_header.nlmsg_type != NLMSG_DONE) { + switch (acmnlmsg->mad.mgmt_class) { + case UMAD_CLASS_SUBN_ADM: + if (acm_nl_supported_sa_mad(&acmnlmsg->sa_mad)) { + acm_nl_process_sa_mad(client, acmnlmsg); + } else { + acm_log(1, "WARN SA: mtd %x att %x rmpp %x\n", + acmnlmsg->sa_mad.method, + acmnlmsg->sa_mad.attr_id, + acmnlmsg->sa_mad.rmpp_version); + acm_nl_process_invalid_mad(client, acmnlmsg); + } + break; + default: + /* Not supported*/ + acm_log(1, "WARN - invalid cls %x mtd %x\n", + acmnlmsg->mad.mgmt_class, acmnlmsg->mad.method); + acm_nl_process_invalid_mad(client, acmnlmsg); + break; + } + } else { + free(acmnlmsg); + } +} + +static int acm_init_nl(void) +{ + struct sockaddr_nl src_addr; + int ret; + SOCKET nl_rcv_socket; + + nl_rcv_socket = socket(PF_NETLINK, SOCK_RAW, NETLINK_RDMA); + if (nl_rcv_socket == INVALID_SOCKET) { + acm_log(0, "ERROR - unable to allocate netlink recv socket\n"); + return socket_errno(); + } + + memset(&src_addr, 0, sizeof(src_addr)); + src_addr.nl_family = AF_NETLINK; + src_addr.nl_pid = getpid(); + src_addr.nl_groups = (1 << (RDMA_NL_GROUP_MAD - 1)); + + ret = bind(nl_rcv_socket, (struct sockaddr *)&src_addr, + sizeof(src_addr)); + if (ret == SOCKET_ERROR) { + acm_log(0, "ERROR - unable to bind netlink socket\n"); + return socket_errno(); + } + + /* init nl client structure */ + client_array[NL_CLIENT_INDEX].sock = nl_rcv_socket; + return 0; +} + static void acm_server(void) { fd_set readfds; @@ -1360,12 +1568,14 @@ static void acm_server(void) acm_log(0, "ERROR - server listen failed\n"); return; } + ret = acm_init_nl(); + if (ret) + acm_log(1, "Warn - Netlink init failed\n"); while (1) { n = (int) listen_socket; FD_ZERO(&readfds); FD_SET(listen_socket, &readfds); - n = max(n, (int) ip_mon_socket); FD_SET(ip_mon_socket, &readfds); @@ -1399,7 +1609,10 @@ static void acm_server(void) if (client_array[i].sock != INVALID_SOCKET && FD_ISSET(client_array[i].sock, &readfds)) { acm_log(2, "receiving from client %d\n", i); - acm_svr_receive(&client_array[i]); + if (i == NL_CLIENT_INDEX) + acm_nl_receive(&client_array[i]); + else + acm_svr_receive(&client_array[i]); } }