From patchwork Mon May 18 19:01:14 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Wan, Kaike" X-Patchwork-Id: 6431701 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork1.web.kernel.org (Postfix) with ESMTP id 6CA669F1CC for ; Mon, 18 May 2015 19:01:40 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id 274E0205DF for ; Mon, 18 May 2015 19:01:39 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 9D96D205DC for ; Mon, 18 May 2015 19:01:37 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754941AbbERTBg (ORCPT ); Mon, 18 May 2015 15:01:36 -0400 Received: from mga09.intel.com ([134.134.136.24]:32994 "EHLO mga09.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754935AbbERTBe (ORCPT ); Mon, 18 May 2015 15:01:34 -0400 Received: from orsmga002.jf.intel.com ([10.7.209.21]) by orsmga102.jf.intel.com with ESMTP; 18 May 2015 12:01:23 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.13,454,1427785200"; d="scan'208";a="731308943" Received: from phlsvsds.ph.intel.com ([10.228.195.38]) by orsmga002.jf.intel.com with ESMTP; 18 May 2015 12:01:22 -0700 Received: from phlsvsds.ph.intel.com (localhost.localdomain [127.0.0.1]) by phlsvsds.ph.intel.com (8.13.8/8.13.8) with ESMTP id t4IJ1LHs023632; Mon, 18 May 2015 15:01:21 -0400 Received: (from kaikewan@localhost) by phlsvsds.ph.intel.com (8.13.8/8.13.8/Submit) id t4IJ1Lic023629; Mon, 18 May 2015 15:01:21 -0400 X-Authentication-Warning: phlsvsds.ph.intel.com: kaikewan set sender to kaike.wan@intel.com using -f From: kaike.wan@intel.com To: linux-rdma@vger.kernel.org Cc: Kaike Wan , John Fleck , Ira Weiny Subject: [PATCH 1/1] ibacm: Add support for pathrecord query through netlink Date: Mon, 18 May 2015 15:01:14 -0400 Message-Id: <1431975674-23602-1-git-send-email-kaike.wan@intel.com> X-Mailer: git-send-email 1.7.1 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, T_RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Kaike Wan This patch enables ibacm to process pathrecord queries through netlink. Since ibacm can cache pathrecords, this implementation provides an easy pathrecord cache for kernel components and therefore offers great performance advantage on large fabric systems. At this time, only non-RMPP SA pathrecord queries are supported. All other queries will be returned with errors. Signed-off-by: Kaike Wan Signed-off-by: John Fleck Signed-off-by: Ira Weiny Reviewed-by: Sean Hefty --- src/acm.c | 219 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 216 insertions(+), 3 deletions(-) diff --git a/src/acm.c b/src/acm.c index 7649725..360ceca 100644 --- a/src/acm.c +++ b/src/acm.c @@ -46,6 +46,8 @@ #include #include #include +#include +#include #include #include #include @@ -55,6 +57,7 @@ #include #include #include +#include #include #include "acm_mad.h" #include "acm_util.h" @@ -66,6 +69,14 @@ #define MAX_EP_ADDR 4 #define NL_MSG_BUF_SIZE 4096 #define ACM_PROV_NAME_SIZE 64 +#define NL_CLIENT_INDEX 0 +#ifndef RDMA_NL_GROUP_MAD + #define RDMA_NL_GROUP_MAD 3 + #define RDMA_NL_MAD 4 + #define RDMA_NL_MAD_REQUEST 0 +#endif +#define RDMA_NL_TYPE_MAD_REQUEST \ + RDMA_NL_GET_TYPE(RDMA_NL_MAD, RDMA_NL_MAD_REQUEST) struct acmc_subnet { DLIST_ENTRY entry; @@ -151,6 +162,14 @@ struct acmc_sa_req { struct acm_sa_mad mad; }; +struct acm_nl_resolve_msg { + struct nlmsghdr nlmsg_header; + union { + struct acm_mad mad; + struct ib_sa_mad sa_mad; + }; +}; + static char def_prov_name[ACM_PROV_NAME_SIZE] = "ibacmp"; static DLIST_ENTRY provider_list; static struct acmc_prov *def_provider = NULL; @@ -172,6 +191,7 @@ static struct acmc_ep *acm_find_ep(struct acmc_port *port, uint16_t pkey); static int acm_ep_insert_addr(struct acmc_ep *ep, const char *name, uint8_t *addr, size_t addr_len, uint8_t addr_type); static void acm_event_handler(struct acmc_device *dev); +static int acm_nl_send(SOCKET sock, struct acm_msg *msg); static struct sa_data { int timeout; @@ -466,7 +486,11 @@ int acm_resolve_response(uint64_t id, struct acm_msg *msg) goto release; } - ret = send(client->sock, (char *) msg, msg->hdr.length, 0); + if (id == NL_CLIENT_INDEX) + ret = acm_nl_send(client->sock, msg); + else + ret = send(client->sock, (char *) msg, msg->hdr.length, 0); + if (ret != msg->hdr.length) acm_log(0, "ERROR - failed to send response\n"); else @@ -597,6 +621,8 @@ static void acm_svr_accept(void) } for (i = 0; i < FD_SETSIZE - 1; i++) { + if (i == NL_CLIENT_INDEX) + continue; if (!atomic_get(&client_array[i].refcnt)) break; } @@ -1346,6 +1372,188 @@ static void acm_ipnl_handler(void) } } +static int acm_nl_send(SOCKET sock, struct acm_msg *msg) +{ + struct sockaddr_nl dst_addr; + struct acm_nl_resolve_msg acmnlmsg; + struct acm_nl_resolve_msg *orig; + struct ib_sa_mad *sa_mad = &acmnlmsg.sa_mad; + int ret; + int datalen; + + orig = (struct acm_nl_resolve_msg *) msg->hdr.tid; + + memset(&dst_addr, 0, sizeof(dst_addr)); + dst_addr.nl_family = AF_NETLINK; + dst_addr.nl_groups = (1 << (RDMA_NL_GROUP_MAD - 1)); + + memset(&acmnlmsg, 0, sizeof(acmnlmsg)); + datalen = sizeof(acmnlmsg); + acmnlmsg.nlmsg_header.nlmsg_len = datalen; + acmnlmsg.nlmsg_header.nlmsg_pid = getpid(); + acmnlmsg.nlmsg_header.nlmsg_type = RDMA_NL_TYPE_MAD_REQUEST; + acmnlmsg.nlmsg_header.nlmsg_flags = NLM_F_REQUEST; + acmnlmsg.nlmsg_header.nlmsg_seq = orig->nlmsg_header.nlmsg_seq; + + if (msg->hdr.status != ACM_STATUS_SUCCESS) { + acm_log(2, "acm status no success = %d\n", msg->hdr.status); + memcpy(&acmnlmsg.mad, &orig->mad, + sizeof(acmnlmsg.mad)); + acmnlmsg.mad.method |= IB_METHOD_RESP; + sa_mad->status = htons(UMAD_SA_STATUS_NO_RECORDS); + } else { + acm_log(2, "acm status success\n"); + memcpy(sa_mad, &orig->sa_mad, + sizeof(*sa_mad) - sizeof(sa_mad->data)); + sa_mad->method |= IB_METHOD_RESP; + sa_mad->status = UMAD_SA_STATUS_SUCCESS; + memcpy(sa_mad->data, &msg->resolve_data[0].info.path, + ACM_MSG_EP_LENGTH); + } + + ret = sendto(sock, &acmnlmsg, datalen, 0, + (const struct sockaddr *)&dst_addr, + (socklen_t)sizeof(dst_addr)); + if (ret != datalen) { + acm_log(0, "ERROR - sendto = %d errno = %d\n", ret, errno); + ret = -1; + } else { + ret = msg->hdr.length; + } + + free(orig); + + return ret; +} + +static inline int acm_nl_supported_sa_mad(struct ib_sa_mad *sa_mad) +{ + return (sa_mad->rmpp_version == 0 && + sa_mad->method == UMAD_METHOD_GET && + sa_mad->attr_id == htons(UMAD_SA_ATTR_PATH_REC)); +} + +static void acm_nl_process_sa_mad(struct acmc_client *client, + struct acm_nl_resolve_msg *acmnlmsg) +{ + struct acm_msg msg; + struct ib_sa_mad *sa_mad = &acmnlmsg->sa_mad; + + acm_format_name(2, log_data, sizeof(log_data), ACM_ADDRESS_GID, + &sa_mad->data[8], sizeof(union ibv_gid)); + acm_log(2, "path dgid %s\n", log_data); + acm_format_name(2, log_data, sizeof(log_data), ACM_ADDRESS_GID, + &sa_mad->data[24], sizeof(union ibv_gid)); + acm_log(2, "path sgid %s\n", log_data); + atomic_inc(&counter[ACM_CNTR_RESOLVE]); + + memset(&msg, 0, sizeof(msg)); + msg.hdr.opcode = ACM_OP_RESOLVE; + msg.hdr.version = ACM_VERSION; + msg.hdr.length = ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH; + msg.hdr.status = ACM_STATUS_SUCCESS; + msg.hdr.tid = (uint64_t) acmnlmsg; + msg.resolve_data[0].type = ACM_EP_INFO_PATH; + msg.resolve_data[0].flags = (ACM_EP_FLAG_SOURCE | ACM_EP_FLAG_DEST); + memcpy(&msg.resolve_data[0].info.path, &sa_mad->data, + sizeof(struct ibv_path_record)); + + acm_svr_resolve(client, &msg); +} + +static void acm_nl_process_invalid_mad(struct acmc_client *client, + struct acm_nl_resolve_msg *acmnlmsg) +{ + struct acm_msg msg; + + memset(&msg, 0, sizeof(msg)); + msg.hdr.opcode = ACM_OP_RESOLVE; + msg.hdr.version = ACM_VERSION; + msg.hdr.length = ACM_MSG_HDR_LENGTH; + msg.hdr.status = ACM_STATUS_EINVAL; + msg.hdr.tid = (uint64_t) acmnlmsg; + + acm_nl_send(client->sock, &msg); +} + +static void acm_nl_receive(struct acmc_client *client) +{ + struct acm_nl_resolve_msg *acmnlmsg; + int datalen = sizeof(*acmnlmsg); + int ret; + + acmnlmsg = calloc(1, sizeof(*acmnlmsg)); + if (!acmnlmsg) { + acm_log(0, "Out of memory for recving nl msg.\n"); + return; + } + ret = recv(client->sock, acmnlmsg, datalen, 0); + if (ret < 0 && errno == ENOBUFS) { + acm_log(0, "ENOBUFS returned from netlink receive.\n"); + free(acmnlmsg); + return; + } + + acm_log(2, "nlmsg: len %d type 0x%x flags 0x%x seq %d pid %d\n", + acmnlmsg->nlmsg_header.nlmsg_len, + acmnlmsg->nlmsg_header.nlmsg_type, + acmnlmsg->nlmsg_header.nlmsg_flags, + acmnlmsg->nlmsg_header.nlmsg_seq, + acmnlmsg->nlmsg_header.nlmsg_pid); + if (acmnlmsg->nlmsg_header.nlmsg_type != NLMSG_DONE) { + switch (acmnlmsg->mad.mgmt_class) { + case UMAD_CLASS_SUBN_ADM: + if (acm_nl_supported_sa_mad(&acmnlmsg->sa_mad)) { + acm_nl_process_sa_mad(client, acmnlmsg); + } else { + acm_log(1, "WARN SA: mtd %x att %x rmpp %x\n", + acmnlmsg->sa_mad.method, + acmnlmsg->sa_mad.attr_id, + acmnlmsg->sa_mad.rmpp_version); + acm_nl_process_invalid_mad(client, acmnlmsg); + } + break; + default: + /* Not supported*/ + acm_log(1, "WARN - invalid cls %x mtd %x\n", + acmnlmsg->mad.mgmt_class, acmnlmsg->mad.method); + acm_nl_process_invalid_mad(client, acmnlmsg); + break; + } + } else { + free(acmnlmsg); + } +} + +static int acm_init_nl(void) +{ + struct sockaddr_nl src_addr; + int ret; + SOCKET nl_rcv_socket; + + nl_rcv_socket = socket(PF_NETLINK, SOCK_RAW, NETLINK_RDMA); + if (nl_rcv_socket == INVALID_SOCKET) { + acm_log(0, "ERROR - unable to allocate netlink recv socket\n"); + return socket_errno(); + } + + memset(&src_addr, 0, sizeof(src_addr)); + src_addr.nl_family = AF_NETLINK; + src_addr.nl_pid = getpid(); + src_addr.nl_groups = (1 << (RDMA_NL_GROUP_MAD - 1)); + + ret = bind(nl_rcv_socket, (struct sockaddr *)&src_addr, + sizeof(src_addr)); + if (ret == SOCKET_ERROR) { + acm_log(0, "ERROR - unable to bind netlink socket\n"); + return socket_errno(); + } + + /* init nl client structure */ + client_array[NL_CLIENT_INDEX].sock = nl_rcv_socket; + return 0; +} + static void acm_server(void) { fd_set readfds; @@ -1360,12 +1568,14 @@ static void acm_server(void) acm_log(0, "ERROR - server listen failed\n"); return; } + ret = acm_init_nl(); + if (ret) + acm_log(1, "Warn - Netlink init failed\n"); while (1) { n = (int) listen_socket; FD_ZERO(&readfds); FD_SET(listen_socket, &readfds); - n = max(n, (int) ip_mon_socket); FD_SET(ip_mon_socket, &readfds); @@ -1399,7 +1609,10 @@ static void acm_server(void) if (client_array[i].sock != INVALID_SOCKET && FD_ISSET(client_array[i].sock, &readfds)) { acm_log(2, "receiving from client %d\n", i); - acm_svr_receive(&client_array[i]); + if (i == NL_CLIENT_INDEX) + acm_nl_receive(&client_array[i]); + else + acm_svr_receive(&client_array[i]); } }