From patchwork Thu Aug 8 23:22:45 2013 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Hefty, Sean" X-Patchwork-Id: 2841477 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork2.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.19.201]) by patchwork2.web.kernel.org (Postfix) with ESMTP id A418FBF546 for ; Thu, 8 Aug 2013 23:23:08 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id 45267203B6 for ; Thu, 8 Aug 2013 23:23:07 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id CF1EF203AF for ; Thu, 8 Aug 2013 23:23:05 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S966880Ab3HHXW7 (ORCPT ); Thu, 8 Aug 2013 19:22:59 -0400 Received: from mga03.intel.com ([143.182.124.21]:43226 "EHLO mga03.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S966696Ab3HHXW5 (ORCPT ); Thu, 8 Aug 2013 19:22:57 -0400 Received: from azsmga002.ch.intel.com ([10.2.17.35]) by azsmga101.ch.intel.com with ESMTP; 08 Aug 2013 16:22:56 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.89,842,1367996400"; d="scan'208";a="279661471" Received: from cst-linux.jf.intel.com ([10.23.221.72]) by AZSMGA002.ch.intel.com with ESMTP; 08 Aug 2013 16:22:56 -0700 From: sean.hefty@intel.com To: linux-rdma@vger.kernel.org Cc: Sean Hefty Subject: [PATCH librdmacm] cmtime: Add example program that times rdma cm calls Date: Thu, 8 Aug 2013 16:22:45 -0700 Message-Id: <1376004165-29921-1-git-send-email-sean.hefty@intel.com> X-Mailer: git-send-email 1.7.3 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Sean Hefty cmtime is a new sample program that measures how long it takes for each step in the connection process to complete. It can be used to analyze the performance of the various CM steps. Signed-off-by: Sean Hefty --- Makefile.am | 4 +- examples/cmtime.c | 498 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 501 insertions(+), 1 deletions(-) create mode 100644 examples/cmtime.c diff --git a/Makefile.am b/Makefile.am index e1be9f3..4e3dee7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,7 +29,7 @@ src_librspreload_la_LIBADD = $(top_builddir)/src/librdmacm.la bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey \ examples/rdma_client examples/rdma_server examples/rdma_xclient \ examples/rdma_xserver examples/rstream examples/rcopy \ - examples/riostream examples/udpong + examples/riostream examples/udpong examples/cmtime examples_ucmatose_SOURCES = examples/cmatose.c examples/common.c examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la examples_rping_SOURCES = examples/rping.c @@ -54,6 +54,8 @@ examples_rcopy_SOURCES = examples/rcopy.c examples_rcopy_LDADD = $(top_builddir)/src/librdmacm.la examples_udpong_SOURCES = examples/udpong.c examples/common.c examples_udpong_LDADD = $(top_builddir)/src/librdmacm.la +examples_cmtime_SOURCES = examples/cmtime.c examples/common.c +examples_cmtime_LDADD = $(top_builddir)/src/librdmacm.la librdmacmincludedir = $(includedir)/rdma infinibandincludedir = $(includedir)/infiniband diff --git a/examples/cmtime.c b/examples/cmtime.c new file mode 100644 index 0000000..f97ab9e --- /dev/null +++ b/examples/cmtime.c @@ -0,0 +1,498 @@ +/* + * Copyright (c) 2013 Intel Corporation. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "common.h" + +static struct rdma_addrinfo hints, *rai; +static struct rdma_event_channel *channel; +static char *port = "7471"; +static char *dst_addr; +static char *src_addr; + +enum step { + STEP_CREATE_ID, + STEP_RESOLVE_ADDR, + STEP_RESOLVE_ROUTE, + STEP_CREATE_QP, + STEP_CONNECT, + STEP_DISCONNECT, + STEP_DESTROY, + STEP_CNT +}; + +static char *step_str[] = { + "create id", + "resolve addr", + "resolve route", + "create qp", + "connect", + "disconnect", + "destroy" +}; + +struct node { + struct rdma_cm_id *id; + struct timeval times[STEP_CNT][2]; + int error; +}; + +static struct node *nodes; +static struct timeval times[STEP_CNT][2]; +static int connections = 100; +static int left[STEP_CNT]; +static struct ibv_qp_init_attr init_qp_attr; +static struct rdma_conn_param conn_param; + +#define start_perf(n, s) gettimeofday(&((n)->times[s][0]), NULL) +#define end_perf(n, s) gettimeofday(&((n)->times[s][1]), NULL) +#define start_time(s) gettimeofday(×[s][0], NULL) +#define end_time(s) gettimeofday(×[s][1], NULL) + +static int zero_time(struct timeval *t) +{ + return !(t->tv_sec || t->tv_usec); +} + +static float diff_us(struct timeval *end, struct timeval *start) +{ + return (end->tv_sec - start->tv_sec) * 1000000. + (end->tv_usec - start->tv_usec); +} + +static void show_perf(void) +{ + int c, i; + float us, max[STEP_CNT], min[STEP_CNT]; + + for (i = 0; i < STEP_CNT; i++) { + max[i] = 0; + min[i] = 999999999.; + for (c = 0; c < connections; c++) { + if (!zero_time(&nodes[c].times[i][0]) && + !zero_time(&nodes[c].times[i][1])) { + us = diff_us(&nodes[c].times[i][1], &nodes[c].times[i][0]); + if (us > max[i]) + max[i] = us; + if (us < min[i]) + min[i] = us; + } + } + } + + printf("step total ms max ms min us us / conn\n"); + for (i = 0; i < STEP_CNT; i++) { + us = diff_us(×[i][1], ×[i][0]); + printf("%-13s: %11.2f%11.2f%11.2f%11.2f\n", step_str[i], us / 1000., + max[i] / 1000., min[i], us / connections); + } +} + +static void addr_handler(struct node *n) +{ + end_perf(n, STEP_RESOLVE_ADDR); + left[STEP_RESOLVE_ADDR]--; +} + +static void route_handler(struct node *n) +{ + end_perf(n, STEP_RESOLVE_ROUTE); + left[STEP_RESOLVE_ROUTE]--; +} + +static void conn_handler(struct node *n) +{ + end_perf(n, STEP_CONNECT); + left[STEP_CONNECT]--; +} + +static void disc_handler(struct node *n) +{ + end_perf(n, STEP_DISCONNECT); + left[STEP_DISCONNECT]--; +} + +static int req_handler(struct rdma_cm_id *id) +{ + int ret; + + ret = rdma_create_qp(id, NULL, &init_qp_attr); + if (ret) { + perror("failure creating qp"); + goto err; + } + + ret = rdma_accept(id, NULL); + if (ret) { + perror("failure accepting"); + goto err; + } + return 0; + +err: + printf("failing connection request\n"); + rdma_reject(id, NULL, 0); + return ret; +} + +static void cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) +{ + struct node *n = id->context; + + switch (event->event) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + addr_handler(n); + break; + case RDMA_CM_EVENT_ROUTE_RESOLVED: + route_handler(n); + break; + case RDMA_CM_EVENT_CONNECT_REQUEST: + if (req_handler(id)) { + rdma_ack_cm_event(event); + rdma_destroy_id(id); + return; + } + break; + case RDMA_CM_EVENT_ESTABLISHED: + if (n) + conn_handler(n); + break; + case RDMA_CM_EVENT_ADDR_ERROR: + printf("RDMA_CM_EVENT_ADDR_ERROR, error: %d\n", event->status); + addr_handler(n); + n->error = 1; + break; + case RDMA_CM_EVENT_ROUTE_ERROR: + printf("RDMA_CM_EVENT_ROUTE_ERROR, error: %d\n", event->status); + route_handler(n); + n->error = 1; + break; + case RDMA_CM_EVENT_CONNECT_ERROR: + case RDMA_CM_EVENT_UNREACHABLE: + case RDMA_CM_EVENT_REJECTED: + printf("event: %s, error: %d\n", + rdma_event_str(event->event), event->status); + conn_handler(n); + n->error = 1; + break; + case RDMA_CM_EVENT_DISCONNECTED: + if (!n) { + rdma_disconnect(id); + rdma_ack_cm_event(event); + rdma_destroy_id(id); + return; + } + disc_handler(n); + break; + case RDMA_CM_EVENT_DEVICE_REMOVAL: + /* Cleanup will occur after test completes. */ + break; + default: + break; + } + rdma_ack_cm_event(event); +} + +static int alloc_nodes(void) +{ + int ret, i; + + nodes = calloc(sizeof *nodes, connections); + if (!nodes) + return -ENOMEM; + + printf("creating id\n"); + start_time(STEP_CREATE_ID); + for (i = 0; i < connections; i++) { + start_perf(&nodes[i], STEP_CREATE_ID); + if (dst_addr) { + ret = rdma_create_id(channel, &nodes[i].id, &nodes[i], + hints.ai_port_space); + if (ret) + goto err; + } + end_perf(&nodes[i], STEP_CREATE_ID); + } + end_time(STEP_CREATE_ID); + return 0; + +err: + while (--i >= 0) + rdma_destroy_id(nodes[i].id); + free(nodes); + return ret; +} + +static void cleanup_nodes(void) +{ + int i; + + printf("destroying id\n"); + start_time(STEP_DESTROY); + for (i = 0; i < connections; i++) { + start_perf(&nodes[i], STEP_DESTROY); + if (nodes[i].id) + rdma_destroy_id(nodes[i].id); + end_perf(&nodes[i], STEP_DESTROY); + } + end_time(STEP_DESTROY); +} + +static int process_events(int *left) +{ + struct rdma_cm_event *event; + int ret = 0; + + while ((!left || *left) && !ret) { + ret = rdma_get_cm_event(channel, &event); + if (!ret) { + cma_handler(event->id, event); + } else { + perror("failure in rdma_get_cm_event in connect events"); + ret = errno; + } + } + + return ret; +} + +static int run_server(void) +{ + struct rdma_cm_id *listen_id; + int ret; + + ret = rdma_create_id(channel, &listen_id, NULL, hints.ai_port_space); + if (ret) { + perror("listen request failed"); + return ret; + } + + ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &rai); + if (ret) { + perror("getrdmaaddr error"); + goto out; + } + + ret = rdma_bind_addr(listen_id, rai->ai_src_addr); + if (ret) { + perror("bind address failed"); + goto out; + } + + ret = rdma_listen(listen_id, 0); + if (ret) { + perror("failure trying to listen"); + goto out; + } + + process_events(NULL); + out: + rdma_destroy_id(listen_id); + return ret; +} + +static int run_client(void) +{ + int i, ret; + + ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &rai); + if (ret) { + perror("getaddrinfo error"); + return ret; + } + + conn_param.responder_resources = 1; + conn_param.initiator_depth = 1; + conn_param.retry_count = 5; + conn_param.private_data = rai->ai_connect; + conn_param.private_data_len = rai->ai_connect_len; + + printf("resolving address\n"); + start_time(STEP_RESOLVE_ADDR); + for (i = 0; i < connections; i++) { + start_perf(&nodes[i], STEP_RESOLVE_ADDR); + ret = rdma_resolve_addr(nodes[i].id, rai->ai_src_addr, + rai->ai_dst_addr, 2000); + if (ret) { + perror("failure getting addr"); + nodes[i].error = 1; + continue; + } + left[STEP_RESOLVE_ADDR]++; + } + ret = process_events(&left[STEP_RESOLVE_ADDR]); + if (ret) + return ret; + end_time(STEP_RESOLVE_ADDR); + + printf("resolving route\n"); + start_time(STEP_RESOLVE_ROUTE); + for (i = 0; i < connections; i++) { + if (nodes[i].error) + continue; + start_perf(&nodes[i], STEP_RESOLVE_ROUTE); + ret = rdma_resolve_route(nodes[i].id, 2000); + if (ret) { + perror("failure resolving route"); + nodes[i].error = 1; + continue; + } + left[STEP_RESOLVE_ROUTE]++; + } + ret = process_events(&left[STEP_RESOLVE_ROUTE]); + if (ret) + return ret; + end_time(STEP_RESOLVE_ROUTE); + + printf("creating qp\n"); + start_time(STEP_CREATE_QP); + for (i = 0; i < connections; i++) { + if (nodes[i].error) + continue; + start_perf(&nodes[i], STEP_CREATE_QP); + ret = rdma_create_qp(nodes[i].id, NULL, &init_qp_attr); + if (ret) { + perror("failure creating qp"); + nodes[i].error = 1; + continue; + } + end_perf(&nodes[i], STEP_CREATE_QP); + } + end_time(STEP_CREATE_QP); + + printf("connecting\n"); + start_time(STEP_CONNECT); + for (i = 0; i < connections; i++) { + if (nodes[i].error) + continue; + start_perf(&nodes[i], STEP_CONNECT); + ret = rdma_connect(nodes[i].id, &conn_param); + if (ret) { + perror("failure rconnecting"); + nodes[i].error = 1; + continue; + } + left[STEP_CONNECT]++; + } + ret = process_events(&left[STEP_CONNECT]); + if (ret) + return ret; + end_time(STEP_CONNECT); + + printf("disconnecting\n"); + start_time(STEP_DISCONNECT); + for (i = 0; i < connections; i++) { + if (nodes[i].error) + continue; + start_perf(&nodes[i], STEP_DISCONNECT); + rdma_disconnect(nodes[i].id); + left[STEP_DISCONNECT]++; + } + ret = process_events(&left[STEP_DISCONNECT]); + if (ret) + return ret; + end_time(STEP_DISCONNECT); + + return ret; +} + +int main(int argc, char **argv) +{ + int op, ret; + + hints.ai_port_space = RDMA_PS_TCP; + hints.ai_qp_type = IBV_QPT_RC; + while ((op = getopt(argc, argv, "s:b:c:p:")) != -1) { + switch (op) { + case 's': + dst_addr = optarg; + break; + case 'b': + src_addr = optarg; + break; + case 'c': + connections = atoi(optarg); + break; + case 'p': + port = optarg; + break; + default: + printf("usage: %s\n", argv[0]); + printf("\t[-s server_address]\n"); + printf("\t[-b bind_address]\n"); + printf("\t[-c connections]\n"); + printf("\t[-p port_number]\n"); + exit(1); + } + } + + init_qp_attr.cap.max_send_wr = 1; + init_qp_attr.cap.max_recv_wr = 1; + init_qp_attr.cap.max_send_sge = 1; + init_qp_attr.cap.max_recv_sge = 1; + init_qp_attr.qp_type = IBV_QPT_RC; + + channel = rdma_create_event_channel(); + if (!channel) { + printf("failed to create event channel\n"); + exit(1); + } + + if (dst_addr) { + alloc_nodes(); + ret = run_client(); + } else { + hints.ai_flags |= RAI_PASSIVE; + ret = run_server(); + } + + cleanup_nodes(); + rdma_destroy_event_channel(channel); + if (rai) + rdma_freeaddrinfo(rai); + + show_perf(); + free(nodes); + return ret; +}