From patchwork Mon Jun 7 12:40:57 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Hal Rosenstock X-Patchwork-Id: 104746 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter.kernel.org (8.14.3/8.14.3) with ESMTP id o57CeoO2013308 for ; Mon, 7 Jun 2010 12:40:50 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752818Ab0FGMj5 (ORCPT ); Mon, 7 Jun 2010 08:39:57 -0400 Received: from qmta01.westchester.pa.mail.comcast.net ([76.96.62.16]:32911 "EHLO qmta01.westchester.pa.mail.comcast.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752595Ab0FGMjz (ORCPT ); Mon, 7 Jun 2010 08:39:55 -0400 Received: from omta08.westchester.pa.mail.comcast.net ([76.96.62.12]) by qmta01.westchester.pa.mail.comcast.net with comcast id SybN1e0030Fqzac510fv3h; Mon, 07 Jun 2010 12:39:55 +0000 Received: from hal.comcast.net ([75.69.247.31]) by omta08.westchester.pa.mail.comcast.net with comcast id T0fu1e00K0hNrtn3U0fvWe; Mon, 07 Jun 2010 12:39:55 +0000 Received: from hal.comcast.net (localhost.localdomain [127.0.0.1]) by hal.comcast.net (8.14.3/8.14.3) with ESMTP id o57CfnuN002102; Mon, 7 Jun 2010 08:41:55 -0400 Received: (from hnrose@localhost) by hal.comcast.net (8.14.3/8.14.3/Submit) id o57Cev5q002070; Mon, 7 Jun 2010 08:40:57 -0400 Date: Mon, 7 Jun 2010 08:40:57 -0400 From: Hal Rosenstock To: sashak@voltaire.com Cc: linux-rdma@vger.kernel.org Subject: [PATCHv2] opensm: Add a rate based mechanism for SMP transactions Message-ID: <20100607124057.GA2060@comcast.net> MIME-Version: 1.0 Content-Disposition: inline User-Agent: Mutt/1.5.19 (2009-01-05) Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter.kernel.org [140.211.167.41]); Mon, 07 Jun 2010 12:40:51 +0000 (UTC) diff --git a/opensm/include/opensm/osm_base.h b/opensm/include/opensm/osm_base.h index e0d6c66..16241ec 100644 --- a/opensm/include/opensm/osm_base.h +++ b/opensm/include/opensm/osm_base.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * @@ -440,6 +440,30 @@ BEGIN_C_DECLS */ #define OSM_DEFAULT_SMP_MAX_ON_WIRE 4 /***********/ +/****d* OpenSM: Base/OSM_DEFAULT_SMP_RATE_MAX +* NAME +* OSM_DEFAULT_SMP_RATE_MAX +* +* DESCRIPTION +* Specifies the default maximum number of outstanding rate based SMPs. +* +* SYNOPSIS +*/ +#define OSM_DEFAULT_SMP_RATE_MAX 100 +/***********/ +/****d* OpenSM: Base/OSM_DEFAULT_SMP_RATE +* NAME +* OSM_DEFAULT_SMP_RATE +* +* DESCRIPTION +* Specifies the default rate (in usec) for rate based SMPs. +* The default rate is 0. A value of 0 (or EVENT_NO_TIMEOUT) +* disables the rate based SMP mechanism. +* +* SYNOPSIS +*/ +#define OSM_DEFAULT_SMP_RATE 0 +/***********/ /****d* OpenSM: Base/OSM_SM_DEFAULT_QP0_RCV_SIZE * NAME * OSM_SM_DEFAULT_QP0_RCV_SIZE diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 4e8c862..49acf2d 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. @@ -149,6 +149,8 @@ typedef struct osm_subn_opt { ib_net16_t m_key_lease_period; uint32_t sweep_interval; uint32_t max_wire_smps; + uint32_t max_rate_based_smps; + uint32_t rate_based_smp_usecs; uint32_t transaction_timeout; uint32_t transaction_retries; uint8_t sm_priority; @@ -264,6 +266,14 @@ typedef struct osm_subn_opt { * max_wire_smps * The maximum number of SMPs sent in parallel. Default is 4. * +* max_rate_based_smps +* The maximum number of rate based SMPs allowed to be outstanding. +* Default is 1000. +* +* rate_based_smp_usecs +* The wait time in usec for rate based SMPs. Default is 0 +* (disabled). +* * transaction_timeout * The maximum time in milliseconds allowed for a transaction * to complete. Default is 200. diff --git a/opensm/include/opensm/osm_vl15intf.h b/opensm/include/opensm/osm_vl15intf.h index 15ed56c..887733f 100644 --- a/opensm/include/opensm/osm_vl15intf.h +++ b/opensm/include/opensm/osm_vl15intf.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -117,6 +117,8 @@ typedef struct osm_vl15 { osm_thread_state_t thread_state; osm_vl15_state_t state; uint32_t max_wire_smps; + uint32_t max_rate_based_smps; + uint32_t rate_based_smp_usecs; cl_event_t signal; cl_thread_t poller; cl_qlist_t rfifo; @@ -137,6 +139,12 @@ typedef struct osm_vl15 { * max_wire_smps * Maximum number of VL15 MADs allowed on the wire at one time. * +* max_rate_based_smps +* Maximum number of rate based SMPs allowed to be outstanding. +* +* rate_based_smp_usecs +* Wait time in usec for rate based SMPs. +* * signal * Event on which the poller sleeps. * @@ -243,7 +251,9 @@ void osm_vl15_destroy(IN osm_vl15_t * p_vl15, IN struct osm_mad_pool *p_pool); */ ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl15, IN osm_vendor_t * p_vend, IN osm_log_t * p_log, IN osm_stats_t * p_stats, - IN int32_t max_wire_smps); + IN int32_t max_wire_smps, + IN int32_t max_rate_based_smps, + IN uint32_t rate_based_smp_usecs); /* * PARAMETERS * p_vl15 @@ -261,6 +271,14 @@ ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl15, IN osm_vendor_t * p_vend, * max_wire_smps * [in] Maximum number of MADs allowed on the wire at one time. * +* max_rate_based_smps +* [in] Maximum number of rate based SMPs allowed to be +* outstanding. +* +* rate_based_smp_usecs +* [in] Wait time in usec for rate based SMPs. +* +* * RETURN VALUES * IB_SUCCESS if the VL15 object was initialized successfully. * diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index 0093aa7..83b94b0 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. @@ -743,8 +743,8 @@ int main(int argc, char *argv[]) case 'n': opt.max_wire_smps = strtoul(optarg, NULL, 0); if (opt.max_wire_smps == 0 || - opt.max_wire_smps > 0x7FFFFFFF) - opt.max_wire_smps = 0x7FFFFFFF; + opt.max_wire_smps > 0x3FFFFFFF) + opt.max_wire_smps = 0x3FFFFFFF; printf(" Max wire smp's = %d\n", opt.max_wire_smps); break; diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c index d0f39da..bf2014f 100644 --- a/opensm/opensm/osm_opensm.c +++ b/opensm/opensm/osm_opensm.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -392,7 +392,9 @@ ib_api_status_t osm_opensm_init(IN osm_opensm_t * p_osm, status = osm_vl15_init(&p_osm->vl15, p_osm->p_vendor, &p_osm->log, &p_osm->stats, - p_opt->max_wire_smps); + p_opt->max_wire_smps, + p_opt->max_rate_based_smps, + p_opt->rate_based_smp_usecs); if (status != IB_SUCCESS) goto Exit; diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ba2c812..6aaf6dc 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. @@ -297,6 +297,8 @@ static const opt_rec_t opt_tbl[] = { { "m_key_lease_period", OPT_OFFSET(m_key_lease_period), opts_parse_net16, NULL, 1 }, { "sweep_interval", OPT_OFFSET(sweep_interval), opts_parse_uint32, NULL, 1 }, { "max_wire_smps", OPT_OFFSET(max_wire_smps), opts_parse_uint32, NULL, 1 }, + { "max_rate_based_smps", OPT_OFFSET(max_rate_based_smps), opts_parse_uint32, NULL, 1 }, + { "rate_based_smp_usecs", OPT_OFFSET(rate_based_smp_usecs), opts_parse_uint32, NULL, 1 }, { "console", OPT_OFFSET(console), opts_parse_charp, NULL, 0 }, { "console_port", OPT_OFFSET(console_port), opts_parse_uint16, NULL, 0 }, { "transaction_timeout", OPT_OFFSET(transaction_timeout), opts_parse_uint32, NULL, 0 }, @@ -670,6 +672,8 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * p_opt) p_opt->m_key_lease_period = 0; p_opt->sweep_interval = OSM_DEFAULT_SWEEP_INTERVAL_SECS; p_opt->max_wire_smps = OSM_DEFAULT_SMP_MAX_ON_WIRE; + p_opt->max_rate_based_smps = OSM_DEFAULT_SMP_RATE_MAX; + p_opt->rate_based_smp_usecs = OSM_DEFAULT_SMP_RATE; p_opt->console = strdup(OSM_DEFAULT_CONSOLE); p_opt->console_port = OSM_DEFAULT_CONSOLE_PORT; p_opt->transaction_timeout = OSM_DEFAULT_TRANS_TIMEOUT_MILLISEC; @@ -1066,14 +1070,22 @@ int osm_subn_verify_config(IN osm_subn_opt_t * p_opts) } if (p_opts->max_wire_smps == 0) - p_opts->max_wire_smps = 0x7FFFFFFF; - else if (p_opts->max_wire_smps > 0x7FFFFFFF) { + p_opts->max_wire_smps = 0x3FFFFFFF; + else if (p_opts->max_wire_smps > 0x3FFFFFFF) { log_report(" Invalid Cached Option Value: max_wire_smps = %u," " Using Default: %u\n", p_opts->max_wire_smps, OSM_DEFAULT_SMP_MAX_ON_WIRE); p_opts->max_wire_smps = OSM_DEFAULT_SMP_MAX_ON_WIRE; } + if (p_opts->max_rate_based_smps > 0x3FFFFFFF) { + log_report(" Invalid Cached Option Value:" + " max_rate_based_smps = %u, Using Default: %u", + p_opts->max_rate_based_smps, + OSM_DEFAULT_SMP_RATE_MAX); + p_opts->max_rate_based_smps = OSM_DEFAULT_SMP_RATE_MAX; + } + if (strcmp(p_opts->console, OSM_DISABLE_CONSOLE) && strcmp(p_opts->console, OSM_LOCAL_CONSOLE) #ifdef ENABLE_OSM_CONSOLE_SOCKET @@ -1482,6 +1494,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts) "#\n# TIMING AND THREADING OPTIONS\n#\n" "# Maximum number of SMPs sent in parallel\n" "max_wire_smps %u\n\n" + "# Maximum number of rate based SMPs allowed to be outstanding\n" + "max_rate_based_smps %u\n\n" + "# The rate in [usec] at which rate based SMPs are sent\n" + "# A value of 0 disables the rate based SMP mechanism\n" + "rate_based_smp_usecs %u\n\n" "# The maximum time in [msec] allowed for a transaction to complete\n" "transaction_timeout %u\n\n" "# The maximum number of retries allowed for a transaction to complete\n" @@ -1494,6 +1511,8 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts) "# Use a single thread for handling SA queries\n" "single_thread %s\n\n", p_opts->max_wire_smps, + p_opts->max_rate_based_smps, + p_opts->rate_based_smp_usecs, p_opts->transaction_timeout, p_opts->transaction_retries, p_opts->max_msg_fifo_timeout, diff --git a/opensm/opensm/osm_vl15intf.c b/opensm/opensm/osm_vl15intf.c index ff9e4db..fdcca44 100644 --- a/opensm/opensm/osm_vl15intf.c +++ b/opensm/opensm/osm_vl15intf.c @@ -1,7 +1,7 @@ /* * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2006,2009 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -113,6 +113,8 @@ static void vl15_poller(IN void *p_ptr) osm_madw_t *p_madw; osm_vl15_t *p_vl = p_ptr; cl_qlist_t *p_fifo; + int32_t max_smps = p_vl->max_wire_smps; + int32_t max_smps2 = p_vl->max_wire_smps + p_vl->max_rate_based_smps; OSM_LOG_ENTER(p_vl->p_log); @@ -155,17 +157,22 @@ static void vl15_poller(IN void *p_ptr) status = cl_event_wait_on(&p_vl->signal, EVENT_NO_TIMEOUT, TRUE); - while (p_vl->p_stats->qp0_mads_outstanding_on_wire >= - (int32_t) p_vl->max_wire_smps && + while (p_vl->p_stats->qp0_mads_outstanding_on_wire >= max_smps && p_vl->thread_state == OSM_THREAD_STATE_RUN) { status = cl_event_wait_on(&p_vl->signal, - EVENT_NO_TIMEOUT, TRUE); - if (status != CL_SUCCESS) { + p_vl->rate_based_smp_usecs, + TRUE); + if (status == CL_TIMEOUT) { + if (max_smps < max_smps2) + max_smps++; + break; + } else if (status != CL_SUCCESS) { OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E02: " "Event wait failed (%s)\n", CL_STATUS_MSG(status)); break; } + max_smps = p_vl->max_wire_smps; } } @@ -236,7 +243,9 @@ void osm_vl15_destroy(IN osm_vl15_t * p_vl, IN struct osm_mad_pool *p_pool) ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl, IN osm_vendor_t * p_vend, IN osm_log_t * p_log, IN osm_stats_t * p_stats, - IN int32_t max_wire_smps) + IN int32_t max_wire_smps, + IN int32_t max_rate_based_smps, + IN uint32_t rate_based_smp_usecs) { ib_api_status_t status = IB_SUCCESS; @@ -246,6 +255,10 @@ ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl, IN osm_vendor_t * p_vend, p_vl->p_log = p_log; p_vl->p_stats = p_stats; p_vl->max_wire_smps = max_wire_smps; + p_vl->max_rate_based_smps = rate_based_smp_usecs && max_rate_based_smps ? + max_rate_based_smps : 0; + p_vl->rate_based_smp_usecs = rate_based_smp_usecs && max_rate_based_smps ? + rate_based_smp_usecs : EVENT_NO_TIMEOUT; status = cl_event_init(&p_vl->signal, FALSE); if (status != IB_SUCCESS)