From patchwork Wed Aug 1 14:52:28 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jim Foraker X-Patchwork-Id: 1264511 X-Patchwork-Delegate: alexne@voltaire.com Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-process-083081@patchwork1.kernel.org Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by patchwork1.kernel.org (Postfix) with ESMTP id 4D6B93FC23 for ; Wed, 1 Aug 2012 14:52:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1755197Ab2HAOwz (ORCPT ); Wed, 1 Aug 2012 10:52:55 -0400 Received: from nspiron-1.llnl.gov ([128.115.41.81]:57070 "EHLO nspiron-1.llnl.gov" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754992Ab2HAOwy (ORCPT ); Wed, 1 Aug 2012 10:52:54 -0400 X-Attachments: Received: from auk75.llnl.gov ([10.253.135.81]) by nspiron-1.llnl.gov with ESMTP; 01 Aug 2012 07:52:54 -0700 From: Jim Foraker To: linux-rdma@vger.kernel.org Cc: weiny2@llnl.gov, alexne@meallanox.com, Jim Foraker Subject: [PATCH 2/9 v2] opensm: Allow recovery of subnets with misset mkeys Date: Wed, 1 Aug 2012 07:52:28 -0700 Message-Id: <1343832755-26753-2-git-send-email-foraker1@llnl.gov> X-Mailer: git-send-email 1.7.9.2 In-Reply-To: <1343832755-26753-1-git-send-email-foraker1@llnl.gov> References: <1343832537.26423.8.camel@auk75.llnl.gov> <1343832755-26753-1-git-send-email-foraker1@llnl.gov> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org Allow the initialization of endpoints that already have an mkey configured that is different than that listed in the configuration file. Signed-off-by: Jim Foraker --- opensm/osm_req.c | 101 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 8 deletions(-) diff --git a/opensm/osm_req.c b/opensm/osm_req.c index 51220f3..d397b14 100644 --- a/opensm/osm_req.c +++ b/opensm/osm_req.c @@ -61,7 +61,86 @@ #include /********************************************************************** - The plock MAY or MAY NOT be held before calling this function. + The plock must be held before calling this function. +**********************************************************************/ +static ib_net64_t req_determine_mkey(IN osm_sm_t * sm, + IN const osm_dr_path_t * p_path) +{ + osm_node_t *p_node; + osm_port_t *p_sm_port; + osm_physp_t *p_physp; + ib_net64_t dest_port_guid, m_key; + uint8_t hop; + + OSM_LOG_ENTER(sm->p_log); + + p_physp = NULL; + + p_sm_port = osm_get_port_by_guid(sm->p_subn, sm->p_subn->sm_port_guid); + + /* hop_count == 0: destination port guid is SM */ + if (p_path->hop_count == 0) { + if (p_sm_port != NULL) + dest_port_guid = sm->p_subn->sm_port_guid; + else + dest_port_guid = sm->p_subn->opt.guid; + goto Remote_Guid; + } + + if (p_sm_port) + p_physp = p_sm_port->p_physp; + + /* hop_count == 1: outgoing physp is SM physp */ + for (hop = 2; p_physp && hop <= p_path->hop_count; hop++) { + p_physp = p_physp->p_remote_physp; + if (!p_physp) + break; + p_node = p_physp->p_node; + p_physp = osm_node_get_physp_ptr(p_node, p_path->path[hop]); + } + + /* At this point, p_physp points at the outgoing physp on the + last hop, or NULL if we don't know it. + */ + if (!p_physp) { + OSM_LOG(sm->p_log, OSM_LOG_ERROR, + "req_determine_mkey: ERR 1107: " + "Outgoing physp is null on non-hop_0!\n"); + dest_port_guid = 0; + goto Remote_Guid; + } + + if (p_physp->p_remote_physp) { + dest_port_guid = p_physp->p_remote_physp->port_guid; + goto Remote_Guid; + } + +Remote_Guid: + if (dest_port_guid) { + if (!osm_db_guid2mkey_get(sm->p_subn->p_g2m, + cl_ntoh64(dest_port_guid), &m_key)) { + m_key = cl_hton64(m_key); + OSM_LOG(sm->p_log, OSM_LOG_DEBUG, + "Found mkey for guid 0x%" + PRIx64 "\n", cl_ntoh64(dest_port_guid)); + } else { + OSM_LOG(sm->p_log, OSM_LOG_DEBUG, + "Target port mkey unknown, using default\n"); + m_key = sm->p_subn->opt.m_key; + } + } else { + OSM_LOG(sm->p_log, OSM_LOG_DEBUG, + "Target port guid unknown, using default\n"); + m_key = sm->p_subn->opt.m_key; + } + + OSM_LOG_EXIT(sm->p_log); + + return m_key; +} + +/********************************************************************** + The plock must be held before calling this function. **********************************************************************/ ib_api_status_t osm_req_get(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path, IN ib_net16_t attr_id, IN ib_net32_t attr_mod, @@ -71,6 +150,7 @@ ib_api_status_t osm_req_get(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path, osm_madw_t *p_madw; ib_api_status_t status = IB_SUCCESS; ib_net64_t tid; + ib_net64_t m_key; CL_ASSERT(sm); @@ -95,15 +175,17 @@ ib_api_status_t osm_req_get(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path, } tid = cl_hton64((uint64_t) cl_atomic_inc(&sm->sm_trans_id)); + m_key = req_determine_mkey(sm, p_path); OSM_LOG(sm->p_log, OSM_LOG_DEBUG, - "Getting %s (0x%X), modifier 0x%X, TID 0x%" PRIx64 "\n", + "Getting %s (0x%X), modifier 0x%X, TID 0x%" PRIx64 + ", MKey 0x%016" PRIx64 "\n", ib_get_sm_attr_str(attr_id), cl_ntoh16(attr_id), - cl_ntoh32(attr_mod), cl_ntoh64(tid)); + cl_ntoh32(attr_mod), cl_ntoh64(tid), cl_ntoh64(m_key)); ib_smp_init_new(osm_madw_get_smp_ptr(p_madw), IB_MAD_METHOD_GET, tid, attr_id, attr_mod, p_path->hop_count, - sm->p_subn->opt.m_key, p_path->path, + m_key, p_path->path, IB_LID_PERMISSIVE, IB_LID_PERMISSIVE); p_madw->mad_addr.dest_lid = IB_LID_PERMISSIVE; @@ -128,7 +210,7 @@ Exit: } /********************************************************************** - The plock MAY or MAY NOT be held before calling this function. + The plock must be held before calling this function. **********************************************************************/ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path, IN const uint8_t * p_payload, @@ -140,6 +222,7 @@ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path, osm_madw_t *p_madw; ib_api_status_t status = IB_SUCCESS; ib_net64_t tid; + ib_net64_t m_key; CL_ASSERT(sm); @@ -165,15 +248,17 @@ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path, } tid = cl_hton64((uint64_t) cl_atomic_inc(&sm->sm_trans_id)); + m_key = req_determine_mkey(sm, p_path); OSM_LOG(sm->p_log, OSM_LOG_DEBUG, - "Setting %s (0x%X), modifier 0x%X, TID 0x%" PRIx64 "\n", + "Setting %s (0x%X), modifier 0x%X, TID 0x%" PRIx64 + ", MKey 0x%016" PRIx64 "\n", ib_get_sm_attr_str(attr_id), cl_ntoh16(attr_id), - cl_ntoh32(attr_mod), cl_ntoh64(tid)); + cl_ntoh32(attr_mod), cl_ntoh64(tid), cl_ntoh64(m_key)); ib_smp_init_new(osm_madw_get_smp_ptr(p_madw), IB_MAD_METHOD_SET, tid, attr_id, attr_mod, p_path->hop_count, - sm->p_subn->opt.m_key, p_path->path, + m_key, p_path->path, IB_LID_PERMISSIVE, IB_LID_PERMISSIVE); p_madw->mad_addr.dest_lid = IB_LID_PERMISSIVE;