From patchwork Sun Oct 7 23:19:37 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: NeilBrown X-Patchwork-Id: 10629799 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 13F3514DB for ; Sun, 7 Oct 2018 23:30:05 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 03A7528AD0 for ; Sun, 7 Oct 2018 23:30:05 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id EBF3128CC0; Sun, 7 Oct 2018 23:30:04 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-2.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_NONE autolearn=ham version=3.3.1 Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 6FBE428AD0 for ; Sun, 7 Oct 2018 23:30:04 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 2A346861766; Sun, 7 Oct 2018 16:30:04 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from mx1.suse.de (mx2.suse.de [195.135.220.15]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 88BAD8616E5 for ; Sun, 7 Oct 2018 16:30:02 -0700 (PDT) X-Virus-Scanned: by amavisd-new at test-mx.suse.de Received: from relay2.suse.de (unknown [195.135.220.254]) by mx1.suse.de (Postfix) with ESMTP id 9BCE8AD2C; Sun, 7 Oct 2018 23:30:01 +0000 (UTC) From: NeilBrown To: Oleg Drokin , Doug Oucharek , James Simmons , Andreas Dilger Date: Mon, 08 Oct 2018 10:19:37 +1100 Message-ID: <153895437770.16383.3391026679795909640.stgit@noble> In-Reply-To: <153895417139.16383.3791701638653772865.stgit@noble> References: <153895417139.16383.3791701638653772865.stgit@noble> User-Agent: StGit/0.17.1-dirty MIME-Version: 1.0 Subject: [lustre-devel] [PATCH 04/24] lustre: lnet: automatic sizing of router pinger buffers X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Amir Shehata , Olaf Weber , Lustre Development List Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" X-Virus-Scanned: ClamAV using ClamSMTP From: Olaf Weber The router pinger uses fixed-size buffers to receive the data returned by a ping. When a router has more than 16 interfaces (including loopback) this means the data for some interfaces is dropped. Detect this situation, and track the number of remote NIs in the lnet_rc_data_t structure. lnet_create_rc_data_locked() becomes lnet_update_rc_data_locked(), and modified to replace an existing ping buffer if one is present. It is now also called by lnet_ping_router_locked() when the existing ping buffer is too small. WC-bug-id: https://jira.whamcloud.com/browse/LU-9480 Signed-off-by: Olaf Weber Reviewed-on: https://review.whamcloud.com/25774 Reviewed-by: Olaf Weber Reviewed-by: Amir Shehata Tested-by: Amir Shehata Signed-off-by: NeilBrown Reviewed-by: James Simmons Reviewed-by: James Simmons --- .../staging/lustre/include/linux/lnet/lib-types.h | 4 - drivers/staging/lustre/lnet/lnet/router.c | 90 +++++++++++++------- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h index ab8c6d66cdbf..d1d17ededd06 100644 --- a/drivers/staging/lustre/include/linux/lnet/lib-types.h +++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h @@ -411,8 +411,6 @@ struct lnet_ping_buffer { /* router checker data, per router */ -#define LNET_MAX_RTR_NIS LNET_INTERFACES_MIN -#define LNET_RTR_PINGINFO_SIZE LNET_PING_INFO_SIZE(LNET_MAX_RTR_NIS) struct lnet_rc_data { /* chain on the_lnet.ln_zombie_rcd or ln_deathrow_rcd */ struct list_head rcd_list; @@ -422,6 +420,8 @@ struct lnet_rc_data { struct lnet_peer_ni *rcd_gateway; /* ping buffer */ struct lnet_ping_buffer *rcd_pingbuffer; + /* desired size of buffer */ + int rcd_nnis; }; struct lnet_peer_ni { diff --git a/drivers/staging/lustre/lnet/lnet/router.c b/drivers/staging/lustre/lnet/lnet/router.c index e97957ce9252..86cce27e10d8 100644 --- a/drivers/staging/lustre/lnet/lnet/router.c +++ b/drivers/staging/lustre/lnet/lnet/router.c @@ -678,8 +678,11 @@ lnet_parse_rc_info(struct lnet_rc_data *rcd) /* Determine the number of NIs for which there is data. */ nnis = pbuf->pb_info.pi_nnis; - if (pbuf->pb_nnis < nnis) + if (pbuf->pb_nnis < nnis) { + if (rcd->rcd_nnis < nnis) + rcd->rcd_nnis = nnis; nnis = pbuf->pb_nnis; + } list_for_each_entry(rte, &gw->lpni_routes, lr_gwlist) { int down = 0; @@ -911,28 +914,47 @@ lnet_destroy_rc_data(struct lnet_rc_data *rcd) } static struct lnet_rc_data * -lnet_create_rc_data_locked(struct lnet_peer_ni *gateway) +lnet_update_rc_data_locked(struct lnet_peer_ni *gateway) { - struct lnet_rc_data *rcd = NULL; - struct lnet_ping_buffer *pbuf; + struct lnet_handle_md mdh; + struct lnet_rc_data *rcd; + struct lnet_ping_buffer *pbuf = NULL; struct lnet_md md; + int nnis = LNET_INTERFACES_MIN; int rc; int i; + rcd = gateway->lpni_rcd; + if (rcd) { + nnis = rcd->rcd_nnis; + mdh = rcd->rcd_mdh; + LNetInvalidateMDHandle(&rcd->rcd_mdh); + pbuf = rcd->rcd_pingbuffer; + rcd->rcd_pingbuffer = NULL; + } else { + LNetInvalidateMDHandle(&mdh); + } + lnet_net_unlock(gateway->lpni_cpt); - rcd = kzalloc(sizeof(*rcd), GFP_NOFS); - if (!rcd) - goto out; + if (rcd) { + LNetMDUnlink(mdh); + lnet_ping_buffer_decref(pbuf); + } else { + rcd = kzalloc(sizeof(*rcd), GFP_NOFS); + if (!rcd) + goto out; - LNetInvalidateMDHandle(&rcd->rcd_mdh); - INIT_LIST_HEAD(&rcd->rcd_list); + LNetInvalidateMDHandle(&rcd->rcd_mdh); + INIT_LIST_HEAD(&rcd->rcd_list); + rcd->rcd_nnis = nnis; + } - pbuf = lnet_ping_buffer_alloc(LNET_MAX_RTR_NIS, GFP_NOFS); + pbuf = lnet_ping_buffer_alloc(nnis, GFP_NOFS); if (!pbuf) goto out; - for (i = 0; i < LNET_MAX_RTR_NIS; i++) { + for (i = 0; i < nnis; i++) { pbuf->pb_info.pi_ni[i].ns_nid = LNET_NID_ANY; pbuf->pb_info.pi_ni[i].ns_status = LNET_NI_STATUS_INVALID; } @@ -940,7 +962,7 @@ lnet_create_rc_data_locked(struct lnet_peer_ni *gateway) md.start = &pbuf->pb_info; md.user_ptr = rcd; - md.length = LNET_RTR_PINGINFO_SIZE; + md.length = LNET_PING_INFO_SIZE(nnis); md.threshold = LNET_MD_THRESH_INF; md.options = LNET_MD_TRUNCATE; md.eq_handle = the_lnet.ln_rc_eqh; @@ -949,33 +971,37 @@ lnet_create_rc_data_locked(struct lnet_peer_ni *gateway) rc = LNetMDBind(md, LNET_UNLINK, &rcd->rcd_mdh); if (rc < 0) { CERROR("Can't bind MD: %d\n", rc); - goto out; + goto out_ping_buffer_decref; } LASSERT(!rc); lnet_net_lock(gateway->lpni_cpt); - /* router table changed or someone has created rcd for this gateway */ - if (!lnet_isrouter(gateway) || gateway->lpni_rcd) { - lnet_net_unlock(gateway->lpni_cpt); - goto out; + /* Check if this is still a router. */ + if (!lnet_isrouter(gateway)) + goto out_unlock; + /* Check if someone else installed router data. */ + if (gateway->lpni_rcd && gateway->lpni_rcd != rcd) + goto out_unlock; + + /* Install and/or update the router data. */ + if (!gateway->lpni_rcd) { + lnet_peer_ni_addref_locked(gateway); + rcd->rcd_gateway = gateway; + gateway->lpni_rcd = rcd; } - - lnet_peer_ni_addref_locked(gateway); - rcd->rcd_gateway = gateway; - gateway->lpni_rcd = rcd; gateway->lpni_ping_notsent = 0; return rcd; - out: - if (rcd) { - if (!LNetMDHandleIsInvalid(rcd->rcd_mdh)) { - rc = LNetMDUnlink(rcd->rcd_mdh); - LASSERT(!rc); - } +out_unlock: + lnet_net_unlock(gateway->lpni_cpt); + rc = LNetMDUnlink(mdh); + LASSERT(!rc); +out_ping_buffer_decref: + lnet_ping_buffer_decref(pbuf); +out: + if (rcd && rcd != gateway->lpni_rcd) lnet_destroy_rc_data(rcd); - } - lnet_net_lock(gateway->lpni_cpt); return gateway->lpni_rcd; } @@ -1018,9 +1044,9 @@ lnet_ping_router_locked(struct lnet_peer_ni *rtr) return; } - rcd = rtr->lpni_rcd ? - rtr->lpni_rcd : lnet_create_rc_data_locked(rtr); - + rcd = rtr->lpni_rcd; + if (!rcd || rcd->rcd_nnis > rcd->rcd_pingbuffer->pb_nnis) + rcd = lnet_update_rc_data_locked(rtr); if (!rcd) return;