diff mbox series

[27/34] lnet: make it possible to add a new interface to a network

Message ID 153628137234.8267.14872362382875902424.stgit@noble (mailing list archive)
State New, archived
Headers show
Series Beginning of multi-rail support for drivers/staging/lustre | expand

Commit Message

NeilBrown Sept. 7, 2018, 12:49 a.m. UTC
lnet_startup_lndnet() is enhanced to cope if the net already
exists.

This is part of
    8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
       LU-7734 lnet: Multi-Rail local NI split

Signed-off-by: NeilBrown <neilb@suse.com>
---
 .../staging/lustre/include/linux/lnet/lib-lnet.h   |    3 +
 drivers/staging/lustre/lnet/lnet/api-ni.c          |   69 +++++++++++++++-----
 drivers/staging/lustre/lnet/lnet/config.c          |   12 ++-
 3 files changed, 61 insertions(+), 23 deletions(-)

Comments

Doug Oucharek Sept. 12, 2018, 4:38 a.m. UTC | #1
Reviewed-by: Doug Oucharek <dougso@me.com>

Doug

On 9/6/18, 5:55 PM, "NeilBrown" <neilb@suse.com> wrote:

    lnet_startup_lndnet() is enhanced to cope if the net already
    exists.
    
    This is part of
        8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
           LU-7734 lnet: Multi-Rail local NI split
    
    Signed-off-by: NeilBrown <neilb@suse.com>
    ---
     .../staging/lustre/include/linux/lnet/lib-lnet.h   |    3 +
     drivers/staging/lustre/lnet/lnet/api-ni.c          |   69 +++++++++++++++-----
     drivers/staging/lustre/lnet/lnet/config.c          |   12 ++-
     3 files changed, 61 insertions(+), 23 deletions(-)
    
    diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    index 6401d9a37b23..905213fc16c7 100644
    --- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    +++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    @@ -630,7 +630,8 @@ void lnet_swap_pinginfo(struct lnet_ping_info *info);
     int lnet_parse_ip2nets(char **networksp, char *ip2nets);
     int lnet_parse_routes(char *route_str, int *im_a_router);
     int lnet_parse_networks(struct list_head *nilist, char *networks);
    -bool lnet_net_unique(__u32 net, struct list_head *nilist);
    +bool lnet_net_unique(__u32 net_id, struct list_head *nilist,
    +		     struct lnet_net **net);
     
     int lnet_nid2peer_locked(struct lnet_peer **lpp, lnet_nid_t nid, int cpt);
     struct lnet_peer *lnet_find_peer_locked(struct lnet_peer_table *ptable,
    diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
    index 0dfd3004f735..042ab0d9e318 100644
    --- a/drivers/staging/lustre/lnet/lnet/api-ni.c
    +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
    @@ -1298,14 +1298,9 @@ lnet_startup_lndni(struct lnet_ni *ni, struct lnet_lnd_tunables *tun)
     		goto failed0;
     	}
     
    -	lnet_net_lock(LNET_LOCK_EX);
    -	/* refcount for ln_nis */
    -	lnet_ni_addref_locked(ni, 0);
    -	list_add_tail(&ni->ni_net->net_list, &the_lnet.ln_nets);
    -	lnet_net_unlock(LNET_LOCK_EX);
    -
     	ni->ni_state = LNET_NI_STATE_ACTIVE;
     
    +	/* We keep a reference on the loopback net through the loopback NI */
     	if (net->net_lnd->lnd_type == LOLND) {
     		lnet_ni_addref(ni);
     		LASSERT(!the_lnet.ln_loni);
    @@ -1360,6 +1355,7 @@ static int
     lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     {
     	struct lnet_ni		*ni;
    +	struct lnet_net		*net_l = NULL;
     	struct list_head	local_ni_list;
     	int			rc;
     	int			ni_count = 0;
    @@ -1368,8 +1364,14 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     
     	INIT_LIST_HEAD(&local_ni_list);
     
    -	/* Make sure this new NI is unique. */
    -	if (lnet_net_unique(net->net_id, &the_lnet.ln_nets)) {
    +	/*
    +	 * make sure that this net is unique. If it isn't then
    +	 * we are adding interfaces to an already existing network, and
    +	 * 'net' is just a convenient way to pass in the list.
    +	 * if it is unique we need to find the LND and load it if
    +	 * necessary.
    +	 */
    +	if (lnet_net_unique(net->net_id, &the_lnet.ln_nets, &net_l)) {
     		lnd_type = LNET_NETTYP(net->net_id);
     
     		LASSERT(libcfs_isknown_lnd(lnd_type));
    @@ -1400,23 +1402,41 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     		net->net_lnd = lnd;
     
     		mutex_unlock(&the_lnet.ln_lnd_mutex);
    -	} else {
    -		if (lnd_type == LOLND) {
    -			lnet_net_free(net);
    -			return 0;
    -		}
     
    -		CERROR("Net %s is not unique\n",
    -		       libcfs_net2str(net->net_id));
    -		rc = -EEXIST;
    -		goto failed0;
    +		net_l = net;
     	}
     
    +	/*
    +	 * net_l: if the network being added is unique then net_l
    +	 *        will point to that network
    +	 *        if the network being added is not unique then
    +	 *        net_l points to the existing network.
    +	 *
    +	 * When we enter the loop below, we'll pick NIs off he
    +	 * network beign added and start them up, then add them to
    +	 * a local ni list. Once we've successfully started all
    +	 * the NIs then we join the local NI list (of started up
    +	 * networks) with the net_l->net_ni_list, which should
    +	 * point to the correct network to add the new ni list to
    +	 *
    +	 * If any of the new NIs fail to start up, then we want to
    +	 * iterate through the local ni list, which should include
    +	 * any NIs which were successfully started up, and shut
    +	 * them down.
    +	 *
    +	 * After than we want to delete the network being added,
    +	 * to avoid a memory leak.
    +	 */
    +
     	while (!list_empty(&net->net_ni_added)) {
     		ni = list_entry(net->net_ni_added.next, struct lnet_ni,
     				ni_netlist);
     		list_del_init(&ni->ni_netlist);
     
    +		/* adjust the pointer the parent network, just in case it
    +		 * the net is a duplicate */
    +		ni->ni_net = net_l;
    +
     		rc = lnet_startup_lndni(ni, tun);
     
     		if (rc < 0)
    @@ -1427,9 +1447,22 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     
     		ni_count++;
     	}
    +
     	lnet_net_lock(LNET_LOCK_EX);
    -	list_splice_tail(&local_ni_list, &net->net_ni_list);
    +	list_splice_tail(&local_ni_list, &net_l->net_ni_list);
     	lnet_net_unlock(LNET_LOCK_EX);
    +
    +	/* if the network is not unique then we don't want to keep
    +	 * it around after we're done. Free it. Otherwise add that
    +	 * net to the global the_lnet.ln_nets */
    +	if (net_l != net && net_l != NULL) {
    +		lnet_net_free(net);
    +	} else {
    +		lnet_net_lock(LNET_LOCK_EX);
    +		list_add_tail(&net->net_list, &the_lnet.ln_nets);
    +		lnet_net_unlock(LNET_LOCK_EX);
    +	}
    +
     	return ni_count;
     
     failed1:
    diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
    index f886dcfc6d6e..fcae50676422 100644
    --- a/drivers/staging/lustre/lnet/lnet/config.c
    +++ b/drivers/staging/lustre/lnet/lnet/config.c
    @@ -79,13 +79,17 @@ lnet_issep(char c)
     }
     
     bool
    -lnet_net_unique(__u32 net, struct list_head *netlist)
    +lnet_net_unique(__u32 net_id, struct list_head *netlist,
    +		struct lnet_net **net)
     {
    -	struct lnet_net	 *net_l;
    +	struct lnet_net  *net_l;
     
     	list_for_each_entry(net_l, netlist, net_list) {
    -		if (net_l->net_id == net)
    +		if (net_l->net_id == net_id) {
    +			if (net != NULL)
    +				*net = net_l;
     			return false;
    +		}
     	}
     
     	return true;
    @@ -309,7 +313,7 @@ lnet_net_alloc(__u32 net_id, struct list_head *net_list)
     {
     	struct lnet_net		*net;
     
    -	if (!lnet_net_unique(net_id, net_list)) {
    +	if (!lnet_net_unique(net_id, net_list, NULL)) {
     		CERROR("Duplicate net %s. Ignore\n",
     		       libcfs_net2str(net_id));
     		return NULL;
diff mbox series

Patch

diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
index 6401d9a37b23..905213fc16c7 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
@@ -630,7 +630,8 @@  void lnet_swap_pinginfo(struct lnet_ping_info *info);
 int lnet_parse_ip2nets(char **networksp, char *ip2nets);
 int lnet_parse_routes(char *route_str, int *im_a_router);
 int lnet_parse_networks(struct list_head *nilist, char *networks);
-bool lnet_net_unique(__u32 net, struct list_head *nilist);
+bool lnet_net_unique(__u32 net_id, struct list_head *nilist,
+		     struct lnet_net **net);
 
 int lnet_nid2peer_locked(struct lnet_peer **lpp, lnet_nid_t nid, int cpt);
 struct lnet_peer *lnet_find_peer_locked(struct lnet_peer_table *ptable,
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index 0dfd3004f735..042ab0d9e318 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -1298,14 +1298,9 @@  lnet_startup_lndni(struct lnet_ni *ni, struct lnet_lnd_tunables *tun)
 		goto failed0;
 	}
 
-	lnet_net_lock(LNET_LOCK_EX);
-	/* refcount for ln_nis */
-	lnet_ni_addref_locked(ni, 0);
-	list_add_tail(&ni->ni_net->net_list, &the_lnet.ln_nets);
-	lnet_net_unlock(LNET_LOCK_EX);
-
 	ni->ni_state = LNET_NI_STATE_ACTIVE;
 
+	/* We keep a reference on the loopback net through the loopback NI */
 	if (net->net_lnd->lnd_type == LOLND) {
 		lnet_ni_addref(ni);
 		LASSERT(!the_lnet.ln_loni);
@@ -1360,6 +1355,7 @@  static int
 lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 {
 	struct lnet_ni		*ni;
+	struct lnet_net		*net_l = NULL;
 	struct list_head	local_ni_list;
 	int			rc;
 	int			ni_count = 0;
@@ -1368,8 +1364,14 @@  lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 
 	INIT_LIST_HEAD(&local_ni_list);
 
-	/* Make sure this new NI is unique. */
-	if (lnet_net_unique(net->net_id, &the_lnet.ln_nets)) {
+	/*
+	 * make sure that this net is unique. If it isn't then
+	 * we are adding interfaces to an already existing network, and
+	 * 'net' is just a convenient way to pass in the list.
+	 * if it is unique we need to find the LND and load it if
+	 * necessary.
+	 */
+	if (lnet_net_unique(net->net_id, &the_lnet.ln_nets, &net_l)) {
 		lnd_type = LNET_NETTYP(net->net_id);
 
 		LASSERT(libcfs_isknown_lnd(lnd_type));
@@ -1400,23 +1402,41 @@  lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 		net->net_lnd = lnd;
 
 		mutex_unlock(&the_lnet.ln_lnd_mutex);
-	} else {
-		if (lnd_type == LOLND) {
-			lnet_net_free(net);
-			return 0;
-		}
 
-		CERROR("Net %s is not unique\n",
-		       libcfs_net2str(net->net_id));
-		rc = -EEXIST;
-		goto failed0;
+		net_l = net;
 	}
 
+	/*
+	 * net_l: if the network being added is unique then net_l
+	 *        will point to that network
+	 *        if the network being added is not unique then
+	 *        net_l points to the existing network.
+	 *
+	 * When we enter the loop below, we'll pick NIs off he
+	 * network beign added and start them up, then add them to
+	 * a local ni list. Once we've successfully started all
+	 * the NIs then we join the local NI list (of started up
+	 * networks) with the net_l->net_ni_list, which should
+	 * point to the correct network to add the new ni list to
+	 *
+	 * If any of the new NIs fail to start up, then we want to
+	 * iterate through the local ni list, which should include
+	 * any NIs which were successfully started up, and shut
+	 * them down.
+	 *
+	 * After than we want to delete the network being added,
+	 * to avoid a memory leak.
+	 */
+
 	while (!list_empty(&net->net_ni_added)) {
 		ni = list_entry(net->net_ni_added.next, struct lnet_ni,
 				ni_netlist);
 		list_del_init(&ni->ni_netlist);
 
+		/* adjust the pointer the parent network, just in case it
+		 * the net is a duplicate */
+		ni->ni_net = net_l;
+
 		rc = lnet_startup_lndni(ni, tun);
 
 		if (rc < 0)
@@ -1427,9 +1447,22 @@  lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 
 		ni_count++;
 	}
+
 	lnet_net_lock(LNET_LOCK_EX);
-	list_splice_tail(&local_ni_list, &net->net_ni_list);
+	list_splice_tail(&local_ni_list, &net_l->net_ni_list);
 	lnet_net_unlock(LNET_LOCK_EX);
+
+	/* if the network is not unique then we don't want to keep
+	 * it around after we're done. Free it. Otherwise add that
+	 * net to the global the_lnet.ln_nets */
+	if (net_l != net && net_l != NULL) {
+		lnet_net_free(net);
+	} else {
+		lnet_net_lock(LNET_LOCK_EX);
+		list_add_tail(&net->net_list, &the_lnet.ln_nets);
+		lnet_net_unlock(LNET_LOCK_EX);
+	}
+
 	return ni_count;
 
 failed1:
diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index f886dcfc6d6e..fcae50676422 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -79,13 +79,17 @@  lnet_issep(char c)
 }
 
 bool
-lnet_net_unique(__u32 net, struct list_head *netlist)
+lnet_net_unique(__u32 net_id, struct list_head *netlist,
+		struct lnet_net **net)
 {
-	struct lnet_net	 *net_l;
+	struct lnet_net  *net_l;
 
 	list_for_each_entry(net_l, netlist, net_list) {
-		if (net_l->net_id == net)
+		if (net_l->net_id == net_id) {
+			if (net != NULL)
+				*net = net_l;
 			return false;
+		}
 	}
 
 	return true;
@@ -309,7 +313,7 @@  lnet_net_alloc(__u32 net_id, struct list_head *net_list)
 {
 	struct lnet_net		*net;
 
-	if (!lnet_net_unique(net_id, net_list)) {
+	if (!lnet_net_unique(net_id, net_list, NULL)) {
 		CERROR("Duplicate net %s. Ignore\n",
 		       libcfs_net2str(net_id));
 		return NULL;