[09/34] lnet: add list of cpts to lnet_net.
diff mbox series

Message ID 153628137159.8267.921309094971745898.stgit@noble
State New
Headers show
Series
  • Beginning of multi-rail support for drivers/staging/lustre
Related show

Commit Message

NeilBrown Sept. 7, 2018, 12:49 a.m. UTC
struct lnet_net now has a list of cpts, which is the union
of the cpts for each lnet_ni.

This is part of
    8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
       LU-7734 lnet: Multi-Rail local NI split

Signed-off-by: NeilBrown <neilb@suse.com>
---
 .../staging/lustre/include/linux/lnet/lib-types.h  |    6 +
 drivers/staging/lustre/lnet/lnet/config.c          |  164 ++++++++++++++++++++
 2 files changed, 170 insertions(+)

Comments

Doug Oucharek Sept. 10, 2018, 11:28 p.m. UTC | #1
I agree with a comment from James Simmons: __u32 should only be used when the variable is being shared with user space.  We need to start converting all uses of __uXX in LNet to just uXX.  Perhaps that should be a set of future patches once all of MR/DD has landed?

Reviewed-by: Doug Oucharek <dougso@me.com<mailto:dougso@me.com>>

Doug

On Sep 6, 2018, at 5:49 PM, NeilBrown <neilb@suse.com<mailto:neilb@suse.com>> wrote:

struct lnet_net now has a list of cpts, which is the union
of the cpts for each lnet_ni.

This is part of
   8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
      LU-7734 lnet: Multi-Rail local NI split

Signed-off-by: NeilBrown <neilb@suse.com<mailto:neilb@suse.com>>
---
.../staging/lustre/include/linux/lnet/lib-types.h  |    6 +
drivers/staging/lustre/lnet/lnet/config.c          |  164 ++++++++++++++++++++
2 files changed, 170 insertions(+)

diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
index 2d2c066a11ba..22957d142cc0 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
@@ -266,6 +266,12 @@ struct lnet_net {
* lnet/include/lnet/nidstr.h */
__u32 net_id;

+ /* total number of CPTs in the array */
+ __u32 net_ncpts;
+
+ /* cumulative CPTs of all NIs in this net */
+ __u32 *net_cpts;
+
/* network tunables */
struct lnet_ioctl_config_lnd_cmn_tunables net_tunables;

diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index e83bdbec11e3..380a3fb1caba 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -91,11 +91,169 @@ lnet_net_unique(__u32 net, struct list_head *netlist)
return true;
}

+static bool
+in_array(__u32 *array, __u32 size, __u32 value)
+{
+ int i;
+
+ for (i = 0; i < size; i++) {
+ if (array[i] == value)
+ return false;
+ }
+
+ return true;
+}
+
+static int
+lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)
+{
+ __u32 *added_cpts = NULL;
+ int i, j = 0, rc = 0;
+
+ /*
+ * no need to go futher since a subset of the NIs already exist on
+ * all CPTs
+ */
+ if (net->net_ncpts == LNET_CPT_NUMBER)
+ return 0;
+
+ if (cpts == NULL) {
+ /* there is an NI which will exist on all CPTs */
+ if (net->net_cpts != NULL)
+ kvfree(net->net_cpts);
+ net->net_cpts = NULL;
+ net->net_ncpts = LNET_CPT_NUMBER;
+ return 0;
+ }
+
+ if (net->net_cpts == NULL) {
+ net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts),
+      GFP_KERNEL);
+ if (net->net_cpts == NULL)
+ return -ENOMEM;
+ memcpy(net->net_cpts, cpts, ncpts);
+ return 0;
+ }
+
+ added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts),
+   GFP_KERNEL);
+ if (added_cpts == NULL)
+ return -ENOMEM;
+
+ for (i = 0; i < ncpts; i++) {
+ if (!in_array(net->net_cpts, net->net_ncpts, cpts[i])) {
+ added_cpts[j] = cpts[i];
+ j++;
+ }
+ }
+
+ /* append the new cpts if any to the list of cpts in the net */
+ if (j > 0) {
+ __u32 *array = NULL, *loc;
+ __u32 total_entries = j + net->net_ncpts;
+
+ array = kmalloc_array(total_entries, sizeof(*net->net_cpts),
+      GFP_KERNEL);
+ if (array == NULL) {
+ rc = -ENOMEM;
+ goto failed;
+ }
+
+ memcpy(array, net->net_cpts,
+       net->net_ncpts * sizeof(*net->net_cpts));
+ loc = array + net->net_ncpts;
+ memcpy(loc, added_cpts, j * sizeof(*net->net_cpts));
+
+ kfree(net->net_cpts);
+ net->net_ncpts = total_entries;
+ net->net_cpts = array;
+ }
+
+failed:
+ kfree(added_cpts);
+
+ return rc;
+}
+
+static void
+lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)
+{
+ struct lnet_ni *ni;
+ int rc;
+
+ /*
+ * Operation Assumption:
+ * This function is called after an NI has been removed from
+ * its parent net.
+ *
+ * if we're removing an NI which exists on all CPTs then
+ * we have to check if any of the other NIs on this net also
+ * exists on all CPTs. If none, then we need to build our Net CPT
+ * list based on the remaining NIs.
+ *
+ * If the NI being removed exist on a subset of the CPTs then we
+ * alo rebuild the Net CPT list based on the remaining NIs, which
+ * should resutl in the expected Net CPT list.
+ */
+
+ /*
+ * sometimes this function can be called due to some failure
+ * creating an NI, before any of the cpts are allocated, so check
+ * for that case and don't do anything
+ */
+ if (ncpts == 0)
+ return;
+
+ if (ncpts == LNET_CPT_NUMBER) {
+ /*
+ * first iteration through the NI list in the net to see
+ * if any of the NIs exist on all the CPTs. If one is
+ * found then our job is done.
+ */
+ list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+ if (ni->ni_ncpts == LNET_CPT_NUMBER)
+ return;
+ }
+ }
+
+ /*
+ * Rebuild the Net CPT list again, thereby only including only the
+ * CPTs which the remaining NIs are associated with.
+ */
+ if (net->net_cpts != NULL) {
+ kfree(net->net_cpts);
+ net->net_cpts = NULL;
+ }
+
+ list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+ rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts,
+  net);
+ if (rc != 0) {
+ CERROR("Out of Memory\n");
+ /*
+ * do our best to keep on going. Delete
+ * the net cpts and set it to NULL. This
+ * way we can keep on going but less
+ * efficiently, since memory accesses might be
+ * accross CPT lines.
+ */
+ if (net->net_cpts != NULL) {
+ kfree(net->net_cpts);
+ net->net_cpts = NULL;
+ net->net_ncpts = LNET_CPT_NUMBER;
+ }
+ return;
+ }
+ }
+}
+
void
lnet_ni_free(struct lnet_ni *ni)
{
int i;

+ lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net);
+
if (ni->ni_refs)
cfs_percpt_free(ni->ni_refs);

@@ -128,6 +286,9 @@ lnet_net_free(struct lnet_net *net)
lnet_ni_free(ni);
}

+ if (net->net_cpts != NULL)
+ kfree(net->net_cpts);
+
kfree(net);
}

@@ -229,6 +390,9 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
ni->ni_net_ns = NULL;

ni->ni_last_alive = ktime_get_real_seconds();
+ rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, net);
+ if (rc != 0)
+ goto failed;
list_add_tail(&ni->ni_netlist, &net->net_ni_list);

return ni;
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
</head>
<body style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;" class="">
I agree with a comment from James Simmons: __u32 should only be used when the variable is being shared with user space. &nbsp;We need to start converting all uses of __uXX in LNet to just uXX. &nbsp;Perhaps that should be a set of future patches once all of MR/DD has
 landed?
<div class=""><br class="">
</div>
<div class="">Reviewed-by: Doug Oucharek &lt;<a href="mailto:dougso@me.com" class="">dougso@me.com</a>&gt;</div>
<div class=""><br class="">
</div>
<div class="">Doug</div>
<div class=""><br class="">
</div>
<div class="">
<div>
<blockquote type="cite" class="">
<div class="">On Sep 6, 2018, at 5:49 PM, NeilBrown &lt;<a href="mailto:neilb@suse.com" class="">neilb@suse.com</a>&gt; wrote:</div>
<br class="Apple-interchange-newline">
<div class="">
<div class="">struct lnet_net now has a list of cpts, which is the union<br class="">
of the cpts for each lnet_ni.<br class="">
<br class="">
This is part of<br class="">
&nbsp;&nbsp;&nbsp;8cbb8cd3e771e7f7e0f99cafc19fad32770dc015<br class="">
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;LU-7734 lnet: Multi-Rail local NI split<br class="">
<br class="">
Signed-off-by: NeilBrown &lt;<a href="mailto:neilb@suse.com" class="">neilb@suse.com</a>&gt;<br class="">
---<br class="">
.../staging/lustre/include/linux/lnet/lib-types.h &nbsp;| &nbsp;&nbsp;&nbsp;6 &#43;<br class="">
drivers/staging/lustre/lnet/lnet/config.c &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;| &nbsp;164 &#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;&#43;<br class="">
2 files changed, 170 insertions(&#43;)<br class="">
<br class="">
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h<br class="">
index 2d2c066a11ba..22957d142cc0 100644<br class="">
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h<br class="">
&#43;&#43;&#43; b/drivers/staging/lustre/include/linux/lnet/lib-types.h<br class="">
@@ -266,6 &#43;266,12 @@ struct lnet_net {<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>* lnet/include/lnet/nidstr.h */<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>__u32<span class="Apple-tab-span" style="white-space:pre">
</span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>net_id;<br class="">
<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>/* total number of CPTs in the array */<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>__u32<span class="Apple-tab-span" style="white-space:pre">
</span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>net_ncpts;<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>/* cumulative CPTs of all NIs in this net */<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>__u32<span class="Apple-tab-span" style="white-space:pre">
</span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>*net_cpts;<br class="">
&#43;<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>/* network tunables */<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>struct lnet_ioctl_config_lnd_cmn_tunables net_tunables;<br class="">
<br class="">
diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c<br class="">
index e83bdbec11e3..380a3fb1caba 100644<br class="">
--- a/drivers/staging/lustre/lnet/lnet/config.c<br class="">
&#43;&#43;&#43; b/drivers/staging/lustre/lnet/lnet/config.c<br class="">
@@ -91,11 &#43;91,169 @@ lnet_net_unique(__u32 net, struct list_head *netlist)<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>return true;<br class="">
}<br class="">
<br class="">
&#43;static bool<br class="">
&#43;in_array(__u32 *array, __u32 size, __u32 value)<br class="">
&#43;{<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>int i;<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>for (i = 0; i &lt; size; i&#43;&#43;) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (array[i] == value)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>return false;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>return true;<br class="">
&#43;}<br class="">
&#43;<br class="">
&#43;static int<br class="">
&#43;lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)<br class="">
&#43;{<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>__u32 *added_cpts = NULL;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>int i, j = 0, rc = 0;<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>/*<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* no need to go futher since a subset of the NIs already exist on<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* all CPTs<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>*/<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (net-&gt;net_ncpts == LNET_CPT_NUMBER)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return 0;<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (cpts == NULL) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>/* there is an NI which will exist on all CPTs */<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (net-&gt;net_cpts != NULL)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>kvfree(net-&gt;net_cpts);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net-&gt;net_cpts = NULL;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net-&gt;net_ncpts = LNET_CPT_NUMBER;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return 0;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (net-&gt;net_cpts == NULL) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net-&gt;net_cpts = kmalloc_array(ncpts, sizeof(net-&gt;net_cpts),<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;GFP_KERNEL);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (net-&gt;net_cpts == NULL)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>return -ENOMEM;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>memcpy(net-&gt;net_cpts, cpts, ncpts);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return 0;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts),<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>&nbsp;&nbsp;GFP_KERNEL);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (added_cpts == NULL)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return -ENOMEM;<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>for (i = 0; i &lt; ncpts; i&#43;&#43;) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (!in_array(net-&gt;net_cpts, net-&gt;net_ncpts, cpts[i])) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>added_cpts[j] = cpts[i];<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>j&#43;&#43;;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>/* append the new cpts if any to the list of cpts in the net */<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (j &gt; 0) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>__u32 *array = NULL, *loc;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>__u32 total_entries = j &#43; net-&gt;net_ncpts;<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>array = kmalloc_array(total_entries, sizeof(*net-&gt;net_cpts),<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;GFP_KERNEL);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (array == NULL) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>rc = -ENOMEM;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>goto failed;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>memcpy(array, net-&gt;net_cpts,<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;net-&gt;net_ncpts * sizeof(*net-&gt;net_cpts));<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>loc = array &#43; net-&gt;net_ncpts;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>memcpy(loc, added_cpts, j * sizeof(*net-&gt;net_cpts));<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>kfree(net-&gt;net_cpts);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net-&gt;net_ncpts = total_entries;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net-&gt;net_cpts = array;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class="">
&#43;<br class="">
&#43;failed:<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>kfree(added_cpts);<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>return rc;<br class="">
&#43;}<br class="">
&#43;<br class="">
&#43;static void<br class="">
&#43;lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)<br class="">
&#43;{<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>struct lnet_ni *ni;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>int rc;<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>/*<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* Operation Assumption:<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>*<span class="Apple-tab-span" style="white-space:pre">
</span>This function is called after an NI has been removed from<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>*<span class="Apple-tab-span" style="white-space:pre">
</span>its parent net.<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>*<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* if we're removing an NI which exists on all CPTs then<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* we have to check if any of the other NIs on this net also<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* exists on all CPTs. If none, then we need to build our Net CPT<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* list based on the remaining NIs.<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>*<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* If the NI being removed exist on a subset of the CPTs then we<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* alo rebuild the Net CPT list based on the remaining NIs, which<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* should resutl in the expected Net CPT list.<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>*/<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>/*<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* sometimes this function can be called due to some failure<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* creating an NI, before any of the cpts are allocated, so check<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* for that case and don't do anything<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>*/<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (ncpts == 0)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return;<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (ncpts == LNET_CPT_NUMBER) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>/*<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>* first iteration through the NI list in the net to see<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>* if any of the NIs exist on all the CPTs. If one is<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>* found then our job is done.<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>*/<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>list_for_each_entry(ni, &amp;net-&gt;net_ni_list, ni_netlist) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>if (ni-&gt;ni_ncpts == LNET_CPT_NUMBER)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>return;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>/*<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* Rebuild the Net CPT list again, thereby only including only the<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>* CPTs which the remaining NIs are associated with.<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>*/<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (net-&gt;net_cpts != NULL) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>kfree(net-&gt;net_cpts);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net-&gt;net_cpts = NULL;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class="">
&#43;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>list_for_each_entry(ni, &amp;net-&gt;net_ni_list, ni_netlist) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>rc = lnet_net_append_cpts(ni-&gt;ni_cpts, ni-&gt;ni_ncpts,<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>&nbsp;net);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (rc != 0) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>CERROR(&quot;Out of Memory\n&quot;);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>/*<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* do our best to keep on going. Delete<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* the net cpts and set it to NULL. This<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* way we can keep on going but less<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* efficiently, since memory accesses might be<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* accross CPT lines.<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>*/<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>if (net-&gt;net_cpts != NULL) {<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>kfree(net-&gt;net_cpts);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>net-&gt;net_cpts =
 NULL;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>net-&gt;net_ncpts
 = LNET_CPT_NUMBER;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>return;<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class="">
&#43;}<br class="">
&#43;<br class="">
void<br class="">
lnet_ni_free(struct lnet_ni *ni)<br class="">
{<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>int i;<br class="">
<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>lnet_net_remove_cpts(ni-&gt;ni_cpts, ni-&gt;ni_ncpts, ni-&gt;ni_net);<br class="">
&#43;<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>if (ni-&gt;ni_refs)<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>cfs_percpt_free(ni-&gt;ni_refs);<br class="">
<br class="">
@@ -128,6 &#43;286,9 @@ lnet_net_free(struct lnet_net *net)<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>lnet_ni_free(ni);<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>}<br class="">
<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (net-&gt;net_cpts != NULL)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>kfree(net-&gt;net_cpts);<br class="">
&#43;<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>kfree(net);<br class="">
}<br class="">
<br class="">
@@ -229,6 &#43;390,9 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>ni-&gt;ni_net_ns = NULL;<br class="">
<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>ni-&gt;ni_last_alive = ktime_get_real_seconds();<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>rc = lnet_net_append_cpts(ni-&gt;ni_cpts, ni-&gt;ni_ncpts, net);<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span>if (rc != 0)<br class="">
&#43;<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>goto failed;<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>list_add_tail(&amp;ni-&gt;ni_netlist, &amp;net-&gt;net_ni_list);<br class="">
<br class="">
<span class="Apple-tab-span" style="white-space:pre"></span>return ni;<br class="">
<br class="">
<br class="">
</div>
</div>
</blockquote>
</div>
<br class="">
</div>
</body>
</html>
James Simmons Sept. 11, 2018, 1:02 a.m. UTC | #2
> struct lnet_net now has a list of cpts, which is the union
> of the cpts for each lnet_ni.

Reviewed-by: James Simmons <jsimmons@infradead.org>
The below needs fixing based on response to cover letter.
 
> This is part of
>     8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
>        LU-7734 lnet: Multi-Rail local NI split
>
> Signed-off-by: NeilBrown <neilb@suse.com>
> ---
>  .../staging/lustre/include/linux/lnet/lib-types.h  |    6 +
>  drivers/staging/lustre/lnet/lnet/config.c          |  164 ++++++++++++++++++++
>  2 files changed, 170 insertions(+)
> 
> diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
> index 2d2c066a11ba..22957d142cc0 100644
> --- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
> +++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
> @@ -266,6 +266,12 @@ struct lnet_net {
>  	 * lnet/include/lnet/nidstr.h */
>  	__u32			net_id;
>  
> +	/* total number of CPTs in the array */
> +	__u32			net_ncpts;
> +
> +	/* cumulative CPTs of all NIs in this net */
> +	__u32			*net_cpts;
> +
>  	/* network tunables */
>  	struct lnet_ioctl_config_lnd_cmn_tunables net_tunables;
>  
> diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
> index e83bdbec11e3..380a3fb1caba 100644
> --- a/drivers/staging/lustre/lnet/lnet/config.c
> +++ b/drivers/staging/lustre/lnet/lnet/config.c
> @@ -91,11 +91,169 @@ lnet_net_unique(__u32 net, struct list_head *netlist)
>  	return true;
>  }
>  
> +static bool
> +in_array(__u32 *array, __u32 size, __u32 value)
> +{
> +	int i;
> +
> +	for (i = 0; i < size; i++) {
> +		if (array[i] == value)
> +			return false;
> +	}
> +
> +	return true;
> +}
> +
> +static int
> +lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)
> +{
> +	__u32 *added_cpts = NULL;
> +	int i, j = 0, rc = 0;
> +
> +	/*
> +	 * no need to go futher since a subset of the NIs already exist on
> +	 * all CPTs
> +	 */
> +	if (net->net_ncpts == LNET_CPT_NUMBER)
> +		return 0;
> +
> +	if (cpts == NULL) {
> +		/* there is an NI which will exist on all CPTs */
> +		if (net->net_cpts != NULL)
> +			kvfree(net->net_cpts);
> +		net->net_cpts = NULL;
> +		net->net_ncpts = LNET_CPT_NUMBER;
> +		return 0;
> +	}
> +
> +	if (net->net_cpts == NULL) {
> +		net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts),
> +					      GFP_KERNEL);
> +		if (net->net_cpts == NULL)
> +			return -ENOMEM;
> +		memcpy(net->net_cpts, cpts, ncpts);
> +		return 0;
> +	}
> +
> +	added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts),
> +				   GFP_KERNEL);
> +	if (added_cpts == NULL)
> +		return -ENOMEM;
> +
> +	for (i = 0; i < ncpts; i++) {
> +		if (!in_array(net->net_cpts, net->net_ncpts, cpts[i])) {
> +			added_cpts[j] = cpts[i];
> +			j++;
> +		}
> +	}
> +
> +	/* append the new cpts if any to the list of cpts in the net */
> +	if (j > 0) {
> +		__u32 *array = NULL, *loc;
> +		__u32 total_entries = j + net->net_ncpts;
> +
> +		array = kmalloc_array(total_entries, sizeof(*net->net_cpts),
> +				      GFP_KERNEL);
> +		if (array == NULL) {
> +			rc = -ENOMEM;
> +			goto failed;
> +		}
> +
> +		memcpy(array, net->net_cpts,
> +		       net->net_ncpts * sizeof(*net->net_cpts));
> +		loc = array + net->net_ncpts;
> +		memcpy(loc, added_cpts, j * sizeof(*net->net_cpts));
> +
> +		kfree(net->net_cpts);
> +		net->net_ncpts = total_entries;
> +		net->net_cpts = array;
> +	}
> +
> +failed:
> +	kfree(added_cpts);
> +
> +	return rc;
> +}
> +
> +static void
> +lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)
> +{
> +	struct lnet_ni *ni;
> +	int rc;
> +
> +	/*
> +	 * Operation Assumption:
> +	 *	This function is called after an NI has been removed from
> +	 *	its parent net.
> +	 *
> +	 * if we're removing an NI which exists on all CPTs then
> +	 * we have to check if any of the other NIs on this net also
> +	 * exists on all CPTs. If none, then we need to build our Net CPT
> +	 * list based on the remaining NIs.
> +	 *
> +	 * If the NI being removed exist on a subset of the CPTs then we
> +	 * alo rebuild the Net CPT list based on the remaining NIs, which
> +	 * should resutl in the expected Net CPT list.
> +	 */
> +
> +	/*
> +	 * sometimes this function can be called due to some failure
> +	 * creating an NI, before any of the cpts are allocated, so check
> +	 * for that case and don't do anything
> +	 */
> +	if (ncpts == 0)
> +		return;
> +
> +	if (ncpts == LNET_CPT_NUMBER) {
> +		/*
> +		 * first iteration through the NI list in the net to see
> +		 * if any of the NIs exist on all the CPTs. If one is
> +		 * found then our job is done.
> +		 */
> +		list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
> +			if (ni->ni_ncpts == LNET_CPT_NUMBER)
> +				return;
> +		}
> +	}
> +
> +	/*
> +	 * Rebuild the Net CPT list again, thereby only including only the
> +	 * CPTs which the remaining NIs are associated with.
> +	 */
> +	if (net->net_cpts != NULL) {
> +		kfree(net->net_cpts);
> +		net->net_cpts = NULL;
> +	}
> +
> +	list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
> +		rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts,
> +					  net);
> +		if (rc != 0) {
> +			CERROR("Out of Memory\n");
> +			/*
> +			 * do our best to keep on going. Delete
> +			 * the net cpts and set it to NULL. This
> +			 * way we can keep on going but less
> +			 * efficiently, since memory accesses might be
> +			 * accross CPT lines.
> +			 */
> +			if (net->net_cpts != NULL) {
> +				kfree(net->net_cpts);
> +				net->net_cpts = NULL;
> +				net->net_ncpts = LNET_CPT_NUMBER;
> +			}
> +			return;
> +		}
> +	}
> +}
> +
>  void
>  lnet_ni_free(struct lnet_ni *ni)
>  {
>  	int i;
>  
> +	lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net);
> +
>  	if (ni->ni_refs)
>  		cfs_percpt_free(ni->ni_refs);
>  
> @@ -128,6 +286,9 @@ lnet_net_free(struct lnet_net *net)
>  		lnet_ni_free(ni);
>  	}
>  
> +	if (net->net_cpts != NULL)
> +		kfree(net->net_cpts);
> +
>  	kfree(net);
>  }
>  
> @@ -229,6 +390,9 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
>  		ni->ni_net_ns = NULL;
>  
>  	ni->ni_last_alive = ktime_get_real_seconds();
> +	rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, net);
> +	if (rc != 0)
> +		goto failed;
>  	list_add_tail(&ni->ni_netlist, &net->net_ni_list);
>  
>  	return ni;
> 
> 
>
NeilBrown Sept. 12, 2018, 2:16 a.m. UTC | #3
On Mon, Sep 10 2018, Doug Oucharek wrote:

> I agree with a comment from James Simmons: __u32 should only be used when the variable is being shared with user space.  We need to start converting all uses of __uXX in LNet to just uXX.  Perhaps that should be a set of future patches once all of MR/DD has landed?
>

That seems reasonable.  I dont't think this series adds significant new
uses of __u32.
Changing them all to u32 should, as you suggest, come later.

> Reviewed-by: Doug Oucharek <dougso@me.com<mailto:dougso@me.com>>

Thanks,
NeilBrown

Patch
diff mbox series

diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
index 2d2c066a11ba..22957d142cc0 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
@@ -266,6 +266,12 @@  struct lnet_net {
 	 * lnet/include/lnet/nidstr.h */
 	__u32			net_id;
 
+	/* total number of CPTs in the array */
+	__u32			net_ncpts;
+
+	/* cumulative CPTs of all NIs in this net */
+	__u32			*net_cpts;
+
 	/* network tunables */
 	struct lnet_ioctl_config_lnd_cmn_tunables net_tunables;
 
diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index e83bdbec11e3..380a3fb1caba 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -91,11 +91,169 @@  lnet_net_unique(__u32 net, struct list_head *netlist)
 	return true;
 }
 
+static bool
+in_array(__u32 *array, __u32 size, __u32 value)
+{
+	int i;
+
+	for (i = 0; i < size; i++) {
+		if (array[i] == value)
+			return false;
+	}
+
+	return true;
+}
+
+static int
+lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)
+{
+	__u32 *added_cpts = NULL;
+	int i, j = 0, rc = 0;
+
+	/*
+	 * no need to go futher since a subset of the NIs already exist on
+	 * all CPTs
+	 */
+	if (net->net_ncpts == LNET_CPT_NUMBER)
+		return 0;
+
+	if (cpts == NULL) {
+		/* there is an NI which will exist on all CPTs */
+		if (net->net_cpts != NULL)
+			kvfree(net->net_cpts);
+		net->net_cpts = NULL;
+		net->net_ncpts = LNET_CPT_NUMBER;
+		return 0;
+	}
+
+	if (net->net_cpts == NULL) {
+		net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts),
+					      GFP_KERNEL);
+		if (net->net_cpts == NULL)
+			return -ENOMEM;
+		memcpy(net->net_cpts, cpts, ncpts);
+		return 0;
+	}
+
+	added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts),
+				   GFP_KERNEL);
+	if (added_cpts == NULL)
+		return -ENOMEM;
+
+	for (i = 0; i < ncpts; i++) {
+		if (!in_array(net->net_cpts, net->net_ncpts, cpts[i])) {
+			added_cpts[j] = cpts[i];
+			j++;
+		}
+	}
+
+	/* append the new cpts if any to the list of cpts in the net */
+	if (j > 0) {
+		__u32 *array = NULL, *loc;
+		__u32 total_entries = j + net->net_ncpts;
+
+		array = kmalloc_array(total_entries, sizeof(*net->net_cpts),
+				      GFP_KERNEL);
+		if (array == NULL) {
+			rc = -ENOMEM;
+			goto failed;
+		}
+
+		memcpy(array, net->net_cpts,
+		       net->net_ncpts * sizeof(*net->net_cpts));
+		loc = array + net->net_ncpts;
+		memcpy(loc, added_cpts, j * sizeof(*net->net_cpts));
+
+		kfree(net->net_cpts);
+		net->net_ncpts = total_entries;
+		net->net_cpts = array;
+	}
+
+failed:
+	kfree(added_cpts);
+
+	return rc;
+}
+
+static void
+lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)
+{
+	struct lnet_ni *ni;
+	int rc;
+
+	/*
+	 * Operation Assumption:
+	 *	This function is called after an NI has been removed from
+	 *	its parent net.
+	 *
+	 * if we're removing an NI which exists on all CPTs then
+	 * we have to check if any of the other NIs on this net also
+	 * exists on all CPTs. If none, then we need to build our Net CPT
+	 * list based on the remaining NIs.
+	 *
+	 * If the NI being removed exist on a subset of the CPTs then we
+	 * alo rebuild the Net CPT list based on the remaining NIs, which
+	 * should resutl in the expected Net CPT list.
+	 */
+
+	/*
+	 * sometimes this function can be called due to some failure
+	 * creating an NI, before any of the cpts are allocated, so check
+	 * for that case and don't do anything
+	 */
+	if (ncpts == 0)
+		return;
+
+	if (ncpts == LNET_CPT_NUMBER) {
+		/*
+		 * first iteration through the NI list in the net to see
+		 * if any of the NIs exist on all the CPTs. If one is
+		 * found then our job is done.
+		 */
+		list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+			if (ni->ni_ncpts == LNET_CPT_NUMBER)
+				return;
+		}
+	}
+
+	/*
+	 * Rebuild the Net CPT list again, thereby only including only the
+	 * CPTs which the remaining NIs are associated with.
+	 */
+	if (net->net_cpts != NULL) {
+		kfree(net->net_cpts);
+		net->net_cpts = NULL;
+	}
+
+	list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+		rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts,
+					  net);
+		if (rc != 0) {
+			CERROR("Out of Memory\n");
+			/*
+			 * do our best to keep on going. Delete
+			 * the net cpts and set it to NULL. This
+			 * way we can keep on going but less
+			 * efficiently, since memory accesses might be
+			 * accross CPT lines.
+			 */
+			if (net->net_cpts != NULL) {
+				kfree(net->net_cpts);
+				net->net_cpts = NULL;
+				net->net_ncpts = LNET_CPT_NUMBER;
+			}
+			return;
+		}
+	}
+}
+
 void
 lnet_ni_free(struct lnet_ni *ni)
 {
 	int i;
 
+	lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net);
+
 	if (ni->ni_refs)
 		cfs_percpt_free(ni->ni_refs);
 
@@ -128,6 +286,9 @@  lnet_net_free(struct lnet_net *net)
 		lnet_ni_free(ni);
 	}
 
+	if (net->net_cpts != NULL)
+		kfree(net->net_cpts);
+
 	kfree(net);
 }
 
@@ -229,6 +390,9 @@  lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
 		ni->ni_net_ns = NULL;
 
 	ni->ni_last_alive = ktime_get_real_seconds();
+	rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, net);
+	if (rc != 0)
+		goto failed;
 	list_add_tail(&ni->ni_netlist, &net->net_ni_list);
 
 	return ni;