| Message ID | 153628137159.8267.921309094971745898.stgit@noble (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Beginning of multi-rail support for drivers/staging/lustre | expand |
I agree with a comment from James Simmons: __u32 should only be used when the variable is being shared with user space. We need to start converting all uses of __uXX in LNet to just uXX. Perhaps that should be a set of future patches once all of MR/DD has landed? Reviewed-by: Doug Oucharek <dougso@me.com<mailto:dougso@me.com>> Doug On Sep 6, 2018, at 5:49 PM, NeilBrown <neilb@suse.com<mailto:neilb@suse.com>> wrote: struct lnet_net now has a list of cpts, which is the union of the cpts for each lnet_ni. This is part of 8cbb8cd3e771e7f7e0f99cafc19fad32770dc015 LU-7734 lnet: Multi-Rail local NI split Signed-off-by: NeilBrown <neilb@suse.com<mailto:neilb@suse.com>> --- .../staging/lustre/include/linux/lnet/lib-types.h | 6 + drivers/staging/lustre/lnet/lnet/config.c | 164 ++++++++++++++++++++ 2 files changed, 170 insertions(+) diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h index 2d2c066a11ba..22957d142cc0 100644 --- a/drivers/staging/lustre/include/linux/lnet/lib-types.h +++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h @@ -266,6 +266,12 @@ struct lnet_net { * lnet/include/lnet/nidstr.h */ __u32 net_id; + /* total number of CPTs in the array */ + __u32 net_ncpts; + + /* cumulative CPTs of all NIs in this net */ + __u32 *net_cpts; + /* network tunables */ struct lnet_ioctl_config_lnd_cmn_tunables net_tunables; diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c index e83bdbec11e3..380a3fb1caba 100644 --- a/drivers/staging/lustre/lnet/lnet/config.c +++ b/drivers/staging/lustre/lnet/lnet/config.c @@ -91,11 +91,169 @@ lnet_net_unique(__u32 net, struct list_head *netlist) return true; } +static bool +in_array(__u32 *array, __u32 size, __u32 value) +{ + int i; + + for (i = 0; i < size; i++) { + if (array[i] == value) + return false; + } + + return true; +} + +static int +lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net) +{ + __u32 *added_cpts = NULL; + int i, j = 0, rc = 0; + + /* + * no need to go futher since a subset of the NIs already exist on + * all CPTs + */ + if (net->net_ncpts == LNET_CPT_NUMBER) + return 0; + + if (cpts == NULL) { + /* there is an NI which will exist on all CPTs */ + if (net->net_cpts != NULL) + kvfree(net->net_cpts); + net->net_cpts = NULL; + net->net_ncpts = LNET_CPT_NUMBER; + return 0; + } + + if (net->net_cpts == NULL) { + net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts), + GFP_KERNEL); + if (net->net_cpts == NULL) + return -ENOMEM; + memcpy(net->net_cpts, cpts, ncpts); + return 0; + } + + added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts), + GFP_KERNEL); + if (added_cpts == NULL) + return -ENOMEM; + + for (i = 0; i < ncpts; i++) { + if (!in_array(net->net_cpts, net->net_ncpts, cpts[i])) { + added_cpts[j] = cpts[i]; + j++; + } + } + + /* append the new cpts if any to the list of cpts in the net */ + if (j > 0) { + __u32 *array = NULL, *loc; + __u32 total_entries = j + net->net_ncpts; + + array = kmalloc_array(total_entries, sizeof(*net->net_cpts), + GFP_KERNEL); + if (array == NULL) { + rc = -ENOMEM; + goto failed; + } + + memcpy(array, net->net_cpts, + net->net_ncpts * sizeof(*net->net_cpts)); + loc = array + net->net_ncpts; + memcpy(loc, added_cpts, j * sizeof(*net->net_cpts)); + + kfree(net->net_cpts); + net->net_ncpts = total_entries; + net->net_cpts = array; + } + +failed: + kfree(added_cpts); + + return rc; +} + +static void +lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net) +{ + struct lnet_ni *ni; + int rc; + + /* + * Operation Assumption: + * This function is called after an NI has been removed from + * its parent net. + * + * if we're removing an NI which exists on all CPTs then + * we have to check if any of the other NIs on this net also + * exists on all CPTs. If none, then we need to build our Net CPT + * list based on the remaining NIs. + * + * If the NI being removed exist on a subset of the CPTs then we + * alo rebuild the Net CPT list based on the remaining NIs, which + * should resutl in the expected Net CPT list. + */ + + /* + * sometimes this function can be called due to some failure + * creating an NI, before any of the cpts are allocated, so check + * for that case and don't do anything + */ + if (ncpts == 0) + return; + + if (ncpts == LNET_CPT_NUMBER) { + /* + * first iteration through the NI list in the net to see + * if any of the NIs exist on all the CPTs. If one is + * found then our job is done. + */ + list_for_each_entry(ni, &net->net_ni_list, ni_netlist) { + if (ni->ni_ncpts == LNET_CPT_NUMBER) + return; + } + } + + /* + * Rebuild the Net CPT list again, thereby only including only the + * CPTs which the remaining NIs are associated with. + */ + if (net->net_cpts != NULL) { + kfree(net->net_cpts); + net->net_cpts = NULL; + } + + list_for_each_entry(ni, &net->net_ni_list, ni_netlist) { + rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, + net); + if (rc != 0) { + CERROR("Out of Memory\n"); + /* + * do our best to keep on going. Delete + * the net cpts and set it to NULL. This + * way we can keep on going but less + * efficiently, since memory accesses might be + * accross CPT lines. + */ + if (net->net_cpts != NULL) { + kfree(net->net_cpts); + net->net_cpts = NULL; + net->net_ncpts = LNET_CPT_NUMBER; + } + return; + } + } +} + void lnet_ni_free(struct lnet_ni *ni) { int i; + lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net); + if (ni->ni_refs) cfs_percpt_free(ni->ni_refs); @@ -128,6 +286,9 @@ lnet_net_free(struct lnet_net *net) lnet_ni_free(ni); } + if (net->net_cpts != NULL) + kfree(net->net_cpts); + kfree(net); } @@ -229,6 +390,9 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface) ni->ni_net_ns = NULL; ni->ni_last_alive = ktime_get_real_seconds(); + rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, net); + if (rc != 0) + goto failed; list_add_tail(&ni->ni_netlist, &net->net_ni_list); return ni; <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=us-ascii"> </head> <body style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;" class=""> I agree with a comment from James Simmons: __u32 should only be used when the variable is being shared with user space. We need to start converting all uses of __uXX in LNet to just uXX. Perhaps that should be a set of future patches once all of MR/DD has landed? <div class=""><br class=""> </div> <div class="">Reviewed-by: Doug Oucharek <<a href="mailto:dougso@me.com" class="">dougso@me.com</a>></div> <div class=""><br class=""> </div> <div class="">Doug</div> <div class=""><br class=""> </div> <div class=""> <div> <blockquote type="cite" class=""> <div class="">On Sep 6, 2018, at 5:49 PM, NeilBrown <<a href="mailto:neilb@suse.com" class="">neilb@suse.com</a>> wrote:</div> <br class="Apple-interchange-newline"> <div class=""> <div class="">struct lnet_net now has a list of cpts, which is the union<br class=""> of the cpts for each lnet_ni.<br class=""> <br class=""> This is part of<br class=""> 8cbb8cd3e771e7f7e0f99cafc19fad32770dc015<br class=""> LU-7734 lnet: Multi-Rail local NI split<br class=""> <br class=""> Signed-off-by: NeilBrown <<a href="mailto:neilb@suse.com" class="">neilb@suse.com</a>><br class=""> ---<br class=""> .../staging/lustre/include/linux/lnet/lib-types.h | 6 +<br class=""> drivers/staging/lustre/lnet/lnet/config.c | 164 ++++++++++++++++++++<br class=""> 2 files changed, 170 insertions(+)<br class=""> <br class=""> diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h<br class=""> index 2d2c066a11ba..22957d142cc0 100644<br class=""> --- a/drivers/staging/lustre/include/linux/lnet/lib-types.h<br class=""> +++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h<br class=""> @@ -266,6 +266,12 @@ struct lnet_net {<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>* lnet/include/lnet/nidstr.h */<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>__u32<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>net_id;<br class=""> <br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>/* total number of CPTs in the array */<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>__u32<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>net_ncpts;<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>/* cumulative CPTs of all NIs in this net */<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>__u32<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>*net_cpts;<br class=""> +<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>/* network tunables */<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>struct lnet_ioctl_config_lnd_cmn_tunables net_tunables;<br class=""> <br class=""> diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c<br class=""> index e83bdbec11e3..380a3fb1caba 100644<br class=""> --- a/drivers/staging/lustre/lnet/lnet/config.c<br class=""> +++ b/drivers/staging/lustre/lnet/lnet/config.c<br class=""> @@ -91,11 +91,169 @@ lnet_net_unique(__u32 net, struct list_head *netlist)<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>return true;<br class=""> }<br class=""> <br class=""> +static bool<br class=""> +in_array(__u32 *array, __u32 size, __u32 value)<br class=""> +{<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>int i;<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>for (i = 0; i < size; i++) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (array[i] == value)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>return false;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>return true;<br class=""> +}<br class=""> +<br class=""> +static int<br class=""> +lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)<br class=""> +{<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>__u32 *added_cpts = NULL;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>int i, j = 0, rc = 0;<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>/*<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* no need to go futher since a subset of the NIs already exist on<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* all CPTs<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>*/<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (net->net_ncpts == LNET_CPT_NUMBER)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return 0;<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (cpts == NULL) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>/* there is an NI which will exist on all CPTs */<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (net->net_cpts != NULL)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>kvfree(net->net_cpts);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net->net_cpts = NULL;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net->net_ncpts = LNET_CPT_NUMBER;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return 0;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (net->net_cpts == NULL) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts),<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span> GFP_KERNEL);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (net->net_cpts == NULL)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>return -ENOMEM;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>memcpy(net->net_cpts, cpts, ncpts);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return 0;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts),<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span> GFP_KERNEL);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (added_cpts == NULL)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return -ENOMEM;<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>for (i = 0; i < ncpts; i++) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (!in_array(net->net_cpts, net->net_ncpts, cpts[i])) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>added_cpts[j] = cpts[i];<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>j++;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>/* append the new cpts if any to the list of cpts in the net */<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (j > 0) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>__u32 *array = NULL, *loc;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>__u32 total_entries = j + net->net_ncpts;<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>array = kmalloc_array(total_entries, sizeof(*net->net_cpts),<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span> GFP_KERNEL);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (array == NULL) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>rc = -ENOMEM;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>goto failed;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>memcpy(array, net->net_cpts,<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span> net->net_ncpts * sizeof(*net->net_cpts));<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>loc = array + net->net_ncpts;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>memcpy(loc, added_cpts, j * sizeof(*net->net_cpts));<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>kfree(net->net_cpts);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net->net_ncpts = total_entries;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net->net_cpts = array;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class=""> +<br class=""> +failed:<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>kfree(added_cpts);<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>return rc;<br class=""> +}<br class=""> +<br class=""> +static void<br class=""> +lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)<br class=""> +{<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>struct lnet_ni *ni;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>int rc;<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>/*<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* Operation Assumption:<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>*<span class="Apple-tab-span" style="white-space:pre"> </span>This function is called after an NI has been removed from<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>*<span class="Apple-tab-span" style="white-space:pre"> </span>its parent net.<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>*<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* if we're removing an NI which exists on all CPTs then<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* we have to check if any of the other NIs on this net also<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* exists on all CPTs. If none, then we need to build our Net CPT<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* list based on the remaining NIs.<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>*<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* If the NI being removed exist on a subset of the CPTs then we<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* alo rebuild the Net CPT list based on the remaining NIs, which<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* should resutl in the expected Net CPT list.<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>*/<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>/*<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* sometimes this function can be called due to some failure<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* creating an NI, before any of the cpts are allocated, so check<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* for that case and don't do anything<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>*/<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (ncpts == 0)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>return;<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (ncpts == LNET_CPT_NUMBER) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>/*<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>* first iteration through the NI list in the net to see<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>* if any of the NIs exist on all the CPTs. If one is<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>* found then our job is done.<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>*/<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>if (ni->ni_ncpts == LNET_CPT_NUMBER)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>return;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>/*<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* Rebuild the Net CPT list again, thereby only including only the<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>* CPTs which the remaining NIs are associated with.<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>*/<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (net->net_cpts != NULL) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>kfree(net->net_cpts);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>net->net_cpts = NULL;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class=""> +<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts,<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span> net);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>if (rc != 0) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>CERROR("Out of Memory\n");<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>/*<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* do our best to keep on going. Delete<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* the net cpts and set it to NULL. This<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* way we can keep on going but less<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* efficiently, since memory accesses might be<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>* accross CPT lines.<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>*/<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>if (net->net_cpts != NULL) {<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>kfree(net->net_cpts);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>net->net_cpts = NULL;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>net->net_ncpts = LNET_CPT_NUMBER;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>return;<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>}<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>}<br class=""> +}<br class=""> +<br class=""> void<br class=""> lnet_ni_free(struct lnet_ni *ni)<br class=""> {<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>int i;<br class=""> <br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net);<br class=""> +<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>if (ni->ni_refs)<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>cfs_percpt_free(ni->ni_refs);<br class=""> <br class=""> @@ -128,6 +286,9 @@ lnet_net_free(struct lnet_net *net)<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>lnet_ni_free(ni);<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>}<br class=""> <br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (net->net_cpts != NULL)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>kfree(net->net_cpts);<br class=""> +<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>kfree(net);<br class=""> }<br class=""> <br class=""> @@ -229,6 +390,9 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span><span class="Apple-tab-span" style="white-space:pre"></span>ni->ni_net_ns = NULL;<br class=""> <br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>ni->ni_last_alive = ktime_get_real_seconds();<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, net);<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span>if (rc != 0)<br class=""> +<span class="Apple-tab-span" style="white-space:pre"> </span><span class="Apple-tab-span" style="white-space:pre"></span>goto failed;<br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>list_add_tail(&ni->ni_netlist, &net->net_ni_list);<br class=""> <br class=""> <span class="Apple-tab-span" style="white-space:pre"></span>return ni;<br class=""> <br class=""> <br class=""> </div> </div> </blockquote> </div> <br class=""> </div> </body> </html>
> struct lnet_net now has a list of cpts, which is the union > of the cpts for each lnet_ni. Reviewed-by: James Simmons <jsimmons@infradead.org> The below needs fixing based on response to cover letter. > This is part of > 8cbb8cd3e771e7f7e0f99cafc19fad32770dc015 > LU-7734 lnet: Multi-Rail local NI split > > Signed-off-by: NeilBrown <neilb@suse.com> > --- > .../staging/lustre/include/linux/lnet/lib-types.h | 6 + > drivers/staging/lustre/lnet/lnet/config.c | 164 ++++++++++++++++++++ > 2 files changed, 170 insertions(+) > > diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h > index 2d2c066a11ba..22957d142cc0 100644 > --- a/drivers/staging/lustre/include/linux/lnet/lib-types.h > +++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h > @@ -266,6 +266,12 @@ struct lnet_net { > * lnet/include/lnet/nidstr.h */ > __u32 net_id; > > + /* total number of CPTs in the array */ > + __u32 net_ncpts; > + > + /* cumulative CPTs of all NIs in this net */ > + __u32 *net_cpts; > + > /* network tunables */ > struct lnet_ioctl_config_lnd_cmn_tunables net_tunables; > > diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c > index e83bdbec11e3..380a3fb1caba 100644 > --- a/drivers/staging/lustre/lnet/lnet/config.c > +++ b/drivers/staging/lustre/lnet/lnet/config.c > @@ -91,11 +91,169 @@ lnet_net_unique(__u32 net, struct list_head *netlist) > return true; > } > > +static bool > +in_array(__u32 *array, __u32 size, __u32 value) > +{ > + int i; > + > + for (i = 0; i < size; i++) { > + if (array[i] == value) > + return false; > + } > + > + return true; > +} > + > +static int > +lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net) > +{ > + __u32 *added_cpts = NULL; > + int i, j = 0, rc = 0; > + > + /* > + * no need to go futher since a subset of the NIs already exist on > + * all CPTs > + */ > + if (net->net_ncpts == LNET_CPT_NUMBER) > + return 0; > + > + if (cpts == NULL) { > + /* there is an NI which will exist on all CPTs */ > + if (net->net_cpts != NULL) > + kvfree(net->net_cpts); > + net->net_cpts = NULL; > + net->net_ncpts = LNET_CPT_NUMBER; > + return 0; > + } > + > + if (net->net_cpts == NULL) { > + net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts), > + GFP_KERNEL); > + if (net->net_cpts == NULL) > + return -ENOMEM; > + memcpy(net->net_cpts, cpts, ncpts); > + return 0; > + } > + > + added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts), > + GFP_KERNEL); > + if (added_cpts == NULL) > + return -ENOMEM; > + > + for (i = 0; i < ncpts; i++) { > + if (!in_array(net->net_cpts, net->net_ncpts, cpts[i])) { > + added_cpts[j] = cpts[i]; > + j++; > + } > + } > + > + /* append the new cpts if any to the list of cpts in the net */ > + if (j > 0) { > + __u32 *array = NULL, *loc; > + __u32 total_entries = j + net->net_ncpts; > + > + array = kmalloc_array(total_entries, sizeof(*net->net_cpts), > + GFP_KERNEL); > + if (array == NULL) { > + rc = -ENOMEM; > + goto failed; > + } > + > + memcpy(array, net->net_cpts, > + net->net_ncpts * sizeof(*net->net_cpts)); > + loc = array + net->net_ncpts; > + memcpy(loc, added_cpts, j * sizeof(*net->net_cpts)); > + > + kfree(net->net_cpts); > + net->net_ncpts = total_entries; > + net->net_cpts = array; > + } > + > +failed: > + kfree(added_cpts); > + > + return rc; > +} > + > +static void > +lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net) > +{ > + struct lnet_ni *ni; > + int rc; > + > + /* > + * Operation Assumption: > + * This function is called after an NI has been removed from > + * its parent net. > + * > + * if we're removing an NI which exists on all CPTs then > + * we have to check if any of the other NIs on this net also > + * exists on all CPTs. If none, then we need to build our Net CPT > + * list based on the remaining NIs. > + * > + * If the NI being removed exist on a subset of the CPTs then we > + * alo rebuild the Net CPT list based on the remaining NIs, which > + * should resutl in the expected Net CPT list. > + */ > + > + /* > + * sometimes this function can be called due to some failure > + * creating an NI, before any of the cpts are allocated, so check > + * for that case and don't do anything > + */ > + if (ncpts == 0) > + return; > + > + if (ncpts == LNET_CPT_NUMBER) { > + /* > + * first iteration through the NI list in the net to see > + * if any of the NIs exist on all the CPTs. If one is > + * found then our job is done. > + */ > + list_for_each_entry(ni, &net->net_ni_list, ni_netlist) { > + if (ni->ni_ncpts == LNET_CPT_NUMBER) > + return; > + } > + } > + > + /* > + * Rebuild the Net CPT list again, thereby only including only the > + * CPTs which the remaining NIs are associated with. > + */ > + if (net->net_cpts != NULL) { > + kfree(net->net_cpts); > + net->net_cpts = NULL; > + } > + > + list_for_each_entry(ni, &net->net_ni_list, ni_netlist) { > + rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, > + net); > + if (rc != 0) { > + CERROR("Out of Memory\n"); > + /* > + * do our best to keep on going. Delete > + * the net cpts and set it to NULL. This > + * way we can keep on going but less > + * efficiently, since memory accesses might be > + * accross CPT lines. > + */ > + if (net->net_cpts != NULL) { > + kfree(net->net_cpts); > + net->net_cpts = NULL; > + net->net_ncpts = LNET_CPT_NUMBER; > + } > + return; > + } > + } > +} > + > void > lnet_ni_free(struct lnet_ni *ni) > { > int i; > > + lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net); > + > if (ni->ni_refs) > cfs_percpt_free(ni->ni_refs); > > @@ -128,6 +286,9 @@ lnet_net_free(struct lnet_net *net) > lnet_ni_free(ni); > } > > + if (net->net_cpts != NULL) > + kfree(net->net_cpts); > + > kfree(net); > } > > @@ -229,6 +390,9 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface) > ni->ni_net_ns = NULL; > > ni->ni_last_alive = ktime_get_real_seconds(); > + rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, net); > + if (rc != 0) > + goto failed; > list_add_tail(&ni->ni_netlist, &net->net_ni_list); > > return ni; > > >
On Mon, Sep 10 2018, Doug Oucharek wrote: > I agree with a comment from James Simmons: __u32 should only be used when the variable is being shared with user space. We need to start converting all uses of __uXX in LNet to just uXX. Perhaps that should be a set of future patches once all of MR/DD has landed? > That seems reasonable. I dont't think this series adds significant new uses of __u32. Changing them all to u32 should, as you suggest, come later. > Reviewed-by: Doug Oucharek <dougso@me.com<mailto:dougso@me.com>> Thanks, NeilBrown
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h index 2d2c066a11ba..22957d142cc0 100644 --- a/drivers/staging/lustre/include/linux/lnet/lib-types.h +++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h @@ -266,6 +266,12 @@ struct lnet_net { * lnet/include/lnet/nidstr.h */ __u32 net_id; + /* total number of CPTs in the array */ + __u32 net_ncpts; + + /* cumulative CPTs of all NIs in this net */ + __u32 *net_cpts; + /* network tunables */ struct lnet_ioctl_config_lnd_cmn_tunables net_tunables; diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c index e83bdbec11e3..380a3fb1caba 100644 --- a/drivers/staging/lustre/lnet/lnet/config.c +++ b/drivers/staging/lustre/lnet/lnet/config.c @@ -91,11 +91,169 @@ lnet_net_unique(__u32 net, struct list_head *netlist) return true; } +static bool +in_array(__u32 *array, __u32 size, __u32 value) +{ + int i; + + for (i = 0; i < size; i++) { + if (array[i] == value) + return false; + } + + return true; +} + +static int +lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net) +{ + __u32 *added_cpts = NULL; + int i, j = 0, rc = 0; + + /* + * no need to go futher since a subset of the NIs already exist on + * all CPTs + */ + if (net->net_ncpts == LNET_CPT_NUMBER) + return 0; + + if (cpts == NULL) { + /* there is an NI which will exist on all CPTs */ + if (net->net_cpts != NULL) + kvfree(net->net_cpts); + net->net_cpts = NULL; + net->net_ncpts = LNET_CPT_NUMBER; + return 0; + } + + if (net->net_cpts == NULL) { + net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts), + GFP_KERNEL); + if (net->net_cpts == NULL) + return -ENOMEM; + memcpy(net->net_cpts, cpts, ncpts); + return 0; + } + + added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts), + GFP_KERNEL); + if (added_cpts == NULL) + return -ENOMEM; + + for (i = 0; i < ncpts; i++) { + if (!in_array(net->net_cpts, net->net_ncpts, cpts[i])) { + added_cpts[j] = cpts[i]; + j++; + } + } + + /* append the new cpts if any to the list of cpts in the net */ + if (j > 0) { + __u32 *array = NULL, *loc; + __u32 total_entries = j + net->net_ncpts; + + array = kmalloc_array(total_entries, sizeof(*net->net_cpts), + GFP_KERNEL); + if (array == NULL) { + rc = -ENOMEM; + goto failed; + } + + memcpy(array, net->net_cpts, + net->net_ncpts * sizeof(*net->net_cpts)); + loc = array + net->net_ncpts; + memcpy(loc, added_cpts, j * sizeof(*net->net_cpts)); + + kfree(net->net_cpts); + net->net_ncpts = total_entries; + net->net_cpts = array; + } + +failed: + kfree(added_cpts); + + return rc; +} + +static void +lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net) +{ + struct lnet_ni *ni; + int rc; + + /* + * Operation Assumption: + * This function is called after an NI has been removed from + * its parent net. + * + * if we're removing an NI which exists on all CPTs then + * we have to check if any of the other NIs on this net also + * exists on all CPTs. If none, then we need to build our Net CPT + * list based on the remaining NIs. + * + * If the NI being removed exist on a subset of the CPTs then we + * alo rebuild the Net CPT list based on the remaining NIs, which + * should resutl in the expected Net CPT list. + */ + + /* + * sometimes this function can be called due to some failure + * creating an NI, before any of the cpts are allocated, so check + * for that case and don't do anything + */ + if (ncpts == 0) + return; + + if (ncpts == LNET_CPT_NUMBER) { + /* + * first iteration through the NI list in the net to see + * if any of the NIs exist on all the CPTs. If one is + * found then our job is done. + */ + list_for_each_entry(ni, &net->net_ni_list, ni_netlist) { + if (ni->ni_ncpts == LNET_CPT_NUMBER) + return; + } + } + + /* + * Rebuild the Net CPT list again, thereby only including only the + * CPTs which the remaining NIs are associated with. + */ + if (net->net_cpts != NULL) { + kfree(net->net_cpts); + net->net_cpts = NULL; + } + + list_for_each_entry(ni, &net->net_ni_list, ni_netlist) { + rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, + net); + if (rc != 0) { + CERROR("Out of Memory\n"); + /* + * do our best to keep on going. Delete + * the net cpts and set it to NULL. This + * way we can keep on going but less + * efficiently, since memory accesses might be + * accross CPT lines. + */ + if (net->net_cpts != NULL) { + kfree(net->net_cpts); + net->net_cpts = NULL; + net->net_ncpts = LNET_CPT_NUMBER; + } + return; + } + } +} + void lnet_ni_free(struct lnet_ni *ni) { int i; + lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net); + if (ni->ni_refs) cfs_percpt_free(ni->ni_refs); @@ -128,6 +286,9 @@ lnet_net_free(struct lnet_net *net) lnet_ni_free(ni); } + if (net->net_cpts != NULL) + kfree(net->net_cpts); + kfree(net); } @@ -229,6 +390,9 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface) ni->ni_net_ns = NULL; ni->ni_last_alive = ktime_get_real_seconds(); + rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, net); + if (rc != 0) + goto failed; list_add_tail(&ni->ni_netlist, &net->net_ni_list); return ni;
struct lnet_net now has a list of cpts, which is the union of the cpts for each lnet_ni. This is part of 8cbb8cd3e771e7f7e0f99cafc19fad32770dc015 LU-7734 lnet: Multi-Rail local NI split Signed-off-by: NeilBrown <neilb@suse.com> --- .../staging/lustre/include/linux/lnet/lib-types.h | 6 + drivers/staging/lustre/lnet/lnet/config.c | 164 ++++++++++++++++++++ 2 files changed, 170 insertions(+)