| Message ID | 20230330224726.662344-4-paulmck@kernel.org (mailing list archive) |
|---|---|
| State | Accepted |
| Commit | 95433f7263011e0e6e83caef85d98896dd99cab7 |
| Headers | show |
| Series | Further shrink srcu_struct to promote cache locality | expand |
On Thu, Mar 30, 2023 at 03:47:10PM -0700, Paul E. McKenney wrote: > The current srcu_struct structure is on the order of 200 bytes in size > (depending on architecture and .config), which is much better than the > old-style 26K bytes, but still all too inconvenient when one is trying > to achieve good cache locality on a fastpath involving SRCU readers. > > However, only a few fields in srcu_struct are used by SRCU readers. > The remaining fields could be offloaded to a new srcu_update > structure, thus shrinking the srcu_struct structure down to a few > tens of bytes. This commit begins this noble quest, a quest that is > complicated by open-coded initialization of the srcu_struct within the > srcu_notifier_head structure. This complication is addressed by updating > the srcu_notifier_head structure's open coding, given that there does > not appear to be a straightforward way of abstracting that initialization. > > This commit moves only the ->node pointer to srcu_update. Later commits > will move additional fields. > > [ paulmck: Fold in qiang1.zhang@intel.com's memory-leak fix. ] > > Link: https://lore.kernel.org/all/20230320055751.4120251-1-qiang1.zhang@intel.com/ > Suggested-by: Christoph Hellwig <hch@lst.de> [..] > @@ -236,8 +236,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) > */ > static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) > { > + if (!is_static) > + ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL); > + if (!ssp->srcu_sup) > + return -ENOMEM; > ssp->srcu_size_state = SRCU_SIZE_SMALL; > - ssp->node = NULL; > + ssp->srcu_sup->node = NULL; > mutex_init(&ssp->srcu_cb_mutex); > mutex_init(&ssp->srcu_gp_mutex); > ssp->srcu_idx = 0; > @@ -249,8 +253,11 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) > ssp->sda_is_static = is_static; > if (!is_static) > ssp->sda = alloc_percpu(struct srcu_data); > - if (!ssp->sda) > + if (!ssp->sda) { > + if (!is_static) > + kfree(ssp->srcu_sup); > return -ENOMEM; > + } > init_srcu_struct_data(ssp); > ssp->srcu_gp_seq_needed_exp = 0; > ssp->srcu_last_gp_end = ktime_get_mono_fast_ns(); > @@ -259,6 +266,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) [1] Here there is an if (!init_srcu_struct_nodes(...)) that the diff does not show. > if (!ssp->sda_is_static) { > free_percpu(ssp->sda); > ssp->sda = NULL; > + kfree(ssp->srcu_sup); > return -ENOMEM; > } > } else { Just a comment about the original code with reference to [1]. Here if allocations in init_srcu_struct_nodes() fail, it will return false and execute the "if (!ssp->sda_is_is_static)" bit. So if the allocation in [1] fails, then if sda_is_static is true, we return -ENOMEM, however if sda_is_static is false, we do the following: ssp->srcu_sup->srcu_ssp = ssp; smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */ return 0; Is that really correct? In other words, if init_srcu_struct_nodes() returns false, then passing along the return value of init_srcu_struct_nodes() back to the caller of init_srcu_struct_fields() depends on whether is_static = true or false. That seems a bit wrong to me, init_srcu_struct_fields() should always return -ENOMEM when init_srcu_struct_nodes() fails to allocate memory IMHO, whether is_static is true or not. Sorry if I missed something subtle, and if the code is correct to begin with. Also I feel the return paths could be made better to also fix the above issue I mentioned. How about the following diff on top of the series, would it work? Thanks! ---8<----------------------- diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index a887cfc89894..1975d06986fa 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -255,29 +255,30 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) ssp->srcu_sup->sda_is_static = is_static; if (!is_static) ssp->sda = alloc_percpu(struct srcu_data); - if (!ssp->sda) { - if (!is_static) - kfree(ssp->srcu_sup); - return -ENOMEM; - } + if (!ssp->sda) + goto err_free_sup; init_srcu_struct_data(ssp); ssp->srcu_sup->srcu_gp_seq_needed_exp = 0; ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns(); if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { - if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) { - if (!ssp->srcu_sup->sda_is_static) { - free_percpu(ssp->sda); - ssp->sda = NULL; - kfree(ssp->srcu_sup); - return -ENOMEM; - } - } else { + if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) + goto err_free_sda; + else WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); - } } ssp->srcu_sup->srcu_ssp = ssp; smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */ return 0; + +err_free_sda: + if (!is_static) { + free_percpu(ssp->sda); + ssp->sda = NULL; + } +err_free_sup: + if (!is_static) + kfree(ssp->srcu_sup); + return -ENOMEM; } #ifdef CONFIG_DEBUG_LOCK_ALLOC
On Tue, Apr 04, 2023 at 12:35:08AM +0000, Joel Fernandes wrote: > On Thu, Mar 30, 2023 at 03:47:10PM -0700, Paul E. McKenney wrote: > > The current srcu_struct structure is on the order of 200 bytes in size > > (depending on architecture and .config), which is much better than the > > old-style 26K bytes, but still all too inconvenient when one is trying > > to achieve good cache locality on a fastpath involving SRCU readers. > > > > However, only a few fields in srcu_struct are used by SRCU readers. > > The remaining fields could be offloaded to a new srcu_update > > structure, thus shrinking the srcu_struct structure down to a few > > tens of bytes. This commit begins this noble quest, a quest that is > > complicated by open-coded initialization of the srcu_struct within the > > srcu_notifier_head structure. This complication is addressed by updating > > the srcu_notifier_head structure's open coding, given that there does > > not appear to be a straightforward way of abstracting that initialization. > > > > This commit moves only the ->node pointer to srcu_update. Later commits > > will move additional fields. > > > > [ paulmck: Fold in qiang1.zhang@intel.com's memory-leak fix. ] > > > > Link: https://lore.kernel.org/all/20230320055751.4120251-1-qiang1.zhang@intel.com/ > > Suggested-by: Christoph Hellwig <hch@lst.de> > > [..] > > @@ -236,8 +236,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) > > */ > > static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) > > { > > + if (!is_static) > > + ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL); > > + if (!ssp->srcu_sup) > > + return -ENOMEM; > > ssp->srcu_size_state = SRCU_SIZE_SMALL; > > - ssp->node = NULL; > > + ssp->srcu_sup->node = NULL; > > mutex_init(&ssp->srcu_cb_mutex); > > mutex_init(&ssp->srcu_gp_mutex); > > ssp->srcu_idx = 0; > > @@ -249,8 +253,11 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) > > ssp->sda_is_static = is_static; > > if (!is_static) > > ssp->sda = alloc_percpu(struct srcu_data); > > - if (!ssp->sda) > > + if (!ssp->sda) { > > + if (!is_static) > > + kfree(ssp->srcu_sup); > > return -ENOMEM; > > + } > > init_srcu_struct_data(ssp); > > ssp->srcu_gp_seq_needed_exp = 0; > > ssp->srcu_last_gp_end = ktime_get_mono_fast_ns(); > > @@ -259,6 +266,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) > > [1] Here there is an if (!init_srcu_struct_nodes(...)) that the diff does not show. > > > if (!ssp->sda_is_static) { > > free_percpu(ssp->sda); > > ssp->sda = NULL; > > + kfree(ssp->srcu_sup); > > return -ENOMEM; > > } > > } else { > > > Just a comment about the original code with reference to [1]. > > Here if allocations in init_srcu_struct_nodes() fail, it will return false > and execute the "if (!ssp->sda_is_is_static)" bit. > > So if the allocation in [1] fails, then if sda_is_static is true, we return > -ENOMEM, however if sda_is_static is false, we do the following: > > ssp->srcu_sup->srcu_ssp = ssp; > smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */ > return 0; > > Is that really correct? Doesn't look like it, now that you mention it. Good catch, thank you! > In other words, if init_srcu_struct_nodes() returns false, then passing along > the return value of init_srcu_struct_nodes() back to the caller of > init_srcu_struct_fields() depends on whether is_static = true or false. That > seems a bit wrong to me, init_srcu_struct_fields() should always return > -ENOMEM when init_srcu_struct_nodes() fails to allocate memory IMHO, whether > is_static is true or not. > > Sorry if I missed something subtle, and if the code is correct to begin with. > Also I feel the return paths could be made better to also fix the above issue > I mentioned. How about the following diff on top of the series, would it > work? Your restructuring looks like an excellent step forward, but given the late date, it might be best to avoid being in a rush. I -could- make the following small patch: if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) { if (!ssp->srcu_sup->sda_is_static) { free_percpu(ssp->sda); ssp->sda = NULL; kfree(ssp->srcu_sup); } return -ENOMEM; } else { WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); } } Except that this is a pre-existing bug that as far as we know no one is hitting, so the risk doesn't seem to stack up. After all, if you are hitting memory exhaustion at boot or on module load, this bug is probably the least of your problems. Even this fix looks to be v6.5 material to me. So would you be willing to send a patch like the above that fixes the bug, and another like you have below to get a better error-path structure? No hurry, the end of this month is perfectly fine. And again, good catch! Thanx, Paul > Thanks! > > ---8<----------------------- > > diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c > index a887cfc89894..1975d06986fa 100644 > --- a/kernel/rcu/srcutree.c > +++ b/kernel/rcu/srcutree.c > @@ -255,29 +255,30 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) > ssp->srcu_sup->sda_is_static = is_static; > if (!is_static) > ssp->sda = alloc_percpu(struct srcu_data); > - if (!ssp->sda) { > - if (!is_static) > - kfree(ssp->srcu_sup); > - return -ENOMEM; > - } > + if (!ssp->sda) > + goto err_free_sup; > init_srcu_struct_data(ssp); > ssp->srcu_sup->srcu_gp_seq_needed_exp = 0; > ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns(); > if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { > - if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) { > - if (!ssp->srcu_sup->sda_is_static) { > - free_percpu(ssp->sda); > - ssp->sda = NULL; > - kfree(ssp->srcu_sup); > - return -ENOMEM; > - } > - } else { > + if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) > + goto err_free_sda; > + else > WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); > - } > } > ssp->srcu_sup->srcu_ssp = ssp; > smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */ > return 0; > + > +err_free_sda: > + if (!is_static) { > + free_percpu(ssp->sda); > + ssp->sda = NULL; > + } > +err_free_sup: > + if (!is_static) > + kfree(ssp->srcu_sup); > + return -ENOMEM; > } > > #ifdef CONFIG_DEBUG_LOCK_ALLOC
On Mon, Apr 3, 2023 at 9:06 PM Paul E. McKenney <paulmck@kernel.org> wrote: [..] > > In other words, if init_srcu_struct_nodes() returns false, then passing along > > the return value of init_srcu_struct_nodes() back to the caller of > > init_srcu_struct_fields() depends on whether is_static = true or false. That > > seems a bit wrong to me, init_srcu_struct_fields() should always return > > -ENOMEM when init_srcu_struct_nodes() fails to allocate memory IMHO, whether > > is_static is true or not. > > > > Sorry if I missed something subtle, and if the code is correct to begin with. > > Also I feel the return paths could be made better to also fix the above issue > > I mentioned. How about the following diff on top of the series, would it > > work? > > Your restructuring looks like an excellent step forward, but given the late > date, it might be best to avoid being in a rush. > > I -could- make the following small patch: > > if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { > if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) { > if (!ssp->srcu_sup->sda_is_static) { > free_percpu(ssp->sda); > ssp->sda = NULL; > kfree(ssp->srcu_sup); > } > return -ENOMEM; > } else { > WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); > } > } > > Except that this is a pre-existing bug that as far as we know no one > is hitting, so the risk doesn't seem to stack up. After all, if you > are hitting memory exhaustion at boot or on module load, this bug is > probably the least of your problems. Even this fix looks to be v6.5 > material to me. > > So would you be willing to send a patch like the above that fixes the > bug, and another like you have below to get a better error-path > structure? No hurry, the end of this month is perfectly fine. Yes, I am happy to send patches along these lines by the end of the month. I will make a note to do so. > And again, good catch! Thanks! - Joel
diff --git a/include/linux/notifier.h b/include/linux/notifier.h index aef88c2d1173..2aba75145144 100644 --- a/include/linux/notifier.h +++ b/include/linux/notifier.h @@ -73,6 +73,9 @@ struct raw_notifier_head { struct srcu_notifier_head { struct mutex mutex; +#ifdef CONFIG_TREE_SRCU + struct srcu_usage srcuu; +#endif struct srcu_struct srcu; struct notifier_block __rcu *head; }; @@ -107,7 +110,7 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); { \ .mutex = __MUTEX_INITIALIZER(name.mutex), \ .head = NULL, \ - .srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \ + .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \ } #define ATOMIC_NOTIFIER_HEAD(name) \ diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 5aa5e0faf6a1..ebd72491af99 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -31,7 +31,7 @@ struct srcu_struct { void srcu_drive_gp(struct work_struct *wp); -#define __SRCU_STRUCT_INIT(name, __ignored) \ +#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored) \ { \ .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ .srcu_cb_tail = &name.srcu_cb_head, \ @@ -44,9 +44,9 @@ void srcu_drive_gp(struct work_struct *wp); * Tree SRCU, which needs some per-CPU data. */ #define DEFINE_SRCU(name) \ - struct srcu_struct name = __SRCU_STRUCT_INIT(name, name) + struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name) #define DEFINE_STATIC_SRCU(name) \ - static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name) + static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name) void synchronize_srcu(struct srcu_struct *ssp); diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index 3ce6deee1dbe..276f325f1296 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -57,11 +57,17 @@ struct srcu_node { int grphi; /* Biggest CPU for node. */ }; +/* + * Per-SRCU-domain structure, update-side data linked from srcu_struct. + */ +struct srcu_usage { + struct srcu_node *node; /* Combining tree. */ +}; + /* * Per-SRCU-domain structure, similar in function to rcu_state. */ struct srcu_struct { - struct srcu_node *node; /* Combining tree. */ struct srcu_node *level[RCU_NUM_LVLS + 1]; /* First node at each level. */ int srcu_size_state; /* Small-to-big transition state. */ @@ -90,6 +96,7 @@ struct srcu_struct { unsigned long reschedule_count; struct delayed_work work; struct lockdep_map dep_map; + struct srcu_usage *srcu_sup; /* Update-side data. */ }; /* Values for size state variable (->srcu_size_state). */ @@ -108,24 +115,24 @@ struct srcu_struct { #define SRCU_STATE_SCAN1 1 #define SRCU_STATE_SCAN2 2 -#define __SRCU_STRUCT_INIT_COMMON(name) \ +#define __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ .srcu_gp_seq_needed = -1UL, \ .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \ + .srcu_sup = &usage_name, \ __SRCU_DEP_MAP_INIT(name) -#define __SRCU_STRUCT_INIT_MODULE(name) \ +#define __SRCU_STRUCT_INIT_MODULE(name, usage_name) \ { \ - __SRCU_STRUCT_INIT_COMMON(name) \ + __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ } -#define __SRCU_STRUCT_INIT(name, pcpu_name) \ +#define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name) \ { \ .sda = &pcpu_name, \ - __SRCU_STRUCT_INIT_COMMON(name) \ + __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ } - /* * Define and initialize a srcu struct at build time. * Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it. @@ -147,15 +154,17 @@ struct srcu_struct { */ #ifdef MODULE # define __DEFINE_SRCU(name, is_static) \ - is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name); \ + static struct srcu_usage name##_srcu_usage; \ + is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage); \ extern struct srcu_struct * const __srcu_struct_##name; \ struct srcu_struct * const __srcu_struct_##name \ __section("___srcu_struct_ptrs") = &name #else # define __DEFINE_SRCU(name, is_static) \ static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data); \ + static struct srcu_usage name##_srcu_usage; \ is_static struct srcu_struct name = \ - __SRCU_STRUCT_INIT(name, name##_srcu_data) + __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data) #endif #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */) #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 115616ac3bfa..8d18d4bf0e29 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -341,11 +341,13 @@ extern void rcu_init_geometry(void); * specified state structure (for SRCU) or the only rcu_state structure * (for RCU). */ -#define srcu_for_each_node_breadth_first(sp, rnp) \ +#define _rcu_for_each_node_breadth_first(sp, rnp) \ for ((rnp) = &(sp)->node[0]; \ (rnp) < &(sp)->node[rcu_num_nodes]; (rnp)++) #define rcu_for_each_node_breadth_first(rnp) \ - srcu_for_each_node_breadth_first(&rcu_state, rnp) + _rcu_for_each_node_breadth_first(&rcu_state, rnp) +#define srcu_for_each_node_breadth_first(ssp, rnp) \ + _rcu_for_each_node_breadth_first(ssp->srcu_sup, rnp) /* * Scan the leaves of the rcu_node hierarchy for the rcu_state structure. diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 7e6e7dfb1a87..049e20dbec76 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -173,12 +173,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) /* Initialize geometry if it has not already been initialized. */ rcu_init_geometry(); - ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), gfp_flags); - if (!ssp->node) + ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags); + if (!ssp->srcu_sup->node) return false; /* Work out the overall tree geometry. */ - ssp->level[0] = &ssp->node[0]; + ssp->level[0] = &ssp->srcu_sup->node[0]; for (i = 1; i < rcu_num_lvls; i++) ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1]; rcu_init_levelspread(levelspread, num_rcu_lvl); @@ -195,7 +195,7 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ; snp->grplo = -1; snp->grphi = -1; - if (snp == &ssp->node[0]) { + if (snp == &ssp->srcu_sup->node[0]) { /* Root node, special case. */ snp->srcu_parent = NULL; continue; @@ -236,8 +236,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) */ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) { + if (!is_static) + ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL); + if (!ssp->srcu_sup) + return -ENOMEM; ssp->srcu_size_state = SRCU_SIZE_SMALL; - ssp->node = NULL; + ssp->srcu_sup->node = NULL; mutex_init(&ssp->srcu_cb_mutex); mutex_init(&ssp->srcu_gp_mutex); ssp->srcu_idx = 0; @@ -249,8 +253,11 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) ssp->sda_is_static = is_static; if (!is_static) ssp->sda = alloc_percpu(struct srcu_data); - if (!ssp->sda) + if (!ssp->sda) { + if (!is_static) + kfree(ssp->srcu_sup); return -ENOMEM; + } init_srcu_struct_data(ssp); ssp->srcu_gp_seq_needed_exp = 0; ssp->srcu_last_gp_end = ktime_get_mono_fast_ns(); @@ -259,6 +266,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) if (!ssp->sda_is_static) { free_percpu(ssp->sda); ssp->sda = NULL; + kfree(ssp->srcu_sup); return -ENOMEM; } } else { @@ -656,13 +664,15 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) rcu_seq_current(&ssp->srcu_gp_seq), ssp->srcu_gp_seq_needed); return; /* Caller forgot to stop doing call_srcu()? */ } + kfree(ssp->srcu_sup->node); + ssp->srcu_sup->node = NULL; + ssp->srcu_size_state = SRCU_SIZE_SMALL; if (!ssp->sda_is_static) { free_percpu(ssp->sda); ssp->sda = NULL; + kfree(ssp->srcu_sup); + ssp->srcu_sup = NULL; } - kfree(ssp->node); - ssp->node = NULL; - ssp->srcu_size_state = SRCU_SIZE_SMALL; } EXPORT_SYMBOL_GPL(cleanup_srcu_struct);