@@ -114,13 +114,14 @@ static inline void rcu_user_exit(void) { }
#ifdef CONFIG_RCU_NOCB_CPU
void rcu_init_nohz(void);
-int rcu_nocb_cpu_offload(int cpu);
-int rcu_nocb_cpu_deoffload(int cpu);
+int rcu_nocb_cpumask_update(struct cpumask *cpumask, bool offload);
void rcu_nocb_flush_deferred_wakeup(void);
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
static inline void rcu_init_nohz(void) { }
-static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
-static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
+static inline int rcu_nocb_cpumask_update(struct cpumask *cpumask, bool offload)
+{
+ return -EINVAL;
+}
static inline void rcu_nocb_flush_deferred_wakeup(void) { }
#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
@@ -1887,10 +1887,10 @@ static int rcu_nocb_toggle(void *arg)
r = torture_random(&rand);
cpu = (r >> 4) % (maxcpu + 1);
if (r & 0x1) {
- rcu_nocb_cpu_offload(cpu);
+ rcu_nocb_cpumask_update(cpumask_of(cpu), true);
atomic_long_inc(&n_nocb_offload);
} else {
- rcu_nocb_cpu_deoffload(cpu);
+ rcu_nocb_cpumask_update(cpumask_of(cpu), false);
atomic_long_inc(&n_nocb_deoffload);
}
toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval;
@@ -1084,29 +1084,23 @@ static long rcu_nocb_rdp_deoffload(void *arg)
return 0;
}
-int rcu_nocb_cpu_deoffload(int cpu)
+static int rcu_nocb_cpu_deoffload(int cpu)
{
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
int ret = 0;
- cpus_read_lock();
- mutex_lock(&rcu_state.barrier_mutex);
- if (rcu_rdp_is_offloaded(rdp)) {
- if (cpu_online(cpu)) {
- ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);
- if (!ret)
- cpumask_clear_cpu(cpu, rcu_nocb_mask);
- } else {
- pr_info("NOCB: Can't CB-deoffload an offline CPU\n");
- ret = -EINVAL;
- }
- }
- mutex_unlock(&rcu_state.barrier_mutex);
- cpus_read_unlock();
+ if (cpu_is_offline(cpu))
+ return -EINVAL;
+
+ if (!rcu_rdp_is_offloaded(rdp))
+ return 0;
+
+ ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);
+ if (!ret)
+ cpumask_clear_cpu(cpu, rcu_nocb_mask);
return ret;
}
-EXPORT_SYMBOL_GPL(rcu_nocb_cpu_deoffload);
static long rcu_nocb_rdp_offload(void *arg)
{
@@ -1117,12 +1111,6 @@ static long rcu_nocb_rdp_offload(void *arg)
struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id());
- /*
- * For now we only support re-offload, ie: the rdp must have been
- * offloaded on boot first.
- */
- if (!rdp->nocb_gp_rdp)
- return -EINVAL;
if (WARN_ON_ONCE(!rdp_gp->nocb_gp_kthread))
return -EINVAL;
@@ -1169,29 +1157,77 @@ static long rcu_nocb_rdp_offload(void *arg)
return 0;
}
-int rcu_nocb_cpu_offload(int cpu)
+static int rcu_nocb_cpu_offload(int cpu)
{
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
- int ret = 0;
+ int ret;
+
+ if (cpu_is_offline(cpu))
+ return -EINVAL;
+
+ if (rcu_rdp_is_offloaded(rdp))
+ return 0;
+
+ ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);
+ if (!ret)
+ cpumask_set_cpu(cpu, rcu_nocb_mask);
+
+ return ret;
+}
+
+int rcu_nocb_cpumask_update(struct cpumask *cpumask, bool offload)
+{
+ int cpu;
+ int err = 0;
+ int err_cpu;
+ cpumask_var_t saved_nocb_mask;
+
+ if (!alloc_cpumask_var(&saved_nocb_mask, GFP_KERNEL))
+ return -ENOMEM;
+
+ cpumask_copy(saved_nocb_mask, rcu_nocb_mask);
cpus_read_lock();
mutex_lock(&rcu_state.barrier_mutex);
- if (!rcu_rdp_is_offloaded(rdp)) {
- if (cpu_online(cpu)) {
- ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);
- if (!ret)
- cpumask_set_cpu(cpu, rcu_nocb_mask);
+ for_each_cpu(cpu, cpumask) {
+ if (offload) {
+ err = rcu_nocb_cpu_offload(cpu);
+ if (err < 0) {
+ err_cpu = cpu;
+ pr_err("NOCB: offload cpu %d failed (%d)\n", cpu, err);
+ break;
+ }
} else {
- pr_info("NOCB: Can't CB-offload an offline CPU\n");
- ret = -EINVAL;
+ err = rcu_nocb_cpu_deoffload(cpu);
+ if (err < 0) {
+ err_cpu = cpu;
+ pr_err("NOCB: deoffload cpu %d failed (%d)\n", cpu, err);
+ break;
+ }
}
}
+
+ /* Rollback in case of error */
+ if (err < 0) {
+ err_cpu = cpu;
+ for_each_cpu(cpu, cpumask) {
+ if (err_cpu == cpu)
+ break;
+ if (cpumask_test_cpu(cpu, saved_nocb_mask))
+ WARN_ON_ONCE(rcu_nocb_cpu_offload(cpu));
+ else
+ WARN_ON_ONCE(rcu_nocb_cpu_deoffload(cpu));
+ }
+ }
+
mutex_unlock(&rcu_state.barrier_mutex);
cpus_read_unlock();
- return ret;
+ free_cpumask_var(saved_nocb_mask);
+
+ return err;
}
-EXPORT_SYMBOL_GPL(rcu_nocb_cpu_offload);
+EXPORT_SYMBOL_GPL(rcu_nocb_cpumask_update);
void __init rcu_init_nohz(void)
{
Currently the interface to toggle callbacks offloading state only takes a single CPU per call. Now driving RCU NOCB through cpusets requires to be able to change the offloading state of a whole set of CPUs. To make it easier, extend the (de-)offloading interface to support a cpumask. Signed-off-by: Frederic Weisbecker <frederic@kernel.org> Cc: Zefan Li <lizefan.x@bytedance.com> Cc: Tejun Heo <tj@kernel.org> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Paul E. McKenney <paulmck@kernel.org> Cc: Phil Auld <pauld@redhat.com> Cc: Nicolas Saenz Julienne <nsaenz@kernel.org> Cc: Marcelo Tosatti <mtosatti@redhat.com> Cc: Paul Gortmaker <paul.gortmaker@windriver.com> Cc: Waiman Long <longman@redhat.com> Cc: Daniel Bristot de Oliveira <bristot@kernel.org> Cc: Peter Zijlstra <peterz@infradead.org> --- include/linux/rcupdate.h | 9 ++-- kernel/rcu/rcutorture.c | 4 +- kernel/rcu/tree_nocb.h | 102 ++++++++++++++++++++++++++------------- 3 files changed, 76 insertions(+), 39 deletions(-)