| Message ID | 20220620224503.3841196-7-paulmck@kernel.org (mailing list archive) |
|---|---|
| State | Accepted |
| Commit | 0578e14c945b1739e15c0e993280151fa5b99ca2 |
| Headers | show |
| Series | Callback-offload (nocb) updates for v5.20 | expand |
On 6/21/2022 4:15 AM, Paul E. McKenney wrote: > From: Zqiang <qiang1.zhang@intel.com> > > Currently, if the 'rcu_nocb_poll' kernel boot parameter is enabled, all > rcuog kthreads enter polling mode. However, if all of a given group > of rcuo kthreads correspond to CPUs that have been de-offloaded, the > corresponding rcuog kthread will nonetheless still wake up periodically, > unnecessarily consuming power and perturbing workloads. Fortunately, > this situation is easily detected by the fact that the rcuog kthread's > CPU's rcu_data structure's ->nocb_head_rdp list is empty. > > This commit saves power and avoids unnecessarily perturbing workloads > by putting an rcuog kthread to sleep during any time period when all of > its rcuo kthreads' CPUs are de-offloaded. > > Co-developed-by: Frederic Weisbecker <frederic@kernel.org> > Signed-off-by: Frederic Weisbecker <frederic@kernel.org> > Signed-off-by: Zqiang <qiang1.zhang@intel.com> > Signed-off-by: Paul E. McKenney <paulmck@kernel.org> > --- Reviewed-by: Neeraj Upadhyay <quic_neeraju@quicinc.com> Thanks Neeraj > kernel/rcu/tree_nocb.h | 24 +++++++++++++++++++----- > 1 file changed, 19 insertions(+), 5 deletions(-) > > diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h > index fa8e4f82e60c0..a8f574d8850d2 100644 > --- a/kernel/rcu/tree_nocb.h > +++ b/kernel/rcu/tree_nocb.h > @@ -584,6 +584,14 @@ static int nocb_gp_toggle_rdp(struct rcu_data *rdp, > return ret; > } > > +static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) > +{ > + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); > + swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, > + !READ_ONCE(my_rdp->nocb_gp_sleep)); > + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); > +} > + > /* > * No-CBs GP kthreads come here to wait for additional callbacks to show up > * or for grace periods to end. > @@ -701,13 +709,19 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) > /* Polling, so trace if first poll in the series. */ > if (gotcbs) > trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); > - schedule_timeout_idle(1); > + if (list_empty(&my_rdp->nocb_head_rdp)) { > + raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); > + if (!my_rdp->nocb_toggling_rdp) > + WRITE_ONCE(my_rdp->nocb_gp_sleep, true); > + raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); > + /* Wait for any offloading rdp */ > + nocb_gp_sleep(my_rdp, cpu); > + } else { > + schedule_timeout_idle(1); > + } > } else if (!needwait_gp) { > /* Wait for callbacks to appear. */ > - trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); > - swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, > - !READ_ONCE(my_rdp->nocb_gp_sleep)); > - trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); > + nocb_gp_sleep(my_rdp, cpu); > } else { > rnp = my_rdp->mynode; > trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait"));
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index fa8e4f82e60c0..a8f574d8850d2 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -584,6 +584,14 @@ static int nocb_gp_toggle_rdp(struct rcu_data *rdp, return ret; } +static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) +{ + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); + swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, + !READ_ONCE(my_rdp->nocb_gp_sleep)); + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); +} + /* * No-CBs GP kthreads come here to wait for additional callbacks to show up * or for grace periods to end. @@ -701,13 +709,19 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) /* Polling, so trace if first poll in the series. */ if (gotcbs) trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); - schedule_timeout_idle(1); + if (list_empty(&my_rdp->nocb_head_rdp)) { + raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); + if (!my_rdp->nocb_toggling_rdp) + WRITE_ONCE(my_rdp->nocb_gp_sleep, true); + raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); + /* Wait for any offloading rdp */ + nocb_gp_sleep(my_rdp, cpu); + } else { + schedule_timeout_idle(1); + } } else if (!needwait_gp) { /* Wait for callbacks to appear. */ - trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); - swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, - !READ_ONCE(my_rdp->nocb_gp_sleep)); - trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); + nocb_gp_sleep(my_rdp, cpu); } else { rnp = my_rdp->mynode; trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait"));