@@ -419,6 +419,7 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp)
// if there has not been an increase in callbacks, limit dequeuing
// to CPU 0. Note the matching RCU read-side critical section in
// call_rcu_tasks_generic().
+ rcu_read_lock();
if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {
raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
if (rtp->percpu_enqueue_lim > 1) {
@@ -443,6 +444,7 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp)
}
raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
}
+ rcu_read_unlock();
return needgpcb;
}
Assume that the current RCU-task belongs to per-CPU callback queuing mode. the 'rcu_task_cb_adjust' variable is true and the conditions for converting to a single CPU-0 queue mode have been met. (ncbsnz == 0 && ncbs < rcu_task_collapse_lim) CPU0 CPU1 rcu_tasks_one_gp() rcu_tasks_need_gpcb() invoke call_rcu_tasks_generic() enqueue callback to CPU1 (CPU1 n_cbs not equal zero) if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) if (rtp->percpu_enqueue_lim > 1) rtp->percpu_enqueue_lim = 1; rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu(); A full RCU grace period has passed (it means that poll_state_synchronize_rcu rtp->percpu_dequeue_gpseq) maybe return true) if (rcu_task_cb_adjust && !ncbsnz && poll_state_synchronize_rcu( rtp->percpu_dequeue_gpseq) { if (rtp->percpu_enqueue_lim < rtp->percpu_dequeue_lim) rtp->percpu_dequeue_lim = 1 for (cpu = rtp->percpu_dequeue_lim; cpu < nr_cpu_ids; cpu++) find CPU1 n_cbs is not zero trigger warning } The above scenario will not only trigger WARN_ONCE(), but also set the rcu_tasks structure's->percpu_dequeue_lim is one when CPU1 still have callbacks, which will cause the callback of CPU1 to have no chance to be called. This commit put get_state_synchronize_rcu() and poll_state_synchronize_rcu() into a RCU read critical section. it means that current readers will block completion of the current or next RCU grace period. this ensures that after we snapshot current RCU gp number and then polling it will return false. This will lead us to judge per-CPU callback numbers again after a grace period of RCU tasks, until the callbacks of other CPUs(except CPU0) are executed and the specified RCU grace period has completed, we have just completed the conversion of single queue. Signed-off-by: Zqiang <qiang1.zhang@intel.com> --- kernel/rcu/tasks.h | 2 ++ 1 file changed, 2 insertions(+)