Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 /* SPDX-License-Identifier: GPL-2.0+ */
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 * Internal non-public definitions that provide either classic
23 if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread) in rcu_current_is_nocb_kthread()
26 if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread) in rcu_current_is_nocb_kthread()
33 * Offload callback processing from the boot-time-specified set of CPUs
35 * created that pull the callbacks from the corresponding CPU, wait for
39 * invoke callbacks. Each GP kthread invokes its own CBs. The no-CBs CPUs
42 * in which case each kthread actively polls its CPU. (Which isn't so great
43 * for energy efficiency, but which does reduce RCU's overhead on that CPU.)
46 * adaptive-idle work, which would seriously reduce OS jitter on CPUs
47 * running CPU-bound user-mode computations.
49 * Offloading of callbacks can also be used as an energy-efficiency
51 * about entering dyntick-idle mode.
56 * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters.
64 pr_warn("rcu_nocbs= bad CPU range, all CPUs set\n"); in rcu_nocb_setup()
81 * Don't bother bypassing ->cblist if the call_rcu() rate is low.
83 * on ->nocb_lock, which only can happen at high call_rcu() rates.
89 * Acquire the specified rcu_data structure's ->nocb_bypass_lock. If the
93 __acquires(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_lock()
96 if (raw_spin_trylock(&rdp->nocb_bypass_lock)) in rcu_nocb_bypass_lock()
102 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
103 raw_spin_lock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_lock()
108 * ->nocb_bypass_lock.
113 return raw_spin_trylock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_trylock()
117 * Release the specified rcu_data structure's ->nocb_bypass_lock.
120 __releases(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_unlock()
123 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_unlock()
127 * Acquire the specified rcu_data structure's ->nocb_lock, but only
128 * if it corresponds to a no-CBs CPU.
135 raw_spin_lock(&rdp->nocb_lock); in rcu_nocb_lock()
139 * Release the specified rcu_data structure's ->nocb_lock, but only
140 * if it corresponds to a no-CBs CPU.
146 raw_spin_unlock(&rdp->nocb_lock); in rcu_nocb_unlock()
151 * Release the specified rcu_data structure's ->nocb_lock and restore
152 * interrupts, but only if it corresponds to a no-CBs CPU.
159 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_unlock_irqrestore()
165 /* Lockdep check that ->cblist may be safely accessed. */
170 lockdep_assert_held(&rdp->nocb_lock); in rcu_lockdep_assert_cblist_protected()
174 * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
184 return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1]; in rcu_nocb_gp_get()
189 init_swait_queue_head(&rnp->nocb_gp_wq[0]); in rcu_init_one_nocb()
190 init_swait_queue_head(&rnp->nocb_gp_wq[1]); in rcu_init_one_nocb()
196 __releases(rdp_gp->nocb_gp_lock) in __wake_nocb_gp()
200 if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) { in __wake_nocb_gp()
201 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags); in __wake_nocb_gp()
202 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __wake_nocb_gp()
207 if (rdp_gp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) { in __wake_nocb_gp()
208 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); in __wake_nocb_gp()
209 del_timer(&rdp_gp->nocb_timer); in __wake_nocb_gp()
212 if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) { in __wake_nocb_gp()
213 WRITE_ONCE(rdp_gp->nocb_gp_sleep, false); in __wake_nocb_gp()
216 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags); in __wake_nocb_gp()
218 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in __wake_nocb_gp()
219 swake_up_one_online(&rdp_gp->nocb_gp_wq); in __wake_nocb_gp()
231 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp()
233 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); in wake_nocb_gp()
263 * Arrange to wake the GP kthread for this NOCB group at some future
270 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp_defer()
272 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); in wake_nocb_gp_defer()
276 * callback storms, no need to wake up too early. in wake_nocb_gp_defer()
279 rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) { in wake_nocb_gp_defer()
280 mod_timer(&rdp_gp->nocb_timer, jiffies + rcu_get_jiffies_lazy_flush()); in wake_nocb_gp_defer()
281 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype); in wake_nocb_gp_defer()
283 mod_timer(&rdp_gp->nocb_timer, jiffies + 2); in wake_nocb_gp_defer()
284 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype); in wake_nocb_gp_defer()
286 if (rdp_gp->nocb_defer_wakeup < RCU_NOCB_WAKE) in wake_nocb_gp_defer()
287 mod_timer(&rdp_gp->nocb_timer, jiffies + 1); in wake_nocb_gp_defer()
288 if (rdp_gp->nocb_defer_wakeup < waketype) in wake_nocb_gp_defer()
289 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype); in wake_nocb_gp_defer()
292 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags); in wake_nocb_gp_defer()
294 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
298 * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
299 * However, if there is a callback to be enqueued and if ->nocb_bypass
300 * proves to be initially empty, just return false because the no-CB GP
316 lockdep_assert_held(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
317 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { in rcu_nocb_do_flush_bypass()
318 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
321 /* Note: ->cblist.len already accounts for ->nocb_bypass contents. */ in rcu_nocb_do_flush_bypass()
323 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_do_flush_bypass()
327 * ->cblist so that we can take advantage of the grace-period that will in rcu_nocb_do_flush_bypass()
332 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
335 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
336 WRITE_ONCE(rdp->lazy_len, 0); in rcu_nocb_do_flush_bypass()
338 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); in rcu_nocb_do_flush_bypass()
339 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_do_flush_bypass()
345 * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
346 * However, if there is a callback to be enqueued and if ->nocb_bypass
347 * proves to be initially empty, just return false because the no-CB GP
363 * If the ->nocb_bypass_lock is immediately available, flush the
364 * ->nocb_bypass queue into ->cblist.
376 * See whether it is appropriate to use the ->nocb_bypass list in order
377 * to control contention on ->nocb_lock. A limited number of direct
378 * enqueues are permitted into ->cblist per jiffy. If ->nocb_bypass
379 * is non-empty, further callbacks must be placed into ->nocb_bypass,
381 * back to direct use of ->cblist. However, ->nocb_bypass should not be
382 * used if ->cblist is empty, because otherwise callbacks can be stranded
383 * on ->nocb_bypass because we cannot count on the current CPU ever again
384 * invoking call_rcu(). The general rule is that if ->nocb_bypass is
385 * non-empty, the corresponding no-CBs grace-period kthread must not be
389 * as doing so would confuse the auto-initialization code. Besides
391 * there is only one CPU in operation.
400 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
401 bool bypass_is_lazy = (ncbs == READ_ONCE(rdp->lazy_len)); in rcu_nocb_try_bypass()
408 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
412 // Don't use ->nocb_bypass during early boot. in rcu_nocb_try_bypass()
415 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
416 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
421 // moving back from ->nocb_bypass to ->cblist. in rcu_nocb_try_bypass()
422 if (j == rdp->nocb_nobypass_last) { in rcu_nocb_try_bypass()
423 c = rdp->nocb_nobypass_count + 1; in rcu_nocb_try_bypass()
425 WRITE_ONCE(rdp->nocb_nobypass_last, j); in rcu_nocb_try_bypass()
426 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy; in rcu_nocb_try_bypass()
427 if (ULONG_CMP_LT(rdp->nocb_nobypass_count, in rcu_nocb_try_bypass()
433 WRITE_ONCE(rdp->nocb_nobypass_count, c); in rcu_nocb_try_bypass()
435 // If there hasn't yet been all that many ->cblist enqueues in rcu_nocb_try_bypass()
436 // this jiffy, tell the caller to enqueue onto ->cblist. But flush in rcu_nocb_try_bypass()
437 // ->nocb_bypass first. in rcu_nocb_try_bypass()
439 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) { in rcu_nocb_try_bypass()
441 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
443 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
447 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
451 // If ->nocb_bypass has been used too long or is too full, in rcu_nocb_try_bypass()
452 // flush ->nocb_bypass to ->cblist. in rcu_nocb_try_bypass()
453 if ((ncbs && !bypass_is_lazy && j != READ_ONCE(rdp->nocb_bypass_first)) || in rcu_nocb_try_bypass()
455 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + rcu_get_jiffies_lazy_flush()))) || in rcu_nocb_try_bypass()
458 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
462 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
464 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
467 if (j != rdp->nocb_gp_adv_time && in rcu_nocb_try_bypass()
468 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in rcu_nocb_try_bypass()
469 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in rcu_nocb_try_bypass()
470 rcu_advance_cbs_nowake(rdp->mynode, rdp); in rcu_nocb_try_bypass()
471 rdp->nocb_gp_adv_time = j; in rcu_nocb_try_bypass()
475 // Don't wait for the wake up timer as it may be too far ahead. in rcu_nocb_try_bypass()
476 // Wake up the GP thread now instead, if the cblist was empty. in rcu_nocb_try_bypass()
484 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
485 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_try_bypass()
486 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_try_bypass()
489 WRITE_ONCE(rdp->lazy_len, rdp->lazy_len + 1); in rcu_nocb_try_bypass()
492 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_try_bypass()
493 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
497 // A wake up of the grace period kthread or timer adjustment in rcu_nocb_try_bypass()
503 // b. The new CB is non-lazy. in rcu_nocb_try_bypass()
505 // No-CBs GP kthread might be indefinitely asleep, if so, wake. in rcu_nocb_try_bypass()
507 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_nocb_try_bypass()
508 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
512 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
521 * Awaken the no-CBs grace-period kthread if needed, either due to it
524 * If warranted, also wake up the kthread servicing this CPUs queues.
528 __releases(rdp->nocb_lock) in __call_rcu_nocb_wake()
536 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in __call_rcu_nocb_wake()
539 t = READ_ONCE(rdp->nocb_gp_kthread); in __call_rcu_nocb_wake()
542 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
547 len = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_nocb_wake()
548 bypass_len = rcu_cblist_n_cbs(&rdp->nocb_bypass); in __call_rcu_nocb_wake()
549 lazy_len = READ_ONCE(rdp->lazy_len); in __call_rcu_nocb_wake()
551 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
557 } else if (!irqs_disabled_flags(flags) && cpu_online(rdp->cpu)) { in __call_rcu_nocb_wake()
561 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
565 * Don't do the wake-up upfront on fragile paths. in __call_rcu_nocb_wake()
567 * (soft-)IRQs. Rely on the final deferred wake-up from in __call_rcu_nocb_wake()
574 } else if (len > rdp->qlen_last_fqs_check + qhimark) { in __call_rcu_nocb_wake()
576 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
578 if (j != rdp->nocb_gp_adv_time && in __call_rcu_nocb_wake()
579 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in __call_rcu_nocb_wake()
580 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in __call_rcu_nocb_wake()
581 rcu_advance_cbs_nowake(rdp->mynode, rdp); in __call_rcu_nocb_wake()
582 rdp->nocb_gp_adv_time = j; in __call_rcu_nocb_wake()
585 if ((rdp->nocb_cb_sleep || in __call_rcu_nocb_wake()
586 !rcu_segcblist_ready_cbs(&rdp->cblist)) && in __call_rcu_nocb_wake()
587 !timer_pending(&rdp_gp->nocb_timer)) { in __call_rcu_nocb_wake()
593 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
597 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
615 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_gp_toggle_rdp()
619 * Locking orders future de-offloaded callbacks enqueue against previous in nocb_gp_toggle_rdp()
623 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in nocb_gp_toggle_rdp()
627 * We will handle this rdp until it ever gets de-offloaded. in nocb_gp_toggle_rdp()
629 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp); in nocb_gp_toggle_rdp()
633 * De-offloading. Clear our flag and notify the de-offload worker. in nocb_gp_toggle_rdp()
634 * We will ignore this rdp until it ever gets re-offloaded. in nocb_gp_toggle_rdp()
636 list_del(&rdp->nocb_entry_rdp); in nocb_gp_toggle_rdp()
639 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in nocb_gp_toggle_rdp()
642 static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) in nocb_gp_sleep() argument
644 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); in nocb_gp_sleep()
645 swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, in nocb_gp_sleep()
646 !READ_ONCE(my_rdp->nocb_gp_sleep)); in nocb_gp_sleep()
647 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); in nocb_gp_sleep()
651 * No-CBs GP kthreads come here to wait for additional callbacks to show up
657 int __maybe_unused cpu = my_rdp->cpu; in nocb_gp_wait() local
674 * and the global grace-period kthread are awakened if needed. in nocb_gp_wait()
676 WARN_ON_ONCE(my_rdp->nocb_gp_rdp != my_rdp); in nocb_gp_wait()
679 * CPU is de-offloaded and added to the list before that CPU is in nocb_gp_wait()
680 * (re-)offloaded. If the following loop happens to be referencing in nocb_gp_wait()
682 * CPU is de-offloaded and then immediately re-offloaded, this in nocb_gp_wait()
687 * entire loop is forced after a given CPU's rcu_data structure in nocb_gp_wait()
688 * is added to the list, so the skipped-over rcu_data structures in nocb_gp_wait()
691 list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { in nocb_gp_wait()
696 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
698 lockdep_assert_held(&rdp->nocb_lock); in nocb_gp_wait()
699 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
700 lazy_ncbs = READ_ONCE(rdp->lazy_len); in nocb_gp_wait()
703 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + rcu_get_jiffies_lazy_flush()) || in nocb_gp_wait()
707 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) || in nocb_gp_wait()
710 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { in nocb_gp_wait()
718 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
719 lazy_ncbs = READ_ONCE(rdp->lazy_len); in nocb_gp_wait()
723 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
730 rnp = rdp->mynode; in nocb_gp_wait()
734 if (!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
736 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_gp_wait()
737 rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) { in nocb_gp_wait()
740 wasempty = rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
746 !rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
748 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { in nocb_gp_wait()
753 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
756 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_gp_wait()
757 needwake = rdp->nocb_cb_sleep; in nocb_gp_wait()
758 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_gp_wait()
764 swake_up_one(&rdp->nocb_cb_wq); in nocb_gp_wait()
771 my_rdp->nocb_gp_bypass = bypass; in nocb_gp_wait()
772 my_rdp->nocb_gp_gp = needwait_gp; in nocb_gp_wait()
773 my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq : 0; in nocb_gp_wait()
775 // At least one child with non-empty ->nocb_bypass, so set in nocb_gp_wait()
776 // timer in order to avoid stranding its callbacks. in nocb_gp_wait()
778 // If bypass list only has lazy CBs. Add a deferred lazy wake up. in nocb_gp_wait()
782 // Otherwise add a deferred bypass wake up. in nocb_gp_wait()
792 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); in nocb_gp_wait()
793 if (list_empty(&my_rdp->nocb_head_rdp)) { in nocb_gp_wait()
794 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); in nocb_gp_wait()
795 if (!my_rdp->nocb_toggling_rdp) in nocb_gp_wait()
796 WRITE_ONCE(my_rdp->nocb_gp_sleep, true); in nocb_gp_wait()
797 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); in nocb_gp_wait()
799 nocb_gp_sleep(my_rdp, cpu); in nocb_gp_wait()
805 nocb_gp_sleep(my_rdp, cpu); in nocb_gp_wait()
807 rnp = my_rdp->mynode; in nocb_gp_wait()
810 rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1], in nocb_gp_wait()
811 rcu_seq_done(&rnp->gp_seq, wait_gp_seq) || in nocb_gp_wait()
812 !READ_ONCE(my_rdp->nocb_gp_sleep)); in nocb_gp_wait()
817 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); in nocb_gp_wait()
818 // (De-)queue an rdp to/from the group if its nocb state is changing in nocb_gp_wait()
819 rdp_toggling = my_rdp->nocb_toggling_rdp; in nocb_gp_wait()
821 my_rdp->nocb_toggling_rdp = NULL; in nocb_gp_wait()
823 if (my_rdp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) { in nocb_gp_wait()
824 WRITE_ONCE(my_rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); in nocb_gp_wait()
825 del_timer(&my_rdp->nocb_timer); in nocb_gp_wait()
827 WRITE_ONCE(my_rdp->nocb_gp_sleep, true); in nocb_gp_wait()
828 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); in nocb_gp_wait()
830 rdp_toggling = READ_ONCE(my_rdp->nocb_toggling_rdp); in nocb_gp_wait()
839 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); in nocb_gp_wait()
840 my_rdp->nocb_toggling_rdp = NULL; in nocb_gp_wait()
841 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); in nocb_gp_wait()
847 swake_up_one(&rdp_toggling->nocb_state_wq); in nocb_gp_wait()
850 my_rdp->nocb_gp_seq = -1; in nocb_gp_wait()
855 * No-CBs grace-period-wait kthread. There is one of these per group
856 * of CPUs, but only once at least one CPU in that group has come online
860 * that then have callback-invocation work to do.
867 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1); in rcu_nocb_gp_kthread()
876 return !READ_ONCE(rdp->nocb_cb_sleep) || kthread_should_park(); in nocb_cb_wait_cond()
880 * Invoke any ready callbacks from the corresponding no-CBs CPU,
885 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_cb_wait()
889 struct rcu_node *rnp = rdp->mynode; in nocb_cb_wait()
891 swait_event_interruptible_exclusive(rdp->nocb_cb_wq, in nocb_cb_wait()
895 } else if (READ_ONCE(rdp->nocb_cb_sleep)) { in nocb_cb_wait()
897 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
907 * transitioning to/from NOCB mode, a self-requeuing callback might in nocb_cb_wait()
917 rcu_seq_done(&rnp->gp_seq, cur_gp_seq) && in nocb_cb_wait()
919 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); in nocb_cb_wait()
924 WRITE_ONCE(rdp->nocb_cb_sleep, true); in nocb_cb_wait()
925 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
927 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_cb_wait()
936 * Per-rcu_data kthread, but only for no-CBs CPUs. Repeatedly invoke
955 return READ_ONCE(rdp->nocb_defer_wakeup) >= level; in rcu_nocb_need_deferred_wakeup()
962 __releases(rdp_gp->nocb_gp_lock) in do_nocb_deferred_wakeup_common()
968 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags); in do_nocb_deferred_wakeup_common()
972 ndw = rdp_gp->nocb_defer_wakeup; in do_nocb_deferred_wakeup_common()
974 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
979 /* Do a deferred wakeup of rcu_nocb_kthread() from a timer handler. */
985 WARN_ON_ONCE(rdp->nocb_gp_rdp != rdp); in do_nocb_deferred_wakeup_timer()
986 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_deferred_wakeup_timer()
988 raw_spin_lock_irqsave(&rdp->nocb_gp_lock, flags); in do_nocb_deferred_wakeup_timer()
989 smp_mb__after_spinlock(); /* Timer expire before wakeup. */ in do_nocb_deferred_wakeup_timer()
995 * This means we do an inexact common-case check. Note that if
996 * we miss, ->nocb_timer will eventually clean things up.
1001 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in do_nocb_deferred_wakeup()
1006 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); in do_nocb_deferred_wakeup()
1018 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_queue_toggle_rdp()
1022 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); in rcu_nocb_queue_toggle_rdp()
1024 WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp); in rcu_nocb_queue_toggle_rdp()
1025 if (rdp_gp->nocb_gp_sleep) { in rcu_nocb_queue_toggle_rdp()
1026 rdp_gp->nocb_gp_sleep = false; in rcu_nocb_queue_toggle_rdp()
1029 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags); in rcu_nocb_queue_toggle_rdp()
1042 * while the ->nocb_lock is held. in rcu_nocb_rdp_deoffload_wait_cond()
1044 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload_wait_cond()
1045 ret = !rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_deoffload_wait_cond()
1046 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload_wait_cond()
1055 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_rdp_deoffload()
1057 /* CPU must be offline, unless it's early boot */ in rcu_nocb_rdp_deoffload()
1058 WARN_ON_ONCE(cpu_online(rdp->cpu) && rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_deoffload()
1060 pr_info("De-offloading %d\n", rdp->cpu); in rcu_nocb_rdp_deoffload()
1069 if (rdp->nocb_cb_kthread) in rcu_nocb_rdp_deoffload()
1070 kthread_park(rdp->nocb_cb_kthread); in rcu_nocb_rdp_deoffload()
1073 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_rdp_deoffload()
1074 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_nocb_rdp_deoffload()
1079 mutex_lock(&rdp_gp->nocb_gp_kthread_mutex); in rcu_nocb_rdp_deoffload()
1081 if (rdp_gp->nocb_gp_kthread) { in rcu_nocb_rdp_deoffload()
1083 wake_up_process(rdp_gp->nocb_gp_kthread); in rcu_nocb_rdp_deoffload()
1085 swait_event_exclusive(rdp->nocb_state_wq, in rcu_nocb_rdp_deoffload()
1093 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload()
1094 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_deoffload()
1095 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload()
1097 list_del(&rdp->nocb_entry_rdp); in rcu_nocb_rdp_deoffload()
1100 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex); in rcu_nocb_rdp_deoffload()
1105 int rcu_nocb_cpu_deoffload(int cpu) in rcu_nocb_cpu_deoffload() argument
1107 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_deoffload()
1113 if (!cpu_online(cpu)) { in rcu_nocb_cpu_deoffload()
1116 cpumask_clear_cpu(cpu, rcu_nocb_mask); in rcu_nocb_cpu_deoffload()
1118 pr_info("NOCB: Cannot CB-deoffload online CPU %d\n", rdp->cpu); in rcu_nocb_cpu_deoffload()
1119 ret = -EINVAL; in rcu_nocb_cpu_deoffload()
1134 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_offload_wait_cond()
1135 ret = rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_offload_wait_cond()
1136 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_offload_wait_cond()
1144 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_rdp_offload()
1146 WARN_ON_ONCE(cpu_online(rdp->cpu)); in rcu_nocb_rdp_offload()
1148 * For now we only support re-offload, ie: the rdp must have been in rcu_nocb_rdp_offload()
1151 if (!rdp->nocb_gp_rdp) in rcu_nocb_rdp_offload()
1152 return -EINVAL; in rcu_nocb_rdp_offload()
1154 if (WARN_ON_ONCE(!rdp_gp->nocb_gp_kthread)) in rcu_nocb_rdp_offload()
1155 return -EINVAL; in rcu_nocb_rdp_offload()
1157 pr_info("Offloading %d\n", rdp->cpu); in rcu_nocb_rdp_offload()
1159 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_rdp_offload()
1160 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_nocb_rdp_offload()
1164 wake_up_process(rdp_gp->nocb_gp_kthread); in rcu_nocb_rdp_offload()
1166 swait_event_exclusive(rdp->nocb_state_wq, in rcu_nocb_rdp_offload()
1169 kthread_unpark(rdp->nocb_cb_kthread); in rcu_nocb_rdp_offload()
1174 int rcu_nocb_cpu_offload(int cpu) in rcu_nocb_cpu_offload() argument
1176 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_offload()
1182 if (!cpu_online(cpu)) { in rcu_nocb_cpu_offload()
1185 cpumask_set_cpu(cpu, rcu_nocb_mask); in rcu_nocb_cpu_offload()
1187 pr_info("NOCB: Cannot CB-offload online CPU %d\n", rdp->cpu); in rcu_nocb_cpu_offload()
1188 ret = -EINVAL; in rcu_nocb_cpu_offload()
1202 int cpu; in lazy_rcu_shrink_count() local
1208 /* Protect rcu_nocb_mask against concurrent (de-)offloading. */ in lazy_rcu_shrink_count()
1213 for_each_cpu(cpu, rcu_nocb_mask) { in lazy_rcu_shrink_count()
1214 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_count()
1216 count += READ_ONCE(rdp->lazy_len); in lazy_rcu_shrink_count()
1227 int cpu; in lazy_rcu_shrink_scan() local
1234 * Protect against concurrent (de-)offloading. Otherwise nocb locking in lazy_rcu_shrink_scan()
1248 for_each_cpu(cpu, rcu_nocb_mask) { in lazy_rcu_shrink_scan()
1249 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_scan()
1255 if (!READ_ONCE(rdp->lazy_len)) in lazy_rcu_shrink_scan()
1264 _count = READ_ONCE(rdp->lazy_len); in lazy_rcu_shrink_scan()
1272 sc->nr_to_scan -= _count; in lazy_rcu_shrink_scan()
1274 if (sc->nr_to_scan <= 0) in lazy_rcu_shrink_scan()
1286 int cpu; in rcu_init_nohz() local
1316 lazy_rcu_shrinker = shrinker_alloc(0, "rcu-lazy"); in rcu_init_nohz()
1320 lazy_rcu_shrinker->count_objects = lazy_rcu_shrink_count; in rcu_init_nohz()
1321 lazy_rcu_shrinker->scan_objects = lazy_rcu_shrink_scan; in rcu_init_nohz()
1338 pr_info("\tPoll for callbacks from no-CBs CPUs.\n"); in rcu_init_nohz()
1340 for_each_cpu(cpu, rcu_nocb_mask) { in rcu_init_nohz()
1341 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
1342 if (rcu_segcblist_empty(&rdp->cblist)) in rcu_init_nohz()
1343 rcu_segcblist_init(&rdp->cblist); in rcu_init_nohz()
1344 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_init_nohz()
1349 /* Initialize per-rcu_data variables for no-CBs CPUs. */
1352 init_swait_queue_head(&rdp->nocb_cb_wq); in rcu_boot_init_nocb_percpu_data()
1353 init_swait_queue_head(&rdp->nocb_gp_wq); in rcu_boot_init_nocb_percpu_data()
1354 init_swait_queue_head(&rdp->nocb_state_wq); in rcu_boot_init_nocb_percpu_data()
1355 raw_spin_lock_init(&rdp->nocb_lock); in rcu_boot_init_nocb_percpu_data()
1356 raw_spin_lock_init(&rdp->nocb_bypass_lock); in rcu_boot_init_nocb_percpu_data()
1357 raw_spin_lock_init(&rdp->nocb_gp_lock); in rcu_boot_init_nocb_percpu_data()
1358 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
1359 rcu_cblist_init(&rdp->nocb_bypass); in rcu_boot_init_nocb_percpu_data()
1360 WRITE_ONCE(rdp->lazy_len, 0); in rcu_boot_init_nocb_percpu_data()
1361 mutex_init(&rdp->nocb_gp_kthread_mutex); in rcu_boot_init_nocb_percpu_data()
1365 * If the specified CPU is a no-CBs CPU that does not already have its
1367 * for this CPU's group has not yet been created, spawn it as well.
1369 static void rcu_spawn_cpu_nocb_kthread(int cpu) in rcu_spawn_cpu_nocb_kthread() argument
1371 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_cpu_nocb_kthread()
1380 if (rdp->nocb_cb_kthread) in rcu_spawn_cpu_nocb_kthread()
1385 rdp_gp = rdp->nocb_gp_rdp; in rcu_spawn_cpu_nocb_kthread()
1386 mutex_lock(&rdp_gp->nocb_gp_kthread_mutex); in rcu_spawn_cpu_nocb_kthread()
1387 if (!rdp_gp->nocb_gp_kthread) { in rcu_spawn_cpu_nocb_kthread()
1389 "rcuog/%d", rdp_gp->cpu); in rcu_spawn_cpu_nocb_kthread()
1391 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex); in rcu_spawn_cpu_nocb_kthread()
1394 WRITE_ONCE(rdp_gp->nocb_gp_kthread, t); in rcu_spawn_cpu_nocb_kthread()
1398 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex); in rcu_spawn_cpu_nocb_kthread()
1400 /* Spawn the kthread for this CPU. */ in rcu_spawn_cpu_nocb_kthread()
1402 "rcuo%c/%d", rcu_state.abbr, cpu); in rcu_spawn_cpu_nocb_kthread()
1414 WRITE_ONCE(rdp->nocb_cb_kthread, t); in rcu_spawn_cpu_nocb_kthread()
1415 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); in rcu_spawn_cpu_nocb_kthread()
1421 * because the number of callbacks should be 0 for a non-boot CPU, in rcu_spawn_cpu_nocb_kthread()
1425 WARN_ON_ONCE(system_state > SYSTEM_BOOTING && rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_spawn_cpu_nocb_kthread()
1429 cpumask_clear_cpu(cpu, rcu_nocb_mask); in rcu_spawn_cpu_nocb_kthread()
1434 /* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */
1435 static int rcu_nocb_gp_stride = -1;
1439 * Initialize GP-CB relationships for all no-CBs CPU.
1443 int cpu; in rcu_organize_nocb_kthreads() local
1454 if (ls == -1) { in rcu_organize_nocb_kthreads()
1461 * Should the corresponding CPU come online in the future, then in rcu_organize_nocb_kthreads()
1464 for_each_possible_cpu(cpu) { in rcu_organize_nocb_kthreads()
1465 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
1466 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
1469 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
1471 INIT_LIST_HEAD(&rdp->nocb_head_rdp); in rcu_organize_nocb_kthreads()
1478 pr_alert("%s: No-CB GP kthread CPU %d:", in rcu_organize_nocb_kthreads()
1479 __func__, cpu); in rcu_organize_nocb_kthreads()
1485 pr_cont(" %d", cpu); in rcu_organize_nocb_kthreads()
1487 rdp->nocb_gp_rdp = rdp_gp; in rcu_organize_nocb_kthreads()
1488 if (cpumask_test_cpu(cpu, rcu_nocb_mask)) in rcu_organize_nocb_kthreads()
1489 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp); in rcu_organize_nocb_kthreads()
1502 WARN_ON(sched_setaffinity(current->pid, rcu_nocb_mask)); in rcu_bind_current_to_nocb()
1506 // The ->on_cpu field is available only in CONFIG_SMP=y, so...
1510 return tsp && task_is_running(tsp) && !tsp->on_cpu ? "!" : ""; in show_rcu_should_be_on_cpu()
1520 * Dump out nocb grace-period kthread state for the specified rcu_data
1525 struct rcu_node *rnp = rdp->mynode; in show_rcu_nocb_gp_state()
1527 pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu %c CPU %d%s\n", in show_rcu_nocb_gp_state()
1528 rdp->cpu, in show_rcu_nocb_gp_state()
1529 "kK"[!!rdp->nocb_gp_kthread], in show_rcu_nocb_gp_state()
1530 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], in show_rcu_nocb_gp_state()
1531 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_gp_state()
1532 "tT"[timer_pending(&rdp->nocb_timer)], in show_rcu_nocb_gp_state()
1533 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_gp_state()
1534 ".W"[swait_active(&rdp->nocb_gp_wq)], in show_rcu_nocb_gp_state()
1535 ".W"[swait_active(&rnp->nocb_gp_wq[0])], in show_rcu_nocb_gp_state()
1536 ".W"[swait_active(&rnp->nocb_gp_wq[1])], in show_rcu_nocb_gp_state()
1537 ".B"[!!rdp->nocb_gp_bypass], in show_rcu_nocb_gp_state()
1538 ".G"[!!rdp->nocb_gp_gp], in show_rcu_nocb_gp_state()
1539 (long)rdp->nocb_gp_seq, in show_rcu_nocb_gp_state()
1540 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops), in show_rcu_nocb_gp_state()
1541 rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.', in show_rcu_nocb_gp_state()
1542 rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, in show_rcu_nocb_gp_state()
1543 show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread)); in show_rcu_nocb_gp_state()
1552 struct rcu_segcblist *rsclp = &rdp->cblist; in show_rcu_nocb_state()
1556 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
1559 nocb_next_rdp = list_next_or_null_rcu(&rdp->nocb_gp_rdp->nocb_head_rdp, in show_rcu_nocb_state()
1560 &rdp->nocb_entry_rdp, in show_rcu_nocb_state()
1564 sprintf(bufw, "%ld", rsclp->gp_seq[RCU_WAIT_TAIL]); in show_rcu_nocb_state()
1565 sprintf(bufr, "%ld", rsclp->gp_seq[RCU_NEXT_READY_TAIL]); in show_rcu_nocb_state()
1566 pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%c%s%c%s%c%c q%ld %c CPU %d%s\n", in show_rcu_nocb_state()
1567 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
1568 nocb_next_rdp ? nocb_next_rdp->cpu : -1, in show_rcu_nocb_state()
1569 "kK"[!!rdp->nocb_cb_kthread], in show_rcu_nocb_state()
1570 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)], in show_rcu_nocb_state()
1571 "lL"[raw_spin_is_locked(&rdp->nocb_lock)], in show_rcu_nocb_state()
1572 "sS"[!!rdp->nocb_cb_sleep], in show_rcu_nocb_state()
1573 ".W"[swait_active(&rdp->nocb_cb_wq)], in show_rcu_nocb_state()
1574 jiffies - rdp->nocb_bypass_first, in show_rcu_nocb_state()
1575 jiffies - rdp->nocb_nobypass_last, in show_rcu_nocb_state()
1576 rdp->nocb_nobypass_count, in show_rcu_nocb_state()
1583 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], in show_rcu_nocb_state()
1584 rcu_segcblist_n_cbs(&rdp->cblist), in show_rcu_nocb_state()
1585 rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.', in show_rcu_nocb_state()
1586 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1, in show_rcu_nocb_state()
1587 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); in show_rcu_nocb_state()
1590 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
1593 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock); in show_rcu_nocb_state()
1594 wassleep = swait_active(&rdp->nocb_gp_wq); in show_rcu_nocb_state()
1595 if (!rdp->nocb_gp_sleep && !waslocked && !wassleep) in show_rcu_nocb_state()
1598 pr_info(" nocb GP activity on CB-only CPU!!! %c%c%c %c\n", in show_rcu_nocb_state()
1600 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_state()
1601 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_state()
1607 /* No ->nocb_lock to acquire. */
1612 /* No ->nocb_lock to release. */
1617 /* No ->nocb_lock to release. */
1624 /* Lockdep check that ->cblist may be safely accessed. */
1680 static void rcu_spawn_cpu_nocb_kthread(int cpu) in rcu_spawn_cpu_nocb_kthread() argument