Lines Matching refs:rdp
20 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) in rcu_current_is_nocb_kthread() argument
23 if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread) in rcu_current_is_nocb_kthread()
26 if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread) in rcu_current_is_nocb_kthread()
92 static void rcu_nocb_bypass_lock(struct rcu_data *rdp) in rcu_nocb_bypass_lock() argument
93 __acquires(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_lock()
96 if (raw_spin_trylock(&rdp->nocb_bypass_lock)) in rcu_nocb_bypass_lock()
102 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
103 raw_spin_lock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_lock()
110 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) in rcu_nocb_bypass_trylock() argument
113 return raw_spin_trylock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_trylock()
119 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp) in rcu_nocb_bypass_unlock() argument
120 __releases(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_unlock()
123 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_unlock()
130 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
133 if (!rcu_rdp_is_offloaded(rdp)) in rcu_nocb_lock()
135 raw_spin_lock(&rdp->nocb_lock); in rcu_nocb_lock()
142 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
144 if (rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_unlock()
146 raw_spin_unlock(&rdp->nocb_lock); in rcu_nocb_unlock()
154 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
157 if (rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_unlock_irqrestore()
159 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_unlock_irqrestore()
166 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
169 if (rcu_rdp_is_offloaded(rdp)) in rcu_lockdep_assert_cblist_protected()
170 lockdep_assert_held(&rdp->nocb_lock); in rcu_lockdep_assert_cblist_protected()
194 struct rcu_data *rdp, in __wake_nocb_gp()
202 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __wake_nocb_gp()
218 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in __wake_nocb_gp()
228 static bool wake_nocb_gp(struct rcu_data *rdp, bool force) in wake_nocb_gp() argument
231 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp()
234 return __wake_nocb_gp(rdp_gp, rdp, force, flags); in wake_nocb_gp()
266 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, in wake_nocb_gp_defer() argument
270 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp_defer()
279 rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) { in wake_nocb_gp_defer()
294 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
308 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp_in, in rcu_nocb_do_flush_bypass() argument
314 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp)); in rcu_nocb_do_flush_bypass()
315 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_do_flush_bypass()
316 lockdep_assert_held(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
317 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { in rcu_nocb_do_flush_bypass()
318 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
323 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_do_flush_bypass()
332 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
335 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
336 WRITE_ONCE(rdp->lazy_len, 0); in rcu_nocb_do_flush_bypass()
338 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); in rcu_nocb_do_flush_bypass()
339 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_do_flush_bypass()
340 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_do_flush_bypass()
352 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
355 if (!rcu_rdp_is_offloaded(rdp)) in rcu_nocb_flush_bypass()
357 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_flush_bypass()
358 rcu_nocb_bypass_lock(rdp); in rcu_nocb_flush_bypass()
359 return rcu_nocb_do_flush_bypass(rdp, rhp, j, lazy); in rcu_nocb_flush_bypass()
366 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j) in rcu_nocb_try_flush_bypass() argument
368 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_try_flush_bypass()
369 if (!rcu_rdp_is_offloaded(rdp) || in rcu_nocb_try_flush_bypass()
370 !rcu_nocb_bypass_trylock(rdp)) in rcu_nocb_try_flush_bypass()
372 WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j, false)); in rcu_nocb_try_flush_bypass()
393 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
400 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
401 bool bypass_is_lazy = (ncbs == READ_ONCE(rdp->lazy_len)); in rcu_nocb_try_bypass()
407 if (!rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_try_bypass()
408 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
414 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
415 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
416 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
422 if (j == rdp->nocb_nobypass_last) { in rcu_nocb_try_bypass()
423 c = rdp->nocb_nobypass_count + 1; in rcu_nocb_try_bypass()
425 WRITE_ONCE(rdp->nocb_nobypass_last, j); in rcu_nocb_try_bypass()
426 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy; in rcu_nocb_try_bypass()
427 if (ULONG_CMP_LT(rdp->nocb_nobypass_count, in rcu_nocb_try_bypass()
433 WRITE_ONCE(rdp->nocb_nobypass_count, c); in rcu_nocb_try_bypass()
439 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) { in rcu_nocb_try_bypass()
440 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
441 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
443 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
446 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j, false)); in rcu_nocb_try_bypass()
447 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
453 if ((ncbs && !bypass_is_lazy && j != READ_ONCE(rdp->nocb_bypass_first)) || in rcu_nocb_try_bypass()
455 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + rcu_get_jiffies_lazy_flush()))) || in rcu_nocb_try_bypass()
457 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
458 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
460 if (!rcu_nocb_flush_bypass(rdp, rhp, j, lazy)) { in rcu_nocb_try_bypass()
462 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
464 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
467 if (j != rdp->nocb_gp_adv_time && in rcu_nocb_try_bypass()
468 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in rcu_nocb_try_bypass()
469 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in rcu_nocb_try_bypass()
470 rcu_advance_cbs_nowake(rdp->mynode, rdp); in rcu_nocb_try_bypass()
471 rdp->nocb_gp_adv_time = j; in rcu_nocb_try_bypass()
477 __call_rcu_nocb_wake(rdp, *was_alldone, flags); in rcu_nocb_try_bypass()
483 rcu_nocb_bypass_lock(rdp); in rcu_nocb_try_bypass()
484 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
485 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_try_bypass()
486 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_try_bypass()
489 WRITE_ONCE(rdp->lazy_len, rdp->lazy_len + 1); in rcu_nocb_try_bypass()
492 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_try_bypass()
493 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
495 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_try_bypass()
506 rcu_nocb_lock(rdp); // Rare during call_rcu() flood. in rcu_nocb_try_bypass()
507 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_nocb_try_bypass()
508 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
510 __call_rcu_nocb_wake(rdp, true, flags); in rcu_nocb_try_bypass()
512 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
514 rcu_nocb_unlock(rdp); in rcu_nocb_try_bypass()
526 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, in __call_rcu_nocb_wake() argument
528 __releases(rdp->nocb_lock) in __call_rcu_nocb_wake()
536 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in __call_rcu_nocb_wake()
539 t = READ_ONCE(rdp->nocb_gp_kthread); in __call_rcu_nocb_wake()
541 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
542 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
547 len = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_nocb_wake()
548 bypass_len = rcu_cblist_n_cbs(&rdp->nocb_bypass); in __call_rcu_nocb_wake()
549 lazy_len = READ_ONCE(rdp->lazy_len); in __call_rcu_nocb_wake()
551 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
554 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
555 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_LAZY, in __call_rcu_nocb_wake()
557 } else if (!irqs_disabled_flags(flags) && cpu_online(rdp->cpu)) { in __call_rcu_nocb_wake()
559 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
560 wake_nocb_gp(rdp, false); in __call_rcu_nocb_wake()
561 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
570 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
571 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE, in __call_rcu_nocb_wake()
574 } else if (len > rdp->qlen_last_fqs_check + qhimark) { in __call_rcu_nocb_wake()
576 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
578 if (j != rdp->nocb_gp_adv_time && in __call_rcu_nocb_wake()
579 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in __call_rcu_nocb_wake()
580 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in __call_rcu_nocb_wake()
581 rcu_advance_cbs_nowake(rdp->mynode, rdp); in __call_rcu_nocb_wake()
582 rdp->nocb_gp_adv_time = j; in __call_rcu_nocb_wake()
585 if ((rdp->nocb_cb_sleep || in __call_rcu_nocb_wake()
586 !rcu_segcblist_ready_cbs(&rdp->cblist)) && in __call_rcu_nocb_wake()
588 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
589 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE, in __call_rcu_nocb_wake()
592 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
593 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
596 rcu_nocb_unlock(rdp); in __call_rcu_nocb_wake()
597 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
601 static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, in call_rcu_nocb() argument
606 if (!rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) { in call_rcu_nocb()
608 rcutree_enqueue(rdp, head, func); in call_rcu_nocb()
609 __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */ in call_rcu_nocb()
613 static void nocb_gp_toggle_rdp(struct rcu_data *rdp_gp, struct rcu_data *rdp) in nocb_gp_toggle_rdp() argument
615 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_gp_toggle_rdp()
623 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in nocb_gp_toggle_rdp()
629 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp); in nocb_gp_toggle_rdp()
636 list_del(&rdp->nocb_entry_rdp); in nocb_gp_toggle_rdp()
639 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in nocb_gp_toggle_rdp()
666 struct rcu_data *rdp, *rdp_toggling = NULL; in nocb_gp_wait() local
691 list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { in nocb_gp_wait()
696 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
697 rcu_nocb_lock_irqsave(rdp, flags); in nocb_gp_wait()
698 lockdep_assert_held(&rdp->nocb_lock); in nocb_gp_wait()
699 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
700 lazy_ncbs = READ_ONCE(rdp->lazy_len); in nocb_gp_wait()
703 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + rcu_get_jiffies_lazy_flush()) || in nocb_gp_wait()
707 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) || in nocb_gp_wait()
710 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { in nocb_gp_wait()
711 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
717 (void)rcu_nocb_try_flush_bypass(rdp, j); in nocb_gp_wait()
718 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
719 lazy_ncbs = READ_ONCE(rdp->lazy_len); in nocb_gp_wait()
723 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
730 rnp = rdp->mynode; in nocb_gp_wait()
734 if (!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
736 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_gp_wait()
739 needwake_gp = rcu_advance_cbs(rnp, rdp); in nocb_gp_wait()
740 wasempty = rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
746 !rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
748 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { in nocb_gp_wait()
753 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
756 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_gp_wait()
757 needwake = rdp->nocb_cb_sleep; in nocb_gp_wait()
758 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_gp_wait()
762 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
764 swake_up_one(&rdp->nocb_cb_wq); in nocb_gp_wait()
864 struct rcu_data *rdp = arg; in rcu_nocb_gp_kthread() local
867 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1); in rcu_nocb_gp_kthread()
868 nocb_gp_wait(rdp); in rcu_nocb_gp_kthread()
874 static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) in nocb_cb_wait_cond() argument
876 return !READ_ONCE(rdp->nocb_cb_sleep) || kthread_should_park(); in nocb_cb_wait_cond()
883 static void nocb_cb_wait(struct rcu_data *rdp) in nocb_cb_wait() argument
885 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_cb_wait()
889 struct rcu_node *rnp = rdp->mynode; in nocb_cb_wait()
891 swait_event_interruptible_exclusive(rdp->nocb_cb_wq, in nocb_cb_wait()
892 nocb_cb_wait_cond(rdp)); in nocb_cb_wait()
895 } else if (READ_ONCE(rdp->nocb_cb_sleep)) { in nocb_cb_wait()
897 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
900 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp)); in nocb_cb_wait()
912 rcu_do_batch(rdp); in nocb_cb_wait()
915 rcu_nocb_lock_irqsave(rdp, flags); in nocb_cb_wait()
919 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); in nocb_cb_wait()
924 WRITE_ONCE(rdp->nocb_cb_sleep, true); in nocb_cb_wait()
925 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
927 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_cb_wait()
930 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_cb_wait()
941 struct rcu_data *rdp = arg; in rcu_nocb_cb_kthread() local
946 nocb_cb_wait(rdp); in rcu_nocb_cb_kthread()
953 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) in rcu_nocb_need_deferred_wakeup() argument
955 return READ_ONCE(rdp->nocb_defer_wakeup) >= level; in rcu_nocb_need_deferred_wakeup()
960 struct rcu_data *rdp, int level, in do_nocb_deferred_wakeup_common()
973 ret = __wake_nocb_gp(rdp_gp, rdp, ndw == RCU_NOCB_WAKE_FORCE, flags); in do_nocb_deferred_wakeup_common()
974 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
983 struct rcu_data *rdp = from_timer(rdp, t, nocb_timer); in do_nocb_deferred_wakeup_timer() local
985 WARN_ON_ONCE(rdp->nocb_gp_rdp != rdp); in do_nocb_deferred_wakeup_timer()
986 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_deferred_wakeup_timer()
988 raw_spin_lock_irqsave(&rdp->nocb_gp_lock, flags); in do_nocb_deferred_wakeup_timer()
990 do_nocb_deferred_wakeup_common(rdp, rdp, RCU_NOCB_WAKE_BYPASS, flags); in do_nocb_deferred_wakeup_timer()
998 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
1001 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in do_nocb_deferred_wakeup()
1007 return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags); in do_nocb_deferred_wakeup()
1016 static int rcu_nocb_queue_toggle_rdp(struct rcu_data *rdp) in rcu_nocb_queue_toggle_rdp() argument
1018 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_queue_toggle_rdp()
1024 WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp); in rcu_nocb_queue_toggle_rdp()
1034 static bool rcu_nocb_rdp_deoffload_wait_cond(struct rcu_data *rdp) in rcu_nocb_rdp_deoffload_wait_cond() argument
1044 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload_wait_cond()
1045 ret = !rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_deoffload_wait_cond()
1046 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload_wait_cond()
1051 static int rcu_nocb_rdp_deoffload(struct rcu_data *rdp) in rcu_nocb_rdp_deoffload() argument
1055 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_rdp_deoffload()
1058 WARN_ON_ONCE(cpu_online(rdp->cpu) && rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_deoffload()
1060 pr_info("De-offloading %d\n", rdp->cpu); in rcu_nocb_rdp_deoffload()
1069 if (rdp->nocb_cb_kthread) in rcu_nocb_rdp_deoffload()
1070 kthread_park(rdp->nocb_cb_kthread); in rcu_nocb_rdp_deoffload()
1072 rcu_nocb_lock_irqsave(rdp, flags); in rcu_nocb_rdp_deoffload()
1073 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_rdp_deoffload()
1074 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_nocb_rdp_deoffload()
1075 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_nocb_rdp_deoffload()
1077 wake_gp = rcu_nocb_queue_toggle_rdp(rdp); in rcu_nocb_rdp_deoffload()
1085 swait_event_exclusive(rdp->nocb_state_wq, in rcu_nocb_rdp_deoffload()
1086 rcu_nocb_rdp_deoffload_wait_cond(rdp)); in rcu_nocb_rdp_deoffload()
1093 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload()
1094 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_deoffload()
1095 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload()
1097 list_del(&rdp->nocb_entry_rdp); in rcu_nocb_rdp_deoffload()
1107 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_deoffload() local
1112 if (rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_cpu_deoffload()
1114 ret = rcu_nocb_rdp_deoffload(rdp); in rcu_nocb_cpu_deoffload()
1118 pr_info("NOCB: Cannot CB-deoffload online CPU %d\n", rdp->cpu); in rcu_nocb_cpu_deoffload()
1129 static bool rcu_nocb_rdp_offload_wait_cond(struct rcu_data *rdp) in rcu_nocb_rdp_offload_wait_cond() argument
1134 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_offload_wait_cond()
1135 ret = rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_nocb_rdp_offload_wait_cond()
1136 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_offload_wait_cond()
1141 static int rcu_nocb_rdp_offload(struct rcu_data *rdp) in rcu_nocb_rdp_offload() argument
1144 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rcu_nocb_rdp_offload()
1146 WARN_ON_ONCE(cpu_online(rdp->cpu)); in rcu_nocb_rdp_offload()
1151 if (!rdp->nocb_gp_rdp) in rcu_nocb_rdp_offload()
1157 pr_info("Offloading %d\n", rdp->cpu); in rcu_nocb_rdp_offload()
1159 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_rdp_offload()
1160 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_nocb_rdp_offload()
1162 wake_gp = rcu_nocb_queue_toggle_rdp(rdp); in rcu_nocb_rdp_offload()
1166 swait_event_exclusive(rdp->nocb_state_wq, in rcu_nocb_rdp_offload()
1167 rcu_nocb_rdp_offload_wait_cond(rdp)); in rcu_nocb_rdp_offload()
1169 kthread_unpark(rdp->nocb_cb_kthread); in rcu_nocb_rdp_offload()
1176 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_offload() local
1181 if (!rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_cpu_offload()
1183 ret = rcu_nocb_rdp_offload(rdp); in rcu_nocb_cpu_offload()
1187 pr_info("NOCB: Cannot CB-offload online CPU %d\n", rdp->cpu); in rcu_nocb_cpu_offload()
1214 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_count() local
1216 count += READ_ONCE(rdp->lazy_len); in lazy_rcu_shrink_count()
1249 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_scan() local
1252 if (WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp))) in lazy_rcu_shrink_scan()
1255 if (!READ_ONCE(rdp->lazy_len)) in lazy_rcu_shrink_scan()
1258 rcu_nocb_lock_irqsave(rdp, flags); in lazy_rcu_shrink_scan()
1264 _count = READ_ONCE(rdp->lazy_len); in lazy_rcu_shrink_scan()
1266 rcu_nocb_unlock_irqrestore(rdp, flags); in lazy_rcu_shrink_scan()
1269 rcu_nocb_try_flush_bypass(rdp, jiffies); in lazy_rcu_shrink_scan()
1270 rcu_nocb_unlock_irqrestore(rdp, flags); in lazy_rcu_shrink_scan()
1271 wake_nocb_gp(rdp, false); in lazy_rcu_shrink_scan()
1287 struct rcu_data *rdp; in rcu_init_nohz() local
1341 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
1342 if (rcu_segcblist_empty(&rdp->cblist)) in rcu_init_nohz()
1343 rcu_segcblist_init(&rdp->cblist); in rcu_init_nohz()
1344 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_OFFLOADED); in rcu_init_nohz()
1350 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
1352 init_swait_queue_head(&rdp->nocb_cb_wq); in rcu_boot_init_nocb_percpu_data()
1353 init_swait_queue_head(&rdp->nocb_gp_wq); in rcu_boot_init_nocb_percpu_data()
1354 init_swait_queue_head(&rdp->nocb_state_wq); in rcu_boot_init_nocb_percpu_data()
1355 raw_spin_lock_init(&rdp->nocb_lock); in rcu_boot_init_nocb_percpu_data()
1356 raw_spin_lock_init(&rdp->nocb_bypass_lock); in rcu_boot_init_nocb_percpu_data()
1357 raw_spin_lock_init(&rdp->nocb_gp_lock); in rcu_boot_init_nocb_percpu_data()
1358 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
1359 rcu_cblist_init(&rdp->nocb_bypass); in rcu_boot_init_nocb_percpu_data()
1360 WRITE_ONCE(rdp->lazy_len, 0); in rcu_boot_init_nocb_percpu_data()
1361 mutex_init(&rdp->nocb_gp_kthread_mutex); in rcu_boot_init_nocb_percpu_data()
1371 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_cpu_nocb_kthread() local
1380 if (rdp->nocb_cb_kthread) in rcu_spawn_cpu_nocb_kthread()
1385 rdp_gp = rdp->nocb_gp_rdp; in rcu_spawn_cpu_nocb_kthread()
1401 t = kthread_create(rcu_nocb_cb_kthread, rdp, in rcu_spawn_cpu_nocb_kthread()
1406 if (rcu_rdp_is_offloaded(rdp)) in rcu_spawn_cpu_nocb_kthread()
1414 WRITE_ONCE(rdp->nocb_cb_kthread, t); in rcu_spawn_cpu_nocb_kthread()
1415 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); in rcu_spawn_cpu_nocb_kthread()
1425 WARN_ON_ONCE(system_state > SYSTEM_BOOTING && rcu_segcblist_n_cbs(&rdp->cblist)); in rcu_spawn_cpu_nocb_kthread()
1427 if (rcu_rdp_is_offloaded(rdp)) { in rcu_spawn_cpu_nocb_kthread()
1428 rcu_nocb_rdp_deoffload(rdp); in rcu_spawn_cpu_nocb_kthread()
1449 struct rcu_data *rdp; in rcu_organize_nocb_kthreads() local
1465 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
1466 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
1469 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
1470 rdp_gp = rdp; in rcu_organize_nocb_kthreads()
1471 INIT_LIST_HEAD(&rdp->nocb_head_rdp); in rcu_organize_nocb_kthreads()
1487 rdp->nocb_gp_rdp = rdp_gp; in rcu_organize_nocb_kthreads()
1489 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp); in rcu_organize_nocb_kthreads()
1523 static void show_rcu_nocb_gp_state(struct rcu_data *rdp) in show_rcu_nocb_gp_state() argument
1525 struct rcu_node *rnp = rdp->mynode; in show_rcu_nocb_gp_state()
1528 rdp->cpu, in show_rcu_nocb_gp_state()
1529 "kK"[!!rdp->nocb_gp_kthread], in show_rcu_nocb_gp_state()
1530 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], in show_rcu_nocb_gp_state()
1531 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_gp_state()
1532 "tT"[timer_pending(&rdp->nocb_timer)], in show_rcu_nocb_gp_state()
1533 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_gp_state()
1534 ".W"[swait_active(&rdp->nocb_gp_wq)], in show_rcu_nocb_gp_state()
1537 ".B"[!!rdp->nocb_gp_bypass], in show_rcu_nocb_gp_state()
1538 ".G"[!!rdp->nocb_gp_gp], in show_rcu_nocb_gp_state()
1539 (long)rdp->nocb_gp_seq, in show_rcu_nocb_gp_state()
1540 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops), in show_rcu_nocb_gp_state()
1541 rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.', in show_rcu_nocb_gp_state()
1542 rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, in show_rcu_nocb_gp_state()
1543 show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread)); in show_rcu_nocb_gp_state()
1547 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument
1552 struct rcu_segcblist *rsclp = &rdp->cblist; in show_rcu_nocb_state()
1556 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
1557 show_rcu_nocb_gp_state(rdp); in show_rcu_nocb_state()
1559 nocb_next_rdp = list_next_or_null_rcu(&rdp->nocb_gp_rdp->nocb_head_rdp, in show_rcu_nocb_state()
1560 &rdp->nocb_entry_rdp, in show_rcu_nocb_state()
1561 typeof(*rdp), in show_rcu_nocb_state()
1567 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
1569 "kK"[!!rdp->nocb_cb_kthread], in show_rcu_nocb_state()
1570 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)], in show_rcu_nocb_state()
1571 "lL"[raw_spin_is_locked(&rdp->nocb_lock)], in show_rcu_nocb_state()
1572 "sS"[!!rdp->nocb_cb_sleep], in show_rcu_nocb_state()
1573 ".W"[swait_active(&rdp->nocb_cb_wq)], in show_rcu_nocb_state()
1574 jiffies - rdp->nocb_bypass_first, in show_rcu_nocb_state()
1575 jiffies - rdp->nocb_nobypass_last, in show_rcu_nocb_state()
1576 rdp->nocb_nobypass_count, in show_rcu_nocb_state()
1583 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], in show_rcu_nocb_state()
1584 rcu_segcblist_n_cbs(&rdp->cblist), in show_rcu_nocb_state()
1585 rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.', in show_rcu_nocb_state()
1586 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1, in show_rcu_nocb_state()
1587 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); in show_rcu_nocb_state()
1590 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
1593 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock); in show_rcu_nocb_state()
1594 wassleep = swait_active(&rdp->nocb_gp_wq); in show_rcu_nocb_state()
1595 if (!rdp->nocb_gp_sleep && !waslocked && !wassleep) in show_rcu_nocb_state()
1600 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_state()
1601 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_state()
1608 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
1613 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
1618 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
1625 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
1643 static bool wake_nocb_gp(struct rcu_data *rdp, bool force) in wake_nocb_gp() argument
1648 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
1654 static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, in call_rcu_nocb() argument
1660 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, in __call_rcu_nocb_wake() argument
1666 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
1670 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) in rcu_nocb_need_deferred_wakeup() argument
1675 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
1684 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument