Lines Matching refs:cblist
245 if (rcu_segcblist_is_enabled(&rdp->cblist)) in rcu_get_n_cbs_cpu()
246 return rcu_segcblist_n_cbs(&rdp->cblist); in rcu_get_n_cbs_cpu()
697 return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && in rcu_needs_cpu()
1144 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_accelerate_cbs()
1147 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPreAcc")); in rcu_accelerate_cbs()
1160 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) in rcu_accelerate_cbs()
1164 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) in rcu_accelerate_cbs()
1169 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); in rcu_accelerate_cbs()
1191 (void)rcu_segcblist_accelerate(&rdp->cblist, c); in rcu_accelerate_cbs_unlocked()
1217 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_advance_cbs()
1224 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
2505 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { in rcu_do_batch()
2507 rcu_segcblist_n_cbs(&rdp->cblist), 0); in rcu_do_batch()
2509 !rcu_segcblist_empty(&rdp->cblist), in rcu_do_batch()
2528 pending = rcu_segcblist_get_seglen(&rdp->cblist, RCU_DONE_TAIL); in rcu_do_batch()
2543 rcu_segcblist_n_cbs(&rdp->cblist), bl); in rcu_do_batch()
2544 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2546 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2548 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbDequeued")); in rcu_do_batch()
2607 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2608 rcu_segcblist_add_len(&rdp->cblist, -count); in rcu_do_batch()
2611 count = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2626 empty = rcu_segcblist_empty(&rdp->cblist); in rcu_do_batch()
2630 WARN_ON_ONCE(count == 0 && rcu_segcblist_n_segment_cbs(&rdp->cblist) != 0); in rcu_do_batch()
2631 WARN_ON_ONCE(!empty && rcu_segcblist_n_segment_cbs(&rdp->cblist) == 0); in rcu_do_batch()
2811 rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp)) { in rcu_core()
2813 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_core()
2821 if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_ready_cbs(&rdp->cblist) && in rcu_core()
2825 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_core()
2954 rcu_segcblist_enqueue(&rdp->cblist, head); in rcutree_enqueue()
2958 rcu_segcblist_n_cbs(&rdp->cblist)); in rcutree_enqueue()
2961 rcu_segcblist_n_cbs(&rdp->cblist)); in rcutree_enqueue()
2962 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued")); in rcutree_enqueue()
2990 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > in call_rcu_core()
3003 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in call_rcu_core()
3006 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in call_rcu_core()
3029 if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) in check_cb_ovld_locked()
3052 ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) == in check_cb_ovld()
3092 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { in __call_rcu_common()
3098 if (rcu_segcblist_empty(&rdp->cblist)) in __call_rcu_common()
3099 rcu_segcblist_init(&rdp->cblist); in __call_rcu_common()
4439 rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_pending()
4443 if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && in rcu_pending()
4445 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_pending()
4512 was_alldone = rcu_rdp_is_offloaded(rdp) && !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_barrier_entrain()
4514 wake_nocb = was_alldone && rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_barrier_entrain()
4515 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) { in rcu_barrier_entrain()
4600 if (!rcu_segcblist_n_cbs(&rdp->cblist)) { in rcu_barrier()
4984 if (!rcu_segcblist_is_enabled(&rdp->cblist)) in rcutree_prepare_cpu()
4985 rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ in rcutree_prepare_cpu()
5231 if (rcu_segcblist_empty(&rdp->cblist)) { in rcutree_migrate_callbacks()
5246 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); in rcutree_migrate_callbacks()
5249 rcu_segcblist_disable(&rdp->cblist); in rcutree_migrate_callbacks()
5250 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != !rcu_segcblist_n_cbs(&my_rdp->cblist)); in rcutree_migrate_callbacks()
5263 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || in rcutree_migrate_callbacks()
5264 !rcu_segcblist_empty(&rdp->cblist), in rcutree_migrate_callbacks()
5266 cpu, rcu_segcblist_n_cbs(&rdp->cblist), in rcutree_migrate_callbacks()
5267 rcu_segcblist_first_cb(&rdp->cblist)); in rcutree_migrate_callbacks()