1 // SPDX-License-Identifier: GPL-2.0-only
7  *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
20 #include <linux/cpu.h>
38    - No shared variables, all the data are CPU local.
39    - If a softirq needs serialization, let it serialize itself
41    - Even if softirq is serialized, only local cpu is marked for
42      execution. Hence, we get something sort of weak cpu binding.
47    - NET RX softirq. It is multithreaded and does not require
49    - NET TX softirq. It kicks software netdevice queues, hence
52    - Tasklets: serialized wrt itself.
65 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
70  * we cannot loop indefinitely here to avoid userspace starvation,
95  * to a per CPU counter and to task::softirqs_disabled_cnt.
97  * - count is changed by SOFTIRQ_OFFSET on entering or leaving softirq
100  * - count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
110  * also in per CPU softirq_ctrl::cnt. This is necessary to allow tasks in a
117  * The per CPU counter prevents pointless wakeups of ksoftirqd in case that
130  * local_bh_blocked() - Check for idle whether BH processing is blocked
132  * Returns false if the per CPU softirq::cnt is 0 otherwise true.
136  * softirq_ctrl::lock was the only running task on the CPU and blocks on
152 	if (!current->softirq_disable_cnt) {  in __local_bh_disable_ip()
163 	 * Track the per CPU softirq disabled state. On RT this is per CPU  in __local_bh_disable_ip()
171 	current->softirq_disable_cnt = newcnt;  in __local_bh_disable_ip()
186 	DEBUG_LOCKS_WARN_ON(current->softirq_disable_cnt !=  in __local_bh_enable()
196 	current->softirq_disable_cnt = newcnt;  in __local_bh_enable()
229 	 * If this was called from non preemptible context, wake up the  in __local_bh_enable_ip()
253  * to acquire the per CPU local lock for reentrancy protection.
299  * This one is for softirq.c-internal use, where hardirqs are disabled
313 	 * is set and before current->softirq_enabled is cleared.  in __local_bh_disable_ip()
327 		current->preempt_disable_ip = get_lock_parent_ip();  in __local_bh_disable_ip()
349  * Special-case - softirqs can safely be enabled by __do_softirq(),
350  * without processing still-pending softirqs:
375 	__preempt_count_sub(cnt - 1);  in __local_bh_enable_ip()
471  * The two things to balance is latency against fairness -
482  * not miss-qualify lock contexts and miss possible deadlocks.
514 	unsigned long old_flags = current->flags;  in handle_softirqs()
526 	current->flags &= ~PF_MEMALLOC;  in handle_softirqs()
546 		h += softirq_bit - 1;  in handle_softirqs()
548 		vec_nr = h - softirq_vec;  in handle_softirqs()
554 		h->action();  in handle_softirqs()
558 			       vec_nr, softirq_to_name[vec_nr], h->action,  in handle_softirqs()
574 		    --max_restart)  in handle_softirqs()
592  * irq_enter_rcu - Enter an interrupt context with RCU watching
606  * irq_enter - Enter an interrupt context including RCU update
617 	int cpu = smp_processor_id();  in tick_irq_exit()  local
619 	/* Make sure that timer wheel updates are propagated */  in tick_irq_exit()
620 	if ((sched_core_idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {  in tick_irq_exit()
643  * irq_exit_rcu() - Exit an interrupt context without updating RCU
655  * irq_exit - Exit an interrupt context, update RCU and lockdep
676 	 * (this also catches softirq-disabled code). We will  in raise_softirq_irqoff()
680 	 * Otherwise we wake up ksoftirqd to make sure we  in raise_softirq_irqoff()
728 	t->next = NULL;  in __tasklet_schedule_common()
729 	*head->tail = t;  in __tasklet_schedule_common()
730 	head->tail = &(t->next);  in __tasklet_schedule_common()
751 	if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) {  in tasklet_clear_sched()
752 		wake_up_var(&t->state);  in tasklet_clear_sched()
757 		  t->use_callback ? "callback" : "func",  in tasklet_clear_sched()
758 		  t->use_callback ? (void *)t->callback : (void *)t->func);  in tasklet_clear_sched()
769 	list = tl_head->head;  in tasklet_action_common()
770 	tl_head->head = NULL;  in tasklet_action_common()
771 	tl_head->tail = &tl_head->head;  in tasklet_action_common()
777 		list = list->next;  in tasklet_action_common()
780 			if (!atomic_read(&t->count)) {  in tasklet_action_common()
782 					if (t->use_callback) {  in tasklet_action_common()
783 						trace_tasklet_entry(t, t->callback);  in tasklet_action_common()
784 						t->callback(t);  in tasklet_action_common()
785 						trace_tasklet_exit(t, t->callback);  in tasklet_action_common()
787 						trace_tasklet_entry(t, t->func);  in tasklet_action_common()
788 						t->func(t->data);  in tasklet_action_common()
789 						trace_tasklet_exit(t, t->func);  in tasklet_action_common()
799 		t->next = NULL;  in tasklet_action_common()
800 		*tl_head->tail = t;  in tasklet_action_common()
801 		tl_head->tail = &t->next;  in tasklet_action_common()
822 	t->next = NULL;  in tasklet_setup()
823 	t->state = 0;  in tasklet_setup()
824 	atomic_set(&t->count, 0);  in tasklet_setup()
825 	t->callback = callback;  in tasklet_setup()
826 	t->use_callback = true;  in tasklet_setup()
827 	t->data = 0;  in tasklet_setup()
834 	t->next = NULL;  in tasklet_init()
835 	t->state = 0;  in tasklet_init()
836 	atomic_set(&t->count, 0);  in tasklet_init()
837 	t->func = func;  in tasklet_init()
838 	t->use_callback = false;  in tasklet_init()
839 	t->data = data;  in tasklet_init()
850 	while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {  in tasklet_unlock_spin_wait()
855 			 * running. If the tasklet runs on a different CPU  in tasklet_unlock_spin_wait()
874 	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state))  in tasklet_kill()
875 		wait_var_event(&t->state, !test_bit(TASKLET_STATE_SCHED, &t->state));  in tasklet_kill()
886 	clear_bit(TASKLET_STATE_RUN, &t->state);  in tasklet_unlock()
888 	wake_up_var(&t->state);  in tasklet_unlock()
894 	wait_var_event(&t->state, !test_bit(TASKLET_STATE_RUN, &t->state));  in tasklet_unlock_wait()
901 	int cpu;  in softirq_init()  local
903 	for_each_possible_cpu(cpu) {  in softirq_init()
904 		per_cpu(tasklet_vec, cpu).tail =  in softirq_init()
905 			&per_cpu(tasklet_vec, cpu).head;  in softirq_init()
906 		per_cpu(tasklet_hi_vec, cpu).tail =  in softirq_init()
907 			&per_cpu(tasklet_hi_vec, cpu).head;  in softirq_init()
914 static int ksoftirqd_should_run(unsigned int cpu)  in ksoftirqd_should_run()  argument
919 static void run_ksoftirqd(unsigned int cpu)  in run_ksoftirqd()  argument
936 static int takeover_tasklets(unsigned int cpu)  in takeover_tasklets()  argument
938 	workqueue_softirq_dead(cpu);  in takeover_tasklets()
940 	/* CPU is dead, so no lock needed. */  in takeover_tasklets()
943 	/* Find end, append list for that CPU. */  in takeover_tasklets()
944 	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {  in takeover_tasklets()
945 		*__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;  in takeover_tasklets()
946 		__this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);  in takeover_tasklets()
947 		per_cpu(tasklet_vec, cpu).head = NULL;  in takeover_tasklets()
948 		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;  in takeover_tasklets()
952 	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {  in takeover_tasklets()
953 		*__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;  in takeover_tasklets()
954 		__this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);  in takeover_tasklets()
955 		per_cpu(tasklet_hi_vec, cpu).head = NULL;  in takeover_tasklets()
956 		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;  in takeover_tasklets()