36 #include <linux/percpu-rwsem.h>
48  * struct cpuhp_cpu_state - Per cpu hotplug state storage
57  * @node:	Remote CPU node; for multi-instance, do a
59  * @last:	For multi-instance rollback, remember how far we got
63  * @done_up:	Signal completion to the issuer of the task for cpu-up
64  * @done_down:	Signal completion to the issuer of the task for cpu-down
96 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
98 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
118  * struct cpuhp_step - Hotplug state machine step
129 		int		(*multi)(unsigned int cpu,  member
134 		int		(*multi)(unsigned int cpu,  member
154 	return bringup ? !step->startup.single : !step->teardown.single;  in cpuhp_step_empty()
158  * cpuhp_invoke_callback - Invoke the callbacks for a given state
162  * @node:	For multi-instance, do a single entry callback for install/remove
163  * @lastp:	For multi-instance rollback, remember how far we got
179 	if (st->fail == state) {  in cpuhp_invoke_callback()
180 		st->fail = CPUHP_INVALID;  in cpuhp_invoke_callback()
181 		return -EAGAIN;  in cpuhp_invoke_callback()
189 	if (!step->multi_instance) {  in cpuhp_invoke_callback()
191 		cb = bringup ? step->startup.single : step->teardown.single;  in cpuhp_invoke_callback()
193 		trace_cpuhp_enter(cpu, st->target, state, cb);  in cpuhp_invoke_callback()
195 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
198 	cbm = bringup ? step->startup.multi : step->teardown.multi;  in cpuhp_invoke_callback()
203 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
205 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
211 	hlist_for_each(node, &step->list) {  in cpuhp_invoke_callback()
215 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
217 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
232 	cbm = !bringup ? step->startup.multi : step->teardown.multi;  in cpuhp_invoke_callback()
236 	hlist_for_each(node, &step->list) {  in cpuhp_invoke_callback()
237 		if (!cnt--)  in cpuhp_invoke_callback()
240 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
242 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
263 	struct completion *done = bringup ? &st->done_up : &st->done_down;  in wait_for_ap_thread()
269 	struct completion *done = bringup ? &st->done_up : &st->done_down;  in complete_ap_thread()
293  * cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
329 		} else if (now - start < NSEC_PER_MSEC) {  in cpuhp_wait_for_sync_state()
345  * cpuhp_ap_report_dead - Update synchronization state to DEAD
386  * cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
445 		ret = -EIO;  in cpuhp_bp_sync_alive()
477  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
574 	cpu_hotplug_disabled--;  in __cpu_hotplug_enable()
598  * Architectures that need SMT-specific errata handling during SMT hotplug
711 	enum cpuhp_state prev_state = st->state;  in cpuhp_set_state()
712 	bool bringup = st->state < target;  in cpuhp_set_state()
714 	st->rollback = false;  in cpuhp_set_state()
715 	st->last = NULL;  in cpuhp_set_state()
717 	st->target = target;  in cpuhp_set_state()
718 	st->single = false;  in cpuhp_set_state()
719 	st->bringup = bringup;  in cpuhp_set_state()
730 	bool bringup = !st->bringup;  in cpuhp_reset_state()
732 	st->target = prev_state;  in cpuhp_reset_state()
738 	if (st->rollback)  in cpuhp_reset_state()
741 	st->rollback = true;  in cpuhp_reset_state()
744 	 * If we have st->last we need to undo partial multi_instance of this  in cpuhp_reset_state()
747 	if (!st->last) {  in cpuhp_reset_state()
748 		if (st->bringup)  in cpuhp_reset_state()
749 			st->state--;  in cpuhp_reset_state()
751 			st->state++;  in cpuhp_reset_state()
754 	st->bringup = bringup;  in cpuhp_reset_state()
762 	if (!st->single && st->state == st->target)  in __cpuhp_kick_ap()
765 	st->result = 0;  in __cpuhp_kick_ap()
771 	st->should_run = true;  in __cpuhp_kick_ap()
772 	wake_up_process(st->thread);  in __cpuhp_kick_ap()
773 	wait_for_ap_thread(st, st->bringup);  in __cpuhp_kick_ap()
784 	if ((ret = st->result)) {  in cpuhp_kick_ap()
799 		return -ECANCELED;  in bringup_wait_for_ap_online()
802 	kthread_unpark(st->thread);  in bringup_wait_for_ap_online()
812 		return -ECANCELED;  in bringup_wait_for_ap_online()
820 		return -EAGAIN;  in cpuhp_kick_ap_alive()
847 	if (st->target <= CPUHP_AP_ONLINE_IDLE)  in cpuhp_bringup_ap()
850 	return cpuhp_kick_ap(cpu, st, st->target);  in cpuhp_bringup_ap()
864 		return -EAGAIN;  in bringup_cpu()
891 	if (st->target <= CPUHP_AP_ONLINE_IDLE)  in bringup_cpu()
894 	return cpuhp_kick_ap(cpu, st, st->target);  in bringup_cpu()
905 	struct mm_struct *mm = idle->active_mm;  in finish_cpu()
912 		idle->active_mm = &init_mm;  in finish_cpu()
925  * st->state will be modified ahead of time, to match state_to_run, as if it
935 			if (st->state >= target)  in cpuhp_next_state()
938 			*state_to_run = ++st->state;  in cpuhp_next_state()
940 			if (st->state <= target)  in cpuhp_next_state()
943 			*state_to_run = st->state--;  in cpuhp_next_state()
972 				cpuhp_get_step(st->state)->name,  in __cpuhp_invoke_callback_range()
973 				st->state, err);  in __cpuhp_invoke_callback_range()
974 			ret = -1;  in __cpuhp_invoke_callback_range()
1011 	return st->state <= CPUHP_BRINGUP_CPU;  in can_rollback_cpu()
1017 	enum cpuhp_state prev_state = st->state;  in cpuhp_up_callbacks()
1023 			 ret, cpu, cpuhp_get_step(st->state)->name,  in cpuhp_up_callbacks()
1024 			 st->state);  in cpuhp_up_callbacks()
1041 	return st->should_run;  in cpuhp_should_run()
1052  *  - single: runs st->cb_state
1053  *  - up:     runs ++st->state, while st->state < st->target
1054  *  - down:   runs st->state--, while st->state > st->target
1061 	bool bringup = st->bringup;  in cpuhp_thread_fun()
1064 	if (WARN_ON_ONCE(!st->should_run))  in cpuhp_thread_fun()
1068 	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures  in cpuhp_thread_fun()
1069 	 * that if we see ->should_run we also see the rest of the state.  in cpuhp_thread_fun()
1081 	if (st->single) {  in cpuhp_thread_fun()
1082 		state = st->cb_state;  in cpuhp_thread_fun()
1083 		st->should_run = false;  in cpuhp_thread_fun()
1085 		st->should_run = cpuhp_next_state(bringup, &state, st, st->target);  in cpuhp_thread_fun()
1086 		if (!st->should_run)  in cpuhp_thread_fun()
1094 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);  in cpuhp_thread_fun()
1100 		WARN_ON_ONCE(st->result);  in cpuhp_thread_fun()
1102 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);  in cpuhp_thread_fun()
1105 	if (st->result) {  in cpuhp_thread_fun()
1111 		WARN_ON_ONCE(st->rollback);  in cpuhp_thread_fun()
1112 		st->should_run = false;  in cpuhp_thread_fun()
1119 	if (!st->should_run)  in cpuhp_thread_fun()
1144 	if (!st->thread)  in cpuhp_invoke_ap_callback()
1147 	st->rollback = false;  in cpuhp_invoke_ap_callback()
1148 	st->last = NULL;  in cpuhp_invoke_ap_callback()
1150 	st->node = node;  in cpuhp_invoke_ap_callback()
1151 	st->bringup = bringup;  in cpuhp_invoke_ap_callback()
1152 	st->cb_state = state;  in cpuhp_invoke_ap_callback()
1153 	st->single = true;  in cpuhp_invoke_ap_callback()
1160 	if ((ret = st->result) && st->last) {  in cpuhp_invoke_ap_callback()
1161 		st->rollback = true;  in cpuhp_invoke_ap_callback()
1162 		st->bringup = !bringup;  in cpuhp_invoke_ap_callback()
1171 	st->node = st->last = NULL;  in cpuhp_invoke_ap_callback()
1178 	enum cpuhp_state prev_state = st->state;  in cpuhp_kick_ap_work()
1187 	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);  in cpuhp_kick_ap_work()
1188 	ret = cpuhp_kick_ap(cpu, st, st->target);  in cpuhp_kick_ap_work()
1189 	trace_cpuhp_exit(cpu, st->state, prev_state, ret);  in cpuhp_kick_ap_work()
1209 		init_completion(&st->done_up);  in cpuhp_init_state()
1210 		init_completion(&st->done_down);  in cpuhp_init_state()
1227  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
1232  * trivial, there are various non-obvious corner cases, which this function
1245 	 * their mm mask. -- Peter Zijlstra  in clear_tasks_mm_cpumask()
1247 	 * full-fledged tasklist_lock.  in clear_tasks_mm_cpumask()
1261 		arch_clear_mm_cpumask_cpu(cpu, t->mm);  in clear_tasks_mm_cpumask()
1271 	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);  in take_cpu_down()
1281 	 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.  in take_cpu_down()
1283 	WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));  in take_cpu_down()
1301 	kthread_park(st->thread);  in takedown_cpu()
1317 		kthread_unpark(st->thread);  in takedown_cpu()
1330 	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);  in takedown_cpu()
1344 	 * Callbacks must be re-integrated right away to the RCU state machine.  in takedown_cpu()
1364 	BUG_ON(st->state != CPUHP_AP_OFFLINE);  in cpuhp_report_idle_dead()
1367 	st->state = CPUHP_AP_IDLE_DEAD;  in cpuhp_report_idle_dead()
1379 	enum cpuhp_state prev_state = st->state;  in cpuhp_down_callbacks()
1385 			 ret, cpu, cpuhp_get_step(st->state)->name,  in cpuhp_down_callbacks()
1386 			 st->state);  in cpuhp_down_callbacks()
1390 		if (st->state < prev_state)  in cpuhp_down_callbacks()
1406 		return -EBUSY;  in _cpu_down()
1409 		return -EINVAL;  in _cpu_down()
1420 	if (st->state > CPUHP_TEARDOWN_CPU) {  in _cpu_down()
1421 		st->target = max((int)target, CPUHP_TEARDOWN_CPU);  in _cpu_down()
1434 		if (st->state > CPUHP_TEARDOWN_CPU)  in _cpu_down()
1437 		st->target = target;  in _cpu_down()
1444 	if (ret && st->state < prev_state) {  in _cpu_down()
1445 		if (st->state == CPUHP_TEARDOWN_CPU) {  in _cpu_down()
1473 	return _cpu_down(work->cpu, 0, work->target);  in __cpu_down_maps_locked()
1485 		return -EOPNOTSUPP;  in cpu_down_maps_locked()
1487 		return -EBUSY;  in cpu_down_maps_locked()
1491 	 * CPU to prevent a deadlock against cfs_b->period_timer.  in cpu_down_maps_locked()
1499 	return -EBUSY;  in cpu_down_maps_locked()
1513  * cpu_device_down - Bring down a cpu device
1518  * Other subsystems should use remove_cpu() instead.
1524 	return cpu_down(dev->id, CPUHP_OFFLINE);  in cpu_device_down()
1560 			pr_err("Failed to offline CPU%d - error=%d",  in smp_shutdown_nonboot_cpus()
1586  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1595 	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);  in notify_cpu_starting()
1627 	st->state = CPUHP_AP_ONLINE_IDLE;  in cpuhp_online_idle()
1641 		ret = -EINVAL;  in _cpu_up()
1649 	if (st->state >= target)  in _cpu_up()
1652 	if (st->state == CPUHP_OFFLINE) {  in _cpu_up()
1674 	if (st->state > CPUHP_BRINGUP_CPU) {  in _cpu_up()
1702 		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",  in cpu_up()
1704 		return -EINVAL;  in cpu_up()
1714 		err = -EBUSY;  in cpu_up()
1718 		err = -EPERM;  in cpu_up()
1729  * cpu_device_up - Bring up a cpu device
1734  * Other subsystems should use add_cpu() instead.
1740 	return cpu_up(dev->id, CPUHP_ONLINE);  in cpu_device_up()
1756  * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1773 			pr_err("Failed to bring hibernate-CPU up!\n");  in bringup_hibernate_cpu()
1797 		if (!--ncpus)  in cpuhp_bringup_mask()
1869 		ncpus -= num_online_cpus();  in cpuhp_bringup_cpus_parallel()
1877 	/* Bring the not-yet started CPUs up */  in cpuhp_bringup_cpus_parallel()
1907 	if (primary == -1) {  in freeze_secondary_cpus()
1917 	 * We take down all of the non-boot CPUs in one shot to avoid races  in freeze_secondary_cpus()
1922 	pr_info("Disabling non-boot CPUs ...\n");  in freeze_secondary_cpus()
1923 	for (cpu = nr_cpu_ids - 1; cpu >= 0; cpu--) {  in freeze_secondary_cpus()
1929 			error = -EBUSY;  in freeze_secondary_cpus()
1947 		pr_err("Non-boot CPUs are not disabled\n");  in freeze_secondary_cpus()
1978 	pr_info("Enabling non-boot CPUs ...\n");  in thaw_secondary_cpus()
2003 		return -ENOMEM;  in alloc_frozen_cpus()
2111 	 * On the tear-down path, timers_dead_cpu() must be invoked
2143 	 * All-in-one CPU bringup state which includes the kick alive.
2274 		return -EINVAL;  in cpuhp_cb_check()
2298 		return -EINVAL;  in cpuhp_reserve_state()
2302 		if (!step->name)  in cpuhp_reserve_state()
2306 	return -ENOSPC;  in cpuhp_reserve_state()
2335 	if (name && sp->name)  in cpuhp_store_callbacks()
2336 		return -EBUSY;  in cpuhp_store_callbacks()
2338 	sp->startup.single = startup;  in cpuhp_store_callbacks()
2339 	sp->teardown.single = teardown;  in cpuhp_store_callbacks()
2340 	sp->name = name;  in cpuhp_store_callbacks()
2341 	sp->multi_instance = multi_instance;  in cpuhp_store_callbacks()
2342 	INIT_HLIST_HEAD(&sp->list);  in cpuhp_store_callbacks()
2348 	return cpuhp_get_step(state)->teardown.single;  in cpuhp_get_teardown_cb()
2396 		int cpustate = st->state;  in cpuhp_rollback_install()
2418 	if (sp->multi_instance == false)  in __cpuhp_state_add_instance_cpuslocked()
2419 		return -EINVAL;  in __cpuhp_state_add_instance_cpuslocked()
2423 	if (!invoke || !sp->startup.multi)  in __cpuhp_state_add_instance_cpuslocked()
2432 		int cpustate = st->state;  in __cpuhp_state_add_instance_cpuslocked()
2439 			if (sp->teardown.multi)  in __cpuhp_state_add_instance_cpuslocked()
2446 	hlist_add_head(node, &sp->list);  in __cpuhp_state_add_instance_cpuslocked()
2465  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2494 		return -EINVAL;  in __cpuhp_setup_state_cpuslocked()
2516 		int cpustate = st->state;  in __cpuhp_setup_state_cpuslocked()
2565 	if (!sp->multi_instance)  in __cpuhp_state_remove_instance()
2566 		return -EINVAL;  in __cpuhp_state_remove_instance()
2580 		int cpustate = st->state;  in __cpuhp_state_remove_instance()
2596  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2615 	if (sp->multi_instance) {  in __cpuhp_remove_state_cpuslocked()
2616 		WARN(!hlist_empty(&sp->list),  in __cpuhp_remove_state_cpuslocked()
2632 		int cpustate = st->state;  in __cpuhp_remove_state_cpuslocked()
2656 	dev->offline = true;  in cpuhp_offline_cpu_device()
2658 	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);  in cpuhp_offline_cpu_device()
2665 	dev->offline = false;  in cpuhp_online_cpu_device()
2667 	kobject_uevent(&dev->kobj, KOBJ_ONLINE);  in cpuhp_online_cpu_device()
2743 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in state_show()
2745 	return sprintf(buf, "%d\n", st->state);  in state_show()
2752 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in target_store()
2762 		return -EINVAL;  in target_store()
2765 		return -EINVAL;  in target_store()
2774 	ret = !sp->name || sp->cant_stop ? -EINVAL : 0;  in target_store()
2779 	if (st->state < target)  in target_store()
2780 		ret = cpu_up(dev->id, target);  in target_store()
2781 	else if (st->state > target)  in target_store()
2782 		ret = cpu_down(dev->id, target);  in target_store()
2783 	else if (WARN_ON(st->target != target))  in target_store()
2784 		st->target = target;  in target_store()
2793 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in target_show()
2795 	return sprintf(buf, "%d\n", st->target);  in target_show()
2802 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in fail_store()
2811 		st->fail = fail;  in fail_store()
2816 		return -EINVAL;  in fail_store()
2822 		return -EINVAL;  in fail_store()
2830 	if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)  in fail_store()
2831 		return -EINVAL;  in fail_store()
2838 	if (!sp->startup.single && !sp->teardown.single)  in fail_store()
2839 		ret = -EINVAL;  in fail_store()
2844 	st->fail = fail;  in fail_store()
2852 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in fail_show()
2854 	return sprintf(buf, "%d\n", st->fail);  in fail_show()
2882 		if (sp->name) {  in states_show()
2883 			cur = sprintf(buf, "%3d: %s\n", i, sp->name);  in states_show()
2921 		return -EPERM;  in __store_smt_control()
2924 		return -ENODEV;  in __store_smt_control()
2941 			return -EINVAL;  in __store_smt_control()
2943 		return -EINVAL;  in __store_smt_control()
2969 	return -ENODEV;  in __store_smt_control()
3029 	int ret = -ENODEV;  in cpu_smt_sysfs_init()
3033 		ret = sysfs_create_group(&dev_root->kobj, &cpuhp_smt_attr_group);  in cpu_smt_sysfs_init()
3050 		ret = sysfs_create_group(&dev_root->kobj, &cpuhp_cpu_root_attr_group);  in cpuhp_sysfs_init()
3061 		ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);  in cpuhp_sysfs_init()
3078 /* cpu_bit_bitmap[0] is empty - so we can back into it */