1 // SPDX-License-Identifier: GPL-2.0-or-later
13 #include <asm/asm-offsets.h>
17 #include <asm/mips-cps.h>
20 #include <asm/pm-cps.h>
22 #include <asm/smp-cps.h>
26  * cps_nc_entry_fn - type of a generated non-coherent state entry function
28  * @nc_ready_count: pointer to a non-coherent mapping of the core ready_count
30  * The code entering & exiting non-coherent states is generated at runtime
33  * core-specific code particularly for cache routines. If coupled_coherence
34  * is non-zero and this is the entry function for the CPS_PM_NC_WAIT state,
35  * returns the number of VPEs that were in the wait state at the point this
42  * The entry point of the generated non-coherent idle state entry/exit
43  * functions. Actually per-core rather than per-CPU.
52  * Indicates the number of coupled VPEs ready to operate in a non-coherent
53  * state. Actually per-core rather than per-CPU.
61  * Used to synchronize entry to deep idle states. Actually per-core rather
62  * than per-CPU.
66 /* Saved CPU state across the CPS_PM_POWER_GATED state */
69 /* A somewhat arbitrary number of labels & relocs for uasm */
70 static struct uasm_label labels[32];  variable
73 bool cps_pm_support_state(enum cps_pm_state state)  in cps_pm_support_state()  argument
75 	return test_bit(state, state_support);  in cps_pm_support_state()
104 int cps_pm_enter_state(enum cps_pm_state state)  in cps_pm_enter_state()  argument
116 	/* Check that there is an entry function for this state */  in cps_pm_enter_state()
117 	entry = per_cpu(nc_asm_enter, core)[state];  in cps_pm_enter_state()
119 		return -EINVAL;  in cps_pm_enter_state()
136 	if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) {  in cps_pm_enter_state()
139 			return -EINVAL;  in cps_pm_enter_state()
142 		vpe_cfg = &core_cfg->vpe_config[cpu_vpe_id(&current_cpu_data)];  in cps_pm_enter_state()
143 		vpe_cfg->pc = (unsigned long)mips_cps_pm_restore;  in cps_pm_enter_state()
144 		vpe_cfg->gp = (unsigned long)current_thread_info();  in cps_pm_enter_state()
145 		vpe_cfg->sp = 0;  in cps_pm_enter_state()
152 	/* Create a non-coherent mapping of the core ready_count */  in cps_pm_enter_state()
159 	/* Ensure ready_count is zero-initialised before the assembly runs */  in cps_pm_enter_state()
166 	/* Remove the non-coherent mapping of ready_count */  in cps_pm_enter_state()
173 	 * If this VPE is the first to leave the non-coherent wait state then  in cps_pm_enter_state()
178 	 * idle state.  in cps_pm_enter_state()
180 	if (coupled_coherence && (state == CPS_PM_NC_WAIT) && (left == online))  in cps_pm_enter_state()
191 	unsigned cache_size = cache->ways << cache->waybit;  in cps_gen_cache_routine()
196 	if (cache->flags & MIPS_CACHE_NOT_PRESENT)  in cps_gen_cache_routine()
215 			uasm_i_addiu(pp, GPR_T0, GPR_T0, cache->linesz);  in cps_gen_cache_routine()
217 			uasm_i_cache(pp, op, i * cache->linesz, GPR_T0);  in cps_gen_cache_routine()
223 		uasm_i_addiu(pp, GPR_T0, GPR_T0, unroll_lines * cache->linesz);  in cps_gen_cache_routine()
238 	unsigned line_size = cpu_info->dcache.linesz;  in cps_gen_flush_fsb()
240 	unsigned revision = cpu_info->processor_id & PRID_REV_MASK;  in cps_gen_flush_fsb()
246 	switch (__get_cpu_type(cpu_info->cputype)) {  in cps_gen_flush_fsb()
258 		return -1;  in cps_gen_flush_fsb()
268 	 * stuck in the D3 (ClrBus) state whilst entering a low power state.  in cps_gen_flush_fsb()
293 	 * Invalidate the new D-cache entries so that the cache will need  in cps_gen_flush_fsb()
336 static void *cps_gen_entry_code(unsigned cpu, enum cps_pm_state state)  in cps_gen_entry_code()  argument
338 	struct uasm_label *l = labels;  in cps_gen_entry_code()
366 	/* Clear labels & relocs ready for (re)use */  in cps_gen_entry_code()
367 	memset(labels, 0, sizeof(labels));  in cps_gen_entry_code()
370 	if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) {  in cps_gen_entry_code()
376 		 * Save CPU state. Note the non-standard calling convention  in cps_gen_entry_code()
406 		 * If this is the last VPE to become ready for non-coherence  in cps_gen_entry_code()
412 		if (state < CPS_PM_POWER_GATED) {  in cps_gen_entry_code()
415 			 * for non-coherence. It needs to wait until coherence  in cps_gen_entry_code()
419 			uasm_i_addiu(&p, GPR_T1, GPR_ZERO, -1);  in cps_gen_entry_code()
455 	 * This is the point of no return - this VPE will now proceed to  in cps_gen_entry_code()
492 	if (state >= CPS_PM_CLOCK_GATED) {  in cps_gen_entry_code()
499 		switch (state) {  in cps_gen_entry_code()
516 		if (state == CPS_PM_POWER_GATED) {  in cps_gen_entry_code()
535 	if (state == CPS_PM_NC_WAIT) {  in cps_gen_entry_code()
557 	 * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs  in cps_gen_entry_code()
558 	 * will run this. The first will actually re-enable coherence & the  in cps_gen_entry_code()
572 	if (coupled_coherence && (state == CPS_PM_NC_WAIT)) {  in cps_gen_entry_code()
577 		uasm_i_addiu(&p, GPR_T2, GPR_T1, -1);  in cps_gen_entry_code()
580 		uasm_i_andi(&p, GPR_V0, GPR_T1, (1 << fls(smp_num_siblings)) - 1);  in cps_gen_entry_code()
586 	if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) {  in cps_gen_entry_code()
596 		 * power-up command to the CPC in order to resume operation.  in cps_gen_entry_code()
598 		 * idle state and the one that disables coherence might as well  in cps_gen_entry_code()
599 		 * be the one to re-enable it. The rest will continue from here  in cps_gen_entry_code()
614 	BUG_ON((p - buf) > max_instrs);  in cps_gen_entry_code()
615 	BUG_ON((l - labels) > ARRAY_SIZE(labels));  in cps_gen_entry_code()
616 	BUG_ON((r - relocs) > ARRAY_SIZE(relocs));  in cps_gen_entry_code()
619 	uasm_resolve_relocs(relocs, labels);  in cps_gen_entry_code()
632 	enum cps_pm_state state;  in cps_pm_online_cpu()  local
636 	for (state = CPS_PM_NC_WAIT; state < CPS_PM_STATE_COUNT; state++) {  in cps_pm_online_cpu()
637 		if (per_cpu(nc_asm_enter, core)[state])  in cps_pm_online_cpu()
639 		if (!test_bit(state, state_support))  in cps_pm_online_cpu()
642 		entry_fn = cps_gen_entry_code(cpu, state);  in cps_pm_online_cpu()
644 			pr_err("Failed to generate core %u state %u entry\n",  in cps_pm_online_cpu()
645 			       core, state);  in cps_pm_online_cpu()
646 			clear_bit(state, state_support);  in cps_pm_online_cpu()
649 		per_cpu(nc_asm_enter, core)[state] = entry_fn;  in cps_pm_online_cpu()
656 			return -ENOMEM;  in cps_pm_online_cpu()
675 		 * instead put the cores into clock-off state. In this state  in cps_pm_power_notifier()
683 			pr_warn("JTAG probe is connected - abort suspend\n");  in cps_pm_power_notifier()
694 	/* A CM is required for all non-coherent states */  in cps_pm_init()
696 		pr_warn("pm-cps: no CM, non-coherent states unavailable\n");  in cps_pm_init()
702 	 * non-coherent core then the VPE may end up processing interrupts  in cps_pm_init()
703 	 * whilst non-coherent. That would be bad.  in cps_pm_init()
708 		pr_warn("pm-cps: non-coherent wait unavailable\n");  in cps_pm_init()
716 			pr_warn("pm-cps: CPC does not support clock gating\n");  in cps_pm_init()
722 			pr_warn("pm-cps: CPS SMP not in use, power gating unavailable\n");  in cps_pm_init()
724 		pr_warn("pm-cps: no CPC, clock & power gating unavailable\n");  in cps_pm_init()