Lines Matching +full:no +full:- +full:tick +full:- +full:in +full:- +full:suspend
1 // SPDX-License-Identifier: GPL-2.0-only
3 * intel_idle.c - native hardware idle loop for modern Intel processors
5 * Copyright (c) 2013 - 2020, Intel Corporation.
12 * in lieu of the legacy ACPI processor_idle driver. The intent is to
23 * for preventing entry into deep C-states
25 * CPU will flush caches as needed when entering a C-state via MWAIT
26 * (in contrast to entering ACPI C3, in which case the WBINVD
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
35 * Have not seen issues with suspend, but may need same workaround here.
39 /* un-comment DEBUG to enable pr_debug() statements */
47 #include <linux/tick.h>
55 #include <asm/intel-family.h>
57 #include <asm/spec-ctrl.h>
67 static int max_cstate = CPUIDLE_STATE_MAX - 1;
87 * Hardware C-state auto-demotion may not always be optimal.
102 * Enable interrupts before entering the C-state. On some platforms and for
103 * some C-states, this may measurably decrease interrupt latency.
119 * Initialize large xstate for the C6-state entrance.
124 * Ignore the sub-state when matching mwait hints between the ACPI _CST and
130 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
131 * the C-state (top nibble) and sub-state (bottom nibble)
143 struct cpuidle_state *state = &drv->states[index]; in __intel_idle()
144 unsigned long eax = flg2MWAIT(state->flags); in __intel_idle()
153 * intel_idle - Ask the processor to enter the given idle state.
161 * If the local APIC timer is not known to be reliable in the target idle state,
162 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
204 * intel_idle_s2idle - Ask the processor to enter the given idle state.
212 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
213 * scheduler tick and suspended scheduler clock on the target CPU.
219 struct cpuidle_state *state = &drv->states[index]; in intel_idle_s2idle()
220 unsigned long eax = flg2MWAIT(state->flags); in intel_idle_s2idle()
222 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) in intel_idle_s2idle()
829 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
830 * But in this case there is effectively no C1, because C1 requests are
832 * and C1E requests end up with C1, so there is effectively no C1E.
1285 * C6, and this is indicated in the CPUID mwait leaf.
1625 unsigned long eax = flg2MWAIT(state->flags); in intel_idle_state_needs_timer_stop()
1631 * Switch over to one-shot tick broadcast if the target C-state in intel_idle_state_needs_timer_stop()
1644 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1651 * intel_idle_cst_usable - Check if the _CST information can be used.
1653 * Check if all of the C-states listed by _CST in the max_cstate range are
1666 if (cx->entry_method != ACPI_CSTATE_FFH) in intel_idle_cst_usable()
1688 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) in intel_idle_acpi_cst_extract()
1719 if (intel_idle_max_cstate_reached(cstate - 1)) in intel_idle_init_cstates_acpi()
1724 state = &drv->states[drv->state_count++]; in intel_idle_init_cstates_acpi()
1726 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); in intel_idle_init_cstates_acpi()
1727 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); in intel_idle_init_cstates_acpi()
1728 state->exit_latency = cx->latency; in intel_idle_init_cstates_acpi()
1730 * For C1-type C-states use the same number for both the exit in intel_idle_init_cstates_acpi()
1732 * C1 in the majority of the static C-states tables above. in intel_idle_init_cstates_acpi()
1733 * For the other types of C-states, however, set the target in intel_idle_init_cstates_acpi()
1735 * a reasonable balance between energy-efficiency and in intel_idle_init_cstates_acpi()
1736 * performance in the majority of interesting cases. in intel_idle_init_cstates_acpi()
1738 state->target_residency = cx->latency; in intel_idle_init_cstates_acpi()
1739 if (cx->type > ACPI_STATE_C1) in intel_idle_init_cstates_acpi()
1740 state->target_residency *= 3; in intel_idle_init_cstates_acpi()
1742 state->flags = MWAIT2flg(cx->address); in intel_idle_init_cstates_acpi()
1743 if (cx->type > ACPI_STATE_C2) in intel_idle_init_cstates_acpi()
1744 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; in intel_idle_init_cstates_acpi()
1747 state->flags |= CPUIDLE_FLAG_OFF; in intel_idle_init_cstates_acpi()
1750 state->flags |= CPUIDLE_FLAG_TIMER_STOP; in intel_idle_init_cstates_acpi()
1752 state->enter = intel_idle; in intel_idle_init_cstates_acpi()
1753 state->enter_s2idle = intel_idle_s2idle; in intel_idle_init_cstates_acpi()
1762 * If there are no _CST C-states, do not disable any C-states by in intel_idle_off_by_default()
1799 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1801 * Tune IVT multi-socket targets.
1806 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ in ivt_idle_state_table_update()
1828 * irtl_2_usec - IRTL to microseconds conversion.
1849 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1897 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1899 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1911 /* if PC10 not present in CPUID.MWAIT.EDX */ in sklh_idle_state_table_update()
1917 /* PC10 is not enabled in PKG C-state limit */ in sklh_idle_state_table_update()
1934 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ in sklh_idle_state_table_update()
1935 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ in sklh_idle_state_table_update()
1939 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1949 * 000b: C0/C1 (no package C-state support) in skx_idle_state_table_update()
1951 * 010b: C6 (non-retention) in skx_idle_state_table_update()
1953 * 111b: No Package C state limits. in skx_idle_state_table_update()
1959 * is disabled in BIOS. This is consistent in skx_idle_state_table_update()
1969 * adl_idle_state_table_update - Adjust AlderLake idle states table.
1988 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
1995 * By default, the C6 state assumes the worst-case scenario of package in spr_idle_state_table_update()
2015 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ in intel_idle_verify_cstate()
2020 mark_tsc_unstable("TSC halts in idle states deeper than C2"); in intel_idle_verify_cstate()
2027 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { in state_update_enter_method()
2032 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); in state_update_enter_method()
2033 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); in state_update_enter_method()
2034 state->enter = intel_idle_xstate; in state_update_enter_method()
2039 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { in state_update_enter_method()
2041 * IBRS mitigation requires that C-states are entered in state_update_enter_method()
2044 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) in state_update_enter_method()
2045 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; in state_update_enter_method()
2046 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); in state_update_enter_method()
2047 state->enter = intel_idle_ibrs; in state_update_enter_method()
2051 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { in state_update_enter_method()
2052 state->enter = intel_idle_irq; in state_update_enter_method()
2058 state->enter = intel_idle_irq; in state_update_enter_method()
2114 drv->states[drv->state_count] = cpuidle_state_table[cstate]; in intel_idle_init_cstates_icpu()
2115 state = &drv->states[drv->state_count]; in intel_idle_init_cstates_icpu()
2120 if ((disabled_states_mask & BIT(drv->state_count)) || in intel_idle_init_cstates_icpu()
2121 ((icpu->use_acpi || force_use_acpi) && in intel_idle_init_cstates_icpu()
2122 intel_idle_off_by_default(state->flags, mwait_hint) && in intel_idle_init_cstates_icpu()
2123 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) in intel_idle_init_cstates_icpu()
2124 state->flags |= CPUIDLE_FLAG_OFF; in intel_idle_init_cstates_icpu()
2127 state->flags |= CPUIDLE_FLAG_TIMER_STOP; in intel_idle_init_cstates_icpu()
2129 drv->state_count++; in intel_idle_init_cstates_icpu()
2132 if (icpu->byt_auto_demotion_disable_flag) { in intel_idle_init_cstates_icpu()
2139 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2147 drv->states[0].flags |= CPUIDLE_FLAG_OFF; in intel_idle_cpuidle_driver_init()
2149 drv->state_count = 1; in intel_idle_cpuidle_driver_init()
2151 if (icpu && icpu->state_table) in intel_idle_cpuidle_driver_init()
2185 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2188 * Register a cpuidle device object for @cpu and update its MSRs in accordance
2196 dev->cpu = cpu; in intel_idle_cpu_init()
2200 return -EIO; in intel_idle_cpu_init()
2224 * driver in this case in intel_idle_cpu_online()
2227 if (!dev->registered) in intel_idle_cpu_online()
2234 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2252 return -ENODEV; in intel_idle_init()
2256 return -EPERM; in intel_idle_init()
2262 pr_debug("Please enable MWAIT in BIOS SETUP\n"); in intel_idle_init()
2263 return -ENODEV; in intel_idle_init()
2268 return -ENODEV; in intel_idle_init()
2272 return -ENODEV; in intel_idle_init()
2279 return -ENODEV; in intel_idle_init()
2283 icpu = (const struct idle_cpu *)id->driver_data; in intel_idle_init()
2285 if (icpu->state_table) in intel_idle_init()
2286 cpuidle_state_table = icpu->state_table; in intel_idle_init()
2288 return -ENODEV; in intel_idle_init()
2290 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; in intel_idle_init()
2291 if (icpu->disable_promotion_to_c1e) in intel_idle_init()
2293 if (icpu->use_acpi || force_use_acpi) in intel_idle_init()
2296 return -ENODEV; in intel_idle_init()
2304 return -ENOMEM; in intel_idle_init()
2312 drv ? drv->name : "none"); in intel_idle_init()
2321 pr_debug("Local APIC timer is reliable in %s\n", in intel_idle_init()
2322 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); in intel_idle_init()
2338 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2339 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2344 * The positions of the bits that are set in this number are the indices of the
2346 * corresponding idle state directories in sysfs, "state0", "state1" ...
2352 * Some platforms come with mutually exclusive C-states, so that if one is
2353 * enabled, the other C-states must not be used. Example: C1 and C1E on
2355 * preferred C-states among the groups of mutually exclusive C-states - the
2356 * selected C-states will be registered, the other C-states from the mutually
2357 * exclusive group won't be registered. If the platform has no mutually
2358 * exclusive C-states, this parameter has no effect.
2363 * Debugging option that forces the driver to enter all C-states with
2364 * interrupts enabled. Does not apply to C-states with