183 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached)  in amd_pstate_get_epp()  argument
190 			epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,  in amd_pstate_get_epp()
197 		ret = cppc_get_epp_perf(cpudata->cpu, &epp);  in amd_pstate_get_epp()
207 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata)  in amd_pstate_get_energy_pref_index()  argument
212 	epp = amd_pstate_get_epp(cpudata, 0);  in amd_pstate_get_energy_pref_index()
236 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,  in pstate_update_perf()  argument
240 		wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));  in pstate_update_perf()
242 		wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,  in pstate_update_perf()
243 			      READ_ONCE(cpudata->cppc_req_cached));  in pstate_update_perf()
248 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,  in amd_pstate_update_perf()  argument
252 	static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,  in amd_pstate_update_perf()
256 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp)  in amd_pstate_set_epp()  argument
262 		u64 value = READ_ONCE(cpudata->cppc_req_cached);  in amd_pstate_set_epp()
266 		WRITE_ONCE(cpudata->cppc_req_cached, value);  in amd_pstate_set_epp()
268 		ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);  in amd_pstate_set_epp()
270 			cpudata->epp_cached = epp;  in amd_pstate_set_epp()
272 		amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U,  in amd_pstate_set_epp()
273 					     cpudata->max_limit_perf, false);  in amd_pstate_set_epp()
276 		ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);  in amd_pstate_set_epp()
281 		cpudata->epp_cached = epp;  in amd_pstate_set_epp()
287 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata,  in amd_pstate_set_energy_pref_index()  argument
294 		epp = cpudata->epp_default;  in amd_pstate_set_energy_pref_index()
299 	if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {  in amd_pstate_set_energy_pref_index()
304 	ret = amd_pstate_set_epp(cpudata, epp);  in amd_pstate_set_energy_pref_index()
369 static int pstate_init_perf(struct amd_cpudata *cpudata)  in pstate_init_perf()  argument
373 	int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,  in pstate_init_perf()
378 	WRITE_ONCE(cpudata->highest_perf, AMD_CPPC_HIGHEST_PERF(cap1));  in pstate_init_perf()
379 	WRITE_ONCE(cpudata->max_limit_perf, AMD_CPPC_HIGHEST_PERF(cap1));  in pstate_init_perf()
380 	WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));  in pstate_init_perf()
381 	WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));  in pstate_init_perf()
382 	WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));  in pstate_init_perf()
383 	WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1));  in pstate_init_perf()
384 	WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1));  in pstate_init_perf()
388 static int cppc_init_perf(struct amd_cpudata *cpudata)  in cppc_init_perf()  argument
392 	int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);  in cppc_init_perf()
396 	WRITE_ONCE(cpudata->highest_perf, cppc_perf.highest_perf);  in cppc_init_perf()
397 	WRITE_ONCE(cpudata->max_limit_perf, cppc_perf.highest_perf);  in cppc_init_perf()
398 	WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);  in cppc_init_perf()
399 	WRITE_ONCE(cpudata->lowest_nonlinear_perf,  in cppc_init_perf()
401 	WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);  in cppc_init_perf()
402 	WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf);  in cppc_init_perf()
403 	WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf);  in cppc_init_perf()
408 	ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf);  in cppc_init_perf()
414 	ret = cppc_set_auto_sel(cpudata->cpu,  in cppc_init_perf()
425 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)  in amd_pstate_init_perf()  argument
427 	return static_call(amd_pstate_init_perf)(cpudata);  in amd_pstate_init_perf()
430 static void cppc_update_perf(struct amd_cpudata *cpudata,  in cppc_update_perf()  argument
440 	cppc_set_perf(cpudata->cpu, &perf_ctrls);  in cppc_update_perf()
443 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)  in amd_pstate_sample()  argument
453 	if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {  in amd_pstate_sample()
460 	cpudata->cur.aperf = aperf;  in amd_pstate_sample()
461 	cpudata->cur.mperf = mperf;  in amd_pstate_sample()
462 	cpudata->cur.tsc =  tsc;  in amd_pstate_sample()
463 	cpudata->cur.aperf -= cpudata->prev.aperf;  in amd_pstate_sample()
464 	cpudata->cur.mperf -= cpudata->prev.mperf;  in amd_pstate_sample()
465 	cpudata->cur.tsc -= cpudata->prev.tsc;  in amd_pstate_sample()
467 	cpudata->prev.aperf = aperf;  in amd_pstate_sample()
468 	cpudata->prev.mperf = mperf;  in amd_pstate_sample()
469 	cpudata->prev.tsc = tsc;  in amd_pstate_sample()
471 	cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);  in amd_pstate_sample()
476 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,  in amd_pstate_update()  argument
480 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu);  in amd_pstate_update()
481 	u64 prev = READ_ONCE(cpudata->cppc_req_cached);  in amd_pstate_update()
482 	u32 nominal_perf = READ_ONCE(cpudata->nominal_perf);  in amd_pstate_update()
485 	min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,  in amd_pstate_update()
486 			cpudata->max_limit_perf);  in amd_pstate_update()
487 	max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,  in amd_pstate_update()
488 			cpudata->max_limit_perf);  in amd_pstate_update()
491 	max_freq = READ_ONCE(cpudata->max_limit_freq);  in amd_pstate_update()
506 	if (!cpudata->boost_supported)  in amd_pstate_update()
512 	if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {  in amd_pstate_update()
513 		trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,  in amd_pstate_update()
514 			cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,  in amd_pstate_update()
515 				cpudata->cpu, (value != prev), fast_switch);  in amd_pstate_update()
521 	WRITE_ONCE(cpudata->cppc_req_cached, value);  in amd_pstate_update()
523 	amd_pstate_update_perf(cpudata, min_perf, des_perf,  in amd_pstate_update()
540 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_update_min_max_limit()  local
542 	if (cpudata->boost_supported && !policy->boost_enabled)  in amd_pstate_update_min_max_limit()
543 		max_perf = READ_ONCE(cpudata->nominal_perf);  in amd_pstate_update_min_max_limit()
545 		max_perf = READ_ONCE(cpudata->highest_perf);  in amd_pstate_update_min_max_limit()
550 	lowest_perf = READ_ONCE(cpudata->lowest_perf);  in amd_pstate_update_min_max_limit()
557 	WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);  in amd_pstate_update_min_max_limit()
558 	WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);  in amd_pstate_update_min_max_limit()
559 	WRITE_ONCE(cpudata->max_limit_freq, policy->max);  in amd_pstate_update_min_max_limit()
560 	WRITE_ONCE(cpudata->min_limit_freq, policy->min);  in amd_pstate_update_min_max_limit()
569 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_update_freq()  local
572 	if (!cpudata->max_freq)  in amd_pstate_update_freq()
575 	if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)  in amd_pstate_update_freq()
578 	cap_perf = READ_ONCE(cpudata->highest_perf);  in amd_pstate_update_freq()
579 	min_perf = READ_ONCE(cpudata->lowest_perf);  in amd_pstate_update_freq()
586 				     cpudata->max_freq);  in amd_pstate_update_freq()
597 	amd_pstate_update(cpudata, min_perf, des_perf,  in amd_pstate_update_freq()
629 	struct amd_cpudata *cpudata;  in amd_pstate_adjust_perf()  local
634 	cpudata = policy->driver_data;  in amd_pstate_adjust_perf()
636 	if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)  in amd_pstate_adjust_perf()
640 	cap_perf = READ_ONCE(cpudata->highest_perf);  in amd_pstate_adjust_perf()
641 	lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);  in amd_pstate_adjust_perf()
647 	min_perf = READ_ONCE(cpudata->lowest_perf);  in amd_pstate_adjust_perf()
660 	amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true,  in amd_pstate_adjust_perf()
667 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_cpu_boost_update()  local
672 	highest_perf = READ_ONCE(cpudata->highest_perf);  in amd_pstate_cpu_boost_update()
673 	nominal_perf = READ_ONCE(cpudata->nominal_perf);  in amd_pstate_cpu_boost_update()
674 	nominal_freq = READ_ONCE(cpudata->nominal_freq);  in amd_pstate_cpu_boost_update()
675 	max_freq = READ_ONCE(cpudata->max_freq);  in amd_pstate_cpu_boost_update()
678 		u64 value = READ_ONCE(cpudata->cppc_req_cached);  in amd_pstate_cpu_boost_update()
682 		WRITE_ONCE(cpudata->cppc_req_cached, value);  in amd_pstate_cpu_boost_update()
684 		wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);  in amd_pstate_cpu_boost_update()
687 		ret = cppc_set_perf(cpudata->cpu, &perf_ctrls);  in amd_pstate_cpu_boost_update()
691 				cpudata->cpu, ret);  in amd_pstate_cpu_boost_update()
704 		ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq);  in amd_pstate_cpu_boost_update()
706 			pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu);  in amd_pstate_cpu_boost_update()
714 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_set_boost()  local
717 	if (!cpudata->boost_supported) {  in amd_pstate_set_boost()
723 	WRITE_ONCE(cpudata->boost_state, !ret ? state : false);  in amd_pstate_set_boost()
731 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata)  in amd_pstate_init_boost_support()  argument
749 	ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val);  in amd_pstate_init_boost_support()
757 		cpudata->boost_supported = true;  in amd_pstate_init_boost_support()
762 	cpudata->boost_supported = false;  in amd_pstate_init_boost_support()
783 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata)  in amd_pstate_init_prefcore()  argument
789 	cpudata->hw_prefcore = true;  in amd_pstate_init_prefcore()
796 	sched_set_itmt_core_prio((int)READ_ONCE(cpudata->highest_perf), cpudata->cpu);  in amd_pstate_init_prefcore()
804 	struct amd_cpudata *cpudata;  in amd_pstate_update_limits()  local
812 	cpudata = policy->driver_data;  in amd_pstate_update_limits()
822 	prev_high = READ_ONCE(cpudata->prefcore_ranking);  in amd_pstate_update_limits()
825 		WRITE_ONCE(cpudata->prefcore_ranking, cur_high);  in amd_pstate_update_limits()
885 static int amd_pstate_init_freq(struct amd_cpudata *cpudata)  in amd_pstate_init_freq()  argument
895 	ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);  in amd_pstate_init_freq()
909 	nominal_perf = READ_ONCE(cpudata->nominal_perf);  in amd_pstate_init_freq()
911 	ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator);  in amd_pstate_init_freq()
917 	lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);  in amd_pstate_init_freq()
922 	WRITE_ONCE(cpudata->min_freq, min_freq);  in amd_pstate_init_freq()
923 	WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq);  in amd_pstate_init_freq()
924 	WRITE_ONCE(cpudata->nominal_freq, nominal_freq);  in amd_pstate_init_freq()
925 	WRITE_ONCE(cpudata->max_freq, max_freq);  in amd_pstate_init_freq()
952 	struct amd_cpudata *cpudata;  in amd_pstate_cpu_init()  local
963 	cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);  in amd_pstate_cpu_init()
964 	if (!cpudata)  in amd_pstate_cpu_init()
967 	cpudata->cpu = policy->cpu;  in amd_pstate_cpu_init()
969 	ret = amd_pstate_init_perf(cpudata);  in amd_pstate_cpu_init()
973 	amd_pstate_init_prefcore(cpudata);  in amd_pstate_cpu_init()
975 	ret = amd_pstate_init_freq(cpudata);  in amd_pstate_cpu_init()
979 	ret = amd_pstate_init_boost_support(cpudata);  in amd_pstate_cpu_init()
983 	min_freq = READ_ONCE(cpudata->min_freq);  in amd_pstate_cpu_init()
984 	max_freq = READ_ONCE(cpudata->max_freq);  in amd_pstate_cpu_init()
995 	policy->boost_enabled = READ_ONCE(cpudata->boost_supported);  in amd_pstate_cpu_init()
1003 	ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],  in amd_pstate_cpu_init()
1010 	ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],  in amd_pstate_cpu_init()
1017 	cpudata->max_limit_freq = max_freq;  in amd_pstate_cpu_init()
1018 	cpudata->min_limit_freq = min_freq;  in amd_pstate_cpu_init()
1020 	policy->driver_data = cpudata;  in amd_pstate_cpu_init()
1028 	freq_qos_remove_request(&cpudata->req[0]);  in amd_pstate_cpu_init()
1030 	kfree(cpudata);  in amd_pstate_cpu_init()
1036 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_cpu_exit()  local
1038 	freq_qos_remove_request(&cpudata->req[1]);  in amd_pstate_cpu_exit()
1039 	freq_qos_remove_request(&cpudata->req[0]);  in amd_pstate_cpu_exit()
1041 	kfree(cpudata);  in amd_pstate_cpu_exit()
1077 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_max_freq()  local
1079 	max_freq = READ_ONCE(cpudata->max_freq);  in show_amd_pstate_max_freq()
1090 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_lowest_nonlinear_freq()  local
1092 	freq = READ_ONCE(cpudata->lowest_nonlinear_freq);  in show_amd_pstate_lowest_nonlinear_freq()
1107 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_highest_perf()  local
1109 	perf = READ_ONCE(cpudata->highest_perf);  in show_amd_pstate_highest_perf()
1118 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_prefcore_ranking()  local
1120 	perf = READ_ONCE(cpudata->prefcore_ranking);  in show_amd_pstate_prefcore_ranking()
1129 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_hw_prefcore()  local
1131 	hw_prefcore = READ_ONCE(cpudata->hw_prefcore);  in show_amd_pstate_hw_prefcore()
1141 	struct amd_cpudata *cpudata = policy->driver_data;  in show_energy_performance_available_preferences()  local
1143 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)  in show_energy_performance_available_preferences()
1158 	struct amd_cpudata *cpudata = policy->driver_data;  in store_energy_performance_preference()  local
1171 	ret = amd_pstate_set_energy_pref_index(cpudata, ret);  in store_energy_performance_preference()
1180 	struct amd_cpudata *cpudata = policy->driver_data;  in show_energy_performance_preference()  local
1183 	preference = amd_pstate_get_energy_pref_index(cpudata);  in show_energy_performance_preference()
1415 	struct amd_cpudata *cpudata;  in amd_pstate_epp_cpu_init()  local
1428 	cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);  in amd_pstate_epp_cpu_init()
1429 	if (!cpudata)  in amd_pstate_epp_cpu_init()
1432 	cpudata->cpu = policy->cpu;  in amd_pstate_epp_cpu_init()
1433 	cpudata->epp_policy = 0;  in amd_pstate_epp_cpu_init()
1435 	ret = amd_pstate_init_perf(cpudata);  in amd_pstate_epp_cpu_init()
1439 	amd_pstate_init_prefcore(cpudata);  in amd_pstate_epp_cpu_init()
1441 	ret = amd_pstate_init_freq(cpudata);  in amd_pstate_epp_cpu_init()
1445 	ret = amd_pstate_init_boost_support(cpudata);  in amd_pstate_epp_cpu_init()
1449 	min_freq = READ_ONCE(cpudata->min_freq);  in amd_pstate_epp_cpu_init()
1450 	max_freq = READ_ONCE(cpudata->max_freq);  in amd_pstate_epp_cpu_init()
1457 	policy->driver_data = cpudata;  in amd_pstate_epp_cpu_init()
1459 	cpudata->epp_cached = cpudata->epp_default = amd_pstate_get_epp(cpudata, 0);  in amd_pstate_epp_cpu_init()
1464 	policy->boost_enabled = READ_ONCE(cpudata->boost_supported);  in amd_pstate_epp_cpu_init()
1477 		ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);  in amd_pstate_epp_cpu_init()
1480 		WRITE_ONCE(cpudata->cppc_req_cached, value);  in amd_pstate_epp_cpu_init()
1482 		ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value);  in amd_pstate_epp_cpu_init()
1485 		WRITE_ONCE(cpudata->cppc_cap1_cached, value);  in amd_pstate_epp_cpu_init()
1491 	kfree(cpudata);  in amd_pstate_epp_cpu_init()
1497 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_cpu_exit()  local
1499 	if (cpudata) {  in amd_pstate_epp_cpu_exit()
1500 		kfree(cpudata);  in amd_pstate_epp_cpu_exit()
1509 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_update_limit()  local
1514 	if (cpudata->boost_supported && !policy->boost_enabled)  in amd_pstate_epp_update_limit()
1515 		max_perf = READ_ONCE(cpudata->nominal_perf);  in amd_pstate_epp_update_limit()
1517 		max_perf = READ_ONCE(cpudata->highest_perf);  in amd_pstate_epp_update_limit()
1518 	min_perf = READ_ONCE(cpudata->lowest_perf);  in amd_pstate_epp_update_limit()
1528 	WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);  in amd_pstate_epp_update_limit()
1529 	WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);  in amd_pstate_epp_update_limit()
1531 	max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,  in amd_pstate_epp_update_limit()
1532 			cpudata->max_limit_perf);  in amd_pstate_epp_update_limit()
1533 	min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,  in amd_pstate_epp_update_limit()
1534 			cpudata->max_limit_perf);  in amd_pstate_epp_update_limit()
1535 	value = READ_ONCE(cpudata->cppc_req_cached);  in amd_pstate_epp_update_limit()
1537 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)  in amd_pstate_epp_update_limit()
1551 	cpudata->epp_policy = cpudata->policy;  in amd_pstate_epp_update_limit()
1554 	epp = amd_pstate_get_epp(cpudata, value);  in amd_pstate_epp_update_limit()
1563 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)  in amd_pstate_epp_update_limit()
1572 	WRITE_ONCE(cpudata->cppc_req_cached, value);  in amd_pstate_epp_update_limit()
1573 	return amd_pstate_set_epp(cpudata, epp);  in amd_pstate_epp_update_limit()
1578 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_set_policy()  local
1587 	cpudata->policy = policy->policy;  in amd_pstate_epp_set_policy()
1602 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata)  in amd_pstate_epp_reenable()  argument
1612 	value = READ_ONCE(cpudata->cppc_req_cached);  in amd_pstate_epp_reenable()
1613 	max_perf = READ_ONCE(cpudata->highest_perf);  in amd_pstate_epp_reenable()
1616 		wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);  in amd_pstate_epp_reenable()
1619 		perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached);  in amd_pstate_epp_reenable()
1620 		cppc_set_perf(cpudata->cpu, &perf_ctrls);  in amd_pstate_epp_reenable()
1626 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_cpu_online()  local
1628 	pr_debug("AMD CPU Core %d going online\n", cpudata->cpu);  in amd_pstate_epp_cpu_online()
1631 		amd_pstate_epp_reenable(cpudata);  in amd_pstate_epp_cpu_online()
1632 		cpudata->suspended = false;  in amd_pstate_epp_cpu_online()
1640 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_offline()  local
1645 	min_perf = READ_ONCE(cpudata->lowest_perf);  in amd_pstate_epp_offline()
1646 	value = READ_ONCE(cpudata->cppc_req_cached);  in amd_pstate_epp_offline()
1650 		cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN;  in amd_pstate_epp_offline()
1657 		wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);  in amd_pstate_epp_offline()
1662 		cppc_set_perf(cpudata->cpu, &perf_ctrls);  in amd_pstate_epp_offline()
1669 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_cpu_offline()  local
1671 	pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu);  in amd_pstate_epp_cpu_offline()
1673 	if (cpudata->suspended)  in amd_pstate_epp_cpu_offline()
1691 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_suspend()  local
1699 	cpudata->suspended = true;  in amd_pstate_epp_suspend()
1711 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_resume()  local
1713 	if (cpudata->suspended) {  in amd_pstate_epp_resume()
1717 		amd_pstate_epp_reenable(cpudata);  in amd_pstate_epp_resume()
1721 		cpudata->suspended = false;  in amd_pstate_epp_resume()