1  // SPDX-License-Identifier: GPL-2.0-only
2  /*
3   * Apple SoC CPU cluster performance state driver
4   *
5   * Copyright The Asahi Linux Contributors
6   *
7   * Based on scpi-cpufreq.c
8   */
9  
10  #include <linux/bitfield.h>
11  #include <linux/bitops.h>
12  #include <linux/cpu.h>
13  #include <linux/cpufreq.h>
14  #include <linux/cpumask.h>
15  #include <linux/delay.h>
16  #include <linux/err.h>
17  #include <linux/io.h>
18  #include <linux/iopoll.h>
19  #include <linux/module.h>
20  #include <linux/of.h>
21  #include <linux/of_address.h>
22  #include <linux/pm_opp.h>
23  #include <linux/slab.h>
24  
25  #define APPLE_DVFS_CMD			0x20
26  #define APPLE_DVFS_CMD_BUSY		BIT(31)
27  #define APPLE_DVFS_CMD_SET		BIT(25)
28  #define APPLE_DVFS_CMD_PS2		GENMASK(16, 12)
29  #define APPLE_DVFS_CMD_PS1		GENMASK(4, 0)
30  
31  /* Same timebase as CPU counter (24MHz) */
32  #define APPLE_DVFS_LAST_CHG_TIME	0x38
33  
34  /*
35   * Apple ran out of bits and had to shift this in T8112...
36   */
37  #define APPLE_DVFS_STATUS			0x50
38  #define APPLE_DVFS_STATUS_CUR_PS_T8103		GENMASK(7, 4)
39  #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103	4
40  #define APPLE_DVFS_STATUS_TGT_PS_T8103		GENMASK(3, 0)
41  #define APPLE_DVFS_STATUS_CUR_PS_T8112		GENMASK(9, 5)
42  #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112	5
43  #define APPLE_DVFS_STATUS_TGT_PS_T8112		GENMASK(4, 0)
44  
45  /*
46   * Div is +1, base clock is 12MHz on existing SoCs.
47   * For documentation purposes. We use the OPP table to
48   * get the frequency.
49   */
50  #define APPLE_DVFS_PLL_STATUS		0xc0
51  #define APPLE_DVFS_PLL_FACTOR		0xc8
52  #define APPLE_DVFS_PLL_FACTOR_MULT	GENMASK(31, 16)
53  #define APPLE_DVFS_PLL_FACTOR_DIV	GENMASK(15, 0)
54  
55  #define APPLE_DVFS_TRANSITION_TIMEOUT 100
56  
57  struct apple_soc_cpufreq_info {
58  	u64 max_pstate;
59  	u64 cur_pstate_mask;
60  	u64 cur_pstate_shift;
61  };
62  
63  struct apple_cpu_priv {
64  	struct device *cpu_dev;
65  	void __iomem *reg_base;
66  	const struct apple_soc_cpufreq_info *info;
67  };
68  
69  static struct cpufreq_driver apple_soc_cpufreq_driver;
70  
71  static const struct apple_soc_cpufreq_info soc_t8103_info = {
72  	.max_pstate = 15,
73  	.cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,
74  	.cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,
75  };
76  
77  static const struct apple_soc_cpufreq_info soc_t8112_info = {
78  	.max_pstate = 31,
79  	.cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,
80  	.cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,
81  };
82  
83  static const struct apple_soc_cpufreq_info soc_default_info = {
84  	.max_pstate = 15,
85  	.cur_pstate_mask = 0, /* fallback */
86  };
87  
88  static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = {
89  	{
90  		.compatible = "apple,t8103-cluster-cpufreq",
91  		.data = &soc_t8103_info,
92  	},
93  	{
94  		.compatible = "apple,t8112-cluster-cpufreq",
95  		.data = &soc_t8112_info,
96  	},
97  	{
98  		.compatible = "apple,cluster-cpufreq",
99  		.data = &soc_default_info,
100  	},
101  	{}
102  };
103  
apple_soc_cpufreq_get_rate(unsigned int cpu)104  static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)
105  {
106  	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
107  	struct apple_cpu_priv *priv = policy->driver_data;
108  	struct cpufreq_frequency_table *p;
109  	unsigned int pstate;
110  
111  	if (priv->info->cur_pstate_mask) {
112  		u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS);
113  
114  		pstate = (reg & priv->info->cur_pstate_mask) >>  priv->info->cur_pstate_shift;
115  	} else {
116  		/*
117  		 * For the fallback case we might not know the layout of DVFS_STATUS,
118  		 * so just use the command register value (which ignores boost limitations).
119  		 */
120  		u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);
121  
122  		pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);
123  	}
124  
125  	cpufreq_for_each_valid_entry(p, policy->freq_table)
126  		if (p->driver_data == pstate)
127  			return p->frequency;
128  
129  	dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",
130  		pstate);
131  	return 0;
132  }
133  
apple_soc_cpufreq_set_target(struct cpufreq_policy * policy,unsigned int index)134  static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
135  					unsigned int index)
136  {
137  	struct apple_cpu_priv *priv = policy->driver_data;
138  	unsigned int pstate = policy->freq_table[index].driver_data;
139  	u64 reg;
140  
141  	/* Fallback for newer SoCs */
142  	if (index > priv->info->max_pstate)
143  		index = priv->info->max_pstate;
144  
145  	if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,
146  				      !(reg & APPLE_DVFS_CMD_BUSY), 2,
147  				      APPLE_DVFS_TRANSITION_TIMEOUT)) {
148  		return -EIO;
149  	}
150  
151  	reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2);
152  	reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate);
153  	reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
154  	reg |= APPLE_DVFS_CMD_SET;
155  
156  	writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
157  
158  	return 0;
159  }
160  
apple_soc_cpufreq_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)161  static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,
162  						  unsigned int target_freq)
163  {
164  	if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)
165  		return 0;
166  
167  	return policy->freq_table[policy->cached_resolved_idx].frequency;
168  }
169  
apple_soc_cpufreq_find_cluster(struct cpufreq_policy * policy,void __iomem ** reg_base,const struct apple_soc_cpufreq_info ** info)170  static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,
171  					  void __iomem **reg_base,
172  					  const struct apple_soc_cpufreq_info **info)
173  {
174  	struct of_phandle_args args;
175  	const struct of_device_id *match;
176  	int ret = 0;
177  
178  	ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
179  						 "#performance-domain-cells",
180  						 policy->cpus, &args);
181  	if (ret < 0)
182  		return ret;
183  
184  	match = of_match_node(apple_soc_cpufreq_of_match, args.np);
185  	of_node_put(args.np);
186  	if (!match)
187  		return -ENODEV;
188  
189  	*info = match->data;
190  
191  	*reg_base = of_iomap(args.np, 0);
192  	if (!*reg_base)
193  		return -ENOMEM;
194  
195  	return 0;
196  }
197  
198  static struct freq_attr *apple_soc_cpufreq_hw_attr[] = {
199  	&cpufreq_freq_attr_scaling_available_freqs,
200  	NULL, /* Filled in below if boost is enabled */
201  	NULL,
202  };
203  
apple_soc_cpufreq_init(struct cpufreq_policy * policy)204  static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
205  {
206  	int ret, i;
207  	unsigned int transition_latency;
208  	void __iomem *reg_base;
209  	struct device *cpu_dev;
210  	struct apple_cpu_priv *priv;
211  	const struct apple_soc_cpufreq_info *info;
212  	struct cpufreq_frequency_table *freq_table;
213  
214  	cpu_dev = get_cpu_device(policy->cpu);
215  	if (!cpu_dev) {
216  		pr_err("failed to get cpu%d device\n", policy->cpu);
217  		return -ENODEV;
218  	}
219  
220  	ret = dev_pm_opp_of_add_table(cpu_dev);
221  	if (ret < 0) {
222  		dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
223  		return ret;
224  	}
225  
226  	ret = apple_soc_cpufreq_find_cluster(policy, &reg_base, &info);
227  	if (ret) {
228  		dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
229  		return ret;
230  	}
231  
232  	ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
233  	if (ret) {
234  		dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
235  		goto out_iounmap;
236  	}
237  
238  	ret = dev_pm_opp_get_opp_count(cpu_dev);
239  	if (ret <= 0) {
240  		dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
241  		ret = -EPROBE_DEFER;
242  		goto out_free_opp;
243  	}
244  
245  	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
246  	if (!priv) {
247  		ret = -ENOMEM;
248  		goto out_free_opp;
249  	}
250  
251  	ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
252  	if (ret) {
253  		dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
254  		goto out_free_priv;
255  	}
256  
257  	/* Get OPP levels (p-state indexes) and stash them in driver_data */
258  	for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
259  		unsigned long rate = freq_table[i].frequency * 1000 + 999;
260  		struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);
261  
262  		if (IS_ERR(opp)) {
263  			ret = PTR_ERR(opp);
264  			goto out_free_cpufreq_table;
265  		}
266  		freq_table[i].driver_data = dev_pm_opp_get_level(opp);
267  		dev_pm_opp_put(opp);
268  	}
269  
270  	priv->cpu_dev = cpu_dev;
271  	priv->reg_base = reg_base;
272  	priv->info = info;
273  	policy->driver_data = priv;
274  	policy->freq_table = freq_table;
275  
276  	transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
277  	if (!transition_latency)
278  		transition_latency = CPUFREQ_ETERNAL;
279  
280  	policy->cpuinfo.transition_latency = transition_latency;
281  	policy->dvfs_possible_from_any_cpu = true;
282  	policy->fast_switch_possible = true;
283  	policy->suspend_freq = freq_table[0].frequency;
284  
285  	if (policy_has_boost_freq(policy)) {
286  		ret = cpufreq_enable_boost_support();
287  		if (ret) {
288  			dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);
289  		} else {
290  			apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;
291  			apple_soc_cpufreq_driver.boost_enabled = true;
292  		}
293  	}
294  
295  	return 0;
296  
297  out_free_cpufreq_table:
298  	dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
299  out_free_priv:
300  	kfree(priv);
301  out_free_opp:
302  	dev_pm_opp_remove_all_dynamic(cpu_dev);
303  out_iounmap:
304  	iounmap(reg_base);
305  	return ret;
306  }
307  
apple_soc_cpufreq_exit(struct cpufreq_policy * policy)308  static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
309  {
310  	struct apple_cpu_priv *priv = policy->driver_data;
311  
312  	dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
313  	dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
314  	iounmap(priv->reg_base);
315  	kfree(priv);
316  }
317  
318  static struct cpufreq_driver apple_soc_cpufreq_driver = {
319  	.name		= "apple-cpufreq",
320  	.flags		= CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
321  			  CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,
322  	.verify		= cpufreq_generic_frequency_table_verify,
323  	.get		= apple_soc_cpufreq_get_rate,
324  	.init		= apple_soc_cpufreq_init,
325  	.exit		= apple_soc_cpufreq_exit,
326  	.target_index	= apple_soc_cpufreq_set_target,
327  	.fast_switch	= apple_soc_cpufreq_fast_switch,
328  	.register_em	= cpufreq_register_em_with_opp,
329  	.attr		= apple_soc_cpufreq_hw_attr,
330  	.suspend	= cpufreq_generic_suspend,
331  };
332  
apple_soc_cpufreq_module_init(void)333  static int __init apple_soc_cpufreq_module_init(void)
334  {
335  	if (!of_machine_is_compatible("apple,arm-platform"))
336  		return -ENODEV;
337  
338  	return cpufreq_register_driver(&apple_soc_cpufreq_driver);
339  }
340  module_init(apple_soc_cpufreq_module_init);
341  
apple_soc_cpufreq_module_exit(void)342  static void __exit apple_soc_cpufreq_module_exit(void)
343  {
344  	cpufreq_unregister_driver(&apple_soc_cpufreq_driver);
345  }
346  module_exit(apple_soc_cpufreq_module_exit);
347  
348  MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);
349  MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
350  MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");
351  MODULE_LICENSE("GPL");
352