1 // SPDX-License-Identifier: GPL-2.0-only
2 //
3 // Copyright (C) 2020-2021 Samuel Holland <samuel@sholland.org>
4 //
5
6 #include <linux/clk.h>
7 #include <linux/devfreq.h>
8 #include <linux/err.h>
9 #include <linux/io.h>
10 #include <linux/iopoll.h>
11 #include <linux/module.h>
12 #include <linux/of.h>
13 #include <linux/platform_device.h>
14 #include <linux/property.h>
15
16 #define MBUS_CR 0x0000
17 #define MBUS_CR_GET_DRAM_TYPE(x) (((x) >> 16) & 0x7)
18 #define MBUS_CR_DRAM_TYPE_DDR2 2
19 #define MBUS_CR_DRAM_TYPE_DDR3 3
20 #define MBUS_CR_DRAM_TYPE_DDR4 4
21 #define MBUS_CR_DRAM_TYPE_LPDDR2 6
22 #define MBUS_CR_DRAM_TYPE_LPDDR3 7
23
24 #define MBUS_TMR 0x000c
25 #define MBUS_TMR_PERIOD(x) ((x) - 1)
26
27 #define MBUS_PMU_CFG 0x009c
28 #define MBUS_PMU_CFG_PERIOD(x) (((x) - 1) << 16)
29 #define MBUS_PMU_CFG_UNIT (0x3 << 1)
30 #define MBUS_PMU_CFG_UNIT_B (0x0 << 1)
31 #define MBUS_PMU_CFG_UNIT_KB (0x1 << 1)
32 #define MBUS_PMU_CFG_UNIT_MB (0x2 << 1)
33 #define MBUS_PMU_CFG_ENABLE (0x1 << 0)
34
35 #define MBUS_PMU_BWCR(n) (0x00a0 + (0x04 * (n)))
36
37 #define MBUS_TOTAL_BWCR MBUS_PMU_BWCR(5)
38 #define MBUS_TOTAL_BWCR_H616 MBUS_PMU_BWCR(13)
39
40 #define MBUS_MDFSCR 0x0100
41 #define MBUS_MDFSCR_BUFFER_TIMING (0x1 << 15)
42 #define MBUS_MDFSCR_PAD_HOLD (0x1 << 13)
43 #define MBUS_MDFSCR_BYPASS (0x1 << 4)
44 #define MBUS_MDFSCR_MODE (0x1 << 1)
45 #define MBUS_MDFSCR_MODE_DFS (0x0 << 1)
46 #define MBUS_MDFSCR_MODE_CFS (0x1 << 1)
47 #define MBUS_MDFSCR_START (0x1 << 0)
48
49 #define MBUS_MDFSMRMR 0x0108
50
51 #define DRAM_PWRCTL 0x0004
52 #define DRAM_PWRCTL_SELFREF_EN (0x1 << 0)
53
54 #define DRAM_RFSHTMG 0x0090
55 #define DRAM_RFSHTMG_TREFI(x) ((x) << 16)
56 #define DRAM_RFSHTMG_TRFC(x) ((x) << 0)
57
58 #define DRAM_VTFCR 0x00b8
59 #define DRAM_VTFCR_VTF_ENABLE (0x3 << 8)
60
61 #define DRAM_ODTMAP 0x0120
62
63 #define DRAM_DX_MAX 4
64
65 #define DRAM_DXnGCR0(n) (0x0344 + 0x80 * (n))
66 #define DRAM_DXnGCR0_DXODT (0x3 << 4)
67 #define DRAM_DXnGCR0_DXODT_DYNAMIC (0x0 << 4)
68 #define DRAM_DXnGCR0_DXODT_ENABLED (0x1 << 4)
69 #define DRAM_DXnGCR0_DXODT_DISABLED (0x2 << 4)
70 #define DRAM_DXnGCR0_DXEN (0x1 << 0)
71
72 struct sun8i_a33_mbus_variant {
73 u32 min_dram_divider;
74 u32 max_dram_divider;
75 u32 odt_freq_mhz;
76 };
77
78 struct sun8i_a33_mbus {
79 const struct sun8i_a33_mbus_variant *variant;
80 void __iomem *reg_dram;
81 void __iomem *reg_mbus;
82 struct clk *clk_bus;
83 struct clk *clk_dram;
84 struct clk *clk_mbus;
85 struct devfreq *devfreq_dram;
86 struct devfreq_simple_ondemand_data gov_data;
87 struct devfreq_dev_profile profile;
88 u32 data_width;
89 u32 nominal_bw;
90 u32 odtmap;
91 u32 tREFI_ns;
92 u32 tRFC_ns;
93 unsigned long freq_table[];
94 };
95
96 /*
97 * The unit for this value is (MBUS clock cycles / MBUS_TMR_PERIOD). When
98 * MBUS_TMR_PERIOD is programmed to match the MBUS clock frequency in MHz, as
99 * it is during DRAM init and during probe, the resulting unit is microseconds.
100 */
101 static int pmu_period = 50000;
102 module_param(pmu_period, int, 0644);
103 MODULE_PARM_DESC(pmu_period, "Bandwidth measurement period (microseconds)");
104
sun8i_a33_mbus_get_peak_bw(struct sun8i_a33_mbus * priv)105 static u32 sun8i_a33_mbus_get_peak_bw(struct sun8i_a33_mbus *priv)
106 {
107 /* Returns the peak transfer (in KiB) during any single PMU period. */
108 return readl_relaxed(priv->reg_mbus + MBUS_TOTAL_BWCR);
109 }
110
sun8i_a33_mbus_restart_pmu_counters(struct sun8i_a33_mbus * priv)111 static void sun8i_a33_mbus_restart_pmu_counters(struct sun8i_a33_mbus *priv)
112 {
113 u32 pmu_cfg = MBUS_PMU_CFG_PERIOD(pmu_period) | MBUS_PMU_CFG_UNIT_KB;
114
115 /* All PMU counters are cleared on a disable->enable transition. */
116 writel_relaxed(pmu_cfg,
117 priv->reg_mbus + MBUS_PMU_CFG);
118 writel_relaxed(pmu_cfg | MBUS_PMU_CFG_ENABLE,
119 priv->reg_mbus + MBUS_PMU_CFG);
120
121 }
122
sun8i_a33_mbus_update_nominal_bw(struct sun8i_a33_mbus * priv,u32 ddr_freq_mhz)123 static void sun8i_a33_mbus_update_nominal_bw(struct sun8i_a33_mbus *priv,
124 u32 ddr_freq_mhz)
125 {
126 /*
127 * Nominal bandwidth (KiB per PMU period):
128 *
129 * DDR transfers microseconds KiB
130 * ------------- * ------------ * --------
131 * microsecond PMU period transfer
132 */
133 priv->nominal_bw = ddr_freq_mhz * pmu_period * priv->data_width / 1024;
134 }
135
sun8i_a33_mbus_set_dram_freq(struct sun8i_a33_mbus * priv,unsigned long freq)136 static int sun8i_a33_mbus_set_dram_freq(struct sun8i_a33_mbus *priv,
137 unsigned long freq)
138 {
139 u32 ddr_freq_mhz = freq / USEC_PER_SEC; /* DDR */
140 u32 dram_freq_mhz = ddr_freq_mhz / 2; /* SDR */
141 u32 mctl_freq_mhz = dram_freq_mhz / 2; /* HDR */
142 u32 dxodt, mdfscr, pwrctl, vtfcr;
143 u32 i, tREFI_32ck, tRFC_ck;
144 int ret;
145
146 /* The rate change is not effective until the MDFS process runs. */
147 ret = clk_set_rate(priv->clk_dram, freq);
148 if (ret)
149 return ret;
150
151 /* Disable automatic self-refesh and VTF before starting MDFS. */
152 pwrctl = readl_relaxed(priv->reg_dram + DRAM_PWRCTL) &
153 ~DRAM_PWRCTL_SELFREF_EN;
154 writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);
155 vtfcr = readl_relaxed(priv->reg_dram + DRAM_VTFCR);
156 writel_relaxed(vtfcr & ~DRAM_VTFCR_VTF_ENABLE,
157 priv->reg_dram + DRAM_VTFCR);
158
159 /* Set up MDFS and enable double buffering for timing registers. */
160 mdfscr = MBUS_MDFSCR_MODE_DFS |
161 MBUS_MDFSCR_BYPASS |
162 MBUS_MDFSCR_PAD_HOLD |
163 MBUS_MDFSCR_BUFFER_TIMING;
164 writel(mdfscr, priv->reg_mbus + MBUS_MDFSCR);
165
166 /* Update the buffered copy of RFSHTMG. */
167 tREFI_32ck = priv->tREFI_ns * mctl_freq_mhz / 1000 / 32;
168 tRFC_ck = DIV_ROUND_UP(priv->tRFC_ns * mctl_freq_mhz, 1000);
169 writel(DRAM_RFSHTMG_TREFI(tREFI_32ck) | DRAM_RFSHTMG_TRFC(tRFC_ck),
170 priv->reg_dram + DRAM_RFSHTMG);
171
172 /* Enable ODT if needed, or disable it to save power. */
173 if (priv->odtmap && dram_freq_mhz > priv->variant->odt_freq_mhz) {
174 dxodt = DRAM_DXnGCR0_DXODT_DYNAMIC;
175 writel(priv->odtmap, priv->reg_dram + DRAM_ODTMAP);
176 } else {
177 dxodt = DRAM_DXnGCR0_DXODT_DISABLED;
178 writel(0, priv->reg_dram + DRAM_ODTMAP);
179 }
180 for (i = 0; i < DRAM_DX_MAX; ++i) {
181 void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);
182
183 writel((readl(reg) & ~DRAM_DXnGCR0_DXODT) | dxodt, reg);
184 }
185
186 dev_dbg(priv->devfreq_dram->dev.parent,
187 "Setting DRAM to %u MHz, tREFI=%u, tRFC=%u, ODT=%s\n",
188 dram_freq_mhz, tREFI_32ck, tRFC_ck,
189 dxodt == DRAM_DXnGCR0_DXODT_DYNAMIC ? "dynamic" : "disabled");
190
191 /* Trigger hardware MDFS. */
192 writel(mdfscr | MBUS_MDFSCR_START, priv->reg_mbus + MBUS_MDFSCR);
193 ret = readl_poll_timeout_atomic(priv->reg_mbus + MBUS_MDFSCR, mdfscr,
194 !(mdfscr & MBUS_MDFSCR_START), 10, 1000);
195 if (ret)
196 return ret;
197
198 /* Disable double buffering. */
199 writel(0, priv->reg_mbus + MBUS_MDFSCR);
200
201 /* Restore VTF configuration. */
202 writel_relaxed(vtfcr, priv->reg_dram + DRAM_VTFCR);
203
204 /* Enable automatic self-refresh at the lowest frequency only. */
205 if (freq == priv->freq_table[0])
206 pwrctl |= DRAM_PWRCTL_SELFREF_EN;
207 writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);
208
209 sun8i_a33_mbus_restart_pmu_counters(priv);
210 sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq_mhz);
211
212 return 0;
213 }
214
sun8i_a33_mbus_set_dram_target(struct device * dev,unsigned long * freq,u32 flags)215 static int sun8i_a33_mbus_set_dram_target(struct device *dev,
216 unsigned long *freq, u32 flags)
217 {
218 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
219 struct devfreq *devfreq = priv->devfreq_dram;
220 struct dev_pm_opp *opp;
221 int ret;
222
223 opp = devfreq_recommended_opp(dev, freq, flags);
224 if (IS_ERR(opp))
225 return PTR_ERR(opp);
226
227 dev_pm_opp_put(opp);
228
229 if (*freq == devfreq->previous_freq)
230 return 0;
231
232 ret = sun8i_a33_mbus_set_dram_freq(priv, *freq);
233 if (ret) {
234 dev_warn(dev, "failed to set DRAM frequency: %d\n", ret);
235 *freq = devfreq->previous_freq;
236 }
237
238 return ret;
239 }
240
sun8i_a33_mbus_get_dram_status(struct device * dev,struct devfreq_dev_status * stat)241 static int sun8i_a33_mbus_get_dram_status(struct device *dev,
242 struct devfreq_dev_status *stat)
243 {
244 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
245
246 stat->busy_time = sun8i_a33_mbus_get_peak_bw(priv);
247 stat->total_time = priv->nominal_bw;
248 stat->current_frequency = priv->devfreq_dram->previous_freq;
249
250 sun8i_a33_mbus_restart_pmu_counters(priv);
251
252 dev_dbg(dev, "Using %lu/%lu (%lu%%) at %lu MHz\n",
253 stat->busy_time, stat->total_time,
254 DIV_ROUND_CLOSEST(stat->busy_time * 100, stat->total_time),
255 stat->current_frequency / USEC_PER_SEC);
256
257 return 0;
258 }
259
sun8i_a33_mbus_hw_init(struct device * dev,struct sun8i_a33_mbus * priv,unsigned long ddr_freq)260 static int sun8i_a33_mbus_hw_init(struct device *dev,
261 struct sun8i_a33_mbus *priv,
262 unsigned long ddr_freq)
263 {
264 u32 i, mbus_cr, mbus_freq_mhz;
265
266 /* Choose tREFI and tRFC to match the configured DRAM type. */
267 mbus_cr = readl_relaxed(priv->reg_mbus + MBUS_CR);
268 switch (MBUS_CR_GET_DRAM_TYPE(mbus_cr)) {
269 case MBUS_CR_DRAM_TYPE_DDR2:
270 case MBUS_CR_DRAM_TYPE_DDR3:
271 case MBUS_CR_DRAM_TYPE_DDR4:
272 priv->tREFI_ns = 7800;
273 priv->tRFC_ns = 350;
274 break;
275 case MBUS_CR_DRAM_TYPE_LPDDR2:
276 case MBUS_CR_DRAM_TYPE_LPDDR3:
277 priv->tREFI_ns = 3900;
278 priv->tRFC_ns = 210;
279 break;
280 default:
281 return -EINVAL;
282 }
283
284 /* Save ODTMAP so it can be restored when raising the frequency. */
285 priv->odtmap = readl_relaxed(priv->reg_dram + DRAM_ODTMAP);
286
287 /* Compute the DRAM data bus width by counting enabled DATx8 blocks. */
288 for (i = 0; i < DRAM_DX_MAX; ++i) {
289 void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);
290
291 if (!(readl_relaxed(reg) & DRAM_DXnGCR0_DXEN))
292 break;
293 }
294 priv->data_width = i;
295
296 dev_dbg(dev, "Detected %u-bit %sDDRx with%s ODT\n",
297 priv->data_width * 8,
298 MBUS_CR_GET_DRAM_TYPE(mbus_cr) > 4 ? "LP" : "",
299 priv->odtmap ? "" : "out");
300
301 /* Program MBUS_TMR such that the PMU period unit is microseconds. */
302 mbus_freq_mhz = clk_get_rate(priv->clk_mbus) / USEC_PER_SEC;
303 writel_relaxed(MBUS_TMR_PERIOD(mbus_freq_mhz),
304 priv->reg_mbus + MBUS_TMR);
305
306 /* "Master Ready Mask Register" bits must be set or MDFS will block. */
307 writel_relaxed(0xffffffff, priv->reg_mbus + MBUS_MDFSMRMR);
308
309 sun8i_a33_mbus_restart_pmu_counters(priv);
310 sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq / USEC_PER_SEC);
311
312 return 0;
313 }
314
sun8i_a33_mbus_suspend(struct device * dev)315 static int __maybe_unused sun8i_a33_mbus_suspend(struct device *dev)
316 {
317 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
318
319 clk_disable_unprepare(priv->clk_bus);
320
321 return 0;
322 }
323
sun8i_a33_mbus_resume(struct device * dev)324 static int __maybe_unused sun8i_a33_mbus_resume(struct device *dev)
325 {
326 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
327
328 return clk_prepare_enable(priv->clk_bus);
329 }
330
sun8i_a33_mbus_probe(struct platform_device * pdev)331 static int sun8i_a33_mbus_probe(struct platform_device *pdev)
332 {
333 const struct sun8i_a33_mbus_variant *variant;
334 struct device *dev = &pdev->dev;
335 struct sun8i_a33_mbus *priv;
336 unsigned long base_freq;
337 unsigned int max_state;
338 const char *err;
339 int i, ret;
340
341 variant = device_get_match_data(dev);
342 if (!variant)
343 return -EINVAL;
344
345 max_state = variant->max_dram_divider - variant->min_dram_divider + 1;
346
347 priv = devm_kzalloc(dev, struct_size(priv, freq_table, max_state), GFP_KERNEL);
348 if (!priv)
349 return -ENOMEM;
350
351 platform_set_drvdata(pdev, priv);
352
353 priv->variant = variant;
354
355 priv->reg_dram = devm_platform_ioremap_resource_byname(pdev, "dram");
356 if (IS_ERR(priv->reg_dram))
357 return PTR_ERR(priv->reg_dram);
358
359 priv->reg_mbus = devm_platform_ioremap_resource_byname(pdev, "mbus");
360 if (IS_ERR(priv->reg_mbus))
361 return PTR_ERR(priv->reg_mbus);
362
363 priv->clk_bus = devm_clk_get(dev, "bus");
364 if (IS_ERR(priv->clk_bus))
365 return dev_err_probe(dev, PTR_ERR(priv->clk_bus),
366 "failed to get bus clock\n");
367
368 priv->clk_dram = devm_clk_get(dev, "dram");
369 if (IS_ERR(priv->clk_dram))
370 return dev_err_probe(dev, PTR_ERR(priv->clk_dram),
371 "failed to get dram clock\n");
372
373 priv->clk_mbus = devm_clk_get(dev, "mbus");
374 if (IS_ERR(priv->clk_mbus))
375 return dev_err_probe(dev, PTR_ERR(priv->clk_mbus),
376 "failed to get mbus clock\n");
377
378 ret = clk_prepare_enable(priv->clk_bus);
379 if (ret)
380 return dev_err_probe(dev, ret,
381 "failed to enable bus clock\n");
382
383 /* Lock the DRAM clock rate to keep priv->nominal_bw in sync. */
384 ret = clk_rate_exclusive_get(priv->clk_dram);
385 if (ret) {
386 err = "failed to lock dram clock rate\n";
387 goto err_disable_bus;
388 }
389
390 /* Lock the MBUS clock rate to keep MBUS_TMR_PERIOD in sync. */
391 ret = clk_rate_exclusive_get(priv->clk_mbus);
392 if (ret) {
393 err = "failed to lock mbus clock rate\n";
394 goto err_unlock_dram;
395 }
396
397 priv->gov_data.upthreshold = 10;
398 priv->gov_data.downdifferential = 5;
399
400 priv->profile.initial_freq = clk_get_rate(priv->clk_dram);
401 priv->profile.polling_ms = 1000;
402 priv->profile.target = sun8i_a33_mbus_set_dram_target;
403 priv->profile.get_dev_status = sun8i_a33_mbus_get_dram_status;
404 priv->profile.freq_table = priv->freq_table;
405 priv->profile.max_state = max_state;
406
407 ret = devm_pm_opp_set_clkname(dev, "dram");
408 if (ret) {
409 err = "failed to add OPP table\n";
410 goto err_unlock_mbus;
411 }
412
413 base_freq = clk_get_rate(clk_get_parent(priv->clk_dram));
414 for (i = 0; i < max_state; ++i) {
415 unsigned int div = variant->max_dram_divider - i;
416
417 priv->freq_table[i] = base_freq / div;
418
419 ret = dev_pm_opp_add(dev, priv->freq_table[i], 0);
420 if (ret) {
421 err = "failed to add OPPs\n";
422 goto err_remove_opps;
423 }
424 }
425
426 ret = sun8i_a33_mbus_hw_init(dev, priv, priv->profile.initial_freq);
427 if (ret) {
428 err = "failed to init hardware\n";
429 goto err_remove_opps;
430 }
431
432 priv->devfreq_dram = devfreq_add_device(dev, &priv->profile,
433 DEVFREQ_GOV_SIMPLE_ONDEMAND,
434 &priv->gov_data);
435 if (IS_ERR(priv->devfreq_dram)) {
436 ret = PTR_ERR(priv->devfreq_dram);
437 err = "failed to add devfreq device\n";
438 goto err_remove_opps;
439 }
440
441 /*
442 * This must be set manually after registering the devfreq device,
443 * because there is no way to select a dynamic OPP as the suspend OPP.
444 */
445 priv->devfreq_dram->suspend_freq = priv->freq_table[0];
446
447 return 0;
448
449 err_remove_opps:
450 dev_pm_opp_remove_all_dynamic(dev);
451 err_unlock_mbus:
452 clk_rate_exclusive_put(priv->clk_mbus);
453 err_unlock_dram:
454 clk_rate_exclusive_put(priv->clk_dram);
455 err_disable_bus:
456 clk_disable_unprepare(priv->clk_bus);
457
458 return dev_err_probe(dev, ret, err);
459 }
460
sun8i_a33_mbus_remove(struct platform_device * pdev)461 static void sun8i_a33_mbus_remove(struct platform_device *pdev)
462 {
463 struct sun8i_a33_mbus *priv = platform_get_drvdata(pdev);
464 unsigned long initial_freq = priv->profile.initial_freq;
465 struct device *dev = &pdev->dev;
466 int ret;
467
468 devfreq_remove_device(priv->devfreq_dram);
469
470 ret = sun8i_a33_mbus_set_dram_freq(priv, initial_freq);
471 if (ret)
472 dev_warn(dev, "failed to restore DRAM frequency: %d\n", ret);
473
474 dev_pm_opp_remove_all_dynamic(dev);
475 clk_rate_exclusive_put(priv->clk_mbus);
476 clk_rate_exclusive_put(priv->clk_dram);
477 clk_disable_unprepare(priv->clk_bus);
478 }
479
480 static const struct sun8i_a33_mbus_variant sun50i_a64_mbus = {
481 .min_dram_divider = 1,
482 .max_dram_divider = 4,
483 .odt_freq_mhz = 400,
484 };
485
486 static const struct of_device_id sun8i_a33_mbus_of_match[] = {
487 { .compatible = "allwinner,sun50i-a64-mbus", .data = &sun50i_a64_mbus },
488 { .compatible = "allwinner,sun50i-h5-mbus", .data = &sun50i_a64_mbus },
489 { },
490 };
491 MODULE_DEVICE_TABLE(of, sun8i_a33_mbus_of_match);
492
493 static SIMPLE_DEV_PM_OPS(sun8i_a33_mbus_pm_ops,
494 sun8i_a33_mbus_suspend, sun8i_a33_mbus_resume);
495
496 static struct platform_driver sun8i_a33_mbus_driver = {
497 .probe = sun8i_a33_mbus_probe,
498 .remove_new = sun8i_a33_mbus_remove,
499 .driver = {
500 .name = "sun8i-a33-mbus",
501 .of_match_table = sun8i_a33_mbus_of_match,
502 .pm = pm_ptr(&sun8i_a33_mbus_pm_ops),
503 },
504 };
505 module_platform_driver(sun8i_a33_mbus_driver);
506
507 MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>");
508 MODULE_DESCRIPTION("Allwinner sun8i/sun50i MBUS DEVFREQ Driver");
509 MODULE_LICENSE("GPL v2");
510