1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2021-2022 Linaro Ltd
5 * Author: Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>, based on
6 * previous work of Thara Gopinath and msm-4.9 downstream sources.
7 */
8
9 #include <linux/err.h>
10 #include <linux/interconnect.h>
11 #include <linux/interrupt.h>
12 #include <linux/io.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/of.h>
16 #include <linux/platform_device.h>
17 #include <linux/pm_opp.h>
18 #include <linux/regmap.h>
19 #include <linux/sizes.h>
20 #define CREATE_TRACE_POINTS
21 #include "trace_icc-bwmon.h"
22
23 /*
24 * The BWMON samples data throughput within 'sample_ms' time. With three
25 * configurable thresholds (Low, Medium and High) gives four windows (called
26 * zones) of current bandwidth:
27 *
28 * Zone 0: byte count < THRES_LO
29 * Zone 1: THRES_LO < byte count < THRES_MED
30 * Zone 2: THRES_MED < byte count < THRES_HIGH
31 * Zone 3: THRES_HIGH < byte count
32 *
33 * Zones 0 and 2 are not used by this driver.
34 */
35
36 /* Internal sampling clock frequency */
37 #define HW_TIMER_HZ 19200000
38
39 #define BWMON_V4_GLOBAL_IRQ_CLEAR 0x108
40 #define BWMON_V4_GLOBAL_IRQ_ENABLE 0x10c
41 /*
42 * All values here and further are matching regmap fields, so without absolute
43 * register offsets.
44 */
45 #define BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE BIT(0)
46
47 /*
48 * Starting with SDM845, the BWMON4 register space has changed a bit:
49 * the global registers were jammed into the beginning of the monitor region.
50 * To keep the proper offsets, one would have to map <GLOBAL_BASE 0x200> and
51 * <GLOBAL_BASE+0x100 0x300>, which is straight up wrong.
52 * To facilitate for that, while allowing the older, arguably more proper
53 * implementations to work, offset the global registers by -0x100 to avoid
54 * having to map half of the global registers twice.
55 */
56 #define BWMON_V4_845_OFFSET 0x100
57 #define BWMON_V4_GLOBAL_IRQ_CLEAR_845 (BWMON_V4_GLOBAL_IRQ_CLEAR - BWMON_V4_845_OFFSET)
58 #define BWMON_V4_GLOBAL_IRQ_ENABLE_845 (BWMON_V4_GLOBAL_IRQ_ENABLE - BWMON_V4_845_OFFSET)
59
60 #define BWMON_V4_IRQ_STATUS 0x100
61 #define BWMON_V4_IRQ_CLEAR 0x108
62
63 #define BWMON_V4_IRQ_ENABLE 0x10c
64 #define BWMON_IRQ_ENABLE_MASK (BIT(1) | BIT(3))
65 #define BWMON_V5_IRQ_STATUS 0x000
66 #define BWMON_V5_IRQ_CLEAR 0x008
67 #define BWMON_V5_IRQ_ENABLE 0x00c
68
69 #define BWMON_V4_ENABLE 0x2a0
70 #define BWMON_V5_ENABLE 0x010
71 #define BWMON_ENABLE_ENABLE BIT(0)
72
73 #define BWMON_V4_CLEAR 0x2a4
74 #define BWMON_V5_CLEAR 0x014
75 #define BWMON_CLEAR_CLEAR BIT(0)
76 #define BWMON_CLEAR_CLEAR_ALL BIT(1)
77
78 #define BWMON_V4_SAMPLE_WINDOW 0x2a8
79 #define BWMON_V5_SAMPLE_WINDOW 0x020
80
81 #define BWMON_V4_THRESHOLD_HIGH 0x2ac
82 #define BWMON_V4_THRESHOLD_MED 0x2b0
83 #define BWMON_V4_THRESHOLD_LOW 0x2b4
84 #define BWMON_V5_THRESHOLD_HIGH 0x024
85 #define BWMON_V5_THRESHOLD_MED 0x028
86 #define BWMON_V5_THRESHOLD_LOW 0x02c
87
88 #define BWMON_V4_ZONE_ACTIONS 0x2b8
89 #define BWMON_V5_ZONE_ACTIONS 0x030
90 /*
91 * Actions to perform on some zone 'z' when current zone hits the threshold:
92 * Increment counter of zone 'z'
93 */
94 #define BWMON_ZONE_ACTIONS_INCREMENT(z) (0x2 << ((z) * 2))
95 /* Clear counter of zone 'z' */
96 #define BWMON_ZONE_ACTIONS_CLEAR(z) (0x1 << ((z) * 2))
97
98 /* Zone 0 threshold hit: Clear zone count */
99 #define BWMON_ZONE_ACTIONS_ZONE0 (BWMON_ZONE_ACTIONS_CLEAR(0))
100
101 /* Zone 1 threshold hit: Increment zone count & clear lower zones */
102 #define BWMON_ZONE_ACTIONS_ZONE1 (BWMON_ZONE_ACTIONS_INCREMENT(1) | \
103 BWMON_ZONE_ACTIONS_CLEAR(0))
104
105 /* Zone 2 threshold hit: Increment zone count & clear lower zones */
106 #define BWMON_ZONE_ACTIONS_ZONE2 (BWMON_ZONE_ACTIONS_INCREMENT(2) | \
107 BWMON_ZONE_ACTIONS_CLEAR(1) | \
108 BWMON_ZONE_ACTIONS_CLEAR(0))
109
110 /* Zone 3 threshold hit: Increment zone count & clear lower zones */
111 #define BWMON_ZONE_ACTIONS_ZONE3 (BWMON_ZONE_ACTIONS_INCREMENT(3) | \
112 BWMON_ZONE_ACTIONS_CLEAR(2) | \
113 BWMON_ZONE_ACTIONS_CLEAR(1) | \
114 BWMON_ZONE_ACTIONS_CLEAR(0))
115
116 /*
117 * There is no clear documentation/explanation of BWMON_V4_THRESHOLD_COUNT
118 * register. Based on observations, this is number of times one threshold has to
119 * be reached, to trigger interrupt in given zone.
120 *
121 * 0xff are maximum values meant to ignore the zones 0 and 2.
122 */
123 #define BWMON_V4_THRESHOLD_COUNT 0x2bc
124 #define BWMON_V5_THRESHOLD_COUNT 0x034
125 #define BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT 0xff
126 #define BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT 0xff
127
128 #define BWMON_V4_ZONE_MAX(zone) (0x2e0 + 4 * (zone))
129 #define BWMON_V5_ZONE_MAX(zone) (0x044 + 4 * (zone))
130
131 /* Quirks for specific BWMON types */
132 #define BWMON_HAS_GLOBAL_IRQ BIT(0)
133 #define BWMON_NEEDS_FORCE_CLEAR BIT(1)
134
135 enum bwmon_fields {
136 /* Global region fields, keep them at the top */
137 F_GLOBAL_IRQ_CLEAR,
138 F_GLOBAL_IRQ_ENABLE,
139 F_NUM_GLOBAL_FIELDS,
140
141 /* Monitor region fields */
142 F_IRQ_STATUS = F_NUM_GLOBAL_FIELDS,
143 F_IRQ_CLEAR,
144 F_IRQ_ENABLE,
145 F_ENABLE,
146 F_CLEAR,
147 F_SAMPLE_WINDOW,
148 F_THRESHOLD_HIGH,
149 F_THRESHOLD_MED,
150 F_THRESHOLD_LOW,
151 F_ZONE_ACTIONS_ZONE0,
152 F_ZONE_ACTIONS_ZONE1,
153 F_ZONE_ACTIONS_ZONE2,
154 F_ZONE_ACTIONS_ZONE3,
155 F_THRESHOLD_COUNT_ZONE0,
156 F_THRESHOLD_COUNT_ZONE1,
157 F_THRESHOLD_COUNT_ZONE2,
158 F_THRESHOLD_COUNT_ZONE3,
159 F_ZONE0_MAX,
160 F_ZONE1_MAX,
161 F_ZONE2_MAX,
162 F_ZONE3_MAX,
163
164 F_NUM_FIELDS
165 };
166
167 struct icc_bwmon_data {
168 unsigned int sample_ms;
169 unsigned int count_unit_kb; /* kbytes */
170 u8 zone1_thres_count;
171 u8 zone3_thres_count;
172 unsigned int quirks;
173
174 const struct regmap_config *regmap_cfg;
175 const struct reg_field *regmap_fields;
176
177 const struct regmap_config *global_regmap_cfg;
178 const struct reg_field *global_regmap_fields;
179 };
180
181 struct icc_bwmon {
182 struct device *dev;
183 const struct icc_bwmon_data *data;
184 int irq;
185
186 struct regmap_field *regs[F_NUM_FIELDS];
187 struct regmap_field *global_regs[F_NUM_GLOBAL_FIELDS];
188
189 unsigned int max_bw_kbps;
190 unsigned int min_bw_kbps;
191 unsigned int target_kbps;
192 unsigned int current_kbps;
193 };
194
195 /* BWMON v4 */
196 static const struct reg_field msm8998_bwmon_reg_fields[] = {
197 [F_GLOBAL_IRQ_CLEAR] = {},
198 [F_GLOBAL_IRQ_ENABLE] = {},
199 [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7),
200 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7),
201 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7),
202 /* F_ENABLE covers entire register to disable other features */
203 [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31),
204 [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1),
205 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23),
206 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11),
207 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11),
208 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11),
209 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7),
210 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15),
211 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23),
212 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31),
213 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7),
214 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15),
215 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23),
216 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31),
217 [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11),
218 [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11),
219 [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11),
220 [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11),
221 };
222
223 static const struct regmap_range msm8998_bwmon_reg_noread_ranges[] = {
224 regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR),
225 regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR),
226 };
227
228 static const struct regmap_access_table msm8998_bwmon_reg_read_table = {
229 .no_ranges = msm8998_bwmon_reg_noread_ranges,
230 .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_reg_noread_ranges),
231 };
232
233 static const struct regmap_range msm8998_bwmon_reg_volatile_ranges[] = {
234 regmap_reg_range(BWMON_V4_IRQ_STATUS, BWMON_V4_IRQ_STATUS),
235 regmap_reg_range(BWMON_V4_ZONE_MAX(0), BWMON_V4_ZONE_MAX(3)),
236 };
237
238 static const struct regmap_access_table msm8998_bwmon_reg_volatile_table = {
239 .yes_ranges = msm8998_bwmon_reg_volatile_ranges,
240 .n_yes_ranges = ARRAY_SIZE(msm8998_bwmon_reg_volatile_ranges),
241 };
242
243 static const struct reg_field msm8998_bwmon_global_reg_fields[] = {
244 [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR, 0, 0),
245 [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE, 0, 0),
246 };
247
248 static const struct regmap_range msm8998_bwmon_global_reg_noread_ranges[] = {
249 regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR, BWMON_V4_GLOBAL_IRQ_CLEAR),
250 };
251
252 static const struct regmap_access_table msm8998_bwmon_global_reg_read_table = {
253 .no_ranges = msm8998_bwmon_global_reg_noread_ranges,
254 .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_global_reg_noread_ranges),
255 };
256
257 /*
258 * Fill the cache for non-readable registers only as rest does not really
259 * matter and can be read from the device.
260 */
261 static const struct reg_default msm8998_bwmon_reg_defaults[] = {
262 { BWMON_V4_IRQ_CLEAR, 0x0 },
263 { BWMON_V4_CLEAR, 0x0 },
264 };
265
266 static const struct reg_default msm8998_bwmon_global_reg_defaults[] = {
267 { BWMON_V4_GLOBAL_IRQ_CLEAR, 0x0 },
268 };
269
270 static const struct regmap_config msm8998_bwmon_regmap_cfg = {
271 .reg_bits = 32,
272 .reg_stride = 4,
273 .val_bits = 32,
274 /*
275 * No concurrent access expected - driver has one interrupt handler,
276 * regmap is not shared, no driver or user-space API.
277 */
278 .disable_locking = true,
279 .rd_table = &msm8998_bwmon_reg_read_table,
280 .volatile_table = &msm8998_bwmon_reg_volatile_table,
281 .reg_defaults = msm8998_bwmon_reg_defaults,
282 .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_reg_defaults),
283 /*
284 * Cache is necessary for using regmap fields with non-readable
285 * registers.
286 */
287 .cache_type = REGCACHE_MAPLE,
288 };
289
290 static const struct regmap_config msm8998_bwmon_global_regmap_cfg = {
291 .reg_bits = 32,
292 .reg_stride = 4,
293 .val_bits = 32,
294 /*
295 * No concurrent access expected - driver has one interrupt handler,
296 * regmap is not shared, no driver or user-space API.
297 */
298 .disable_locking = true,
299 .rd_table = &msm8998_bwmon_global_reg_read_table,
300 .reg_defaults = msm8998_bwmon_global_reg_defaults,
301 .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_global_reg_defaults),
302 /*
303 * Cache is necessary for using regmap fields with non-readable
304 * registers.
305 */
306 .cache_type = REGCACHE_MAPLE,
307 };
308
309 static const struct reg_field sdm845_cpu_bwmon_reg_fields[] = {
310 [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0, 0),
311 [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE_845, 0, 0),
312 [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7),
313 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7),
314 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7),
315 /* F_ENABLE covers entire register to disable other features */
316 [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31),
317 [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1),
318 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23),
319 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11),
320 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11),
321 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11),
322 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7),
323 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15),
324 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23),
325 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31),
326 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7),
327 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15),
328 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23),
329 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31),
330 [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11),
331 [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11),
332 [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11),
333 [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11),
334 };
335
336 static const struct regmap_range sdm845_cpu_bwmon_reg_noread_ranges[] = {
337 regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR_845, BWMON_V4_GLOBAL_IRQ_CLEAR_845),
338 regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR),
339 regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR),
340 };
341
342 static const struct regmap_access_table sdm845_cpu_bwmon_reg_read_table = {
343 .no_ranges = sdm845_cpu_bwmon_reg_noread_ranges,
344 .n_no_ranges = ARRAY_SIZE(sdm845_cpu_bwmon_reg_noread_ranges),
345 };
346
347 /*
348 * Fill the cache for non-readable registers only as rest does not really
349 * matter and can be read from the device.
350 */
351 static const struct reg_default sdm845_cpu_bwmon_reg_defaults[] = {
352 { BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0x0 },
353 { BWMON_V4_IRQ_CLEAR, 0x0 },
354 { BWMON_V4_CLEAR, 0x0 },
355 };
356
357 static const struct regmap_config sdm845_cpu_bwmon_regmap_cfg = {
358 .reg_bits = 32,
359 .reg_stride = 4,
360 .val_bits = 32,
361 /*
362 * No concurrent access expected - driver has one interrupt handler,
363 * regmap is not shared, no driver or user-space API.
364 */
365 .disable_locking = true,
366 .rd_table = &sdm845_cpu_bwmon_reg_read_table,
367 .volatile_table = &msm8998_bwmon_reg_volatile_table,
368 .reg_defaults = sdm845_cpu_bwmon_reg_defaults,
369 .num_reg_defaults = ARRAY_SIZE(sdm845_cpu_bwmon_reg_defaults),
370 /*
371 * Cache is necessary for using regmap fields with non-readable
372 * registers.
373 */
374 .cache_type = REGCACHE_MAPLE,
375 };
376
377 /* BWMON v5 */
378 static const struct reg_field sdm845_llcc_bwmon_reg_fields[] = {
379 [F_GLOBAL_IRQ_CLEAR] = {},
380 [F_GLOBAL_IRQ_ENABLE] = {},
381 [F_IRQ_STATUS] = REG_FIELD(BWMON_V5_IRQ_STATUS, 0, 3),
382 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V5_IRQ_CLEAR, 0, 3),
383 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V5_IRQ_ENABLE, 0, 3),
384 /* F_ENABLE covers entire register to disable other features */
385 [F_ENABLE] = REG_FIELD(BWMON_V5_ENABLE, 0, 31),
386 [F_CLEAR] = REG_FIELD(BWMON_V5_CLEAR, 0, 1),
387 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V5_SAMPLE_WINDOW, 0, 19),
388 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V5_THRESHOLD_HIGH, 0, 11),
389 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V5_THRESHOLD_MED, 0, 11),
390 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V5_THRESHOLD_LOW, 0, 11),
391 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 0, 7),
392 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 8, 15),
393 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 16, 23),
394 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 24, 31),
395 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 0, 7),
396 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 8, 15),
397 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 16, 23),
398 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 24, 31),
399 [F_ZONE0_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(0), 0, 11),
400 [F_ZONE1_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(1), 0, 11),
401 [F_ZONE2_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(2), 0, 11),
402 [F_ZONE3_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(3), 0, 11),
403 };
404
405 static const struct regmap_range sdm845_llcc_bwmon_reg_noread_ranges[] = {
406 regmap_reg_range(BWMON_V5_IRQ_CLEAR, BWMON_V5_IRQ_CLEAR),
407 regmap_reg_range(BWMON_V5_CLEAR, BWMON_V5_CLEAR),
408 };
409
410 static const struct regmap_access_table sdm845_llcc_bwmon_reg_read_table = {
411 .no_ranges = sdm845_llcc_bwmon_reg_noread_ranges,
412 .n_no_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_noread_ranges),
413 };
414
415 static const struct regmap_range sdm845_llcc_bwmon_reg_volatile_ranges[] = {
416 regmap_reg_range(BWMON_V5_IRQ_STATUS, BWMON_V5_IRQ_STATUS),
417 regmap_reg_range(BWMON_V5_ZONE_MAX(0), BWMON_V5_ZONE_MAX(3)),
418 };
419
420 static const struct regmap_access_table sdm845_llcc_bwmon_reg_volatile_table = {
421 .yes_ranges = sdm845_llcc_bwmon_reg_volatile_ranges,
422 .n_yes_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_volatile_ranges),
423 };
424
425 /*
426 * Fill the cache for non-readable registers only as rest does not really
427 * matter and can be read from the device.
428 */
429 static const struct reg_default sdm845_llcc_bwmon_reg_defaults[] = {
430 { BWMON_V5_IRQ_CLEAR, 0x0 },
431 { BWMON_V5_CLEAR, 0x0 },
432 };
433
434 static const struct regmap_config sdm845_llcc_bwmon_regmap_cfg = {
435 .reg_bits = 32,
436 .reg_stride = 4,
437 .val_bits = 32,
438 /*
439 * No concurrent access expected - driver has one interrupt handler,
440 * regmap is not shared, no driver or user-space API.
441 */
442 .disable_locking = true,
443 .rd_table = &sdm845_llcc_bwmon_reg_read_table,
444 .volatile_table = &sdm845_llcc_bwmon_reg_volatile_table,
445 .reg_defaults = sdm845_llcc_bwmon_reg_defaults,
446 .num_reg_defaults = ARRAY_SIZE(sdm845_llcc_bwmon_reg_defaults),
447 /*
448 * Cache is necessary for using regmap fields with non-readable
449 * registers.
450 */
451 .cache_type = REGCACHE_MAPLE,
452 };
453
bwmon_clear_counters(struct icc_bwmon * bwmon,bool clear_all)454 static void bwmon_clear_counters(struct icc_bwmon *bwmon, bool clear_all)
455 {
456 unsigned int val = BWMON_CLEAR_CLEAR;
457
458 if (clear_all)
459 val |= BWMON_CLEAR_CLEAR_ALL;
460 /*
461 * Clear counters. The order and barriers are
462 * important. Quoting downstream Qualcomm msm-4.9 tree:
463 *
464 * The counter clear and IRQ clear bits are not in the same 4KB
465 * region. So, we need to make sure the counter clear is completed
466 * before we try to clear the IRQ or do any other counter operations.
467 */
468 regmap_field_force_write(bwmon->regs[F_CLEAR], val);
469 if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR)
470 regmap_field_force_write(bwmon->regs[F_CLEAR], 0);
471 }
472
bwmon_clear_irq(struct icc_bwmon * bwmon)473 static void bwmon_clear_irq(struct icc_bwmon *bwmon)
474 {
475 struct regmap_field *global_irq_clr;
476
477 if (bwmon->data->global_regmap_fields)
478 global_irq_clr = bwmon->global_regs[F_GLOBAL_IRQ_CLEAR];
479 else
480 global_irq_clr = bwmon->regs[F_GLOBAL_IRQ_CLEAR];
481
482 /*
483 * Clear zone and global interrupts. The order and barriers are
484 * important. Quoting downstream Qualcomm msm-4.9 tree:
485 *
486 * Synchronize the local interrupt clear in mon_irq_clear()
487 * with the global interrupt clear here. Otherwise, the CPU
488 * may reorder the two writes and clear the global interrupt
489 * before the local interrupt, causing the global interrupt
490 * to be retriggered by the local interrupt still being high.
491 *
492 * Similarly, because the global registers are in a different
493 * region than the local registers, we need to ensure any register
494 * writes to enable the monitor after this call are ordered with the
495 * clearing here so that local writes don't happen before the
496 * interrupt is cleared.
497 */
498 regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], BWMON_IRQ_ENABLE_MASK);
499 if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR)
500 regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], 0);
501 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
502 regmap_field_force_write(global_irq_clr,
503 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE);
504 }
505
bwmon_disable(struct icc_bwmon * bwmon)506 static void bwmon_disable(struct icc_bwmon *bwmon)
507 {
508 struct regmap_field *global_irq_en;
509
510 if (bwmon->data->global_regmap_fields)
511 global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE];
512 else
513 global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE];
514
515 /* Disable interrupts. Strict ordering, see bwmon_clear_irq(). */
516 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
517 regmap_field_write(global_irq_en, 0x0);
518 regmap_field_write(bwmon->regs[F_IRQ_ENABLE], 0x0);
519
520 /*
521 * Disable bwmon. Must happen before bwmon_clear_irq() to avoid spurious
522 * IRQ.
523 */
524 regmap_field_write(bwmon->regs[F_ENABLE], 0x0);
525 }
526
bwmon_enable(struct icc_bwmon * bwmon,unsigned int irq_enable)527 static void bwmon_enable(struct icc_bwmon *bwmon, unsigned int irq_enable)
528 {
529 struct regmap_field *global_irq_en;
530
531 if (bwmon->data->global_regmap_fields)
532 global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE];
533 else
534 global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE];
535
536 /* Enable interrupts */
537 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
538 regmap_field_write(global_irq_en,
539 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE);
540
541 regmap_field_write(bwmon->regs[F_IRQ_ENABLE], irq_enable);
542
543 /* Enable bwmon */
544 regmap_field_write(bwmon->regs[F_ENABLE], BWMON_ENABLE_ENABLE);
545 }
546
bwmon_kbps_to_count(struct icc_bwmon * bwmon,unsigned int kbps)547 static unsigned int bwmon_kbps_to_count(struct icc_bwmon *bwmon,
548 unsigned int kbps)
549 {
550 return kbps / bwmon->data->count_unit_kb;
551 }
552
bwmon_set_threshold(struct icc_bwmon * bwmon,struct regmap_field * reg,unsigned int kbps)553 static void bwmon_set_threshold(struct icc_bwmon *bwmon,
554 struct regmap_field *reg, unsigned int kbps)
555 {
556 unsigned int thres;
557
558 thres = mult_frac(bwmon_kbps_to_count(bwmon, kbps),
559 bwmon->data->sample_ms, MSEC_PER_SEC);
560 regmap_field_write(reg, thres);
561 }
562
bwmon_start(struct icc_bwmon * bwmon)563 static void bwmon_start(struct icc_bwmon *bwmon)
564 {
565 const struct icc_bwmon_data *data = bwmon->data;
566 u32 bw_low = 0;
567 int window;
568
569 /* No need to check for errors, as this must have succeeded before. */
570 dev_pm_opp_put(dev_pm_opp_find_bw_ceil(bwmon->dev, &bw_low, 0));
571
572 bwmon_clear_counters(bwmon, true);
573
574 window = mult_frac(bwmon->data->sample_ms, HW_TIMER_HZ, MSEC_PER_SEC);
575 /* Maximum sampling window: 0xffffff for v4 and 0xfffff for v5 */
576 regmap_field_write(bwmon->regs[F_SAMPLE_WINDOW], window);
577
578 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH], bw_low);
579 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED], bw_low);
580 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_LOW], 0);
581
582 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE0],
583 BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT);
584 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE1],
585 data->zone1_thres_count);
586 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE2],
587 BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT);
588 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE3],
589 data->zone3_thres_count);
590
591 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE0],
592 BWMON_ZONE_ACTIONS_ZONE0);
593 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE1],
594 BWMON_ZONE_ACTIONS_ZONE1);
595 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE2],
596 BWMON_ZONE_ACTIONS_ZONE2);
597 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE3],
598 BWMON_ZONE_ACTIONS_ZONE3);
599
600 bwmon_clear_irq(bwmon);
601 bwmon_enable(bwmon, BWMON_IRQ_ENABLE_MASK);
602 }
603
bwmon_intr(int irq,void * dev_id)604 static irqreturn_t bwmon_intr(int irq, void *dev_id)
605 {
606 struct icc_bwmon *bwmon = dev_id;
607 unsigned int status, max;
608 int zone;
609
610 if (regmap_field_read(bwmon->regs[F_IRQ_STATUS], &status))
611 return IRQ_NONE;
612
613 status &= BWMON_IRQ_ENABLE_MASK;
614 if (!status) {
615 /*
616 * Only zone 1 and zone 3 interrupts are enabled but zone 2
617 * threshold could be hit and trigger interrupt even if not
618 * enabled.
619 * Such spurious interrupt might come with valuable max count or
620 * not, so solution would be to always check all
621 * BWMON_ZONE_MAX() registers to find the highest value.
622 * Such case is currently ignored.
623 */
624 return IRQ_NONE;
625 }
626
627 bwmon_disable(bwmon);
628
629 zone = get_bitmask_order(status) - 1;
630 /*
631 * Zone max bytes count register returns count units within sampling
632 * window. Downstream kernel for BWMONv4 (called BWMON type 2 in
633 * downstream) always increments the max bytes count by one.
634 */
635 if (regmap_field_read(bwmon->regs[F_ZONE0_MAX + zone], &max))
636 return IRQ_NONE;
637
638 max += 1;
639 max *= bwmon->data->count_unit_kb;
640 bwmon->target_kbps = mult_frac(max, MSEC_PER_SEC, bwmon->data->sample_ms);
641
642 return IRQ_WAKE_THREAD;
643 }
644
bwmon_intr_thread(int irq,void * dev_id)645 static irqreturn_t bwmon_intr_thread(int irq, void *dev_id)
646 {
647 struct icc_bwmon *bwmon = dev_id;
648 unsigned int irq_enable = 0;
649 struct dev_pm_opp *opp, *target_opp;
650 unsigned int bw_kbps, up_kbps, down_kbps, meas_kbps;
651
652 bw_kbps = bwmon->target_kbps;
653 meas_kbps = bwmon->target_kbps;
654
655 target_opp = dev_pm_opp_find_bw_ceil(bwmon->dev, &bw_kbps, 0);
656 if (IS_ERR(target_opp) && PTR_ERR(target_opp) == -ERANGE)
657 target_opp = dev_pm_opp_find_bw_floor(bwmon->dev, &bw_kbps, 0);
658
659 bwmon->target_kbps = bw_kbps;
660
661 bw_kbps--;
662 opp = dev_pm_opp_find_bw_floor(bwmon->dev, &bw_kbps, 0);
663 if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE)
664 down_kbps = bwmon->target_kbps;
665 else
666 down_kbps = bw_kbps;
667
668 up_kbps = bwmon->target_kbps + 1;
669
670 if (bwmon->target_kbps >= bwmon->max_bw_kbps)
671 irq_enable = BIT(1);
672 else if (bwmon->target_kbps <= bwmon->min_bw_kbps)
673 irq_enable = BIT(3);
674 else
675 irq_enable = BWMON_IRQ_ENABLE_MASK;
676
677 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH],
678 up_kbps);
679 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED],
680 down_kbps);
681 bwmon_clear_counters(bwmon, false);
682 bwmon_clear_irq(bwmon);
683 bwmon_enable(bwmon, irq_enable);
684
685 trace_qcom_bwmon_update(dev_name(bwmon->dev), meas_kbps, up_kbps, down_kbps);
686 if (bwmon->target_kbps == bwmon->current_kbps)
687 goto out;
688
689 dev_pm_opp_set_opp(bwmon->dev, target_opp);
690 bwmon->current_kbps = bwmon->target_kbps;
691
692 out:
693 dev_pm_opp_put(target_opp);
694 if (!IS_ERR(opp))
695 dev_pm_opp_put(opp);
696
697 return IRQ_HANDLED;
698 }
699
bwmon_init_regmap(struct platform_device * pdev,struct icc_bwmon * bwmon)700 static int bwmon_init_regmap(struct platform_device *pdev,
701 struct icc_bwmon *bwmon)
702 {
703 struct device *dev = &pdev->dev;
704 void __iomem *base;
705 struct regmap *map;
706 int ret;
707
708 /* Map the monitor base */
709 base = devm_platform_ioremap_resource(pdev, 0);
710 if (IS_ERR(base))
711 return dev_err_probe(dev, PTR_ERR(base),
712 "failed to map bwmon registers\n");
713
714 map = devm_regmap_init_mmio(dev, base, bwmon->data->regmap_cfg);
715 if (IS_ERR(map))
716 return dev_err_probe(dev, PTR_ERR(map),
717 "failed to initialize regmap\n");
718
719 BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_global_reg_fields) != F_NUM_GLOBAL_FIELDS);
720 BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_reg_fields) != F_NUM_FIELDS);
721 BUILD_BUG_ON(ARRAY_SIZE(sdm845_cpu_bwmon_reg_fields) != F_NUM_FIELDS);
722 BUILD_BUG_ON(ARRAY_SIZE(sdm845_llcc_bwmon_reg_fields) != F_NUM_FIELDS);
723
724 ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->regs,
725 bwmon->data->regmap_fields,
726 F_NUM_FIELDS);
727 if (ret)
728 return ret;
729
730 if (bwmon->data->global_regmap_cfg) {
731 /* Map the global base, if separate */
732 base = devm_platform_ioremap_resource(pdev, 1);
733 if (IS_ERR(base))
734 return dev_err_probe(dev, PTR_ERR(base),
735 "failed to map bwmon global registers\n");
736
737 map = devm_regmap_init_mmio(dev, base, bwmon->data->global_regmap_cfg);
738 if (IS_ERR(map))
739 return dev_err_probe(dev, PTR_ERR(map),
740 "failed to initialize global regmap\n");
741
742 ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->global_regs,
743 bwmon->data->global_regmap_fields,
744 F_NUM_GLOBAL_FIELDS);
745 }
746
747 return ret;
748 }
749
bwmon_probe(struct platform_device * pdev)750 static int bwmon_probe(struct platform_device *pdev)
751 {
752 struct device *dev = &pdev->dev;
753 struct dev_pm_opp *opp;
754 struct icc_bwmon *bwmon;
755 int ret;
756
757 bwmon = devm_kzalloc(dev, sizeof(*bwmon), GFP_KERNEL);
758 if (!bwmon)
759 return -ENOMEM;
760
761 bwmon->data = of_device_get_match_data(dev);
762
763 ret = bwmon_init_regmap(pdev, bwmon);
764 if (ret)
765 return ret;
766
767 bwmon->irq = platform_get_irq(pdev, 0);
768 if (bwmon->irq < 0)
769 return bwmon->irq;
770
771 ret = devm_pm_opp_of_add_table(dev);
772 if (ret)
773 return dev_err_probe(dev, ret, "failed to add OPP table\n");
774
775 bwmon->max_bw_kbps = UINT_MAX;
776 opp = dev_pm_opp_find_bw_floor(dev, &bwmon->max_bw_kbps, 0);
777 if (IS_ERR(opp))
778 return dev_err_probe(dev, PTR_ERR(opp), "failed to find max peak bandwidth\n");
779 dev_pm_opp_put(opp);
780
781 bwmon->min_bw_kbps = 0;
782 opp = dev_pm_opp_find_bw_ceil(dev, &bwmon->min_bw_kbps, 0);
783 if (IS_ERR(opp))
784 return dev_err_probe(dev, PTR_ERR(opp), "failed to find min peak bandwidth\n");
785 dev_pm_opp_put(opp);
786
787 bwmon->dev = dev;
788
789 bwmon_disable(bwmon);
790
791 /*
792 * SoCs with multiple cpu-bwmon instances can end up using a shared interrupt
793 * line. Using the devm_ variant might result in the IRQ handler being executed
794 * after bwmon_disable in bwmon_remove()
795 */
796 ret = request_threaded_irq(bwmon->irq, bwmon_intr, bwmon_intr_thread,
797 IRQF_ONESHOT | IRQF_SHARED, dev_name(dev), bwmon);
798 if (ret)
799 return dev_err_probe(dev, ret, "failed to request IRQ\n");
800
801 platform_set_drvdata(pdev, bwmon);
802 bwmon_start(bwmon);
803
804 return 0;
805 }
806
bwmon_remove(struct platform_device * pdev)807 static void bwmon_remove(struct platform_device *pdev)
808 {
809 struct icc_bwmon *bwmon = platform_get_drvdata(pdev);
810
811 bwmon_disable(bwmon);
812 free_irq(bwmon->irq, bwmon);
813 }
814
815 static const struct icc_bwmon_data msm8998_bwmon_data = {
816 .sample_ms = 4,
817 .count_unit_kb = 1024,
818 .zone1_thres_count = 16,
819 .zone3_thres_count = 1,
820 .quirks = BWMON_HAS_GLOBAL_IRQ,
821 .regmap_fields = msm8998_bwmon_reg_fields,
822 .regmap_cfg = &msm8998_bwmon_regmap_cfg,
823 .global_regmap_fields = msm8998_bwmon_global_reg_fields,
824 .global_regmap_cfg = &msm8998_bwmon_global_regmap_cfg,
825 };
826
827 static const struct icc_bwmon_data sdm845_cpu_bwmon_data = {
828 .sample_ms = 4,
829 .count_unit_kb = 64,
830 .zone1_thres_count = 16,
831 .zone3_thres_count = 1,
832 .quirks = BWMON_HAS_GLOBAL_IRQ,
833 .regmap_fields = sdm845_cpu_bwmon_reg_fields,
834 .regmap_cfg = &sdm845_cpu_bwmon_regmap_cfg,
835 };
836
837 static const struct icc_bwmon_data sdm845_llcc_bwmon_data = {
838 .sample_ms = 4,
839 .count_unit_kb = 1024,
840 .zone1_thres_count = 16,
841 .zone3_thres_count = 1,
842 .regmap_fields = sdm845_llcc_bwmon_reg_fields,
843 .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg,
844 };
845
846 static const struct icc_bwmon_data sc7280_llcc_bwmon_data = {
847 .sample_ms = 4,
848 .count_unit_kb = 64,
849 .zone1_thres_count = 16,
850 .zone3_thres_count = 1,
851 .quirks = BWMON_NEEDS_FORCE_CLEAR,
852 .regmap_fields = sdm845_llcc_bwmon_reg_fields,
853 .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg,
854 };
855
856 static const struct of_device_id bwmon_of_match[] = {
857 /* BWMONv4, separate monitor and global register spaces */
858 { .compatible = "qcom,msm8998-bwmon", .data = &msm8998_bwmon_data },
859 /* BWMONv4, unified register space */
860 { .compatible = "qcom,sdm845-bwmon", .data = &sdm845_cpu_bwmon_data },
861 /* BWMONv5 */
862 { .compatible = "qcom,sdm845-llcc-bwmon", .data = &sdm845_llcc_bwmon_data },
863 { .compatible = "qcom,sc7280-llcc-bwmon", .data = &sc7280_llcc_bwmon_data },
864
865 /* Compatibles kept for legacy reasons */
866 { .compatible = "qcom,sc7280-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },
867 { .compatible = "qcom,sc8280xp-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },
868 { .compatible = "qcom,sm8550-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },
869 {}
870 };
871 MODULE_DEVICE_TABLE(of, bwmon_of_match);
872
873 static struct platform_driver bwmon_driver = {
874 .probe = bwmon_probe,
875 .remove_new = bwmon_remove,
876 .driver = {
877 .name = "qcom-bwmon",
878 .of_match_table = bwmon_of_match,
879 },
880 };
881 module_platform_driver(bwmon_driver);
882
883 MODULE_AUTHOR("Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>");
884 MODULE_DESCRIPTION("QCOM BWMON driver");
885 MODULE_LICENSE("GPL");
886