1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Linaro Ltd.
4 *
5 * Author: Stanimir Varbanov <stanimir.varbanov@linaro.org>
6 */
7 #include <linux/clk.h>
8 #include <linux/interconnect.h>
9 #include <linux/iopoll.h>
10 #include <linux/kernel.h>
11 #include <linux/pm_domain.h>
12 #include <linux/pm_opp.h>
13 #include <linux/pm_runtime.h>
14 #include <linux/reset.h>
15 #include <linux/types.h>
16 #include <media/v4l2-mem2mem.h>
17
18 #include "core.h"
19 #include "hfi_parser.h"
20 #include "hfi_venus_io.h"
21 #include "pm_helpers.h"
22 #include "hfi_platform.h"
23
24 static bool legacy_binding;
25
core_clks_get(struct venus_core * core)26 static int core_clks_get(struct venus_core *core)
27 {
28 const struct venus_resources *res = core->res;
29 struct device *dev = core->dev;
30 unsigned int i;
31
32 for (i = 0; i < res->clks_num; i++) {
33 core->clks[i] = devm_clk_get(dev, res->clks[i]);
34 if (IS_ERR(core->clks[i]))
35 return PTR_ERR(core->clks[i]);
36 }
37
38 return 0;
39 }
40
core_clks_enable(struct venus_core * core)41 static int core_clks_enable(struct venus_core *core)
42 {
43 const struct venus_resources *res = core->res;
44 const struct freq_tbl *freq_tbl = core->res->freq_tbl;
45 unsigned int freq_tbl_size = core->res->freq_tbl_size;
46 unsigned long freq;
47 unsigned int i;
48 int ret;
49
50 if (!freq_tbl)
51 return -EINVAL;
52
53 freq = freq_tbl[freq_tbl_size - 1].freq;
54
55 for (i = 0; i < res->clks_num; i++) {
56 if (IS_V6(core)) {
57 ret = clk_set_rate(core->clks[i], freq);
58 if (ret)
59 goto err;
60 }
61
62 ret = clk_prepare_enable(core->clks[i]);
63 if (ret)
64 goto err;
65 }
66
67 return 0;
68 err:
69 while (i--)
70 clk_disable_unprepare(core->clks[i]);
71
72 return ret;
73 }
74
core_clks_disable(struct venus_core * core)75 static void core_clks_disable(struct venus_core *core)
76 {
77 const struct venus_resources *res = core->res;
78 unsigned int i = res->clks_num;
79
80 while (i--)
81 clk_disable_unprepare(core->clks[i]);
82 }
83
core_clks_set_rate(struct venus_core * core,unsigned long freq)84 static int core_clks_set_rate(struct venus_core *core, unsigned long freq)
85 {
86 int ret;
87
88 ret = dev_pm_opp_set_rate(core->dev, freq);
89 if (ret)
90 return ret;
91
92 ret = clk_set_rate(core->vcodec0_clks[0], freq);
93 if (ret)
94 return ret;
95
96 ret = clk_set_rate(core->vcodec1_clks[0], freq);
97 if (ret)
98 return ret;
99
100 return 0;
101 }
102
vcodec_clks_get(struct venus_core * core,struct device * dev,struct clk ** clks,const char * const * id)103 static int vcodec_clks_get(struct venus_core *core, struct device *dev,
104 struct clk **clks, const char * const *id)
105 {
106 const struct venus_resources *res = core->res;
107 unsigned int i;
108
109 for (i = 0; i < res->vcodec_clks_num; i++) {
110 if (!id[i])
111 continue;
112 clks[i] = devm_clk_get(dev, id[i]);
113 if (IS_ERR(clks[i]))
114 return PTR_ERR(clks[i]);
115 }
116
117 return 0;
118 }
119
vcodec_clks_enable(struct venus_core * core,struct clk ** clks)120 static int vcodec_clks_enable(struct venus_core *core, struct clk **clks)
121 {
122 const struct venus_resources *res = core->res;
123 unsigned int i;
124 int ret;
125
126 for (i = 0; i < res->vcodec_clks_num; i++) {
127 ret = clk_prepare_enable(clks[i]);
128 if (ret)
129 goto err;
130 }
131
132 return 0;
133 err:
134 while (i--)
135 clk_disable_unprepare(clks[i]);
136
137 return ret;
138 }
139
vcodec_clks_disable(struct venus_core * core,struct clk ** clks)140 static void vcodec_clks_disable(struct venus_core *core, struct clk **clks)
141 {
142 const struct venus_resources *res = core->res;
143 unsigned int i = res->vcodec_clks_num;
144
145 while (i--)
146 clk_disable_unprepare(clks[i]);
147 }
148
load_per_instance(struct venus_inst * inst)149 static u32 load_per_instance(struct venus_inst *inst)
150 {
151 u32 mbs;
152
153 if (!inst || !(inst->state >= INST_INIT && inst->state < INST_STOP))
154 return 0;
155
156 mbs = (ALIGN(inst->width, 16) / 16) * (ALIGN(inst->height, 16) / 16);
157
158 return mbs * inst->fps;
159 }
160
load_per_type(struct venus_core * core,u32 session_type)161 static u32 load_per_type(struct venus_core *core, u32 session_type)
162 {
163 struct venus_inst *inst = NULL;
164 u32 mbs_per_sec = 0;
165
166 list_for_each_entry(inst, &core->instances, list) {
167 if (inst->session_type != session_type)
168 continue;
169
170 mbs_per_sec += load_per_instance(inst);
171 }
172
173 return mbs_per_sec;
174 }
175
mbs_to_bw(struct venus_inst * inst,u32 mbs,u32 * avg,u32 * peak)176 static void mbs_to_bw(struct venus_inst *inst, u32 mbs, u32 *avg, u32 *peak)
177 {
178 const struct venus_resources *res = inst->core->res;
179 const struct bw_tbl *bw_tbl;
180 unsigned int num_rows, i;
181
182 *avg = 0;
183 *peak = 0;
184
185 if (mbs == 0)
186 return;
187
188 if (inst->session_type == VIDC_SESSION_TYPE_ENC) {
189 num_rows = res->bw_tbl_enc_size;
190 bw_tbl = res->bw_tbl_enc;
191 } else if (inst->session_type == VIDC_SESSION_TYPE_DEC) {
192 num_rows = res->bw_tbl_dec_size;
193 bw_tbl = res->bw_tbl_dec;
194 } else {
195 return;
196 }
197
198 if (!bw_tbl || num_rows == 0)
199 return;
200
201 for (i = 0; i < num_rows; i++) {
202 if (i != 0 && mbs > bw_tbl[i].mbs_per_sec)
203 break;
204
205 if (inst->dpb_fmt & HFI_COLOR_FORMAT_10_BIT_BASE) {
206 *avg = bw_tbl[i].avg_10bit;
207 *peak = bw_tbl[i].peak_10bit;
208 } else {
209 *avg = bw_tbl[i].avg;
210 *peak = bw_tbl[i].peak;
211 }
212 }
213 }
214
load_scale_bw(struct venus_core * core)215 static int load_scale_bw(struct venus_core *core)
216 {
217 struct venus_inst *inst = NULL;
218 u32 mbs_per_sec, avg, peak, total_avg = 0, total_peak = 0;
219
220 list_for_each_entry(inst, &core->instances, list) {
221 mbs_per_sec = load_per_instance(inst);
222 mbs_to_bw(inst, mbs_per_sec, &avg, &peak);
223 total_avg += avg;
224 total_peak += peak;
225 }
226
227 /*
228 * keep minimum bandwidth vote for "video-mem" path,
229 * so that clks can be disabled during vdec_session_release().
230 * Actual bandwidth drop will be done during device supend
231 * so that device can power down without any warnings.
232 */
233
234 if (!total_avg && !total_peak)
235 total_avg = kbps_to_icc(1000);
236
237 dev_dbg(core->dev, VDBGL "total: avg_bw: %u, peak_bw: %u\n",
238 total_avg, total_peak);
239
240 return icc_set_bw(core->video_path, total_avg, total_peak);
241 }
242
load_scale_v1(struct venus_inst * inst)243 static int load_scale_v1(struct venus_inst *inst)
244 {
245 struct venus_core *core = inst->core;
246 const struct freq_tbl *table = core->res->freq_tbl;
247 unsigned int num_rows = core->res->freq_tbl_size;
248 unsigned long freq = table[0].freq;
249 struct device *dev = core->dev;
250 u32 mbs_per_sec;
251 unsigned int i;
252 int ret = 0;
253
254 mutex_lock(&core->lock);
255 mbs_per_sec = load_per_type(core, VIDC_SESSION_TYPE_ENC) +
256 load_per_type(core, VIDC_SESSION_TYPE_DEC);
257
258 if (mbs_per_sec > core->res->max_load)
259 dev_warn(dev, "HW is overloaded, needed: %d max: %d\n",
260 mbs_per_sec, core->res->max_load);
261
262 if (!mbs_per_sec && num_rows > 1) {
263 freq = table[num_rows - 1].freq;
264 goto set_freq;
265 }
266
267 for (i = 0; i < num_rows; i++) {
268 if (mbs_per_sec > table[i].load)
269 break;
270 freq = table[i].freq;
271 }
272
273 set_freq:
274
275 ret = core_clks_set_rate(core, freq);
276 if (ret) {
277 dev_err(dev, "failed to set clock rate %lu (%d)\n",
278 freq, ret);
279 goto exit;
280 }
281
282 ret = load_scale_bw(core);
283 if (ret) {
284 dev_err(dev, "failed to set bandwidth (%d)\n",
285 ret);
286 goto exit;
287 }
288
289 exit:
290 mutex_unlock(&core->lock);
291 return ret;
292 }
293
core_get_v1(struct venus_core * core)294 static int core_get_v1(struct venus_core *core)
295 {
296 int ret;
297
298 ret = core_clks_get(core);
299 if (ret)
300 return ret;
301
302 ret = devm_pm_opp_set_clkname(core->dev, "core");
303 if (ret)
304 return ret;
305
306 return 0;
307 }
308
core_put_v1(struct venus_core * core)309 static void core_put_v1(struct venus_core *core)
310 {
311 }
312
core_power_v1(struct venus_core * core,int on)313 static int core_power_v1(struct venus_core *core, int on)
314 {
315 int ret = 0;
316
317 if (on == POWER_ON)
318 ret = core_clks_enable(core);
319 else
320 core_clks_disable(core);
321
322 return ret;
323 }
324
325 static const struct venus_pm_ops pm_ops_v1 = {
326 .core_get = core_get_v1,
327 .core_put = core_put_v1,
328 .core_power = core_power_v1,
329 .load_scale = load_scale_v1,
330 };
331
332 static void
vcodec_control_v3(struct venus_core * core,u32 session_type,bool enable)333 vcodec_control_v3(struct venus_core *core, u32 session_type, bool enable)
334 {
335 void __iomem *ctrl;
336
337 if (session_type == VIDC_SESSION_TYPE_DEC)
338 ctrl = core->wrapper_base + WRAPPER_VDEC_VCODEC_POWER_CONTROL;
339 else
340 ctrl = core->wrapper_base + WRAPPER_VENC_VCODEC_POWER_CONTROL;
341
342 if (enable)
343 writel(0, ctrl);
344 else
345 writel(1, ctrl);
346 }
347
vdec_get_v3(struct device * dev)348 static int vdec_get_v3(struct device *dev)
349 {
350 struct venus_core *core = dev_get_drvdata(dev);
351
352 return vcodec_clks_get(core, dev, core->vcodec0_clks,
353 core->res->vcodec0_clks);
354 }
355
vdec_power_v3(struct device * dev,int on)356 static int vdec_power_v3(struct device *dev, int on)
357 {
358 struct venus_core *core = dev_get_drvdata(dev);
359 int ret = 0;
360
361 vcodec_control_v3(core, VIDC_SESSION_TYPE_DEC, true);
362
363 if (on == POWER_ON)
364 ret = vcodec_clks_enable(core, core->vcodec0_clks);
365 else
366 vcodec_clks_disable(core, core->vcodec0_clks);
367
368 vcodec_control_v3(core, VIDC_SESSION_TYPE_DEC, false);
369
370 return ret;
371 }
372
venc_get_v3(struct device * dev)373 static int venc_get_v3(struct device *dev)
374 {
375 struct venus_core *core = dev_get_drvdata(dev);
376
377 return vcodec_clks_get(core, dev, core->vcodec1_clks,
378 core->res->vcodec1_clks);
379 }
380
venc_power_v3(struct device * dev,int on)381 static int venc_power_v3(struct device *dev, int on)
382 {
383 struct venus_core *core = dev_get_drvdata(dev);
384 int ret = 0;
385
386 vcodec_control_v3(core, VIDC_SESSION_TYPE_ENC, true);
387
388 if (on == POWER_ON)
389 ret = vcodec_clks_enable(core, core->vcodec1_clks);
390 else
391 vcodec_clks_disable(core, core->vcodec1_clks);
392
393 vcodec_control_v3(core, VIDC_SESSION_TYPE_ENC, false);
394
395 return ret;
396 }
397
398 static const struct venus_pm_ops pm_ops_v3 = {
399 .core_get = core_get_v1,
400 .core_put = core_put_v1,
401 .core_power = core_power_v1,
402 .vdec_get = vdec_get_v3,
403 .vdec_power = vdec_power_v3,
404 .venc_get = venc_get_v3,
405 .venc_power = venc_power_v3,
406 .load_scale = load_scale_v1,
407 };
408
vcodec_control_v4(struct venus_core * core,u32 coreid,bool enable)409 static int vcodec_control_v4(struct venus_core *core, u32 coreid, bool enable)
410 {
411 void __iomem *ctrl, *stat;
412 u32 val;
413 int ret;
414
415 if (IS_V6(core))
416 return dev_pm_genpd_set_hwmode(core->pmdomains->pd_devs[coreid], !enable);
417 else if (coreid == VIDC_CORE_ID_1) {
418 ctrl = core->wrapper_base + WRAPPER_VCODEC0_MMCC_POWER_CONTROL;
419 stat = core->wrapper_base + WRAPPER_VCODEC0_MMCC_POWER_STATUS;
420 } else {
421 ctrl = core->wrapper_base + WRAPPER_VCODEC1_MMCC_POWER_CONTROL;
422 stat = core->wrapper_base + WRAPPER_VCODEC1_MMCC_POWER_STATUS;
423 }
424
425 if (enable) {
426 writel(0, ctrl);
427
428 ret = readl_poll_timeout(stat, val, val & BIT(1), 1, 100);
429 if (ret)
430 return ret;
431 } else {
432 writel(1, ctrl);
433
434 ret = readl_poll_timeout(stat, val, !(val & BIT(1)), 1, 100);
435 if (ret)
436 return ret;
437 }
438
439 return 0;
440 }
441
poweroff_coreid(struct venus_core * core,unsigned int coreid_mask)442 static int poweroff_coreid(struct venus_core *core, unsigned int coreid_mask)
443 {
444 int ret;
445
446 if (coreid_mask & VIDC_CORE_ID_1) {
447 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, true);
448 if (ret)
449 return ret;
450
451 vcodec_clks_disable(core, core->vcodec0_clks);
452
453 if (!IS_V6(core)) {
454 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, false);
455 if (ret)
456 return ret;
457 }
458
459 ret = pm_runtime_put_sync(core->pmdomains->pd_devs[1]);
460 if (ret < 0)
461 return ret;
462 }
463
464 if (coreid_mask & VIDC_CORE_ID_2) {
465 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, true);
466 if (ret)
467 return ret;
468
469 vcodec_clks_disable(core, core->vcodec1_clks);
470
471 if (!IS_V6(core)) {
472 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, false);
473 if (ret)
474 return ret;
475 }
476
477 ret = pm_runtime_put_sync(core->pmdomains->pd_devs[2]);
478 if (ret < 0)
479 return ret;
480 }
481
482 return 0;
483 }
484
poweron_coreid(struct venus_core * core,unsigned int coreid_mask)485 static int poweron_coreid(struct venus_core *core, unsigned int coreid_mask)
486 {
487 int ret;
488
489 if (coreid_mask & VIDC_CORE_ID_1) {
490 ret = pm_runtime_get_sync(core->pmdomains->pd_devs[1]);
491 if (ret < 0)
492 return ret;
493
494 if (!IS_V6(core)) {
495 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, true);
496 if (ret)
497 return ret;
498 }
499
500 ret = vcodec_clks_enable(core, core->vcodec0_clks);
501 if (ret)
502 return ret;
503
504 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, false);
505 if (ret < 0)
506 return ret;
507 }
508
509 if (coreid_mask & VIDC_CORE_ID_2) {
510 ret = pm_runtime_get_sync(core->pmdomains->pd_devs[2]);
511 if (ret < 0)
512 return ret;
513
514 if (!IS_V6(core)) {
515 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, true);
516 if (ret)
517 return ret;
518 }
519
520 ret = vcodec_clks_enable(core, core->vcodec1_clks);
521 if (ret)
522 return ret;
523
524 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, false);
525 if (ret < 0)
526 return ret;
527 }
528
529 return 0;
530 }
531
power_save_mode_enable(struct venus_inst * inst,bool enable)532 static inline int power_save_mode_enable(struct venus_inst *inst,
533 bool enable)
534 {
535 struct venc_controls *enc_ctr = &inst->controls.enc;
536 const u32 ptype = HFI_PROPERTY_CONFIG_VENC_PERF_MODE;
537 u32 venc_mode;
538 int ret = 0;
539
540 if (inst->session_type != VIDC_SESSION_TYPE_ENC)
541 return 0;
542
543 if (enc_ctr->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CQ)
544 enable = false;
545
546 venc_mode = enable ? HFI_VENC_PERFMODE_POWER_SAVE :
547 HFI_VENC_PERFMODE_MAX_QUALITY;
548
549 ret = hfi_session_set_property(inst, ptype, &venc_mode);
550 if (ret)
551 return ret;
552
553 inst->flags = enable ? inst->flags | VENUS_LOW_POWER :
554 inst->flags & ~VENUS_LOW_POWER;
555
556 return ret;
557 }
558
move_core_to_power_save_mode(struct venus_core * core,u32 core_id)559 static int move_core_to_power_save_mode(struct venus_core *core,
560 u32 core_id)
561 {
562 struct venus_inst *inst = NULL;
563
564 mutex_lock(&core->lock);
565 list_for_each_entry(inst, &core->instances, list) {
566 if (inst->clk_data.core_id == core_id &&
567 inst->session_type == VIDC_SESSION_TYPE_ENC)
568 power_save_mode_enable(inst, true);
569 }
570 mutex_unlock(&core->lock);
571 return 0;
572 }
573
574 static void
min_loaded_core(struct venus_inst * inst,u32 * min_coreid,u32 * min_load,bool low_power)575 min_loaded_core(struct venus_inst *inst, u32 *min_coreid, u32 *min_load, bool low_power)
576 {
577 u32 mbs_per_sec, load, core1_load = 0, core2_load = 0;
578 u32 cores_max = core_num_max(inst);
579 struct venus_core *core = inst->core;
580 struct venus_inst *inst_pos;
581 unsigned long vpp_freq;
582 u32 coreid;
583
584 mutex_lock(&core->lock);
585
586 list_for_each_entry(inst_pos, &core->instances, list) {
587 if (inst_pos == inst)
588 continue;
589
590 if (inst_pos->state != INST_START)
591 continue;
592
593 if (inst->session_type == VIDC_SESSION_TYPE_DEC)
594 vpp_freq = inst_pos->clk_data.vpp_freq;
595 else if (inst->session_type == VIDC_SESSION_TYPE_ENC)
596 vpp_freq = low_power ? inst_pos->clk_data.low_power_freq :
597 inst_pos->clk_data.vpp_freq;
598 else
599 continue;
600
601 coreid = inst_pos->clk_data.core_id;
602
603 mbs_per_sec = load_per_instance(inst_pos);
604 load = mbs_per_sec * vpp_freq;
605
606 if ((coreid & VIDC_CORE_ID_3) == VIDC_CORE_ID_3) {
607 core1_load += load / 2;
608 core2_load += load / 2;
609 } else if (coreid & VIDC_CORE_ID_1) {
610 core1_load += load;
611 } else if (coreid & VIDC_CORE_ID_2) {
612 core2_load += load;
613 }
614 }
615
616 *min_coreid = core1_load <= core2_load ?
617 VIDC_CORE_ID_1 : VIDC_CORE_ID_2;
618 *min_load = min(core1_load, core2_load);
619
620 if (cores_max < VIDC_CORE_ID_2 || core->res->vcodec_num < 2) {
621 *min_coreid = VIDC_CORE_ID_1;
622 *min_load = core1_load;
623 }
624
625 mutex_unlock(&core->lock);
626 }
627
decide_core(struct venus_inst * inst)628 static int decide_core(struct venus_inst *inst)
629 {
630 const u32 ptype = HFI_PROPERTY_CONFIG_VIDEOCORES_USAGE;
631 struct venus_core *core = inst->core;
632 u32 min_coreid, min_load, cur_inst_load;
633 u32 min_lp_coreid, min_lp_load, cur_inst_lp_load;
634 struct hfi_videocores_usage_type cu;
635 unsigned long max_freq;
636 int ret = 0;
637
638 if (legacy_binding) {
639 if (inst->session_type == VIDC_SESSION_TYPE_DEC)
640 cu.video_core_enable_mask = VIDC_CORE_ID_1;
641 else
642 cu.video_core_enable_mask = VIDC_CORE_ID_2;
643
644 goto done;
645 }
646
647 if (inst->clk_data.core_id != VIDC_CORE_ID_DEFAULT)
648 return 0;
649
650 cur_inst_load = load_per_instance(inst);
651 cur_inst_load *= inst->clk_data.vpp_freq;
652 /*TODO : divide this inst->load by work_route */
653
654 cur_inst_lp_load = load_per_instance(inst);
655 cur_inst_lp_load *= inst->clk_data.low_power_freq;
656 /*TODO : divide this inst->load by work_route */
657
658 max_freq = core->res->freq_tbl[0].freq;
659
660 min_loaded_core(inst, &min_coreid, &min_load, false);
661 min_loaded_core(inst, &min_lp_coreid, &min_lp_load, true);
662
663 if (cur_inst_load + min_load <= max_freq) {
664 inst->clk_data.core_id = min_coreid;
665 cu.video_core_enable_mask = min_coreid;
666 } else if (cur_inst_lp_load + min_load <= max_freq) {
667 /* Move current instance to LP and return */
668 inst->clk_data.core_id = min_coreid;
669 cu.video_core_enable_mask = min_coreid;
670 power_save_mode_enable(inst, true);
671 } else if (cur_inst_lp_load + min_lp_load <= max_freq) {
672 /* Move all instances to LP mode and return */
673 inst->clk_data.core_id = min_lp_coreid;
674 cu.video_core_enable_mask = min_lp_coreid;
675 move_core_to_power_save_mode(core, min_lp_coreid);
676 } else {
677 dev_warn(core->dev, "HW can't support this load");
678 return -EINVAL;
679 }
680
681 done:
682 ret = hfi_session_set_property(inst, ptype, &cu);
683 if (ret)
684 return ret;
685
686 return ret;
687 }
688
acquire_core(struct venus_inst * inst)689 static int acquire_core(struct venus_inst *inst)
690 {
691 struct venus_core *core = inst->core;
692 unsigned int coreid_mask = 0;
693
694 if (inst->core_acquired)
695 return 0;
696
697 inst->core_acquired = true;
698
699 if (inst->clk_data.core_id & VIDC_CORE_ID_1) {
700 if (core->core0_usage_count++)
701 return 0;
702
703 coreid_mask = VIDC_CORE_ID_1;
704 }
705
706 if (inst->clk_data.core_id & VIDC_CORE_ID_2) {
707 if (core->core1_usage_count++)
708 return 0;
709
710 coreid_mask |= VIDC_CORE_ID_2;
711 }
712
713 return poweron_coreid(core, coreid_mask);
714 }
715
release_core(struct venus_inst * inst)716 static int release_core(struct venus_inst *inst)
717 {
718 struct venus_core *core = inst->core;
719 unsigned int coreid_mask = 0;
720 int ret;
721
722 if (!inst->core_acquired)
723 return 0;
724
725 if (inst->clk_data.core_id & VIDC_CORE_ID_1) {
726 if (--core->core0_usage_count)
727 goto done;
728
729 coreid_mask = VIDC_CORE_ID_1;
730 }
731
732 if (inst->clk_data.core_id & VIDC_CORE_ID_2) {
733 if (--core->core1_usage_count)
734 goto done;
735
736 coreid_mask |= VIDC_CORE_ID_2;
737 }
738
739 ret = poweroff_coreid(core, coreid_mask);
740 if (ret)
741 return ret;
742
743 done:
744 inst->clk_data.core_id = VIDC_CORE_ID_DEFAULT;
745 inst->core_acquired = false;
746 return 0;
747 }
748
coreid_power_v4(struct venus_inst * inst,int on)749 static int coreid_power_v4(struct venus_inst *inst, int on)
750 {
751 struct venus_core *core = inst->core;
752 int ret;
753
754 if (legacy_binding)
755 return 0;
756
757 if (on == POWER_ON) {
758 ret = decide_core(inst);
759 if (ret)
760 return ret;
761
762 mutex_lock(&core->lock);
763 ret = acquire_core(inst);
764 mutex_unlock(&core->lock);
765 } else {
766 mutex_lock(&core->lock);
767 ret = release_core(inst);
768 mutex_unlock(&core->lock);
769 }
770
771 return ret;
772 }
773
vdec_get_v4(struct device * dev)774 static int vdec_get_v4(struct device *dev)
775 {
776 struct venus_core *core = dev_get_drvdata(dev);
777
778 if (!legacy_binding)
779 return 0;
780
781 return vcodec_clks_get(core, dev, core->vcodec0_clks,
782 core->res->vcodec0_clks);
783 }
784
vdec_put_v4(struct device * dev)785 static void vdec_put_v4(struct device *dev)
786 {
787 struct venus_core *core = dev_get_drvdata(dev);
788 unsigned int i;
789
790 if (!legacy_binding)
791 return;
792
793 for (i = 0; i < core->res->vcodec_clks_num; i++)
794 core->vcodec0_clks[i] = NULL;
795 }
796
vdec_power_v4(struct device * dev,int on)797 static int vdec_power_v4(struct device *dev, int on)
798 {
799 struct venus_core *core = dev_get_drvdata(dev);
800 int ret;
801
802 if (!legacy_binding)
803 return 0;
804
805 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, true);
806 if (ret)
807 return ret;
808
809 if (on == POWER_ON)
810 ret = vcodec_clks_enable(core, core->vcodec0_clks);
811 else
812 vcodec_clks_disable(core, core->vcodec0_clks);
813
814 vcodec_control_v4(core, VIDC_CORE_ID_1, false);
815
816 return ret;
817 }
818
venc_get_v4(struct device * dev)819 static int venc_get_v4(struct device *dev)
820 {
821 struct venus_core *core = dev_get_drvdata(dev);
822
823 if (!legacy_binding)
824 return 0;
825
826 return vcodec_clks_get(core, dev, core->vcodec1_clks,
827 core->res->vcodec1_clks);
828 }
829
venc_put_v4(struct device * dev)830 static void venc_put_v4(struct device *dev)
831 {
832 struct venus_core *core = dev_get_drvdata(dev);
833 unsigned int i;
834
835 if (!legacy_binding)
836 return;
837
838 for (i = 0; i < core->res->vcodec_clks_num; i++)
839 core->vcodec1_clks[i] = NULL;
840 }
841
venc_power_v4(struct device * dev,int on)842 static int venc_power_v4(struct device *dev, int on)
843 {
844 struct venus_core *core = dev_get_drvdata(dev);
845 int ret;
846
847 if (!legacy_binding)
848 return 0;
849
850 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, true);
851 if (ret)
852 return ret;
853
854 if (on == POWER_ON)
855 ret = vcodec_clks_enable(core, core->vcodec1_clks);
856 else
857 vcodec_clks_disable(core, core->vcodec1_clks);
858
859 vcodec_control_v4(core, VIDC_CORE_ID_2, false);
860
861 return ret;
862 }
863
vcodec_domains_get(struct venus_core * core)864 static int vcodec_domains_get(struct venus_core *core)
865 {
866 int ret;
867 struct device **opp_virt_dev;
868 struct device *dev = core->dev;
869 const struct venus_resources *res = core->res;
870 struct dev_pm_domain_attach_data vcodec_data = {
871 .pd_names = res->vcodec_pmdomains,
872 .num_pd_names = res->vcodec_pmdomains_num,
873 .pd_flags = PD_FLAG_NO_DEV_LINK,
874 };
875
876 if (!res->vcodec_pmdomains_num)
877 goto skip_pmdomains;
878
879 ret = devm_pm_domain_attach_list(dev, &vcodec_data, &core->pmdomains);
880 if (ret < 0)
881 return ret;
882
883 skip_pmdomains:
884 if (!core->res->opp_pmdomain)
885 return 0;
886
887 /* Attach the power domain for setting performance state */
888 ret = devm_pm_opp_attach_genpd(dev, res->opp_pmdomain, &opp_virt_dev);
889 if (ret)
890 goto opp_attach_err;
891
892 core->opp_pmdomain = *opp_virt_dev;
893 core->opp_dl_venus = device_link_add(dev, core->opp_pmdomain,
894 DL_FLAG_RPM_ACTIVE |
895 DL_FLAG_PM_RUNTIME |
896 DL_FLAG_STATELESS);
897 if (!core->opp_dl_venus) {
898 ret = -ENODEV;
899 goto opp_attach_err;
900 }
901
902 return 0;
903
904 opp_attach_err:
905 return ret;
906 }
907
vcodec_domains_put(struct venus_core * core)908 static void vcodec_domains_put(struct venus_core *core)
909 {
910 if (!core->has_opp_table)
911 return;
912
913 if (core->opp_dl_venus)
914 device_link_del(core->opp_dl_venus);
915 }
916
core_resets_reset(struct venus_core * core)917 static int core_resets_reset(struct venus_core *core)
918 {
919 const struct venus_resources *res = core->res;
920 unsigned int i;
921 int ret;
922
923 if (!res->resets_num)
924 return 0;
925
926 for (i = 0; i < res->resets_num; i++) {
927 ret = reset_control_assert(core->resets[i]);
928 if (ret)
929 goto err;
930
931 usleep_range(150, 250);
932 ret = reset_control_deassert(core->resets[i]);
933 if (ret)
934 goto err;
935 }
936
937 err:
938 return ret;
939 }
940
core_resets_get(struct venus_core * core)941 static int core_resets_get(struct venus_core *core)
942 {
943 struct device *dev = core->dev;
944 const struct venus_resources *res = core->res;
945 unsigned int i;
946 int ret;
947
948 if (!res->resets_num)
949 return 0;
950
951 for (i = 0; i < res->resets_num; i++) {
952 core->resets[i] =
953 devm_reset_control_get_exclusive(dev, res->resets[i]);
954 if (IS_ERR(core->resets[i])) {
955 ret = PTR_ERR(core->resets[i]);
956 return ret;
957 }
958 }
959
960 return 0;
961 }
962
core_get_v4(struct venus_core * core)963 static int core_get_v4(struct venus_core *core)
964 {
965 struct device *dev = core->dev;
966 const struct venus_resources *res = core->res;
967 int ret;
968
969 ret = core_clks_get(core);
970 if (ret)
971 return ret;
972
973 if (!res->vcodec_pmdomains_num)
974 legacy_binding = true;
975
976 dev_info(dev, "%s legacy binding\n", legacy_binding ? "" : "non");
977
978 ret = vcodec_clks_get(core, dev, core->vcodec0_clks, res->vcodec0_clks);
979 if (ret)
980 return ret;
981
982 ret = vcodec_clks_get(core, dev, core->vcodec1_clks, res->vcodec1_clks);
983 if (ret)
984 return ret;
985
986 ret = core_resets_get(core);
987 if (ret)
988 return ret;
989
990 if (legacy_binding)
991 return 0;
992
993 ret = devm_pm_opp_set_clkname(dev, "core");
994 if (ret)
995 return ret;
996
997 ret = vcodec_domains_get(core);
998 if (ret)
999 return ret;
1000
1001 if (core->res->opp_pmdomain) {
1002 ret = devm_pm_opp_of_add_table(dev);
1003 if (!ret) {
1004 core->has_opp_table = true;
1005 } else if (ret != -ENODEV) {
1006 dev_err(dev, "invalid OPP table in device tree\n");
1007 return ret;
1008 }
1009 }
1010
1011 return 0;
1012 }
1013
core_put_v4(struct venus_core * core)1014 static void core_put_v4(struct venus_core *core)
1015 {
1016 if (legacy_binding)
1017 return;
1018
1019 vcodec_domains_put(core);
1020 }
1021
core_power_v4(struct venus_core * core,int on)1022 static int core_power_v4(struct venus_core *core, int on)
1023 {
1024 struct device *dev = core->dev;
1025 struct device *pmctrl = core->pmdomains ?
1026 core->pmdomains->pd_devs[0] : NULL;
1027 int ret = 0;
1028
1029 if (on == POWER_ON) {
1030 if (pmctrl) {
1031 ret = pm_runtime_resume_and_get(pmctrl);
1032 if (ret < 0) {
1033 return ret;
1034 }
1035 }
1036
1037 ret = core_resets_reset(core);
1038 if (ret) {
1039 if (pmctrl)
1040 pm_runtime_put_sync(pmctrl);
1041 return ret;
1042 }
1043
1044 ret = core_clks_enable(core);
1045 if (ret < 0 && pmctrl)
1046 pm_runtime_put_sync(pmctrl);
1047 } else {
1048 /* Drop the performance state vote */
1049 if (core->opp_pmdomain)
1050 dev_pm_opp_set_rate(dev, 0);
1051
1052 core_clks_disable(core);
1053
1054 ret = core_resets_reset(core);
1055
1056 if (pmctrl)
1057 pm_runtime_put_sync(pmctrl);
1058 }
1059
1060 return ret;
1061 }
1062
calculate_inst_freq(struct venus_inst * inst,unsigned long filled_len)1063 static unsigned long calculate_inst_freq(struct venus_inst *inst,
1064 unsigned long filled_len)
1065 {
1066 unsigned long vpp_freq_per_mb = 0, vpp_freq = 0, vsp_freq = 0;
1067 u32 fps = (u32)inst->fps;
1068 u32 mbs_per_sec;
1069
1070 mbs_per_sec = load_per_instance(inst);
1071
1072 if (inst->state != INST_START)
1073 return 0;
1074
1075 if (inst->session_type == VIDC_SESSION_TYPE_ENC) {
1076 vpp_freq_per_mb = inst->flags & VENUS_LOW_POWER ?
1077 inst->clk_data.low_power_freq :
1078 inst->clk_data.vpp_freq;
1079
1080 vpp_freq = mbs_per_sec * vpp_freq_per_mb;
1081 } else {
1082 vpp_freq = mbs_per_sec * inst->clk_data.vpp_freq;
1083 }
1084
1085 /* 21 / 20 is overhead factor */
1086 vpp_freq += vpp_freq / 20;
1087 vsp_freq = mbs_per_sec * inst->clk_data.vsp_freq;
1088
1089 /* 10 / 7 is overhead factor */
1090 if (inst->session_type == VIDC_SESSION_TYPE_ENC)
1091 vsp_freq += (inst->controls.enc.bitrate * 10) / 7;
1092 else
1093 vsp_freq += ((fps * filled_len * 8) * 10) / 7;
1094
1095 return max(vpp_freq, vsp_freq);
1096 }
1097
load_scale_v4(struct venus_inst * inst)1098 static int load_scale_v4(struct venus_inst *inst)
1099 {
1100 struct venus_core *core = inst->core;
1101 const struct freq_tbl *table = core->res->freq_tbl;
1102 unsigned int num_rows = core->res->freq_tbl_size;
1103 struct device *dev = core->dev;
1104 unsigned long freq = 0, freq_core1 = 0, freq_core2 = 0;
1105 unsigned long filled_len = 0;
1106 int i, ret = 0;
1107
1108 for (i = 0; i < inst->num_input_bufs; i++)
1109 filled_len = max(filled_len, inst->payloads[i]);
1110
1111 if (inst->session_type == VIDC_SESSION_TYPE_DEC && !filled_len)
1112 return ret;
1113
1114 freq = calculate_inst_freq(inst, filled_len);
1115 inst->clk_data.freq = freq;
1116
1117 mutex_lock(&core->lock);
1118 list_for_each_entry(inst, &core->instances, list) {
1119 if (inst->clk_data.core_id == VIDC_CORE_ID_1) {
1120 freq_core1 += inst->clk_data.freq;
1121 } else if (inst->clk_data.core_id == VIDC_CORE_ID_2) {
1122 freq_core2 += inst->clk_data.freq;
1123 } else if (inst->clk_data.core_id == VIDC_CORE_ID_3) {
1124 freq_core1 += inst->clk_data.freq;
1125 freq_core2 += inst->clk_data.freq;
1126 }
1127 }
1128
1129 freq = max(freq_core1, freq_core2);
1130
1131 if (freq > table[0].freq) {
1132 dev_dbg(dev, VDBGL "requested clock rate: %lu scaling clock rate : %lu\n",
1133 freq, table[0].freq);
1134
1135 freq = table[0].freq;
1136 goto set_freq;
1137 }
1138
1139 for (i = num_rows - 1 ; i >= 0; i--) {
1140 if (freq <= table[i].freq) {
1141 freq = table[i].freq;
1142 break;
1143 }
1144 }
1145
1146 set_freq:
1147
1148 ret = core_clks_set_rate(core, freq);
1149 if (ret) {
1150 dev_err(dev, "failed to set clock rate %lu (%d)\n",
1151 freq, ret);
1152 goto exit;
1153 }
1154
1155 ret = load_scale_bw(core);
1156 if (ret) {
1157 dev_err(dev, "failed to set bandwidth (%d)\n",
1158 ret);
1159 goto exit;
1160 }
1161
1162 exit:
1163 mutex_unlock(&core->lock);
1164 return ret;
1165 }
1166
1167 static const struct venus_pm_ops pm_ops_v4 = {
1168 .core_get = core_get_v4,
1169 .core_put = core_put_v4,
1170 .core_power = core_power_v4,
1171 .vdec_get = vdec_get_v4,
1172 .vdec_put = vdec_put_v4,
1173 .vdec_power = vdec_power_v4,
1174 .venc_get = venc_get_v4,
1175 .venc_put = venc_put_v4,
1176 .venc_power = venc_power_v4,
1177 .coreid_power = coreid_power_v4,
1178 .load_scale = load_scale_v4,
1179 };
1180
venus_pm_get(enum hfi_version version)1181 const struct venus_pm_ops *venus_pm_get(enum hfi_version version)
1182 {
1183 switch (version) {
1184 case HFI_VERSION_1XX:
1185 default:
1186 return &pm_ops_v1;
1187 case HFI_VERSION_3XX:
1188 return &pm_ops_v3;
1189 case HFI_VERSION_4XX:
1190 case HFI_VERSION_6XX:
1191 return &pm_ops_v4;
1192 }
1193
1194 return NULL;
1195 }
1196