Lines Matching +full:emc +full:- +full:mode +full:- +full:1
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved.
14 #include "tegra210-emc.h"
15 #include "tegra210-mc.h"
20 #define INFO (1 << 0)
21 #define STEPS (1 << 1)
22 #define SUB_STEPS (1 << 2)
23 #define PRELOCK (1 << 3)
24 #define PRELOCK_STEPS (1 << 4)
25 #define ACTIVE_EN (1 << 5)
26 #define PRAMP_UP (1 << 6)
27 #define PRAMP_DN (1 << 7)
28 #define EMA_WRITES (1 << 10)
29 #define EMA_UPDATES (1 << 11)
30 #define PER_TRAIN (1 << 16)
31 #define CC_PRINT (1 << 17)
32 #define CCFIFO (1 << 29)
33 #define REGS (1 << 30)
34 #define REG_LISTS (1 << 31)
36 #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__) argument
42 DVFS_SEQUENCE = 1,
53 * PTFV defines - basically just indexes into the per table PTFV array.
56 #define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1
67 #define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0)
78 ({ next->ptfv_list[(dev)] = \
79 next->ptfv_list[(dev)] / \
80 next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
86 ({ next->ptfv_list[(dev)] += \
93 ((timing)->ptfv_list[(dev)] / \
102 next->ptfv_list[dqs] = \
104 (next->ptfv_list[dqs] * \
105 next->ptfv_list[w])) / \
106 (next->ptfv_list[w] + 1); \
108 emc_dbg(emc, EMA_UPDATES, "%s: (s=%u) EMA: %u\n", \
109 __stringify(dev), nval, next->ptfv_list[dqs]); \
114 ((timing)->ptfv_list[(dev)])
119 u32 *curr = &timing->current_dram_clktree[idx]; in tegra210_emc_compare_update_delay()
120 u32 rate_mhz = timing->rate / 1000; in tegra210_emc_compare_update_delay()
123 tmdel = abs(*curr - measured); in tegra210_emc_compare_update_delay()
125 if (tmdel * 128 * rate_mhz / 1000000 > timing->tree_margin) { in tegra210_emc_compare_update_delay()
133 static void tegra210_emc_get_clktree_delay(struct tegra210_emc *emc, in tegra210_emc_get_clktree_delay() argument
136 struct tegra210_emc_timing *curr = emc->last; in tegra210_emc_get_clktree_delay()
137 u32 rate_mhz = curr->rate / 1000; in tegra210_emc_get_clktree_delay()
142 clocks = tegra210_emc_actual_osc_clocks(curr->run_clocks); in tegra210_emc_get_clktree_delay()
145 tegra210_emc_start_periodic_compensation(emc); in tegra210_emc_get_clktree_delay()
148 for (d = 0; d < emc->num_devices; d++) { in tegra210_emc_get_clktree_delay()
150 msb = tegra210_emc_mrr_read(emc, 2 - d, 19); in tegra210_emc_get_clktree_delay()
151 lsb = tegra210_emc_mrr_read(emc, 2 - d, 18); in tegra210_emc_get_clktree_delay()
153 for (c = 0; c < emc->num_channels; c++) { in tegra210_emc_get_clktree_delay()
165 /* C[c]D[d]U[1] */ in tegra210_emc_get_clktree_delay()
182 static bool periodic_compensation_handler(struct tegra210_emc *emc, u32 type, in periodic_compensation_handler() argument
188 (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) in periodic_compensation_handler()
190 u32 i, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; in periodic_compensation_handler()
194 if (!next->periodic_training) in periodic_compensation_handler()
198 if (last->periodic_training && in periodic_compensation_handler()
199 (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & in periodic_compensation_handler()
215 tegra210_emc_get_clktree_delay(emc, delay); in periodic_compensation_handler()
231 tegra210_emc_get_clktree_delay(emc, delay); in periodic_compensation_handler()
243 static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc) in tegra210_emc_r21021_periodic_compensation() argument
258 struct tegra210_emc_timing *last = emc->last; in tegra210_emc_r21021_periodic_compensation()
261 if (last->periodic_training) { in tegra210_emc_r21021_periodic_compensation()
262 emc_dbg(emc, PER_TRAIN, "Periodic training starting\n"); in tegra210_emc_r21021_periodic_compensation()
264 value = emc_readl(emc, EMC_DBG); in tegra210_emc_r21021_periodic_compensation()
265 emc_cfg_o = emc_readl(emc, EMC_CFG); in tegra210_emc_r21021_periodic_compensation()
272 * 1. Power optimizations should be off. in tegra210_emc_r21021_periodic_compensation()
274 emc_writel(emc, emc_cfg, EMC_CFG); in tegra210_emc_r21021_periodic_compensation()
277 tegra210_emc_dll_disable(emc); in tegra210_emc_r21021_periodic_compensation()
279 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_periodic_compensation()
280 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, in tegra210_emc_r21021_periodic_compensation()
284 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_periodic_compensation()
285 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, in tegra210_emc_r21021_periodic_compensation()
289 emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE); in tegra210_emc_r21021_periodic_compensation()
292 emc_writel(emc, value, EMC_CFG_UPDATE); in tegra210_emc_r21021_periodic_compensation()
295 * 2. osc kick off - this assumes training and dvfs have set in tegra210_emc_r21021_periodic_compensation()
303 if (periodic_compensation_handler(emc, PERIODIC_TRAINING_SEQUENCE, in tegra210_emc_r21021_periodic_compensation()
311 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", in tegra210_emc_r21021_periodic_compensation()
313 emc_writel(emc, value, list[i]); in tegra210_emc_r21021_periodic_compensation()
317 emc_writel(emc, emc_cfg_o, EMC_CFG); in tegra210_emc_r21021_periodic_compensation()
322 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_periodic_compensation()
325 emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE); in tegra210_emc_r21021_periodic_compensation()
328 tegra210_emc_dll_enable(emc); in tegra210_emc_r21021_periodic_compensation()
337 static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc) in tegra210_emc_r21021_set_clock() argument
354 * necessarily correspond to the actual timing values in the EMC at the in tegra210_emc_r21021_set_clock()
359 struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next; in tegra210_emc_r21021_set_clock()
364 u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4]; in tegra210_emc_r21021_set_clock()
376 emc_dbg(emc, INFO, "Running clock change.\n"); in tegra210_emc_r21021_set_clock()
379 fake = tegra210_emc_find_timing(emc, last->rate * 1000UL); in tegra210_emc_r21021_set_clock()
382 value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK; in tegra210_emc_r21021_set_clock()
385 if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31)) in tegra210_emc_r21021_set_clock()
388 if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 && in tegra210_emc_r21021_set_clock()
389 last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) || in tegra210_emc_r21021_set_clock()
396 if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) && in tegra210_emc_r21021_set_clock()
400 emc_readl(emc, EMC_CFG); in tegra210_emc_r21021_set_clock()
401 emc_readl(emc, EMC_AUTO_CAL_CONFIG); in tegra210_emc_r21021_set_clock()
403 src_clk_period = 1000000000 / last->rate; in tegra210_emc_r21021_set_clock()
404 dst_clk_period = 1000000000 / next->rate; in tegra210_emc_r21021_set_clock()
407 tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4; in tegra210_emc_r21021_set_clock()
413 emc_dbg = emc_readl(emc, EMC_DBG); in tegra210_emc_r21021_set_clock()
414 emc_pin = emc_readl(emc, EMC_PIN); in tegra210_emc_r21021_set_clock()
415 emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK); in tegra210_emc_r21021_set_clock()
417 emc_cfg = next->burst_regs[EMC_CFG_INDEX]; in tegra210_emc_r21021_set_clock()
420 emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl; in tegra210_emc_r21021_set_clock()
427 emc_dbg(emc, INFO, "Clock change version: %d\n", in tegra210_emc_r21021_set_clock()
429 emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type); in tegra210_emc_r21021_set_clock()
430 emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices); in tegra210_emc_r21021_set_clock()
431 emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc); in tegra210_emc_r21021_set_clock()
432 emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src); in tegra210_emc_r21021_set_clock()
433 emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate, in tegra210_emc_r21021_set_clock()
434 next->rate); in tegra210_emc_r21021_set_clock()
435 emc_dbg(emc, INFO, "last period: %u, next period: %u\n", in tegra210_emc_r21021_set_clock()
437 emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor); in tegra210_emc_r21021_set_clock()
438 emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels); in tegra210_emc_r21021_set_clock()
439 emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode); in tegra210_emc_r21021_set_clock()
442 * Step 1: in tegra210_emc_r21021_set_clock()
445 emc_dbg(emc, STEPS, "Step 1\n"); in tegra210_emc_r21021_set_clock()
446 emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n"); in tegra210_emc_r21021_set_clock()
448 value = emc_readl(emc, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
450 emc_writel(emc, value, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
452 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_set_clock()
454 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_set_clock()
455 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL, in tegra210_emc_r21021_set_clock()
458 emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n"); in tegra210_emc_r21021_set_clock()
460 emc_auto_cal_config = next->emc_auto_cal_config; in tegra210_emc_r21021_set_clock()
466 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); in tegra210_emc_r21021_set_clock()
467 emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */ in tegra210_emc_r21021_set_clock()
469 emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n"); in tegra210_emc_r21021_set_clock()
471 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
472 emc_writel(emc, emc_cfg, EMC_CFG); in tegra210_emc_r21021_set_clock()
473 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); in tegra210_emc_r21021_set_clock()
474 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
476 if (next->periodic_training) { in tegra210_emc_r21021_set_clock()
479 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_set_clock()
480 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, in tegra210_emc_r21021_set_clock()
484 for (i = 0; i < emc->num_channels; i++) in tegra210_emc_r21021_set_clock()
485 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, in tegra210_emc_r21021_set_clock()
489 if (periodic_compensation_handler(emc, DVFS_SEQUENCE, fake, next)) in tegra210_emc_r21021_set_clock()
493 emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS); in tegra210_emc_r21021_set_clock()
494 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
495 emc_writel(emc, emc_cfg, EMC_CFG); in tegra210_emc_r21021_set_clock()
496 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); in tegra210_emc_r21021_set_clock()
497 emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, in tegra210_emc_r21021_set_clock()
499 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp & in tegra210_emc_r21021_set_clock()
504 ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
506 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
508 ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
510 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
513 (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
516 (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
521 emc_writel(emc, last->burst_regs in tegra210_emc_r21021_set_clock()
527 emc_writel(emc, last->burst_regs in tegra210_emc_r21021_set_clock()
534 if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & in tegra210_emc_r21021_set_clock()
536 ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & in tegra210_emc_r21021_set_clock()
537 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) || in tegra210_emc_r21021_set_clock()
538 (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & in tegra210_emc_r21021_set_clock()
540 ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & in tegra210_emc_r21021_set_clock()
543 next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; in tegra210_emc_r21021_set_clock()
545 last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; in tegra210_emc_r21021_set_clock()
556 emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL); in tegra210_emc_r21021_set_clock()
557 udelay(1); in tegra210_emc_r21021_set_clock()
559 udelay(1); in tegra210_emc_r21021_set_clock()
562 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
568 emc_dbg(emc, STEPS, "Step 2\n"); in tegra210_emc_r21021_set_clock()
570 if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & in tegra210_emc_r21021_set_clock()
572 emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n"); in tegra210_emc_r21021_set_clock()
573 value = tegra210_emc_dll_prelock(emc, clksrc); in tegra210_emc_r21021_set_clock()
574 emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value); in tegra210_emc_r21021_set_clock()
576 emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n"); in tegra210_emc_r21021_set_clock()
577 tegra210_emc_dll_disable(emc); in tegra210_emc_r21021_set_clock()
584 emc_dbg(emc, STEPS, "Step 3\n"); in tegra210_emc_r21021_set_clock()
586 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
587 emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2); in tegra210_emc_r21021_set_clock()
588 emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3); in tegra210_emc_r21021_set_clock()
589 emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4); in tegra210_emc_r21021_set_clock()
590 emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5); in tegra210_emc_r21021_set_clock()
591 emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6); in tegra210_emc_r21021_set_clock()
592 emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7); in tegra210_emc_r21021_set_clock()
593 emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8); in tegra210_emc_r21021_set_clock()
594 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
598 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); in tegra210_emc_r21021_set_clock()
604 emc_dbg(emc, STEPS, "Step 4\n"); in tegra210_emc_r21021_set_clock()
607 ccfifo_writel(emc, 1, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
609 emc_writel(emc, next->emc_cfg_2, EMC_CFG_2); in tegra210_emc_r21021_set_clock()
615 emc_dbg(emc, STEPS, "Step 5\n"); in tegra210_emc_r21021_set_clock()
618 zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period)); in tegra210_emc_r21021_set_clock()
620 zq_wait_long = max(next->min_mrs_wait, in tegra210_emc_r21021_set_clock()
630 * Training code - removed. in tegra210_emc_r21021_set_clock()
632 emc_dbg(emc, STEPS, "Step 6\n"); in tegra210_emc_r21021_set_clock()
638 emc_dbg(emc, STEPS, "Step 7\n"); in tegra210_emc_r21021_set_clock()
639 emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P"); in tegra210_emc_r21021_set_clock()
665 * something for the form: f_ceil(N + .5) = N + 1; in tegra210_emc_r21021_set_clock()
667 tRPST = (last->emc_mrw & 0x80) >> 7; in tegra210_emc_r21021_set_clock()
668 tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) + in tegra210_emc_r21021_set_clock()
670 1 + nRTP; in tegra210_emc_r21021_set_clock()
672 emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM, in tegra210_emc_r21021_set_clock()
673 next->burst_regs[EMC_RP_INDEX]); in tegra210_emc_r21021_set_clock()
675 if (last->burst_regs[EMC_RP_INDEX] < tRTM) { in tegra210_emc_r21021_set_clock()
676 if (tRTM > (last->burst_regs[EMC_R2P_INDEX] + in tegra210_emc_r21021_set_clock()
677 last->burst_regs[EMC_RP_INDEX])) { in tegra210_emc_r21021_set_clock()
678 R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX]; in tegra210_emc_r21021_set_clock()
679 RP_war = last->burst_regs[EMC_RP_INDEX]; in tegra210_emc_r21021_set_clock()
680 TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; in tegra210_emc_r21021_set_clock()
684 last->burst_regs[EMC_RP_INDEX] - 63; in tegra210_emc_r21021_set_clock()
692 R2P_war = last->burst_regs[EMC_R2P_INDEX]; in tegra210_emc_r21021_set_clock()
693 RP_war = last->burst_regs[EMC_RP_INDEX]; in tegra210_emc_r21021_set_clock()
694 TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; in tegra210_emc_r21021_set_clock()
698 W2P_war = last->burst_regs[EMC_W2P_INDEX] in tegra210_emc_r21021_set_clock()
699 + deltaTWATM - RP_war; in tegra210_emc_r21021_set_clock()
701 RP_war = RP_war + W2P_war - 63; in tegra210_emc_r21021_set_clock()
707 W2P_war = last->burst_regs[ in tegra210_emc_r21021_set_clock()
711 if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) || in tegra210_emc_r21021_set_clock()
712 (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) || in tegra210_emc_r21021_set_clock()
713 (last->burst_regs[EMC_RP_INDEX] ^ RP_war) || in tegra210_emc_r21021_set_clock()
714 (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) { in tegra210_emc_r21021_set_clock()
715 emc_writel(emc, RP_war, EMC_RP); in tegra210_emc_r21021_set_clock()
716 emc_writel(emc, R2P_war, EMC_R2P); in tegra210_emc_r21021_set_clock()
717 emc_writel(emc, W2P_war, EMC_W2P); in tegra210_emc_r21021_set_clock()
718 emc_writel(emc, TRPab_war, EMC_TRPAB); in tegra210_emc_r21021_set_clock()
721 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_set_clock()
723 emc_dbg(emc, INFO, "Skipped WAR\n"); in tegra210_emc_r21021_set_clock()
728 mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80; in tegra210_emc_r21021_set_clock()
729 mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00; in tegra210_emc_r21021_set_clock()
731 mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40; in tegra210_emc_r21021_set_clock()
732 mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0; in tegra210_emc_r21021_set_clock()
736 emc_writel(emc, mr13_flip_fspwr, EMC_MRW3); in tegra210_emc_r21021_set_clock()
737 emc_writel(emc, next->emc_mrw, EMC_MRW); in tegra210_emc_r21021_set_clock()
738 emc_writel(emc, next->emc_mrw2, EMC_MRW2); in tegra210_emc_r21021_set_clock()
745 emc_dbg(emc, STEPS, "Step 8\n"); in tegra210_emc_r21021_set_clock()
746 emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n"); in tegra210_emc_r21021_set_clock()
748 for (i = 0; i < next->num_burst; i++) { in tegra210_emc_r21021_set_clock()
749 const u16 *offsets = emc->offsets->burst; in tegra210_emc_r21021_set_clock()
755 value = next->burst_regs[i]; in tegra210_emc_r21021_set_clock()
812 emc_writel(emc, value, offset); in tegra210_emc_r21021_set_clock()
815 /* SW addition: do EMC refresh adjustment here. */ in tegra210_emc_r21021_set_clock()
816 tegra210_emc_adjust_timing(emc, next); in tegra210_emc_r21021_set_clock()
820 (next->run_clocks & EMC_MRW_MRW_OP_MASK); in tegra210_emc_r21021_set_clock()
821 emc_writel(emc, value, EMC_MRW); in tegra210_emc_r21021_set_clock()
825 emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n"); in tegra210_emc_r21021_set_clock()
827 for (i = 0; i < next->num_burst_per_ch; i++) { in tegra210_emc_r21021_set_clock()
829 emc->offsets->burst_per_channel; in tegra210_emc_r21021_set_clock()
847 /* Filter out second channel if not in DUAL_CHANNEL mode. */ in tegra210_emc_r21021_set_clock()
848 if (emc->num_channels < 2 && burst[i].bank >= 1) in tegra210_emc_r21021_set_clock()
851 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
852 next->burst_reg_per_ch[i], burst[i].offset); in tegra210_emc_r21021_set_clock()
853 emc_channel_writel(emc, burst[i].bank, in tegra210_emc_r21021_set_clock()
854 next->burst_reg_per_ch[i], in tegra210_emc_r21021_set_clock()
859 emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n"); in tegra210_emc_r21021_set_clock()
861 for (i = 0; i < next->vref_num; i++) { in tegra210_emc_r21021_set_clock()
863 emc->offsets->vref_per_channel; in tegra210_emc_r21021_set_clock()
868 if (emc->num_channels < 2 && vref[i].bank >= 1) in tegra210_emc_r21021_set_clock()
871 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
872 next->vref_perch_regs[i], vref[i].offset); in tegra210_emc_r21021_set_clock()
873 emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i], in tegra210_emc_r21021_set_clock()
878 emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n"); in tegra210_emc_r21021_set_clock()
880 for (i = 0; i < next->num_trim; i++) { in tegra210_emc_r21021_set_clock()
881 const u16 *offsets = emc->offsets->trim; in tegra210_emc_r21021_set_clock()
898 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
900 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", in tegra210_emc_r21021_set_clock()
902 emc_writel(emc, value, offsets[i]); in tegra210_emc_r21021_set_clock()
904 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
905 next->trim_regs[i], offsets[i]); in tegra210_emc_r21021_set_clock()
906 emc_writel(emc, next->trim_regs[i], offsets[i]); in tegra210_emc_r21021_set_clock()
911 emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n"); in tegra210_emc_r21021_set_clock()
913 for (i = 0; i < next->num_trim_per_ch; i++) { in tegra210_emc_r21021_set_clock()
915 &emc->offsets->trim_per_channel[0]; in tegra210_emc_r21021_set_clock()
921 if (emc->num_channels < 2 && trim[i].bank >= 1) in tegra210_emc_r21021_set_clock()
938 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
940 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset, in tegra210_emc_r21021_set_clock()
942 emc_channel_writel(emc, trim[i].bank, value, offset); in tegra210_emc_r21021_set_clock()
944 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
945 next->trim_perch_regs[i], offset); in tegra210_emc_r21021_set_clock()
946 emc_channel_writel(emc, trim[i].bank, in tegra210_emc_r21021_set_clock()
947 next->trim_perch_regs[i], offset); in tegra210_emc_r21021_set_clock()
951 emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n"); in tegra210_emc_r21021_set_clock()
953 for (i = 0; i < next->num_mc_regs; i++) { in tegra210_emc_r21021_set_clock()
954 const u16 *offsets = emc->offsets->burst_mc; in tegra210_emc_r21021_set_clock()
955 u32 *values = next->burst_mc_regs; in tegra210_emc_r21021_set_clock()
957 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
959 mc_writel(emc->mc, values[i], offsets[i]); in tegra210_emc_r21021_set_clock()
963 if (next->rate < last->rate) { in tegra210_emc_r21021_set_clock()
964 const u16 *la = emc->offsets->la_scale; in tegra210_emc_r21021_set_clock()
966 emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n"); in tegra210_emc_r21021_set_clock()
968 for (i = 0; i < next->num_up_down; i++) { in tegra210_emc_r21021_set_clock()
969 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, in tegra210_emc_r21021_set_clock()
970 next->la_scale_regs[i], la[i]); in tegra210_emc_r21021_set_clock()
971 mc_writel(emc->mc, next->la_scale_regs[i], la[i]); in tegra210_emc_r21021_set_clock()
976 mc_readl(emc->mc, MC_EMEM_ADR_CFG); in tegra210_emc_r21021_set_clock()
982 emc_dbg(emc, STEPS, "Step 9\n"); in tegra210_emc_r21021_set_clock()
984 value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX]; in tegra210_emc_r21021_set_clock()
988 emc_writel(emc, 0, EMC_ZCAL_INTERVAL); in tegra210_emc_r21021_set_clock()
989 emc_writel(emc, value, EMC_ZCAL_WAIT_CNT); in tegra210_emc_r21021_set_clock()
994 emc_writel(emc, value, EMC_DBG); in tegra210_emc_r21021_set_clock()
995 emc_writel(emc, 0, EMC_ZCAL_INTERVAL); in tegra210_emc_r21021_set_clock()
996 emc_writel(emc, emc_dbg, EMC_DBG); in tegra210_emc_r21021_set_clock()
1003 emc_dbg(emc, STEPS, "Step 10\n"); in tegra210_emc_r21021_set_clock()
1007 ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1009 ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1013 ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0); in tegra210_emc_r21021_set_clock()
1014 ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] & in tegra210_emc_r21021_set_clock()
1016 (last->burst_regs[EMC_MRW6_INDEX] & in tegra210_emc_r21021_set_clock()
1018 ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] & in tegra210_emc_r21021_set_clock()
1020 (last->burst_regs[EMC_MRW14_INDEX] & in tegra210_emc_r21021_set_clock()
1023 if (emc->num_devices > 1) { in tegra210_emc_r21021_set_clock()
1024 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1025 (next->burst_regs[EMC_MRW7_INDEX] & in tegra210_emc_r21021_set_clock()
1027 (last->burst_regs[EMC_MRW7_INDEX] & in tegra210_emc_r21021_set_clock()
1029 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1030 (next->burst_regs[EMC_MRW15_INDEX] & in tegra210_emc_r21021_set_clock()
1032 (last->burst_regs[EMC_MRW15_INDEX] & in tegra210_emc_r21021_set_clock()
1037 if (emc->num_devices < 2) in tegra210_emc_r21021_set_clock()
1038 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1043 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1048 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1056 value = (1000 * fake->dram_timings[T_RP]) / src_clk_period; in tegra210_emc_r21021_set_clock()
1057 ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value); in tegra210_emc_r21021_set_clock()
1058 ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period); in tegra210_emc_r21021_set_clock()
1065 delay += (1000 * fake->dram_timings[T_RP]) / in tegra210_emc_r21021_set_clock()
1067 delay += 4000 * fake->dram_timings[T_RFC]; in tegra210_emc_r21021_set_clock()
1070 ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV | in tegra210_emc_r21021_set_clock()
1077 value = 1; in tegra210_emc_r21021_set_clock()
1089 delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) + in tegra210_emc_r21021_set_clock()
1090 (1000 * fake->dram_timings[T_RFC] / src_clk_period)); in tegra210_emc_r21021_set_clock()
1100 emc_dbg(emc, STEPS, "Step 11\n"); in tegra210_emc_r21021_set_clock()
1102 ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay); in tegra210_emc_r21021_set_clock()
1105 ccfifo_writel(emc, value, EMC_DBG, 0); in tegra210_emc_r21021_set_clock()
1107 ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period, in tegra210_emc_r21021_set_clock()
1112 * And finally - trigger the clock change. in tegra210_emc_r21021_set_clock()
1114 emc_dbg(emc, STEPS, "Step 12\n"); in tegra210_emc_r21021_set_clock()
1116 ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0); in tegra210_emc_r21021_set_clock()
1118 ccfifo_writel(emc, value, EMC_DBG, 0); in tegra210_emc_r21021_set_clock()
1124 emc_dbg(emc, STEPS, "Step 13\n"); in tegra210_emc_r21021_set_clock()
1126 ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0); in tegra210_emc_r21021_set_clock()
1127 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); in tegra210_emc_r21021_set_clock()
1133 emc_dbg(emc, STEPS, "Step 14\n"); in tegra210_emc_r21021_set_clock()
1138 if (emc->num_devices <= 1) in tegra210_emc_r21021_set_clock()
1143 ccfifo_writel(emc, value, EMC_PIN, 0); in tegra210_emc_r21021_set_clock()
1150 emc_dbg(emc, STEPS, "Step 15\n"); in tegra210_emc_r21021_set_clock()
1155 zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t; in tegra210_emc_r21021_set_clock()
1157 zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - in tegra210_emc_r21021_set_clock()
1158 div_o3(1000 * next->dram_timings[T_PDEX], in tegra210_emc_r21021_set_clock()
1162 emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj); in tegra210_emc_r21021_set_clock()
1163 emc_dbg(emc, INFO, "dst_clk_period = %u\n", in tegra210_emc_r21021_set_clock()
1165 emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n", in tegra210_emc_r21021_set_clock()
1166 next->dram_timings[T_PDEX]); in tegra210_emc_r21021_set_clock()
1167 emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n", in tegra210_emc_r21021_set_clock()
1171 delay = div_o3(1000 * next->dram_timings[T_PDEX], in tegra210_emc_r21021_set_clock()
1174 if (emc->num_devices < 2) { in tegra210_emc_r21021_set_clock()
1176 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1182 ccfifo_writel(emc, value, EMC_MRW3, delay); in tegra210_emc_r21021_set_clock()
1183 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1184 ccfifo_writel(emc, 0, EMC_REF, 0); in tegra210_emc_r21021_set_clock()
1185 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1191 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1196 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1200 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1205 ccfifo_writel(emc, value, EMC_MRW3, 0); in tegra210_emc_r21021_set_clock()
1206 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1207 ccfifo_writel(emc, 0, EMC_REF, 0); in tegra210_emc_r21021_set_clock()
1209 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1214 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD, in tegra210_emc_r21021_set_clock()
1218 ccfifo_writel(emc, value, EMC_MRW3, delay); in tegra210_emc_r21021_set_clock()
1219 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1220 ccfifo_writel(emc, 0, EMC_REF, 0); in tegra210_emc_r21021_set_clock()
1222 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, in tegra210_emc_r21021_set_clock()
1228 ccfifo_writel(emc, 0, 0, 10); in tegra210_emc_r21021_set_clock()
1239 emc_dbg(emc, STEPS, "Step 17\n"); in tegra210_emc_r21021_set_clock()
1242 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); in tegra210_emc_r21021_set_clock()
1248 emc_dbg(emc, STEPS, "Step 18\n"); in tegra210_emc_r21021_set_clock()
1251 ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0); in tegra210_emc_r21021_set_clock()
1252 ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0); in tegra210_emc_r21021_set_clock()
1254 ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0); in tegra210_emc_r21021_set_clock()
1257 ccfifo_writel(emc, next->emc_emrs & in tegra210_emc_r21021_set_clock()
1259 ccfifo_writel(emc, next->emc_emrs2 & in tegra210_emc_r21021_set_clock()
1261 ccfifo_writel(emc, next->emc_mrs | in tegra210_emc_r21021_set_clock()
1269 emc_dbg(emc, STEPS, "Step 19\n"); in tegra210_emc_r21021_set_clock()
1279 ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0); in tegra210_emc_r21021_set_clock()
1282 ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | in tegra210_emc_r21021_set_clock()
1288 if (emc->num_devices > 1) { in tegra210_emc_r21021_set_clock()
1289 value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | in tegra210_emc_r21021_set_clock()
1293 ccfifo_writel(emc, value, EMC_MRW, 0); in tegra210_emc_r21021_set_clock()
1298 ccfifo_writel(emc, value | in tegra210_emc_r21021_set_clock()
1303 if (emc->num_devices > 1) { in tegra210_emc_r21021_set_clock()
1304 value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | in tegra210_emc_r21021_set_clock()
1306 ccfifo_writel(emc, value, EMC_ZQ_CAL, 0); in tegra210_emc_r21021_set_clock()
1312 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1315 delay = (1250000 - ramp_up_wait) / dst_clk_period; in tegra210_emc_r21021_set_clock()
1319 ccfifo_writel(emc, in tegra210_emc_r21021_set_clock()
1320 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], in tegra210_emc_r21021_set_clock()
1322 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1329 emc_dbg(emc, STEPS, "Step 20\n"); in tegra210_emc_r21021_set_clock()
1332 ccfifo_writel(emc, 0, EMC_REF, 0); in tegra210_emc_r21021_set_clock()
1335 ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0); in tegra210_emc_r21021_set_clock()
1336 ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2); in tegra210_emc_r21021_set_clock()
1343 emc_dbg(emc, STEPS, "Step 21\n"); in tegra210_emc_r21021_set_clock()
1346 ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, in tegra210_emc_r21021_set_clock()
1349 ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], in tegra210_emc_r21021_set_clock()
1353 ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] & in tegra210_emc_r21021_set_clock()
1356 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); in tegra210_emc_r21021_set_clock()
1363 emc_dbg(emc, STEPS, "Step 22\n"); in tegra210_emc_r21021_set_clock()
1365 ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0); in tegra210_emc_r21021_set_clock()
1369 emc_writel(emc, in tegra210_emc_r21021_set_clock()
1370 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
1374 emc_writel(emc, in tegra210_emc_r21021_set_clock()
1375 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & in tegra210_emc_r21021_set_clock()
1383 emc_dbg(emc, STEPS, "Step 23\n"); in tegra210_emc_r21021_set_clock()
1385 value = emc_readl(emc, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
1392 emc_writel(emc, value, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
1394 tegra210_emc_do_clock_change(emc, clksrc); in tegra210_emc_r21021_set_clock()
1405 emc_dbg(emc, STEPS, "Step 25\n"); in tegra210_emc_r21021_set_clock()
1407 if (next->rate > last->rate) { in tegra210_emc_r21021_set_clock()
1408 for (i = 0; i < next->num_up_down; i++) in tegra210_emc_r21021_set_clock()
1409 mc_writel(emc->mc, next->la_scale_regs[i], in tegra210_emc_r21021_set_clock()
1410 emc->offsets->la_scale[i]); in tegra210_emc_r21021_set_clock()
1412 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_set_clock()
1419 emc_dbg(emc, STEPS, "Step 26\n"); in tegra210_emc_r21021_set_clock()
1422 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1423 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], in tegra210_emc_r21021_set_clock()
1425 emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], in tegra210_emc_r21021_set_clock()
1427 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1434 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1436 emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX], in tegra210_emc_r21021_set_clock()
1439 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], in tegra210_emc_r21021_set_clock()
1441 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1448 emc_dbg(emc, STEPS, "Step 27\n"); in tegra210_emc_r21021_set_clock()
1450 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1451 emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG); in tegra210_emc_r21021_set_clock()
1452 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1453 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp, in tegra210_emc_r21021_set_clock()
1455 emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); in tegra210_emc_r21021_set_clock()
1461 emc_dbg(emc, STEPS, "Step 28\n"); in tegra210_emc_r21021_set_clock()
1463 tegra210_emc_set_shadow_bypass(emc, ACTIVE); in tegra210_emc_r21021_set_clock()
1464 emc_writel(emc, in tegra210_emc_r21021_set_clock()
1465 next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], in tegra210_emc_r21021_set_clock()
1467 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); in tegra210_emc_r21021_set_clock()
1473 emc_dbg(emc, STEPS, "Step 29\n"); in tegra210_emc_r21021_set_clock()
1475 emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 | in tegra210_emc_r21021_set_clock()
1484 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, in tegra210_emc_r21021_set_clock()
1486 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, in tegra210_emc_r21021_set_clock()
1488 emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0); in tegra210_emc_r21021_set_clock()
1492 * Re-enable autocal. in tegra210_emc_r21021_set_clock()
1494 emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n"); in tegra210_emc_r21021_set_clock()
1496 if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) { in tegra210_emc_r21021_set_clock()
1497 value = emc_readl(emc, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
1504 emc_writel(emc, value, EMC_CFG_DIG_DLL); in tegra210_emc_r21021_set_clock()
1505 tegra210_emc_timing_update(emc); in tegra210_emc_r21021_set_clock()
1508 emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); in tegra210_emc_r21021_set_clock()