1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include <linux/align.h>
7 #include <linux/bitfield.h>
8 #include <linux/log2.h>
9 #include <linux/sizes.h>
10
11 #include "xe_lmtt_types.h"
12 #include "xe_macros.h"
13
14 /**
15 * DOC: Multi-Level LMTT Structure
16 *
17 * LMHAW (Local Memory Host Address Width) is 48 bit (256TB)
18 *
19 * LMGAW (Local Memory Guest Address Width) is 48 bit (256TB)
20 *
21 * The following figure illustrates the structure and function of the ML LMTT::
22 *
23 * LMTT L3 Directory
24 * (1 Entry per VF) LMTT L1 Leaf
25 * +-----------+ +-----------+
26 * | | LMTT L2 (per VF) | |
27 * | | +-----------+ | |
28 * | | | | index: +===========+
29 * | | | | GDPA --> | PTE | => LMEM PF offset
30 * | | | | 34:21 +===========+
31 * | | index: | | | |
32 * | | LMEM VF +===========+ | |
33 * | | offset -> | PTE | ----------> +-----------+
34 * | | GAW-1:35 +===========+ / \.
35 * index: +===========+ | | / \.
36 * VFID --> | PDE | ---------> +-----------+ / \.
37 * +===========+ / / / \.
38 * | | / / / \.
39 * +-----------+ <== [LMTT Directory Ptr] / \.
40 * / \ / / / \.
41 * / \ / / +-----------+-----------------+------+---+
42 * / /\ / | 31:HAW-16 | HAW-17:5 | 4:1 | 0 |
43 * / / \ / +===========+=================+======+===+
44 * / / \ / | Reserved | LMEM Page (2MB) | Rsvd | V |
45 * / / +-----------+-----------------+------+---+
46 * / /
47 * +-----------+-----------------+------+---+
48 * | 63:HAW-12 | HAW-13:4 | 3:1 | 0 |
49 * +===========+=================+======+===+
50 * | Reserved | LMTT Ptr (64KB) | Rsvd | V |
51 * +-----------+-----------------+------+---+
52 *
53 */
54
55 typedef u64 lmtt_ml_pde_t;
56 typedef u32 lmtt_ml_pte_t;
57
58 #define LMTT_ML_HAW 48 /* 256 TiB */
59
60 #define LMTT_ML_PDE_MAX_NUM 64 /* SRIOV with PF and 63 VFs, index 0 (PF) is unused */
61 #define LMTT_ML_PDE_LMTT_PTR GENMASK_ULL(LMTT_ML_HAW - 13, 4)
62 #define LMTT_ML_PDE_VALID BIT(0)
63
64 #define LMTT_ML_PDE_L2_SHIFT 35
65 #define LMTT_ML_PDE_L2_MAX_NUM BIT_ULL(LMTT_ML_HAW - 35)
66
67 #define LMTT_ML_PTE_MAX_NUM BIT(35 - ilog2(SZ_2M))
68 #define LMTT_ML_PTE_LMEM_PAGE GENMASK(LMTT_ML_HAW - 17, 5)
69 #define LMTT_ML_PTE_VALID BIT(0)
70
lmtt_ml_root_pd_level(void)71 static unsigned int lmtt_ml_root_pd_level(void)
72 {
73 return 2; /* implementation is 0-based */
74 }
75
lmtt_ml_pte_num(unsigned int level)76 static unsigned int lmtt_ml_pte_num(unsigned int level)
77 {
78 switch (level) {
79 case 2:
80 return LMTT_ML_PDE_MAX_NUM;
81 case 1:
82 BUILD_BUG_ON(LMTT_ML_HAW == 48 && LMTT_ML_PDE_L2_MAX_NUM != SZ_8K);
83 return LMTT_ML_PDE_L2_MAX_NUM;
84 case 0:
85 BUILD_BUG_ON(LMTT_ML_PTE_MAX_NUM != SZ_16K);
86 return LMTT_ML_PTE_MAX_NUM;
87 default:
88 return 0;
89 }
90 }
91
lmtt_ml_pte_size(unsigned int level)92 static unsigned int lmtt_ml_pte_size(unsigned int level)
93 {
94 switch (level) {
95 case 2:
96 case 1:
97 return sizeof(lmtt_ml_pde_t);
98 case 0:
99 return sizeof(lmtt_ml_pte_t);
100 default:
101 return 0;
102 }
103 }
104
lmtt_ml_pte_shift(unsigned int level)105 static unsigned int lmtt_ml_pte_shift(unsigned int level)
106 {
107 switch (level) {
108 case 1:
109 BUILD_BUG_ON(BIT_ULL(LMTT_ML_PDE_L2_SHIFT) != SZ_32G);
110 return ilog2(SZ_32G);
111 case 0:
112 return ilog2(SZ_2M);
113 default:
114 return 0;
115 }
116 }
117
lmtt_ml_pte_index(u64 addr,unsigned int level)118 static unsigned int lmtt_ml_pte_index(u64 addr, unsigned int level)
119 {
120 addr >>= lmtt_ml_pte_shift(level);
121
122 switch (level) {
123 case 1:
124 /* SZ_32G increments */
125 BUILD_BUG_ON_NOT_POWER_OF_2(LMTT_ML_PDE_L2_MAX_NUM);
126 return addr & (LMTT_ML_PDE_L2_MAX_NUM - 1);
127 case 0:
128 /* SZ_2M increments */
129 BUILD_BUG_ON_NOT_POWER_OF_2(LMTT_ML_PTE_MAX_NUM);
130 return addr & (LMTT_ML_PTE_MAX_NUM - 1);
131 default:
132 return 0;
133 }
134 }
135
lmtt_ml_pte_encode(unsigned long offset,unsigned int level)136 static u64 lmtt_ml_pte_encode(unsigned long offset, unsigned int level)
137 {
138 switch (level) {
139 case 0:
140 XE_WARN_ON(!IS_ALIGNED(offset, SZ_2M));
141 XE_WARN_ON(!FIELD_FIT(LMTT_ML_PTE_LMEM_PAGE, offset / SZ_2M));
142 return FIELD_PREP(LMTT_ML_PTE_LMEM_PAGE, offset / SZ_2M) | LMTT_ML_PTE_VALID;
143 case 1:
144 case 2:
145 XE_WARN_ON(!IS_ALIGNED(offset, SZ_64K));
146 XE_WARN_ON(!FIELD_FIT(LMTT_ML_PDE_LMTT_PTR, offset / SZ_64K));
147 return FIELD_PREP(LMTT_ML_PDE_LMTT_PTR, offset / SZ_64K) | LMTT_ML_PDE_VALID;
148 default:
149 XE_WARN_ON(true);
150 return 0;
151 }
152 }
153
154 const struct xe_lmtt_ops lmtt_ml_ops = {
155 .lmtt_root_pd_level = lmtt_ml_root_pd_level,
156 .lmtt_pte_num = lmtt_ml_pte_num,
157 .lmtt_pte_size = lmtt_ml_pte_size,
158 .lmtt_pte_shift = lmtt_ml_pte_shift,
159 .lmtt_pte_index = lmtt_ml_pte_index,
160 .lmtt_pte_encode = lmtt_ml_pte_encode,
161 };
162