1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 */
9
10 #ifndef _ARM_SMMU_H
11 #define _ARM_SMMU_H
12
13 #include <linux/atomic.h>
14 #include <linux/bitfield.h>
15 #include <linux/bits.h>
16 #include <linux/clk.h>
17 #include <linux/device.h>
18 #include <linux/io-64-nonatomic-hi-lo.h>
19 #include <linux/io-pgtable.h>
20 #include <linux/iommu.h>
21 #include <linux/irqreturn.h>
22 #include <linux/mutex.h>
23 #include <linux/spinlock.h>
24 #include <linux/types.h>
25
26 /* Configuration registers */
27 #define ARM_SMMU_GR0_sCR0 0x0
28 #define ARM_SMMU_sCR0_VMID16EN BIT(31)
29 #define ARM_SMMU_sCR0_BSU GENMASK(15, 14)
30 #define ARM_SMMU_sCR0_FB BIT(13)
31 #define ARM_SMMU_sCR0_PTM BIT(12)
32 #define ARM_SMMU_sCR0_VMIDPNE BIT(11)
33 #define ARM_SMMU_sCR0_USFCFG BIT(10)
34 #define ARM_SMMU_sCR0_GCFGFIE BIT(5)
35 #define ARM_SMMU_sCR0_GCFGFRE BIT(4)
36 #define ARM_SMMU_sCR0_EXIDENABLE BIT(3)
37 #define ARM_SMMU_sCR0_GFIE BIT(2)
38 #define ARM_SMMU_sCR0_GFRE BIT(1)
39 #define ARM_SMMU_sCR0_CLIENTPD BIT(0)
40
41 /* Auxiliary Configuration register */
42 #define ARM_SMMU_GR0_sACR 0x10
43
44 /* Identification registers */
45 #define ARM_SMMU_GR0_ID0 0x20
46 #define ARM_SMMU_ID0_S1TS BIT(30)
47 #define ARM_SMMU_ID0_S2TS BIT(29)
48 #define ARM_SMMU_ID0_NTS BIT(28)
49 #define ARM_SMMU_ID0_SMS BIT(27)
50 #define ARM_SMMU_ID0_ATOSNS BIT(26)
51 #define ARM_SMMU_ID0_PTFS_NO_AARCH32 BIT(25)
52 #define ARM_SMMU_ID0_PTFS_NO_AARCH32S BIT(24)
53 #define ARM_SMMU_ID0_NUMIRPT GENMASK(23, 16)
54 #define ARM_SMMU_ID0_CTTW BIT(14)
55 #define ARM_SMMU_ID0_NUMSIDB GENMASK(12, 9)
56 #define ARM_SMMU_ID0_EXIDS BIT(8)
57 #define ARM_SMMU_ID0_NUMSMRG GENMASK(7, 0)
58
59 #define ARM_SMMU_GR0_ID1 0x24
60 #define ARM_SMMU_ID1_PAGESIZE BIT(31)
61 #define ARM_SMMU_ID1_NUMPAGENDXB GENMASK(30, 28)
62 #define ARM_SMMU_ID1_NUMS2CB GENMASK(23, 16)
63 #define ARM_SMMU_ID1_NUMCB GENMASK(7, 0)
64
65 #define ARM_SMMU_GR0_ID2 0x28
66 #define ARM_SMMU_ID2_VMID16 BIT(15)
67 #define ARM_SMMU_ID2_PTFS_64K BIT(14)
68 #define ARM_SMMU_ID2_PTFS_16K BIT(13)
69 #define ARM_SMMU_ID2_PTFS_4K BIT(12)
70 #define ARM_SMMU_ID2_UBS GENMASK(11, 8)
71 #define ARM_SMMU_ID2_OAS GENMASK(7, 4)
72 #define ARM_SMMU_ID2_IAS GENMASK(3, 0)
73
74 #define ARM_SMMU_GR0_ID3 0x2c
75 #define ARM_SMMU_GR0_ID4 0x30
76 #define ARM_SMMU_GR0_ID5 0x34
77 #define ARM_SMMU_GR0_ID6 0x38
78
79 #define ARM_SMMU_GR0_ID7 0x3c
80 #define ARM_SMMU_ID7_MAJOR GENMASK(7, 4)
81 #define ARM_SMMU_ID7_MINOR GENMASK(3, 0)
82
83 #define ARM_SMMU_GR0_sGFSR 0x48
84 #define ARM_SMMU_sGFSR_USF BIT(1)
85
86 #define ARM_SMMU_GR0_sGFSYNR0 0x50
87 #define ARM_SMMU_GR0_sGFSYNR1 0x54
88 #define ARM_SMMU_GR0_sGFSYNR2 0x58
89
90 /* Global TLB invalidation */
91 #define ARM_SMMU_GR0_TLBIVMID 0x64
92 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
93 #define ARM_SMMU_GR0_TLBIALLH 0x6c
94 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
95
96 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
97 #define ARM_SMMU_sTLBGSTATUS_GSACTIVE BIT(0)
98
99 /* Stream mapping registers */
100 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
101 #define ARM_SMMU_SMR_VALID BIT(31)
102 #define ARM_SMMU_SMR_MASK GENMASK(31, 16)
103 #define ARM_SMMU_SMR_ID GENMASK(15, 0)
104
105 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
106 #define ARM_SMMU_S2CR_PRIVCFG GENMASK(25, 24)
107 enum arm_smmu_s2cr_privcfg {
108 S2CR_PRIVCFG_DEFAULT,
109 S2CR_PRIVCFG_DIPAN,
110 S2CR_PRIVCFG_UNPRIV,
111 S2CR_PRIVCFG_PRIV,
112 };
113 #define ARM_SMMU_S2CR_TYPE GENMASK(17, 16)
114 enum arm_smmu_s2cr_type {
115 S2CR_TYPE_TRANS,
116 S2CR_TYPE_BYPASS,
117 S2CR_TYPE_FAULT,
118 };
119 #define ARM_SMMU_S2CR_EXIDVALID BIT(10)
120 #define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0)
121
122 /* Context bank attribute registers */
123 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
124 #define ARM_SMMU_CBAR_IRPTNDX GENMASK(31, 24)
125 #define ARM_SMMU_CBAR_TYPE GENMASK(17, 16)
126 enum arm_smmu_cbar_type {
127 CBAR_TYPE_S2_TRANS,
128 CBAR_TYPE_S1_TRANS_S2_BYPASS,
129 CBAR_TYPE_S1_TRANS_S2_FAULT,
130 CBAR_TYPE_S1_TRANS_S2_TRANS,
131 };
132 #define ARM_SMMU_CBAR_S1_MEMATTR GENMASK(15, 12)
133 #define ARM_SMMU_CBAR_S1_MEMATTR_WB 0xf
134 #define ARM_SMMU_CBAR_S1_BPSHCFG GENMASK(9, 8)
135 #define ARM_SMMU_CBAR_S1_BPSHCFG_NSH 3
136 #define ARM_SMMU_CBAR_VMID GENMASK(7, 0)
137
138 #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
139 #define ARM_SMMU_CBFRSYNRA_SID GENMASK(15, 0)
140
141 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
142 #define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16)
143 #define ARM_SMMU_CBA2R_VA64 BIT(0)
144
145 #define ARM_SMMU_CB_SCTLR 0x0
146 #define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12)
147 #define ARM_SMMU_SCTLR_CFCFG BIT(7)
148 #define ARM_SMMU_SCTLR_HUPCF BIT(8)
149 #define ARM_SMMU_SCTLR_CFIE BIT(6)
150 #define ARM_SMMU_SCTLR_CFRE BIT(5)
151 #define ARM_SMMU_SCTLR_E BIT(4)
152 #define ARM_SMMU_SCTLR_AFE BIT(2)
153 #define ARM_SMMU_SCTLR_TRE BIT(1)
154 #define ARM_SMMU_SCTLR_M BIT(0)
155
156 #define ARM_SMMU_CB_ACTLR 0x4
157
158 #define ARM_SMMU_CB_RESUME 0x8
159 #define ARM_SMMU_RESUME_TERMINATE BIT(0)
160
161 #define ARM_SMMU_CB_TCR2 0x10
162 #define ARM_SMMU_TCR2_SEP GENMASK(17, 15)
163 #define ARM_SMMU_TCR2_SEP_UPSTREAM 0x7
164 #define ARM_SMMU_TCR2_AS BIT(4)
165 #define ARM_SMMU_TCR2_PASIZE GENMASK(3, 0)
166
167 #define ARM_SMMU_CB_TTBR0 0x20
168 #define ARM_SMMU_CB_TTBR1 0x28
169 #define ARM_SMMU_TTBRn_ASID GENMASK_ULL(63, 48)
170
171 #define ARM_SMMU_CB_TCR 0x30
172 #define ARM_SMMU_TCR_EAE BIT(31)
173 #define ARM_SMMU_TCR_EPD1 BIT(23)
174 #define ARM_SMMU_TCR_A1 BIT(22)
175 #define ARM_SMMU_TCR_TG0 GENMASK(15, 14)
176 #define ARM_SMMU_TCR_SH0 GENMASK(13, 12)
177 #define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10)
178 #define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8)
179 #define ARM_SMMU_TCR_EPD0 BIT(7)
180 #define ARM_SMMU_TCR_T0SZ GENMASK(5, 0)
181
182 #define ARM_SMMU_VTCR_RES1 BIT(31)
183 #define ARM_SMMU_VTCR_PS GENMASK(18, 16)
184 #define ARM_SMMU_VTCR_TG0 ARM_SMMU_TCR_TG0
185 #define ARM_SMMU_VTCR_SH0 ARM_SMMU_TCR_SH0
186 #define ARM_SMMU_VTCR_ORGN0 ARM_SMMU_TCR_ORGN0
187 #define ARM_SMMU_VTCR_IRGN0 ARM_SMMU_TCR_IRGN0
188 #define ARM_SMMU_VTCR_SL0 GENMASK(7, 6)
189 #define ARM_SMMU_VTCR_T0SZ ARM_SMMU_TCR_T0SZ
190
191 #define ARM_SMMU_CB_CONTEXTIDR 0x34
192 #define ARM_SMMU_CB_S1_MAIR0 0x38
193 #define ARM_SMMU_CB_S1_MAIR1 0x3c
194
195 #define ARM_SMMU_CB_PAR 0x50
196 #define ARM_SMMU_CB_PAR_F BIT(0)
197
198 #define ARM_SMMU_CB_FSR 0x58
199 #define ARM_SMMU_CB_FSR_MULTI BIT(31)
200 #define ARM_SMMU_CB_FSR_SS BIT(30)
201 #define ARM_SMMU_CB_FSR_FORMAT GENMASK(10, 9)
202 #define ARM_SMMU_CB_FSR_UUT BIT(8)
203 #define ARM_SMMU_CB_FSR_ASF BIT(7)
204 #define ARM_SMMU_CB_FSR_TLBLKF BIT(6)
205 #define ARM_SMMU_CB_FSR_TLBMCF BIT(5)
206 #define ARM_SMMU_CB_FSR_EF BIT(4)
207 #define ARM_SMMU_CB_FSR_PF BIT(3)
208 #define ARM_SMMU_CB_FSR_AFF BIT(2)
209 #define ARM_SMMU_CB_FSR_TF BIT(1)
210
211 #define ARM_SMMU_CB_FSR_IGN (ARM_SMMU_CB_FSR_AFF | \
212 ARM_SMMU_CB_FSR_ASF | \
213 ARM_SMMU_CB_FSR_TLBMCF | \
214 ARM_SMMU_CB_FSR_TLBLKF)
215
216 #define ARM_SMMU_CB_FSR_FAULT (ARM_SMMU_CB_FSR_MULTI | \
217 ARM_SMMU_CB_FSR_SS | \
218 ARM_SMMU_CB_FSR_UUT | \
219 ARM_SMMU_CB_FSR_EF | \
220 ARM_SMMU_CB_FSR_PF | \
221 ARM_SMMU_CB_FSR_TF | \
222 ARM_SMMU_CB_FSR_IGN)
223
224 #define ARM_SMMU_CB_FAR 0x60
225
226 #define ARM_SMMU_CB_FSYNR0 0x68
227 #define ARM_SMMU_CB_FSYNR0_PLVL GENMASK(1, 0)
228 #define ARM_SMMU_CB_FSYNR0_WNR BIT(4)
229 #define ARM_SMMU_CB_FSYNR0_PNU BIT(5)
230 #define ARM_SMMU_CB_FSYNR0_IND BIT(6)
231 #define ARM_SMMU_CB_FSYNR0_NSATTR BIT(8)
232 #define ARM_SMMU_CB_FSYNR0_PTWF BIT(10)
233 #define ARM_SMMU_CB_FSYNR0_AFR BIT(11)
234 #define ARM_SMMU_CB_FSYNR0_S1CBNDX GENMASK(23, 16)
235
236 #define ARM_SMMU_CB_FSYNR1 0x6c
237
238 #define ARM_SMMU_CB_S1_TLBIVA 0x600
239 #define ARM_SMMU_CB_S1_TLBIASID 0x610
240 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
241 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
242 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
243 #define ARM_SMMU_CB_TLBSYNC 0x7f0
244 #define ARM_SMMU_CB_TLBSTATUS 0x7f4
245 #define ARM_SMMU_CB_ATS1PR 0x800
246
247 #define ARM_SMMU_CB_ATSR 0x8f0
248 #define ARM_SMMU_CB_ATSR_ACTIVE BIT(0)
249
250 #define ARM_SMMU_RESUME_TERMINATE BIT(0)
251
252 /* Maximum number of context banks per SMMU */
253 #define ARM_SMMU_MAX_CBS 128
254
255 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
256 #define TLB_SPIN_COUNT 10
257
258 /* Shared driver definitions */
259 enum arm_smmu_arch_version {
260 ARM_SMMU_V1,
261 ARM_SMMU_V1_64K,
262 ARM_SMMU_V2,
263 };
264
265 enum arm_smmu_implementation {
266 GENERIC_SMMU,
267 ARM_MMU500,
268 CAVIUM_SMMUV2,
269 QCOM_SMMUV2,
270 };
271
272 struct arm_smmu_s2cr {
273 struct iommu_group *group;
274 int count;
275 enum arm_smmu_s2cr_type type;
276 enum arm_smmu_s2cr_privcfg privcfg;
277 u8 cbndx;
278 };
279
280 struct arm_smmu_smr {
281 u16 mask;
282 u16 id;
283 bool valid;
284 bool pinned;
285 };
286
287 struct arm_smmu_device {
288 struct device *dev;
289
290 void __iomem *base;
291 phys_addr_t ioaddr;
292 unsigned int numpage;
293 unsigned int pgshift;
294
295 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
296 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
297 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
298 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
299 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
300 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
301 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
302 #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
303 #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
304 #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
305 #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
306 #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
307 #define ARM_SMMU_FEAT_EXIDS (1 << 12)
308 u32 features;
309
310 enum arm_smmu_arch_version version;
311 enum arm_smmu_implementation model;
312 const struct arm_smmu_impl *impl;
313
314 u32 num_context_banks;
315 u32 num_s2_context_banks;
316 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
317 struct arm_smmu_cb *cbs;
318 atomic_t irptndx;
319
320 u32 num_mapping_groups;
321 u16 streamid_mask;
322 u16 smr_mask_mask;
323 struct arm_smmu_smr *smrs;
324 struct arm_smmu_s2cr *s2crs;
325 struct mutex stream_map_mutex;
326
327 unsigned long va_size;
328 unsigned long ipa_size;
329 unsigned long pa_size;
330 unsigned long pgsize_bitmap;
331
332 int num_context_irqs;
333 int num_clks;
334 unsigned int *irqs;
335 struct clk_bulk_data *clks;
336
337 spinlock_t global_sync_lock;
338
339 /* IOMMU core code handle */
340 struct iommu_device iommu;
341 };
342
343 enum arm_smmu_context_fmt {
344 ARM_SMMU_CTX_FMT_NONE,
345 ARM_SMMU_CTX_FMT_AARCH64,
346 ARM_SMMU_CTX_FMT_AARCH32_L,
347 ARM_SMMU_CTX_FMT_AARCH32_S,
348 };
349
350 struct arm_smmu_cfg {
351 u8 cbndx;
352 u8 irptndx;
353 union {
354 u16 asid;
355 u16 vmid;
356 };
357 enum arm_smmu_cbar_type cbar;
358 enum arm_smmu_context_fmt fmt;
359 bool flush_walk_prefer_tlbiasid;
360 };
361 #define ARM_SMMU_INVALID_IRPTNDX 0xff
362
363 struct arm_smmu_cb {
364 u64 ttbr[2];
365 u32 tcr[2];
366 u32 mair[2];
367 struct arm_smmu_cfg *cfg;
368 };
369
370 enum arm_smmu_domain_stage {
371 ARM_SMMU_DOMAIN_S1 = 0,
372 ARM_SMMU_DOMAIN_S2,
373 ARM_SMMU_DOMAIN_NESTED,
374 };
375
376 struct arm_smmu_domain {
377 struct arm_smmu_device *smmu;
378 struct io_pgtable_ops *pgtbl_ops;
379 unsigned long pgtbl_quirks;
380 const struct iommu_flush_ops *flush_ops;
381 struct arm_smmu_cfg cfg;
382 enum arm_smmu_domain_stage stage;
383 struct mutex init_mutex; /* Protects smmu pointer */
384 spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
385 struct iommu_domain domain;
386 };
387
388 struct arm_smmu_master_cfg {
389 struct arm_smmu_device *smmu;
390 s16 smendx[];
391 };
392
arm_smmu_lpae_tcr(const struct io_pgtable_cfg * cfg)393 static inline u32 arm_smmu_lpae_tcr(const struct io_pgtable_cfg *cfg)
394 {
395 u32 tcr = FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) |
396 FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) |
397 FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) |
398 FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) |
399 FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
400
401 /*
402 * When TTBR1 is selected shift the TCR fields by 16 bits and disable
403 * translation in TTBR0
404 */
405 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
406 tcr = (tcr << 16) & ~ARM_SMMU_TCR_A1;
407 tcr |= ARM_SMMU_TCR_EPD0;
408 } else
409 tcr |= ARM_SMMU_TCR_EPD1;
410
411 return tcr;
412 }
413
arm_smmu_lpae_tcr2(const struct io_pgtable_cfg * cfg)414 static inline u32 arm_smmu_lpae_tcr2(const struct io_pgtable_cfg *cfg)
415 {
416 return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) |
417 FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
418 }
419
arm_smmu_lpae_vtcr(const struct io_pgtable_cfg * cfg)420 static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg)
421 {
422 return ARM_SMMU_VTCR_RES1 |
423 FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) |
424 FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) |
425 FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) |
426 FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) |
427 FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) |
428 FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) |
429 FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
430 }
431
432 /* Implementation details, yay! */
433 struct arm_smmu_impl {
434 u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
435 void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
436 u32 val);
437 u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset);
438 void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset,
439 u64 val);
440 int (*cfg_probe)(struct arm_smmu_device *smmu);
441 int (*reset)(struct arm_smmu_device *smmu);
442 int (*init_context)(struct arm_smmu_domain *smmu_domain,
443 struct io_pgtable_cfg *cfg, struct device *dev);
444 void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
445 int status);
446 int (*def_domain_type)(struct device *dev);
447 irqreturn_t (*global_fault)(int irq, void *dev);
448 irqreturn_t (*context_fault)(int irq, void *dev);
449 bool context_fault_needs_threaded_irq;
450 int (*alloc_context_bank)(struct arm_smmu_domain *smmu_domain,
451 struct arm_smmu_device *smmu,
452 struct device *dev, int start);
453 void (*write_s2cr)(struct arm_smmu_device *smmu, int idx);
454 void (*write_sctlr)(struct arm_smmu_device *smmu, int idx, u32 reg);
455 void (*probe_finalize)(struct arm_smmu_device *smmu, struct device *dev);
456 };
457
458 #define INVALID_SMENDX -1
459 #define cfg_smendx(cfg, fw, i) \
460 (i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i])
461 #define for_each_cfg_sme(cfg, fw, i, idx) \
462 for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i)
463
__arm_smmu_alloc_bitmap(unsigned long * map,int start,int end)464 static inline int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
465 {
466 int idx;
467
468 do {
469 idx = find_next_zero_bit(map, end, start);
470 if (idx == end)
471 return -ENOSPC;
472 } while (test_and_set_bit(idx, map));
473
474 return idx;
475 }
476
arm_smmu_page(struct arm_smmu_device * smmu,int n)477 static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
478 {
479 return smmu->base + (n << smmu->pgshift);
480 }
481
arm_smmu_readl(struct arm_smmu_device * smmu,int page,int offset)482 static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
483 {
484 if (smmu->impl && unlikely(smmu->impl->read_reg))
485 return smmu->impl->read_reg(smmu, page, offset);
486 return readl_relaxed(arm_smmu_page(smmu, page) + offset);
487 }
488
arm_smmu_writel(struct arm_smmu_device * smmu,int page,int offset,u32 val)489 static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
490 int offset, u32 val)
491 {
492 if (smmu->impl && unlikely(smmu->impl->write_reg))
493 smmu->impl->write_reg(smmu, page, offset, val);
494 else
495 writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
496 }
497
arm_smmu_readq(struct arm_smmu_device * smmu,int page,int offset)498 static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
499 {
500 if (smmu->impl && unlikely(smmu->impl->read_reg64))
501 return smmu->impl->read_reg64(smmu, page, offset);
502 return readq_relaxed(arm_smmu_page(smmu, page) + offset);
503 }
504
arm_smmu_writeq(struct arm_smmu_device * smmu,int page,int offset,u64 val)505 static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
506 int offset, u64 val)
507 {
508 if (smmu->impl && unlikely(smmu->impl->write_reg64))
509 smmu->impl->write_reg64(smmu, page, offset, val);
510 else
511 writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
512 }
513
514 #define ARM_SMMU_GR0 0
515 #define ARM_SMMU_GR1 1
516 #define ARM_SMMU_CB(s, n) ((s)->numpage + (n))
517
518 #define arm_smmu_gr0_read(s, o) \
519 arm_smmu_readl((s), ARM_SMMU_GR0, (o))
520 #define arm_smmu_gr0_write(s, o, v) \
521 arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
522
523 #define arm_smmu_gr1_read(s, o) \
524 arm_smmu_readl((s), ARM_SMMU_GR1, (o))
525 #define arm_smmu_gr1_write(s, o, v) \
526 arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
527
528 #define arm_smmu_cb_read(s, n, o) \
529 arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
530 #define arm_smmu_cb_write(s, n, o, v) \
531 arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
532 #define arm_smmu_cb_readq(s, n, o) \
533 arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
534 #define arm_smmu_cb_writeq(s, n, o, v) \
535 arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
536
537 struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
538 struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu);
539 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);
540
541 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx);
542 int arm_mmu500_reset(struct arm_smmu_device *smmu);
543
544 struct arm_smmu_context_fault_info {
545 unsigned long iova;
546 u32 fsr;
547 u32 fsynr;
548 u32 cbfrsynra;
549 };
550
551 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
552 struct arm_smmu_context_fault_info *cfi);
553
554 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
555 const struct arm_smmu_context_fault_info *cfi);
556
557 #endif /* _ARM_SMMU_H */
558