1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Memory allocator for buffers shared with the TrustZone.
4 *
5 * Copyright (C) 2023-2024 Linaro Ltd.
6 */
7
8 #include <linux/bug.h>
9 #include <linux/cleanup.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/err.h>
12 #include <linux/firmware/qcom/qcom_tzmem.h>
13 #include <linux/genalloc.h>
14 #include <linux/gfp.h>
15 #include <linux/kernel.h>
16 #include <linux/list.h>
17 #include <linux/mm.h>
18 #include <linux/radix-tree.h>
19 #include <linux/slab.h>
20 #include <linux/spinlock.h>
21 #include <linux/types.h>
22
23 #include "qcom_tzmem.h"
24
25 struct qcom_tzmem_area {
26 struct list_head list;
27 void *vaddr;
28 dma_addr_t paddr;
29 size_t size;
30 void *priv;
31 };
32
33 struct qcom_tzmem_pool {
34 struct gen_pool *genpool;
35 struct list_head areas;
36 enum qcom_tzmem_policy policy;
37 size_t increment;
38 size_t max_size;
39 spinlock_t lock;
40 };
41
42 struct qcom_tzmem_chunk {
43 size_t size;
44 struct qcom_tzmem_pool *owner;
45 };
46
47 static struct device *qcom_tzmem_dev;
48 static RADIX_TREE(qcom_tzmem_chunks, GFP_ATOMIC);
49 static DEFINE_SPINLOCK(qcom_tzmem_chunks_lock);
50
51 #if IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_GENERIC)
52
qcom_tzmem_init(void)53 static int qcom_tzmem_init(void)
54 {
55 return 0;
56 }
57
qcom_tzmem_init_area(struct qcom_tzmem_area * area)58 static int qcom_tzmem_init_area(struct qcom_tzmem_area *area)
59 {
60 return 0;
61 }
62
qcom_tzmem_cleanup_area(struct qcom_tzmem_area * area)63 static void qcom_tzmem_cleanup_area(struct qcom_tzmem_area *area)
64 {
65
66 }
67
68 #elif IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE)
69
70 #include <linux/firmware/qcom/qcom_scm.h>
71 #include <linux/of.h>
72
73 #define QCOM_SHM_BRIDGE_NUM_VM_SHIFT 9
74
75 static bool qcom_tzmem_using_shm_bridge;
76
77 /* List of machines that are known to not support SHM bridge correctly. */
78 static const char *const qcom_tzmem_blacklist[] = {
79 "qcom,sc8180x",
80 "qcom,sdm670", /* failure in GPU firmware loading */
81 "qcom,sdm845", /* reset in rmtfs memory assignment */
82 "qcom,sm8150", /* reset in rmtfs memory assignment */
83 NULL
84 };
85
qcom_tzmem_init(void)86 static int qcom_tzmem_init(void)
87 {
88 const char *const *platform;
89 int ret;
90
91 for (platform = qcom_tzmem_blacklist; *platform; platform++) {
92 if (of_machine_is_compatible(*platform))
93 goto notsupp;
94 }
95
96 ret = qcom_scm_shm_bridge_enable();
97 if (ret == -EOPNOTSUPP)
98 goto notsupp;
99
100 if (!ret)
101 qcom_tzmem_using_shm_bridge = true;
102
103 return ret;
104
105 notsupp:
106 dev_info(qcom_tzmem_dev, "SHM Bridge not supported\n");
107 return 0;
108 }
109
qcom_tzmem_init_area(struct qcom_tzmem_area * area)110 static int qcom_tzmem_init_area(struct qcom_tzmem_area *area)
111 {
112 u64 pfn_and_ns_perm, ipfn_and_s_perm, size_and_flags;
113 int ret;
114
115 if (!qcom_tzmem_using_shm_bridge)
116 return 0;
117
118 pfn_and_ns_perm = (u64)area->paddr | QCOM_SCM_PERM_RW;
119 ipfn_and_s_perm = (u64)area->paddr | QCOM_SCM_PERM_RW;
120 size_and_flags = area->size | (1 << QCOM_SHM_BRIDGE_NUM_VM_SHIFT);
121
122 u64 *handle __free(kfree) = kzalloc(sizeof(*handle), GFP_KERNEL);
123 if (!handle)
124 return -ENOMEM;
125
126 ret = qcom_scm_shm_bridge_create(qcom_tzmem_dev, pfn_and_ns_perm,
127 ipfn_and_s_perm, size_and_flags,
128 QCOM_SCM_VMID_HLOS, handle);
129 if (ret)
130 return ret;
131
132 area->priv = no_free_ptr(handle);
133
134 return 0;
135 }
136
qcom_tzmem_cleanup_area(struct qcom_tzmem_area * area)137 static void qcom_tzmem_cleanup_area(struct qcom_tzmem_area *area)
138 {
139 u64 *handle = area->priv;
140
141 if (!qcom_tzmem_using_shm_bridge)
142 return;
143
144 qcom_scm_shm_bridge_delete(qcom_tzmem_dev, *handle);
145 kfree(handle);
146 }
147
148 #endif /* CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE */
149
qcom_tzmem_pool_add_memory(struct qcom_tzmem_pool * pool,size_t size,gfp_t gfp)150 static int qcom_tzmem_pool_add_memory(struct qcom_tzmem_pool *pool,
151 size_t size, gfp_t gfp)
152 {
153 int ret;
154
155 struct qcom_tzmem_area *area __free(kfree) = kzalloc(sizeof(*area),
156 gfp);
157 if (!area)
158 return -ENOMEM;
159
160 area->size = PAGE_ALIGN(size);
161
162 area->vaddr = dma_alloc_coherent(qcom_tzmem_dev, area->size,
163 &area->paddr, gfp);
164 if (!area->vaddr)
165 return -ENOMEM;
166
167 ret = qcom_tzmem_init_area(area);
168 if (ret) {
169 dma_free_coherent(qcom_tzmem_dev, area->size,
170 area->vaddr, area->paddr);
171 return ret;
172 }
173
174 ret = gen_pool_add_virt(pool->genpool, (unsigned long)area->vaddr,
175 (phys_addr_t)area->paddr, size, -1);
176 if (ret) {
177 dma_free_coherent(qcom_tzmem_dev, area->size,
178 area->vaddr, area->paddr);
179 return ret;
180 }
181
182 scoped_guard(spinlock_irqsave, &pool->lock)
183 list_add_tail(&area->list, &pool->areas);
184
185 area = NULL;
186 return 0;
187 }
188
189 /**
190 * qcom_tzmem_pool_new() - Create a new TZ memory pool.
191 * @config: Pool configuration.
192 *
193 * Create a new pool of memory suitable for sharing with the TrustZone.
194 *
195 * Must not be used in atomic context.
196 *
197 * Return: New memory pool address or ERR_PTR() on error.
198 */
199 struct qcom_tzmem_pool *
qcom_tzmem_pool_new(const struct qcom_tzmem_pool_config * config)200 qcom_tzmem_pool_new(const struct qcom_tzmem_pool_config *config)
201 {
202 int ret = -ENOMEM;
203
204 might_sleep();
205
206 switch (config->policy) {
207 case QCOM_TZMEM_POLICY_STATIC:
208 if (!config->initial_size)
209 return ERR_PTR(-EINVAL);
210 break;
211 case QCOM_TZMEM_POLICY_MULTIPLIER:
212 if (!config->increment)
213 return ERR_PTR(-EINVAL);
214 break;
215 case QCOM_TZMEM_POLICY_ON_DEMAND:
216 break;
217 default:
218 return ERR_PTR(-EINVAL);
219 }
220
221 struct qcom_tzmem_pool *pool __free(kfree) = kzalloc(sizeof(*pool),
222 GFP_KERNEL);
223 if (!pool)
224 return ERR_PTR(-ENOMEM);
225
226 pool->genpool = gen_pool_create(PAGE_SHIFT, -1);
227 if (!pool->genpool)
228 return ERR_PTR(-ENOMEM);
229
230 gen_pool_set_algo(pool->genpool, gen_pool_best_fit, NULL);
231
232 pool->policy = config->policy;
233 pool->increment = config->increment;
234 pool->max_size = config->max_size;
235 INIT_LIST_HEAD(&pool->areas);
236 spin_lock_init(&pool->lock);
237
238 if (config->initial_size) {
239 ret = qcom_tzmem_pool_add_memory(pool, config->initial_size,
240 GFP_KERNEL);
241 if (ret) {
242 gen_pool_destroy(pool->genpool);
243 return ERR_PTR(ret);
244 }
245 }
246
247 return_ptr(pool);
248 }
249 EXPORT_SYMBOL_GPL(qcom_tzmem_pool_new);
250
251 /**
252 * qcom_tzmem_pool_free() - Destroy a TZ memory pool and free all resources.
253 * @pool: Memory pool to free.
254 *
255 * Must not be called if any of the allocated chunks has not been freed.
256 * Must not be used in atomic context.
257 */
qcom_tzmem_pool_free(struct qcom_tzmem_pool * pool)258 void qcom_tzmem_pool_free(struct qcom_tzmem_pool *pool)
259 {
260 struct qcom_tzmem_area *area, *next;
261 struct qcom_tzmem_chunk *chunk;
262 struct radix_tree_iter iter;
263 bool non_empty = false;
264 void __rcu **slot;
265
266 might_sleep();
267
268 if (!pool)
269 return;
270
271 scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock) {
272 radix_tree_for_each_slot(slot, &qcom_tzmem_chunks, &iter, 0) {
273 chunk = radix_tree_deref_slot_protected(slot,
274 &qcom_tzmem_chunks_lock);
275
276 if (chunk->owner == pool)
277 non_empty = true;
278 }
279 }
280
281 WARN(non_empty, "Freeing TZ memory pool with memory still allocated");
282
283 list_for_each_entry_safe(area, next, &pool->areas, list) {
284 list_del(&area->list);
285 qcom_tzmem_cleanup_area(area);
286 dma_free_coherent(qcom_tzmem_dev, area->size,
287 area->vaddr, area->paddr);
288 kfree(area);
289 }
290
291 gen_pool_destroy(pool->genpool);
292 kfree(pool);
293 }
294 EXPORT_SYMBOL_GPL(qcom_tzmem_pool_free);
295
devm_qcom_tzmem_pool_free(void * data)296 static void devm_qcom_tzmem_pool_free(void *data)
297 {
298 struct qcom_tzmem_pool *pool = data;
299
300 qcom_tzmem_pool_free(pool);
301 }
302
303 /**
304 * devm_qcom_tzmem_pool_new() - Managed variant of qcom_tzmem_pool_new().
305 * @dev: Device managing this resource.
306 * @config: Pool configuration.
307 *
308 * Must not be used in atomic context.
309 *
310 * Return: Address of the managed pool or ERR_PTR() on failure.
311 */
312 struct qcom_tzmem_pool *
devm_qcom_tzmem_pool_new(struct device * dev,const struct qcom_tzmem_pool_config * config)313 devm_qcom_tzmem_pool_new(struct device *dev,
314 const struct qcom_tzmem_pool_config *config)
315 {
316 struct qcom_tzmem_pool *pool;
317 int ret;
318
319 pool = qcom_tzmem_pool_new(config);
320 if (IS_ERR(pool))
321 return pool;
322
323 ret = devm_add_action_or_reset(dev, devm_qcom_tzmem_pool_free, pool);
324 if (ret)
325 return ERR_PTR(ret);
326
327 return pool;
328 }
329 EXPORT_SYMBOL_GPL(devm_qcom_tzmem_pool_new);
330
qcom_tzmem_try_grow_pool(struct qcom_tzmem_pool * pool,size_t requested,gfp_t gfp)331 static bool qcom_tzmem_try_grow_pool(struct qcom_tzmem_pool *pool,
332 size_t requested, gfp_t gfp)
333 {
334 size_t current_size = gen_pool_size(pool->genpool);
335
336 if (pool->max_size && (current_size + requested) > pool->max_size)
337 return false;
338
339 switch (pool->policy) {
340 case QCOM_TZMEM_POLICY_STATIC:
341 return false;
342 case QCOM_TZMEM_POLICY_MULTIPLIER:
343 requested = current_size * pool->increment;
344 break;
345 case QCOM_TZMEM_POLICY_ON_DEMAND:
346 break;
347 }
348
349 return !qcom_tzmem_pool_add_memory(pool, requested, gfp);
350 }
351
352 /**
353 * qcom_tzmem_alloc() - Allocate a memory chunk suitable for sharing with TZ.
354 * @pool: TZ memory pool from which to allocate memory.
355 * @size: Number of bytes to allocate.
356 * @gfp: GFP flags.
357 *
358 * Can be used in any context.
359 *
360 * Return:
361 * Address of the allocated buffer or NULL if no more memory can be allocated.
362 * The buffer must be released using qcom_tzmem_free().
363 */
qcom_tzmem_alloc(struct qcom_tzmem_pool * pool,size_t size,gfp_t gfp)364 void *qcom_tzmem_alloc(struct qcom_tzmem_pool *pool, size_t size, gfp_t gfp)
365 {
366 unsigned long vaddr;
367 int ret;
368
369 if (!size)
370 return NULL;
371
372 size = PAGE_ALIGN(size);
373
374 struct qcom_tzmem_chunk *chunk __free(kfree) = kzalloc(sizeof(*chunk),
375 gfp);
376 if (!chunk)
377 return NULL;
378
379 again:
380 vaddr = gen_pool_alloc(pool->genpool, size);
381 if (!vaddr) {
382 if (qcom_tzmem_try_grow_pool(pool, size, gfp))
383 goto again;
384
385 return NULL;
386 }
387
388 chunk->size = size;
389 chunk->owner = pool;
390
391 scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock) {
392 ret = radix_tree_insert(&qcom_tzmem_chunks, vaddr, chunk);
393 if (ret) {
394 gen_pool_free(pool->genpool, vaddr, size);
395 return NULL;
396 }
397
398 chunk = NULL;
399 }
400
401 return (void *)vaddr;
402 }
403 EXPORT_SYMBOL_GPL(qcom_tzmem_alloc);
404
405 /**
406 * qcom_tzmem_free() - Release a buffer allocated from a TZ memory pool.
407 * @vaddr: Virtual address of the buffer.
408 *
409 * Can be used in any context.
410 */
qcom_tzmem_free(void * vaddr)411 void qcom_tzmem_free(void *vaddr)
412 {
413 struct qcom_tzmem_chunk *chunk;
414
415 scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock)
416 chunk = radix_tree_delete_item(&qcom_tzmem_chunks,
417 (unsigned long)vaddr, NULL);
418
419 if (!chunk) {
420 WARN(1, "Virtual address %p not owned by TZ memory allocator",
421 vaddr);
422 return;
423 }
424
425 scoped_guard(spinlock_irqsave, &chunk->owner->lock)
426 gen_pool_free(chunk->owner->genpool, (unsigned long)vaddr,
427 chunk->size);
428 kfree(chunk);
429 }
430 EXPORT_SYMBOL_GPL(qcom_tzmem_free);
431
432 /**
433 * qcom_tzmem_to_phys() - Map the virtual address of TZ memory to physical.
434 * @vaddr: Virtual address of memory allocated from a TZ memory pool.
435 *
436 * Can be used in any context. The address must point to memory allocated
437 * using qcom_tzmem_alloc().
438 *
439 * Returns:
440 * Physical address mapped from the virtual or 0 if the mapping failed.
441 */
qcom_tzmem_to_phys(void * vaddr)442 phys_addr_t qcom_tzmem_to_phys(void *vaddr)
443 {
444 struct qcom_tzmem_chunk *chunk;
445 struct radix_tree_iter iter;
446 void __rcu **slot;
447 phys_addr_t ret;
448
449 guard(spinlock_irqsave)(&qcom_tzmem_chunks_lock);
450
451 radix_tree_for_each_slot(slot, &qcom_tzmem_chunks, &iter, 0) {
452 chunk = radix_tree_deref_slot_protected(slot,
453 &qcom_tzmem_chunks_lock);
454
455 ret = gen_pool_virt_to_phys(chunk->owner->genpool,
456 (unsigned long)vaddr);
457 if (ret == -1)
458 continue;
459
460 return ret;
461 }
462
463 return 0;
464 }
465 EXPORT_SYMBOL_GPL(qcom_tzmem_to_phys);
466
qcom_tzmem_enable(struct device * dev)467 int qcom_tzmem_enable(struct device *dev)
468 {
469 if (qcom_tzmem_dev)
470 return -EBUSY;
471
472 qcom_tzmem_dev = dev;
473
474 return qcom_tzmem_init();
475 }
476 EXPORT_SYMBOL_GPL(qcom_tzmem_enable);
477
478 MODULE_DESCRIPTION("TrustZone memory allocator for Qualcomm firmware drivers");
479 MODULE_AUTHOR("Bartosz Golaszewski <bartosz.golaszewski@linaro.org>");
480 MODULE_LICENSE("GPL");
481