1 // SPDX-License-Identifier: GPL-2.0
8 #include <linux/dma-map-ops.h>
9 #include <linux/dma-direct.h>
23 /* Size can be defined by the coherent_pool command line */
46 static void dma_atomic_pool_size_add(gfp_t gfp, size_t size)  in dma_atomic_pool_size_add()  argument
49 		pool_size_dma += size;  in dma_atomic_pool_size_add()
51 		pool_size_dma32 += size;  in dma_atomic_pool_size_add()
53 		pool_size_kernel += size;  in dma_atomic_pool_size_add()
58 	unsigned long size;  in cma_in_zone()  local
66 	size = cma_get_size(cma);  in cma_in_zone()
67 	if (!size)  in cma_in_zone()
71 	end = cma_get_base(cma) + size - 1;  in cma_in_zone()
83 	struct page *page = NULL;  in atomic_pool_expand()  local
85 	int ret = -ENOMEM;  in atomic_pool_expand()
93 			page = dma_alloc_from_contiguous(NULL, 1 << order,  in atomic_pool_expand()
95 		if (!page)  in atomic_pool_expand()
96 			page = alloc_pages(gfp, order);  in atomic_pool_expand()
97 	} while (!page && order-- > 0);  in atomic_pool_expand()
98 	if (!page)  in atomic_pool_expand()
101 	arch_dma_prep_coherent(page, pool_size);  in atomic_pool_expand()
104 	addr = dma_common_contiguous_remap(page, pool_size,  in atomic_pool_expand()
110 	addr = page_to_virt(page);  in atomic_pool_expand()
114 	 * shrink so no re-encryption occurs in dma_direct_free().  in atomic_pool_expand()
116 	ret = set_memory_decrypted((unsigned long)page_to_virt(page),  in atomic_pool_expand()
120 	ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),  in atomic_pool_expand()
129 	ret = set_memory_encrypted((unsigned long)page_to_virt(page),  in atomic_pool_expand()
139 	__free_pages(page, order);  in atomic_pool_expand()
205 		ret = -ENOMEM;  in dma_atomic_pool_init()
210 			ret = -ENOMEM;  in dma_atomic_pool_init()
216 			ret = -ENOMEM;  in dma_atomic_pool_init()
240 static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,  in __dma_alloc_from_pool()  argument
247 	addr = gen_pool_alloc(pool, size);  in __dma_alloc_from_pool()
252 	if (phys_addr_ok && !phys_addr_ok(dev, phys, size)) {  in __dma_alloc_from_pool()
253 		gen_pool_free(pool, addr, size);  in __dma_alloc_from_pool()
261 	memset(*cpu_addr, 0, size);  in __dma_alloc_from_pool()
265 struct page *dma_alloc_from_pool(struct device *dev, size_t size,  in dma_alloc_from_pool()  argument
270 	struct page *page;  in dma_alloc_from_pool()  local
273 		page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,  in dma_alloc_from_pool()
275 		if (page)  in dma_alloc_from_pool()
276 			return page;  in dma_alloc_from_pool()
283 bool dma_free_from_pool(struct device *dev, void *start, size_t size)  in dma_free_from_pool()  argument
288 		if (!gen_pool_has_addr(pool, (unsigned long)start, size))  in dma_free_from_pool()
290 		gen_pool_free(pool, (unsigned long)start, size);  in dma_free_from_pool()