1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
5 #include <linux/dma-mapping.h>
26 #define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1)
27 #define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1)
43 		vm->bts[pbe].cpu[bte] = 0;  in lima_vm_unmap_range()
52 	if (!vm->bts[pbe].cpu) {  in lima_vm_map_page()
57 		vm->bts[pbe].cpu = dma_alloc_wc(  in lima_vm_map_page()
58 			vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,  in lima_vm_map_page()
59 			&vm->bts[pbe].dma, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);  in lima_vm_map_page()
60 		if (!vm->bts[pbe].cpu)  in lima_vm_map_page()
61 			return -ENOMEM;  in lima_vm_map_page()
63 		pts = vm->bts[pbe].dma;  in lima_vm_map_page()
64 		pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT);  in lima_vm_map_page()
71 	vm->bts[pbe].cpu[bte] = pa | LIMA_VM_FLAGS_CACHE;  in lima_vm_map_page()
81 	list_for_each_entry(bo_va, &bo->va, list) {  in lima_vm_bo_find()
82 		if (bo_va->vm == vm) {  in lima_vm_bo_find()
95 	int offset = 0, err;  in lima_vm_bo_add()  local
97 	mutex_lock(&bo->lock);  in lima_vm_bo_add()
101 		bo_va->ref_count++;  in lima_vm_bo_add()
102 		mutex_unlock(&bo->lock);  in lima_vm_bo_add()
108 		mutex_unlock(&bo->lock);  in lima_vm_bo_add()
109 		return -ENOENT;  in lima_vm_bo_add()
114 		err = -ENOMEM;  in lima_vm_bo_add()
118 	bo_va->vm = vm;  in lima_vm_bo_add()
119 	bo_va->ref_count = 1;  in lima_vm_bo_add()
121 	mutex_lock(&vm->lock);  in lima_vm_bo_add()
123 	err = drm_mm_insert_node(&vm->mm, &bo_va->node, lima_bo_size(bo));  in lima_vm_bo_add()
127 	for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, 0) {  in lima_vm_bo_add()
129 				       bo_va->node.start + offset);  in lima_vm_bo_add()
133 		offset += PAGE_SIZE;  in lima_vm_bo_add()
136 	mutex_unlock(&vm->lock);  in lima_vm_bo_add()
138 	list_add_tail(&bo_va->list, &bo->va);  in lima_vm_bo_add()
140 	mutex_unlock(&bo->lock);  in lima_vm_bo_add()
144 	if (offset)  in lima_vm_bo_add()
145 		lima_vm_unmap_range(vm, bo_va->node.start, bo_va->node.start + offset - 1);  in lima_vm_bo_add()
146 	drm_mm_remove_node(&bo_va->node);  in lima_vm_bo_add()
148 	mutex_unlock(&vm->lock);  in lima_vm_bo_add()
151 	mutex_unlock(&bo->lock);  in lima_vm_bo_add()
160 	mutex_lock(&bo->lock);  in lima_vm_bo_del()
163 	if (--bo_va->ref_count > 0) {  in lima_vm_bo_del()
164 		mutex_unlock(&bo->lock);  in lima_vm_bo_del()
168 	mutex_lock(&vm->lock);  in lima_vm_bo_del()
170 	size = bo->heap_size ? bo->heap_size : bo_va->node.size;  in lima_vm_bo_del()
171 	lima_vm_unmap_range(vm, bo_va->node.start,  in lima_vm_bo_del()
172 			    bo_va->node.start + size - 1);  in lima_vm_bo_del()
174 	drm_mm_remove_node(&bo_va->node);  in lima_vm_bo_del()
176 	mutex_unlock(&vm->lock);  in lima_vm_bo_del()
178 	list_del(&bo_va->list);  in lima_vm_bo_del()
180 	mutex_unlock(&bo->lock);  in lima_vm_bo_del()
190 	mutex_lock(&bo->lock);  in lima_vm_get_va()
193 	ret = bo_va->node.start;  in lima_vm_get_va()
195 	mutex_unlock(&bo->lock);  in lima_vm_get_va()
208 	vm->dev = dev;  in lima_vm_create()
209 	mutex_init(&vm->lock);  in lima_vm_create()
210 	kref_init(&vm->refcount);  in lima_vm_create()
212 	vm->pd.cpu = dma_alloc_wc(dev->dev, LIMA_PAGE_SIZE, &vm->pd.dma,  in lima_vm_create()
214 	if (!vm->pd.cpu)  in lima_vm_create()
217 	if (dev->dlbu_cpu) {  in lima_vm_create()
219 			vm, dev->dlbu_dma, LIMA_VA_RESERVE_DLBU);  in lima_vm_create()
224 	drm_mm_init(&vm->mm, dev->va_start, dev->va_end - dev->va_start);  in lima_vm_create()
229 	dma_free_wc(dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);  in lima_vm_create()
240 	drm_mm_takedown(&vm->mm);  in lima_vm_release()
243 		if (vm->bts[i].cpu)  in lima_vm_release()
244 			dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,  in lima_vm_release()
245 				    vm->bts[i].cpu, vm->bts[i].dma);  in lima_vm_release()
248 	if (vm->pd.cpu)  in lima_vm_release()
249 		dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);  in lima_vm_release()
259 	if (!vm->pd.cpu)  in lima_vm_print()
262 	pd = vm->pd.cpu;  in lima_vm_print()
264 		if (!vm->bts[i].cpu)  in lima_vm_print()
267 		pt = vm->bts[i].cpu;  in lima_vm_print()
287 	int offset = 0, err;  in lima_vm_map_bo()  local
290 	mutex_lock(&bo->lock);  in lima_vm_map_bo()
294 		err = -ENOENT;  in lima_vm_map_bo()
298 	mutex_lock(&vm->lock);  in lima_vm_map_bo()
300 	base = bo_va->node.start + (pageoff << PAGE_SHIFT);  in lima_vm_map_bo()
301 	for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, pageoff) {  in lima_vm_map_bo()
303 				       base + offset);  in lima_vm_map_bo()
307 		offset += PAGE_SIZE;  in lima_vm_map_bo()
310 	mutex_unlock(&vm->lock);  in lima_vm_map_bo()
312 	mutex_unlock(&bo->lock);  in lima_vm_map_bo()
316 	if (offset)  in lima_vm_map_bo()
317 		lima_vm_unmap_range(vm, base, base + offset - 1);  in lima_vm_map_bo()
318 	mutex_unlock(&vm->lock);  in lima_vm_map_bo()
320 	mutex_unlock(&bo->lock);  in lima_vm_map_bo()