1 // SPDX-License-Identifier: MIT
20  * We use a per-vm interval tree to keep track of vma_resources
37 #define VMA_RES_START(_node) ((_node)->start - (_node)->guard)
38 #define VMA_RES_LAST(_node) ((_node)->start + (_node)->node_size + (_node)->guard - 1)
43 /* Callbacks for the unbind dma-fence. */
46  * i915_vma_resource_alloc - Allocate a vma resource
49  * a -ENOMEM error pointer if allocation fails.
56 	return vma_res ? vma_res : ERR_PTR(-ENOMEM);  in i915_vma_resource_alloc()
60  * i915_vma_resource_free - Free a vma resource
92 	i915_sw_fence_fini(&vma_res->chain);  in unbind_fence_release()
94 	call_rcu(&fence->rcu, unbind_fence_free_rcu);  in unbind_fence_release()
107 	if (!refcount_dec_and_test(&vma_res->hold_count))  in __i915_vma_resource_unhold()
110 	dma_fence_signal(&vma_res->unbind_fence);  in __i915_vma_resource_unhold()
112 	vm = vma_res->vm;  in __i915_vma_resource_unhold()
113 	if (vma_res->wakeref)  in __i915_vma_resource_unhold()
114 		intel_runtime_pm_put(&vm->i915->runtime_pm, vma_res->wakeref);  in __i915_vma_resource_unhold()
116 	vma_res->vm = NULL;  in __i915_vma_resource_unhold()
117 	if (!RB_EMPTY_NODE(&vma_res->rb)) {  in __i915_vma_resource_unhold()
118 		mutex_lock(&vm->mutex);  in __i915_vma_resource_unhold()
119 		vma_res_itree_remove(vma_res, &vm->pending_unbind);  in __i915_vma_resource_unhold()
120 		mutex_unlock(&vm->mutex);  in __i915_vma_resource_unhold()
123 	if (vma_res->bi.pages_rsgt)  in __i915_vma_resource_unhold()
124 		i915_refct_sgt_put(vma_res->bi.pages_rsgt);  in __i915_vma_resource_unhold()
128  * i915_vma_resource_unhold - Unhold the signaling of the vma resource unbind
143 		/* Inefficient open-coded might_lock_irqsave() */  in i915_vma_resource_unhold()
144 		spin_lock_irqsave(&vma_res->lock, irq_flags);  in i915_vma_resource_unhold()
145 		spin_unlock_irqrestore(&vma_res->lock, irq_flags);  in i915_vma_resource_unhold()
152  * i915_vma_resource_hold - Hold the signaling of the vma resource unbind fence.
165 	bool held = refcount_inc_not_zero(&vma_res->hold_count);  in i915_vma_resource_hold()
177 	struct i915_address_space *vm = vma_res->vm;  in i915_vma_resource_unbind_work()
181 	if (likely(!vma_res->skip_pte_rewrite))  in i915_vma_resource_unbind_work()
182 		vma_res->ops->unbind_vma(vm, vma_res);  in i915_vma_resource_unbind_work()
196 		&vma_res->unbind_fence;  in i915_vma_resource_fence_notify()
201 		if (vma_res->immediate_unbind) {  in i915_vma_resource_fence_notify()
202 			i915_vma_resource_unbind_work(&vma_res->work);  in i915_vma_resource_fence_notify()
204 			INIT_WORK(&vma_res->work, i915_vma_resource_unbind_work);  in i915_vma_resource_fence_notify()
205 			queue_work(system_unbound_wq, &vma_res->work);  in i915_vma_resource_fence_notify()
217  * i915_vma_resource_unbind - Unbind a vma resource
219  * @tlb: pointer to vma->obj->mm.tlb associated with the resource
220  *	 to be stored at vma_res->tlb. When not-NULL, it will be used
227  * Return: A refcounted pointer to a dma-fence that signals when unbinding is
233 	struct i915_address_space *vm = vma_res->vm;  in i915_vma_resource_unbind()
235 	vma_res->tlb = tlb;  in i915_vma_resource_unbind()
241 	if (vma_res->needs_wakeref)  in i915_vma_resource_unbind()
242 		vma_res->wakeref = intel_runtime_pm_get_if_in_use(&vm->i915->runtime_pm);  in i915_vma_resource_unbind()
244 	if (atomic_read(&vma_res->chain.pending) <= 1) {  in i915_vma_resource_unbind()
245 		RB_CLEAR_NODE(&vma_res->rb);  in i915_vma_resource_unbind()
246 		vma_res->immediate_unbind = 1;  in i915_vma_resource_unbind()
248 		vma_res_itree_insert(vma_res, &vma_res->vm->pending_unbind);  in i915_vma_resource_unbind()
251 	i915_sw_fence_commit(&vma_res->chain);  in i915_vma_resource_unbind()
253 	return &vma_res->unbind_fence;  in i915_vma_resource_unbind()
257  * __i915_vma_resource_init - Initialize a vma resource.
264 	spin_lock_init(&vma_res->lock);  in __i915_vma_resource_init()
265 	dma_fence_init(&vma_res->unbind_fence, &unbind_fence_ops,  in __i915_vma_resource_init()
266 		       &vma_res->lock, 0, 0);  in __i915_vma_resource_init()
267 	refcount_set(&vma_res->hold_count, 1);  in __i915_vma_resource_init()
268 	i915_sw_fence_init(&vma_res->chain, i915_vma_resource_fence_notify);  in __i915_vma_resource_init()
278 			*start -= I915_GTT_PAGE_SIZE;  in i915_vma_resource_color_adjust_range()
284  * i915_vma_resource_bind_dep_sync - Wait for / sync all unbinds touching a
293  * Return: Zero on success, -ERESTARTSYS if interrupted and @intr==true
301 	u64 last = offset + size - 1;  in i915_vma_resource_bind_dep_sync()
303 	lockdep_assert_held(&vm->mutex);  in i915_vma_resource_bind_dep_sync()
307 	node = vma_res_itree_iter_first(&vm->pending_unbind, offset, last);  in i915_vma_resource_bind_dep_sync()
309 		int ret = dma_fence_wait(&node->unbind_fence, intr);  in i915_vma_resource_bind_dep_sync()
321  * i915_vma_resource_bind_dep_sync_all - Wait for / sync all unbinds of a vm,
337 		mutex_lock(&vm->mutex);  in i915_vma_resource_bind_dep_sync_all()
338 		node = vma_res_itree_iter_first(&vm->pending_unbind, 0,  in i915_vma_resource_bind_dep_sync_all()
341 			fence = dma_fence_get_rcu(&node->unbind_fence);  in i915_vma_resource_bind_dep_sync_all()
342 		mutex_unlock(&vm->mutex);  in i915_vma_resource_bind_dep_sync_all()
356  * i915_vma_resource_bind_dep_await - Have a struct i915_sw_fence await all
369  * wait for the unbind fence to signal, using @intr to judge whether to
377  * Return: Zero on success, -ERESTARTSYS if interrupted and @intr==true
387 	u64 last = offset + size - 1;  in i915_vma_resource_bind_dep_await()
389 	lockdep_assert_held(&vm->mutex);  in i915_vma_resource_bind_dep_await()
394 	node = vma_res_itree_iter_first(&vm->pending_unbind, offset, last);  in i915_vma_resource_bind_dep_await()
399 						    &node->unbind_fence,  in i915_vma_resource_bind_dep_await()
402 			ret = dma_fence_wait(&node->unbind_fence, intr);  in i915_vma_resource_bind_dep_await()
422 		return -ENOMEM;  in i915_vma_resource_module_init()