1 // SPDX-License-Identifier: MIT
8 #include <linux/dma-fence-array.h>
38 	mmdrop(ufence->mm);  in user_fence_destroy()
44 	kref_get(&ufence->refcount);  in user_fence_get()
49 	kref_put(&ufence->refcount, user_fence_destroy);  in user_fence_put()
60 		return ERR_PTR(-EFAULT);  in user_fence_create()
64 		return ERR_PTR(-ENOMEM);  in user_fence_create()
66 	ufence->xe = xe;  in user_fence_create()
67 	kref_init(&ufence->refcount);  in user_fence_create()
68 	ufence->addr = ptr;  in user_fence_create()
69 	ufence->value = value;  in user_fence_create()
70 	ufence->mm = current->mm;  in user_fence_create()
71 	mmgrab(ufence->mm);  in user_fence_create()
80 	if (mmget_not_zero(ufence->mm)) {  in user_fence_worker()
81 		kthread_use_mm(ufence->mm);  in user_fence_worker()
82 		if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))  in user_fence_worker()
84 		kthread_unuse_mm(ufence->mm);  in user_fence_worker()
85 		mmput(ufence->mm);  in user_fence_worker()
88 	wake_up_all(&ufence->xe->ufence_wq);  in user_fence_worker()
89 	WRITE_ONCE(ufence->signalled, 1);  in user_fence_worker()
95 	INIT_WORK(&ufence->worker, user_fence_worker);  in kick_ufence()
96 	queue_work(ufence->xe->ordered_wq, &ufence->worker);  in kick_ufence()
108 			struct xe_sync_entry *sync,  in xe_sync_entry_parse()  argument
120 		return -EFAULT;  in xe_sync_entry_parse()
123 	    XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))  in xe_sync_entry_parse()
124 		return -EINVAL;  in xe_sync_entry_parse()
130 			return -EOPNOTSUPP;  in xe_sync_entry_parse()
133 			return -EINVAL;  in xe_sync_entry_parse()
135 		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);  in xe_sync_entry_parse()
136 		if (XE_IOCTL_DBG(xe, !sync->syncobj))  in xe_sync_entry_parse()
137 			return -ENOENT;  in xe_sync_entry_parse()
140 			sync->fence = drm_syncobj_fence_get(sync->syncobj);  in xe_sync_entry_parse()
141 			if (XE_IOCTL_DBG(xe, !sync->fence))  in xe_sync_entry_parse()
142 				return -EINVAL;  in xe_sync_entry_parse()
148 			return -EOPNOTSUPP;  in xe_sync_entry_parse()
151 			return -EINVAL;  in xe_sync_entry_parse()
154 			return -EINVAL;  in xe_sync_entry_parse()
156 		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);  in xe_sync_entry_parse()
157 		if (XE_IOCTL_DBG(xe, !sync->syncobj))  in xe_sync_entry_parse()
158 			return -ENOENT;  in xe_sync_entry_parse()
161 			sync->chain_fence = dma_fence_chain_alloc();  in xe_sync_entry_parse()
162 			if (!sync->chain_fence)  in xe_sync_entry_parse()
163 				return -ENOMEM;  in xe_sync_entry_parse()
165 			sync->fence = drm_syncobj_fence_get(sync->syncobj);  in xe_sync_entry_parse()
166 			if (XE_IOCTL_DBG(xe, !sync->fence))  in xe_sync_entry_parse()
167 				return -EINVAL;  in xe_sync_entry_parse()
169 			err = dma_fence_chain_find_seqno(&sync->fence,  in xe_sync_entry_parse()
178 			return -EOPNOTSUPP;  in xe_sync_entry_parse()
181 			return -EOPNOTSUPP;  in xe_sync_entry_parse()
184 			return -EINVAL;  in xe_sync_entry_parse()
187 			sync->addr = sync_in.addr;  in xe_sync_entry_parse()
189 			sync->ufence = user_fence_create(xe, sync_in.addr,  in xe_sync_entry_parse()
191 			if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence)))  in xe_sync_entry_parse()
192 				return PTR_ERR(sync->ufence);  in xe_sync_entry_parse()
198 		return -EINVAL;  in xe_sync_entry_parse()
201 	sync->type = sync_in.type;  in xe_sync_entry_parse()
202 	sync->flags = sync_in.flags;  in xe_sync_entry_parse()
203 	sync->timeline_value = sync_in.timeline_value;  in xe_sync_entry_parse()
208 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)  in xe_sync_entry_add_deps()  argument
210 	if (sync->fence)  in xe_sync_entry_add_deps()
211 		return  drm_sched_job_add_dependency(&job->drm,  in xe_sync_entry_add_deps()
212 						     dma_fence_get(sync->fence));  in xe_sync_entry_add_deps()
217 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)  in xe_sync_entry_signal()  argument
219 	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))  in xe_sync_entry_signal()
222 	if (sync->chain_fence) {  in xe_sync_entry_signal()
223 		drm_syncobj_add_point(sync->syncobj, sync->chain_fence,  in xe_sync_entry_signal()
224 				      fence, sync->timeline_value);  in xe_sync_entry_signal()
229 		sync->chain_fence = NULL;  in xe_sync_entry_signal()
230 	} else if (sync->syncobj) {  in xe_sync_entry_signal()
231 		drm_syncobj_replace_fence(sync->syncobj, fence);  in xe_sync_entry_signal()
232 	} else if (sync->ufence) {  in xe_sync_entry_signal()
236 		user_fence_get(sync->ufence);  in xe_sync_entry_signal()
237 		err = dma_fence_add_callback(fence, &sync->ufence->cb,  in xe_sync_entry_signal()
239 		if (err == -ENOENT) {  in xe_sync_entry_signal()
240 			kick_ufence(sync->ufence, fence);  in xe_sync_entry_signal()
243 			user_fence_put(sync->ufence);  in xe_sync_entry_signal()
249 void xe_sync_entry_cleanup(struct xe_sync_entry *sync)  in xe_sync_entry_cleanup()  argument
251 	if (sync->syncobj)  in xe_sync_entry_cleanup()
252 		drm_syncobj_put(sync->syncobj);  in xe_sync_entry_cleanup()
253 	dma_fence_put(sync->fence);  in xe_sync_entry_cleanup()
254 	dma_fence_chain_free(sync->chain_fence);  in xe_sync_entry_cleanup()
255 	if (sync->ufence)  in xe_sync_entry_cleanup()
256 		user_fence_put(sync->ufence);  in xe_sync_entry_cleanup()
260  * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
261  * @sync: input syncs
266  * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
267  * and return a composite fence of all in-fences + last fence. If no in-fences
271  * Return: fence on success, ERR_PTR(-ENOMEM) on failure
274 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,  in xe_sync_in_fence_get()  argument
282 	lockdep_assert_held(&vm->lock);  in xe_sync_in_fence_get()
284 	/* Count in-fences */  in xe_sync_in_fence_get()
286 		if (sync[i].fence) {  in xe_sync_in_fence_get()
288 			fence = sync[i].fence;  in xe_sync_in_fence_get()
299 	fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL);  in xe_sync_in_fence_get()
301 		return ERR_PTR(-ENOMEM);  in xe_sync_in_fence_get()
303 		if (sync[i].fence) {  in xe_sync_in_fence_get()
304 			dma_fence_get(sync[i].fence);  in xe_sync_in_fence_get()
305 			fences[current_fence++] = sync[i].fence;  in xe_sync_in_fence_get()
310 				    vm->composite_fence_ctx,  in xe_sync_in_fence_get()
311 				    vm->composite_fence_seqno++,  in xe_sync_in_fence_get()
314 		--vm->composite_fence_seqno;  in xe_sync_in_fence_get()
318 	return &cf->base;  in xe_sync_in_fence_get()
322 		dma_fence_put(fences[--current_fence]);  in xe_sync_in_fence_get()
326 	return ERR_PTR(-ENOMEM);  in xe_sync_in_fence_get()
330  * __xe_sync_ufence_get() - Get user fence from user fence
345  * xe_sync_ufence_get() - Get user fence from sync
346  * @sync: input sync
348  * Get a user fence reference from sync.
352 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)  in xe_sync_ufence_get()  argument
354 	user_fence_get(sync->ufence);  in xe_sync_ufence_get()
356 	return sync->ufence;  in xe_sync_ufence_get()
360  * xe_sync_ufence_put() - Put user fence reference
370  * xe_sync_ufence_get_status() - Get user fence status
373  * Return: 1 if signalled, 0 not signalled, <0 on error
377 	return READ_ONCE(ufence->signalled);  in xe_sync_ufence_get_status()