277 static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf)  in get_pagefault()  argument
282 	spin_lock_irq(&pf_queue->lock);  in get_pagefault()
283 	if (pf_queue->tail != pf_queue->head) {  in get_pagefault()
285 			(pf_queue->data + pf_queue->tail);  in get_pagefault()
303 		pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) %  in get_pagefault()
304 			pf_queue->num_dw;  in get_pagefault()
307 	spin_unlock_irq(&pf_queue->lock);  in get_pagefault()
312 static bool pf_queue_full(struct pf_queue *pf_queue)  in pf_queue_full()  argument
314 	lockdep_assert_held(&pf_queue->lock);  in pf_queue_full()
316 	return CIRC_SPACE(pf_queue->head, pf_queue->tail,  in pf_queue_full()
317 			  pf_queue->num_dw) <=  in pf_queue_full()
325 	struct pf_queue *pf_queue;  in xe_guc_pagefault_handler()  local
334 	pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE);  in xe_guc_pagefault_handler()
339 	xe_gt_assert(gt, !(pf_queue->num_dw % PF_MSG_LEN_DW));  in xe_guc_pagefault_handler()
341 	spin_lock_irqsave(&pf_queue->lock, flags);  in xe_guc_pagefault_handler()
342 	full = pf_queue_full(pf_queue);  in xe_guc_pagefault_handler()
344 		memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32));  in xe_guc_pagefault_handler()
345 		pf_queue->head = (pf_queue->head + len) %  in xe_guc_pagefault_handler()
346 			pf_queue->num_dw;  in xe_guc_pagefault_handler()
347 		queue_work(gt->usm.pf_wq, &pf_queue->worker);  in xe_guc_pagefault_handler()
351 	spin_unlock_irqrestore(&pf_queue->lock, flags);  in xe_guc_pagefault_handler()
360 	struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker);  in pf_queue_work_func()  local
361 	struct xe_gt *gt = pf_queue->gt;  in pf_queue_work_func()
370 	while (get_pagefault(pf_queue, &pf)) {  in pf_queue_work_func()
392 		    pf_queue->tail != pf_queue->head) {  in pf_queue_work_func()
413 static int xe_alloc_pf_queue(struct xe_gt *gt, struct pf_queue *pf_queue)  in xe_alloc_pf_queue()  argument
427 	pf_queue->num_dw =  in xe_alloc_pf_queue()
430 	pf_queue->gt = gt;  in xe_alloc_pf_queue()
431 	pf_queue->data = devm_kcalloc(xe->drm.dev, pf_queue->num_dw,  in xe_alloc_pf_queue()
433 	if (!pf_queue->data)  in xe_alloc_pf_queue()
436 	spin_lock_init(&pf_queue->lock);  in xe_alloc_pf_queue()
437 	INIT_WORK(&pf_queue->worker, pf_queue_work_func);  in xe_alloc_pf_queue()
451 		ret = xe_alloc_pf_queue(gt, &gt->usm.pf_queue[i]);  in xe_gt_pagefault_init()
486 		spin_lock_irq(&gt->usm.pf_queue[i].lock);  in xe_gt_pagefault_reset()
487 		gt->usm.pf_queue[i].head = 0;  in xe_gt_pagefault_reset()
488 		gt->usm.pf_queue[i].tail = 0;  in xe_gt_pagefault_reset()
489 		spin_unlock_irq(&gt->usm.pf_queue[i].lock);  in xe_gt_pagefault_reset()