19 static bool irq_enable(struct intel_breadcrumbs *b)  in irq_enable()  argument
21 	return intel_engine_irq_enable(b->irq_engine);  in irq_enable()
24 static void irq_disable(struct intel_breadcrumbs *b)  in irq_disable()  argument
26 	intel_engine_irq_disable(b->irq_engine);  in irq_disable()
29 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_arm_irq()  argument
37 	wakeref = intel_gt_pm_get_if_awake(b->irq_engine->gt);  in __intel_breadcrumbs_arm_irq()
47 	WRITE_ONCE(b->irq_armed, wakeref);  in __intel_breadcrumbs_arm_irq()
50 	if (!b->irq_enabled++ && b->irq_enable(b))  in __intel_breadcrumbs_arm_irq()
51 		irq_work_queue(&b->irq_work);  in __intel_breadcrumbs_arm_irq()
54 static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)  in intel_breadcrumbs_arm_irq()  argument
56 	if (!b->irq_engine)  in intel_breadcrumbs_arm_irq()
59 	spin_lock(&b->irq_lock);  in intel_breadcrumbs_arm_irq()
60 	if (!b->irq_armed)  in intel_breadcrumbs_arm_irq()
61 		__intel_breadcrumbs_arm_irq(b);  in intel_breadcrumbs_arm_irq()
62 	spin_unlock(&b->irq_lock);  in intel_breadcrumbs_arm_irq()
65 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_disarm_irq()  argument
67 	intel_wakeref_t wakeref = b->irq_armed;  in __intel_breadcrumbs_disarm_irq()
69 	GEM_BUG_ON(!b->irq_enabled);  in __intel_breadcrumbs_disarm_irq()
70 	if (!--b->irq_enabled)  in __intel_breadcrumbs_disarm_irq()
71 		b->irq_disable(b);  in __intel_breadcrumbs_disarm_irq()
73 	WRITE_ONCE(b->irq_armed, 0);  in __intel_breadcrumbs_disarm_irq()
74 	intel_gt_pm_put_async(b->irq_engine->gt, wakeref);  in __intel_breadcrumbs_disarm_irq()
77 static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)  in intel_breadcrumbs_disarm_irq()  argument
79 	spin_lock(&b->irq_lock);  in intel_breadcrumbs_disarm_irq()
80 	if (b->irq_armed)  in intel_breadcrumbs_disarm_irq()
81 		__intel_breadcrumbs_disarm_irq(b);  in intel_breadcrumbs_disarm_irq()
82 	spin_unlock(&b->irq_lock);  in intel_breadcrumbs_disarm_irq()
85 static void add_signaling_context(struct intel_breadcrumbs *b,  in add_signaling_context()  argument
90 	spin_lock(&b->signalers_lock);  in add_signaling_context()
91 	list_add_rcu(&ce->signal_link, &b->signalers);  in add_signaling_context()
92 	spin_unlock(&b->signalers_lock);  in add_signaling_context()
95 static bool remove_signaling_context(struct intel_breadcrumbs *b,  in remove_signaling_context()  argument
103 	spin_lock(&b->signalers_lock);  in remove_signaling_context()
105 	spin_unlock(&b->signalers_lock);  in remove_signaling_context()
157 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)  in add_retire()  argument
159 	if (b->irq_engine)  in add_retire()
160 		intel_engine_add_retire(b->irq_engine, tl);  in add_retire()
172 	struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);  in signal_irq_work()  local
178 	if (unlikely(!llist_empty(&b->signaled_requests)))  in signal_irq_work()
179 		signal = llist_del_all(&b->signaled_requests);  in signal_irq_work()
206 	if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers))  in signal_irq_work()
207 		intel_breadcrumbs_disarm_irq(b);  in signal_irq_work()
210 	atomic_inc(&b->signaler_active);  in signal_irq_work()
211 	list_for_each_entry_rcu(ce, &b->signalers, signal_link) {  in signal_irq_work()
231 			release = remove_signaling_context(b, ce);  in signal_irq_work()
235 					add_retire(b, ce->timeline);  in signal_irq_work()
246 	atomic_dec(&b->signaler_active);  in signal_irq_work()
267 	if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers))  in signal_irq_work()
268 		intel_breadcrumbs_arm_irq(b);  in signal_irq_work()
271 	if (READ_ONCE(b->irq_armed) && !atomic_read(&b->active))  in signal_irq_work()
272 		intel_breadcrumbs_disarm_irq(b);  in signal_irq_work()
278 	struct intel_breadcrumbs *b;  in intel_breadcrumbs_create()  local
280 	b = kzalloc(sizeof(*b), GFP_KERNEL);  in intel_breadcrumbs_create()
281 	if (!b)  in intel_breadcrumbs_create()
284 	kref_init(&b->ref);  in intel_breadcrumbs_create()
286 	spin_lock_init(&b->signalers_lock);  in intel_breadcrumbs_create()
287 	INIT_LIST_HEAD(&b->signalers);  in intel_breadcrumbs_create()
288 	init_llist_head(&b->signaled_requests);  in intel_breadcrumbs_create()
290 	spin_lock_init(&b->irq_lock);  in intel_breadcrumbs_create()
291 	init_irq_work(&b->irq_work, signal_irq_work);  in intel_breadcrumbs_create()
293 	b->irq_engine = irq_engine;  in intel_breadcrumbs_create()
294 	b->irq_enable = irq_enable;  in intel_breadcrumbs_create()
295 	b->irq_disable = irq_disable;  in intel_breadcrumbs_create()
297 	return b;  in intel_breadcrumbs_create()
300 void intel_breadcrumbs_reset(struct intel_breadcrumbs *b)  in intel_breadcrumbs_reset()  argument
304 	if (!b->irq_engine)  in intel_breadcrumbs_reset()
307 	spin_lock_irqsave(&b->irq_lock, flags);  in intel_breadcrumbs_reset()
309 	if (b->irq_enabled)  in intel_breadcrumbs_reset()
310 		b->irq_enable(b);  in intel_breadcrumbs_reset()
312 		b->irq_disable(b);  in intel_breadcrumbs_reset()
314 	spin_unlock_irqrestore(&b->irq_lock, flags);  in intel_breadcrumbs_reset()
317 void __intel_breadcrumbs_park(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_park()  argument
319 	if (!READ_ONCE(b->irq_armed))  in __intel_breadcrumbs_park()
323 	irq_work_queue(&b->irq_work);  in __intel_breadcrumbs_park()
328 	struct intel_breadcrumbs *b = container_of(kref, typeof(*b), ref);  in intel_breadcrumbs_free()  local
330 	irq_work_sync(&b->irq_work);  in intel_breadcrumbs_free()
331 	GEM_BUG_ON(!list_empty(&b->signalers));  in intel_breadcrumbs_free()
332 	GEM_BUG_ON(b->irq_armed);  in intel_breadcrumbs_free()
334 	kfree(b);  in intel_breadcrumbs_free()
338 			       struct intel_breadcrumbs *b)  in irq_signal_request()  argument
344 	if (llist_add(&rq->signal_node, &b->signaled_requests))  in irq_signal_request()
345 		irq_work_queue(&b->irq_work);  in irq_signal_request()
350 	struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;  in insert_breadcrumb()  local
363 		irq_signal_request(rq, b);  in insert_breadcrumb()
369 		add_signaling_context(b, ce);  in insert_breadcrumb()
406 	if (!READ_ONCE(b->irq_armed) || __i915_request_is_complete(rq))  in insert_breadcrumb()
407 		irq_work_queue(&b->irq_work);  in insert_breadcrumb()
437 	struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;  in i915_request_cancel_breadcrumb()  local
448 	release = remove_signaling_context(b, ce);  in i915_request_cancel_breadcrumb()
454 		irq_signal_request(rq, b);  in i915_request_cancel_breadcrumb()
460 				      struct intel_breadcrumbs *b)  in intel_context_remove_breadcrumbs()  argument
478 		irq_signal_request(rq, b);  in intel_context_remove_breadcrumbs()
481 	release = remove_signaling_context(b, ce);  in intel_context_remove_breadcrumbs()
488 	while (atomic_read(&b->signaler_active))  in intel_context_remove_breadcrumbs()
492 static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p)  in print_signals()  argument
500 	list_for_each_entry_rcu(ce, &b->signalers, signal_link) {  in print_signals()
515 	struct intel_breadcrumbs *b;  in intel_engine_print_breadcrumbs()  local
517 	b = engine->breadcrumbs;  in intel_engine_print_breadcrumbs()
518 	if (!b)  in intel_engine_print_breadcrumbs()
521 	drm_printf(p, "IRQ: %s\n", str_enabled_disabled(b->irq_armed));  in intel_engine_print_breadcrumbs()
522 	if (!list_empty(&b->signalers))  in intel_engine_print_breadcrumbs()
523 		print_signals(b, p);  in intel_engine_print_breadcrumbs()