27 static struct page *hwsp_page(struct intel_timeline *tl)  in hwsp_page()  argument
29 	struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj;  in hwsp_page()
35 static unsigned long hwsp_cacheline(struct intel_timeline *tl)  in hwsp_cacheline()  argument
37 	unsigned long address = (unsigned long)page_address(hwsp_page(tl));  in hwsp_cacheline()
39 	return (address + offset_in_page(tl->hwsp_offset)) / TIMELINE_SEQNO_BYTES;  in hwsp_cacheline()
42 static int selftest_tl_pin(struct intel_timeline *tl)  in selftest_tl_pin()  argument
49 	err = i915_gem_object_lock(tl->hwsp_ggtt->obj, &ww);  in selftest_tl_pin()
51 		err = intel_timeline_pin(tl, &ww);  in selftest_tl_pin()
79 			       struct intel_timeline *tl)  in __mock_hwsp_record()  argument
81 	tl = xchg(&state->history[idx], tl);  in __mock_hwsp_record()
82 	if (tl) {  in __mock_hwsp_record()
83 		radix_tree_delete(&state->cachelines, hwsp_cacheline(tl));  in __mock_hwsp_record()
84 		intel_timeline_unpin(tl);  in __mock_hwsp_record()
85 		intel_timeline_put(tl);  in __mock_hwsp_record()
93 	struct intel_timeline *tl;  in __mock_hwsp_timeline()  local
100 		tl = intel_timeline_create(state->gt);  in __mock_hwsp_timeline()
101 		if (IS_ERR(tl))  in __mock_hwsp_timeline()
102 			return PTR_ERR(tl);  in __mock_hwsp_timeline()
104 		err = selftest_tl_pin(tl);  in __mock_hwsp_timeline()
106 			intel_timeline_put(tl);  in __mock_hwsp_timeline()
110 		cacheline = hwsp_cacheline(tl);  in __mock_hwsp_timeline()
111 		err = radix_tree_insert(&state->cachelines, cacheline, tl);  in __mock_hwsp_timeline()
117 			intel_timeline_unpin(tl);  in __mock_hwsp_timeline()
118 			intel_timeline_put(tl);  in __mock_hwsp_timeline()
123 		__mock_hwsp_record(state, idx, tl);  in __mock_hwsp_timeline()
204 static int __igt_sync(struct intel_timeline *tl,  in __igt_sync()  argument
211 	if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) {  in __igt_sync()
218 		ret = __intel_timeline_sync_set(tl, ctx, p->seqno);  in __igt_sync()
246 	struct intel_timeline tl;  in igt_sync()  local
250 	mock_timeline_init(&tl, 0);  in igt_sync()
256 				ret = __igt_sync(&tl, ctx, p, "1");  in igt_sync()
262 	mock_timeline_fini(&tl);  in igt_sync()
264 	mock_timeline_init(&tl, 0);  in igt_sync()
270 				ret = __igt_sync(&tl, ctx, p, "2");  in igt_sync()
278 	mock_timeline_fini(&tl);  in igt_sync()
290 	struct intel_timeline tl;  in bench_sync()  local
296 	mock_timeline_init(&tl, 0);  in bench_sync()
328 		__intel_timeline_sync_set(&tl, id, 0);  in bench_sync()
343 		if (!__intel_timeline_sync_is_later(&tl, id, 0)) {  in bench_sync()
344 			mock_timeline_fini(&tl);  in bench_sync()
354 	mock_timeline_fini(&tl);  in bench_sync()
357 	mock_timeline_init(&tl, 0);  in bench_sync()
364 		__intel_timeline_sync_set(&tl, count++, 0);  in bench_sync()
374 		if (!__intel_timeline_sync_is_later(&tl, end_time, 0)) {  in bench_sync()
376 			mock_timeline_fini(&tl);  in bench_sync()
384 	mock_timeline_fini(&tl);  in bench_sync()
387 	mock_timeline_init(&tl, 0);  in bench_sync()
398 		if (!__intel_timeline_sync_is_later(&tl, id, seqno))  in bench_sync()
399 			__intel_timeline_sync_set(&tl, id, seqno);  in bench_sync()
407 	mock_timeline_fini(&tl);  in bench_sync()
415 		mock_timeline_init(&tl, 0);  in bench_sync()
427 			__intel_timeline_sync_is_later(&tl, id, 0);  in bench_sync()
428 			__intel_timeline_sync_set(&tl, id, 0);  in bench_sync()
436 		mock_timeline_fini(&tl);  in bench_sync()
485 checked_tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value)  in checked_tl_write()  argument
490 	err = selftest_tl_pin(tl);  in checked_tl_write()
496 	if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) {  in checked_tl_write()
498 		       *tl->hwsp_seqno, tl->seqno);  in checked_tl_write()
499 		intel_timeline_unpin(tl);  in checked_tl_write()
509 	err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value);  in checked_tl_write()
517 	intel_timeline_unpin(tl);  in checked_tl_write()
553 			struct intel_timeline *tl;  in live_hwsp_engine()  local
556 			tl = intel_timeline_create(gt);  in live_hwsp_engine()
557 			if (IS_ERR(tl)) {  in live_hwsp_engine()
558 				err = PTR_ERR(tl);  in live_hwsp_engine()
562 			rq = checked_tl_write(tl, engine, count);  in live_hwsp_engine()
564 				intel_timeline_put(tl);  in live_hwsp_engine()
569 			timelines[count++] = tl;  in live_hwsp_engine()
582 		struct intel_timeline *tl = timelines[n];  in live_hwsp_engine()  local
584 		if (!err && READ_ONCE(*tl->hwsp_seqno) != n) {  in live_hwsp_engine()
586 				      n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno);  in live_hwsp_engine()
590 		intel_timeline_put(tl);  in live_hwsp_engine()
623 			struct intel_timeline *tl;  in live_hwsp_alternate()  local
629 			tl = intel_timeline_create(gt);  in live_hwsp_alternate()
630 			if (IS_ERR(tl)) {  in live_hwsp_alternate()
631 				err = PTR_ERR(tl);  in live_hwsp_alternate()
636 			rq = checked_tl_write(tl, engine, count);  in live_hwsp_alternate()
639 				intel_timeline_put(tl);  in live_hwsp_alternate()
644 			timelines[count++] = tl;  in live_hwsp_alternate()
654 		struct intel_timeline *tl = timelines[n];  in live_hwsp_alternate()  local
656 		if (!err && READ_ONCE(*tl->hwsp_seqno) != n) {  in live_hwsp_alternate()
658 				      n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno);  in live_hwsp_alternate()
662 		intel_timeline_put(tl);  in live_hwsp_alternate()
674 	struct intel_timeline *tl;  in live_hwsp_wrap()  local
683 	tl = intel_timeline_create(gt);  in live_hwsp_wrap()
684 	if (IS_ERR(tl))  in live_hwsp_wrap()
685 		return PTR_ERR(tl);  in live_hwsp_wrap()
687 	if (!tl->has_initial_breadcrumb)  in live_hwsp_wrap()
690 	err = selftest_tl_pin(tl);  in live_hwsp_wrap()
708 		tl->seqno = -4u;  in live_hwsp_wrap()
710 		mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);  in live_hwsp_wrap()
711 		err = intel_timeline_get_seqno(tl, rq, &seqno[0]);  in live_hwsp_wrap()
712 		mutex_unlock(&tl->mutex);  in live_hwsp_wrap()
718 			 seqno[0], tl->hwsp_offset);  in live_hwsp_wrap()
720 		err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]);  in live_hwsp_wrap()
725 		hwsp_seqno[0] = tl->hwsp_seqno;  in live_hwsp_wrap()
727 		mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);  in live_hwsp_wrap()
728 		err = intel_timeline_get_seqno(tl, rq, &seqno[1]);  in live_hwsp_wrap()
729 		mutex_unlock(&tl->mutex);  in live_hwsp_wrap()
735 			 seqno[1], tl->hwsp_offset);  in live_hwsp_wrap()
737 		err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]);  in live_hwsp_wrap()
742 		hwsp_seqno[1] = tl->hwsp_seqno;  in live_hwsp_wrap()
772 	intel_timeline_unpin(tl);  in live_hwsp_wrap()
774 	intel_timeline_put(tl);  in live_hwsp_wrap()
829 			 struct intel_timeline *tl)  in setup_watcher()  argument
839 	i915_gem_object_set_pat_index(obj, tl->hwsp_ggtt->obj->pat_index);  in setup_watcher()
841 						  page_unmask_bits(tl->hwsp_ggtt->obj->mm.mapping));  in setup_watcher()
943 static bool retire_requests(struct intel_timeline *tl)  in retire_requests()  argument
947 	mutex_lock(&tl->mutex);  in retire_requests()
948 	list_for_each_entry_safe(rq, rn, &tl->requests, link)  in retire_requests()
951 	mutex_unlock(&tl->mutex);  in retire_requests()
953 	return !i915_active_fence_isset(&tl->last_request);  in retire_requests()
959 	struct intel_timeline *tl = ce->timeline;  in wrap_timeline()  local
962 	while (tl->seqno >= seqno) { /* Cause a wrap */  in wrap_timeline()
988 	struct intel_timeline *tl;  in live_hwsp_read()  local
1004 	tl = intel_timeline_create(gt);  in live_hwsp_read()
1005 	if (IS_ERR(tl))  in live_hwsp_read()
1006 		return PTR_ERR(tl);  in live_hwsp_read()
1008 	if (!tl->has_initial_breadcrumb)  in live_hwsp_read()
1011 	selftest_tl_pin(tl);  in live_hwsp_read()
1014 		err = setup_watcher(&watcher[i], gt, tl);  in live_hwsp_read()
1050 			ce->timeline = intel_timeline_get(tl);  in live_hwsp_read()
1063 			tl->seqno = -12u + 2 * (count & 3);  in live_hwsp_read()
1064 			__intel_timeline_get_seqno(tl, &dummy);  in live_hwsp_read()
1148 			retire_requests(tl);  in live_hwsp_read()
1169 	intel_timeline_unpin(tl);  in live_hwsp_read()
1175 	intel_timeline_put(tl);  in live_hwsp_read()
1193 		struct intel_timeline *tl = ce->timeline;  in live_hwsp_rollover_kernel()  local
1203 		GEM_BUG_ON(i915_active_fence_isset(&tl->last_request));  in live_hwsp_rollover_kernel()
1204 		tl->seqno = -2u;  in live_hwsp_rollover_kernel()
1205 		WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);  in live_hwsp_rollover_kernel()
1220 			GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);  in live_hwsp_rollover_kernel()
1271 		struct intel_timeline *tl;  in live_hwsp_rollover_user()  local
1283 		tl = ce->timeline;  in live_hwsp_rollover_user()
1284 		if (!tl->has_initial_breadcrumb)  in live_hwsp_rollover_user()
1291 		tl->seqno = -4u;  in live_hwsp_rollover_user()
1292 		WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);  in live_hwsp_rollover_user()
1307 			GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);  in live_hwsp_rollover_user()
1369 			struct intel_timeline *tl;  in live_hwsp_recycle()  local
1372 			tl = intel_timeline_create(gt);  in live_hwsp_recycle()
1373 			if (IS_ERR(tl)) {  in live_hwsp_recycle()
1374 				err = PTR_ERR(tl);  in live_hwsp_recycle()
1378 			rq = checked_tl_write(tl, engine, count);  in live_hwsp_recycle()
1380 				intel_timeline_put(tl);  in live_hwsp_recycle()
1388 				intel_timeline_put(tl);  in live_hwsp_recycle()
1393 			if (READ_ONCE(*tl->hwsp_seqno) != count) {  in live_hwsp_recycle()
1395 					      count, tl->fence_context,  in live_hwsp_recycle()
1396 					      tl->hwsp_offset, *tl->hwsp_seqno);  in live_hwsp_recycle()
1402 			intel_timeline_put(tl);  in live_hwsp_recycle()