Lines Matching +full:cs +full:- +full:x
1 // SPDX-License-Identifier: MIT
3 * Copyright © 2017-2018 Intel Corporation
29 struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj; in hwsp_page()
32 return sg_page(obj->mm.pages->sgl); in hwsp_page()
39 return (address + offset_in_page(tl->hwsp_offset)) / TIMELINE_SEQNO_BYTES; in hwsp_cacheline()
49 err = i915_gem_object_lock(tl->hwsp_ggtt->obj, &ww); in selftest_tl_pin()
53 if (err == -EDEADLK) { in selftest_tl_pin()
81 tl = xchg(&state->history[idx], tl); in __mock_hwsp_record()
83 radix_tree_delete(&state->cachelines, hwsp_cacheline(tl)); in __mock_hwsp_record()
96 while (count--) { in __mock_hwsp_timeline()
100 tl = intel_timeline_create(state->gt); in __mock_hwsp_timeline()
111 err = radix_tree_insert(&state->cachelines, cacheline, tl); in __mock_hwsp_timeline()
113 if (err == -EEXIST) { in __mock_hwsp_timeline()
122 idx = state->count++ % state->max; in __mock_hwsp_timeline()
127 i915_prandom_shuffle(state->history, in __mock_hwsp_timeline()
128 sizeof(*state->history), in __mock_hwsp_timeline()
129 min(state->count, state->max), in __mock_hwsp_timeline()
130 &state->prng); in __mock_hwsp_timeline()
132 count = i915_prandom_u32_max_state(min(state->count, state->max), in __mock_hwsp_timeline()
133 &state->prng); in __mock_hwsp_timeline()
134 while (count--) { in __mock_hwsp_timeline()
135 idx = --state->count % state->max; in __mock_hwsp_timeline()
159 return -ENOMEM; in mock_hwsp_freelist()
175 err = -ENOMEM; in mock_hwsp_freelist()
179 for (p = phases; p->name; p++) { in mock_hwsp_freelist()
180 pr_debug("%s(%s)\n", __func__, p->name); in mock_hwsp_freelist()
182 err = __mock_hwsp_timeline(&state, na, p->flags); in mock_hwsp_freelist()
211 if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) { in __igt_sync()
213 name, p->name, ctx, p->seqno, str_yes_no(p->expected)); in __igt_sync()
214 return -EINVAL; in __igt_sync()
217 if (p->set) { in __igt_sync()
218 ret = __intel_timeline_sync_set(tl, ctx, p->seqno); in __igt_sync()
238 { "INT_MAX-1", INT_MAX-1, true, false }, in igt_sync()
248 int ret = -ENODEV; in igt_sync()
251 for (p = pass; p->name; p++) { in igt_sync()
253 for (offset = -1; offset <= (order > 1); offset++) { in igt_sync()
266 for (offset = -1; offset <= (order > 1); offset++) { in igt_sync()
269 for (p = pass; p->name; p++) { in igt_sync()
299 * and the loop itself becomes a significant factor in the per-iteration in bench_sync()
308 u32 x; in bench_sync() local
311 WRITE_ONCE(x, prandom_u32_state(&prng)); in bench_sync()
340 while (end_time--) { in bench_sync()
346 return -EINVAL; in bench_sync()
367 pr_info("%s: %lu in-order insertions, %lluns/insert\n", in bench_sync()
373 while (end_time--) { in bench_sync()
377 return -EINVAL; in bench_sync()
381 pr_info("%s: %lu in-order lookups, %lluns/lookup\n", in bench_sync()
413 unsigned int mask = BIT(order) - 1; in bench_sync()
422 * implementation, try to identify its phase-changes in bench_sync()
456 u32 *cs; in emit_ggtt_store_dw() local
458 cs = intel_ring_begin(rq, 4); in emit_ggtt_store_dw()
459 if (IS_ERR(cs)) in emit_ggtt_store_dw()
460 return PTR_ERR(cs); in emit_ggtt_store_dw()
462 if (GRAPHICS_VER(rq->i915) >= 8) { in emit_ggtt_store_dw()
463 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in emit_ggtt_store_dw()
464 *cs++ = addr; in emit_ggtt_store_dw()
465 *cs++ = 0; in emit_ggtt_store_dw()
466 *cs++ = value; in emit_ggtt_store_dw()
467 } else if (GRAPHICS_VER(rq->i915) >= 4) { in emit_ggtt_store_dw()
468 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in emit_ggtt_store_dw()
469 *cs++ = 0; in emit_ggtt_store_dw()
470 *cs++ = addr; in emit_ggtt_store_dw()
471 *cs++ = value; in emit_ggtt_store_dw()
473 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; in emit_ggtt_store_dw()
474 *cs++ = addr; in emit_ggtt_store_dw()
475 *cs++ = value; in emit_ggtt_store_dw()
476 *cs++ = MI_NOOP; in emit_ggtt_store_dw()
479 intel_ring_advance(rq, cs); in emit_ggtt_store_dw()
496 if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) { in checked_tl_write()
497 pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n", in checked_tl_write()
498 *tl->hwsp_seqno, tl->seqno); in checked_tl_write()
500 return ERR_PTR(-EINVAL); in checked_tl_write()
509 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value); in checked_tl_write()
543 return -ENOMEM; in live_hwsp_engine()
578 if (igt_flush_test(gt->i915)) in live_hwsp_engine()
579 err = -EIO; in live_hwsp_engine()
584 if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { in live_hwsp_engine()
585 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n", in live_hwsp_engine()
586 n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); in live_hwsp_engine()
588 err = -EINVAL; in live_hwsp_engine()
618 return -ENOMEM; in live_hwsp_alternate()
650 if (igt_flush_test(gt->i915)) in live_hwsp_alternate()
651 err = -EIO; in live_hwsp_alternate()
656 if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { in live_hwsp_alternate()
657 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n", in live_hwsp_alternate()
658 n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); in live_hwsp_alternate()
660 err = -EINVAL; in live_hwsp_alternate()
687 if (!tl->has_initial_breadcrumb) in live_hwsp_wrap()
708 tl->seqno = -4u; in live_hwsp_wrap()
710 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); in live_hwsp_wrap()
712 mutex_unlock(&tl->mutex); in live_hwsp_wrap()
717 pr_debug("seqno[0]:%08x, hwsp_offset:%08x\n", in live_hwsp_wrap()
718 seqno[0], tl->hwsp_offset); in live_hwsp_wrap()
720 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]); in live_hwsp_wrap()
725 hwsp_seqno[0] = tl->hwsp_seqno; in live_hwsp_wrap()
727 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); in live_hwsp_wrap()
729 mutex_unlock(&tl->mutex); in live_hwsp_wrap()
734 pr_debug("seqno[1]:%08x, hwsp_offset:%08x\n", in live_hwsp_wrap()
735 seqno[1], tl->hwsp_offset); in live_hwsp_wrap()
737 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]); in live_hwsp_wrap()
742 hwsp_seqno[1] = tl->hwsp_seqno; in live_hwsp_wrap()
752 err = -EIO; in live_hwsp_wrap()
758 pr_err("Bad timeline values: found (%x, %x), expected (%x, %x)\n", in live_hwsp_wrap()
761 err = -EINVAL; in live_hwsp_wrap()
769 if (igt_flush_test(gt->i915)) in live_hwsp_wrap()
770 err = -EIO; in live_hwsp_wrap()
782 const u32 gpr = i915_mmio_reg_offset(GEN8_RING_CS_GPR(rq->engine->mmio_base, 0)); in emit_read_hwsp()
783 u32 *cs; in emit_read_hwsp() local
785 cs = intel_ring_begin(rq, 12); in emit_read_hwsp()
786 if (IS_ERR(cs)) in emit_read_hwsp()
787 return PTR_ERR(cs); in emit_read_hwsp()
789 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in emit_read_hwsp()
790 *cs++ = *addr; in emit_read_hwsp()
791 *cs++ = 0; in emit_read_hwsp()
792 *cs++ = seqno; in emit_read_hwsp()
795 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_USE_GGTT; in emit_read_hwsp()
796 *cs++ = gpr; in emit_read_hwsp()
797 *cs++ = hwsp; in emit_read_hwsp()
798 *cs++ = 0; in emit_read_hwsp()
800 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; in emit_read_hwsp()
801 *cs++ = gpr; in emit_read_hwsp()
802 *cs++ = *addr; in emit_read_hwsp()
803 *cs++ = 0; in emit_read_hwsp()
806 intel_ring_advance(rq, cs); in emit_read_hwsp()
834 obj = i915_gem_object_create_internal(gt->i915, SZ_2M); in setup_watcher()
839 i915_gem_object_set_pat_index(obj, tl->hwsp_ggtt->obj->pat_index); in setup_watcher()
840 w->map = i915_gem_object_pin_map_unlocked(obj, in setup_watcher()
841 page_unmask_bits(tl->hwsp_ggtt->obj->mm.mapping)); in setup_watcher()
842 if (IS_ERR(w->map)) { in setup_watcher()
844 return PTR_ERR(w->map); in setup_watcher()
853 w->vma = vma; in setup_watcher()
854 w->addr = i915_ggtt_offset(vma); in setup_watcher()
860 /* some light mutex juggling required; think co-routines */ in switch_tl_lock()
863 lockdep_unpin_lock(&from->context->timeline->mutex, from->cookie); in switch_tl_lock()
864 mutex_unlock(&from->context->timeline->mutex); in switch_tl_lock()
868 mutex_lock(&to->context->timeline->mutex); in switch_tl_lock()
869 to->cookie = lockdep_pin_lock(&to->context->timeline->mutex); in switch_tl_lock()
883 ce->ring_size = ringsz; in create_watcher()
884 w->rq = intel_context_create_request(ce); in create_watcher()
886 if (IS_ERR(w->rq)) in create_watcher()
887 return PTR_ERR(w->rq); in create_watcher()
889 w->addr = i915_ggtt_offset(w->vma); in create_watcher()
891 switch_tl_lock(w->rq, NULL); in create_watcher()
899 struct i915_request *rq = fetch_and_zero(&w->rq); in check_watcher()
903 GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size); in check_watcher()
910 err = -ETIME; in check_watcher()
916 end = (w->addr - i915_ggtt_offset(w->vma)) / sizeof(*w->map); in check_watcher()
918 if (!op(w->map[offset + 1], w->map[offset])) { in check_watcher()
919 pr_err("Watcher '%s' found HWSP value %x for seqno %x\n", in check_watcher()
920 name, w->map[offset + 1], w->map[offset]); in check_watcher()
921 err = -EINVAL; in check_watcher()
934 if (w->rq) { in cleanup_watcher()
935 switch_tl_lock(NULL, w->rq); in cleanup_watcher()
937 i915_request_add(w->rq); in cleanup_watcher()
940 i915_vma_unpin_and_release(&w->vma, I915_VMA_RELEASE_MAP); in cleanup_watcher()
947 mutex_lock(&tl->mutex); in retire_requests()
948 list_for_each_entry_safe(rq, rn, &tl->requests, link) in retire_requests()
951 mutex_unlock(&tl->mutex); in retire_requests()
953 return !i915_active_fence_isset(&tl->last_request); in retire_requests()
958 struct intel_context *ce = rq->context; in wrap_timeline()
959 struct intel_timeline *tl = ce->timeline; in wrap_timeline()
960 u32 seqno = rq->fence.seqno; in wrap_timeline()
962 while (tl->seqno >= seqno) { /* Cause a wrap */ in wrap_timeline()
1001 if (GRAPHICS_VER(gt->i915) < 8) /* CS convenience [SRM/LRM] */ in live_hwsp_read()
1008 if (!tl->has_initial_breadcrumb) in live_hwsp_read()
1036 err = -ENOMEM; in live_hwsp_read()
1050 ce->timeline = intel_timeline_get(tl); in live_hwsp_read()
1063 tl->seqno = -12u + 2 * (count & 3); in live_hwsp_read()
1074 err = i915_sw_fence_await_dma_fence(&rq->submit, in live_hwsp_read()
1075 &watcher[0].rq->fence, 0, in live_hwsp_read()
1088 rq->fence.seqno, hwsp, in live_hwsp_read()
1102 rq->fence.seqno, hwsp, in live_hwsp_read()
1123 err = i915_sw_fence_await_dma_fence(&watcher[1].rq->submit, in live_hwsp_read()
1124 &rq->fence, 0, in live_hwsp_read()
1144 err = -ETIME; in live_hwsp_read()
1152 if (8 * watcher[1].rq->ring->emit > in live_hwsp_read()
1153 3 * watcher[1].rq->ring->size) in live_hwsp_read()
1157 count < (PAGE_SIZE / TIMELINE_SEQNO_BYTES - 1) / 2); in live_hwsp_read()
1159 pr_info("%s: simulated %lu wraps\n", engine->name, count); in live_hwsp_read()
1171 if (igt_flush_test(gt->i915)) in live_hwsp_read()
1172 err = -EIO; in live_hwsp_read()
1192 struct intel_context *ce = engine->kernel_context; in live_hwsp_rollover_kernel()
1193 struct intel_timeline *tl = ce->timeline; in live_hwsp_rollover_kernel()
1199 err = -EIO; in live_hwsp_rollover_kernel()
1203 GEM_BUG_ON(i915_active_fence_isset(&tl->last_request)); in live_hwsp_rollover_kernel()
1204 tl->seqno = -2u; in live_hwsp_rollover_kernel()
1205 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); in live_hwsp_rollover_kernel()
1217 engine->name, in live_hwsp_rollover_kernel()
1218 lower_32_bits(this->fence.seqno)); in live_hwsp_rollover_kernel()
1220 GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl); in live_hwsp_rollover_kernel()
1227 GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno); in live_hwsp_rollover_kernel()
1231 err = -EIO; in live_hwsp_rollover_kernel()
1237 pr_err("Pre-wrap request not completed!\n"); in live_hwsp_rollover_kernel()
1238 err = -EINVAL; in live_hwsp_rollover_kernel()
1251 if (igt_flush_test(gt->i915)) in live_hwsp_rollover_kernel()
1252 err = -EIO; in live_hwsp_rollover_kernel()
1283 tl = ce->timeline; in live_hwsp_rollover_user()
1284 if (!tl->has_initial_breadcrumb) in live_hwsp_rollover_user()
1291 tl->seqno = -4u; in live_hwsp_rollover_user()
1292 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); in live_hwsp_rollover_user()
1304 engine->name, in live_hwsp_rollover_user()
1305 lower_32_bits(this->fence.seqno)); in live_hwsp_rollover_user()
1307 GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl); in live_hwsp_rollover_user()
1314 GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno); in live_hwsp_rollover_user()
1318 err = -EIO; in live_hwsp_rollover_user()
1324 pr_err("Pre-wrap request not completed!\n"); in live_hwsp_rollover_user()
1325 err = -EINVAL; in live_hwsp_rollover_user()
1339 if (igt_flush_test(gt->i915)) in live_hwsp_rollover_user()
1340 err = -EIO; in live_hwsp_rollover_user()
1389 err = -EIO; in live_hwsp_recycle()
1393 if (READ_ONCE(*tl->hwsp_seqno) != count) { in live_hwsp_recycle()
1394 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x found 0x%x\n", in live_hwsp_recycle()
1395 count, tl->fence_context, in live_hwsp_recycle()
1396 tl->hwsp_offset, *tl->hwsp_seqno); in live_hwsp_recycle()
1398 err = -EINVAL; in live_hwsp_recycle()