56 static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,  in __xe_pt_empty_pte()  argument
59 	struct xe_device *xe = tile_to_xe(tile);  in __xe_pt_empty_pte()
61 	u8 id = tile->id;  in __xe_pt_empty_pte()
85  * @tile: The tile to create for.
97 struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,  in xe_pt_create()  argument
115 	bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,  in xe_pt_create()
117 				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |  in xe_pt_create()
131 	xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);  in xe_pt_create()
143  * @tile: The tile the scratch pagetable of which to use.
147  * Populate the page-table bo of @pt with entries pointing into the tile's
150 void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,  in xe_pt_populate_empty()  argument
164 		empty = __xe_pt_empty_pte(tile, vm, pt->level);  in xe_pt_populate_empty()
259 	/** @tile: The tile we're building for. */
260 	struct xe_tile *tile;  member
544 		xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1);  in xe_pt_stage_bind_entry()
552 			xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child);  in xe_pt_stage_bind_entry()
563 		if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&  in xe_pt_stage_bind_entry()
587  * @tile: The tile we're building for.
603 xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,  in xe_pt_stage_bind()  argument
606 	struct xe_device *xe = tile_to_xe(tile);  in xe_pt_stage_bind()
618 		.tile = tile,  in xe_pt_stage_bind()
625 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];  in xe_pt_stage_bind()
752 	/** @tile: The tile we're building for */
753 	struct xe_tile *tile;  member
781 		xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap,  in xe_pt_zap_ptes_entry()
796  * @tile: The tile we're zapping for.
810 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)  in xe_pt_zap_ptes()  argument
818 		.tile = tile,  in xe_pt_zap_ptes()
820 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];  in xe_pt_zap_ptes()
823 	if (!(pt_mask & BIT(tile->id)))  in xe_pt_zap_ptes()
833 xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,  in xe_vm_populate_pgtable()  argument
844 			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *  in xe_vm_populate_pgtable()
982 xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,  in xe_pt_prepare_bind()  argument
988 	err = xe_pt_stage_bind(tile, vma, entries, num_entries);  in xe_pt_prepare_bind()
990 		xe_tile_assert(tile, *num_entries);  in xe_pt_prepare_bind()
1388 	/** @tile: The tile we're unbinding from. */
1389 	struct xe_tile *tile;  member
1498  * @tile: The tile we're unbinding for.
1509 static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma,  in xe_pt_stage_unbind()  argument
1518 		.tile = tile,  in xe_pt_stage_unbind()
1523 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];  in xe_pt_stage_unbind()
1533 				  struct xe_tile *tile, struct iosys_map *map,  in xe_migrate_clear_pgtable_callback()  argument
1538 	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);  in xe_migrate_clear_pgtable_callback()
1543 			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *  in xe_migrate_clear_pgtable_callback()
1641 static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,  in bind_op_prepare()  argument
1657 	pt_op->rebind = BIT(tile->id) & vma->tile_present;  in bind_op_prepare()
1659 	err = vma_reserve_fences(tile_to_xe(tile), vma);  in bind_op_prepare()
1663 	err = xe_pt_prepare_bind(tile, vma, pt_op->entries,  in bind_op_prepare()
1666 		xe_tile_assert(tile, pt_op->num_entries <=  in bind_op_prepare()
1668 		xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,  in bind_op_prepare()
1694 		vma->tile_staged |= BIT(tile->id);  in bind_op_prepare()
1705 static int unbind_op_prepare(struct xe_tile *tile,  in unbind_op_prepare()  argument
1713 	if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))  in unbind_op_prepare()
1733 	err = vma_reserve_fences(tile_to_xe(tile), vma);  in unbind_op_prepare()
1737 	pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries);  in unbind_op_prepare()
1739 	xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,  in unbind_op_prepare()
1752 		      struct xe_tile *tile,  in op_prepare()  argument
1765 		err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma);  in op_prepare()
1769 		err = unbind_op_prepare(tile, pt_update_ops,  in op_prepare()
1773 			err = bind_op_prepare(vm, tile, pt_update_ops,  in op_prepare()
1778 			err = bind_op_prepare(vm, tile, pt_update_ops,  in op_prepare()
1784 		err = unbind_op_prepare(tile, pt_update_ops,  in op_prepare()
1788 		err = bind_op_prepare(vm, tile, pt_update_ops,  in op_prepare()
1809  * @tile: Tile of PT update operations
1818 int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_prepare()  argument
1821 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_prepare()
1823 	int shift = tile->media_gt ? 1 : 0;  in xe_pt_update_ops_prepare()
1832 				      tile_to_xe(tile)->info.tile_count << shift);  in xe_pt_update_ops_prepare()
1837 		err = op_prepare(vops->vm, tile, pt_update_ops, op);  in xe_pt_update_ops_prepare()
1843 	xe_tile_assert(tile, pt_update_ops->current_op <=  in xe_pt_update_ops_prepare()
1855 static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,  in bind_op_commit()  argument
1871 	vma->tile_present |= BIT(tile->id);  in bind_op_commit()
1872 	vma->tile_staged &= ~BIT(tile->id);  in bind_op_commit()
1888 static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,  in unbind_op_commit()  argument
1904 	vma->tile_present &= ~BIT(tile->id);  in unbind_op_commit()
1918 		      struct xe_tile *tile,  in op_commit()  argument
1930 		bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence,  in op_commit()
1934 		unbind_op_commit(vm, tile, pt_update_ops,  in op_commit()
1939 			bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,  in op_commit()
1942 			bind_op_commit(vm, tile, pt_update_ops, op->remap.next,  in op_commit()
1946 		unbind_op_commit(vm, tile, pt_update_ops,  in op_commit()
1950 		bind_op_commit(vm, tile, pt_update_ops,  in op_commit()
1972  * @tile: Tile of PT update operations
1982 xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_run()  argument
1986 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_run()
1999 		.tile_id = tile->id,  in xe_pt_update_ops_run()
2006 		xe_tile_assert(tile, xe_vm_in_fault_mode(vm));  in xe_pt_update_ops_run()
2023 		if (tile->media_gt) {  in xe_pt_update_ops_run()
2048 	fence = xe_migrate_update_pgtables(tile->migrate, &update);  in xe_pt_update_ops_run()
2063 	if (xe_range_fence_insert(&vm->rftree[tile->id], rfence,  in xe_pt_update_ops_run()
2073 		invalidation_fence_init(tile->primary_gt, ifence, fence,  in xe_pt_update_ops_run()
2077 			invalidation_fence_init(tile->media_gt, mfence, fence,  in xe_pt_update_ops_run()
2099 			op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL);  in xe_pt_update_ops_run()
2112 			op_commit(vops->vm, tile, pt_update_ops, op,  in xe_pt_update_ops_run()
2129 	if (err != -EAGAIN && tile->id)  in xe_pt_update_ops_run()
2137  * @tile: Tile of PT update operations
2142 void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_fini()  argument
2145 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_fini()
2156 	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);  in xe_pt_update_ops_fini()
2161  * @tile: Tile of PT update operations
2166 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_abort()  argument
2169 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_abort()
2191 	xe_pt_update_ops_fini(tile, vops);  in xe_pt_update_ops_abort()