1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include "xe_migrate.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/sizes.h> 10 11 #include <drm/drm_managed.h> 12 #include <drm/ttm/ttm_tt.h> 13 #include <uapi/drm/xe_drm.h> 14 15 #include <generated/xe_wa_oob.h> 16 17 #include "instructions/xe_gpu_commands.h" 18 #include "instructions/xe_mi_commands.h" 19 #include "regs/xe_gtt_defs.h" 20 #include "tests/xe_test.h" 21 #include "xe_assert.h" 22 #include "xe_bb.h" 23 #include "xe_bo.h" 24 #include "xe_exec_queue.h" 25 #include "xe_ggtt.h" 26 #include "xe_gt.h" 27 #include "xe_hw_engine.h" 28 #include "xe_lrc.h" 29 #include "xe_map.h" 30 #include "xe_mocs.h" 31 #include "xe_pt.h" 32 #include "xe_res_cursor.h" 33 #include "xe_sched_job.h" 34 #include "xe_sync.h" 35 #include "xe_trace_bo.h" 36 #include "xe_vm.h" 37 38 /** 39 * struct xe_migrate - migrate context. 40 */ 41 struct xe_migrate { 42 /** @q: Default exec queue used for migration */ 43 struct xe_exec_queue *q; 44 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */ 45 struct xe_tile *tile; 46 /** @job_mutex: Timeline mutex for @eng. */ 47 struct mutex job_mutex; 48 /** @pt_bo: Page-table buffer object. */ 49 struct xe_bo *pt_bo; 50 /** @batch_base_ofs: VM offset of the migration batch buffer */ 51 u64 batch_base_ofs; 52 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */ 53 u64 usm_batch_base_ofs; 54 /** @cleared_mem_ofs: VM offset of @cleared_bo. */ 55 u64 cleared_mem_ofs; 56 /** 57 * @fence: dma-fence representing the last migration job batch. 58 * Protected by @job_mutex. 59 */ 60 struct dma_fence *fence; 61 /** 62 * @vm_update_sa: For integrated, used to suballocate page-tables 63 * out of the pt_bo. 64 */ 65 struct drm_suballoc_manager vm_update_sa; 66 /** @min_chunk_size: For dgfx, Minimum chunk size */ 67 u64 min_chunk_size; 68 }; 69 70 #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ 71 #define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ 72 #define NUM_KERNEL_PDE 15 73 #define NUM_PT_SLOTS 32 74 #define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M 75 #define MAX_NUM_PTE 512 76 #define IDENTITY_OFFSET 256ULL 77 78 /* 79 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest 80 * legal value accepted. Since that instruction field is always stored in 81 * (val-2) format, this translates to 0x400 dwords for the true maximum length 82 * of the instruction. Subtracting the instruction header (1 dword) and 83 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values. 84 */ 85 #define MAX_PTE_PER_SDI 0x1FE 86 87 /** 88 * xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue. 89 * @tile: The tile. 90 * 91 * Returns the default migrate exec queue of this tile. 92 * 93 * Return: The default migrate exec queue 94 */ xe_tile_migrate_exec_queue(struct xe_tile * tile)95 struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile) 96 { 97 return tile->migrate->q; 98 } 99 xe_migrate_fini(struct drm_device * dev,void * arg)100 static void xe_migrate_fini(struct drm_device *dev, void *arg) 101 { 102 struct xe_migrate *m = arg; 103 104 xe_vm_lock(m->q->vm, false); 105 xe_bo_unpin(m->pt_bo); 106 xe_vm_unlock(m->q->vm); 107 108 dma_fence_put(m->fence); 109 xe_bo_put(m->pt_bo); 110 drm_suballoc_manager_fini(&m->vm_update_sa); 111 mutex_destroy(&m->job_mutex); 112 xe_vm_close_and_put(m->q->vm); 113 xe_exec_queue_put(m->q); 114 } 115 xe_migrate_vm_addr(u64 slot,u32 level)116 static u64 xe_migrate_vm_addr(u64 slot, u32 level) 117 { 118 XE_WARN_ON(slot >= NUM_PT_SLOTS); 119 120 /* First slot is reserved for mapping of PT bo and bb, start from 1 */ 121 return (slot + 1ULL) << xe_pt_shift(level + 1); 122 } 123 xe_migrate_vram_ofs(struct xe_device * xe,u64 addr,bool is_comp_pte)124 static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte) 125 { 126 /* 127 * Remove the DPA to get a correct offset into identity table for the 128 * migrate offset 129 */ 130 u64 identity_offset = IDENTITY_OFFSET; 131 132 if (GRAPHICS_VER(xe) >= 20 && is_comp_pte) 133 identity_offset += DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G); 134 135 addr -= xe->mem.vram.dpa_base; 136 return addr + (identity_offset << xe_pt_shift(2)); 137 } 138 xe_migrate_program_identity(struct xe_device * xe,struct xe_vm * vm,struct xe_bo * bo,u64 map_ofs,u64 vram_offset,u16 pat_index,u64 pt_2m_ofs)139 static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo, 140 u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs) 141 { 142 u64 pos, ofs, flags; 143 u64 entry; 144 /* XXX: Unclear if this should be usable_size? */ 145 u64 vram_limit = xe->mem.vram.actual_physical_size + 146 xe->mem.vram.dpa_base; 147 u32 level = 2; 148 149 ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8; 150 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, 151 true, 0); 152 153 xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M)); 154 155 /* 156 * Use 1GB pages when possible, last chunk always use 2M 157 * pages as mixing reserved memory (stolen, WOCPM) with a single 158 * mapping is not allowed on certain platforms. 159 */ 160 for (pos = xe->mem.vram.dpa_base; pos < vram_limit; 161 pos += SZ_1G, ofs += 8) { 162 if (pos + SZ_1G >= vram_limit) { 163 entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs, 164 pat_index); 165 xe_map_wr(xe, &bo->vmap, ofs, u64, entry); 166 167 flags = vm->pt_ops->pte_encode_addr(xe, 0, 168 pat_index, 169 level - 1, 170 true, 0); 171 172 for (ofs = pt_2m_ofs; pos < vram_limit; 173 pos += SZ_2M, ofs += 8) 174 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 175 break; /* Ensure pos == vram_limit assert correct */ 176 } 177 178 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 179 } 180 181 xe_assert(xe, pos == vram_limit); 182 } 183 xe_migrate_prepare_vm(struct xe_tile * tile,struct xe_migrate * m,struct xe_vm * vm)184 static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, 185 struct xe_vm *vm) 186 { 187 struct xe_device *xe = tile_to_xe(tile); 188 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 189 u8 id = tile->id; 190 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; 191 #define VRAM_IDENTITY_MAP_COUNT 2 192 u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT; 193 #undef VRAM_IDENTITY_MAP_COUNT 194 u32 map_ofs, level, i; 195 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo; 196 u64 entry, pt29_ofs; 197 198 /* Can't bump NUM_PT_SLOTS too high */ 199 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE); 200 /* Must be a multiple of 64K to support all platforms */ 201 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K); 202 /* And one slot reserved for the 4KiB page table updates */ 203 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); 204 205 /* Need to be sure everything fits in the first PT, or create more */ 206 xe_tile_assert(tile, m->batch_base_ofs + batch->size < SZ_2M); 207 208 bo = xe_bo_create_pin_map(vm->xe, tile, vm, 209 num_entries * XE_PAGE_SIZE, 210 ttm_bo_type_kernel, 211 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 212 XE_BO_FLAG_PINNED); 213 if (IS_ERR(bo)) 214 return PTR_ERR(bo); 215 216 /* PT30 & PT31 reserved for 2M identity map */ 217 pt29_ofs = bo->size - 3 * XE_PAGE_SIZE; 218 entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs, pat_index); 219 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); 220 221 map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; 222 223 /* Map the entire BO in our level 0 pt */ 224 for (i = 0, level = 0; i < num_entries; level++) { 225 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE, 226 pat_index, 0); 227 228 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); 229 230 if (vm->flags & XE_VM_FLAG_64K) 231 i += 16; 232 else 233 i += 1; 234 } 235 236 if (!IS_DGFX(xe)) { 237 /* Write out batch too */ 238 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE; 239 for (i = 0; i < batch->size; 240 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 241 XE_PAGE_SIZE) { 242 entry = vm->pt_ops->pte_encode_bo(batch, i, 243 pat_index, 0); 244 245 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 246 entry); 247 level++; 248 } 249 if (xe->info.has_usm) { 250 xe_tile_assert(tile, batch->size == SZ_1M); 251 252 batch = tile->primary_gt->usm.bb_pool->bo; 253 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M; 254 xe_tile_assert(tile, batch->size == SZ_512K); 255 256 for (i = 0; i < batch->size; 257 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 258 XE_PAGE_SIZE) { 259 entry = vm->pt_ops->pte_encode_bo(batch, i, 260 pat_index, 0); 261 262 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 263 entry); 264 level++; 265 } 266 } 267 } else { 268 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 269 270 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false); 271 272 if (xe->info.has_usm) { 273 batch = tile->primary_gt->usm.bb_pool->bo; 274 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 275 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false); 276 } 277 } 278 279 for (level = 1; level < num_level; level++) { 280 u32 flags = 0; 281 282 if (vm->flags & XE_VM_FLAG_64K && level == 1) 283 flags = XE_PDE_64K; 284 285 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) * 286 XE_PAGE_SIZE, pat_index); 287 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64, 288 entry | flags); 289 } 290 291 /* Write PDE's that point to our BO. */ 292 for (i = 0; i < map_ofs / PAGE_SIZE; i++) { 293 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE, 294 pat_index); 295 296 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + 297 (i + 1) * 8, u64, entry); 298 } 299 300 /* Set up a 1GiB NULL mapping at 255GiB offset. */ 301 level = 2; 302 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64, 303 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) 304 | XE_PTE_NULL); 305 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level)); 306 307 /* Identity map the entire vram at 256GiB offset */ 308 if (IS_DGFX(xe)) { 309 u64 pt30_ofs = bo->size - 2 * XE_PAGE_SIZE; 310 311 xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET, 312 pat_index, pt30_ofs); 313 xe_assert(xe, xe->mem.vram.actual_physical_size <= 314 (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G); 315 316 /* 317 * Identity map the entire vram for compressed pat_index for xe2+ 318 * if flat ccs is enabled. 319 */ 320 if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) { 321 u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION]; 322 u64 vram_offset = IDENTITY_OFFSET + 323 DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G); 324 u64 pt31_ofs = bo->size - XE_PAGE_SIZE; 325 326 xe_assert(xe, xe->mem.vram.actual_physical_size <= (MAX_NUM_PTE - 327 IDENTITY_OFFSET - IDENTITY_OFFSET / 2) * SZ_1G); 328 xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset, 329 comp_pat_index, pt31_ofs); 330 } 331 } 332 333 /* 334 * Example layout created above, with root level = 3: 335 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's 336 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's 337 * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's 338 * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map] 339 * 340 * This makes the lowest part of the VM point to the pagetables. 341 * Hence the lowest 2M in the vm should point to itself, with a few writes 342 * and flushes, other parts of the VM can be used either for copying and 343 * clearing. 344 * 345 * For performance, the kernel reserves PDE's, so about 20 are left 346 * for async VM updates. 347 * 348 * To make it easier to work, each scratch PT is put in slot (1 + PT #) 349 * everywhere, this allows lockless updates to scratch pages by using 350 * the different addresses in VM. 351 */ 352 #define NUM_VMUSA_UNIT_PER_PAGE 32 353 #define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) 354 #define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) 355 drm_suballoc_manager_init(&m->vm_update_sa, 356 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) * 357 NUM_VMUSA_UNIT_PER_PAGE, 0); 358 359 m->pt_bo = bo; 360 return 0; 361 } 362 363 /* 364 * Including the reserved copy engine is required to avoid deadlocks due to 365 * migrate jobs servicing the faults gets stuck behind the job that faulted. 366 */ xe_migrate_usm_logical_mask(struct xe_gt * gt)367 static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt) 368 { 369 u32 logical_mask = 0; 370 struct xe_hw_engine *hwe; 371 enum xe_hw_engine_id id; 372 373 for_each_hw_engine(hwe, gt, id) { 374 if (hwe->class != XE_ENGINE_CLASS_COPY) 375 continue; 376 377 if (xe_gt_is_usm_hwe(gt, hwe)) 378 logical_mask |= BIT(hwe->logical_instance); 379 } 380 381 return logical_mask; 382 } 383 xe_migrate_needs_ccs_emit(struct xe_device * xe)384 static bool xe_migrate_needs_ccs_emit(struct xe_device *xe) 385 { 386 return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)); 387 } 388 389 /** 390 * xe_migrate_init() - Initialize a migrate context 391 * @tile: Back-pointer to the tile we're initializing for. 392 * 393 * Return: Pointer to a migrate context on success. Error pointer on error. 394 */ xe_migrate_init(struct xe_tile * tile)395 struct xe_migrate *xe_migrate_init(struct xe_tile *tile) 396 { 397 struct xe_device *xe = tile_to_xe(tile); 398 struct xe_gt *primary_gt = tile->primary_gt; 399 struct xe_migrate *m; 400 struct xe_vm *vm; 401 int err; 402 403 m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL); 404 if (!m) 405 return ERR_PTR(-ENOMEM); 406 407 m->tile = tile; 408 409 /* Special layout, prepared below.. */ 410 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | 411 XE_VM_FLAG_SET_TILE_ID(tile)); 412 if (IS_ERR(vm)) 413 return ERR_CAST(vm); 414 415 xe_vm_lock(vm, false); 416 err = xe_migrate_prepare_vm(tile, m, vm); 417 xe_vm_unlock(vm); 418 if (err) { 419 xe_vm_close_and_put(vm); 420 return ERR_PTR(err); 421 } 422 423 if (xe->info.has_usm) { 424 struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt, 425 XE_ENGINE_CLASS_COPY, 426 primary_gt->usm.reserved_bcs_instance, 427 false); 428 u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt); 429 430 if (!hwe || !logical_mask) 431 return ERR_PTR(-EINVAL); 432 433 /* 434 * XXX: Currently only reserving 1 (likely slow) BCS instance on 435 * PVC, may want to revisit if performance is needed. 436 */ 437 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe, 438 EXEC_QUEUE_FLAG_KERNEL | 439 EXEC_QUEUE_FLAG_PERMANENT | 440 EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0); 441 } else { 442 m->q = xe_exec_queue_create_class(xe, primary_gt, vm, 443 XE_ENGINE_CLASS_COPY, 444 EXEC_QUEUE_FLAG_KERNEL | 445 EXEC_QUEUE_FLAG_PERMANENT, 0); 446 } 447 if (IS_ERR(m->q)) { 448 xe_vm_close_and_put(vm); 449 return ERR_CAST(m->q); 450 } 451 452 mutex_init(&m->job_mutex); 453 fs_reclaim_acquire(GFP_KERNEL); 454 might_lock(&m->job_mutex); 455 fs_reclaim_release(GFP_KERNEL); 456 457 err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m); 458 if (err) 459 return ERR_PTR(err); 460 461 if (IS_DGFX(xe)) { 462 if (xe_migrate_needs_ccs_emit(xe)) 463 /* min chunk size corresponds to 4K of CCS Metadata */ 464 m->min_chunk_size = SZ_4K * SZ_64K / 465 xe_device_ccs_bytes(xe, SZ_64K); 466 else 467 /* Somewhat arbitrary to avoid a huge amount of blits */ 468 m->min_chunk_size = SZ_64K; 469 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size); 470 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n", 471 (unsigned long long)m->min_chunk_size); 472 } 473 474 return m; 475 } 476 max_mem_transfer_per_pass(struct xe_device * xe)477 static u64 max_mem_transfer_per_pass(struct xe_device *xe) 478 { 479 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) 480 return MAX_CCS_LIMITED_TRANSFER; 481 482 return MAX_PREEMPTDISABLE_TRANSFER; 483 } 484 xe_migrate_res_sizes(struct xe_migrate * m,struct xe_res_cursor * cur)485 static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur) 486 { 487 struct xe_device *xe = tile_to_xe(m->tile); 488 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining); 489 490 if (mem_type_is_vram(cur->mem_type)) { 491 /* 492 * VRAM we want to blit in chunks with sizes aligned to 493 * min_chunk_size in order for the offset to CCS metadata to be 494 * page-aligned. If it's the last chunk it may be smaller. 495 * 496 * Another constraint is that we need to limit the blit to 497 * the VRAM block size, unless size is smaller than 498 * min_chunk_size. 499 */ 500 u64 chunk = max_t(u64, cur->size, m->min_chunk_size); 501 502 size = min_t(u64, size, chunk); 503 if (size > m->min_chunk_size) 504 size = round_down(size, m->min_chunk_size); 505 } 506 507 return size; 508 } 509 xe_migrate_allow_identity(u64 size,const struct xe_res_cursor * cur)510 static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur) 511 { 512 /* If the chunk is not fragmented, allow identity map. */ 513 return cur->size >= size; 514 } 515 516 #define PTE_UPDATE_FLAG_IS_VRAM BIT(0) 517 #define PTE_UPDATE_FLAG_IS_COMP_PTE BIT(1) 518 pte_update_size(struct xe_migrate * m,u32 flags,struct ttm_resource * res,struct xe_res_cursor * cur,u64 * L0,u64 * L0_ofs,u32 * L0_pt,u32 cmd_size,u32 pt_ofs,u32 avail_pts)519 static u32 pte_update_size(struct xe_migrate *m, 520 u32 flags, 521 struct ttm_resource *res, 522 struct xe_res_cursor *cur, 523 u64 *L0, u64 *L0_ofs, u32 *L0_pt, 524 u32 cmd_size, u32 pt_ofs, u32 avail_pts) 525 { 526 u32 cmds = 0; 527 bool is_vram = PTE_UPDATE_FLAG_IS_VRAM & flags; 528 bool is_comp_pte = PTE_UPDATE_FLAG_IS_COMP_PTE & flags; 529 530 *L0_pt = pt_ofs; 531 if (is_vram && xe_migrate_allow_identity(*L0, cur)) { 532 /* Offset into identity map. */ 533 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile), 534 cur->start + vram_region_gpu_offset(res), 535 is_comp_pte); 536 cmds += cmd_size; 537 } else { 538 /* Clip L0 to available size */ 539 u64 size = min(*L0, (u64)avail_pts * SZ_2M); 540 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT; 541 542 *L0 = size; 543 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); 544 545 /* MI_STORE_DATA_IMM */ 546 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI); 547 548 /* PDE qwords */ 549 cmds += num_4k_pages * 2; 550 551 /* Each chunk has a single blit command */ 552 cmds += cmd_size; 553 } 554 555 return cmds; 556 } 557 emit_pte(struct xe_migrate * m,struct xe_bb * bb,u32 at_pt,bool is_vram,bool is_comp_pte,struct xe_res_cursor * cur,u32 size,struct ttm_resource * res)558 static void emit_pte(struct xe_migrate *m, 559 struct xe_bb *bb, u32 at_pt, 560 bool is_vram, bool is_comp_pte, 561 struct xe_res_cursor *cur, 562 u32 size, struct ttm_resource *res) 563 { 564 struct xe_device *xe = tile_to_xe(m->tile); 565 struct xe_vm *vm = m->q->vm; 566 u16 pat_index; 567 u32 ptes; 568 u64 ofs = (u64)at_pt * XE_PAGE_SIZE; 569 u64 cur_ofs; 570 571 /* Indirect access needs compression enabled uncached PAT index */ 572 if (GRAPHICS_VERx100(xe) >= 2000) 573 pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] : 574 xe->pat.idx[XE_CACHE_WB]; 575 else 576 pat_index = xe->pat.idx[XE_CACHE_WB]; 577 578 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); 579 580 while (ptes) { 581 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 582 583 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 584 bb->cs[bb->len++] = ofs; 585 bb->cs[bb->len++] = 0; 586 587 cur_ofs = ofs; 588 ofs += chunk * 8; 589 ptes -= chunk; 590 591 while (chunk--) { 592 u64 addr, flags = 0; 593 bool devmem = false; 594 595 addr = xe_res_dma(cur) & PAGE_MASK; 596 if (is_vram) { 597 if (vm->flags & XE_VM_FLAG_64K) { 598 u64 va = cur_ofs * XE_PAGE_SIZE / 8; 599 600 xe_assert(xe, (va & (SZ_64K - 1)) == 601 (addr & (SZ_64K - 1))); 602 603 flags |= XE_PTE_PS64; 604 } 605 606 addr += vram_region_gpu_offset(res); 607 devmem = true; 608 } 609 610 addr = vm->pt_ops->pte_encode_addr(m->tile->xe, 611 addr, pat_index, 612 0, devmem, flags); 613 bb->cs[bb->len++] = lower_32_bits(addr); 614 bb->cs[bb->len++] = upper_32_bits(addr); 615 616 xe_res_next(cur, min_t(u32, size, PAGE_SIZE)); 617 cur_ofs += 8; 618 } 619 } 620 } 621 622 #define EMIT_COPY_CCS_DW 5 emit_copy_ccs(struct xe_gt * gt,struct xe_bb * bb,u64 dst_ofs,bool dst_is_indirect,u64 src_ofs,bool src_is_indirect,u32 size)623 static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, 624 u64 dst_ofs, bool dst_is_indirect, 625 u64 src_ofs, bool src_is_indirect, 626 u32 size) 627 { 628 struct xe_device *xe = gt_to_xe(gt); 629 u32 *cs = bb->cs + bb->len; 630 u32 num_ccs_blks; 631 u32 num_pages; 632 u32 ccs_copy_size; 633 u32 mocs; 634 635 if (GRAPHICS_VERx100(xe) >= 2000) { 636 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE); 637 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1)); 638 639 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1); 640 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index); 641 642 } else { 643 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), 644 NUM_CCS_BYTES_PER_BLOCK); 645 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1)); 646 647 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1); 648 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index); 649 } 650 651 *cs++ = XY_CTRL_SURF_COPY_BLT | 652 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | 653 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | 654 ccs_copy_size; 655 *cs++ = lower_32_bits(src_ofs); 656 *cs++ = upper_32_bits(src_ofs) | mocs; 657 *cs++ = lower_32_bits(dst_ofs); 658 *cs++ = upper_32_bits(dst_ofs) | mocs; 659 660 bb->len = cs - bb->cs; 661 } 662 663 #define EMIT_COPY_DW 10 emit_copy(struct xe_gt * gt,struct xe_bb * bb,u64 src_ofs,u64 dst_ofs,unsigned int size,unsigned int pitch)664 static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, 665 u64 src_ofs, u64 dst_ofs, unsigned int size, 666 unsigned int pitch) 667 { 668 struct xe_device *xe = gt_to_xe(gt); 669 u32 mocs = 0; 670 u32 tile_y = 0; 671 672 xe_gt_assert(gt, size / pitch <= S16_MAX); 673 xe_gt_assert(gt, pitch / 4 <= S16_MAX); 674 xe_gt_assert(gt, pitch <= U16_MAX); 675 676 if (GRAPHICS_VER(xe) >= 20) 677 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index); 678 679 if (GRAPHICS_VERx100(xe) >= 1250) 680 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; 681 682 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); 683 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; 684 bb->cs[bb->len++] = 0; 685 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; 686 bb->cs[bb->len++] = lower_32_bits(dst_ofs); 687 bb->cs[bb->len++] = upper_32_bits(dst_ofs); 688 bb->cs[bb->len++] = 0; 689 bb->cs[bb->len++] = pitch | mocs; 690 bb->cs[bb->len++] = lower_32_bits(src_ofs); 691 bb->cs[bb->len++] = upper_32_bits(src_ofs); 692 } 693 xe_migrate_batch_base(struct xe_migrate * m,bool usm)694 static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) 695 { 696 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; 697 } 698 xe_migrate_ccs_copy(struct xe_migrate * m,struct xe_bb * bb,u64 src_ofs,bool src_is_indirect,u64 dst_ofs,bool dst_is_indirect,u32 dst_size,u64 ccs_ofs,bool copy_ccs)699 static u32 xe_migrate_ccs_copy(struct xe_migrate *m, 700 struct xe_bb *bb, 701 u64 src_ofs, bool src_is_indirect, 702 u64 dst_ofs, bool dst_is_indirect, u32 dst_size, 703 u64 ccs_ofs, bool copy_ccs) 704 { 705 struct xe_gt *gt = m->tile->primary_gt; 706 u32 flush_flags = 0; 707 708 if (!copy_ccs && dst_is_indirect) { 709 /* 710 * If the src is already in vram, then it should already 711 * have been cleared by us, or has been populated by the 712 * user. Make sure we copy the CCS aux state as-is. 713 * 714 * Otherwise if the bo doesn't have any CCS metadata attached, 715 * we still need to clear it for security reasons. 716 */ 717 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs; 718 719 emit_copy_ccs(gt, bb, 720 dst_ofs, true, 721 ccs_src_ofs, src_is_indirect, dst_size); 722 723 flush_flags = MI_FLUSH_DW_CCS; 724 } else if (copy_ccs) { 725 if (!src_is_indirect) 726 src_ofs = ccs_ofs; 727 else if (!dst_is_indirect) 728 dst_ofs = ccs_ofs; 729 730 xe_gt_assert(gt, src_is_indirect || dst_is_indirect); 731 732 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs, 733 src_is_indirect, dst_size); 734 if (dst_is_indirect) 735 flush_flags = MI_FLUSH_DW_CCS; 736 } 737 738 return flush_flags; 739 } 740 741 /** 742 * xe_migrate_copy() - Copy content of TTM resources. 743 * @m: The migration context. 744 * @src_bo: The buffer object @src is currently bound to. 745 * @dst_bo: If copying between resources created for the same bo, set this to 746 * the same value as @src_bo. If copying between buffer objects, set it to 747 * the buffer object @dst is currently bound to. 748 * @src: The source TTM resource. 749 * @dst: The dst TTM resource. 750 * @copy_only_ccs: If true copy only CCS metadata 751 * 752 * Copies the contents of @src to @dst: On flat CCS devices, 753 * the CCS metadata is copied as well if needed, or if not present, 754 * the CCS metadata of @dst is cleared for security reasons. 755 * 756 * Return: Pointer to a dma_fence representing the last copy batch, or 757 * an error pointer on failure. If there is a failure, any copy operation 758 * started by the function call has been synced. 759 */ xe_migrate_copy(struct xe_migrate * m,struct xe_bo * src_bo,struct xe_bo * dst_bo,struct ttm_resource * src,struct ttm_resource * dst,bool copy_only_ccs)760 struct dma_fence *xe_migrate_copy(struct xe_migrate *m, 761 struct xe_bo *src_bo, 762 struct xe_bo *dst_bo, 763 struct ttm_resource *src, 764 struct ttm_resource *dst, 765 bool copy_only_ccs) 766 { 767 struct xe_gt *gt = m->tile->primary_gt; 768 struct xe_device *xe = gt_to_xe(gt); 769 struct dma_fence *fence = NULL; 770 u64 size = src_bo->size; 771 struct xe_res_cursor src_it, dst_it, ccs_it; 772 u64 src_L0_ofs, dst_L0_ofs; 773 u32 src_L0_pt, dst_L0_pt; 774 u64 src_L0, dst_L0; 775 int pass = 0; 776 int err; 777 bool src_is_pltt = src->mem_type == XE_PL_TT; 778 bool dst_is_pltt = dst->mem_type == XE_PL_TT; 779 bool src_is_vram = mem_type_is_vram(src->mem_type); 780 bool dst_is_vram = mem_type_is_vram(dst->mem_type); 781 bool copy_ccs = xe_device_has_flat_ccs(xe) && 782 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo); 783 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); 784 bool use_comp_pat = xe_device_has_flat_ccs(xe) && 785 GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram; 786 787 /* Copying CCS between two different BOs is not supported yet. */ 788 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo)) 789 return ERR_PTR(-EINVAL); 790 791 if (src_bo != dst_bo && XE_WARN_ON(src_bo->size != dst_bo->size)) 792 return ERR_PTR(-EINVAL); 793 794 if (!src_is_vram) 795 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); 796 else 797 xe_res_first(src, 0, size, &src_it); 798 if (!dst_is_vram) 799 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it); 800 else 801 xe_res_first(dst, 0, size, &dst_it); 802 803 if (copy_system_ccs) 804 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), 805 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), 806 &ccs_it); 807 808 while (size) { 809 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ 810 struct xe_sched_job *job; 811 struct xe_bb *bb; 812 u32 flush_flags = 0; 813 u32 update_idx; 814 u64 ccs_ofs, ccs_size; 815 u32 ccs_pt; 816 u32 pte_flags; 817 818 bool usm = xe->info.has_usm; 819 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 820 821 src_L0 = xe_migrate_res_sizes(m, &src_it); 822 dst_L0 = xe_migrate_res_sizes(m, &dst_it); 823 824 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", 825 pass++, src_L0, dst_L0); 826 827 src_L0 = min(src_L0, dst_L0); 828 829 pte_flags = src_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 830 pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0; 831 batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0, 832 &src_L0_ofs, &src_L0_pt, 0, 0, 833 avail_pts); 834 835 pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 836 batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0, 837 &dst_L0_ofs, &dst_L0_pt, 0, 838 avail_pts, avail_pts); 839 840 if (copy_system_ccs) { 841 ccs_size = xe_device_ccs_bytes(xe, src_L0); 842 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, 843 &ccs_ofs, &ccs_pt, 0, 844 2 * avail_pts, 845 avail_pts); 846 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); 847 } 848 849 /* Add copy commands size here */ 850 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + 851 ((xe_migrate_needs_ccs_emit(xe) ? EMIT_COPY_CCS_DW : 0)); 852 853 bb = xe_bb_new(gt, batch_size, usm); 854 if (IS_ERR(bb)) { 855 err = PTR_ERR(bb); 856 goto err_sync; 857 } 858 859 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it)) 860 xe_res_next(&src_it, src_L0); 861 else 862 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs, 863 &src_it, src_L0, src); 864 865 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it)) 866 xe_res_next(&dst_it, src_L0); 867 else 868 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs, 869 &dst_it, src_L0, dst); 870 871 if (copy_system_ccs) 872 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); 873 874 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 875 update_idx = bb->len; 876 877 if (!copy_only_ccs) 878 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE); 879 880 if (xe_migrate_needs_ccs_emit(xe)) 881 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, 882 IS_DGFX(xe) ? src_is_vram : src_is_pltt, 883 dst_L0_ofs, 884 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt, 885 src_L0, ccs_ofs, copy_ccs); 886 887 job = xe_bb_create_migration_job(m->q, bb, 888 xe_migrate_batch_base(m, usm), 889 update_idx); 890 if (IS_ERR(job)) { 891 err = PTR_ERR(job); 892 goto err; 893 } 894 895 xe_sched_job_add_migrate_flush(job, flush_flags); 896 if (!fence) { 897 err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv, 898 DMA_RESV_USAGE_BOOKKEEP); 899 if (!err && src_bo != dst_bo) 900 err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv, 901 DMA_RESV_USAGE_BOOKKEEP); 902 if (err) 903 goto err_job; 904 } 905 906 mutex_lock(&m->job_mutex); 907 xe_sched_job_arm(job); 908 dma_fence_put(fence); 909 fence = dma_fence_get(&job->drm.s_fence->finished); 910 xe_sched_job_push(job); 911 912 dma_fence_put(m->fence); 913 m->fence = dma_fence_get(fence); 914 915 mutex_unlock(&m->job_mutex); 916 917 xe_bb_free(bb, fence); 918 size -= src_L0; 919 continue; 920 921 err_job: 922 xe_sched_job_put(job); 923 err: 924 xe_bb_free(bb, NULL); 925 926 err_sync: 927 /* Sync partial copy if any. FIXME: under job_mutex? */ 928 if (fence) { 929 dma_fence_wait(fence, false); 930 dma_fence_put(fence); 931 } 932 933 return ERR_PTR(err); 934 } 935 936 return fence; 937 } 938 emit_clear_link_copy(struct xe_gt * gt,struct xe_bb * bb,u64 src_ofs,u32 size,u32 pitch)939 static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 940 u32 size, u32 pitch) 941 { 942 struct xe_device *xe = gt_to_xe(gt); 943 u32 *cs = bb->cs + bb->len; 944 u32 len = PVC_MEM_SET_CMD_LEN_DW; 945 946 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); 947 *cs++ = pitch - 1; 948 *cs++ = (size / pitch) - 1; 949 *cs++ = pitch - 1; 950 *cs++ = lower_32_bits(src_ofs); 951 *cs++ = upper_32_bits(src_ofs); 952 if (GRAPHICS_VERx100(xe) >= 2000) 953 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 954 else 955 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 956 957 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 958 959 bb->len += len; 960 } 961 emit_clear_main_copy(struct xe_gt * gt,struct xe_bb * bb,u64 src_ofs,u32 size,u32 pitch,bool is_vram)962 static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, 963 u64 src_ofs, u32 size, u32 pitch, bool is_vram) 964 { 965 struct xe_device *xe = gt_to_xe(gt); 966 u32 *cs = bb->cs + bb->len; 967 u32 len = XY_FAST_COLOR_BLT_DW; 968 969 if (GRAPHICS_VERx100(xe) < 1250) 970 len = 11; 971 972 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | 973 (len - 2); 974 if (GRAPHICS_VERx100(xe) >= 2000) 975 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) | 976 (pitch - 1); 977 else 978 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) | 979 (pitch - 1); 980 *cs++ = 0; 981 *cs++ = (size / pitch) << 16 | pitch / 4; 982 *cs++ = lower_32_bits(src_ofs); 983 *cs++ = upper_32_bits(src_ofs); 984 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; 985 *cs++ = 0; 986 *cs++ = 0; 987 *cs++ = 0; 988 *cs++ = 0; 989 990 if (len > 11) { 991 *cs++ = 0; 992 *cs++ = 0; 993 *cs++ = 0; 994 *cs++ = 0; 995 *cs++ = 0; 996 } 997 998 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 999 1000 bb->len += len; 1001 } 1002 has_service_copy_support(struct xe_gt * gt)1003 static bool has_service_copy_support(struct xe_gt *gt) 1004 { 1005 /* 1006 * What we care about is whether the architecture was designed with 1007 * service copy functionality (specifically the new MEM_SET / MEM_COPY 1008 * instructions) so check the architectural engine list rather than the 1009 * actual list since these instructions are usable on BCS0 even if 1010 * all of the actual service copy engines (BCS1-BCS8) have been fused 1011 * off. 1012 */ 1013 return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8, 1014 XE_HW_ENGINE_BCS1); 1015 } 1016 emit_clear_cmd_len(struct xe_gt * gt)1017 static u32 emit_clear_cmd_len(struct xe_gt *gt) 1018 { 1019 if (has_service_copy_support(gt)) 1020 return PVC_MEM_SET_CMD_LEN_DW; 1021 else 1022 return XY_FAST_COLOR_BLT_DW; 1023 } 1024 emit_clear(struct xe_gt * gt,struct xe_bb * bb,u64 src_ofs,u32 size,u32 pitch,bool is_vram)1025 static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 1026 u32 size, u32 pitch, bool is_vram) 1027 { 1028 if (has_service_copy_support(gt)) 1029 emit_clear_link_copy(gt, bb, src_ofs, size, pitch); 1030 else 1031 emit_clear_main_copy(gt, bb, src_ofs, size, pitch, 1032 is_vram); 1033 } 1034 1035 /** 1036 * xe_migrate_clear() - Copy content of TTM resources. 1037 * @m: The migration context. 1038 * @bo: The buffer object @dst is currently bound to. 1039 * @dst: The dst TTM resource to be cleared. 1040 * @clear_flags: flags to specify which data to clear: CCS, BO, or both. 1041 * 1042 * Clear the contents of @dst to zero when XE_MIGRATE_CLEAR_FLAG_BO_DATA is set. 1043 * On flat CCS devices, the CCS metadata is cleared to zero with XE_MIGRATE_CLEAR_FLAG_CCS_DATA. 1044 * Set XE_MIGRATE_CLEAR_FLAG_FULL to clear bo as well as CCS metadata. 1045 * TODO: Eliminate the @bo argument. 1046 * 1047 * Return: Pointer to a dma_fence representing the last clear batch, or 1048 * an error pointer on failure. If there is a failure, any clear operation 1049 * started by the function call has been synced. 1050 */ xe_migrate_clear(struct xe_migrate * m,struct xe_bo * bo,struct ttm_resource * dst,u32 clear_flags)1051 struct dma_fence *xe_migrate_clear(struct xe_migrate *m, 1052 struct xe_bo *bo, 1053 struct ttm_resource *dst, 1054 u32 clear_flags) 1055 { 1056 bool clear_vram = mem_type_is_vram(dst->mem_type); 1057 bool clear_bo_data = XE_MIGRATE_CLEAR_FLAG_BO_DATA & clear_flags; 1058 bool clear_ccs = XE_MIGRATE_CLEAR_FLAG_CCS_DATA & clear_flags; 1059 struct xe_gt *gt = m->tile->primary_gt; 1060 struct xe_device *xe = gt_to_xe(gt); 1061 bool clear_only_system_ccs = false; 1062 struct dma_fence *fence = NULL; 1063 u64 size = bo->size; 1064 struct xe_res_cursor src_it; 1065 struct ttm_resource *src = dst; 1066 int err; 1067 1068 if (WARN_ON(!clear_bo_data && !clear_ccs)) 1069 return NULL; 1070 1071 if (!clear_bo_data && clear_ccs && !IS_DGFX(xe)) 1072 clear_only_system_ccs = true; 1073 1074 if (!clear_vram) 1075 xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it); 1076 else 1077 xe_res_first(src, 0, bo->size, &src_it); 1078 1079 while (size) { 1080 u64 clear_L0_ofs; 1081 u32 clear_L0_pt; 1082 u32 flush_flags = 0; 1083 u64 clear_L0; 1084 struct xe_sched_job *job; 1085 struct xe_bb *bb; 1086 u32 batch_size, update_idx; 1087 u32 pte_flags; 1088 1089 bool usm = xe->info.has_usm; 1090 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1091 1092 clear_L0 = xe_migrate_res_sizes(m, &src_it); 1093 1094 /* Calculate final sizes and batch size.. */ 1095 pte_flags = clear_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 1096 batch_size = 2 + 1097 pte_update_size(m, pte_flags, src, &src_it, 1098 &clear_L0, &clear_L0_ofs, &clear_L0_pt, 1099 clear_bo_data ? emit_clear_cmd_len(gt) : 0, 0, 1100 avail_pts); 1101 1102 if (xe_migrate_needs_ccs_emit(xe)) 1103 batch_size += EMIT_COPY_CCS_DW; 1104 1105 /* Clear commands */ 1106 1107 if (WARN_ON_ONCE(!clear_L0)) 1108 break; 1109 1110 bb = xe_bb_new(gt, batch_size, usm); 1111 if (IS_ERR(bb)) { 1112 err = PTR_ERR(bb); 1113 goto err_sync; 1114 } 1115 1116 size -= clear_L0; 1117 /* Preemption is enabled again by the ring ops. */ 1118 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) 1119 xe_res_next(&src_it, clear_L0); 1120 else 1121 emit_pte(m, bb, clear_L0_pt, clear_vram, clear_only_system_ccs, 1122 &src_it, clear_L0, dst); 1123 1124 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1125 update_idx = bb->len; 1126 1127 if (clear_bo_data) 1128 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram); 1129 1130 if (xe_migrate_needs_ccs_emit(xe)) { 1131 emit_copy_ccs(gt, bb, clear_L0_ofs, true, 1132 m->cleared_mem_ofs, false, clear_L0); 1133 flush_flags = MI_FLUSH_DW_CCS; 1134 } 1135 1136 job = xe_bb_create_migration_job(m->q, bb, 1137 xe_migrate_batch_base(m, usm), 1138 update_idx); 1139 if (IS_ERR(job)) { 1140 err = PTR_ERR(job); 1141 goto err; 1142 } 1143 1144 xe_sched_job_add_migrate_flush(job, flush_flags); 1145 if (!fence) { 1146 /* 1147 * There can't be anything userspace related at this 1148 * point, so we just need to respect any potential move 1149 * fences, which are always tracked as 1150 * DMA_RESV_USAGE_KERNEL. 1151 */ 1152 err = xe_sched_job_add_deps(job, bo->ttm.base.resv, 1153 DMA_RESV_USAGE_KERNEL); 1154 if (err) 1155 goto err_job; 1156 } 1157 1158 mutex_lock(&m->job_mutex); 1159 xe_sched_job_arm(job); 1160 dma_fence_put(fence); 1161 fence = dma_fence_get(&job->drm.s_fence->finished); 1162 xe_sched_job_push(job); 1163 1164 dma_fence_put(m->fence); 1165 m->fence = dma_fence_get(fence); 1166 1167 mutex_unlock(&m->job_mutex); 1168 1169 xe_bb_free(bb, fence); 1170 continue; 1171 1172 err_job: 1173 xe_sched_job_put(job); 1174 err: 1175 xe_bb_free(bb, NULL); 1176 err_sync: 1177 /* Sync partial copies if any. FIXME: job_mutex? */ 1178 if (fence) { 1179 dma_fence_wait(m->fence, false); 1180 dma_fence_put(fence); 1181 } 1182 1183 return ERR_PTR(err); 1184 } 1185 1186 if (clear_ccs) 1187 bo->ccs_cleared = true; 1188 1189 return fence; 1190 } 1191 write_pgtable(struct xe_tile * tile,struct xe_bb * bb,u64 ppgtt_ofs,const struct xe_vm_pgtable_update_op * pt_op,const struct xe_vm_pgtable_update * update,struct xe_migrate_pt_update * pt_update)1192 static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, 1193 const struct xe_vm_pgtable_update_op *pt_op, 1194 const struct xe_vm_pgtable_update *update, 1195 struct xe_migrate_pt_update *pt_update) 1196 { 1197 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1198 u32 chunk; 1199 u32 ofs = update->ofs, size = update->qwords; 1200 1201 /* 1202 * If we have 512 entries (max), we would populate it ourselves, 1203 * and update the PDE above it to the new pointer. 1204 * The only time this can only happen if we have to update the top 1205 * PDE. This requires a BO that is almost vm->size big. 1206 * 1207 * This shouldn't be possible in practice.. might change when 16K 1208 * pages are used. Hence the assert. 1209 */ 1210 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE); 1211 if (!ppgtt_ofs) 1212 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile), 1213 xe_bo_addr(update->pt_bo, 0, 1214 XE_PAGE_SIZE), false); 1215 1216 do { 1217 u64 addr = ppgtt_ofs + ofs * 8; 1218 1219 chunk = min(size, MAX_PTE_PER_SDI); 1220 1221 /* Ensure populatefn can do memset64 by aligning bb->cs */ 1222 if (!(bb->len & 1)) 1223 bb->cs[bb->len++] = MI_NOOP; 1224 1225 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 1226 bb->cs[bb->len++] = lower_32_bits(addr); 1227 bb->cs[bb->len++] = upper_32_bits(addr); 1228 if (pt_op->bind) 1229 ops->populate(pt_update, tile, NULL, bb->cs + bb->len, 1230 ofs, chunk, update); 1231 else 1232 ops->clear(pt_update, tile, NULL, bb->cs + bb->len, 1233 ofs, chunk, update); 1234 1235 bb->len += chunk * 2; 1236 ofs += chunk; 1237 size -= chunk; 1238 } while (size); 1239 } 1240 xe_migrate_get_vm(struct xe_migrate * m)1241 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) 1242 { 1243 return xe_vm_get(m->q->vm); 1244 } 1245 1246 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1247 struct migrate_test_params { 1248 struct xe_test_priv base; 1249 bool force_gpu; 1250 }; 1251 1252 #define to_migrate_test_params(_priv) \ 1253 container_of(_priv, struct migrate_test_params, base) 1254 #endif 1255 1256 static struct dma_fence * xe_migrate_update_pgtables_cpu(struct xe_migrate * m,struct xe_migrate_pt_update * pt_update)1257 xe_migrate_update_pgtables_cpu(struct xe_migrate *m, 1258 struct xe_migrate_pt_update *pt_update) 1259 { 1260 XE_TEST_DECLARE(struct migrate_test_params *test = 1261 to_migrate_test_params 1262 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));) 1263 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1264 struct xe_vm *vm = pt_update->vops->vm; 1265 struct xe_vm_pgtable_update_ops *pt_update_ops = 1266 &pt_update->vops->pt_update_ops[pt_update->tile_id]; 1267 int err; 1268 u32 i, j; 1269 1270 if (XE_TEST_ONLY(test && test->force_gpu)) 1271 return ERR_PTR(-ETIME); 1272 1273 if (ops->pre_commit) { 1274 pt_update->job = NULL; 1275 err = ops->pre_commit(pt_update); 1276 if (err) 1277 return ERR_PTR(err); 1278 } 1279 1280 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1281 const struct xe_vm_pgtable_update_op *pt_op = 1282 &pt_update_ops->ops[i]; 1283 1284 for (j = 0; j < pt_op->num_entries; j++) { 1285 const struct xe_vm_pgtable_update *update = 1286 &pt_op->entries[j]; 1287 1288 if (pt_op->bind) 1289 ops->populate(pt_update, m->tile, 1290 &update->pt_bo->vmap, NULL, 1291 update->ofs, update->qwords, 1292 update); 1293 else 1294 ops->clear(pt_update, m->tile, 1295 &update->pt_bo->vmap, NULL, 1296 update->ofs, update->qwords, update); 1297 } 1298 } 1299 1300 trace_xe_vm_cpu_bind(vm); 1301 xe_device_wmb(vm->xe); 1302 1303 return dma_fence_get_stub(); 1304 } 1305 1306 static struct dma_fence * __xe_migrate_update_pgtables(struct xe_migrate * m,struct xe_migrate_pt_update * pt_update,struct xe_vm_pgtable_update_ops * pt_update_ops)1307 __xe_migrate_update_pgtables(struct xe_migrate *m, 1308 struct xe_migrate_pt_update *pt_update, 1309 struct xe_vm_pgtable_update_ops *pt_update_ops) 1310 { 1311 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1312 struct xe_tile *tile = m->tile; 1313 struct xe_gt *gt = tile->primary_gt; 1314 struct xe_device *xe = tile_to_xe(tile); 1315 struct xe_sched_job *job; 1316 struct dma_fence *fence; 1317 struct drm_suballoc *sa_bo = NULL; 1318 struct xe_bb *bb; 1319 u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0; 1320 u32 num_updates = 0, current_update = 0; 1321 u64 addr; 1322 int err = 0; 1323 bool is_migrate = pt_update_ops->q == m->q; 1324 bool usm = is_migrate && xe->info.has_usm; 1325 1326 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1327 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; 1328 struct xe_vm_pgtable_update *updates = pt_op->entries; 1329 1330 num_updates += pt_op->num_entries; 1331 for (j = 0; j < pt_op->num_entries; ++j) { 1332 u32 num_cmds = DIV_ROUND_UP(updates[j].qwords, 1333 MAX_PTE_PER_SDI); 1334 1335 /* align noop + MI_STORE_DATA_IMM cmd prefix */ 1336 batch_size += 4 * num_cmds + updates[j].qwords * 2; 1337 } 1338 } 1339 1340 /* fixed + PTE entries */ 1341 if (IS_DGFX(xe)) 1342 batch_size += 2; 1343 else 1344 batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) + 1345 num_updates * 2; 1346 1347 bb = xe_bb_new(gt, batch_size, usm); 1348 if (IS_ERR(bb)) 1349 return ERR_CAST(bb); 1350 1351 /* For sysmem PTE's, need to map them in our hole.. */ 1352 if (!IS_DGFX(xe)) { 1353 u32 ptes, ofs; 1354 1355 ppgtt_ofs = NUM_KERNEL_PDE - 1; 1356 if (!is_migrate) { 1357 u32 num_units = DIV_ROUND_UP(num_updates, 1358 NUM_VMUSA_WRITES_PER_UNIT); 1359 1360 if (num_units > m->vm_update_sa.size) { 1361 err = -ENOBUFS; 1362 goto err_bb; 1363 } 1364 sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units, 1365 GFP_KERNEL, true, 0); 1366 if (IS_ERR(sa_bo)) { 1367 err = PTR_ERR(sa_bo); 1368 goto err_bb; 1369 } 1370 1371 ppgtt_ofs = NUM_KERNEL_PDE + 1372 (drm_suballoc_soffset(sa_bo) / 1373 NUM_VMUSA_UNIT_PER_PAGE); 1374 page_ofs = (drm_suballoc_soffset(sa_bo) % 1375 NUM_VMUSA_UNIT_PER_PAGE) * 1376 VM_SA_UPDATE_UNIT_SIZE; 1377 } 1378 1379 /* Map our PT's to gtt */ 1380 i = 0; 1381 j = 0; 1382 ptes = num_updates; 1383 ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs; 1384 while (ptes) { 1385 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 1386 u32 idx = 0; 1387 1388 bb->cs[bb->len++] = MI_STORE_DATA_IMM | 1389 MI_SDI_NUM_QW(chunk); 1390 bb->cs[bb->len++] = ofs; 1391 bb->cs[bb->len++] = 0; /* upper_32_bits */ 1392 1393 for (; i < pt_update_ops->num_ops; ++i) { 1394 struct xe_vm_pgtable_update_op *pt_op = 1395 &pt_update_ops->ops[i]; 1396 struct xe_vm_pgtable_update *updates = pt_op->entries; 1397 1398 for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) { 1399 struct xe_vm *vm = pt_update->vops->vm; 1400 struct xe_bo *pt_bo = updates[j].pt_bo; 1401 1402 if (idx == chunk) 1403 goto next_cmd; 1404 1405 xe_tile_assert(tile, pt_bo->size == SZ_4K); 1406 1407 /* Map a PT at most once */ 1408 if (pt_bo->update_index < 0) 1409 pt_bo->update_index = current_update; 1410 1411 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, 1412 XE_CACHE_WB, 0); 1413 bb->cs[bb->len++] = lower_32_bits(addr); 1414 bb->cs[bb->len++] = upper_32_bits(addr); 1415 } 1416 1417 j = 0; 1418 } 1419 1420 next_cmd: 1421 ptes -= chunk; 1422 ofs += chunk * sizeof(u64); 1423 } 1424 1425 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1426 update_idx = bb->len; 1427 1428 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + 1429 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; 1430 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1431 struct xe_vm_pgtable_update_op *pt_op = 1432 &pt_update_ops->ops[i]; 1433 struct xe_vm_pgtable_update *updates = pt_op->entries; 1434 1435 for (j = 0; j < pt_op->num_entries; ++j) { 1436 struct xe_bo *pt_bo = updates[j].pt_bo; 1437 1438 write_pgtable(tile, bb, addr + 1439 pt_bo->update_index * XE_PAGE_SIZE, 1440 pt_op, &updates[j], pt_update); 1441 } 1442 } 1443 } else { 1444 /* phys pages, no preamble required */ 1445 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1446 update_idx = bb->len; 1447 1448 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1449 struct xe_vm_pgtable_update_op *pt_op = 1450 &pt_update_ops->ops[i]; 1451 struct xe_vm_pgtable_update *updates = pt_op->entries; 1452 1453 for (j = 0; j < pt_op->num_entries; ++j) 1454 write_pgtable(tile, bb, 0, pt_op, &updates[j], 1455 pt_update); 1456 } 1457 } 1458 1459 job = xe_bb_create_migration_job(pt_update_ops->q, bb, 1460 xe_migrate_batch_base(m, usm), 1461 update_idx); 1462 if (IS_ERR(job)) { 1463 err = PTR_ERR(job); 1464 goto err_sa; 1465 } 1466 1467 if (ops->pre_commit) { 1468 pt_update->job = job; 1469 err = ops->pre_commit(pt_update); 1470 if (err) 1471 goto err_job; 1472 } 1473 if (is_migrate) 1474 mutex_lock(&m->job_mutex); 1475 1476 xe_sched_job_arm(job); 1477 fence = dma_fence_get(&job->drm.s_fence->finished); 1478 xe_sched_job_push(job); 1479 1480 if (is_migrate) 1481 mutex_unlock(&m->job_mutex); 1482 1483 xe_bb_free(bb, fence); 1484 drm_suballoc_free(sa_bo, fence); 1485 1486 return fence; 1487 1488 err_job: 1489 xe_sched_job_put(job); 1490 err_sa: 1491 drm_suballoc_free(sa_bo, NULL); 1492 err_bb: 1493 xe_bb_free(bb, NULL); 1494 return ERR_PTR(err); 1495 } 1496 1497 /** 1498 * xe_migrate_update_pgtables() - Pipelined page-table update 1499 * @m: The migrate context. 1500 * @pt_update: PT update arguments 1501 * 1502 * Perform a pipelined page-table update. The update descriptors are typically 1503 * built under the same lock critical section as a call to this function. If 1504 * using the default engine for the updates, they will be performed in the 1505 * order they grab the job_mutex. If different engines are used, external 1506 * synchronization is needed for overlapping updates to maintain page-table 1507 * consistency. Note that the meaing of "overlapping" is that the updates 1508 * touch the same page-table, which might be a higher-level page-directory. 1509 * If no pipelining is needed, then updates may be performed by the cpu. 1510 * 1511 * Return: A dma_fence that, when signaled, indicates the update completion. 1512 */ 1513 struct dma_fence * xe_migrate_update_pgtables(struct xe_migrate * m,struct xe_migrate_pt_update * pt_update)1514 xe_migrate_update_pgtables(struct xe_migrate *m, 1515 struct xe_migrate_pt_update *pt_update) 1516 1517 { 1518 struct xe_vm_pgtable_update_ops *pt_update_ops = 1519 &pt_update->vops->pt_update_ops[pt_update->tile_id]; 1520 struct dma_fence *fence; 1521 1522 fence = xe_migrate_update_pgtables_cpu(m, pt_update); 1523 1524 /* -ETIME indicates a job is needed, anything else is legit error */ 1525 if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME) 1526 return fence; 1527 1528 return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops); 1529 } 1530 1531 /** 1532 * xe_migrate_wait() - Complete all operations using the xe_migrate context 1533 * @m: Migrate context to wait for. 1534 * 1535 * Waits until the GPU no longer uses the migrate context's default engine 1536 * or its page-table objects. FIXME: What about separate page-table update 1537 * engines? 1538 */ xe_migrate_wait(struct xe_migrate * m)1539 void xe_migrate_wait(struct xe_migrate *m) 1540 { 1541 if (m->fence) 1542 dma_fence_wait(m->fence, false); 1543 } 1544 1545 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1546 #include "tests/xe_migrate.c" 1547 #endif 1548