Lines Matching +full:encode +full:- +full:only
1 // SPDX-License-Identifier: MIT
42 * For pre-gen12 platforms pat_index is the same as enum in gen8_pte_encode()
43 * i915_cache_level, so the switch-case here is still valid. in gen8_pte_encode()
90 struct drm_i915_private *i915 = ppgtt->vm.i915; in gen8_ppgtt_notify_vgt()
91 struct intel_uncore *uncore = ppgtt->vm.gt->uncore; in gen8_ppgtt_notify_vgt()
96 atomic_inc(px_used(ppgtt->pd)); /* never remove */ in gen8_ppgtt_notify_vgt()
98 atomic_dec(px_used(ppgtt->pd)); in gen8_ppgtt_notify_vgt()
100 mutex_lock(&i915->vgpu.lock); in gen8_ppgtt_notify_vgt()
102 if (i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_notify_vgt()
103 const u64 daddr = px_dma(ppgtt->pd); in gen8_ppgtt_notify_vgt()
133 mutex_unlock(&i915->vgpu.lock); in gen8_ppgtt_notify_vgt()
137 #define GEN8_PAGE_SIZE (SZ_4K) /* page and page-directory sizes are the same */
158 return GEN8_PDES - *idx; in gen8_pd_range()
160 return i915_pde_index(end, shift) - *idx; in gen8_pd_range()
175 return GEN8_PDES - (start & (GEN8_PDES - 1)); in gen8_pt_count()
177 return end - start; in gen8_pt_count()
182 unsigned int shift = __gen8_pte_shift(vm->top); in gen8_pd_top_count()
184 return (vm->total + (1ull << shift) - 1) >> shift; in gen8_pd_top_count()
192 if (vm->top == 2) in gen8_pdp_for_page_index()
193 return ppgtt->pd; in gen8_pdp_for_page_index()
195 return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top)); in gen8_pdp_for_page_index()
209 void **pde = pd->entry; in __gen8_ppgtt_cleanup()
215 __gen8_ppgtt_cleanup(vm, *pde, GEN8_PDES, lvl - 1); in __gen8_ppgtt_cleanup()
216 } while (pde++, --count); in __gen8_ppgtt_cleanup()
219 free_px(vm, &pd->pt, lvl); in __gen8_ppgtt_cleanup()
226 if (vm->rsvd.obj) in gen8_ppgtt_cleanup()
227 i915_gem_object_put(vm->rsvd.obj); in gen8_ppgtt_cleanup()
229 if (intel_vgpu_active(vm->i915)) in gen8_ppgtt_cleanup()
232 if (ppgtt->pd) in gen8_ppgtt_cleanup()
233 __gen8_ppgtt_cleanup(vm, ppgtt->pd, in gen8_ppgtt_cleanup()
234 gen8_pd_top_count(vm), vm->top); in gen8_ppgtt_cleanup()
243 const struct drm_i915_gem_object * const scratch = vm->scratch[lvl]; in __gen8_ppgtt_clear()
246 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_clear()
248 len = gen8_pd_range(start, end, lvl--, &idx); in __gen8_ppgtt_clear()
255 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_clear()
257 if (atomic_fetch_inc(&pt->used) >> gen8_pd_shift(1) && in __gen8_ppgtt_clear()
280 atomic_read(&pt->used)); in __gen8_ppgtt_clear()
281 GEM_BUG_ON(!count || count >= atomic_read(&pt->used)); in __gen8_ppgtt_clear()
284 if (pt->is_compact) { in __gen8_ppgtt_clear()
293 vm->scratch[0]->encode, in __gen8_ppgtt_clear()
296 atomic_sub(count, &pt->used); in __gen8_ppgtt_clear()
302 } while (idx++, --len); in __gen8_ppgtt_clear()
312 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_clear()
318 __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_clear()
319 start, start + length, vm->top); in gen8_ppgtt_clear()
329 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_alloc()
331 len = gen8_pd_range(*start, end, lvl--, &idx); in __gen8_ppgtt_alloc()
335 GEM_BUG_ON(!len || (idx + len - 1) >> gen8_pd_shift(1)); in __gen8_ppgtt_alloc()
337 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
340 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_alloc()
343 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
348 pt = stash->pt[!!lvl]; in __gen8_ppgtt_alloc()
349 __i915_gem_object_pin_pages(pt->base); in __gen8_ppgtt_alloc()
351 fill_px(pt, vm->scratch[lvl]->encode); in __gen8_ppgtt_alloc()
353 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
354 if (likely(!pd->entry[idx])) { in __gen8_ppgtt_alloc()
355 stash->pt[!!lvl] = pt->stash; in __gen8_ppgtt_alloc()
356 atomic_set(&pt->used, 0); in __gen8_ppgtt_alloc()
359 pt = pd->entry[idx]; in __gen8_ppgtt_alloc()
364 atomic_inc(&pt->used); in __gen8_ppgtt_alloc()
365 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
370 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
371 atomic_dec(&pt->used); in __gen8_ppgtt_alloc()
372 GEM_BUG_ON(!atomic_read(&pt->used)); in __gen8_ppgtt_alloc()
379 atomic_read(&pt->used)); in __gen8_ppgtt_alloc()
381 atomic_add(count, &pt->used); in __gen8_ppgtt_alloc()
383 GEM_BUG_ON(atomic_read(&pt->used) > NALLOC * I915_PDES); in __gen8_ppgtt_alloc()
386 } while (idx++, --len); in __gen8_ppgtt_alloc()
387 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
396 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_alloc()
402 __gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_alloc()
403 &start, start + length, vm->top); in gen8_ppgtt_alloc()
416 len = gen8_pd_range(*start, end, lvl--, &idx); in __gen8_ppgtt_foreach()
418 spin_lock(&pd->lock); in __gen8_ppgtt_foreach()
420 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_foreach()
422 atomic_inc(&pt->used); in __gen8_ppgtt_foreach()
423 spin_unlock(&pd->lock); in __gen8_ppgtt_foreach()
433 spin_lock(&pd->lock); in __gen8_ppgtt_foreach()
434 atomic_dec(&pt->used); in __gen8_ppgtt_foreach()
435 } while (idx++, --len); in __gen8_ppgtt_foreach()
436 spin_unlock(&pd->lock); in __gen8_ppgtt_foreach()
449 __gen8_ppgtt_foreach(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_foreach()
450 &start, start + length, vm->top, in gen8_ppgtt_foreach()
463 const gen8_pte_t pte_encode = ppgtt->vm.pte_encode(0, pat_index, flags); in gen8_ppgtt_insert_pte()
469 GEM_BUG_ON(sg_dma_len(iter->sg) < I915_GTT_PAGE_SIZE); in gen8_ppgtt_insert_pte()
470 vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma; in gen8_ppgtt_insert_pte()
472 iter->dma += I915_GTT_PAGE_SIZE; in gen8_ppgtt_insert_pte()
473 if (iter->dma >= iter->max) { in gen8_ppgtt_insert_pte()
474 iter->sg = __sg_next(iter->sg); in gen8_ppgtt_insert_pte()
475 if (!iter->sg || sg_dma_len(iter->sg) == 0) { in gen8_ppgtt_insert_pte()
480 iter->dma = sg_dma_address(iter->sg); in gen8_ppgtt_insert_pte()
481 iter->max = iter->dma + sg_dma_len(iter->sg); in gen8_ppgtt_insert_pte()
490 pd = pdp->entry[gen8_pd_index(idx, 2)]; in gen8_ppgtt_insert_pte()
509 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); in xehp_ppgtt_insert_huge()
510 unsigned int rem = sg_dma_len(iter->sg); in xehp_ppgtt_insert_huge()
511 u64 start = vma_res->start; in xehp_ppgtt_insert_huge()
512 u64 end = start + vma_res->vma_size; in xehp_ppgtt_insert_huge()
523 gen8_pte_t encode = pte_encode; in xehp_ppgtt_insert_huge() local
531 if (vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_2M && in xehp_ppgtt_insert_huge()
532 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && in xehp_ppgtt_insert_huge()
536 encode |= GEN8_PDE_PS_2M; in xehp_ppgtt_insert_huge()
544 if (vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { in xehp_ppgtt_insert_huge()
546 * Device local-memory on these platforms should in xehp_ppgtt_insert_huge()
549 * page-table needs to be filled we can always in xehp_ppgtt_insert_huge()
550 * safely use the compact-layout. Otherwise fall in xehp_ppgtt_insert_huge()
552 * system memory we only bother with PS64. in xehp_ppgtt_insert_huge()
554 if ((encode & GEN12_PPGTT_PTE_LM) && in xehp_ppgtt_insert_huge()
555 end - start >= SZ_2M && !index) { in xehp_ppgtt_insert_huge()
564 pt->is_compact = true; in xehp_ppgtt_insert_huge()
565 } else if (IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && in xehp_ppgtt_insert_huge()
568 encode |= GEN12_PTE_PS64; in xehp_ppgtt_insert_huge()
582 encode | (iter->dma + i * in xehp_ppgtt_insert_huge()
587 iter->dma += page_size; in xehp_ppgtt_insert_huge()
588 rem -= page_size; in xehp_ppgtt_insert_huge()
589 if (iter->dma >= iter->max) { in xehp_ppgtt_insert_huge()
590 iter->sg = __sg_next(iter->sg); in xehp_ppgtt_insert_huge()
591 if (!iter->sg) in xehp_ppgtt_insert_huge()
594 rem = sg_dma_len(iter->sg); in xehp_ppgtt_insert_huge()
598 iter->dma = sg_dma_address(iter->sg); in xehp_ppgtt_insert_huge()
599 iter->max = iter->dma + rem; in xehp_ppgtt_insert_huge()
601 if (unlikely(!IS_ALIGNED(iter->dma, page_size))) in xehp_ppgtt_insert_huge()
607 vma_res->page_sizes_gtt |= page_size; in xehp_ppgtt_insert_huge()
608 } while (iter->sg && sg_dma_len(iter->sg)); in xehp_ppgtt_insert_huge()
617 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); in gen8_ppgtt_insert_huge()
618 unsigned int rem = sg_dma_len(iter->sg); in gen8_ppgtt_insert_huge()
619 u64 start = vma_res->start; in gen8_ppgtt_insert_huge()
628 gen8_pte_t encode = pte_encode; in gen8_ppgtt_insert_huge() local
629 unsigned int maybe_64K = -1; in gen8_ppgtt_insert_huge()
634 if (vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_2M && in gen8_ppgtt_insert_huge()
635 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && in gen8_ppgtt_insert_huge()
639 encode |= GEN8_PDE_PS_2M; in gen8_ppgtt_insert_huge()
651 vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_64K && in gen8_ppgtt_insert_huge()
652 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && in gen8_ppgtt_insert_huge()
654 rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE)) in gen8_ppgtt_insert_huge()
661 GEM_BUG_ON(sg_dma_len(iter->sg) < page_size); in gen8_ppgtt_insert_huge()
662 vaddr[index++] = encode | iter->dma; in gen8_ppgtt_insert_huge()
665 iter->dma += page_size; in gen8_ppgtt_insert_huge()
666 rem -= page_size; in gen8_ppgtt_insert_huge()
667 if (iter->dma >= iter->max) { in gen8_ppgtt_insert_huge()
668 iter->sg = __sg_next(iter->sg); in gen8_ppgtt_insert_huge()
669 if (!iter->sg) in gen8_ppgtt_insert_huge()
672 rem = sg_dma_len(iter->sg); in gen8_ppgtt_insert_huge()
676 iter->dma = sg_dma_address(iter->sg); in gen8_ppgtt_insert_huge()
677 iter->max = iter->dma + rem; in gen8_ppgtt_insert_huge()
679 if (maybe_64K != -1 && index < I915_PDES && in gen8_ppgtt_insert_huge()
680 !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && in gen8_ppgtt_insert_huge()
682 rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE))) in gen8_ppgtt_insert_huge()
683 maybe_64K = -1; in gen8_ppgtt_insert_huge()
685 if (unlikely(!IS_ALIGNED(iter->dma, page_size))) in gen8_ppgtt_insert_huge()
693 * Is it safe to mark the 2M block as 64K? -- Either we have in gen8_ppgtt_insert_huge()
694 * filled whole page-table with 64K entries, or filled part of in gen8_ppgtt_insert_huge()
698 if (maybe_64K != -1 && in gen8_ppgtt_insert_huge()
701 !iter->sg && IS_ALIGNED(vma_res->start + in gen8_ppgtt_insert_huge()
702 vma_res->node_size, in gen8_ppgtt_insert_huge()
715 * instead - which we detect as missing results during in gen8_ppgtt_insert_huge()
718 if (I915_SELFTEST_ONLY(vm->scrub_64K)) { in gen8_ppgtt_insert_huge()
721 encode = vm->scratch[0]->encode; in gen8_ppgtt_insert_huge()
725 memset64(vaddr + i, encode, 15); in gen8_ppgtt_insert_huge()
731 vma_res->page_sizes_gtt |= page_size; in gen8_ppgtt_insert_huge()
732 } while (iter->sg && sg_dma_len(iter->sg)); in gen8_ppgtt_insert_huge()
743 if (vma_res->bi.page_sizes.sg > I915_GTT_PAGE_SIZE) { in gen8_ppgtt_insert()
744 if (GRAPHICS_VER_FULL(vm->i915) >= IP_VER(12, 55)) in gen8_ppgtt_insert()
749 u64 idx = vma_res->start >> GEN8_PTE_SHIFT; in gen8_ppgtt_insert()
759 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; in gen8_ppgtt_insert()
777 GEM_BUG_ON(pt->is_compact); in gen8_ppgtt_insert_entry()
780 vaddr[gen8_pd_index(idx, 0)] = vm->pte_encode(addr, pat_index, flags); in gen8_ppgtt_insert_entry()
803 if (!pt->is_compact) { in xehp_ppgtt_insert_entry_lm()
806 pt->is_compact = true; in xehp_ppgtt_insert_entry_lm()
810 vaddr[gen8_pd_index(idx, 0) / 16] = vm->pte_encode(addr, pat_index, flags); in xehp_ppgtt_insert_entry_lm()
836 if (vm->has_read_only && vm->gt->vm && !i915_is_ggtt(vm->gt->vm)) { in gen8_init_scratch()
837 struct i915_address_space *clone = vm->gt->vm; in gen8_init_scratch()
839 GEM_BUG_ON(!clone->has_read_only); in gen8_init_scratch()
841 vm->scratch_order = clone->scratch_order; in gen8_init_scratch()
842 for (i = 0; i <= vm->top; i++) in gen8_init_scratch()
843 vm->scratch[i] = i915_gem_object_get(clone->scratch[i]); in gen8_init_scratch()
852 pte_flags = vm->has_read_only; in gen8_init_scratch()
853 if (i915_gem_object_is_lmem(vm->scratch[0])) in gen8_init_scratch()
856 vm->scratch[0]->encode = in gen8_init_scratch()
857 vm->pte_encode(px_dma(vm->scratch[0]), in gen8_init_scratch()
858 i915_gem_get_pat_index(vm->i915, in gen8_init_scratch()
862 for (i = 1; i <= vm->top; i++) { in gen8_init_scratch()
865 obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_init_scratch()
877 fill_px(obj, vm->scratch[i - 1]->encode); in gen8_init_scratch()
878 obj->encode = gen8_pde_encode(px_dma(obj), I915_CACHE_NONE); in gen8_init_scratch()
880 vm->scratch[i] = obj; in gen8_init_scratch()
886 while (i--) in gen8_init_scratch()
887 i915_gem_object_put(vm->scratch[i]); in gen8_init_scratch()
888 vm->scratch[0] = NULL; in gen8_init_scratch()
894 struct i915_address_space *vm = &ppgtt->vm; in gen8_preallocate_top_level_pdp()
895 struct i915_page_directory *pd = ppgtt->pd; in gen8_preallocate_top_level_pdp()
898 GEM_BUG_ON(vm->top != 2); in gen8_preallocate_top_level_pdp()
909 err = map_pt_dma(vm, pde->pt.base); in gen8_preallocate_top_level_pdp()
915 fill_px(pde, vm->scratch[1]->encode); in gen8_preallocate_top_level_pdp()
935 return ERR_PTR(-ENOMEM); in gen8_alloc_top_pd()
937 pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_alloc_top_pd()
938 if (IS_ERR(pd->pt.base)) { in gen8_alloc_top_pd()
939 err = PTR_ERR(pd->pt.base); in gen8_alloc_top_pd()
940 pd->pt.base = NULL; in gen8_alloc_top_pd()
944 err = map_pt_dma(vm, pd->pt.base); in gen8_alloc_top_pd()
948 fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count); in gen8_alloc_top_pd()
959 struct drm_i915_private *i915 = vm->i915; in gen8_init_rsvd()
964 if (!intel_gt_needs_wa_16018031267(vm->gt)) in gen8_init_rsvd()
967 /* The memory will be used only by GPU. */ in gen8_init_rsvd()
986 vm->rsvd.vma = i915_vma_make_unshrinkable(vma); in gen8_init_rsvd()
987 vm->rsvd.obj = obj; in gen8_init_rsvd()
988 vm->total -= vma->node.size; in gen8_init_rsvd()
997 * with a net effect resembling a 2-level page table in normal x86 terms. Each
1011 return ERR_PTR(-ENOMEM); in gen8_ppgtt_create()
1014 ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2; in gen8_ppgtt_create()
1015 ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t)); in gen8_ppgtt_create()
1018 * From bdw, there is hw support for read-only pages in the PPGTT. in gen8_ppgtt_create()
1023 * Gen12 has inherited the same read-only fault issue from gen11. in gen8_ppgtt_create()
1025 ppgtt->vm.has_read_only = !IS_GRAPHICS_VER(gt->i915, 11, 12); in gen8_ppgtt_create()
1027 if (HAS_LMEM(gt->i915)) in gen8_ppgtt_create()
1028 ppgtt->vm.alloc_pt_dma = alloc_pt_lmem; in gen8_ppgtt_create()
1030 ppgtt->vm.alloc_pt_dma = alloc_pt_dma; in gen8_ppgtt_create()
1038 ppgtt->vm.alloc_scratch_dma = alloc_pt_dma; in gen8_ppgtt_create()
1040 if (GRAPHICS_VER(gt->i915) >= 12) in gen8_ppgtt_create()
1041 ppgtt->vm.pte_encode = gen12_pte_encode; in gen8_ppgtt_create()
1043 ppgtt->vm.pte_encode = gen8_pte_encode; in gen8_ppgtt_create()
1045 ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; in gen8_ppgtt_create()
1046 ppgtt->vm.insert_entries = gen8_ppgtt_insert; in gen8_ppgtt_create()
1047 if (HAS_64K_PAGES(gt->i915)) in gen8_ppgtt_create()
1048 ppgtt->vm.insert_page = xehp_ppgtt_insert_entry; in gen8_ppgtt_create()
1050 ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; in gen8_ppgtt_create()
1051 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; in gen8_ppgtt_create()
1052 ppgtt->vm.clear_range = gen8_ppgtt_clear; in gen8_ppgtt_create()
1053 ppgtt->vm.foreach = gen8_ppgtt_foreach; in gen8_ppgtt_create()
1054 ppgtt->vm.cleanup = gen8_ppgtt_cleanup; in gen8_ppgtt_create()
1056 err = gen8_init_scratch(&ppgtt->vm); in gen8_ppgtt_create()
1060 pd = gen8_alloc_top_pd(&ppgtt->vm); in gen8_ppgtt_create()
1065 ppgtt->pd = pd; in gen8_ppgtt_create()
1067 if (!i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_create()
1073 if (intel_vgpu_active(gt->i915)) in gen8_ppgtt_create()
1076 err = gen8_init_rsvd(&ppgtt->vm); in gen8_ppgtt_create()
1083 i915_vm_put(&ppgtt->vm); in gen8_ppgtt_create()