/linux-6.12.1/arch/arm64/mm/ |
D | contpte.c | 24 static inline pte_t *contpte_align_down(pte_t *ptep) in contpte_align_down() argument 26 return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * CONT_PTES); in contpte_align_down() 30 pte_t *ptep, unsigned int nr) in contpte_try_unfold_partial() argument 37 if (ptep != contpte_align_down(ptep) || nr < CONT_PTES) in contpte_try_unfold_partial() 38 contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); in contpte_try_unfold_partial() 40 if (ptep + nr != contpte_align_down(ptep + nr)) { in contpte_try_unfold_partial() 42 pte_t *last_ptep = ptep + nr - 1; in contpte_try_unfold_partial() 50 pte_t *ptep, pte_t pte) in contpte_convert() argument 57 start_ptep = ptep = contpte_align_down(ptep); in contpte_convert() 61 for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) { in contpte_convert() [all …]
|
D | hugetlbpage.c | 83 pte_t *ptep, size_t *pgsize) in find_num_contig() argument 94 if ((pte_t *)pmdp == ptep) { in find_num_contig() 130 pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in huge_ptep_get() argument 134 pte_t orig_pte = __ptep_get(ptep); in huge_ptep_get() 140 for (i = 0; i < ncontig; i++, ptep++) { in huge_ptep_get() 141 pte_t pte = __ptep_get(ptep); in huge_ptep_get() 162 pte_t *ptep, in get_clear_contig() argument 166 pte_t orig_pte = __ptep_get(ptep); in get_clear_contig() 169 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { in get_clear_contig() 170 pte_t pte = __ptep_get_and_clear(mm, addr, ptep); in get_clear_contig() [all …]
|
/linux-6.12.1/arch/s390/mm/ |
D | pgtable.c | 48 pte_t *ptep, int nodat) in ptep_ipte_local() argument 61 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); in ptep_ipte_local() 63 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); in ptep_ipte_local() 68 pte_t *ptep, int nodat) in ptep_ipte_global() argument 81 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); in ptep_ipte_global() 83 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); in ptep_ipte_global() 88 unsigned long addr, pte_t *ptep, in ptep_flush_direct() argument 93 old = *ptep; in ptep_flush_direct() 99 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct() 101 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct() [all …]
|
/linux-6.12.1/arch/riscv/mm/ |
D | hugetlbpage.c | 6 pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in huge_ptep_get() argument 10 pte_t orig_pte = ptep_get(ptep); in huge_ptep_get() 17 for (i = 0; i < pte_num; i++, ptep++) { in huge_ptep_get() 18 pte_t pte = ptep_get(ptep); in huge_ptep_get() 150 pte_t *ptep, in get_clear_contig() argument 153 pte_t orig_pte = ptep_get(ptep); in get_clear_contig() 156 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) { in get_clear_contig() 157 pte_t pte = ptep_get_and_clear(mm, addr, ptep); in get_clear_contig() 171 pte_t *ptep, in get_clear_contig_flush() argument 174 pte_t orig_pte = get_clear_contig(mm, addr, ptep, pte_num); in get_clear_contig_flush() [all …]
|
/linux-6.12.1/drivers/iommu/ |
D | io-pgtable-dart.c | 121 dart_iopte *ptep) in dart_init_pte() argument 128 if (ptep[i] & APPLE_DART_PTE_VALID) { in dart_init_pte() 130 WARN_ON(ptep[i] & APPLE_DART_PTE_VALID); in dart_init_pte() 142 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); in dart_init_pte() 148 dart_iopte *ptep, in dart_install_table() argument 163 old = cmpxchg64_relaxed(ptep, curr, new); in dart_install_table() 190 dart_iopte pte, *ptep; in dart_get_l2() local 193 ptep = data->pgd[tbl]; in dart_get_l2() 194 if (!ptep) in dart_get_l2() 197 ptep += dart_get_l1_index(data, iova); in dart_get_l2() [all …]
|
D | io-pgtable-arm.c | 150 #define iopte_set_writeable_clean(ptep) \ argument 151 set_bit(ARM_LPAE_PTE_AP_RDONLY_BIT, (unsigned long *)(ptep)) 270 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, in __arm_lpae_sync_pte() argument 273 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), in __arm_lpae_sync_pte() 274 sizeof(*ptep) * num_entries, DMA_TO_DEVICE); in __arm_lpae_sync_pte() 277 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries) in __arm_lpae_clear_pte() argument 280 ptep[i] = 0; in __arm_lpae_clear_pte() 283 __arm_lpae_sync_pte(ptep, num_entries, cfg); in __arm_lpae_clear_pte() 289 int lvl, arm_lpae_iopte *ptep); 293 int lvl, int num_entries, arm_lpae_iopte *ptep) in __arm_lpae_init_pte() argument [all …]
|
/linux-6.12.1/arch/s390/include/asm/ |
D | hugetlb.h | 19 pte_t *ptep, pte_t pte, unsigned long sz); 21 pte_t *ptep, pte_t pte); 22 pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 24 unsigned long addr, pte_t *ptep); 49 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument 51 if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) in huge_pte_clear() 52 set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY)); in huge_pte_clear() 54 set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY)); in huge_pte_clear() 58 unsigned long address, pte_t *ptep) in huge_ptep_clear_flush() argument 60 return huge_ptep_get_and_clear(vma->vm_mm, address, ptep); in huge_ptep_clear_flush() [all …]
|
/linux-6.12.1/arch/arm64/include/asm/ |
D | pgtable.h | 59 #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) argument 96 #define __pte_clear(mm, addr, ptep) \ argument 97 __set_pte(ptep, __pte(0)) 316 static inline void __set_pte_nosync(pte_t *ptep, pte_t pte) in __set_pte_nosync() argument 318 WRITE_ONCE(*ptep, pte); in __set_pte_nosync() 321 static inline void __set_pte(pte_t *ptep, pte_t pte) in __set_pte() argument 323 __set_pte_nosync(ptep, pte); in __set_pte() 335 static inline pte_t __ptep_get(pte_t *ptep) in __ptep_get() argument 337 return READ_ONCE(*ptep); in __ptep_get() 359 static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, in __check_safe_pte_update() argument [all …]
|
D | hugetlb.h | 31 pte_t *ptep, pte_t pte, unsigned long sz); 34 unsigned long addr, pte_t *ptep, 38 unsigned long addr, pte_t *ptep); 41 unsigned long addr, pte_t *ptep); 44 unsigned long addr, pte_t *ptep); 47 pte_t *ptep, unsigned long sz); 49 extern pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 55 unsigned long addr, pte_t *ptep); 59 unsigned long addr, pte_t *ptep,
|
/linux-6.12.1/arch/sparc/include/asm/ |
D | hugetlb.h | 17 pte_t *ptep, pte_t pte, unsigned long sz); 19 pte_t *ptep, pte_t pte); 23 pte_t *ptep); 27 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument 29 return *ptep; in huge_ptep_clear_flush() 34 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument 36 pte_t old_pte = *ptep; in huge_ptep_set_wrprotect() 37 __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); in huge_ptep_set_wrprotect() 42 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument 45 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags() [all …]
|
/linux-6.12.1/include/asm-generic/ |
D | hugetlb.h | 62 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument 64 pte_clear(mm, addr, ptep); in huge_pte_clear() 79 pte_t *ptep, pte_t pte, unsigned long sz) in set_huge_pte_at() argument 81 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at() 87 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument 89 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear() 95 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument 97 return ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush() 131 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument 133 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect() [all …]
|
/linux-6.12.1/include/linux/ |
D | pgtable.h | 230 static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) in pte_batch_hint() argument 265 pte_t *ptep, pte_t pte, unsigned int nr) in set_ptes() argument 267 page_table_check_ptes_set(mm, ptep, pte, nr); in set_ptes() 271 set_pte(ptep, pte); in set_ptes() 274 ptep++; in set_ptes() 280 #define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1) argument 284 unsigned long address, pte_t *ptep, 315 static inline pte_t ptep_get(pte_t *ptep) in ptep_get() argument 317 return READ_ONCE(*ptep); in ptep_get() 352 pte_t *ptep) in ptep_test_and_clear_young() argument [all …]
|
/linux-6.12.1/arch/powerpc/include/asm/nohash/32/ |
D | hugetlb-8xx.h | 19 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, 23 static inline pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in huge_ptep_get() argument 25 if (ptep_is_8m_pmdp(mm, addr, ptep)) in huge_ptep_get() 26 ptep = pte_offset_kernel((pmd_t *)ptep, ALIGN_DOWN(addr, SZ_8M)); in huge_ptep_get() 27 return ptep_get(ptep); in huge_ptep_get() 32 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument 34 pte_update(mm, addr, ptep, ~0UL, 0, 1); in huge_pte_clear() 39 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument 44 pte_update(mm, addr, ptep, clr, set, 1); in huge_ptep_set_wrprotect()
|
/linux-6.12.1/arch/xtensa/include/asm/ |
D | pgalloc.h | 24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument 25 (pmd_val(*(pmdp)) = ((unsigned long)ptep)) 35 static inline void ptes_clear(pte_t *ptep) in ptes_clear() argument 40 pte_clear(NULL, 0, ptep + i); in ptes_clear() 45 pte_t *ptep; in pte_alloc_one_kernel() local 47 ptep = (pte_t *)__pte_alloc_one_kernel(mm); in pte_alloc_one_kernel() 48 if (!ptep) in pte_alloc_one_kernel() 50 ptes_clear(ptep); in pte_alloc_one_kernel() 51 return ptep; in pte_alloc_one_kernel()
|
D | pgtable.h | 243 #define pte_clear(mm,addr,ptep) \ argument 244 do { update_pte(ptep, __pte(_PAGE_CA_INVALID | _PAGE_USER)); } while (0) 294 static inline void update_pte(pte_t *ptep, pte_t pteval) in update_pte() argument 296 *ptep = pteval; in update_pte() 298 __asm__ __volatile__ ("dhwb %0, 0" :: "a" (ptep)); in update_pte() 305 static inline void set_pte(pte_t *ptep, pte_t pte) in set_pte() argument 307 update_pte(ptep, pte); in set_pte() 320 pte_t *ptep) in ptep_test_and_clear_young() argument 322 pte_t pte = *ptep; in ptep_test_and_clear_young() 325 update_pte(ptep, pte_mkold(pte)); in ptep_test_and_clear_young() [all …]
|
/linux-6.12.1/arch/riscv/kvm/ |
D | mmu.c | 100 pte_t *ptep; in gstage_get_leaf_entry() local 104 ptep = (pte_t *)kvm->arch.pgd; in gstage_get_leaf_entry() 105 ptep = &ptep[gstage_pte_index(addr, current_level)]; in gstage_get_leaf_entry() 106 while (ptep && pte_val(ptep_get(ptep))) { in gstage_get_leaf_entry() 107 if (gstage_pte_leaf(ptep)) { in gstage_get_leaf_entry() 109 *ptepp = ptep; in gstage_get_leaf_entry() 116 ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); in gstage_get_leaf_entry() 117 ptep = &ptep[gstage_pte_index(addr, current_level)]; in gstage_get_leaf_entry() 119 ptep = NULL; in gstage_get_leaf_entry() 143 pte_t *ptep = &next_ptep[gstage_pte_index(addr, current_level)]; in gstage_set_pte() local [all …]
|
/linux-6.12.1/tools/testing/selftests/kvm/lib/aarch64/ |
D | processor.c | 132 uint64_t *ptep; in _virt_pg_map() local 148 ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8; in _virt_pg_map() 149 if (!*ptep) in _virt_pg_map() 150 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map() 154 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8; in _virt_pg_map() 155 if (!*ptep) in _virt_pg_map() 156 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map() 159 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8; in _virt_pg_map() 160 if (!*ptep) in _virt_pg_map() 161 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map() [all …]
|
/linux-6.12.1/arch/parisc/mm/ |
D | hugetlbpage.c | 123 pte_t *ptep, pte_t entry) in __set_huge_pte_at() argument 132 set_pte(ptep, entry); in __set_huge_pte_at() 133 ptep++; in __set_huge_pte_at() 143 pte_t *ptep, pte_t entry, unsigned long sz) in set_huge_pte_at() argument 145 __set_huge_pte_at(mm, addr, ptep, entry); in set_huge_pte_at() 150 pte_t *ptep) in huge_ptep_get_and_clear() argument 154 entry = *ptep; in huge_ptep_get_and_clear() 155 __set_huge_pte_at(mm, addr, ptep, __pte(0)); in huge_ptep_get_and_clear() 162 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument 166 old_pte = *ptep; in huge_ptep_set_wrprotect() [all …]
|
/linux-6.12.1/arch/powerpc/mm/ |
D | pgtable.c | 192 void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, in set_ptes() argument 215 VM_WARN_ON(pte_hw_valid(*ptep) && !pte_protnone(*ptep)); in set_ptes() 218 __set_pte_at(mm, addr, ptep, pte, 0); in set_ptes() 221 ptep++; in set_ptes() 230 pte_t *ptep = pte_offset_kernel(pmdp, va); in unmap_kernel_page() local 232 pte_clear(&init_mm, va, ptep); in unmap_kernel_page() 244 pte_t *ptep, pte_t entry, int dirty) in ptep_set_access_flags() argument 248 changed = !pte_same(*(ptep), entry); in ptep_set_access_flags() 251 __ptep_set_access_flags(vma, ptep, entry, in ptep_set_access_flags() 259 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument [all …]
|
D | pageattr.c | 19 static pte_basic_t pte_update_delta(pte_t *ptep, unsigned long addr, in pte_update_delta() argument 22 return pte_update(&init_mm, addr, ptep, old & ~new, new & ~old, 0); in pte_update_delta() 31 static int change_page_attr(pte_t *ptep, unsigned long addr, void *data) in change_page_attr() argument 40 pte_update_delta(ptep, addr, _PAGE_KERNEL_RW & ~_PAGE_DIRTY, _PAGE_KERNEL_RO); in change_page_attr() 44 pte_update_delta(ptep, addr, _PAGE_KERNEL_RW & ~_PAGE_DIRTY, _PAGE_KERNEL_ROX); in change_page_attr() 47 pte_update_delta(ptep, addr, _PAGE_KERNEL_RO, _PAGE_KERNEL_RW); in change_page_attr() 50 pte_update_delta(ptep, addr, _PAGE_KERNEL_ROX, _PAGE_KERNEL_RO); in change_page_attr() 53 pte_update_delta(ptep, addr, _PAGE_KERNEL_RO, _PAGE_KERNEL_ROX); in change_page_attr() 56 pte_update(&init_mm, addr, ptep, _PAGE_PRESENT, 0, 0); in change_page_attr() 59 pte_update(&init_mm, addr, ptep, 0, _PAGE_PRESENT, 0); in change_page_attr()
|
/linux-6.12.1/arch/powerpc/include/asm/book3s/64/ |
D | hash.h | 162 pte_t *ptep, unsigned long pte, int huge); 165 static inline unsigned long hash__pte_update_one(pte_t *ptep, unsigned long clr, in hash__pte_update_one() argument 178 : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) in hash__pte_update_one() 179 : "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep), in hash__pte_update_one() 188 pte_t *ptep, unsigned long clr, in hash__pte_update() argument 194 old = hash__pte_update_one(ptep, clr, set); in hash__pte_update() 210 hash__pte_update_one(ptep + i, clr, set); in hash__pte_update() 217 hpte_need_flush(mm, addr, ptep, old, huge); in hash__pte_update() 225 static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry) in hash__ptep_set_access_flags() argument 241 :"=&r" (old), "=&r" (tmp), "=m" (*ptep) in hash__ptep_set_access_flags() [all …]
|
/linux-6.12.1/arch/powerpc/mm/book3s64/ |
D | hugetlbpage.c | 21 pte_t *ptep, unsigned long trap, unsigned long flags, in __hash_page_huge() argument 49 old_pte = pte_val(*ptep); in __hash_page_huge() 74 } while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); in __hash_page_huge() 85 rpte = __real_pte(__pte(old_pte), ptep, offset); in __hash_page_huge() 121 *ptep = __pte(old_pte); in __hash_page_huge() 127 new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset); in __hash_page_huge() 133 *ptep = __pte(new_pte & ~H_PAGE_BUSY); in __hash_page_huge() 139 unsigned long addr, pte_t *ptep) in huge_ptep_modify_prot_start() argument 147 pte_val = pte_update(vma->vm_mm, addr, ptep, in huge_ptep_modify_prot_start() 154 pte_t *ptep, pte_t old_pte, pte_t pte) in huge_ptep_modify_prot_commit() argument [all …]
|
/linux-6.12.1/arch/powerpc/include/asm/nohash/ |
D | pgtable.h | 103 unsigned long addr, pte_t *ptep) in ptep_test_and_clear_young() argument 107 old = pte_update(vma->vm_mm, addr, ptep, _PAGE_ACCESSED, 0, 0); in ptep_test_and_clear_young() 115 pte_t *ptep) in ptep_set_wrprotect() argument 117 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0); in ptep_set_wrprotect() 123 pte_t *ptep) in ptep_get_and_clear() argument 125 return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 0)); in ptep_get_and_clear() 129 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in pte_clear() argument 131 pte_update(mm, addr, ptep, ~0UL, 0, 0); in pte_clear() 137 pte_t *ptep, pte_t entry, in __ptep_set_access_flags() argument 145 pte_update(vma->vm_mm, address, ptep, 0, set, huge); in __ptep_set_access_flags() [all …]
|
/linux-6.12.1/arch/mips/include/asm/ |
D | pgtable.h | 119 static inline void set_pte(pte_t *ptep, pte_t pte) argument 121 ptep->pte_high = pte.pte_high; 123 ptep->pte_low = pte.pte_low; 130 pte_t *buddy = ptep_buddy(ptep); 143 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) argument 150 if (ptep_buddy(ptep)->pte_high & _PAGE_GLOBAL) 153 if (ptep_buddy(ptep)->pte_low & _PAGE_GLOBAL) 157 set_pte(ptep, null); 171 static inline void set_pte(pte_t *ptep, pte_t pteval) argument 173 *ptep = pteval; [all …]
|
D | hugetlb.h | 35 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument 38 pte_t pte = *ptep; in huge_ptep_get_and_clear() 41 set_pte_at(mm, addr, ptep, clear); in huge_ptep_get_and_clear() 47 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument 56 pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_clear_flush() 71 pte_t *ptep, pte_t pte, in huge_ptep_set_access_flags() argument 74 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags() 77 set_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
|