19 #include <linux/mm.h>
42 void load_mm_ldt(struct mm_struct *mm)  in load_mm_ldt()  argument
47 	ldt = READ_ONCE(mm->context.ldt);  in load_mm_ldt()
50 	 * Any change to mm->context.ldt is followed by an IPI to all  in load_mm_ldt()
51 	 * CPUs with the mm active.  The LDT will not be freed until  in load_mm_ldt()
93 	 * Load the LDT if either the old or new mm had an LDT.  in switch_ldt()
95 	 * An mm will never go from having an LDT to not having an LDT.  Two  in switch_ldt()
138 	struct mm_struct *mm = __mm;  in flush_ldt()  local
140 	if (this_cpu_read(cpu_tlbstate.loaded_mm) != mm)  in flush_ldt()
143 	load_mm_ldt(mm);  in flush_ldt()
189 static void do_sanity_check(struct mm_struct *mm,  in do_sanity_check()  argument
193 	if (mm->context.ldt) {  in do_sanity_check()
234 static void map_ldt_struct_to_user(struct mm_struct *mm)  in map_ldt_struct_to_user()  argument
236 	pgd_t *k_pgd = pgd_offset(mm, LDT_BASE_ADDR);  in map_ldt_struct_to_user()
243 	if (boot_cpu_has(X86_FEATURE_PTI) && !mm->context.ldt)  in map_ldt_struct_to_user()
247 static void sanity_check_ldt_mapping(struct mm_struct *mm)  in sanity_check_ldt_mapping()  argument
249 	pgd_t *k_pgd = pgd_offset(mm, LDT_BASE_ADDR);  in sanity_check_ldt_mapping()
259 	do_sanity_check(mm, had_kernel, had_user);  in sanity_check_ldt_mapping()
264 static void map_ldt_struct_to_user(struct mm_struct *mm)  in map_ldt_struct_to_user()  argument
266 	pgd_t *pgd = pgd_offset(mm, LDT_BASE_ADDR);  in map_ldt_struct_to_user()
268 	if (boot_cpu_has(X86_FEATURE_PTI) && !mm->context.ldt)  in map_ldt_struct_to_user()
272 static void sanity_check_ldt_mapping(struct mm_struct *mm)  in sanity_check_ldt_mapping()  argument
274 	pgd_t *pgd = pgd_offset(mm, LDT_BASE_ADDR);  in sanity_check_ldt_mapping()
278 	do_sanity_check(mm, had_kernel, had_user);  in sanity_check_ldt_mapping()
285  * usermode tables for the given mm.
288 map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot)  in map_ldt_struct()  argument
305 	sanity_check_ldt_mapping(mm);  in map_ldt_struct()
324 		 * and account for them in this mm.  in map_ldt_struct()
326 		ptep = get_locked_pte(mm, va, &ptl);  in map_ldt_struct()
338 		set_pte_at(mm, va, ptep, pte);  in map_ldt_struct()
343 	map_ldt_struct_to_user(mm);  in map_ldt_struct()
349 static void unmap_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt)  in unmap_ldt_struct()  argument
369 		ptep = get_locked_pte(mm, va, &ptl);  in unmap_ldt_struct()
371 			pte_clear(mm, va, ptep);  in unmap_ldt_struct()
377 	flush_tlb_mm_range(mm, va, va + nr_pages * PAGE_SIZE, PAGE_SHIFT, false);  in unmap_ldt_struct()
383 map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot)  in map_ldt_struct()  argument
388 static void unmap_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt)  in unmap_ldt_struct()  argument
393 static void free_ldt_pgtables(struct mm_struct *mm)  in free_ldt_pgtables()  argument
409 	tlb_gather_mmu_fullmm(&tlb, mm);  in free_ldt_pgtables()
421 static void install_ldt(struct mm_struct *mm, struct ldt_struct *ldt)  in install_ldt()  argument
423 	mutex_lock(&mm->context.lock);  in install_ldt()
426 	smp_store_release(&mm->context.ldt, ldt);  in install_ldt()
428 	/* Activate the LDT for all CPUs using currents mm. */  in install_ldt()
429 	on_each_cpu_mask(mm_cpumask(mm), flush_ldt, mm, true);  in install_ldt()
431 	mutex_unlock(&mm->context.lock);  in install_ldt()
451 int ldt_dup_context(struct mm_struct *old_mm, struct mm_struct *mm)  in ldt_dup_context()  argument
473 	retval = map_ldt_struct(mm, new_ldt, 0);  in ldt_dup_context()
475 		free_ldt_pgtables(mm);  in ldt_dup_context()
479 	mm->context.ldt = new_ldt;  in ldt_dup_context()
487  * No need to lock the MM as we are the last user
491 void destroy_context_ldt(struct mm_struct *mm)  in destroy_context_ldt()  argument
493 	free_ldt_struct(mm->context.ldt);  in destroy_context_ldt()
494 	mm->context.ldt = NULL;  in destroy_context_ldt()
497 void ldt_arch_exit_mmap(struct mm_struct *mm)  in ldt_arch_exit_mmap()  argument
499 	free_ldt_pgtables(mm);  in ldt_arch_exit_mmap()
504 	struct mm_struct *mm = current->mm;  in read_ldt()  local
508 	down_read(&mm->context.ldt_usr_sem);  in read_ldt()
510 	if (!mm->context.ldt) {  in read_ldt()
518 	entries_size = mm->context.ldt->nr_entries * LDT_ENTRY_SIZE;  in read_ldt()
522 	if (copy_to_user(ptr, mm->context.ldt->entries, entries_size)) {  in read_ldt()
537 	up_read(&mm->context.ldt_usr_sem);  in read_ldt()
580 	struct mm_struct *mm = current->mm;  in write_ldt()  local
619 	if (down_write_killable(&mm->context.ldt_usr_sem))  in write_ldt()
622 	old_ldt       = mm->context.ldt;  in write_ldt()
643 	error = map_ldt_struct(mm, new_ldt, old_ldt ? !old_ldt->slot : 0);  in write_ldt()
651 			free_ldt_pgtables(mm);  in write_ldt()
656 	install_ldt(mm, new_ldt);  in write_ldt()
657 	unmap_ldt_struct(mm, old_ldt);  in write_ldt()
662 	up_write(&mm->context.ldt_usr_sem);  in write_ldt()