mm/nohash/tlb_low_64e.S

1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  *  Low level TLB miss handlers for Book3E
5  *  Copyright (C) 2008-2009
15 #include <asm/asm-offsets.h>
17 #include <asm/exception-64e.h>
18 #include <asm/ppc-opcode.h>
21 #include <asm/feature-fixups.h>
36  * Note that, unlike non-bolted handlers, TLB_EXFRAME is not
95 	/* We pre-test some combination of permissions to avoid double
116 	rldicl.	r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
117 	bne-	dtlb_miss_fault_bolted	/* Bail if fault addr is invalid */
119 	rlwinm	r10,r11,32-19,27,27
120 	rlwimi	r10,r11,32-16,19,19
131 	beq-	tlb_miss_fault_bolted /* KUAP fault */
136  * This is the guts of the TLB miss handler for bolted-linear.
146 	rldicl	r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
153 	rldicl	r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
159 	rldicl	r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
165 	rldicl	r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3
173 	rldicr	r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
174 	bne-	tlb_miss_fault_bolted
180 	 *               - PID already updated by caller if necessary
181 	 *               - TSIZE need change if !base page size, not
188 	rlwimi	r11,r14,32-19,27,31	/* Insert WIMGE */
189 	rlwimi	r15,r14,32-8,22,25	/* Move in U bits */
192 	rlwimi	r15,r14,32-2,26,31	/* Move in BAP bits */
233 	rldicl.	r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
235 	bne-	itlb_miss_fault_bolted
253  * No HES or NV hint on TLB1, so we need to do software round-robin
255  *    with MAS-damage caused by tlbsx
292 	crmove	cr2*4+2,cr0*4+2		/* cr2.eq != 0 if kernel address */
303 	crclr	cr1*4+eq	/* set cr1.eq = 0 for non-recursive */
312 	beq	cr1,3b		/* unlock will happen if cr1.eq = 0 */
324 	 * Erratum A-008139 says that we can't use tlbwe to change
342 	rlwinm	r10,r10,0,0x80000fff  /* tgs,tlpid -> sgs,slpid */
347 	rlwinm	r15,r10,0,0x3fff0000  /* tid -> spid */
348 	rlwimi	r15,r10,20,0x00000003 /* ind,ts -> sind,sas */
382 	rldicl.	r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
383 	bne-	tlb_miss_fault_e6500
385 	rldicl	r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
388 	beq-	tlb_miss_fault_e6500 /* No PGDIR, bail */
391 	rldicl	r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
397 	rldicl	r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
409 	 * MAS 0   :	ESEL needs to be filled by software round-robin
411 	 *               - PID already updated by caller if necessary
412 	 *               - TSIZE for now is base ind page size always
413 	 *               - TID already cleared if necessary
414 	 * MAS 2   :	Default not 2M-aligned, need to be redone
421 	clrrdi	r15,r16,21		/* make EA 2M-aligned */
453 	rlwinm	r15,r14,32-_PAGE_PSIZE_SHIFT,0x1e
458 	 * MAS 0   :	ESEL needs to be filled by software round-robin
459 	 *		 - can be handled by indirect code
461 	 * MAS 2,3+7:	Needs to be redone similar to non-tablewalk handler
469 	li	r10,-0x400
472 	rldicr	r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
473 	rlwimi	r10,r14,32-19,27,31	/* Insert WIMGE */
475 	rlwimi	r15,r14,32-8,22,25	/* Move in U bits */
478 	rlwimi	r15,r14,32-2,26,31	/* Move in BAP bits */
510  * This is the guts of the second-level TLB miss handler for direct
521  * Note that this should only ever be called as a second level handler
524  * EX_TLB_DEAR-EX_TLB_SIZE(r12)
526  * It can be re-entered by the linear mapping miss handler. However, to
527  * avoid too much complication, it will restart the whole fault at level
540 	 * pgdir in the PACA :-).
555 	beq-	virt_page_table_tlb_miss_fault /* KUAP fault */
564 	rldicl	r10,r16,64-(VPTE_INDEX_SIZE+3),VPTE_INDEX_SIZE+3+4
566 	bne-	virt_page_table_tlb_miss_fault
571 	beq-	virt_page_table_tlb_miss_fault
574 	rldicl	r11,r16,64-VPTE_PGD_SHIFT,64-PGD_INDEX_SIZE-3
581 	rldicl	r11,r16,64-VPTE_PUD_SHIFT,64-PUD_INDEX_SIZE-3
588 	rldicl	r11,r16,64-VPTE_PMD_SHIFT,64-PMD_INDEX_SIZE-3
595 	 * a 4K or 64K page from r16 -> r15.
601 	 *               - PID already updated by caller if necessary
602 	 *               - TSIZE for now is base page size always
609 	ori	r10,r11,1		/* Or-in SR */
627 	 * always called as a second level tlb miss for SW load or as a first
628 	 * level TLB miss for HW load, so we should be able to peek at the
637 	 * level as well. Since we are doing that, we don't need to clear or
642 	bne-	virt_page_table_tlb_miss_whacko_fault
649 	cmpdi	cr0,r16,-1
666  * This is the guts of "any" level TLB miss handler for kernel linear
672  * r14 = ESR (data) or -1 (instruction)
678  * In addition we know that we will not re-enter, so in theory, we could
683  * handlers in which case we probably want to do a full restart at level
738 	cmpdi	cr0,r14,-1