Lines Matching +full:1 +full:p
46 mips_xpa_disabled = 1; in xpa_disable()
48 return 1; in xpa_disable()
108 return 1; in use_bbit_insns()
119 return 1; in use_lwx_insns()
136 i += 1; /* Kernel use starts at the top and works down. */ in scratchpad_offset()
167 label_second_part = 1,
206 static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) in uasm_bgezl_hazard() argument
210 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); in uasm_bgezl_hazard()
217 static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) in uasm_bgezl_label() argument
221 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); in uasm_bgezl_label()
274 pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]); in dump_handler()
319 return -1; in allocate_kscratch()
323 kscratch_used_mask |= (1 << r); in allocate_kscratch()
333 static struct work_registers build_get_work_registers(u32 **p) in build_get_work_registers() argument
339 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); in build_get_work_registers()
346 if (num_possible_cpus() > 1) { in build_get_work_registers()
348 UASM_i_CPUID_MFC0(p, GPR_K0, SMP_CPUID_REG); in build_get_work_registers()
349 UASM_i_SRL_SAFE(p, GPR_K0, GPR_K0, SMP_CPUID_REGSHIFT); in build_get_work_registers()
352 UASM_i_SLL(p, GPR_K0, GPR_K0, ilog2(sizeof(struct tlb_reg_save))); in build_get_work_registers()
354 UASM_i_LA(p, GPR_K1, (long)&handler_reg_save); in build_get_work_registers()
355 UASM_i_ADDU(p, GPR_K0, GPR_K0, GPR_K1); in build_get_work_registers()
357 UASM_i_LA(p, GPR_K0, (long)&handler_reg_save); in build_get_work_registers()
359 /* GPR_K0 now points to save area, save $1 and $2 */ in build_get_work_registers()
360 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), GPR_K0); in build_get_work_registers()
361 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), GPR_K0); in build_get_work_registers()
364 r.r2 = 1; in build_get_work_registers()
369 static void build_restore_work_registers(u32 **p) in build_restore_work_registers() argument
372 uasm_i_ehb(p); in build_restore_work_registers()
373 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_restore_work_registers()
376 /* GPR_K0 already points to save area, restore $1 and $2 */ in build_restore_work_registers()
377 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), GPR_K0); in build_restore_work_registers()
378 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), GPR_K0); in build_restore_work_registers()
392 u32 *p; in build_r3000_tlb_refill_handler() local
395 p = tlb_handler; in build_r3000_tlb_refill_handler()
397 uasm_i_mfc0(&p, GPR_K0, C0_BADVADDR); in build_r3000_tlb_refill_handler()
398 uasm_i_lui(&p, GPR_K1, uasm_rel_hi(pgdc)); /* cp0 delay */ in build_r3000_tlb_refill_handler()
399 uasm_i_lw(&p, GPR_K1, uasm_rel_lo(pgdc), GPR_K1); in build_r3000_tlb_refill_handler()
400 uasm_i_srl(&p, GPR_K0, GPR_K0, 22); /* load delay */ in build_r3000_tlb_refill_handler()
401 uasm_i_sll(&p, GPR_K0, GPR_K0, 2); in build_r3000_tlb_refill_handler()
402 uasm_i_addu(&p, GPR_K1, GPR_K1, GPR_K0); in build_r3000_tlb_refill_handler()
403 uasm_i_mfc0(&p, GPR_K0, C0_CONTEXT); in build_r3000_tlb_refill_handler()
404 uasm_i_lw(&p, GPR_K1, 0, GPR_K1); /* cp0 delay */ in build_r3000_tlb_refill_handler()
405 uasm_i_andi(&p, GPR_K0, GPR_K0, 0xffc); /* load delay */ in build_r3000_tlb_refill_handler()
406 uasm_i_addu(&p, GPR_K1, GPR_K1, GPR_K0); in build_r3000_tlb_refill_handler()
407 uasm_i_lw(&p, GPR_K0, 0, GPR_K1); in build_r3000_tlb_refill_handler()
408 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_refill_handler()
409 uasm_i_mtc0(&p, GPR_K0, C0_ENTRYLO0); in build_r3000_tlb_refill_handler()
410 uasm_i_mfc0(&p, GPR_K1, C0_EPC); /* cp0 delay */ in build_r3000_tlb_refill_handler()
411 uasm_i_tlbwr(&p); /* cp0 delay */ in build_r3000_tlb_refill_handler()
412 uasm_i_jr(&p, GPR_K1); in build_r3000_tlb_refill_handler()
413 uasm_i_rfe(&p); /* branch delay */ in build_r3000_tlb_refill_handler()
415 if (p > tlb_handler + 32) in build_r3000_tlb_refill_handler()
419 (unsigned int)(p - tlb_handler)); in build_r3000_tlb_refill_handler()
458 static void __maybe_unused build_tlb_probe_entry(u32 **p) in build_tlb_probe_entry() argument
466 uasm_i_nop(p); in build_tlb_probe_entry()
467 uasm_i_tlbp(p); in build_tlb_probe_entry()
471 uasm_i_tlbp(p); in build_tlb_probe_entry()
476 void build_tlb_write_entry(u32 **p, struct uasm_label **l, in build_tlb_write_entry() argument
489 uasm_i_ehb(p); in build_tlb_write_entry()
490 tlbw(p); in build_tlb_write_entry()
505 uasm_bgezl_hazard(p, r, hazard_instance); in build_tlb_write_entry()
506 tlbw(p); in build_tlb_write_entry()
507 uasm_bgezl_label(l, p, hazard_instance); in build_tlb_write_entry()
509 uasm_i_nop(p); in build_tlb_write_entry()
514 uasm_i_nop(p); in build_tlb_write_entry()
515 tlbw(p); in build_tlb_write_entry()
516 uasm_i_nop(p); in build_tlb_write_entry()
521 uasm_i_nop(p); /* QED specifies 2 nops hazard */ in build_tlb_write_entry()
522 uasm_i_nop(p); /* QED specifies 2 nops hazard */ in build_tlb_write_entry()
523 tlbw(p); in build_tlb_write_entry()
530 uasm_i_nop(p); in build_tlb_write_entry()
531 tlbw(p); in build_tlb_write_entry()
556 uasm_i_nop(p); in build_tlb_write_entry()
559 tlbw(p); in build_tlb_write_entry()
563 uasm_i_nop(p); in build_tlb_write_entry()
564 uasm_i_nop(p); in build_tlb_write_entry()
565 uasm_i_nop(p); in build_tlb_write_entry()
566 uasm_i_nop(p); in build_tlb_write_entry()
567 tlbw(p); in build_tlb_write_entry()
571 tlbw(p); in build_tlb_write_entry()
572 uasm_i_nop(p); in build_tlb_write_entry()
583 static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, in build_convert_pte_to_entrylo() argument
593 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
595 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); in build_convert_pte_to_entrylo()
596 UASM_i_ROTR(p, reg, reg, in build_convert_pte_to_entrylo()
601 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
603 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
610 static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, in build_restore_pagemask() argument
620 uasm_i_ehb(p); in build_restore_pagemask()
624 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); in build_restore_pagemask()
625 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); in build_restore_pagemask()
626 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
627 uasm_il_b(p, r, lid); in build_restore_pagemask()
629 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); in build_restore_pagemask()
630 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
631 uasm_il_b(p, r, lid); in build_restore_pagemask()
633 uasm_i_mtc0(p, 0, C0_PAGEMASK); in build_restore_pagemask()
634 uasm_il_b(p, r, lid); in build_restore_pagemask()
637 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_restore_pagemask()
639 UASM_i_LW(p, 1, scratchpad_offset(0), 0); in build_restore_pagemask()
643 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); in build_restore_pagemask()
644 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); in build_restore_pagemask()
645 uasm_il_b(p, r, lid); in build_restore_pagemask()
646 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
648 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); in build_restore_pagemask()
649 uasm_il_b(p, r, lid); in build_restore_pagemask()
650 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
652 uasm_il_b(p, r, lid); in build_restore_pagemask()
653 uasm_i_mtc0(p, 0, C0_PAGEMASK); in build_restore_pagemask()
658 static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, in build_huge_tlb_write_entry() argument
665 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); in build_huge_tlb_write_entry()
666 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); in build_huge_tlb_write_entry()
667 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_huge_tlb_write_entry()
669 build_tlb_write_entry(p, l, r, wmode); in build_huge_tlb_write_entry()
671 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); in build_huge_tlb_write_entry()
678 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, in build_is_huge_pte() argument
681 UASM_i_LW(p, tmp, 0, pmd); in build_is_huge_pte()
683 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); in build_is_huge_pte()
685 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); in build_is_huge_pte()
686 uasm_il_bnez(p, r, tmp, lid); in build_is_huge_pte()
690 static void build_huge_update_entries(u32 **p, unsigned int pte, in build_huge_update_entries() argument
708 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); in build_huge_update_entries()
710 build_convert_pte_to_entrylo(p, pte); in build_huge_update_entries()
711 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ in build_huge_update_entries()
714 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); in build_huge_update_entries()
716 UASM_i_ADDU(p, pte, pte, tmp); in build_huge_update_entries()
718 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ in build_huge_update_entries()
721 static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, in build_huge_handler_tail() argument
728 UASM_i_SC(p, pte, 0, ptr); in build_huge_handler_tail()
729 uasm_il_beqz(p, r, pte, label_tlb_huge_update); in build_huge_handler_tail()
730 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ in build_huge_handler_tail()
732 UASM_i_SW(p, pte, 0, ptr); in build_huge_handler_tail()
737 UASM_i_MFC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
738 uasm_i_ori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); in build_huge_handler_tail()
739 UASM_i_MTC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
740 build_tlb_write_entry(p, l, r, tlb_indexed); in build_huge_handler_tail()
742 uasm_i_xori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); in build_huge_handler_tail()
743 UASM_i_MTC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
744 build_huge_update_entries(p, pte, ptr); in build_huge_handler_tail()
745 build_huge_tlb_write_entry(p, l, r, pte, tlb_random, 0); in build_huge_handler_tail()
750 build_huge_update_entries(p, pte, ptr); in build_huge_handler_tail()
751 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); in build_huge_handler_tail()
760 void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, in build_get_pmde64() argument
769 uasm_i_dmfc0(p, tmp, C0_BADVADDR); in build_get_pmde64()
783 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); in build_get_pmde64()
784 uasm_il_bnez(p, r, ptr, label_vmalloc); in build_get_pmde64()
786 uasm_il_bltz(p, r, tmp, label_vmalloc); in build_get_pmde64()
790 if (pgd_reg != -1) { in build_get_pmde64()
793 UASM_i_MFC0(p, ptr, C0_PWBASE); in build_get_pmde64()
795 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_get_pmde64()
801 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_get_pmde64()
804 uasm_i_dins(p, ptr, 0, 0, 23); in build_get_pmde64()
807 uasm_i_ori(p, ptr, ptr, ((u64)(CAC_BASE) >> 53)); in build_get_pmde64()
808 uasm_i_drotr(p, ptr, ptr, 11); in build_get_pmde64()
810 UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG); in build_get_pmde64()
811 uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT); in build_get_pmde64()
812 UASM_i_LA_mostly(p, tmp, pgdc); in build_get_pmde64()
813 uasm_i_daddu(p, ptr, ptr, tmp); in build_get_pmde64()
814 uasm_i_dmfc0(p, tmp, C0_BADVADDR); in build_get_pmde64()
815 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pmde64()
817 UASM_i_LA_mostly(p, ptr, pgdc); in build_get_pmde64()
818 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pmde64()
822 uasm_l_vmalloc_done(l, *p); in build_get_pmde64()
825 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); in build_get_pmde64()
827 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); in build_get_pmde64()
828 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ in build_get_pmde64()
830 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pmde64()
831 uasm_i_ld(p, ptr, 0, ptr); /* get pud pointer */ in build_get_pmde64()
832 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */ in build_get_pmde64()
833 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3); in build_get_pmde64()
834 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pud offset */ in build_get_pmde64()
837 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pmde64()
838 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ in build_get_pmde64()
839 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ in build_get_pmde64()
840 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); in build_get_pmde64()
841 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ in build_get_pmde64()
851 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, in build_get_pgd_vmalloc64() argument
861 uasm_l_vmalloc(l, *p); in build_get_pgd_vmalloc64()
865 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); in build_get_pgd_vmalloc64()
866 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); in build_get_pgd_vmalloc64()
867 did_vmalloc_branch = 1; in build_get_pgd_vmalloc64()
870 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); in build_get_pgd_vmalloc64()
875 uasm_il_b(p, r, label_vmalloc_done); in build_get_pgd_vmalloc64()
876 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); in build_get_pgd_vmalloc64()
878 UASM_i_LA_mostly(p, ptr, swpd); in build_get_pgd_vmalloc64()
879 uasm_il_b(p, r, label_vmalloc_done); in build_get_pgd_vmalloc64()
881 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); in build_get_pgd_vmalloc64()
883 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); in build_get_pgd_vmalloc64()
887 uasm_l_large_segbits_fault(l, *p); in build_get_pgd_vmalloc64()
890 uasm_i_ehb(p); in build_get_pgd_vmalloc64()
905 uasm_i_sync(p, 0); in build_get_pgd_vmalloc64()
906 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); in build_get_pgd_vmalloc64()
907 uasm_i_jr(p, ptr); in build_get_pgd_vmalloc64()
911 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_get_pgd_vmalloc64()
913 UASM_i_LW(p, 1, scratchpad_offset(0), 0); in build_get_pgd_vmalloc64()
915 uasm_i_nop(p); in build_get_pgd_vmalloc64()
926 void build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) in build_get_pgde32() argument
928 if (pgd_reg != -1) { in build_get_pgde32()
930 uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg); in build_get_pgde32()
931 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pgde32()
937 uasm_i_mfc0(p, ptr, SMP_CPUID_REG); in build_get_pgde32()
938 UASM_i_LA_mostly(p, tmp, pgdc); in build_get_pgde32()
939 uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT); in build_get_pgde32()
940 uasm_i_addu(p, ptr, tmp, ptr); in build_get_pgde32()
942 UASM_i_LA_mostly(p, ptr, pgdc); in build_get_pgde32()
944 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pgde32()
945 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pgde32()
947 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ in build_get_pgde32()
948 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); in build_get_pgde32()
949 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ in build_get_pgde32()
955 static void build_adjust_context(u32 **p, unsigned int ctx) in build_adjust_context() argument
957 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; in build_adjust_context()
958 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); in build_adjust_context()
961 UASM_i_SRL(p, ctx, ctx, shift); in build_adjust_context()
962 uasm_i_andi(p, ctx, ctx, mask); in build_adjust_context()
965 void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) in build_get_ptep() argument
976 UASM_i_LW(p, ptr, 0, ptr); in build_get_ptep()
977 GET_CONTEXT(p, tmp); /* get context reg */ in build_get_ptep()
981 GET_CONTEXT(p, tmp); /* get context reg */ in build_get_ptep()
982 UASM_i_LW(p, ptr, 0, ptr); in build_get_ptep()
986 build_adjust_context(p, tmp); in build_get_ptep()
987 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ in build_get_ptep()
991 void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) in build_update_entries() argument
1003 uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */ in build_update_entries()
1004 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1005 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); in build_update_entries()
1008 uasm_i_lw(p, tmp, 0, ptep); in build_update_entries()
1009 uasm_i_ext(p, tmp, tmp, 0, 24); in build_update_entries()
1010 uasm_i_mthc0(p, tmp, C0_ENTRYLO0); in build_update_entries()
1013 uasm_i_lw(p, tmp, pte_off_odd, ptep); /* odd pte */ in build_update_entries()
1014 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1015 UASM_i_MTC0(p, tmp, C0_ENTRYLO1); in build_update_entries()
1018 uasm_i_lw(p, tmp, sizeof(pte_t), ptep); in build_update_entries()
1019 uasm_i_ext(p, tmp, tmp, 0, 24); in build_update_entries()
1020 uasm_i_mthc0(p, tmp, C0_ENTRYLO1); in build_update_entries()
1025 UASM_i_LW(p, tmp, pte_off_even, ptep); /* get even pte */ in build_update_entries()
1026 UASM_i_LW(p, ptep, pte_off_odd, ptep); /* get odd pte */ in build_update_entries()
1028 build_tlb_probe_entry(p); in build_update_entries()
1029 build_convert_pte_to_entrylo(p, tmp); in build_update_entries()
1031 UASM_i_MTC0(p, 0, C0_ENTRYLO0); in build_update_entries()
1032 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ in build_update_entries()
1033 build_convert_pte_to_entrylo(p, ptep); in build_update_entries()
1035 uasm_i_mfc0(p, tmp, C0_INDEX); in build_update_entries()
1037 UASM_i_MTC0(p, 0, C0_ENTRYLO1); in build_update_entries()
1038 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ in build_update_entries()
1049 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, in build_fast_tlb_refill_handler() argument
1056 const int scratch = 1; /* Our extra working register */ in build_fast_tlb_refill_handler()
1063 UASM_i_MFC0(p, tmp, C0_BADVADDR); in build_fast_tlb_refill_handler()
1065 if (pgd_reg != -1) in build_fast_tlb_refill_handler()
1066 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_fast_tlb_refill_handler()
1068 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_fast_tlb_refill_handler()
1071 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1073 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1075 uasm_i_dsrl_safe(p, scratch, tmp, in build_fast_tlb_refill_handler()
1077 uasm_il_bnez(p, r, scratch, label_vmalloc); in build_fast_tlb_refill_handler()
1079 if (pgd_reg == -1) { in build_fast_tlb_refill_handler()
1080 vmalloc_branch_delay_filled = 1; in build_fast_tlb_refill_handler()
1082 uasm_i_dins(p, ptr, 0, 0, 23); in build_fast_tlb_refill_handler()
1085 if (pgd_reg != -1) in build_fast_tlb_refill_handler()
1086 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_fast_tlb_refill_handler()
1088 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_fast_tlb_refill_handler()
1090 UASM_i_MFC0(p, tmp, C0_BADVADDR); in build_fast_tlb_refill_handler()
1093 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1095 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1097 if (pgd_reg == -1) in build_fast_tlb_refill_handler()
1099 uasm_i_dins(p, ptr, 0, 0, 23); in build_fast_tlb_refill_handler()
1101 uasm_il_bltz(p, r, tmp, label_vmalloc); in build_fast_tlb_refill_handler()
1104 if (pgd_reg == -1) { in build_fast_tlb_refill_handler()
1105 vmalloc_branch_delay_filled = 1; in build_fast_tlb_refill_handler()
1107 uasm_i_ori(p, ptr, ptr, ((u64)(CAC_BASE) >> 53)); in build_fast_tlb_refill_handler()
1109 uasm_i_drotr(p, ptr, ptr, 11); in build_fast_tlb_refill_handler()
1120 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1122 uasm_l_vmalloc_done(l, *p); in build_fast_tlb_refill_handler()
1132 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1135 GET_CONTEXT(p, tmp); /* get context reg */ in build_fast_tlb_refill_handler()
1137 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); in build_fast_tlb_refill_handler()
1140 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); in build_fast_tlb_refill_handler()
1142 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ in build_fast_tlb_refill_handler()
1143 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ in build_fast_tlb_refill_handler()
1148 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3); in build_fast_tlb_refill_handler()
1149 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3); in build_fast_tlb_refill_handler()
1152 UASM_i_LWX(p, ptr, scratch, ptr); in build_fast_tlb_refill_handler()
1154 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ in build_fast_tlb_refill_handler()
1155 UASM_i_LW(p, ptr, 0, ptr); in build_fast_tlb_refill_handler()
1163 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); in build_fast_tlb_refill_handler()
1164 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); in build_fast_tlb_refill_handler()
1165 GET_CONTEXT(p, tmp); /* get context reg */ in build_fast_tlb_refill_handler()
1168 UASM_i_LWX(p, scratch, scratch, ptr); in build_fast_tlb_refill_handler()
1170 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ in build_fast_tlb_refill_handler()
1171 UASM_i_LW(p, scratch, 0, ptr); in build_fast_tlb_refill_handler()
1175 build_adjust_context(p, tmp); in build_fast_tlb_refill_handler()
1178 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); in build_fast_tlb_refill_handler()
1185 uasm_i_nop(p); in build_fast_tlb_refill_handler()
1193 UASM_i_LWX(p, even, scratch, tmp); in build_fast_tlb_refill_handler()
1194 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); in build_fast_tlb_refill_handler()
1195 UASM_i_LWX(p, odd, scratch, tmp); in build_fast_tlb_refill_handler()
1197 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ in build_fast_tlb_refill_handler()
1200 UASM_i_LW(p, even, 0, ptr); /* get even pte */ in build_fast_tlb_refill_handler()
1201 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ in build_fast_tlb_refill_handler()
1204 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1205 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ in build_fast_tlb_refill_handler()
1206 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1208 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1209 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ in build_fast_tlb_refill_handler()
1210 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1212 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ in build_fast_tlb_refill_handler()
1215 uasm_i_ehb(p); in build_fast_tlb_refill_handler()
1216 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1217 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1218 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1219 rv.restore_scratch = 1; in build_fast_tlb_refill_handler()
1221 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1222 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1223 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1225 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1226 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1227 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1228 rv.restore_scratch = 1; in build_fast_tlb_refill_handler()
1231 uasm_i_eret(p); /* return from trap */ in build_fast_tlb_refill_handler()
1246 u32 *p = tlb_handler; in build_r4000_tlb_refill_handler() local
1260 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, GPR_K0, GPR_K1, in build_r4000_tlb_refill_handler()
1274 uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR); in build_r4000_tlb_refill_handler()
1275 uasm_i_dmfc0(&p, GPR_K1, C0_ENTRYHI); in build_r4000_tlb_refill_handler()
1276 uasm_i_xor(&p, GPR_K0, GPR_K0, GPR_K1); in build_r4000_tlb_refill_handler()
1277 uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0, 62); in build_r4000_tlb_refill_handler()
1278 uasm_i_dsrl_safe(&p, GPR_K0, GPR_K0, 12 + 1); in build_r4000_tlb_refill_handler()
1279 uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_refill_handler()
1280 uasm_i_or(&p, GPR_K0, GPR_K0, GPR_K1); in build_r4000_tlb_refill_handler()
1281 uasm_il_bnez(&p, &r, GPR_K0, label_leave); in build_r4000_tlb_refill_handler()
1286 build_get_pmde64(&p, &l, &r, GPR_K0, GPR_K1); /* get pmd in GPR_K1 */ in build_r4000_tlb_refill_handler()
1288 build_get_pgde32(&p, GPR_K0, GPR_K1); /* get pgd in GPR_K1 */ in build_r4000_tlb_refill_handler()
1292 build_is_huge_pte(&p, &r, GPR_K0, GPR_K1, label_tlb_huge_update); in build_r4000_tlb_refill_handler()
1295 build_get_ptep(&p, GPR_K0, GPR_K1); in build_r4000_tlb_refill_handler()
1296 build_update_entries(&p, GPR_K0, GPR_K1); in build_r4000_tlb_refill_handler()
1297 build_tlb_write_entry(&p, &l, &r, tlb_random); in build_r4000_tlb_refill_handler()
1298 uasm_l_leave(&l, p); in build_r4000_tlb_refill_handler()
1299 uasm_i_eret(&p); /* return from trap */ in build_r4000_tlb_refill_handler()
1302 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_refill_handler()
1304 UASM_i_LW(&p, htlb_info.huge_pte, 0, GPR_K1); in build_r4000_tlb_refill_handler()
1305 build_huge_update_entries(&p, htlb_info.huge_pte, GPR_K1); in build_r4000_tlb_refill_handler()
1306 build_huge_tlb_write_entry(&p, &l, &r, GPR_K0, tlb_random, in build_r4000_tlb_refill_handler()
1311 build_get_pgd_vmalloc64(&p, &l, &r, GPR_K0, GPR_K1, vmalloc_mode); in build_r4000_tlb_refill_handler()
1327 if ((p - tlb_handler) > 64) in build_r4000_tlb_refill_handler()
1334 uasm_copy_handler(relocs, labels, tlb_handler, p, f); in build_r4000_tlb_refill_handler()
1335 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1338 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) in build_r4000_tlb_refill_handler()
1339 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) in build_r4000_tlb_refill_handler()
1347 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { in build_r4000_tlb_refill_handler()
1349 uasm_copy_handler(relocs, labels, tlb_handler, p, f); in build_r4000_tlb_refill_handler()
1350 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1370 split < p - MIPS64_REFILL_INSNS) in build_r4000_tlb_refill_handler()
1371 ov = 1; in build_r4000_tlb_refill_handler()
1386 if (uasm_insn_has_bdelay(relocs, split - 1)) in build_r4000_tlb_refill_handler()
1401 split, split + 1, f); in build_r4000_tlb_refill_handler()
1402 uasm_move_labels(labels, f, f + 1, -1); in build_r4000_tlb_refill_handler()
1409 uasm_copy_handler(relocs, labels, split, p, final_handler); in build_r4000_tlb_refill_handler()
1411 (p - split); in build_r4000_tlb_refill_handler()
1440 pgd_i = PGDIR_SHIFT; /* 1st level PGD */ in setup_pw()
1455 pwctl = 1 << 30; /* Set PWDirExt */ in setup_pw()
1459 write_c0_pwsize(1 << 30 | pgd_w << 24 | pmd_w << 12 | pt_w << 6 | pte_w); in setup_pw()
1462 write_c0_pwsize(1 << 30 | pgd_w << 24 | pt_w << 6 | pte_w); in setup_pw()
1466 pwctl |= (1 << 6 | psn); in setup_pw()
1470 kscratch_used_mask |= (1 << 7); /* KScratch6 is used for KPGD */ in setup_pw()
1475 u32 *p = tlb_handler; in build_loongson3_tlb_refill_handler() local
1484 uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR); in build_loongson3_tlb_refill_handler()
1485 uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0, in build_loongson3_tlb_refill_handler()
1487 uasm_il_beqz(&p, &r, GPR_K1, label_vmalloc); in build_loongson3_tlb_refill_handler()
1488 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1490 uasm_il_bgez(&p, &r, GPR_K0, label_large_segbits_fault); in build_loongson3_tlb_refill_handler()
1491 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1492 uasm_l_vmalloc(&l, p); in build_loongson3_tlb_refill_handler()
1495 uasm_i_dmfc0(&p, GPR_K1, C0_PGD); in build_loongson3_tlb_refill_handler()
1497 uasm_i_lddir(&p, GPR_K0, GPR_K1, 3); /* global page dir */ in build_loongson3_tlb_refill_handler()
1499 uasm_i_lddir(&p, GPR_K1, GPR_K0, 1); /* middle page dir */ in build_loongson3_tlb_refill_handler()
1501 uasm_i_ldpte(&p, GPR_K1, 0); /* even */ in build_loongson3_tlb_refill_handler()
1502 uasm_i_ldpte(&p, GPR_K1, 1); /* odd */ in build_loongson3_tlb_refill_handler()
1503 uasm_i_tlbwr(&p); in build_loongson3_tlb_refill_handler()
1507 uasm_i_lui(&p, GPR_K0, PM_DEFAULT_MASK >> 16); in build_loongson3_tlb_refill_handler()
1508 uasm_i_ori(&p, GPR_K0, GPR_K0, PM_DEFAULT_MASK & 0xffff); in build_loongson3_tlb_refill_handler()
1509 uasm_i_mtc0(&p, GPR_K0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1511 uasm_i_ori(&p, GPR_K0, 0, PM_DEFAULT_MASK); in build_loongson3_tlb_refill_handler()
1512 uasm_i_mtc0(&p, GPR_K0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1514 uasm_i_mtc0(&p, 0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1517 uasm_i_eret(&p); in build_loongson3_tlb_refill_handler()
1520 uasm_l_large_segbits_fault(&l, p); in build_loongson3_tlb_refill_handler()
1521 UASM_i_LA(&p, GPR_K1, (unsigned long)tlb_do_page_fault_0); in build_loongson3_tlb_refill_handler()
1522 uasm_i_jr(&p, GPR_K1); in build_loongson3_tlb_refill_handler()
1523 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1538 u32 *p = (u32 *)msk_isa16_mode((ulong)tlbmiss_handler_setup_pgd); in build_setup_pgd() local
1543 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p); in build_setup_pgd()
1548 if (pgd_reg == -1) { in build_setup_pgd()
1559 UASM_i_SRA(&p, a1, a0, 29); in build_setup_pgd()
1560 UASM_i_ADDIU(&p, a1, a1, 4); in build_setup_pgd()
1561 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); in build_setup_pgd()
1562 uasm_i_nop(&p); in build_setup_pgd()
1563 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); in build_setup_pgd()
1564 uasm_l_tlbl_goaround1(&l, p); in build_setup_pgd()
1565 UASM_i_SLL(&p, a0, a0, 11); in build_setup_pgd()
1566 UASM_i_MTC0(&p, a0, C0_CONTEXT); in build_setup_pgd()
1567 uasm_i_jr(&p, 31); in build_setup_pgd()
1568 uasm_i_ehb(&p); in build_setup_pgd()
1572 UASM_i_MTC0(&p, a0, C0_PWBASE); in build_setup_pgd()
1574 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); in build_setup_pgd()
1575 uasm_i_jr(&p, 31); in build_setup_pgd()
1576 uasm_i_ehb(&p); in build_setup_pgd()
1581 UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG); in build_setup_pgd()
1582 UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT); in build_setup_pgd()
1583 UASM_i_LA_mostly(&p, a2, pgdc); in build_setup_pgd()
1584 UASM_i_ADDU(&p, a2, a2, a1); in build_setup_pgd()
1585 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); in build_setup_pgd()
1587 UASM_i_LA_mostly(&p, a2, pgdc); in build_setup_pgd()
1588 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); in build_setup_pgd()
1592 if (pgd_reg != -1) { in build_setup_pgd()
1593 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); in build_setup_pgd()
1594 uasm_i_jr(&p, 31); in build_setup_pgd()
1595 uasm_i_ehb(&p); in build_setup_pgd()
1597 uasm_i_jr(&p, 31); in build_setup_pgd()
1598 uasm_i_nop(&p); in build_setup_pgd()
1601 if (p >= (u32 *)tlbmiss_handler_setup_pgd_end) in build_setup_pgd()
1606 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd)); in build_setup_pgd()
1613 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) in iPTE_LW() argument
1617 uasm_i_sync(p, 0); in iPTE_LW()
1620 uasm_i_lld(p, pte, 0, ptr); in iPTE_LW()
1623 UASM_i_LL(p, pte, 0, ptr); in iPTE_LW()
1627 uasm_i_ld(p, pte, 0, ptr); in iPTE_LW()
1630 UASM_i_LW(p, pte, 0, ptr); in iPTE_LW()
1635 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, in iPTE_SW() argument
1642 uasm_i_lui(p, scratch, swmode >> 16); in iPTE_SW()
1643 uasm_i_or(p, pte, pte, scratch); in iPTE_SW()
1646 uasm_i_ori(p, pte, pte, mode); in iPTE_SW()
1652 uasm_i_scd(p, pte, 0, ptr); in iPTE_SW()
1655 UASM_i_SC(p, pte, 0, ptr); in iPTE_SW()
1658 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1660 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1665 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1666 uasm_i_ori(p, pte, pte, hwmode); in iPTE_SW()
1668 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1669 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1671 uasm_i_lw(p, pte, 0, ptr); in iPTE_SW()
1673 uasm_i_nop(p); in iPTE_SW()
1675 uasm_i_nop(p); in iPTE_SW()
1680 uasm_i_sd(p, pte, 0, ptr); in iPTE_SW()
1683 UASM_i_SW(p, pte, 0, ptr); in iPTE_SW()
1687 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1688 uasm_i_ori(p, pte, pte, hwmode); in iPTE_SW()
1690 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1691 uasm_i_lw(p, pte, 0, ptr); in iPTE_SW()
1703 build_pte_present(u32 **p, struct uasm_reloc **r, in build_pte_present() argument
1711 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); in build_pte_present()
1712 uasm_i_nop(p); in build_pte_present()
1715 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_present()
1718 uasm_i_andi(p, t, cur, 1); in build_pte_present()
1719 uasm_il_beqz(p, r, t, lid); in build_pte_present()
1722 iPTE_LW(p, pte, ptr); in build_pte_present()
1726 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_present()
1729 uasm_i_andi(p, t, cur, in build_pte_present()
1731 uasm_i_xori(p, t, t, _PAGE_PRESENT >> _PAGE_PRESENT_SHIFT); in build_pte_present()
1732 uasm_il_bnez(p, r, t, lid); in build_pte_present()
1735 iPTE_LW(p, pte, ptr); in build_pte_present()
1741 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, in build_make_valid() argument
1746 iPTE_SW(p, r, pte, ptr, mode, scratch); in build_make_valid()
1754 build_pte_writable(u32 **p, struct uasm_reloc **r, in build_pte_writable() argument
1762 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_writable()
1765 uasm_i_andi(p, t, cur, in build_pte_writable()
1767 uasm_i_xori(p, t, t, in build_pte_writable()
1769 uasm_il_bnez(p, r, t, lid); in build_pte_writable()
1772 iPTE_LW(p, pte, ptr); in build_pte_writable()
1774 uasm_i_nop(p); in build_pte_writable()
1781 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, in build_make_write() argument
1787 iPTE_SW(p, r, pte, ptr, mode, scratch); in build_make_write()
1795 build_pte_modifiable(u32 **p, struct uasm_reloc **r, in build_pte_modifiable() argument
1800 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); in build_pte_modifiable()
1801 uasm_i_nop(p); in build_pte_modifiable()
1804 uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT); in build_pte_modifiable()
1805 uasm_i_andi(p, t, t, 1); in build_pte_modifiable()
1806 uasm_il_beqz(p, r, t, lid); in build_pte_modifiable()
1809 iPTE_LW(p, pte, ptr); in build_pte_modifiable()
1825 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) in build_r3000_pte_reload_tlbwi() argument
1827 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ in build_r3000_pte_reload_tlbwi()
1828 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ in build_r3000_pte_reload_tlbwi()
1829 uasm_i_tlbwi(p); in build_r3000_pte_reload_tlbwi()
1830 uasm_i_jr(p, tmp); in build_r3000_pte_reload_tlbwi()
1831 uasm_i_rfe(p); /* branch delay */ in build_r3000_pte_reload_tlbwi()
1841 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, in build_r3000_tlb_reload_write() argument
1845 uasm_i_mfc0(p, tmp, C0_INDEX); in build_r3000_tlb_reload_write()
1846 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ in build_r3000_tlb_reload_write()
1847 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ in build_r3000_tlb_reload_write()
1848 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ in build_r3000_tlb_reload_write()
1849 uasm_i_tlbwi(p); /* cp0 delay */ in build_r3000_tlb_reload_write()
1850 uasm_i_jr(p, tmp); in build_r3000_tlb_reload_write()
1851 uasm_i_rfe(p); /* branch delay */ in build_r3000_tlb_reload_write()
1852 uasm_l_r3000_write_probe_fail(l, *p); in build_r3000_tlb_reload_write()
1853 uasm_i_tlbwr(p); /* cp0 delay */ in build_r3000_tlb_reload_write()
1854 uasm_i_jr(p, tmp); in build_r3000_tlb_reload_write()
1855 uasm_i_rfe(p); /* branch delay */ in build_r3000_tlb_reload_write()
1859 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, in build_r3000_tlbchange_handler_head() argument
1864 uasm_i_mfc0(p, pte, C0_BADVADDR); in build_r3000_tlbchange_handler_head()
1865 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ in build_r3000_tlbchange_handler_head()
1866 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); in build_r3000_tlbchange_handler_head()
1867 uasm_i_srl(p, pte, pte, 22); /* load delay */ in build_r3000_tlbchange_handler_head()
1868 uasm_i_sll(p, pte, pte, 2); in build_r3000_tlbchange_handler_head()
1869 uasm_i_addu(p, ptr, ptr, pte); in build_r3000_tlbchange_handler_head()
1870 uasm_i_mfc0(p, pte, C0_CONTEXT); in build_r3000_tlbchange_handler_head()
1871 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ in build_r3000_tlbchange_handler_head()
1872 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ in build_r3000_tlbchange_handler_head()
1873 uasm_i_addu(p, ptr, ptr, pte); in build_r3000_tlbchange_handler_head()
1874 uasm_i_lw(p, pte, 0, ptr); in build_r3000_tlbchange_handler_head()
1875 uasm_i_tlbp(p); /* load delay */ in build_r3000_tlbchange_handler_head()
1880 u32 *p = (u32 *)handle_tlbl; in build_r3000_tlb_load_handler() local
1884 memset(p, 0, handle_tlbl_end - (char *)p); in build_r3000_tlb_load_handler()
1888 build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1); in build_r3000_tlb_load_handler()
1889 build_pte_present(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbl); in build_r3000_tlb_load_handler()
1890 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_load_handler()
1891 build_make_valid(&p, &r, GPR_K0, GPR_K1, -1); in build_r3000_tlb_load_handler()
1892 build_r3000_tlb_reload_write(&p, &l, &r, GPR_K0, GPR_K1); in build_r3000_tlb_load_handler()
1894 uasm_l_nopage_tlbl(&l, p); in build_r3000_tlb_load_handler()
1895 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); in build_r3000_tlb_load_handler()
1896 uasm_i_nop(&p); in build_r3000_tlb_load_handler()
1898 if (p >= (u32 *)handle_tlbl_end) in build_r3000_tlb_load_handler()
1903 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r3000_tlb_load_handler()
1910 u32 *p = (u32 *)handle_tlbs; in build_r3000_tlb_store_handler() local
1914 memset(p, 0, handle_tlbs_end - (char *)p); in build_r3000_tlb_store_handler()
1918 build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1); in build_r3000_tlb_store_handler()
1919 build_pte_writable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbs); in build_r3000_tlb_store_handler()
1920 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_store_handler()
1921 build_make_write(&p, &r, GPR_K0, GPR_K1, -1); in build_r3000_tlb_store_handler()
1922 build_r3000_tlb_reload_write(&p, &l, &r, GPR_K0, GPR_K1); in build_r3000_tlb_store_handler()
1924 uasm_l_nopage_tlbs(&l, p); in build_r3000_tlb_store_handler()
1925 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r3000_tlb_store_handler()
1926 uasm_i_nop(&p); in build_r3000_tlb_store_handler()
1928 if (p >= (u32 *)handle_tlbs_end) in build_r3000_tlb_store_handler()
1933 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r3000_tlb_store_handler()
1940 u32 *p = (u32 *)handle_tlbm; in build_r3000_tlb_modify_handler() local
1944 memset(p, 0, handle_tlbm_end - (char *)p); in build_r3000_tlb_modify_handler()
1948 build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1); in build_r3000_tlb_modify_handler()
1949 build_pte_modifiable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbm); in build_r3000_tlb_modify_handler()
1950 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_modify_handler()
1951 build_make_write(&p, &r, GPR_K0, GPR_K1, -1); in build_r3000_tlb_modify_handler()
1952 build_r3000_pte_reload_tlbwi(&p, GPR_K0, GPR_K1); in build_r3000_tlb_modify_handler()
1954 uasm_l_nopage_tlbm(&l, p); in build_r3000_tlb_modify_handler()
1955 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r3000_tlb_modify_handler()
1956 uasm_i_nop(&p); in build_r3000_tlb_modify_handler()
1958 if (p >= (u32 *)handle_tlbm_end) in build_r3000_tlb_modify_handler()
1963 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r3000_tlb_modify_handler()
1993 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, in build_r4000_tlbchange_handler_head() argument
1996 struct work_registers wr = build_get_work_registers(p); in build_r4000_tlbchange_handler_head()
1999 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ in build_r4000_tlbchange_handler_head()
2001 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ in build_r4000_tlbchange_handler_head()
2010 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); in build_r4000_tlbchange_handler_head()
2013 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); in build_r4000_tlbchange_handler_head()
2014 UASM_i_LW(p, wr.r2, 0, wr.r2); in build_r4000_tlbchange_handler_head()
2015 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT - PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2016 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2017 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); in build_r4000_tlbchange_handler_head()
2020 uasm_l_smp_pgtable_change(l, *p); in build_r4000_tlbchange_handler_head()
2022 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ in build_r4000_tlbchange_handler_head()
2024 build_tlb_probe_entry(p); in build_r4000_tlbchange_handler_head()
2027 uasm_i_ehb(p); in build_r4000_tlbchange_handler_head()
2028 uasm_i_mfc0(p, wr.r3, C0_INDEX); in build_r4000_tlbchange_handler_head()
2029 uasm_il_bltz(p, r, wr.r3, label_leave); in build_r4000_tlbchange_handler_head()
2030 uasm_i_nop(p); in build_r4000_tlbchange_handler_head()
2037 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, in build_r4000_tlbchange_handler_tail() argument
2041 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); in build_r4000_tlbchange_handler_tail()
2042 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); in build_r4000_tlbchange_handler_tail()
2043 build_update_entries(p, tmp, ptr); in build_r4000_tlbchange_handler_tail()
2044 build_tlb_write_entry(p, l, r, tlb_indexed); in build_r4000_tlbchange_handler_tail()
2045 uasm_l_leave(l, *p); in build_r4000_tlbchange_handler_tail()
2046 build_restore_work_registers(p); in build_r4000_tlbchange_handler_tail()
2047 uasm_i_eret(p); /* return from trap */ in build_r4000_tlbchange_handler_tail()
2050 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); in build_r4000_tlbchange_handler_tail()
2056 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl); in build_r4000_tlb_load_handler() local
2061 memset(p, 0, handle_tlbl_end - (char *)p); in build_r4000_tlb_load_handler()
2068 uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR); in build_r4000_tlb_load_handler()
2069 uasm_i_dmfc0(&p, GPR_K1, C0_ENTRYHI); in build_r4000_tlb_load_handler()
2070 uasm_i_xor(&p, GPR_K0, GPR_K0, GPR_K1); in build_r4000_tlb_load_handler()
2071 uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0, 62); in build_r4000_tlb_load_handler()
2072 uasm_i_dsrl_safe(&p, GPR_K0, GPR_K0, 12 + 1); in build_r4000_tlb_load_handler()
2073 uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_load_handler()
2074 uasm_i_or(&p, GPR_K0, GPR_K0, GPR_K1); in build_r4000_tlb_load_handler()
2075 uasm_il_bnez(&p, &r, GPR_K0, label_leave); in build_r4000_tlb_load_handler()
2079 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_load_handler()
2080 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2082 build_tlb_probe_entry(&p); in build_r4000_tlb_load_handler()
2090 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), in build_r4000_tlb_load_handler()
2093 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); in build_r4000_tlb_load_handler()
2094 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); in build_r4000_tlb_load_handler()
2096 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2106 uasm_i_tlbr(&p); in build_r4000_tlb_load_handler()
2109 uasm_i_ehb(&p); in build_r4000_tlb_load_handler()
2111 /* Examine entrylo 0 or 1 based on ptr. */ in build_r4000_tlb_load_handler()
2113 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); in build_r4000_tlb_load_handler()
2115 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); in build_r4000_tlb_load_handler()
2116 uasm_i_beqz(&p, wr.r3, 8); in build_r4000_tlb_load_handler()
2119 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); in build_r4000_tlb_load_handler()
2121 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); in build_r4000_tlb_load_handler()
2123 * If the entryLo (now in wr.r3) is valid (bit 1), RI or in build_r4000_tlb_load_handler()
2127 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2128 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2129 uasm_l_tlbl_goaround1(&l, p); in build_r4000_tlb_load_handler()
2131 uasm_i_andi(&p, wr.r3, wr.r3, 2); in build_r4000_tlb_load_handler()
2132 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2133 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2135 uasm_l_tlbl_goaround1(&l, p); in build_r4000_tlb_load_handler()
2137 build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_load_handler()
2138 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2145 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_load_handler()
2146 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2147 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2148 build_tlb_probe_entry(&p); in build_r4000_tlb_load_handler()
2156 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), in build_r4000_tlb_load_handler()
2159 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); in build_r4000_tlb_load_handler()
2160 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2162 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2172 uasm_i_tlbr(&p); in build_r4000_tlb_load_handler()
2175 uasm_i_ehb(&p); in build_r4000_tlb_load_handler()
2177 /* Examine entrylo 0 or 1 based on ptr. */ in build_r4000_tlb_load_handler()
2179 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); in build_r4000_tlb_load_handler()
2181 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); in build_r4000_tlb_load_handler()
2182 uasm_i_beqz(&p, wr.r3, 8); in build_r4000_tlb_load_handler()
2185 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); in build_r4000_tlb_load_handler()
2187 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); in build_r4000_tlb_load_handler()
2189 * If the entryLo (now in wr.r3) is valid (bit 1), RI or in build_r4000_tlb_load_handler()
2193 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2195 uasm_i_andi(&p, wr.r3, wr.r3, 2); in build_r4000_tlb_load_handler()
2196 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2199 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2204 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); in build_r4000_tlb_load_handler()
2206 uasm_l_tlbl_goaround2(&l, p); in build_r4000_tlb_load_handler()
2208 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); in build_r4000_tlb_load_handler()
2209 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); in build_r4000_tlb_load_handler()
2212 uasm_l_nopage_tlbl(&l, p); in build_r4000_tlb_load_handler()
2214 uasm_i_sync(&p, 0); in build_r4000_tlb_load_handler()
2215 build_restore_work_registers(&p); in build_r4000_tlb_load_handler()
2217 if ((unsigned long)tlb_do_page_fault_0 & 1) { in build_r4000_tlb_load_handler()
2218 uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_0)); in build_r4000_tlb_load_handler()
2219 uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_0)); in build_r4000_tlb_load_handler()
2220 uasm_i_jr(&p, GPR_K0); in build_r4000_tlb_load_handler()
2223 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); in build_r4000_tlb_load_handler()
2224 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2226 if (p >= (u32 *)handle_tlbl_end) in build_r4000_tlb_load_handler()
2231 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r4000_tlb_load_handler()
2238 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbs); in build_r4000_tlb_store_handler() local
2243 memset(p, 0, handle_tlbs_end - (char *)p); in build_r4000_tlb_store_handler()
2247 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_store_handler()
2248 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); in build_r4000_tlb_store_handler()
2250 build_tlb_probe_entry(&p); in build_r4000_tlb_store_handler()
2251 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_store_handler()
2252 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2259 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_store_handler()
2260 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2261 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); in build_r4000_tlb_store_handler()
2262 build_tlb_probe_entry(&p); in build_r4000_tlb_store_handler()
2263 uasm_i_ori(&p, wr.r1, wr.r1, in build_r4000_tlb_store_handler()
2265 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); in build_r4000_tlb_store_handler()
2268 uasm_l_nopage_tlbs(&l, p); in build_r4000_tlb_store_handler()
2270 uasm_i_sync(&p, 0); in build_r4000_tlb_store_handler()
2271 build_restore_work_registers(&p); in build_r4000_tlb_store_handler()
2273 if ((unsigned long)tlb_do_page_fault_1 & 1) { in build_r4000_tlb_store_handler()
2274 uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_1)); in build_r4000_tlb_store_handler()
2275 uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_1)); in build_r4000_tlb_store_handler()
2276 uasm_i_jr(&p, GPR_K0); in build_r4000_tlb_store_handler()
2279 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r4000_tlb_store_handler()
2280 uasm_i_nop(&p); in build_r4000_tlb_store_handler()
2282 if (p >= (u32 *)handle_tlbs_end) in build_r4000_tlb_store_handler()
2287 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r4000_tlb_store_handler()
2294 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbm); in build_r4000_tlb_modify_handler() local
2299 memset(p, 0, handle_tlbm_end - (char *)p); in build_r4000_tlb_modify_handler()
2303 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_modify_handler()
2304 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); in build_r4000_tlb_modify_handler()
2306 build_tlb_probe_entry(&p); in build_r4000_tlb_modify_handler()
2308 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_modify_handler()
2309 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2316 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_modify_handler()
2317 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2318 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); in build_r4000_tlb_modify_handler()
2319 build_tlb_probe_entry(&p); in build_r4000_tlb_modify_handler()
2320 uasm_i_ori(&p, wr.r1, wr.r1, in build_r4000_tlb_modify_handler()
2322 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 0); in build_r4000_tlb_modify_handler()
2325 uasm_l_nopage_tlbm(&l, p); in build_r4000_tlb_modify_handler()
2327 uasm_i_sync(&p, 0); in build_r4000_tlb_modify_handler()
2328 build_restore_work_registers(&p); in build_r4000_tlb_modify_handler()
2330 if ((unsigned long)tlb_do_page_fault_1 & 1) { in build_r4000_tlb_modify_handler()
2331 uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_1)); in build_r4000_tlb_modify_handler()
2332 uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_1)); in build_r4000_tlb_modify_handler()
2333 uasm_i_jr(&p, GPR_K0); in build_r4000_tlb_modify_handler()
2336 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r4000_tlb_modify_handler()
2337 uasm_i_nop(&p); in build_r4000_tlb_modify_handler()
2339 if (p >= (u32 *)handle_tlbm_end) in build_r4000_tlb_modify_handler()
2344 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r4000_tlb_modify_handler()
2463 config = 1 << MIPS_PWCTL_PWEN_SHIFT; in config_htw_params()
2513 entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1); in check_pabits()