1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ARM64_KVM_NESTED_H 3 #define __ARM64_KVM_NESTED_H 4 5 #include <linux/bitfield.h> 6 #include <linux/kvm_host.h> 7 #include <asm/kvm_emulate.h> 8 #include <asm/kvm_pgtable.h> 9 vcpu_has_nv(const struct kvm_vcpu * vcpu)10 static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) 11 { 12 return (!__is_defined(__KVM_NVHE_HYPERVISOR__) && 13 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && 14 vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2)); 15 } 16 17 /* Translation helpers from non-VHE EL2 to EL1 */ tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)18 static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2) 19 { 20 return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT; 21 } 22 translate_tcr_el2_to_tcr_el1(u64 tcr)23 static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr) 24 { 25 return TCR_EPD1_MASK | /* disable TTBR1_EL1 */ 26 ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) | 27 tcr_el2_ps_to_tcr_el1_ips(tcr) | 28 (tcr & TCR_EL2_TG0_MASK) | 29 (tcr & TCR_EL2_ORGN0_MASK) | 30 (tcr & TCR_EL2_IRGN0_MASK) | 31 (tcr & TCR_EL2_T0SZ_MASK); 32 } 33 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)34 static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2) 35 { 36 u64 cpacr_el1 = CPACR_ELx_RES1; 37 38 if (cptr_el2 & CPTR_EL2_TTA) 39 cpacr_el1 |= CPACR_ELx_TTA; 40 if (!(cptr_el2 & CPTR_EL2_TFP)) 41 cpacr_el1 |= CPACR_ELx_FPEN; 42 if (!(cptr_el2 & CPTR_EL2_TZ)) 43 cpacr_el1 |= CPACR_ELx_ZEN; 44 45 cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM); 46 47 return cpacr_el1; 48 } 49 translate_sctlr_el2_to_sctlr_el1(u64 val)50 static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val) 51 { 52 /* Only preserve the minimal set of bits we support */ 53 val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA | 54 SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE); 55 val |= SCTLR_EL1_RES1; 56 57 return val; 58 } 59 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)60 static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) 61 { 62 /* Clear the ASID field */ 63 return ttbr0 & ~GENMASK_ULL(63, 48); 64 } 65 66 extern bool forward_smc_trap(struct kvm_vcpu *vcpu); 67 extern void kvm_init_nested(struct kvm *kvm); 68 extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); 69 extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); 70 extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); 71 72 union tlbi_info; 73 74 extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 75 const union tlbi_info *info, 76 void (*)(struct kvm_s2_mmu *, 77 const union tlbi_info *)); 78 extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); 79 extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); 80 81 extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu); 82 83 struct kvm_s2_trans { 84 phys_addr_t output; 85 unsigned long block_size; 86 bool writable; 87 bool readable; 88 int level; 89 u32 esr; 90 u64 desc; 91 }; 92 kvm_s2_trans_output(struct kvm_s2_trans * trans)93 static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans) 94 { 95 return trans->output; 96 } 97 kvm_s2_trans_size(struct kvm_s2_trans * trans)98 static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans) 99 { 100 return trans->block_size; 101 } 102 kvm_s2_trans_esr(struct kvm_s2_trans * trans)103 static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans) 104 { 105 return trans->esr; 106 } 107 kvm_s2_trans_readable(struct kvm_s2_trans * trans)108 static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans) 109 { 110 return trans->readable; 111 } 112 kvm_s2_trans_writable(struct kvm_s2_trans * trans)113 static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans) 114 { 115 return trans->writable; 116 } 117 kvm_s2_trans_executable(struct kvm_s2_trans * trans)118 static inline bool kvm_s2_trans_executable(struct kvm_s2_trans *trans) 119 { 120 return !(trans->desc & BIT(54)); 121 } 122 123 extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 124 struct kvm_s2_trans *result); 125 extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, 126 struct kvm_s2_trans *trans); 127 extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2); 128 extern void kvm_nested_s2_wp(struct kvm *kvm); 129 extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block); 130 extern void kvm_nested_s2_flush(struct kvm *kvm); 131 132 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val); 133 kvm_supported_tlbi_s1e1_op(struct kvm_vcpu * vpcu,u32 instr)134 static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr) 135 { 136 struct kvm *kvm = vpcu->kvm; 137 u8 CRm = sys_reg_CRm(instr); 138 139 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 140 sys_reg_Op1(instr) == TLBI_Op1_EL1)) 141 return false; 142 143 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 144 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 145 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 146 return false; 147 148 if (CRm == TLBI_CRm_nROS && 149 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 150 return false; 151 152 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 153 CRm == TLBI_CRm_RNS) && 154 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 155 return false; 156 157 return true; 158 } 159 kvm_supported_tlbi_s1e2_op(struct kvm_vcpu * vpcu,u32 instr)160 static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr) 161 { 162 struct kvm *kvm = vpcu->kvm; 163 u8 CRm = sys_reg_CRm(instr); 164 165 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 166 sys_reg_Op1(instr) == TLBI_Op1_EL2)) 167 return false; 168 169 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 170 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 171 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 172 return false; 173 174 if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS) 175 return false; 176 177 if (CRm == TLBI_CRm_nROS && 178 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 179 return false; 180 181 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 182 CRm == TLBI_CRm_RNS) && 183 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 184 return false; 185 186 return true; 187 } 188 189 int kvm_init_nv_sysregs(struct kvm *kvm); 190 191 #ifdef CONFIG_ARM64_PTR_AUTH 192 bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr); 193 #else kvm_auth_eretax(struct kvm_vcpu * vcpu,u64 * elr)194 static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr) 195 { 196 /* We really should never execute this... */ 197 WARN_ON_ONCE(1); 198 *elr = 0xbad9acc0debadbad; 199 return false; 200 } 201 #endif 202 203 #define KVM_NV_GUEST_MAP_SZ (KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0) 204 kvm_encode_nested_level(struct kvm_s2_trans * trans)205 static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) 206 { 207 return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level); 208 } 209 210 /* Adjust alignment for the contiguous bit as per StageOA() */ 211 #define contiguous_bit_shift(d, wi, l) \ 212 ({ \ 213 u8 shift = 0; \ 214 \ 215 if ((d) & PTE_CONT) { \ 216 switch (BIT((wi)->pgshift)) { \ 217 case SZ_4K: \ 218 shift = 4; \ 219 break; \ 220 case SZ_16K: \ 221 shift = (l) == 2 ? 5 : 7; \ 222 break; \ 223 case SZ_64K: \ 224 shift = 5; \ 225 break; \ 226 } \ 227 } \ 228 \ 229 shift; \ 230 }) 231 ps_to_output_size(unsigned int ps)232 static inline unsigned int ps_to_output_size(unsigned int ps) 233 { 234 switch (ps) { 235 case 0: return 32; 236 case 1: return 36; 237 case 2: return 40; 238 case 3: return 42; 239 case 4: return 44; 240 case 5: 241 default: 242 return 48; 243 } 244 } 245 246 #endif /* __ARM64_KVM_NESTED_H */ 247