1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Columbia University and Linaro Ltd. 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/bitfield.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 11 #include <asm/kvm_arm.h> 12 #include <asm/kvm_emulate.h> 13 #include <asm/kvm_mmu.h> 14 #include <asm/kvm_nested.h> 15 #include <asm/sysreg.h> 16 17 #include "sys_regs.h" 18 19 /* Protection against the sysreg repainting madness... */ 20 #define NV_FTR(r, f) ID_AA64##r##_EL1_##f 21 22 /* 23 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between 24 * memory usage and potential number of different sets of S2 PTs in 25 * the guests. Running out of S2 MMUs only affects performance (we 26 * will invalidate them more often). 27 */ 28 #define S2_MMU_PER_VCPU 2 29 kvm_init_nested(struct kvm * kvm)30 void kvm_init_nested(struct kvm *kvm) 31 { 32 kvm->arch.nested_mmus = NULL; 33 kvm->arch.nested_mmus_size = 0; 34 } 35 init_nested_s2_mmu(struct kvm * kvm,struct kvm_s2_mmu * mmu)36 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) 37 { 38 /* 39 * We only initialise the IPA range on the canonical MMU, which 40 * defines the contract between KVM and userspace on where the 41 * "hardware" is in the IPA space. This affects the validity of MMIO 42 * exits forwarded to userspace, for example. 43 * 44 * For nested S2s, we use the PARange as exposed to the guest, as it 45 * is allowed to use it at will to expose whatever memory map it 46 * wants to its own guests as it would be on real HW. 47 */ 48 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm)); 49 } 50 kvm_vcpu_init_nested(struct kvm_vcpu * vcpu)51 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) 52 { 53 struct kvm *kvm = vcpu->kvm; 54 struct kvm_s2_mmu *tmp; 55 int num_mmus, ret = 0; 56 57 /* 58 * Let's treat memory allocation failures as benign: If we fail to 59 * allocate anything, return an error and keep the allocated array 60 * alive. Userspace may try to recover by intializing the vcpu 61 * again, and there is no reason to affect the whole VM for this. 62 */ 63 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; 64 tmp = kvrealloc(kvm->arch.nested_mmus, 65 size_mul(sizeof(*kvm->arch.nested_mmus), num_mmus), 66 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 67 if (!tmp) 68 return -ENOMEM; 69 70 /* 71 * If we went through a realocation, adjust the MMU back-pointers in 72 * the previously initialised kvm_pgtable structures. 73 */ 74 if (kvm->arch.nested_mmus != tmp) 75 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) 76 tmp[i].pgt->mmu = &tmp[i]; 77 78 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) 79 ret = init_nested_s2_mmu(kvm, &tmp[i]); 80 81 if (ret) { 82 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) 83 kvm_free_stage2_pgd(&tmp[i]); 84 85 return ret; 86 } 87 88 kvm->arch.nested_mmus_size = num_mmus; 89 kvm->arch.nested_mmus = tmp; 90 91 return 0; 92 } 93 94 struct s2_walk_info { 95 int (*read_desc)(phys_addr_t pa, u64 *desc, void *data); 96 void *data; 97 u64 baddr; 98 unsigned int max_oa_bits; 99 unsigned int pgshift; 100 unsigned int sl; 101 unsigned int t0sz; 102 bool be; 103 }; 104 compute_fsc(int level,u32 fsc)105 static u32 compute_fsc(int level, u32 fsc) 106 { 107 return fsc | (level & 0x3); 108 } 109 esr_s2_fault(struct kvm_vcpu * vcpu,int level,u32 fsc)110 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc) 111 { 112 u32 esr; 113 114 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC; 115 esr |= compute_fsc(level, fsc); 116 return esr; 117 } 118 get_ia_size(struct s2_walk_info * wi)119 static int get_ia_size(struct s2_walk_info *wi) 120 { 121 return 64 - wi->t0sz; 122 } 123 check_base_s2_limits(struct s2_walk_info * wi,int level,int input_size,int stride)124 static int check_base_s2_limits(struct s2_walk_info *wi, 125 int level, int input_size, int stride) 126 { 127 int start_size, ia_size; 128 129 ia_size = get_ia_size(wi); 130 131 /* Check translation limits */ 132 switch (BIT(wi->pgshift)) { 133 case SZ_64K: 134 if (level == 0 || (level == 1 && ia_size <= 42)) 135 return -EFAULT; 136 break; 137 case SZ_16K: 138 if (level == 0 || (level == 1 && ia_size <= 40)) 139 return -EFAULT; 140 break; 141 case SZ_4K: 142 if (level < 0 || (level == 0 && ia_size <= 42)) 143 return -EFAULT; 144 break; 145 } 146 147 /* Check input size limits */ 148 if (input_size > ia_size) 149 return -EFAULT; 150 151 /* Check number of entries in starting level table */ 152 start_size = input_size - ((3 - level) * stride + wi->pgshift); 153 if (start_size < 1 || start_size > stride + 4) 154 return -EFAULT; 155 156 return 0; 157 } 158 159 /* Check if output is within boundaries */ check_output_size(struct s2_walk_info * wi,phys_addr_t output)160 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) 161 { 162 unsigned int output_size = wi->max_oa_bits; 163 164 if (output_size != 48 && (output & GENMASK_ULL(47, output_size))) 165 return -1; 166 167 return 0; 168 } 169 170 /* 171 * This is essentially a C-version of the pseudo code from the ARM ARM 172 * AArch64.TranslationTableWalk function. I strongly recommend looking at 173 * that pseudocode in trying to understand this. 174 * 175 * Must be called with the kvm->srcu read lock held 176 */ walk_nested_s2_pgd(phys_addr_t ipa,struct s2_walk_info * wi,struct kvm_s2_trans * out)177 static int walk_nested_s2_pgd(phys_addr_t ipa, 178 struct s2_walk_info *wi, struct kvm_s2_trans *out) 179 { 180 int first_block_level, level, stride, input_size, base_lower_bound; 181 phys_addr_t base_addr; 182 unsigned int addr_top, addr_bottom; 183 u64 desc; /* page table entry */ 184 int ret; 185 phys_addr_t paddr; 186 187 switch (BIT(wi->pgshift)) { 188 default: 189 case SZ_64K: 190 case SZ_16K: 191 level = 3 - wi->sl; 192 first_block_level = 2; 193 break; 194 case SZ_4K: 195 level = 2 - wi->sl; 196 first_block_level = 1; 197 break; 198 } 199 200 stride = wi->pgshift - 3; 201 input_size = get_ia_size(wi); 202 if (input_size > 48 || input_size < 25) 203 return -EFAULT; 204 205 ret = check_base_s2_limits(wi, level, input_size, stride); 206 if (WARN_ON(ret)) 207 return ret; 208 209 base_lower_bound = 3 + input_size - ((3 - level) * stride + 210 wi->pgshift); 211 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound); 212 213 if (check_output_size(wi, base_addr)) { 214 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 215 return 1; 216 } 217 218 addr_top = input_size - 1; 219 220 while (1) { 221 phys_addr_t index; 222 223 addr_bottom = (3 - level) * stride + wi->pgshift; 224 index = (ipa & GENMASK_ULL(addr_top, addr_bottom)) 225 >> (addr_bottom - 3); 226 227 paddr = base_addr | index; 228 ret = wi->read_desc(paddr, &desc, wi->data); 229 if (ret < 0) 230 return ret; 231 232 /* 233 * Handle reversedescriptors if endianness differs between the 234 * host and the guest hypervisor. 235 */ 236 if (wi->be) 237 desc = be64_to_cpu((__force __be64)desc); 238 else 239 desc = le64_to_cpu((__force __le64)desc); 240 241 /* Check for valid descriptor at this point */ 242 if (!(desc & 1) || ((desc & 3) == 1 && level == 3)) { 243 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 244 out->desc = desc; 245 return 1; 246 } 247 248 /* We're at the final level or block translation level */ 249 if ((desc & 3) == 1 || level == 3) 250 break; 251 252 if (check_output_size(wi, desc)) { 253 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 254 out->desc = desc; 255 return 1; 256 } 257 258 base_addr = desc & GENMASK_ULL(47, wi->pgshift); 259 260 level += 1; 261 addr_top = addr_bottom - 1; 262 } 263 264 if (level < first_block_level) { 265 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 266 out->desc = desc; 267 return 1; 268 } 269 270 if (check_output_size(wi, desc)) { 271 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 272 out->desc = desc; 273 return 1; 274 } 275 276 if (!(desc & BIT(10))) { 277 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS); 278 out->desc = desc; 279 return 1; 280 } 281 282 addr_bottom += contiguous_bit_shift(desc, wi, level); 283 284 /* Calculate and return the result */ 285 paddr = (desc & GENMASK_ULL(47, addr_bottom)) | 286 (ipa & GENMASK_ULL(addr_bottom - 1, 0)); 287 out->output = paddr; 288 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift); 289 out->readable = desc & (0b01 << 6); 290 out->writable = desc & (0b10 << 6); 291 out->level = level; 292 out->desc = desc; 293 return 0; 294 } 295 read_guest_s2_desc(phys_addr_t pa,u64 * desc,void * data)296 static int read_guest_s2_desc(phys_addr_t pa, u64 *desc, void *data) 297 { 298 struct kvm_vcpu *vcpu = data; 299 300 return kvm_read_guest(vcpu->kvm, pa, desc, sizeof(*desc)); 301 } 302 vtcr_to_walk_info(u64 vtcr,struct s2_walk_info * wi)303 static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi) 304 { 305 wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK; 306 307 switch (vtcr & VTCR_EL2_TG0_MASK) { 308 case VTCR_EL2_TG0_4K: 309 wi->pgshift = 12; break; 310 case VTCR_EL2_TG0_16K: 311 wi->pgshift = 14; break; 312 case VTCR_EL2_TG0_64K: 313 default: /* IMPDEF: treat any other value as 64k */ 314 wi->pgshift = 16; break; 315 } 316 317 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 318 /* Global limit for now, should eventually be per-VM */ 319 wi->max_oa_bits = min(get_kvm_ipa_limit(), 320 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr))); 321 } 322 kvm_walk_nested_s2(struct kvm_vcpu * vcpu,phys_addr_t gipa,struct kvm_s2_trans * result)323 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 324 struct kvm_s2_trans *result) 325 { 326 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 327 struct s2_walk_info wi; 328 int ret; 329 330 result->esr = 0; 331 332 if (!vcpu_has_nv(vcpu)) 333 return 0; 334 335 wi.read_desc = read_guest_s2_desc; 336 wi.data = vcpu; 337 wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 338 339 vtcr_to_walk_info(vtcr, &wi); 340 341 wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE; 342 343 ret = walk_nested_s2_pgd(gipa, &wi, result); 344 if (ret) 345 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC); 346 347 return ret; 348 } 349 ttl_to_size(u8 ttl)350 static unsigned int ttl_to_size(u8 ttl) 351 { 352 int level = ttl & 3; 353 int gran = (ttl >> 2) & 3; 354 unsigned int max_size = 0; 355 356 switch (gran) { 357 case TLBI_TTL_TG_4K: 358 switch (level) { 359 case 0: 360 break; 361 case 1: 362 max_size = SZ_1G; 363 break; 364 case 2: 365 max_size = SZ_2M; 366 break; 367 case 3: 368 max_size = SZ_4K; 369 break; 370 } 371 break; 372 case TLBI_TTL_TG_16K: 373 switch (level) { 374 case 0: 375 case 1: 376 break; 377 case 2: 378 max_size = SZ_32M; 379 break; 380 case 3: 381 max_size = SZ_16K; 382 break; 383 } 384 break; 385 case TLBI_TTL_TG_64K: 386 switch (level) { 387 case 0: 388 case 1: 389 /* No 52bit IPA support */ 390 break; 391 case 2: 392 max_size = SZ_512M; 393 break; 394 case 3: 395 max_size = SZ_64K; 396 break; 397 } 398 break; 399 default: /* No size information */ 400 break; 401 } 402 403 return max_size; 404 } 405 406 /* 407 * Compute the equivalent of the TTL field by parsing the shadow PT. The 408 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is 409 * retrieved from first entry carrying the level as a tag. 410 */ get_guest_mapping_ttl(struct kvm_s2_mmu * mmu,u64 addr)411 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) 412 { 413 u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr; 414 kvm_pte_t pte; 415 u8 ttl, level; 416 417 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock); 418 419 switch (vtcr & VTCR_EL2_TG0_MASK) { 420 case VTCR_EL2_TG0_4K: 421 ttl = (TLBI_TTL_TG_4K << 2); 422 break; 423 case VTCR_EL2_TG0_16K: 424 ttl = (TLBI_TTL_TG_16K << 2); 425 break; 426 case VTCR_EL2_TG0_64K: 427 default: /* IMPDEF: treat any other value as 64k */ 428 ttl = (TLBI_TTL_TG_64K << 2); 429 break; 430 } 431 432 tmp = addr; 433 434 again: 435 /* Iteratively compute the block sizes for a particular granule size */ 436 switch (vtcr & VTCR_EL2_TG0_MASK) { 437 case VTCR_EL2_TG0_4K: 438 if (sz < SZ_4K) sz = SZ_4K; 439 else if (sz < SZ_2M) sz = SZ_2M; 440 else if (sz < SZ_1G) sz = SZ_1G; 441 else sz = 0; 442 break; 443 case VTCR_EL2_TG0_16K: 444 if (sz < SZ_16K) sz = SZ_16K; 445 else if (sz < SZ_32M) sz = SZ_32M; 446 else sz = 0; 447 break; 448 case VTCR_EL2_TG0_64K: 449 default: /* IMPDEF: treat any other value as 64k */ 450 if (sz < SZ_64K) sz = SZ_64K; 451 else if (sz < SZ_512M) sz = SZ_512M; 452 else sz = 0; 453 break; 454 } 455 456 if (sz == 0) 457 return 0; 458 459 tmp &= ~(sz - 1); 460 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) 461 goto again; 462 if (!(pte & PTE_VALID)) 463 goto again; 464 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte); 465 if (!level) 466 goto again; 467 468 ttl |= level; 469 470 /* 471 * We now have found some level information in the shadow S2. Check 472 * that the resulting range is actually including the original IPA. 473 */ 474 sz = ttl_to_size(ttl); 475 if (addr < (tmp + sz)) 476 return ttl; 477 478 return 0; 479 } 480 compute_tlb_inval_range(struct kvm_s2_mmu * mmu,u64 val)481 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) 482 { 483 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 484 unsigned long max_size; 485 u8 ttl; 486 487 ttl = FIELD_GET(TLBI_TTL_MASK, val); 488 489 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) { 490 /* No TTL, check the shadow S2 for a hint */ 491 u64 addr = (val & GENMASK_ULL(35, 0)) << 12; 492 ttl = get_guest_mapping_ttl(mmu, addr); 493 } 494 495 max_size = ttl_to_size(ttl); 496 497 if (!max_size) { 498 /* Compute the maximum extent of the invalidation */ 499 switch (mmu->tlb_vtcr & VTCR_EL2_TG0_MASK) { 500 case VTCR_EL2_TG0_4K: 501 max_size = SZ_1G; 502 break; 503 case VTCR_EL2_TG0_16K: 504 max_size = SZ_32M; 505 break; 506 case VTCR_EL2_TG0_64K: 507 default: /* IMPDEF: treat any other value as 64k */ 508 /* 509 * No, we do not support 52bit IPA in nested yet. Once 510 * we do, this should be 4TB. 511 */ 512 max_size = SZ_512M; 513 break; 514 } 515 } 516 517 WARN_ON(!max_size); 518 return max_size; 519 } 520 521 /* 522 * We can have multiple *different* MMU contexts with the same VMID: 523 * 524 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit 525 * 526 * - Multiple vcpus using private S2s (huh huh...), hence differing by the 527 * VBBTR_EL2.BADDR address 528 * 529 * - A combination of the above... 530 * 531 * We can always identify which MMU context to pick at run-time. However, 532 * TLB invalidation involving a VMID must take action on all the TLBs using 533 * this particular VMID. This translates into applying the same invalidation 534 * operation to all the contexts that are using this VMID. Moar phun! 535 */ kvm_s2_mmu_iterate_by_vmid(struct kvm * kvm,u16 vmid,const union tlbi_info * info,void (* tlbi_callback)(struct kvm_s2_mmu *,const union tlbi_info *))536 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 537 const union tlbi_info *info, 538 void (*tlbi_callback)(struct kvm_s2_mmu *, 539 const union tlbi_info *)) 540 { 541 write_lock(&kvm->mmu_lock); 542 543 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 544 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 545 546 if (!kvm_s2_mmu_valid(mmu)) 547 continue; 548 549 if (vmid == get_vmid(mmu->tlb_vttbr)) 550 tlbi_callback(mmu, info); 551 } 552 553 write_unlock(&kvm->mmu_lock); 554 } 555 lookup_s2_mmu(struct kvm_vcpu * vcpu)556 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) 557 { 558 struct kvm *kvm = vcpu->kvm; 559 bool nested_stage2_enabled; 560 u64 vttbr, vtcr, hcr; 561 562 lockdep_assert_held_write(&kvm->mmu_lock); 563 564 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 565 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 566 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); 567 568 nested_stage2_enabled = hcr & HCR_VM; 569 570 /* Don't consider the CnP bit for the vttbr match */ 571 vttbr &= ~VTTBR_CNP_BIT; 572 573 /* 574 * Two possibilities when looking up a S2 MMU context: 575 * 576 * - either S2 is enabled in the guest, and we need a context that is 577 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR, 578 * which makes it safe from a TLB conflict perspective (a broken 579 * guest won't be able to generate them), 580 * 581 * - or S2 is disabled, and we need a context that is S2-disabled 582 * and matches the VMID only, as all TLBs are tagged by VMID even 583 * if S2 translation is disabled. 584 */ 585 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 586 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 587 588 if (!kvm_s2_mmu_valid(mmu)) 589 continue; 590 591 if (nested_stage2_enabled && 592 mmu->nested_stage2_enabled && 593 vttbr == mmu->tlb_vttbr && 594 vtcr == mmu->tlb_vtcr) 595 return mmu; 596 597 if (!nested_stage2_enabled && 598 !mmu->nested_stage2_enabled && 599 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr)) 600 return mmu; 601 } 602 return NULL; 603 } 604 get_s2_mmu_nested(struct kvm_vcpu * vcpu)605 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) 606 { 607 struct kvm *kvm = vcpu->kvm; 608 struct kvm_s2_mmu *s2_mmu; 609 int i; 610 611 lockdep_assert_held_write(&vcpu->kvm->mmu_lock); 612 613 s2_mmu = lookup_s2_mmu(vcpu); 614 if (s2_mmu) 615 goto out; 616 617 /* 618 * Make sure we don't always search from the same point, or we 619 * will always reuse a potentially active context, leaving 620 * free contexts unused. 621 */ 622 for (i = kvm->arch.nested_mmus_next; 623 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); 624 i++) { 625 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; 626 627 if (atomic_read(&s2_mmu->refcnt) == 0) 628 break; 629 } 630 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */ 631 632 /* Set the scene for the next search */ 633 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; 634 635 /* Make sure we don't forget to do the laundry */ 636 if (kvm_s2_mmu_valid(s2_mmu)) 637 s2_mmu->pending_unmap = true; 638 639 /* 640 * The virtual VMID (modulo CnP) will be used as a key when matching 641 * an existing kvm_s2_mmu. 642 * 643 * We cache VTCR at allocation time, once and for all. It'd be great 644 * if the guest didn't screw that one up, as this is not very 645 * forgiving... 646 */ 647 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; 648 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 649 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; 650 651 out: 652 atomic_inc(&s2_mmu->refcnt); 653 654 /* 655 * Set the vCPU request to perform an unmap, even if the pending unmap 656 * originates from another vCPU. This guarantees that the MMU has been 657 * completely unmapped before any vCPU actually uses it, and allows 658 * multiple vCPUs to lend a hand with completing the unmap. 659 */ 660 if (s2_mmu->pending_unmap) 661 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu); 662 663 return s2_mmu; 664 } 665 kvm_init_nested_s2_mmu(struct kvm_s2_mmu * mmu)666 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) 667 { 668 /* CnP being set denotes an invalid entry */ 669 mmu->tlb_vttbr = VTTBR_CNP_BIT; 670 mmu->nested_stage2_enabled = false; 671 atomic_set(&mmu->refcnt, 0); 672 } 673 kvm_vcpu_load_hw_mmu(struct kvm_vcpu * vcpu)674 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) 675 { 676 /* 677 * The vCPU kept its reference on the MMU after the last put, keep 678 * rolling with it. 679 */ 680 if (vcpu->arch.hw_mmu) 681 return; 682 683 if (is_hyp_ctxt(vcpu)) { 684 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 685 } else { 686 write_lock(&vcpu->kvm->mmu_lock); 687 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); 688 write_unlock(&vcpu->kvm->mmu_lock); 689 } 690 } 691 kvm_vcpu_put_hw_mmu(struct kvm_vcpu * vcpu)692 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) 693 { 694 /* 695 * Keep a reference on the associated stage-2 MMU if the vCPU is 696 * scheduling out and not in WFI emulation, suggesting it is likely to 697 * reuse the MMU sometime soon. 698 */ 699 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI)) 700 return; 701 702 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) 703 atomic_dec(&vcpu->arch.hw_mmu->refcnt); 704 705 vcpu->arch.hw_mmu = NULL; 706 } 707 708 /* 709 * Returns non-zero if permission fault is handled by injecting it to the next 710 * level hypervisor. 711 */ kvm_s2_handle_perm_fault(struct kvm_vcpu * vcpu,struct kvm_s2_trans * trans)712 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans) 713 { 714 bool forward_fault = false; 715 716 trans->esr = 0; 717 718 if (!kvm_vcpu_trap_is_permission_fault(vcpu)) 719 return 0; 720 721 if (kvm_vcpu_trap_is_iabt(vcpu)) { 722 forward_fault = !kvm_s2_trans_executable(trans); 723 } else { 724 bool write_fault = kvm_is_write_fault(vcpu); 725 726 forward_fault = ((write_fault && !trans->writable) || 727 (!write_fault && !trans->readable)); 728 } 729 730 if (forward_fault) 731 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM); 732 733 return forward_fault; 734 } 735 kvm_inject_s2_fault(struct kvm_vcpu * vcpu,u64 esr_el2)736 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) 737 { 738 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2); 739 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2); 740 741 return kvm_inject_nested_sync(vcpu, esr_el2); 742 } 743 kvm_nested_s2_wp(struct kvm * kvm)744 void kvm_nested_s2_wp(struct kvm *kvm) 745 { 746 int i; 747 748 lockdep_assert_held_write(&kvm->mmu_lock); 749 750 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 751 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 752 753 if (kvm_s2_mmu_valid(mmu)) 754 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); 755 } 756 } 757 kvm_nested_s2_unmap(struct kvm * kvm,bool may_block)758 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) 759 { 760 int i; 761 762 lockdep_assert_held_write(&kvm->mmu_lock); 763 764 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 765 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 766 767 if (kvm_s2_mmu_valid(mmu)) 768 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); 769 } 770 } 771 kvm_nested_s2_flush(struct kvm * kvm)772 void kvm_nested_s2_flush(struct kvm *kvm) 773 { 774 int i; 775 776 lockdep_assert_held_write(&kvm->mmu_lock); 777 778 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 779 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 780 781 if (kvm_s2_mmu_valid(mmu)) 782 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); 783 } 784 } 785 kvm_arch_flush_shadow_all(struct kvm * kvm)786 void kvm_arch_flush_shadow_all(struct kvm *kvm) 787 { 788 int i; 789 790 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 791 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 792 793 if (!WARN_ON(atomic_read(&mmu->refcnt))) 794 kvm_free_stage2_pgd(mmu); 795 } 796 kvfree(kvm->arch.nested_mmus); 797 kvm->arch.nested_mmus = NULL; 798 kvm->arch.nested_mmus_size = 0; 799 kvm_uninit_stage2_mmu(kvm); 800 } 801 802 /* 803 * Our emulated CPU doesn't support all the possible features. For the 804 * sake of simplicity (and probably mental sanity), wipe out a number 805 * of feature bits we don't intend to support for the time being. 806 * This list should get updated as new features get added to the NV 807 * support, and new extension to the architecture. 808 */ limit_nv_id_regs(struct kvm * kvm)809 static void limit_nv_id_regs(struct kvm *kvm) 810 { 811 u64 val, tmp; 812 813 /* Support everything but TME */ 814 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1); 815 val &= ~NV_FTR(ISAR0, TME); 816 kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1, val); 817 818 /* Support everything but Spec Invalidation and LS64 */ 819 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1); 820 val &= ~(NV_FTR(ISAR1, LS64) | 821 NV_FTR(ISAR1, SPECRES)); 822 kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1, val); 823 824 /* No AMU, MPAM, S-EL2, or RAS */ 825 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1); 826 val &= ~(GENMASK_ULL(55, 52) | 827 NV_FTR(PFR0, AMU) | 828 NV_FTR(PFR0, MPAM) | 829 NV_FTR(PFR0, SEL2) | 830 NV_FTR(PFR0, RAS) | 831 NV_FTR(PFR0, EL3) | 832 NV_FTR(PFR0, EL2) | 833 NV_FTR(PFR0, EL1)); 834 /* 64bit EL1/EL2/EL3 only */ 835 val |= FIELD_PREP(NV_FTR(PFR0, EL1), 0b0001); 836 val |= FIELD_PREP(NV_FTR(PFR0, EL2), 0b0001); 837 val |= FIELD_PREP(NV_FTR(PFR0, EL3), 0b0001); 838 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, val); 839 840 /* Only support BTI, SSBS, CSV2_frac */ 841 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1); 842 val &= (NV_FTR(PFR1, BT) | 843 NV_FTR(PFR1, SSBS) | 844 NV_FTR(PFR1, CSV2_frac)); 845 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1, val); 846 847 /* Hide ECV, ExS, Secure Memory */ 848 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1); 849 val &= ~(NV_FTR(MMFR0, ECV) | 850 NV_FTR(MMFR0, EXS) | 851 NV_FTR(MMFR0, TGRAN4_2) | 852 NV_FTR(MMFR0, TGRAN16_2) | 853 NV_FTR(MMFR0, TGRAN64_2) | 854 NV_FTR(MMFR0, SNSMEM)); 855 856 /* Disallow unsupported S2 page sizes */ 857 switch (PAGE_SIZE) { 858 case SZ_64K: 859 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0001); 860 fallthrough; 861 case SZ_16K: 862 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0001); 863 fallthrough; 864 case SZ_4K: 865 /* Support everything */ 866 break; 867 } 868 /* 869 * Since we can't support a guest S2 page size smaller than 870 * the host's own page size (due to KVM only populating its 871 * own S2 using the kernel's page size), advertise the 872 * limitation using FEAT_GTG. 873 */ 874 switch (PAGE_SIZE) { 875 case SZ_4K: 876 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0010); 877 fallthrough; 878 case SZ_16K: 879 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0010); 880 fallthrough; 881 case SZ_64K: 882 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN64_2), 0b0010); 883 break; 884 } 885 /* Cap PARange to 48bits */ 886 tmp = FIELD_GET(NV_FTR(MMFR0, PARANGE), val); 887 if (tmp > 0b0101) { 888 val &= ~NV_FTR(MMFR0, PARANGE); 889 val |= FIELD_PREP(NV_FTR(MMFR0, PARANGE), 0b0101); 890 } 891 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1, val); 892 893 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1); 894 val &= (NV_FTR(MMFR1, HCX) | 895 NV_FTR(MMFR1, PAN) | 896 NV_FTR(MMFR1, LO) | 897 NV_FTR(MMFR1, HPDS) | 898 NV_FTR(MMFR1, VH) | 899 NV_FTR(MMFR1, VMIDBits)); 900 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1, val); 901 902 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1); 903 val &= ~(NV_FTR(MMFR2, BBM) | 904 NV_FTR(MMFR2, TTL) | 905 GENMASK_ULL(47, 44) | 906 NV_FTR(MMFR2, ST) | 907 NV_FTR(MMFR2, CCIDX) | 908 NV_FTR(MMFR2, VARange)); 909 910 /* Force TTL support */ 911 val |= FIELD_PREP(NV_FTR(MMFR2, TTL), 0b0001); 912 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1, val); 913 914 val = 0; 915 if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1)) 916 val |= FIELD_PREP(NV_FTR(MMFR4, E2H0), 917 ID_AA64MMFR4_EL1_E2H0_NI_NV1); 918 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR4_EL1, val); 919 920 /* Only limited support for PMU, Debug, BPs and WPs */ 921 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1); 922 val &= (NV_FTR(DFR0, PMUVer) | 923 NV_FTR(DFR0, WRPs) | 924 NV_FTR(DFR0, BRPs) | 925 NV_FTR(DFR0, DebugVer)); 926 927 /* Cap Debug to ARMv8.1 */ 928 tmp = FIELD_GET(NV_FTR(DFR0, DebugVer), val); 929 if (tmp > 0b0111) { 930 val &= ~NV_FTR(DFR0, DebugVer); 931 val |= FIELD_PREP(NV_FTR(DFR0, DebugVer), 0b0111); 932 } 933 kvm_set_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1, val); 934 } 935 kvm_vcpu_sanitise_vncr_reg(const struct kvm_vcpu * vcpu,enum vcpu_sysreg sr)936 u64 kvm_vcpu_sanitise_vncr_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg sr) 937 { 938 u64 v = ctxt_sys_reg(&vcpu->arch.ctxt, sr); 939 struct kvm_sysreg_masks *masks; 940 941 masks = vcpu->kvm->arch.sysreg_masks; 942 943 if (masks) { 944 sr -= __VNCR_START__; 945 946 v &= ~masks->mask[sr].res0; 947 v |= masks->mask[sr].res1; 948 } 949 950 return v; 951 } 952 set_sysreg_masks(struct kvm * kvm,int sr,u64 res0,u64 res1)953 static void set_sysreg_masks(struct kvm *kvm, int sr, u64 res0, u64 res1) 954 { 955 int i = sr - __VNCR_START__; 956 957 kvm->arch.sysreg_masks->mask[i].res0 = res0; 958 kvm->arch.sysreg_masks->mask[i].res1 = res1; 959 } 960 kvm_init_nv_sysregs(struct kvm * kvm)961 int kvm_init_nv_sysregs(struct kvm *kvm) 962 { 963 u64 res0, res1; 964 965 lockdep_assert_held(&kvm->arch.config_lock); 966 967 if (kvm->arch.sysreg_masks) 968 return 0; 969 970 kvm->arch.sysreg_masks = kzalloc(sizeof(*(kvm->arch.sysreg_masks)), 971 GFP_KERNEL_ACCOUNT); 972 if (!kvm->arch.sysreg_masks) 973 return -ENOMEM; 974 975 limit_nv_id_regs(kvm); 976 977 /* VTTBR_EL2 */ 978 res0 = res1 = 0; 979 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16)) 980 res0 |= GENMASK(63, 56); 981 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP)) 982 res0 |= VTTBR_CNP_BIT; 983 set_sysreg_masks(kvm, VTTBR_EL2, res0, res1); 984 985 /* VTCR_EL2 */ 986 res0 = GENMASK(63, 32) | GENMASK(30, 20); 987 res1 = BIT(31); 988 set_sysreg_masks(kvm, VTCR_EL2, res0, res1); 989 990 /* VMPIDR_EL2 */ 991 res0 = GENMASK(63, 40) | GENMASK(30, 24); 992 res1 = BIT(31); 993 set_sysreg_masks(kvm, VMPIDR_EL2, res0, res1); 994 995 /* HCR_EL2 */ 996 res0 = BIT(48); 997 res1 = HCR_RW; 998 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, TWED, IMP)) 999 res0 |= GENMASK(63, 59); 1000 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, MTE, MTE2)) 1001 res0 |= (HCR_TID5 | HCR_DCT | HCR_ATA); 1002 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, TTLBxS)) 1003 res0 |= (HCR_TTLBIS | HCR_TTLBOS); 1004 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) && 1005 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2)) 1006 res0 |= HCR_ENSCXT; 1007 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, IMP)) 1008 res0 |= (HCR_TOCU | HCR_TICAB | HCR_TID4); 1009 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1)) 1010 res0 |= HCR_AMVOFFEN; 1011 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1)) 1012 res0 |= HCR_FIEN; 1013 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, FWB, IMP)) 1014 res0 |= HCR_FWB; 1015 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, NV2)) 1016 res0 |= HCR_NV2; 1017 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, IMP)) 1018 res0 |= (HCR_AT | HCR_NV1 | HCR_NV); 1019 if (!(__vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_ADDRESS) && 1020 __vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_GENERIC))) 1021 res0 |= (HCR_API | HCR_APK); 1022 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TME, IMP)) 1023 res0 |= BIT(39); 1024 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) 1025 res0 |= (HCR_TEA | HCR_TERR); 1026 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 1027 res0 |= HCR_TLOR; 1028 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, E2H0, IMP)) 1029 res1 |= HCR_E2H; 1030 set_sysreg_masks(kvm, HCR_EL2, res0, res1); 1031 1032 /* HCRX_EL2 */ 1033 res0 = HCRX_EL2_RES0; 1034 res1 = HCRX_EL2_RES1; 1035 if (!kvm_has_feat(kvm, ID_AA64ISAR3_EL1, PACM, TRIVIAL_IMP)) 1036 res0 |= HCRX_EL2_PACMEn; 1037 if (!kvm_has_feat(kvm, ID_AA64PFR2_EL1, FPMR, IMP)) 1038 res0 |= HCRX_EL2_EnFPM; 1039 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1040 res0 |= HCRX_EL2_GCSEn; 1041 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, SYSREG_128, IMP)) 1042 res0 |= HCRX_EL2_EnIDCP128; 1043 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, DEV_ASYNC)) 1044 res0 |= (HCRX_EL2_EnSDERR | HCRX_EL2_EnSNERR); 1045 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, DF2, IMP)) 1046 res0 |= HCRX_EL2_TMEA; 1047 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP)) 1048 res0 |= HCRX_EL2_D128En; 1049 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1050 res0 |= HCRX_EL2_PTTWI; 1051 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SCTLRX, IMP)) 1052 res0 |= HCRX_EL2_SCTLR2En; 1053 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, TCRX, IMP)) 1054 res0 |= HCRX_EL2_TCR2En; 1055 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP)) 1056 res0 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2); 1057 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, CMOW, IMP)) 1058 res0 |= HCRX_EL2_CMOW; 1059 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, NMI, IMP)) 1060 res0 |= (HCRX_EL2_VFNMI | HCRX_EL2_VINMI | HCRX_EL2_TALLINT); 1061 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP) || 1062 !(read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS)) 1063 res0 |= HCRX_EL2_SMPME; 1064 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)) 1065 res0 |= (HCRX_EL2_FGTnXS | HCRX_EL2_FnXS); 1066 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) 1067 res0 |= HCRX_EL2_EnASR; 1068 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64)) 1069 res0 |= HCRX_EL2_EnALS; 1070 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA)) 1071 res0 |= HCRX_EL2_EnAS0; 1072 set_sysreg_masks(kvm, HCRX_EL2, res0, res1); 1073 1074 /* HFG[RW]TR_EL2 */ 1075 res0 = res1 = 0; 1076 if (!(__vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_ADDRESS) && 1077 __vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_GENERIC))) 1078 res0 |= (HFGxTR_EL2_APDAKey | HFGxTR_EL2_APDBKey | 1079 HFGxTR_EL2_APGAKey | HFGxTR_EL2_APIAKey | 1080 HFGxTR_EL2_APIBKey); 1081 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 1082 res0 |= (HFGxTR_EL2_LORC_EL1 | HFGxTR_EL2_LOREA_EL1 | 1083 HFGxTR_EL2_LORID_EL1 | HFGxTR_EL2_LORN_EL1 | 1084 HFGxTR_EL2_LORSA_EL1); 1085 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) && 1086 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2)) 1087 res0 |= (HFGxTR_EL2_SCXTNUM_EL1 | HFGxTR_EL2_SCXTNUM_EL0); 1088 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP)) 1089 res0 |= HFGxTR_EL2_ICC_IGRPENn_EL1; 1090 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) 1091 res0 |= (HFGxTR_EL2_ERRIDR_EL1 | HFGxTR_EL2_ERRSELR_EL1 | 1092 HFGxTR_EL2_ERXFR_EL1 | HFGxTR_EL2_ERXCTLR_EL1 | 1093 HFGxTR_EL2_ERXSTATUS_EL1 | HFGxTR_EL2_ERXMISCn_EL1 | 1094 HFGxTR_EL2_ERXPFGF_EL1 | HFGxTR_EL2_ERXPFGCTL_EL1 | 1095 HFGxTR_EL2_ERXPFGCDN_EL1 | HFGxTR_EL2_ERXADDR_EL1); 1096 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA)) 1097 res0 |= HFGxTR_EL2_nACCDATA_EL1; 1098 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1099 res0 |= (HFGxTR_EL2_nGCS_EL0 | HFGxTR_EL2_nGCS_EL1); 1100 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP)) 1101 res0 |= (HFGxTR_EL2_nSMPRI_EL1 | HFGxTR_EL2_nTPIDR2_EL0); 1102 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1103 res0 |= HFGxTR_EL2_nRCWMASK_EL1; 1104 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1PIE, IMP)) 1105 res0 |= (HFGxTR_EL2_nPIRE0_EL1 | HFGxTR_EL2_nPIR_EL1); 1106 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1POE, IMP)) 1107 res0 |= (HFGxTR_EL2_nPOR_EL0 | HFGxTR_EL2_nPOR_EL1); 1108 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) 1109 res0 |= HFGxTR_EL2_nS2POR_EL1; 1110 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP)) 1111 res0 |= (HFGxTR_EL2_nMAIR2_EL1 | HFGxTR_EL2_nAMAIR2_EL1); 1112 set_sysreg_masks(kvm, HFGRTR_EL2, res0 | __HFGRTR_EL2_RES0, res1); 1113 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | __HFGWTR_EL2_RES0, res1); 1114 1115 /* HDFG[RW]TR_EL2 */ 1116 res0 = res1 = 0; 1117 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP)) 1118 res0 |= HDFGRTR_EL2_OSDLR_EL1; 1119 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1120 res0 |= (HDFGRTR_EL2_PMEVCNTRn_EL0 | HDFGRTR_EL2_PMEVTYPERn_EL0 | 1121 HDFGRTR_EL2_PMCCFILTR_EL0 | HDFGRTR_EL2_PMCCNTR_EL0 | 1122 HDFGRTR_EL2_PMCNTEN | HDFGRTR_EL2_PMINTEN | 1123 HDFGRTR_EL2_PMOVS | HDFGRTR_EL2_PMSELR_EL0 | 1124 HDFGRTR_EL2_PMMIR_EL1 | HDFGRTR_EL2_PMUSERENR_EL0 | 1125 HDFGRTR_EL2_PMCEIDn_EL0); 1126 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) 1127 res0 |= (HDFGRTR_EL2_PMBLIMITR_EL1 | HDFGRTR_EL2_PMBPTR_EL1 | 1128 HDFGRTR_EL2_PMBSR_EL1 | HDFGRTR_EL2_PMSCR_EL1 | 1129 HDFGRTR_EL2_PMSEVFR_EL1 | HDFGRTR_EL2_PMSFCR_EL1 | 1130 HDFGRTR_EL2_PMSICR_EL1 | HDFGRTR_EL2_PMSIDR_EL1 | 1131 HDFGRTR_EL2_PMSIRR_EL1 | HDFGRTR_EL2_PMSLATFR_EL1 | 1132 HDFGRTR_EL2_PMBIDR_EL1); 1133 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP)) 1134 res0 |= (HDFGRTR_EL2_TRC | HDFGRTR_EL2_TRCAUTHSTATUS | 1135 HDFGRTR_EL2_TRCAUXCTLR | HDFGRTR_EL2_TRCCLAIM | 1136 HDFGRTR_EL2_TRCCNTVRn | HDFGRTR_EL2_TRCID | 1137 HDFGRTR_EL2_TRCIMSPECn | HDFGRTR_EL2_TRCOSLSR | 1138 HDFGRTR_EL2_TRCPRGCTLR | HDFGRTR_EL2_TRCSEQSTR | 1139 HDFGRTR_EL2_TRCSSCSRn | HDFGRTR_EL2_TRCSTATR | 1140 HDFGRTR_EL2_TRCVICTLR); 1141 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP)) 1142 res0 |= (HDFGRTR_EL2_TRBBASER_EL1 | HDFGRTR_EL2_TRBIDR_EL1 | 1143 HDFGRTR_EL2_TRBLIMITR_EL1 | HDFGRTR_EL2_TRBMAR_EL1 | 1144 HDFGRTR_EL2_TRBPTR_EL1 | HDFGRTR_EL2_TRBSR_EL1 | 1145 HDFGRTR_EL2_TRBTRG_EL1); 1146 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP)) 1147 res0 |= (HDFGRTR_EL2_nBRBIDR | HDFGRTR_EL2_nBRBCTL | 1148 HDFGRTR_EL2_nBRBDATA); 1149 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2)) 1150 res0 |= HDFGRTR_EL2_nPMSNEVFR_EL1; 1151 set_sysreg_masks(kvm, HDFGRTR_EL2, res0 | HDFGRTR_EL2_RES0, res1); 1152 1153 /* Reuse the bits from the read-side and add the write-specific stuff */ 1154 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1155 res0 |= (HDFGWTR_EL2_PMCR_EL0 | HDFGWTR_EL2_PMSWINC_EL0); 1156 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP)) 1157 res0 |= HDFGWTR_EL2_TRCOSLAR; 1158 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP)) 1159 res0 |= HDFGWTR_EL2_TRFCR_EL1; 1160 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | HDFGWTR_EL2_RES0, res1); 1161 1162 /* HFGITR_EL2 */ 1163 res0 = HFGITR_EL2_RES0; 1164 res1 = HFGITR_EL2_RES1; 1165 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, DPB, DPB2)) 1166 res0 |= HFGITR_EL2_DCCVADP; 1167 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN2)) 1168 res0 |= (HFGITR_EL2_ATS1E1RP | HFGITR_EL2_ATS1E1WP); 1169 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 1170 res0 |= (HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS | 1171 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS | 1172 HFGITR_EL2_TLBIVAALE1OS | HFGITR_EL2_TLBIVALE1OS | 1173 HFGITR_EL2_TLBIVAAE1OS | HFGITR_EL2_TLBIASIDE1OS | 1174 HFGITR_EL2_TLBIVAE1OS | HFGITR_EL2_TLBIVMALLE1OS); 1175 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 1176 res0 |= (HFGITR_EL2_TLBIRVAALE1 | HFGITR_EL2_TLBIRVALE1 | 1177 HFGITR_EL2_TLBIRVAAE1 | HFGITR_EL2_TLBIRVAE1 | 1178 HFGITR_EL2_TLBIRVAALE1IS | HFGITR_EL2_TLBIRVALE1IS | 1179 HFGITR_EL2_TLBIRVAAE1IS | HFGITR_EL2_TLBIRVAE1IS | 1180 HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS | 1181 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS); 1182 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, IMP)) 1183 res0 |= (HFGITR_EL2_CFPRCTX | HFGITR_EL2_DVPRCTX | 1184 HFGITR_EL2_CPPRCTX); 1185 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP)) 1186 res0 |= (HFGITR_EL2_nBRBINJ | HFGITR_EL2_nBRBIALL); 1187 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1188 res0 |= (HFGITR_EL2_nGCSPUSHM_EL1 | HFGITR_EL2_nGCSSTR_EL1 | 1189 HFGITR_EL2_nGCSEPP); 1190 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, COSP_RCTX)) 1191 res0 |= HFGITR_EL2_COSPRCTX; 1192 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, ATS1A, IMP)) 1193 res0 |= HFGITR_EL2_ATS1E1A; 1194 set_sysreg_masks(kvm, HFGITR_EL2, res0, res1); 1195 1196 /* HAFGRTR_EL2 - not a lot to see here */ 1197 res0 = HAFGRTR_EL2_RES0; 1198 res1 = HAFGRTR_EL2_RES1; 1199 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1)) 1200 res0 |= ~(res0 | res1); 1201 set_sysreg_masks(kvm, HAFGRTR_EL2, res0, res1); 1202 1203 /* SCTLR_EL1 */ 1204 res0 = SCTLR_EL1_RES0; 1205 res1 = SCTLR_EL1_RES1; 1206 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN3)) 1207 res0 |= SCTLR_EL1_EPAN; 1208 set_sysreg_masks(kvm, SCTLR_EL1, res0, res1); 1209 1210 return 0; 1211 } 1212 check_nested_vcpu_requests(struct kvm_vcpu * vcpu)1213 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) 1214 { 1215 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) { 1216 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; 1217 1218 write_lock(&vcpu->kvm->mmu_lock); 1219 if (mmu->pending_unmap) { 1220 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true); 1221 mmu->pending_unmap = false; 1222 } 1223 write_unlock(&vcpu->kvm->mmu_lock); 1224 } 1225 } 1226