1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 Google LLC 4 * Author: Fuad Tabba <tabba@google.com> 5 */ 6 7 #include <linux/kvm_host.h> 8 #include <linux/mm.h> 9 #include <nvhe/fixed_config.h> 10 #include <nvhe/mem_protect.h> 11 #include <nvhe/memory.h> 12 #include <nvhe/pkvm.h> 13 #include <nvhe/trap_handler.h> 14 15 /* Used by icache_is_aliasing(). */ 16 unsigned long __icache_flags; 17 18 /* Used by kvm_get_vttbr(). */ 19 unsigned int kvm_arm_vmid_bits; 20 21 unsigned int kvm_host_sve_max_vl; 22 23 /* 24 * Set trap register values based on features in ID_AA64PFR0. 25 */ pvm_init_traps_aa64pfr0(struct kvm_vcpu * vcpu)26 static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu) 27 { 28 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR0_EL1); 29 u64 hcr_set = HCR_RW; 30 u64 hcr_clear = 0; 31 u64 cptr_set = 0; 32 u64 cptr_clear = 0; 33 34 /* Protected KVM does not support AArch32 guests. */ 35 BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0), 36 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_EL0_IMP); 37 BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1), 38 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_EL1_IMP); 39 40 /* 41 * Linux guests assume support for floating-point and Advanced SIMD. Do 42 * not change the trapping behavior for these from the KVM default. 43 */ 44 BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP), 45 PVM_ID_AA64PFR0_ALLOW)); 46 BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD), 47 PVM_ID_AA64PFR0_ALLOW)); 48 49 if (has_hvhe()) 50 hcr_set |= HCR_E2H; 51 52 /* Trap RAS unless all current versions are supported */ 53 if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), feature_ids) < 54 ID_AA64PFR0_EL1_RAS_V1P1) { 55 hcr_set |= HCR_TERR | HCR_TEA; 56 hcr_clear |= HCR_FIEN; 57 } 58 59 /* Trap AMU */ 60 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU), feature_ids)) { 61 hcr_clear |= HCR_AMVOFFEN; 62 cptr_set |= CPTR_EL2_TAM; 63 } 64 65 /* Trap SVE */ 66 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids)) { 67 if (has_hvhe()) 68 cptr_clear |= CPACR_ELx_ZEN; 69 else 70 cptr_set |= CPTR_EL2_TZ; 71 } 72 73 vcpu->arch.hcr_el2 |= hcr_set; 74 vcpu->arch.hcr_el2 &= ~hcr_clear; 75 vcpu->arch.cptr_el2 |= cptr_set; 76 vcpu->arch.cptr_el2 &= ~cptr_clear; 77 } 78 79 /* 80 * Set trap register values based on features in ID_AA64PFR1. 81 */ pvm_init_traps_aa64pfr1(struct kvm_vcpu * vcpu)82 static void pvm_init_traps_aa64pfr1(struct kvm_vcpu *vcpu) 83 { 84 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR1_EL1); 85 u64 hcr_set = 0; 86 u64 hcr_clear = 0; 87 88 /* Memory Tagging: Trap and Treat as Untagged if not supported. */ 89 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE), feature_ids)) { 90 hcr_set |= HCR_TID5; 91 hcr_clear |= HCR_DCT | HCR_ATA; 92 } 93 94 vcpu->arch.hcr_el2 |= hcr_set; 95 vcpu->arch.hcr_el2 &= ~hcr_clear; 96 } 97 98 /* 99 * Set trap register values based on features in ID_AA64DFR0. 100 */ pvm_init_traps_aa64dfr0(struct kvm_vcpu * vcpu)101 static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu) 102 { 103 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64DFR0_EL1); 104 u64 mdcr_set = 0; 105 u64 mdcr_clear = 0; 106 u64 cptr_set = 0; 107 108 /* Trap/constrain PMU */ 109 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), feature_ids)) { 110 mdcr_set |= MDCR_EL2_TPM | MDCR_EL2_TPMCR; 111 mdcr_clear |= MDCR_EL2_HPME | MDCR_EL2_MTPME | 112 MDCR_EL2_HPMN_MASK; 113 } 114 115 /* Trap Debug */ 116 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), feature_ids)) 117 mdcr_set |= MDCR_EL2_TDRA | MDCR_EL2_TDA | MDCR_EL2_TDE; 118 119 /* Trap OS Double Lock */ 120 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DoubleLock), feature_ids)) 121 mdcr_set |= MDCR_EL2_TDOSA; 122 123 /* Trap SPE */ 124 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer), feature_ids)) { 125 mdcr_set |= MDCR_EL2_TPMS; 126 mdcr_clear |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT; 127 } 128 129 /* Trap Trace Filter */ 130 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceFilt), feature_ids)) 131 mdcr_set |= MDCR_EL2_TTRF; 132 133 /* Trap Trace */ 134 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids)) { 135 if (has_hvhe()) 136 cptr_set |= CPACR_EL1_TTA; 137 else 138 cptr_set |= CPTR_EL2_TTA; 139 } 140 141 /* Trap External Trace */ 142 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_ExtTrcBuff), feature_ids)) 143 mdcr_clear |= MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT; 144 145 vcpu->arch.mdcr_el2 |= mdcr_set; 146 vcpu->arch.mdcr_el2 &= ~mdcr_clear; 147 vcpu->arch.cptr_el2 |= cptr_set; 148 } 149 150 /* 151 * Set trap register values based on features in ID_AA64MMFR0. 152 */ pvm_init_traps_aa64mmfr0(struct kvm_vcpu * vcpu)153 static void pvm_init_traps_aa64mmfr0(struct kvm_vcpu *vcpu) 154 { 155 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64MMFR0_EL1); 156 u64 mdcr_set = 0; 157 158 /* Trap Debug Communications Channel registers */ 159 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_FGT), feature_ids)) 160 mdcr_set |= MDCR_EL2_TDCC; 161 162 vcpu->arch.mdcr_el2 |= mdcr_set; 163 } 164 165 /* 166 * Set trap register values based on features in ID_AA64MMFR1. 167 */ pvm_init_traps_aa64mmfr1(struct kvm_vcpu * vcpu)168 static void pvm_init_traps_aa64mmfr1(struct kvm_vcpu *vcpu) 169 { 170 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64MMFR1_EL1); 171 u64 hcr_set = 0; 172 173 /* Trap LOR */ 174 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_LO), feature_ids)) 175 hcr_set |= HCR_TLOR; 176 177 vcpu->arch.hcr_el2 |= hcr_set; 178 } 179 180 /* 181 * Set baseline trap register values. 182 */ pvm_init_trap_regs(struct kvm_vcpu * vcpu)183 static void pvm_init_trap_regs(struct kvm_vcpu *vcpu) 184 { 185 const u64 hcr_trap_feat_regs = HCR_TID3; 186 const u64 hcr_trap_impdef = HCR_TACR | HCR_TIDCP | HCR_TID1; 187 188 /* 189 * Always trap: 190 * - Feature id registers: to control features exposed to guests 191 * - Implementation-defined features 192 */ 193 vcpu->arch.hcr_el2 |= hcr_trap_feat_regs | hcr_trap_impdef; 194 195 /* Clear res0 and set res1 bits to trap potential new features. */ 196 vcpu->arch.hcr_el2 &= ~(HCR_RES0); 197 vcpu->arch.mdcr_el2 &= ~(MDCR_EL2_RES0); 198 if (!has_hvhe()) { 199 vcpu->arch.cptr_el2 |= CPTR_NVHE_EL2_RES1; 200 vcpu->arch.cptr_el2 &= ~(CPTR_NVHE_EL2_RES0); 201 } 202 } 203 204 /* 205 * Initialize trap register values in protected mode. 206 */ __pkvm_vcpu_init_traps(struct kvm_vcpu * vcpu)207 void __pkvm_vcpu_init_traps(struct kvm_vcpu *vcpu) 208 { 209 pvm_init_trap_regs(vcpu); 210 pvm_init_traps_aa64pfr0(vcpu); 211 pvm_init_traps_aa64pfr1(vcpu); 212 pvm_init_traps_aa64dfr0(vcpu); 213 pvm_init_traps_aa64mmfr0(vcpu); 214 pvm_init_traps_aa64mmfr1(vcpu); 215 } 216 217 /* 218 * Start the VM table handle at the offset defined instead of at 0. 219 * Mainly for sanity checking and debugging. 220 */ 221 #define HANDLE_OFFSET 0x1000 222 vm_handle_to_idx(pkvm_handle_t handle)223 static unsigned int vm_handle_to_idx(pkvm_handle_t handle) 224 { 225 return handle - HANDLE_OFFSET; 226 } 227 idx_to_vm_handle(unsigned int idx)228 static pkvm_handle_t idx_to_vm_handle(unsigned int idx) 229 { 230 return idx + HANDLE_OFFSET; 231 } 232 233 /* 234 * Spinlock for protecting state related to the VM table. Protects writes 235 * to 'vm_table' and 'nr_table_entries' as well as reads and writes to 236 * 'last_hyp_vcpu_lookup'. 237 */ 238 static DEFINE_HYP_SPINLOCK(vm_table_lock); 239 240 /* 241 * The table of VM entries for protected VMs in hyp. 242 * Allocated at hyp initialization and setup. 243 */ 244 static struct pkvm_hyp_vm **vm_table; 245 pkvm_hyp_vm_table_init(void * tbl)246 void pkvm_hyp_vm_table_init(void *tbl) 247 { 248 WARN_ON(vm_table); 249 vm_table = tbl; 250 } 251 252 /* 253 * Return the hyp vm structure corresponding to the handle. 254 */ get_vm_by_handle(pkvm_handle_t handle)255 static struct pkvm_hyp_vm *get_vm_by_handle(pkvm_handle_t handle) 256 { 257 unsigned int idx = vm_handle_to_idx(handle); 258 259 if (unlikely(idx >= KVM_MAX_PVMS)) 260 return NULL; 261 262 return vm_table[idx]; 263 } 264 pkvm_load_hyp_vcpu(pkvm_handle_t handle,unsigned int vcpu_idx)265 struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, 266 unsigned int vcpu_idx) 267 { 268 struct pkvm_hyp_vcpu *hyp_vcpu = NULL; 269 struct pkvm_hyp_vm *hyp_vm; 270 271 hyp_spin_lock(&vm_table_lock); 272 hyp_vm = get_vm_by_handle(handle); 273 if (!hyp_vm || hyp_vm->nr_vcpus <= vcpu_idx) 274 goto unlock; 275 276 hyp_vcpu = hyp_vm->vcpus[vcpu_idx]; 277 hyp_page_ref_inc(hyp_virt_to_page(hyp_vm)); 278 unlock: 279 hyp_spin_unlock(&vm_table_lock); 280 return hyp_vcpu; 281 } 282 pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu * hyp_vcpu)283 void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 284 { 285 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 286 287 hyp_spin_lock(&vm_table_lock); 288 hyp_page_ref_dec(hyp_virt_to_page(hyp_vm)); 289 hyp_spin_unlock(&vm_table_lock); 290 } 291 unpin_host_vcpu(struct kvm_vcpu * host_vcpu)292 static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) 293 { 294 if (host_vcpu) 295 hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); 296 } 297 unpin_host_vcpus(struct pkvm_hyp_vcpu * hyp_vcpus[],unsigned int nr_vcpus)298 static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[], 299 unsigned int nr_vcpus) 300 { 301 int i; 302 303 for (i = 0; i < nr_vcpus; i++) 304 unpin_host_vcpu(hyp_vcpus[i]->host_vcpu); 305 } 306 init_pkvm_hyp_vm(struct kvm * host_kvm,struct pkvm_hyp_vm * hyp_vm,unsigned int nr_vcpus)307 static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, 308 unsigned int nr_vcpus) 309 { 310 hyp_vm->host_kvm = host_kvm; 311 hyp_vm->kvm.created_vcpus = nr_vcpus; 312 hyp_vm->kvm.arch.mmu.vtcr = host_mmu.arch.mmu.vtcr; 313 } 314 init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu * hyp_vcpu,struct pkvm_hyp_vm * hyp_vm,struct kvm_vcpu * host_vcpu,unsigned int vcpu_idx)315 static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, 316 struct pkvm_hyp_vm *hyp_vm, 317 struct kvm_vcpu *host_vcpu, 318 unsigned int vcpu_idx) 319 { 320 int ret = 0; 321 322 if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) 323 return -EBUSY; 324 325 if (host_vcpu->vcpu_idx != vcpu_idx) { 326 ret = -EINVAL; 327 goto done; 328 } 329 330 hyp_vcpu->host_vcpu = host_vcpu; 331 332 hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; 333 hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id); 334 hyp_vcpu->vcpu.vcpu_idx = vcpu_idx; 335 336 hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; 337 hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); 338 done: 339 if (ret) 340 unpin_host_vcpu(host_vcpu); 341 return ret; 342 } 343 find_free_vm_table_entry(struct kvm * host_kvm)344 static int find_free_vm_table_entry(struct kvm *host_kvm) 345 { 346 int i; 347 348 for (i = 0; i < KVM_MAX_PVMS; ++i) { 349 if (!vm_table[i]) 350 return i; 351 } 352 353 return -ENOMEM; 354 } 355 356 /* 357 * Allocate a VM table entry and insert a pointer to the new vm. 358 * 359 * Return a unique handle to the protected VM on success, 360 * negative error code on failure. 361 */ insert_vm_table_entry(struct kvm * host_kvm,struct pkvm_hyp_vm * hyp_vm)362 static pkvm_handle_t insert_vm_table_entry(struct kvm *host_kvm, 363 struct pkvm_hyp_vm *hyp_vm) 364 { 365 struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu; 366 int idx; 367 368 hyp_assert_lock_held(&vm_table_lock); 369 370 /* 371 * Initializing protected state might have failed, yet a malicious 372 * host could trigger this function. Thus, ensure that 'vm_table' 373 * exists. 374 */ 375 if (unlikely(!vm_table)) 376 return -EINVAL; 377 378 idx = find_free_vm_table_entry(host_kvm); 379 if (idx < 0) 380 return idx; 381 382 hyp_vm->kvm.arch.pkvm.handle = idx_to_vm_handle(idx); 383 384 /* VMID 0 is reserved for the host */ 385 atomic64_set(&mmu->vmid.id, idx + 1); 386 387 mmu->arch = &hyp_vm->kvm.arch; 388 mmu->pgt = &hyp_vm->pgt; 389 390 vm_table[idx] = hyp_vm; 391 return hyp_vm->kvm.arch.pkvm.handle; 392 } 393 394 /* 395 * Deallocate and remove the VM table entry corresponding to the handle. 396 */ remove_vm_table_entry(pkvm_handle_t handle)397 static void remove_vm_table_entry(pkvm_handle_t handle) 398 { 399 hyp_assert_lock_held(&vm_table_lock); 400 vm_table[vm_handle_to_idx(handle)] = NULL; 401 } 402 pkvm_get_hyp_vm_size(unsigned int nr_vcpus)403 static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus) 404 { 405 return size_add(sizeof(struct pkvm_hyp_vm), 406 size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus)); 407 } 408 map_donated_memory_noclear(unsigned long host_va,size_t size)409 static void *map_donated_memory_noclear(unsigned long host_va, size_t size) 410 { 411 void *va = (void *)kern_hyp_va(host_va); 412 413 if (!PAGE_ALIGNED(va)) 414 return NULL; 415 416 if (__pkvm_host_donate_hyp(hyp_virt_to_pfn(va), 417 PAGE_ALIGN(size) >> PAGE_SHIFT)) 418 return NULL; 419 420 return va; 421 } 422 map_donated_memory(unsigned long host_va,size_t size)423 static void *map_donated_memory(unsigned long host_va, size_t size) 424 { 425 void *va = map_donated_memory_noclear(host_va, size); 426 427 if (va) 428 memset(va, 0, size); 429 430 return va; 431 } 432 __unmap_donated_memory(void * va,size_t size)433 static void __unmap_donated_memory(void *va, size_t size) 434 { 435 kvm_flush_dcache_to_poc(va, size); 436 WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(va), 437 PAGE_ALIGN(size) >> PAGE_SHIFT)); 438 } 439 unmap_donated_memory(void * va,size_t size)440 static void unmap_donated_memory(void *va, size_t size) 441 { 442 if (!va) 443 return; 444 445 memset(va, 0, size); 446 __unmap_donated_memory(va, size); 447 } 448 unmap_donated_memory_noclear(void * va,size_t size)449 static void unmap_donated_memory_noclear(void *va, size_t size) 450 { 451 if (!va) 452 return; 453 454 __unmap_donated_memory(va, size); 455 } 456 457 /* 458 * Initialize the hypervisor copy of the protected VM state using the 459 * memory donated by the host. 460 * 461 * Unmaps the donated memory from the host at stage 2. 462 * 463 * host_kvm: A pointer to the host's struct kvm. 464 * vm_hva: The host va of the area being donated for the VM state. 465 * Must be page aligned. 466 * pgd_hva: The host va of the area being donated for the stage-2 PGD for 467 * the VM. Must be page aligned. Its size is implied by the VM's 468 * VTCR. 469 * 470 * Return a unique handle to the protected VM on success, 471 * negative error code on failure. 472 */ __pkvm_init_vm(struct kvm * host_kvm,unsigned long vm_hva,unsigned long pgd_hva)473 int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, 474 unsigned long pgd_hva) 475 { 476 struct pkvm_hyp_vm *hyp_vm = NULL; 477 size_t vm_size, pgd_size; 478 unsigned int nr_vcpus; 479 void *pgd = NULL; 480 int ret; 481 482 ret = hyp_pin_shared_mem(host_kvm, host_kvm + 1); 483 if (ret) 484 return ret; 485 486 nr_vcpus = READ_ONCE(host_kvm->created_vcpus); 487 if (nr_vcpus < 1) { 488 ret = -EINVAL; 489 goto err_unpin_kvm; 490 } 491 492 vm_size = pkvm_get_hyp_vm_size(nr_vcpus); 493 pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.mmu.vtcr); 494 495 ret = -ENOMEM; 496 497 hyp_vm = map_donated_memory(vm_hva, vm_size); 498 if (!hyp_vm) 499 goto err_remove_mappings; 500 501 pgd = map_donated_memory_noclear(pgd_hva, pgd_size); 502 if (!pgd) 503 goto err_remove_mappings; 504 505 init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus); 506 507 hyp_spin_lock(&vm_table_lock); 508 ret = insert_vm_table_entry(host_kvm, hyp_vm); 509 if (ret < 0) 510 goto err_unlock; 511 512 ret = kvm_guest_prepare_stage2(hyp_vm, pgd); 513 if (ret) 514 goto err_remove_vm_table_entry; 515 hyp_spin_unlock(&vm_table_lock); 516 517 return hyp_vm->kvm.arch.pkvm.handle; 518 519 err_remove_vm_table_entry: 520 remove_vm_table_entry(hyp_vm->kvm.arch.pkvm.handle); 521 err_unlock: 522 hyp_spin_unlock(&vm_table_lock); 523 err_remove_mappings: 524 unmap_donated_memory(hyp_vm, vm_size); 525 unmap_donated_memory(pgd, pgd_size); 526 err_unpin_kvm: 527 hyp_unpin_shared_mem(host_kvm, host_kvm + 1); 528 return ret; 529 } 530 531 /* 532 * Initialize the hypervisor copy of the protected vCPU state using the 533 * memory donated by the host. 534 * 535 * handle: The handle for the protected vm. 536 * host_vcpu: A pointer to the corresponding host vcpu. 537 * vcpu_hva: The host va of the area being donated for the vcpu state. 538 * Must be page aligned. The size of the area must be equal to 539 * the page-aligned size of 'struct pkvm_hyp_vcpu'. 540 * Return 0 on success, negative error code on failure. 541 */ __pkvm_init_vcpu(pkvm_handle_t handle,struct kvm_vcpu * host_vcpu,unsigned long vcpu_hva)542 int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, 543 unsigned long vcpu_hva) 544 { 545 struct pkvm_hyp_vcpu *hyp_vcpu; 546 struct pkvm_hyp_vm *hyp_vm; 547 unsigned int idx; 548 int ret; 549 550 hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu)); 551 if (!hyp_vcpu) 552 return -ENOMEM; 553 554 hyp_spin_lock(&vm_table_lock); 555 556 hyp_vm = get_vm_by_handle(handle); 557 if (!hyp_vm) { 558 ret = -ENOENT; 559 goto unlock; 560 } 561 562 idx = hyp_vm->nr_vcpus; 563 if (idx >= hyp_vm->kvm.created_vcpus) { 564 ret = -EINVAL; 565 goto unlock; 566 } 567 568 ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx); 569 if (ret) 570 goto unlock; 571 572 hyp_vm->vcpus[idx] = hyp_vcpu; 573 hyp_vm->nr_vcpus++; 574 unlock: 575 hyp_spin_unlock(&vm_table_lock); 576 577 if (ret) { 578 unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu)); 579 return ret; 580 } 581 582 hyp_vcpu->vcpu.arch.cptr_el2 = kvm_get_reset_cptr_el2(&hyp_vcpu->vcpu); 583 584 return 0; 585 } 586 587 static void teardown_donated_memory(struct kvm_hyp_memcache * mc,void * addr,size_t size)588 teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size) 589 { 590 size = PAGE_ALIGN(size); 591 memset(addr, 0, size); 592 593 for (void *start = addr; start < addr + size; start += PAGE_SIZE) 594 push_hyp_memcache(mc, start, hyp_virt_to_phys); 595 596 unmap_donated_memory_noclear(addr, size); 597 } 598 __pkvm_teardown_vm(pkvm_handle_t handle)599 int __pkvm_teardown_vm(pkvm_handle_t handle) 600 { 601 struct kvm_hyp_memcache *mc; 602 struct pkvm_hyp_vm *hyp_vm; 603 struct kvm *host_kvm; 604 unsigned int idx; 605 size_t vm_size; 606 int err; 607 608 hyp_spin_lock(&vm_table_lock); 609 hyp_vm = get_vm_by_handle(handle); 610 if (!hyp_vm) { 611 err = -ENOENT; 612 goto err_unlock; 613 } 614 615 if (WARN_ON(hyp_page_count(hyp_vm))) { 616 err = -EBUSY; 617 goto err_unlock; 618 } 619 620 host_kvm = hyp_vm->host_kvm; 621 622 /* Ensure the VMID is clean before it can be reallocated */ 623 __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu); 624 remove_vm_table_entry(handle); 625 hyp_spin_unlock(&vm_table_lock); 626 627 /* Reclaim guest pages (including page-table pages) */ 628 mc = &host_kvm->arch.pkvm.teardown_mc; 629 reclaim_guest_pages(hyp_vm, mc); 630 unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus); 631 632 /* Push the metadata pages to the teardown memcache */ 633 for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) { 634 struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; 635 636 teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu)); 637 } 638 639 vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus); 640 teardown_donated_memory(mc, hyp_vm, vm_size); 641 hyp_unpin_shared_mem(host_kvm, host_kvm + 1); 642 return 0; 643 644 err_unlock: 645 hyp_spin_unlock(&vm_table_lock); 646 return err; 647 } 648