1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2011. Freescale Inc. All rights reserved. 4 * 5 * Authors: 6 * Alexander Graf <agraf@suse.de> 7 * Paul Mackerras <paulus@samba.org> 8 * 9 * Description: 10 * 11 * Hypercall handling for running PAPR guests in PR KVM on Book 3S 12 * processors. 13 */ 14 15 #include <linux/anon_inodes.h> 16 17 #include <linux/uaccess.h> 18 #include <asm/kvm_ppc.h> 19 #include <asm/kvm_book3s.h> 20 21 #define HPTE_SIZE 16 /* bytes per HPT entry */ 22 get_pteg_addr(struct kvm_vcpu * vcpu,long pte_index)23 static unsigned long get_pteg_addr(struct kvm_vcpu *vcpu, long pte_index) 24 { 25 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu); 26 unsigned long pteg_addr; 27 28 pte_index <<= 4; 29 pte_index &= ((1 << ((vcpu_book3s->sdr1 & 0x1f) + 11)) - 1) << 7 | 0x70; 30 pteg_addr = vcpu_book3s->sdr1 & 0xfffffffffffc0000ULL; 31 pteg_addr |= pte_index; 32 33 return pteg_addr; 34 } 35 kvmppc_h_pr_enter(struct kvm_vcpu * vcpu)36 static int kvmppc_h_pr_enter(struct kvm_vcpu *vcpu) 37 { 38 long flags = kvmppc_get_gpr(vcpu, 4); 39 long pte_index = kvmppc_get_gpr(vcpu, 5); 40 __be64 pteg[2 * 8]; 41 __be64 *hpte; 42 unsigned long pteg_addr, i; 43 long int ret; 44 45 i = pte_index & 7; 46 pte_index &= ~7UL; 47 pteg_addr = get_pteg_addr(vcpu, pte_index); 48 49 mutex_lock(&vcpu->kvm->arch.hpt_mutex); 50 ret = H_FUNCTION; 51 if (copy_from_user(pteg, (void __user *)pteg_addr, sizeof(pteg))) 52 goto done; 53 hpte = pteg; 54 55 ret = H_PTEG_FULL; 56 if (likely((flags & H_EXACT) == 0)) { 57 for (i = 0; ; ++i) { 58 if (i == 8) 59 goto done; 60 if ((be64_to_cpu(*hpte) & HPTE_V_VALID) == 0) 61 break; 62 hpte += 2; 63 } 64 } else { 65 hpte += i * 2; 66 if (*hpte & HPTE_V_VALID) 67 goto done; 68 } 69 70 hpte[0] = cpu_to_be64(kvmppc_get_gpr(vcpu, 6)); 71 hpte[1] = cpu_to_be64(kvmppc_get_gpr(vcpu, 7)); 72 pteg_addr += i * HPTE_SIZE; 73 ret = H_FUNCTION; 74 if (copy_to_user((void __user *)pteg_addr, hpte, HPTE_SIZE)) 75 goto done; 76 kvmppc_set_gpr(vcpu, 4, pte_index | i); 77 ret = H_SUCCESS; 78 79 done: 80 mutex_unlock(&vcpu->kvm->arch.hpt_mutex); 81 kvmppc_set_gpr(vcpu, 3, ret); 82 83 return EMULATE_DONE; 84 } 85 kvmppc_h_pr_remove(struct kvm_vcpu * vcpu)86 static int kvmppc_h_pr_remove(struct kvm_vcpu *vcpu) 87 { 88 unsigned long flags= kvmppc_get_gpr(vcpu, 4); 89 unsigned long pte_index = kvmppc_get_gpr(vcpu, 5); 90 unsigned long avpn = kvmppc_get_gpr(vcpu, 6); 91 unsigned long v = 0, pteg, rb; 92 unsigned long pte[2]; 93 long int ret; 94 95 pteg = get_pteg_addr(vcpu, pte_index); 96 mutex_lock(&vcpu->kvm->arch.hpt_mutex); 97 ret = H_FUNCTION; 98 if (copy_from_user(pte, (void __user *)pteg, sizeof(pte))) 99 goto done; 100 pte[0] = be64_to_cpu((__force __be64)pte[0]); 101 pte[1] = be64_to_cpu((__force __be64)pte[1]); 102 103 ret = H_NOT_FOUND; 104 if ((pte[0] & HPTE_V_VALID) == 0 || 105 ((flags & H_AVPN) && (pte[0] & ~0x7fUL) != avpn) || 106 ((flags & H_ANDCOND) && (pte[0] & avpn) != 0)) 107 goto done; 108 109 ret = H_FUNCTION; 110 if (copy_to_user((void __user *)pteg, &v, sizeof(v))) 111 goto done; 112 113 rb = compute_tlbie_rb(pte[0], pte[1], pte_index); 114 vcpu->arch.mmu.tlbie(vcpu, rb, rb & 1 ? true : false); 115 116 ret = H_SUCCESS; 117 kvmppc_set_gpr(vcpu, 4, pte[0]); 118 kvmppc_set_gpr(vcpu, 5, pte[1]); 119 120 done: 121 mutex_unlock(&vcpu->kvm->arch.hpt_mutex); 122 kvmppc_set_gpr(vcpu, 3, ret); 123 124 return EMULATE_DONE; 125 } 126 127 /* Request defs for kvmppc_h_pr_bulk_remove() */ 128 #define H_BULK_REMOVE_TYPE 0xc000000000000000ULL 129 #define H_BULK_REMOVE_REQUEST 0x4000000000000000ULL 130 #define H_BULK_REMOVE_RESPONSE 0x8000000000000000ULL 131 #define H_BULK_REMOVE_END 0xc000000000000000ULL 132 #define H_BULK_REMOVE_CODE 0x3000000000000000ULL 133 #define H_BULK_REMOVE_SUCCESS 0x0000000000000000ULL 134 #define H_BULK_REMOVE_NOT_FOUND 0x1000000000000000ULL 135 #define H_BULK_REMOVE_PARM 0x2000000000000000ULL 136 #define H_BULK_REMOVE_HW 0x3000000000000000ULL 137 #define H_BULK_REMOVE_RC 0x0c00000000000000ULL 138 #define H_BULK_REMOVE_FLAGS 0x0300000000000000ULL 139 #define H_BULK_REMOVE_ABSOLUTE 0x0000000000000000ULL 140 #define H_BULK_REMOVE_ANDCOND 0x0100000000000000ULL 141 #define H_BULK_REMOVE_AVPN 0x0200000000000000ULL 142 #define H_BULK_REMOVE_PTEX 0x00ffffffffffffffULL 143 #define H_BULK_REMOVE_MAX_BATCH 4 144 kvmppc_h_pr_bulk_remove(struct kvm_vcpu * vcpu)145 static int kvmppc_h_pr_bulk_remove(struct kvm_vcpu *vcpu) 146 { 147 int i; 148 int paramnr = 4; 149 int ret = H_SUCCESS; 150 151 mutex_lock(&vcpu->kvm->arch.hpt_mutex); 152 for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) { 153 unsigned long tsh = kvmppc_get_gpr(vcpu, paramnr+(2*i)); 154 unsigned long tsl = kvmppc_get_gpr(vcpu, paramnr+(2*i)+1); 155 unsigned long pteg, rb, flags; 156 unsigned long pte[2]; 157 unsigned long v = 0; 158 159 if ((tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) { 160 break; /* Exit success */ 161 } else if ((tsh & H_BULK_REMOVE_TYPE) != 162 H_BULK_REMOVE_REQUEST) { 163 ret = H_PARAMETER; 164 break; /* Exit fail */ 165 } 166 167 tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS; 168 tsh |= H_BULK_REMOVE_RESPONSE; 169 170 if ((tsh & H_BULK_REMOVE_ANDCOND) && 171 (tsh & H_BULK_REMOVE_AVPN)) { 172 tsh |= H_BULK_REMOVE_PARM; 173 kvmppc_set_gpr(vcpu, paramnr+(2*i), tsh); 174 ret = H_PARAMETER; 175 break; /* Exit fail */ 176 } 177 178 pteg = get_pteg_addr(vcpu, tsh & H_BULK_REMOVE_PTEX); 179 if (copy_from_user(pte, (void __user *)pteg, sizeof(pte))) { 180 ret = H_FUNCTION; 181 break; 182 } 183 pte[0] = be64_to_cpu((__force __be64)pte[0]); 184 pte[1] = be64_to_cpu((__force __be64)pte[1]); 185 186 /* tsl = AVPN */ 187 flags = (tsh & H_BULK_REMOVE_FLAGS) >> 26; 188 189 if ((pte[0] & HPTE_V_VALID) == 0 || 190 ((flags & H_AVPN) && (pte[0] & ~0x7fUL) != tsl) || 191 ((flags & H_ANDCOND) && (pte[0] & tsl) != 0)) { 192 tsh |= H_BULK_REMOVE_NOT_FOUND; 193 } else { 194 /* Splat the pteg in (userland) hpt */ 195 if (copy_to_user((void __user *)pteg, &v, sizeof(v))) { 196 ret = H_FUNCTION; 197 break; 198 } 199 200 rb = compute_tlbie_rb(pte[0], pte[1], 201 tsh & H_BULK_REMOVE_PTEX); 202 vcpu->arch.mmu.tlbie(vcpu, rb, rb & 1 ? true : false); 203 tsh |= H_BULK_REMOVE_SUCCESS; 204 tsh |= (pte[1] & (HPTE_R_C | HPTE_R_R)) << 43; 205 } 206 kvmppc_set_gpr(vcpu, paramnr+(2*i), tsh); 207 } 208 mutex_unlock(&vcpu->kvm->arch.hpt_mutex); 209 kvmppc_set_gpr(vcpu, 3, ret); 210 211 return EMULATE_DONE; 212 } 213 kvmppc_h_pr_protect(struct kvm_vcpu * vcpu)214 static int kvmppc_h_pr_protect(struct kvm_vcpu *vcpu) 215 { 216 unsigned long flags = kvmppc_get_gpr(vcpu, 4); 217 unsigned long pte_index = kvmppc_get_gpr(vcpu, 5); 218 unsigned long avpn = kvmppc_get_gpr(vcpu, 6); 219 unsigned long rb, pteg, r, v; 220 unsigned long pte[2]; 221 long int ret; 222 223 pteg = get_pteg_addr(vcpu, pte_index); 224 mutex_lock(&vcpu->kvm->arch.hpt_mutex); 225 ret = H_FUNCTION; 226 if (copy_from_user(pte, (void __user *)pteg, sizeof(pte))) 227 goto done; 228 pte[0] = be64_to_cpu((__force __be64)pte[0]); 229 pte[1] = be64_to_cpu((__force __be64)pte[1]); 230 231 ret = H_NOT_FOUND; 232 if ((pte[0] & HPTE_V_VALID) == 0 || 233 ((flags & H_AVPN) && (pte[0] & ~0x7fUL) != avpn)) 234 goto done; 235 236 v = pte[0]; 237 r = pte[1]; 238 r &= ~(HPTE_R_PP0 | HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_HI | 239 HPTE_R_KEY_LO); 240 r |= (flags << 55) & HPTE_R_PP0; 241 r |= (flags << 48) & HPTE_R_KEY_HI; 242 r |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO); 243 244 pte[1] = r; 245 246 rb = compute_tlbie_rb(v, r, pte_index); 247 vcpu->arch.mmu.tlbie(vcpu, rb, rb & 1 ? true : false); 248 pte[0] = (__force u64)cpu_to_be64(pte[0]); 249 pte[1] = (__force u64)cpu_to_be64(pte[1]); 250 ret = H_FUNCTION; 251 if (copy_to_user((void __user *)pteg, pte, sizeof(pte))) 252 goto done; 253 ret = H_SUCCESS; 254 255 done: 256 mutex_unlock(&vcpu->kvm->arch.hpt_mutex); 257 kvmppc_set_gpr(vcpu, 3, ret); 258 259 return EMULATE_DONE; 260 } 261 kvmppc_h_pr_logical_ci_load(struct kvm_vcpu * vcpu)262 static int kvmppc_h_pr_logical_ci_load(struct kvm_vcpu *vcpu) 263 { 264 long rc; 265 266 rc = kvmppc_h_logical_ci_load(vcpu); 267 if (rc == H_TOO_HARD) 268 return EMULATE_FAIL; 269 kvmppc_set_gpr(vcpu, 3, rc); 270 return EMULATE_DONE; 271 } 272 kvmppc_h_pr_logical_ci_store(struct kvm_vcpu * vcpu)273 static int kvmppc_h_pr_logical_ci_store(struct kvm_vcpu *vcpu) 274 { 275 long rc; 276 277 rc = kvmppc_h_logical_ci_store(vcpu); 278 if (rc == H_TOO_HARD) 279 return EMULATE_FAIL; 280 kvmppc_set_gpr(vcpu, 3, rc); 281 return EMULATE_DONE; 282 } 283 kvmppc_h_pr_set_mode(struct kvm_vcpu * vcpu)284 static int kvmppc_h_pr_set_mode(struct kvm_vcpu *vcpu) 285 { 286 unsigned long mflags = kvmppc_get_gpr(vcpu, 4); 287 unsigned long resource = kvmppc_get_gpr(vcpu, 5); 288 289 if (resource == H_SET_MODE_RESOURCE_ADDR_TRANS_MODE) { 290 /* KVM PR does not provide AIL!=0 to guests */ 291 if (mflags == 0) 292 kvmppc_set_gpr(vcpu, 3, H_SUCCESS); 293 else 294 kvmppc_set_gpr(vcpu, 3, H_UNSUPPORTED_FLAG_START - 63); 295 return EMULATE_DONE; 296 } 297 return EMULATE_FAIL; 298 } 299 300 #ifdef CONFIG_SPAPR_TCE_IOMMU kvmppc_h_pr_put_tce(struct kvm_vcpu * vcpu)301 static int kvmppc_h_pr_put_tce(struct kvm_vcpu *vcpu) 302 { 303 unsigned long liobn = kvmppc_get_gpr(vcpu, 4); 304 unsigned long ioba = kvmppc_get_gpr(vcpu, 5); 305 unsigned long tce = kvmppc_get_gpr(vcpu, 6); 306 long rc; 307 308 rc = kvmppc_h_put_tce(vcpu, liobn, ioba, tce); 309 if (rc == H_TOO_HARD) 310 return EMULATE_FAIL; 311 kvmppc_set_gpr(vcpu, 3, rc); 312 return EMULATE_DONE; 313 } 314 kvmppc_h_pr_put_tce_indirect(struct kvm_vcpu * vcpu)315 static int kvmppc_h_pr_put_tce_indirect(struct kvm_vcpu *vcpu) 316 { 317 unsigned long liobn = kvmppc_get_gpr(vcpu, 4); 318 unsigned long ioba = kvmppc_get_gpr(vcpu, 5); 319 unsigned long tce = kvmppc_get_gpr(vcpu, 6); 320 unsigned long npages = kvmppc_get_gpr(vcpu, 7); 321 long rc; 322 323 rc = kvmppc_h_put_tce_indirect(vcpu, liobn, ioba, 324 tce, npages); 325 if (rc == H_TOO_HARD) 326 return EMULATE_FAIL; 327 kvmppc_set_gpr(vcpu, 3, rc); 328 return EMULATE_DONE; 329 } 330 kvmppc_h_pr_stuff_tce(struct kvm_vcpu * vcpu)331 static int kvmppc_h_pr_stuff_tce(struct kvm_vcpu *vcpu) 332 { 333 unsigned long liobn = kvmppc_get_gpr(vcpu, 4); 334 unsigned long ioba = kvmppc_get_gpr(vcpu, 5); 335 unsigned long tce_value = kvmppc_get_gpr(vcpu, 6); 336 unsigned long npages = kvmppc_get_gpr(vcpu, 7); 337 long rc; 338 339 rc = kvmppc_h_stuff_tce(vcpu, liobn, ioba, tce_value, npages); 340 if (rc == H_TOO_HARD) 341 return EMULATE_FAIL; 342 kvmppc_set_gpr(vcpu, 3, rc); 343 return EMULATE_DONE; 344 } 345 346 #else /* CONFIG_SPAPR_TCE_IOMMU */ kvmppc_h_pr_put_tce(struct kvm_vcpu * vcpu)347 static int kvmppc_h_pr_put_tce(struct kvm_vcpu *vcpu) 348 { 349 return EMULATE_FAIL; 350 } 351 kvmppc_h_pr_put_tce_indirect(struct kvm_vcpu * vcpu)352 static int kvmppc_h_pr_put_tce_indirect(struct kvm_vcpu *vcpu) 353 { 354 return EMULATE_FAIL; 355 } 356 kvmppc_h_pr_stuff_tce(struct kvm_vcpu * vcpu)357 static int kvmppc_h_pr_stuff_tce(struct kvm_vcpu *vcpu) 358 { 359 return EMULATE_FAIL; 360 } 361 #endif /* CONFIG_SPAPR_TCE_IOMMU */ 362 kvmppc_h_pr_xics_hcall(struct kvm_vcpu * vcpu,u32 cmd)363 static int kvmppc_h_pr_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd) 364 { 365 long rc = kvmppc_xics_hcall(vcpu, cmd); 366 kvmppc_set_gpr(vcpu, 3, rc); 367 return EMULATE_DONE; 368 } 369 kvmppc_h_pr(struct kvm_vcpu * vcpu,unsigned long cmd)370 int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd) 371 { 372 int rc, idx; 373 374 if (cmd <= MAX_HCALL_OPCODE && 375 !test_bit(cmd/4, vcpu->kvm->arch.enabled_hcalls)) 376 return EMULATE_FAIL; 377 378 switch (cmd) { 379 case H_ENTER: 380 return kvmppc_h_pr_enter(vcpu); 381 case H_REMOVE: 382 return kvmppc_h_pr_remove(vcpu); 383 case H_PROTECT: 384 return kvmppc_h_pr_protect(vcpu); 385 case H_BULK_REMOVE: 386 return kvmppc_h_pr_bulk_remove(vcpu); 387 case H_PUT_TCE: 388 return kvmppc_h_pr_put_tce(vcpu); 389 case H_PUT_TCE_INDIRECT: 390 return kvmppc_h_pr_put_tce_indirect(vcpu); 391 case H_STUFF_TCE: 392 return kvmppc_h_pr_stuff_tce(vcpu); 393 case H_CEDE: 394 kvmppc_set_msr_fast(vcpu, kvmppc_get_msr(vcpu) | MSR_EE); 395 kvm_vcpu_halt(vcpu); 396 vcpu->stat.generic.halt_wakeup++; 397 return EMULATE_DONE; 398 case H_LOGICAL_CI_LOAD: 399 return kvmppc_h_pr_logical_ci_load(vcpu); 400 case H_LOGICAL_CI_STORE: 401 return kvmppc_h_pr_logical_ci_store(vcpu); 402 case H_SET_MODE: 403 return kvmppc_h_pr_set_mode(vcpu); 404 case H_XIRR: 405 case H_CPPR: 406 case H_EOI: 407 case H_IPI: 408 case H_IPOLL: 409 case H_XIRR_X: 410 if (kvmppc_xics_enabled(vcpu)) 411 return kvmppc_h_pr_xics_hcall(vcpu, cmd); 412 break; 413 case H_RTAS: 414 if (list_empty(&vcpu->kvm->arch.rtas_tokens)) 415 break; 416 idx = srcu_read_lock(&vcpu->kvm->srcu); 417 rc = kvmppc_rtas_hcall(vcpu); 418 srcu_read_unlock(&vcpu->kvm->srcu, idx); 419 if (rc) 420 break; 421 kvmppc_set_gpr(vcpu, 3, 0); 422 return EMULATE_DONE; 423 } 424 425 return EMULATE_FAIL; 426 } 427 kvmppc_hcall_impl_pr(unsigned long cmd)428 int kvmppc_hcall_impl_pr(unsigned long cmd) 429 { 430 switch (cmd) { 431 case H_ENTER: 432 case H_REMOVE: 433 case H_PROTECT: 434 case H_BULK_REMOVE: 435 #ifdef CONFIG_SPAPR_TCE_IOMMU 436 case H_GET_TCE: 437 case H_PUT_TCE: 438 case H_PUT_TCE_INDIRECT: 439 case H_STUFF_TCE: 440 #endif 441 case H_CEDE: 442 case H_LOGICAL_CI_LOAD: 443 case H_LOGICAL_CI_STORE: 444 case H_SET_MODE: 445 #ifdef CONFIG_KVM_XICS 446 case H_XIRR: 447 case H_CPPR: 448 case H_EOI: 449 case H_IPI: 450 case H_IPOLL: 451 case H_XIRR_X: 452 #endif 453 return 1; 454 } 455 return 0; 456 } 457 458 /* 459 * List of hcall numbers to enable by default. 460 * For compatibility with old userspace, we enable by default 461 * all hcalls that were implemented before the hcall-enabling 462 * facility was added. Note this list should not include H_RTAS. 463 */ 464 static unsigned int default_hcall_list[] = { 465 H_ENTER, 466 H_REMOVE, 467 H_PROTECT, 468 H_BULK_REMOVE, 469 #ifdef CONFIG_SPAPR_TCE_IOMMU 470 H_GET_TCE, 471 H_PUT_TCE, 472 #endif 473 H_CEDE, 474 H_SET_MODE, 475 #ifdef CONFIG_KVM_XICS 476 H_XIRR, 477 H_CPPR, 478 H_EOI, 479 H_IPI, 480 H_IPOLL, 481 H_XIRR_X, 482 #endif 483 0 484 }; 485 kvmppc_pr_init_default_hcalls(struct kvm * kvm)486 void kvmppc_pr_init_default_hcalls(struct kvm *kvm) 487 { 488 int i; 489 unsigned int hcall; 490 491 for (i = 0; default_hcall_list[i]; ++i) { 492 hcall = default_hcall_list[i]; 493 WARN_ON(!kvmppc_hcall_impl_pr(hcall)); 494 __set_bit(hcall / 4, kvm->arch.enabled_hcalls); 495 } 496 } 497