Lines Matching refs:kvm
47 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
53 struct kvm *kvm; member
118 void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info) in kvmppc_set_hpt() argument
120 atomic64_set(&kvm->arch.mmio_update, 0); in kvmppc_set_hpt()
121 kvm->arch.hpt = *info; in kvmppc_set_hpt()
122 kvm->arch.sdr1 = __pa(info->virt) | (info->order - 18); in kvmppc_set_hpt()
125 info->virt, (long)info->order, kvm->arch.lpid); in kvmppc_set_hpt()
128 int kvmppc_alloc_reset_hpt(struct kvm *kvm, int order) in kvmppc_alloc_reset_hpt() argument
133 mutex_lock(&kvm->arch.mmu_setup_lock); in kvmppc_alloc_reset_hpt()
134 if (kvm->arch.mmu_ready) { in kvmppc_alloc_reset_hpt()
135 kvm->arch.mmu_ready = 0; in kvmppc_alloc_reset_hpt()
138 if (atomic_read(&kvm->arch.vcpus_running)) { in kvmppc_alloc_reset_hpt()
139 kvm->arch.mmu_ready = 1; in kvmppc_alloc_reset_hpt()
143 if (kvm_is_radix(kvm)) { in kvmppc_alloc_reset_hpt()
144 err = kvmppc_switch_mmu_to_hpt(kvm); in kvmppc_alloc_reset_hpt()
149 if (kvm->arch.hpt.order == order) { in kvmppc_alloc_reset_hpt()
153 memset((void *)kvm->arch.hpt.virt, 0, 1ul << order); in kvmppc_alloc_reset_hpt()
157 kvmppc_rmap_reset(kvm); in kvmppc_alloc_reset_hpt()
162 if (kvm->arch.hpt.virt) { in kvmppc_alloc_reset_hpt()
163 kvmppc_free_hpt(&kvm->arch.hpt); in kvmppc_alloc_reset_hpt()
164 kvmppc_rmap_reset(kvm); in kvmppc_alloc_reset_hpt()
170 kvmppc_set_hpt(kvm, &info); in kvmppc_alloc_reset_hpt()
175 cpumask_setall(&kvm->arch.need_tlb_flush); in kvmppc_alloc_reset_hpt()
177 mutex_unlock(&kvm->arch.mmu_setup_lock); in kvmppc_alloc_reset_hpt()
217 struct kvm *kvm = vcpu->kvm; in kvmppc_map_vrma() local
226 if (npages > kvmppc_hpt_mask(&kvm->arch.hpt) + 1) in kvmppc_map_vrma()
227 npages = kvmppc_hpt_mask(&kvm->arch.hpt) + 1; in kvmppc_map_vrma()
238 & kvmppc_hpt_mask(&kvm->arch.hpt); in kvmppc_map_vrma()
248 ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r, in kvmppc_map_vrma()
292 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags, in kvmppc_virtmode_do_h_enter() argument
299 ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel, in kvmppc_virtmode_do_h_enter()
300 kvm->mm->pgd, false, pte_idx_ret); in kvmppc_virtmode_do_h_enter()
344 struct kvm *kvm = vcpu->kvm; in kvmppc_mmu_book3s_64_hv_xlate() local
353 if (kvm_is_radix(vcpu->kvm)) in kvmppc_mmu_book3s_64_hv_xlate()
364 slb_v = vcpu->kvm->arch.vrma_slb_v; in kvmppc_mmu_book3s_64_hv_xlate()
369 index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v, in kvmppc_mmu_book3s_64_hv_xlate()
375 hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4)); in kvmppc_mmu_book3s_64_hv_xlate()
379 gr = kvm->arch.hpt.rev[index].guest_rpte; in kvmppc_mmu_book3s_64_hv_xlate()
447 idx = srcu_read_lock(&vcpu->kvm->srcu); in kvmppc_hv_emulate_mmio()
450 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvmppc_hv_emulate_mmio()
509 struct kvm *kvm = vcpu->kvm; in kvmppc_book3s_hv_page_fault() local
528 if (kvm_is_radix(kvm)) in kvmppc_book3s_hv_page_fault()
541 mmio_update = atomic64_read(&kvm->arch.mmio_update); in kvmppc_book3s_hv_page_fault()
554 hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4)); in kvmppc_book3s_hv_page_fault()
555 rev = &kvm->arch.hpt.rev[index]; in kvmppc_book3s_hv_page_fault()
579 memslot = gfn_to_memslot(kvm, gfn); in kvmppc_book3s_hv_page_fault()
596 mmu_seq = kvm->mmu_invalidate_seq; in kvmppc_book3s_hv_page_fault()
632 spin_lock(&kvm->mmu_lock); in kvmppc_book3s_hv_page_fault()
633 ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift); in kvmppc_book3s_hv_page_fault()
637 spin_unlock(&kvm->mmu_lock); in kvmppc_book3s_hv_page_fault()
696 if (!kvm->arch.mmu_ready) in kvmppc_book3s_hv_page_fault()
711 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) { in kvmppc_book3s_hv_page_fault()
724 kvmppc_invalidate_hpte(kvm, hptep, index); in kvmppc_book3s_hv_page_fault()
728 kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0); in kvmppc_book3s_hv_page_fault()
756 void kvmppc_rmap_reset(struct kvm *kvm) in kvmppc_rmap_reset() argument
762 srcu_idx = srcu_read_lock(&kvm->srcu); in kvmppc_rmap_reset()
763 slots = kvm_memslots(kvm); in kvmppc_rmap_reset()
766 spin_lock(&kvm->mmu_lock); in kvmppc_rmap_reset()
773 spin_unlock(&kvm->mmu_lock); in kvmppc_rmap_reset()
775 srcu_read_unlock(&kvm->srcu, srcu_idx); in kvmppc_rmap_reset()
779 static void kvmppc_unmap_hpte(struct kvm *kvm, unsigned long i, in kvmppc_unmap_hpte() argument
783 __be64 *hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4)); in kvmppc_unmap_hpte()
784 struct revmap_entry *rev = kvm->arch.hpt.rev; in kvmppc_unmap_hpte()
807 kvmppc_invalidate_hpte(kvm, hptep, i); in kvmppc_unmap_hpte()
816 note_hpte_modification(kvm, &rev[i]); in kvmppc_unmap_hpte()
821 static void kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot, in kvm_unmap_rmapp() argument
842 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4)); in kvm_unmap_rmapp()
851 kvmppc_unmap_hpte(kvm, i, memslot, rmapp, gfn); in kvm_unmap_rmapp()
857 bool kvm_unmap_gfn_range_hv(struct kvm *kvm, struct kvm_gfn_range *range) in kvm_unmap_gfn_range_hv() argument
861 if (kvm_is_radix(kvm)) { in kvm_unmap_gfn_range_hv()
863 kvm_unmap_radix(kvm, range->slot, gfn); in kvm_unmap_gfn_range_hv()
866 kvm_unmap_rmapp(kvm, range->slot, gfn); in kvm_unmap_gfn_range_hv()
872 void kvmppc_core_flush_memslot_hv(struct kvm *kvm, in kvmppc_core_flush_memslot_hv() argument
881 if (kvm_is_radix(kvm)) { in kvmppc_core_flush_memslot_hv()
882 kvmppc_radix_flush_memslot(kvm, memslot); in kvmppc_core_flush_memslot_hv()
894 kvm_unmap_rmapp(kvm, memslot, gfn); in kvmppc_core_flush_memslot_hv()
899 static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot, in kvm_age_rmapp() argument
902 struct revmap_entry *rev = kvm->arch.hpt.rev; in kvm_age_rmapp()
922 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4)); in kvm_age_rmapp()
940 kvmppc_clear_ref_hpte(kvm, hptep, i); in kvm_age_rmapp()
943 note_hpte_modification(kvm, &rev[i]); in kvm_age_rmapp()
954 bool kvm_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range) in kvm_age_gfn_hv() argument
959 if (kvm_is_radix(kvm)) { in kvm_age_gfn_hv()
961 ret |= kvm_age_radix(kvm, range->slot, gfn); in kvm_age_gfn_hv()
964 ret |= kvm_age_rmapp(kvm, range->slot, gfn); in kvm_age_gfn_hv()
970 static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot, in kvm_test_age_rmapp() argument
973 struct revmap_entry *rev = kvm->arch.hpt.rev; in kvm_test_age_rmapp()
990 hp = (unsigned long *)(kvm->arch.hpt.virt + (i << 4)); in kvm_test_age_rmapp()
1003 bool kvm_test_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range) in kvm_test_age_gfn_hv() argument
1007 if (kvm_is_radix(kvm)) in kvm_test_age_gfn_hv()
1008 return kvm_test_age_radix(kvm, range->slot, range->start); in kvm_test_age_gfn_hv()
1010 return kvm_test_age_rmapp(kvm, range->slot, range->start); in kvm_test_age_gfn_hv()
1013 static int vcpus_running(struct kvm *kvm) in vcpus_running() argument
1015 return atomic_read(&kvm->arch.vcpus_running) != 0; in vcpus_running()
1022 static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp) in kvm_test_clear_dirty_npages() argument
1024 struct revmap_entry *rev = kvm->arch.hpt.rev; in kvm_test_clear_dirty_npages()
1041 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4)); in kvm_test_clear_dirty_npages()
1060 (!hpte_is_writable(hptep1) || vcpus_running(kvm))) in kvm_test_clear_dirty_npages()
1079 kvmppc_invalidate_hpte(kvm, hptep, i); in kvm_test_clear_dirty_npages()
1086 note_hpte_modification(kvm, &rev[i]); in kvm_test_clear_dirty_npages()
1121 long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm, in kvmppc_hv_get_dirty_log_hpt() argument
1130 int npages = kvm_test_clear_dirty_npages(kvm, rmapp); in kvmppc_hv_get_dirty_log_hpt()
1144 void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa, in kvmppc_pin_guest_page() argument
1154 srcu_idx = srcu_read_lock(&kvm->srcu); in kvmppc_pin_guest_page()
1155 memslot = gfn_to_memslot(kvm, gfn); in kvmppc_pin_guest_page()
1163 srcu_read_unlock(&kvm->srcu, srcu_idx); in kvmppc_pin_guest_page()
1171 srcu_read_unlock(&kvm->srcu, srcu_idx); in kvmppc_pin_guest_page()
1175 void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa, in kvmppc_unpin_guest_page() argument
1190 srcu_idx = srcu_read_lock(&kvm->srcu); in kvmppc_unpin_guest_page()
1191 memslot = gfn_to_memslot(kvm, gfn); in kvmppc_unpin_guest_page()
1194 srcu_read_unlock(&kvm->srcu, srcu_idx); in kvmppc_unpin_guest_page()
1217 struct kvm *kvm = resize->kvm; in resize_hpt_rehash_hpte() local
1218 struct kvm_hpt_info *old = &kvm->arch.hpt; in resize_hpt_rehash_hpte()
1265 int srcu_idx = srcu_read_lock(&kvm->srcu); in resize_hpt_rehash_hpte()
1267 __gfn_to_memslot(kvm_memslots(kvm), gfn); in resize_hpt_rehash_hpte()
1274 kvmppc_unmap_hpte(kvm, idx, memslot, rmapp, gfn); in resize_hpt_rehash_hpte()
1278 srcu_read_unlock(&kvm->srcu, srcu_idx); in resize_hpt_rehash_hpte()
1371 struct kvm *kvm = resize->kvm; in resize_hpt_rehash() local
1375 for (i = 0; i < kvmppc_hpt_npte(&kvm->arch.hpt); i++) { in resize_hpt_rehash()
1386 struct kvm *kvm = resize->kvm; in resize_hpt_pivot() local
1394 spin_lock(&kvm->mmu_lock); in resize_hpt_pivot()
1397 hpt_tmp = kvm->arch.hpt; in resize_hpt_pivot()
1398 kvmppc_set_hpt(kvm, &resize->hpt); in resize_hpt_pivot()
1401 spin_unlock(&kvm->mmu_lock); in resize_hpt_pivot()
1403 synchronize_srcu_expedited(&kvm->srcu); in resize_hpt_pivot()
1406 kvmppc_setup_partition_table(kvm); in resize_hpt_pivot()
1411 static void resize_hpt_release(struct kvm *kvm, struct kvm_resize_hpt *resize) in resize_hpt_release() argument
1413 if (WARN_ON(!mutex_is_locked(&kvm->arch.mmu_setup_lock))) in resize_hpt_release()
1425 if (kvm->arch.resize_hpt == resize) in resize_hpt_release()
1426 kvm->arch.resize_hpt = NULL; in resize_hpt_release()
1434 struct kvm *kvm = resize->kvm; in resize_hpt_prepare_work() local
1440 mutex_lock(&kvm->arch.mmu_setup_lock); in resize_hpt_prepare_work()
1443 if (kvm->arch.resize_hpt == resize) { in resize_hpt_prepare_work()
1447 mutex_unlock(&kvm->arch.mmu_setup_lock); in resize_hpt_prepare_work()
1460 mutex_lock(&kvm->arch.mmu_setup_lock); in resize_hpt_prepare_work()
1468 if (kvm->arch.resize_hpt != resize) in resize_hpt_prepare_work()
1469 resize_hpt_release(kvm, resize); in resize_hpt_prepare_work()
1471 mutex_unlock(&kvm->arch.mmu_setup_lock); in resize_hpt_prepare_work()
1474 int kvm_vm_ioctl_resize_hpt_prepare(struct kvm *kvm, in kvm_vm_ioctl_resize_hpt_prepare() argument
1482 if (flags != 0 || kvm_is_radix(kvm)) in kvm_vm_ioctl_resize_hpt_prepare()
1488 mutex_lock(&kvm->arch.mmu_setup_lock); in kvm_vm_ioctl_resize_hpt_prepare()
1490 resize = kvm->arch.resize_hpt; in kvm_vm_ioctl_resize_hpt_prepare()
1499 resize_hpt_release(kvm, resize); in kvm_vm_ioctl_resize_hpt_prepare()
1505 resize_hpt_release(kvm, resize); in kvm_vm_ioctl_resize_hpt_prepare()
1522 resize->kvm = kvm; in kvm_vm_ioctl_resize_hpt_prepare()
1524 kvm->arch.resize_hpt = resize; in kvm_vm_ioctl_resize_hpt_prepare()
1531 mutex_unlock(&kvm->arch.mmu_setup_lock); in kvm_vm_ioctl_resize_hpt_prepare()
1540 int kvm_vm_ioctl_resize_hpt_commit(struct kvm *kvm, in kvm_vm_ioctl_resize_hpt_commit() argument
1548 if (flags != 0 || kvm_is_radix(kvm)) in kvm_vm_ioctl_resize_hpt_commit()
1554 mutex_lock(&kvm->arch.mmu_setup_lock); in kvm_vm_ioctl_resize_hpt_commit()
1556 resize = kvm->arch.resize_hpt; in kvm_vm_ioctl_resize_hpt_commit()
1560 if (WARN_ON(!kvm->arch.mmu_ready)) in kvm_vm_ioctl_resize_hpt_commit()
1564 kvm->arch.mmu_ready = 0; in kvm_vm_ioctl_resize_hpt_commit()
1587 kvm->arch.mmu_ready = 1; in kvm_vm_ioctl_resize_hpt_commit()
1590 resize_hpt_release(kvm, resize); in kvm_vm_ioctl_resize_hpt_commit()
1591 mutex_unlock(&kvm->arch.mmu_setup_lock); in kvm_vm_ioctl_resize_hpt_commit()
1614 struct kvm *kvm; member
1717 struct kvm *kvm = ctx->kvm; in kvm_htab_read() local
1730 if (kvm_is_radix(kvm)) in kvm_htab_read()
1737 hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE)); in kvm_htab_read()
1738 revp = kvm->arch.hpt.rev + i; in kvm_htab_read()
1753 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) && in kvm_htab_read()
1763 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) && in kvm_htab_read()
1779 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) && in kvm_htab_read()
1800 if (i >= kvmppc_hpt_npte(&kvm->arch.hpt)) { in kvm_htab_read()
1816 struct kvm *kvm = ctx->kvm; in kvm_htab_write() local
1830 if (kvm_is_radix(kvm)) in kvm_htab_write()
1834 mutex_lock(&kvm->arch.mmu_setup_lock); in kvm_htab_write()
1835 mmu_ready = kvm->arch.mmu_ready; in kvm_htab_write()
1837 kvm->arch.mmu_ready = 0; /* temporarily */ in kvm_htab_write()
1840 if (atomic_read(&kvm->arch.vcpus_running)) { in kvm_htab_write()
1841 kvm->arch.mmu_ready = 1; in kvm_htab_write()
1842 mutex_unlock(&kvm->arch.mmu_setup_lock); in kvm_htab_write()
1862 if (i >= kvmppc_hpt_npte(&kvm->arch.hpt) || in kvm_htab_write()
1863 i + hdr.n_valid + hdr.n_invalid > kvmppc_hpt_npte(&kvm->arch.hpt)) in kvm_htab_write()
1866 hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE)); in kvm_htab_write()
1888 kvmppc_do_h_remove(kvm, 0, i, 0, tmp); in kvm_htab_write()
1890 ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r, in kvm_htab_write()
1900 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | in kvm_htab_write()
1904 kvmppc_update_lpcr(kvm, lpcr, in kvm_htab_write()
1907 kvmppc_setup_partition_table(kvm); in kvm_htab_write()
1917 kvmppc_do_h_remove(kvm, 0, i, 0, tmp); in kvm_htab_write()
1927 kvm->arch.mmu_ready = mmu_ready; in kvm_htab_write()
1928 mutex_unlock(&kvm->arch.mmu_setup_lock); in kvm_htab_write()
1941 atomic_dec(&ctx->kvm->arch.hpte_mod_interest); in kvm_htab_release()
1942 kvm_put_kvm(ctx->kvm); in kvm_htab_release()
1954 int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf) in kvm_vm_ioctl_get_htab_fd() argument
1966 kvm_get_kvm(kvm); in kvm_vm_ioctl_get_htab_fd()
1967 ctx->kvm = kvm; in kvm_vm_ioctl_get_htab_fd()
1976 kvm_put_kvm_no_destroy(kvm); in kvm_vm_ioctl_get_htab_fd()
1981 mutex_lock(&kvm->slots_lock); in kvm_vm_ioctl_get_htab_fd()
1982 atomic_inc(&kvm->arch.hpte_mod_interest); in kvm_vm_ioctl_get_htab_fd()
1984 synchronize_srcu_expedited(&kvm->srcu); in kvm_vm_ioctl_get_htab_fd()
1985 mutex_unlock(&kvm->slots_lock); in kvm_vm_ioctl_get_htab_fd()
1992 struct kvm *kvm; member
2002 struct kvm *kvm = inode->i_private; in debugfs_htab_open() local
2009 kvm_get_kvm(kvm); in debugfs_htab_open()
2010 p->kvm = kvm; in debugfs_htab_open()
2021 kvm_put_kvm(p->kvm); in debugfs_htab_release()
2033 struct kvm *kvm; in debugfs_htab_read() local
2036 kvm = p->kvm; in debugfs_htab_read()
2037 if (kvm_is_radix(kvm)) in debugfs_htab_read()
2063 hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE)); in debugfs_htab_read()
2064 for (; len != 0 && i < kvmppc_hpt_npte(&kvm->arch.hpt); in debugfs_htab_read()
2075 gr = kvm->arch.hpt.rev[i].guest_rpte; in debugfs_htab_read()
2123 void kvmppc_mmu_debugfs_init(struct kvm *kvm) in kvmppc_mmu_debugfs_init() argument
2125 debugfs_create_file("htab", 0400, kvm->debugfs_dentry, kvm, in kvmppc_mmu_debugfs_init()