23 static bool kvm_external_write_tracking_enabled(struct kvm *kvm)  in kvm_external_write_tracking_enabled()  argument
30 	return smp_load_acquire(&kvm->arch.external_write_tracking_enabled);  in kvm_external_write_tracking_enabled()
36 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)  in kvm_page_track_write_tracking_enabled()  argument
38 	return kvm_external_write_tracking_enabled(kvm) ||  in kvm_page_track_write_tracking_enabled()
39 	       kvm_shadow_root_allocated(kvm) || !tdp_enabled;  in kvm_page_track_write_tracking_enabled()
60 int kvm_page_track_create_memslot(struct kvm *kvm,  in kvm_page_track_create_memslot()  argument
64 	if (!kvm_page_track_write_tracking_enabled(kvm))  in kvm_page_track_create_memslot()
90 void __kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,  in __kvm_write_track_add_gfn()  argument
93 	lockdep_assert_held_write(&kvm->mmu_lock);  in __kvm_write_track_add_gfn()
95 	lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||  in __kvm_write_track_add_gfn()
96 			    srcu_read_lock_held(&kvm->srcu));  in __kvm_write_track_add_gfn()
98 	if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))  in __kvm_write_track_add_gfn()
109 	if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))  in __kvm_write_track_add_gfn()
110 		kvm_flush_remote_tlbs(kvm);  in __kvm_write_track_add_gfn()
113 void __kvm_write_track_remove_gfn(struct kvm *kvm,  in __kvm_write_track_remove_gfn()  argument
116 	lockdep_assert_held_write(&kvm->mmu_lock);  in __kvm_write_track_remove_gfn()
118 	lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||  in __kvm_write_track_remove_gfn()
119 			    srcu_read_lock_held(&kvm->srcu));  in __kvm_write_track_remove_gfn()
121 	if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))  in __kvm_write_track_remove_gfn()
136 bool kvm_gfn_is_write_tracked(struct kvm *kvm,  in kvm_gfn_is_write_tracked()  argument
144 	if (!kvm_page_track_write_tracking_enabled(kvm))  in kvm_gfn_is_write_tracked()
152 void kvm_page_track_cleanup(struct kvm *kvm)  in kvm_page_track_cleanup()  argument
156 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_cleanup()
160 int kvm_page_track_init(struct kvm *kvm)  in kvm_page_track_init()  argument
164 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_init()
169 static int kvm_enable_external_write_tracking(struct kvm *kvm)  in kvm_enable_external_write_tracking()  argument
175 	mutex_lock(&kvm->slots_arch_lock);  in kvm_enable_external_write_tracking()
182 	if (kvm_page_track_write_tracking_enabled(kvm))  in kvm_enable_external_write_tracking()
185 	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {  in kvm_enable_external_write_tracking()
186 		slots = __kvm_memslots(kvm, i);  in kvm_enable_external_write_tracking()
207 	smp_store_release(&kvm->arch.external_write_tracking_enabled, true);  in kvm_enable_external_write_tracking()
209 	mutex_unlock(&kvm->slots_arch_lock);  in kvm_enable_external_write_tracking()
217 int kvm_page_track_register_notifier(struct kvm *kvm,  in kvm_page_track_register_notifier()  argument
223 	if (!kvm || kvm->mm != current->mm)  in kvm_page_track_register_notifier()
226 	if (!kvm_external_write_tracking_enabled(kvm)) {  in kvm_page_track_register_notifier()
227 		r = kvm_enable_external_write_tracking(kvm);  in kvm_page_track_register_notifier()
232 	kvm_get_kvm(kvm);  in kvm_page_track_register_notifier()
234 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_register_notifier()
236 	write_lock(&kvm->mmu_lock);  in kvm_page_track_register_notifier()
238 	write_unlock(&kvm->mmu_lock);  in kvm_page_track_register_notifier()
247 void kvm_page_track_unregister_notifier(struct kvm *kvm,  in kvm_page_track_unregister_notifier()  argument
252 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_unregister_notifier()
254 	write_lock(&kvm->mmu_lock);  in kvm_page_track_unregister_notifier()
256 	write_unlock(&kvm->mmu_lock);  in kvm_page_track_unregister_notifier()
259 	kvm_put_kvm(kvm);  in kvm_page_track_unregister_notifier()
270 void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes)  in __kvm_page_track_write()  argument
276 	head = &kvm->arch.track_notifier_head;  in __kvm_page_track_write()
293 void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot)  in kvm_page_track_delete_slot()  argument
299 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_delete_slot()
319 int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn)  in kvm_write_track_add_gfn()  argument
324 	idx = srcu_read_lock(&kvm->srcu);  in kvm_write_track_add_gfn()
326 	slot = gfn_to_memslot(kvm, gfn);  in kvm_write_track_add_gfn()
328 		srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_add_gfn()
332 	write_lock(&kvm->mmu_lock);  in kvm_write_track_add_gfn()
333 	__kvm_write_track_add_gfn(kvm, slot, gfn);  in kvm_write_track_add_gfn()
334 	write_unlock(&kvm->mmu_lock);  in kvm_write_track_add_gfn()
336 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_add_gfn()
349 int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn)  in kvm_write_track_remove_gfn()  argument
354 	idx = srcu_read_lock(&kvm->srcu);  in kvm_write_track_remove_gfn()
356 	slot = gfn_to_memslot(kvm, gfn);  in kvm_write_track_remove_gfn()
358 		srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_remove_gfn()
362 	write_lock(&kvm->mmu_lock);  in kvm_write_track_remove_gfn()
363 	__kvm_write_track_remove_gfn(kvm, slot, gfn);  in kvm_write_track_remove_gfn()
364 	write_unlock(&kvm->mmu_lock);  in kvm_write_track_remove_gfn()
366 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_remove_gfn()