36 kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)  in kvm_arch_irqfd_allowed()  argument
46 	struct kvm *kvm = irqfd->kvm;  in irqfd_inject()  local
49 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,  in irqfd_inject()
51 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,  in irqfd_inject()
54 		kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,  in irqfd_inject()
63 				 srcu_read_lock_held(&resampler->kvm->irq_srcu))  in irqfd_resampler_notify()
76 	struct kvm *kvm;  in irqfd_resampler_ack()  local
81 	kvm = resampler->kvm;  in irqfd_resampler_ack()
83 	kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,  in irqfd_resampler_ack()
86 	idx = srcu_read_lock(&kvm->irq_srcu);  in irqfd_resampler_ack()
88 	srcu_read_unlock(&kvm->irq_srcu, idx);  in irqfd_resampler_ack()
95 	struct kvm *kvm = resampler->kvm;  in irqfd_resampler_shutdown()  local
97 	mutex_lock(&kvm->irqfds.resampler_lock);  in irqfd_resampler_shutdown()
103 		kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier);  in irqfd_resampler_shutdown()
108 		kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,  in irqfd_resampler_shutdown()
112 		synchronize_srcu_expedited(&kvm->irq_srcu);  in irqfd_resampler_shutdown()
115 	mutex_unlock(&kvm->irqfds.resampler_lock);  in irqfd_resampler_shutdown()
126 	struct kvm *kvm = irqfd->kvm;  in irqfd_shutdown()  local
130 	synchronize_srcu_expedited(&kvm->irq_srcu);  in irqfd_shutdown()
184 				struct kvm *kvm, int irq_source_id,  in kvm_arch_set_irq_inatomic()  argument
201 	struct kvm *kvm = irqfd->kvm;  in irqfd_wakeup()  local
210 		idx = srcu_read_lock(&kvm->irq_srcu);  in irqfd_wakeup()
216 		if (kvm_arch_set_irq_inatomic(&irq, kvm,  in irqfd_wakeup()
220 		srcu_read_unlock(&kvm->irq_srcu, idx);  in irqfd_wakeup()
228 		spin_lock_irqsave(&kvm->irqfds.lock, iflags);  in irqfd_wakeup()
242 		spin_unlock_irqrestore(&kvm->irqfds.lock, iflags);  in irqfd_wakeup()
258 static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd)  in irqfd_update()  argument
264 	n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);  in irqfd_update()
289 				struct kvm *kvm, unsigned int host_irq,  in kvm_arch_update_irqfd_routing()  argument
304 kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)  in kvm_irqfd_assign()  argument
313 	if (!kvm_arch_intc_initialized(kvm))  in kvm_irqfd_assign()
316 	if (!kvm_arch_irqfd_allowed(kvm, args))  in kvm_irqfd_assign()
323 	irqfd->kvm = kvm;  in kvm_irqfd_assign()
328 	seqcount_spinlock_init(&irqfd->irq_entry_sc, &kvm->irqfds.lock);  in kvm_irqfd_assign()
356 		mutex_lock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
359 				    &kvm->irqfds.resampler_list, link) {  in kvm_irqfd_assign()
371 				mutex_unlock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
375 			resampler->kvm = kvm;  in kvm_irqfd_assign()
381 			list_add_rcu(&resampler->link, &kvm->irqfds.resampler_list);  in kvm_irqfd_assign()
382 			kvm_register_irq_ack_notifier(kvm,  in kvm_irqfd_assign()
388 		synchronize_srcu_expedited(&kvm->irq_srcu);  in kvm_irqfd_assign()
390 		mutex_unlock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
400 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
403 	list_for_each_entry(tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_assign()
408 		spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
412 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_irqfd_assign()
413 	irqfd_update(kvm, irqfd);  in kvm_irqfd_assign()
415 	list_add_tail(&irqfd->list, &kvm->irqfds.items);  in kvm_irqfd_assign()
417 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
442 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irqfd_assign()
468 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)  in kvm_irq_has_notifier()  argument
473 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_irq_has_notifier()
474 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_irq_has_notifier()
476 		hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,  in kvm_irq_has_notifier()
477 					  link, srcu_read_lock_held(&kvm->irq_srcu))  in kvm_irq_has_notifier()
479 				srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irq_has_notifier()
483 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irq_has_notifier()
489 void kvm_notify_acked_gsi(struct kvm *kvm, int gsi)  in kvm_notify_acked_gsi()  argument
493 	hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,  in kvm_notify_acked_gsi()
494 				  link, srcu_read_lock_held(&kvm->irq_srcu))  in kvm_notify_acked_gsi()
499 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)  in kvm_notify_acked_irq()  argument
505 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_notify_acked_irq()
506 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_notify_acked_irq()
508 		kvm_notify_acked_gsi(kvm, gsi);  in kvm_notify_acked_irq()
509 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_acked_irq()
512 void kvm_register_irq_ack_notifier(struct kvm *kvm,  in kvm_register_irq_ack_notifier()  argument
515 	mutex_lock(&kvm->irq_lock);  in kvm_register_irq_ack_notifier()
516 	hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);  in kvm_register_irq_ack_notifier()
517 	mutex_unlock(&kvm->irq_lock);  in kvm_register_irq_ack_notifier()
518 	kvm_arch_post_irq_ack_notifier_list_update(kvm);  in kvm_register_irq_ack_notifier()
521 void kvm_unregister_irq_ack_notifier(struct kvm *kvm,  in kvm_unregister_irq_ack_notifier()  argument
524 	mutex_lock(&kvm->irq_lock);  in kvm_unregister_irq_ack_notifier()
526 	mutex_unlock(&kvm->irq_lock);  in kvm_unregister_irq_ack_notifier()
527 	synchronize_srcu_expedited(&kvm->irq_srcu);  in kvm_unregister_irq_ack_notifier()
528 	kvm_arch_post_irq_ack_notifier_list_update(kvm);  in kvm_unregister_irq_ack_notifier()
535 kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args)  in kvm_irqfd_deassign()  argument
544 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_deassign()
546 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_deassign()
561 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_deassign()
575 kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)  in kvm_irqfd()  argument
581 		return kvm_irqfd_deassign(kvm, args);  in kvm_irqfd()
583 	return kvm_irqfd_assign(kvm, args);  in kvm_irqfd()
591 kvm_irqfd_release(struct kvm *kvm)  in kvm_irqfd_release()  argument
595 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_release()
597 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)  in kvm_irqfd_release()
600 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_release()
614 void kvm_irq_routing_update(struct kvm *kvm)  in kvm_irq_routing_update()  argument
618 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irq_routing_update()
620 	list_for_each_entry(irqfd, &kvm->irqfds.items, list) {  in kvm_irq_routing_update()
626 		irqfd_update(kvm, irqfd);  in kvm_irq_routing_update()
632 					irqfd->kvm, irqfd->producer->irq,  in kvm_irq_routing_update()
639 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irq_routing_update()
642 bool kvm_notify_irqfd_resampler(struct kvm *kvm,  in kvm_notify_irqfd_resampler()  argument
649 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_notify_irqfd_resampler()
650 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_notify_irqfd_resampler()
653 					 &kvm->irqfds.resampler_list, link,  in kvm_notify_irqfd_resampler()
654 					 srcu_read_lock_held(&kvm->irq_srcu)) {  in kvm_notify_irqfd_resampler()
657 				srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_irqfd_resampler()
662 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_irqfd_resampler()
799 ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p)  in ioeventfd_check_collision()  argument
803 	list_for_each_entry(_p, &kvm->ioeventfds, list)  in ioeventfd_check_collision()
824 static int kvm_assign_ioeventfd_idx(struct kvm *kvm,  in kvm_assign_ioeventfd_idx()  argument
855 	mutex_lock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
858 	if (ioeventfd_check_collision(kvm, p)) {  in kvm_assign_ioeventfd_idx()
865 	ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length,  in kvm_assign_ioeventfd_idx()
870 	kvm_get_bus(kvm, bus_idx)->ioeventfd_count++;  in kvm_assign_ioeventfd_idx()
871 	list_add_tail(&p->list, &kvm->ioeventfds);  in kvm_assign_ioeventfd_idx()
873 	mutex_unlock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
878 	mutex_unlock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
888 kvm_deassign_ioeventfd_idx(struct kvm *kvm, enum kvm_bus bus_idx,  in kvm_deassign_ioeventfd_idx()  argument
903 	mutex_lock(&kvm->slots_lock);  in kvm_deassign_ioeventfd_idx()
905 	list_for_each_entry(p, &kvm->ioeventfds, list) {  in kvm_deassign_ioeventfd_idx()
916 		kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);  in kvm_deassign_ioeventfd_idx()
917 		bus = kvm_get_bus(kvm, bus_idx);  in kvm_deassign_ioeventfd_idx()
924 	mutex_unlock(&kvm->slots_lock);  in kvm_deassign_ioeventfd_idx()
931 static int kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)  in kvm_deassign_ioeventfd()  argument
934 	int ret = kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);  in kvm_deassign_ioeventfd()
937 		kvm_deassign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);  in kvm_deassign_ioeventfd()
943 kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)  in kvm_assign_ioeventfd()  argument
973 	ret = kvm_assign_ioeventfd_idx(kvm, bus_idx, args);  in kvm_assign_ioeventfd()
981 		ret = kvm_assign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);  in kvm_assign_ioeventfd()
989 	kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);  in kvm_assign_ioeventfd()
995 kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)  in kvm_ioeventfd()  argument
998 		return kvm_deassign_ioeventfd(kvm, args);  in kvm_ioeventfd()
1000 	return kvm_assign_ioeventfd(kvm, args);  in kvm_ioeventfd()
1004 kvm_eventfd_init(struct kvm *kvm)  in kvm_eventfd_init()  argument
1007 	spin_lock_init(&kvm->irqfds.lock);  in kvm_eventfd_init()
1008 	INIT_LIST_HEAD(&kvm->irqfds.items);  in kvm_eventfd_init()
1009 	INIT_LIST_HEAD(&kvm->irqfds.resampler_list);  in kvm_eventfd_init()
1010 	mutex_init(&kvm->irqfds.resampler_lock);  in kvm_eventfd_init()
1012 	INIT_LIST_HEAD(&kvm->ioeventfds);  in kvm_eventfd_init()