177 static bool (*pirq_needs_eoi)(struct irq_info *info);
259 static void set_info_for_irq(unsigned int irq, struct irq_info *info)  in set_info_for_irq()  argument
262 		legacy_info_ptrs[irq] = info;  in set_info_for_irq()
264 		irq_set_chip_data(irq, info);  in set_info_for_irq()
281 static void channels_on_cpu_dec(struct irq_info *info)  in channels_on_cpu_dec()  argument
283 	if (!info->is_accounted)  in channels_on_cpu_dec()
286 	info->is_accounted = 0;  in channels_on_cpu_dec()
288 	if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids))  in channels_on_cpu_dec()
291 	WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], -1 , 0));  in channels_on_cpu_dec()
294 static void channels_on_cpu_inc(struct irq_info *info)  in channels_on_cpu_inc()  argument
296 	if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids))  in channels_on_cpu_inc()
299 	if (WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], 1,  in channels_on_cpu_inc()
303 	info->is_accounted = 1;  in channels_on_cpu_inc()
315 	struct irq_info *info = container_of(to_rcu_work(work), struct irq_info,  in delayed_free_irq()  local
317 	unsigned int irq = info->irq;  in delayed_free_irq()
322 	kfree(info);  in delayed_free_irq()
328 static int xen_irq_info_common_setup(struct irq_info *info,  in xen_irq_info_common_setup()  argument
335 	BUG_ON(info->type != IRQT_UNBOUND && info->type != type);  in xen_irq_info_common_setup()
337 	info->type = type;  in xen_irq_info_common_setup()
338 	info->evtchn = evtchn;  in xen_irq_info_common_setup()
339 	info->cpu = cpu;  in xen_irq_info_common_setup()
340 	info->mask_reason = EVT_MASK_REASON_EXPLICIT;  in xen_irq_info_common_setup()
341 	raw_spin_lock_init(&info->lock);  in xen_irq_info_common_setup()
343 	ret = set_evtchn_to_irq(evtchn, info->irq);  in xen_irq_info_common_setup()
347 	irq_clear_status_flags(info->irq, IRQ_NOREQUEST | IRQ_NOAUTOEN);  in xen_irq_info_common_setup()
352 static int xen_irq_info_evtchn_setup(struct irq_info *info,  in xen_irq_info_evtchn_setup()  argument
358 	ret = xen_irq_info_common_setup(info, IRQT_EVTCHN, evtchn, 0);  in xen_irq_info_evtchn_setup()
359 	info->u.interdomain = dev;  in xen_irq_info_evtchn_setup()
366 static int xen_irq_info_ipi_setup(struct irq_info *info, unsigned int cpu,  in xen_irq_info_ipi_setup()  argument
369 	info->u.ipi = ipi;  in xen_irq_info_ipi_setup()
371 	per_cpu(ipi_to_irq, cpu)[ipi] = info->irq;  in xen_irq_info_ipi_setup()
374 	return xen_irq_info_common_setup(info, IRQT_IPI, evtchn, 0);  in xen_irq_info_ipi_setup()
377 static int xen_irq_info_virq_setup(struct irq_info *info, unsigned int cpu,  in xen_irq_info_virq_setup()  argument
380 	info->u.virq = virq;  in xen_irq_info_virq_setup()
382 	per_cpu(virq_to_irq, cpu)[virq] = info->irq;  in xen_irq_info_virq_setup()
384 	return xen_irq_info_common_setup(info, IRQT_VIRQ, evtchn, 0);  in xen_irq_info_virq_setup()
387 static int xen_irq_info_pirq_setup(struct irq_info *info, evtchn_port_t evtchn,  in xen_irq_info_pirq_setup()  argument
391 	info->u.pirq.pirq = pirq;  in xen_irq_info_pirq_setup()
392 	info->u.pirq.gsi = gsi;  in xen_irq_info_pirq_setup()
393 	info->u.pirq.domid = domid;  in xen_irq_info_pirq_setup()
394 	info->u.pirq.flags = flags;  in xen_irq_info_pirq_setup()
396 	return xen_irq_info_common_setup(info, IRQT_PIRQ, evtchn, 0);  in xen_irq_info_pirq_setup()
399 static void xen_irq_info_cleanup(struct irq_info *info)  in xen_irq_info_cleanup()  argument
401 	set_evtchn_to_irq(info->evtchn, -1);  in xen_irq_info_cleanup()
402 	xen_evtchn_port_remove(info->evtchn, info->cpu);  in xen_irq_info_cleanup()
403 	info->evtchn = 0;  in xen_irq_info_cleanup()
404 	channels_on_cpu_dec(info);  in xen_irq_info_cleanup()
412 	const struct irq_info *info = NULL;  in evtchn_from_irq()  local
415 		info = info_for_irq(irq);  in evtchn_from_irq()
416 	if (!info)  in evtchn_from_irq()
419 	return info->evtchn;  in evtchn_from_irq()
424 	struct irq_info *info = evtchn_to_info(evtchn);  in irq_from_evtchn()  local
426 	return info ? info->irq : -1;  in irq_from_evtchn()
440 static enum ipi_vector ipi_from_irq(struct irq_info *info)  in ipi_from_irq()  argument
442 	BUG_ON(info == NULL);  in ipi_from_irq()
443 	BUG_ON(info->type != IRQT_IPI);  in ipi_from_irq()
445 	return info->u.ipi;  in ipi_from_irq()
448 static unsigned int virq_from_irq(struct irq_info *info)  in virq_from_irq()  argument
450 	BUG_ON(info == NULL);  in virq_from_irq()
451 	BUG_ON(info->type != IRQT_VIRQ);  in virq_from_irq()
453 	return info->u.virq;  in virq_from_irq()
456 static unsigned int pirq_from_irq(struct irq_info *info)  in pirq_from_irq()  argument
458 	BUG_ON(info == NULL);  in pirq_from_irq()
459 	BUG_ON(info->type != IRQT_PIRQ);  in pirq_from_irq()
461 	return info->u.pirq.pirq;  in pirq_from_irq()
466 	struct irq_info *info = evtchn_to_info(evtchn);  in cpu_from_evtchn()  local
468 	return info ? info->cpu : 0;  in cpu_from_evtchn()
471 static void do_mask(struct irq_info *info, u8 reason)  in do_mask()  argument
475 	raw_spin_lock_irqsave(&info->lock, flags);  in do_mask()
477 	if (!info->mask_reason)  in do_mask()
478 		mask_evtchn(info->evtchn);  in do_mask()
480 	info->mask_reason |= reason;  in do_mask()
482 	raw_spin_unlock_irqrestore(&info->lock, flags);  in do_mask()
485 static void do_unmask(struct irq_info *info, u8 reason)  in do_unmask()  argument
489 	raw_spin_lock_irqsave(&info->lock, flags);  in do_unmask()
491 	info->mask_reason &= ~reason;  in do_unmask()
493 	if (!info->mask_reason)  in do_unmask()
494 		unmask_evtchn(info->evtchn);  in do_unmask()
496 	raw_spin_unlock_irqrestore(&info->lock, flags);  in do_unmask()
500 static bool pirq_check_eoi_map(struct irq_info *info)  in pirq_check_eoi_map()  argument
502 	return test_bit(pirq_from_irq(info), pirq_eoi_map);  in pirq_check_eoi_map()
506 static bool pirq_needs_eoi_flag(struct irq_info *info)  in pirq_needs_eoi_flag()  argument
508 	BUG_ON(info->type != IRQT_PIRQ);  in pirq_needs_eoi_flag()
510 	return info->u.pirq.flags & PIRQ_NEEDS_EOI;  in pirq_needs_eoi_flag()
513 static void bind_evtchn_to_cpu(struct irq_info *info, unsigned int cpu,  in bind_evtchn_to_cpu()  argument
517 		struct irq_data *data = irq_get_irq_data(info->irq);  in bind_evtchn_to_cpu()
523 	xen_evtchn_port_bind_to_cpu(info->evtchn, cpu, info->cpu);  in bind_evtchn_to_cpu()
525 	channels_on_cpu_dec(info);  in bind_evtchn_to_cpu()
526 	info->cpu = cpu;  in bind_evtchn_to_cpu()
527 	channels_on_cpu_inc(info);  in bind_evtchn_to_cpu()
555 static void lateeoi_list_del(struct irq_info *info)  in lateeoi_list_del()  argument
557 	struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);  in lateeoi_list_del()
561 	list_del_init(&info->eoi_list);  in lateeoi_list_del()
565 static void lateeoi_list_add(struct irq_info *info)  in lateeoi_list_add()  argument
567 	struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);  in lateeoi_list_add()
573 	if (now < info->eoi_time)  in lateeoi_list_add()
574 		delay = info->eoi_time - now;  in lateeoi_list_add()
582 	if (!elem || info->eoi_time < elem->eoi_time) {  in lateeoi_list_add()
583 		list_add(&info->eoi_list, &eoi->eoi_list);  in lateeoi_list_add()
584 		mod_delayed_work_on(info->eoi_cpu, system_wq,  in lateeoi_list_add()
588 			if (elem->eoi_time <= info->eoi_time)  in lateeoi_list_add()
591 		list_add(&info->eoi_list, &elem->eoi_list);  in lateeoi_list_add()
597 static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious)  in xen_irq_lateeoi_locked()  argument
603 	evtchn = info->evtchn;  in xen_irq_lateeoi_locked()
604 	if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list))  in xen_irq_lateeoi_locked()
608 		struct xenbus_device *dev = info->u.interdomain;  in xen_irq_lateeoi_locked()
614 		if ((1 << info->spurious_cnt) < (HZ << 2)) {  in xen_irq_lateeoi_locked()
615 			if (info->spurious_cnt != 0xFF)  in xen_irq_lateeoi_locked()
616 				info->spurious_cnt++;  in xen_irq_lateeoi_locked()
618 		if (info->spurious_cnt > threshold) {  in xen_irq_lateeoi_locked()
619 			delay = 1 << (info->spurious_cnt - 1 - threshold);  in xen_irq_lateeoi_locked()
622 			if (!info->eoi_time)  in xen_irq_lateeoi_locked()
623 				info->eoi_cpu = smp_processor_id();  in xen_irq_lateeoi_locked()
624 			info->eoi_time = get_jiffies_64() + delay;  in xen_irq_lateeoi_locked()
631 		info->spurious_cnt = 0;  in xen_irq_lateeoi_locked()
634 	cpu = info->eoi_cpu;  in xen_irq_lateeoi_locked()
635 	if (info->eoi_time &&  in xen_irq_lateeoi_locked()
636 	    (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) {  in xen_irq_lateeoi_locked()
637 		lateeoi_list_add(info);  in xen_irq_lateeoi_locked()
641 	info->eoi_time = 0;  in xen_irq_lateeoi_locked()
644 	smp_store_release(&info->is_active, 0);  in xen_irq_lateeoi_locked()
645 	do_unmask(info, EVT_MASK_REASON_EOI_PENDING);  in xen_irq_lateeoi_locked()
651 	struct irq_info *info;  in xen_irq_lateeoi_worker()  local
662 		info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info,  in xen_irq_lateeoi_worker()
665 		if (info == NULL)  in xen_irq_lateeoi_worker()
668 		if (now < info->eoi_time) {  in xen_irq_lateeoi_worker()
669 			mod_delayed_work_on(info->eoi_cpu, system_wq,  in xen_irq_lateeoi_worker()
671 					    info->eoi_time - now);  in xen_irq_lateeoi_worker()
675 		list_del_init(&info->eoi_list);  in xen_irq_lateeoi_worker()
679 		info->eoi_time = 0;  in xen_irq_lateeoi_worker()
681 		xen_irq_lateeoi_locked(info, false);  in xen_irq_lateeoi_worker()
700 	struct irq_info *info;  in xen_irq_lateeoi()  local
704 	info = info_for_irq(irq);  in xen_irq_lateeoi()
706 	if (info)  in xen_irq_lateeoi()
707 		xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS);  in xen_irq_lateeoi()
715 	struct irq_info *info;  in xen_irq_init()  local
717 	info = kzalloc(sizeof(*info), GFP_KERNEL);  in xen_irq_init()
718 	if (info) {  in xen_irq_init()
719 		info->irq = irq;  in xen_irq_init()
720 		info->type = IRQT_UNBOUND;  in xen_irq_init()
721 		info->refcnt = -1;  in xen_irq_init()
722 		INIT_RCU_WORK(&info->rwork, delayed_free_irq);  in xen_irq_init()
724 		set_info_for_irq(irq, info);  in xen_irq_init()
731 		INIT_LIST_HEAD(&info->eoi_list);  in xen_irq_init()
732 		list_add_tail(&info->list, &xen_irq_list_head);  in xen_irq_init()
735 	return info;  in xen_irq_init()
741 	struct irq_info *info = NULL;  in xen_allocate_irq_dynamic()  local
744 		info = xen_irq_init(irq);  in xen_allocate_irq_dynamic()
745 		if (!info)  in xen_allocate_irq_dynamic()
749 	return info;  in xen_allocate_irq_dynamic()
755 	struct irq_info *info;  in xen_allocate_irq_gsi()  local
772 	info = xen_irq_init(irq);  in xen_allocate_irq_gsi()
773 	if (!info)  in xen_allocate_irq_gsi()
776 	return info;  in xen_allocate_irq_gsi()
779 static void xen_free_irq(struct irq_info *info)  in xen_free_irq()  argument
781 	if (WARN_ON(!info))  in xen_free_irq()
784 	if (!list_empty(&info->eoi_list))  in xen_free_irq()
785 		lateeoi_list_del(info);  in xen_free_irq()
787 	list_del(&info->list);  in xen_free_irq()
789 	WARN_ON(info->refcnt > 0);  in xen_free_irq()
791 	queue_rcu_work(system_wq, &info->rwork);  in xen_free_irq()
795 static void event_handler_exit(struct irq_info *info)  in event_handler_exit()  argument
797 	smp_store_release(&info->is_active, 0);  in event_handler_exit()
798 	clear_evtchn(info->evtchn);  in event_handler_exit()
801 static void pirq_query_unmask(struct irq_info *info)  in pirq_query_unmask()  argument
805 	irq_status.irq = pirq_from_irq(info);  in pirq_query_unmask()
809 	info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;  in pirq_query_unmask()
811 		info->u.pirq.flags |= PIRQ_NEEDS_EOI;  in pirq_query_unmask()
814 static void do_eoi_pirq(struct irq_info *info)  in do_eoi_pirq()  argument
816 	struct physdev_eoi eoi = { .irq = pirq_from_irq(info) };  in do_eoi_pirq()
819 	if (!VALID_EVTCHN(info->evtchn))  in do_eoi_pirq()
822 	event_handler_exit(info);  in do_eoi_pirq()
824 	if (pirq_needs_eoi(info)) {  in do_eoi_pirq()
832 	struct irq_info *info = info_for_irq(data->irq);  in eoi_pirq()  local
834 	do_eoi_pirq(info);  in eoi_pirq()
837 static void do_disable_dynirq(struct irq_info *info)  in do_disable_dynirq()  argument
839 	if (VALID_EVTCHN(info->evtchn))  in do_disable_dynirq()
840 		do_mask(info, EVT_MASK_REASON_EXPLICIT);  in do_disable_dynirq()
845 	struct irq_info *info = info_for_irq(data->irq);  in disable_dynirq()  local
847 	if (info)  in disable_dynirq()
848 		do_disable_dynirq(info);  in disable_dynirq()
853 	struct irq_info *info = info_for_irq(data->irq);  in mask_ack_pirq()  local
855 	if (info) {  in mask_ack_pirq()
856 		do_disable_dynirq(info);  in mask_ack_pirq()
857 		do_eoi_pirq(info);  in mask_ack_pirq()
861 static unsigned int __startup_pirq(struct irq_info *info)  in __startup_pirq()  argument
864 	evtchn_port_t evtchn = info->evtchn;  in __startup_pirq()
870 	bind_pirq.pirq = pirq_from_irq(info);  in __startup_pirq()
872 	bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?  in __startup_pirq()
876 		pr_warn("Failed to obtain physical IRQ %d\n", info->irq);  in __startup_pirq()
881 	pirq_query_unmask(info);  in __startup_pirq()
883 	rc = set_evtchn_to_irq(evtchn, info->irq);  in __startup_pirq()
887 	info->evtchn = evtchn;  in __startup_pirq()
888 	bind_evtchn_to_cpu(info, 0, false);  in __startup_pirq()
895 	do_unmask(info, EVT_MASK_REASON_EXPLICIT);  in __startup_pirq()
897 	do_eoi_pirq(info);  in __startup_pirq()
902 	pr_err("irq%d: Failed to set port to irq mapping (%d)\n", info->irq,  in __startup_pirq()
910 	struct irq_info *info = info_for_irq(data->irq);  in startup_pirq()  local
912 	return __startup_pirq(info);  in startup_pirq()
917 	struct irq_info *info = info_for_irq(data->irq);  in shutdown_pirq()  local
918 	evtchn_port_t evtchn = info->evtchn;  in shutdown_pirq()
920 	BUG_ON(info->type != IRQT_PIRQ);  in shutdown_pirq()
925 	do_mask(info, EVT_MASK_REASON_EXPLICIT);  in shutdown_pirq()
926 	xen_irq_info_cleanup(info);  in shutdown_pirq()
942 	struct irq_info *info;  in xen_irq_from_gsi()  local
944 	list_for_each_entry(info, &xen_irq_list_head, list) {  in xen_irq_from_gsi()
945 		if (info->type != IRQT_PIRQ)  in xen_irq_from_gsi()
948 		if (info->u.pirq.gsi == gsi)  in xen_irq_from_gsi()
949 			return info->irq;  in xen_irq_from_gsi()
956 static void __unbind_from_irq(struct irq_info *info, unsigned int irq)  in __unbind_from_irq()  argument
961 	if (!info) {  in __unbind_from_irq()
966 	if (info->refcnt > 0) {  in __unbind_from_irq()
967 		info->refcnt--;  in __unbind_from_irq()
968 		if (info->refcnt != 0)  in __unbind_from_irq()
972 	evtchn = info->evtchn;  in __unbind_from_irq()
975 		unsigned int cpu = info->cpu;  in __unbind_from_irq()
978 		if (!info->is_static)  in __unbind_from_irq()
981 		switch (info->type) {  in __unbind_from_irq()
983 			per_cpu(virq_to_irq, cpu)[virq_from_irq(info)] = -1;  in __unbind_from_irq()
986 			per_cpu(ipi_to_irq, cpu)[ipi_from_irq(info)] = -1;  in __unbind_from_irq()
987 			per_cpu(ipi_to_evtchn, cpu)[ipi_from_irq(info)] = 0;  in __unbind_from_irq()
990 			dev = info->u.interdomain;  in __unbind_from_irq()
998 		xen_irq_info_cleanup(info);  in __unbind_from_irq()
1004 	xen_free_irq(info);  in __unbind_from_irq()
1020 	struct irq_info *info;  in xen_bind_pirq_gsi_to_irq()  local
1033 	info = xen_allocate_irq_gsi(gsi);  in xen_bind_pirq_gsi_to_irq()
1034 	if (!info)  in xen_bind_pirq_gsi_to_irq()
1037 	irq_op.irq = info->irq;  in xen_bind_pirq_gsi_to_irq()
1045 		xen_free_irq(info);  in xen_bind_pirq_gsi_to_irq()
1050 	ret = xen_irq_info_pirq_setup(info, 0, pirq, gsi, DOMID_SELF,  in xen_bind_pirq_gsi_to_irq()
1053 		__unbind_from_irq(info, info->irq);  in xen_bind_pirq_gsi_to_irq()
1057 	pirq_query_unmask(info);  in xen_bind_pirq_gsi_to_irq()
1074 		irq_set_chip_and_handler_name(info->irq, &xen_pirq_chip,  in xen_bind_pirq_gsi_to_irq()
1077 		irq_set_chip_and_handler_name(info->irq, &xen_pirq_chip,  in xen_bind_pirq_gsi_to_irq()
1080 	ret = info->irq;  in xen_bind_pirq_gsi_to_irq()
1107 	struct irq_info *info;  in xen_bind_pirq_msi_to_irq()  local
1116 		info = xen_irq_init(irq + i);  in xen_bind_pirq_msi_to_irq()
1117 		if (!info) {  in xen_bind_pirq_msi_to_irq()
1124 		ret = xen_irq_info_pirq_setup(info, 0, pirq + i, 0, domid,  in xen_bind_pirq_msi_to_irq()
1139 		info = info_for_irq(irq + nvec);  in xen_bind_pirq_msi_to_irq()
1140 		__unbind_from_irq(info, irq + nvec);  in xen_bind_pirq_msi_to_irq()
1150 	struct irq_info *info = info_for_irq(irq);  in xen_destroy_irq()  local
1160 	if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) {  in xen_destroy_irq()
1161 		unmap_irq.pirq = info->u.pirq.pirq;  in xen_destroy_irq()
1162 		unmap_irq.domid = info->u.pirq.domid;  in xen_destroy_irq()
1168 		if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))  in xen_destroy_irq()
1170 				info->u.pirq.domid, info->u.pirq.pirq);  in xen_destroy_irq()
1177 	xen_free_irq(info);  in xen_destroy_irq()
1186 	struct irq_info *info = info_for_irq(irq);  in xen_pirq_from_irq()  local
1188 	return pirq_from_irq(info);  in xen_pirq_from_irq()
1196 	struct irq_info *info;  in bind_evtchn_to_irq_chip()  local
1203 	info = evtchn_to_info(evtchn);  in bind_evtchn_to_irq_chip()
1205 	if (!info) {  in bind_evtchn_to_irq_chip()
1206 		info = xen_allocate_irq_dynamic();  in bind_evtchn_to_irq_chip()
1207 		if (!info)  in bind_evtchn_to_irq_chip()
1210 		irq_set_chip_and_handler_name(info->irq, chip,  in bind_evtchn_to_irq_chip()
1213 		ret = xen_irq_info_evtchn_setup(info, evtchn, dev);  in bind_evtchn_to_irq_chip()
1215 			__unbind_from_irq(info, info->irq);  in bind_evtchn_to_irq_chip()
1225 		bind_evtchn_to_cpu(info, 0, false);  in bind_evtchn_to_irq_chip()
1226 	} else if (!WARN_ON(info->type != IRQT_EVTCHN)) {  in bind_evtchn_to_irq_chip()
1227 		if (shared && !WARN_ON(info->refcnt < 0))  in bind_evtchn_to_irq_chip()
1228 			info->refcnt++;  in bind_evtchn_to_irq_chip()
1231 	ret = info->irq;  in bind_evtchn_to_irq_chip()
1255 	struct irq_info *info;  in bind_ipi_to_irq()  local
1263 		info = xen_allocate_irq_dynamic();  in bind_ipi_to_irq()
1264 		if (!info)  in bind_ipi_to_irq()
1267 		irq_set_chip_and_handler_name(info->irq, &xen_percpu_chip,  in bind_ipi_to_irq()
1276 		ret = xen_irq_info_ipi_setup(info, cpu, evtchn, ipi);  in bind_ipi_to_irq()
1278 			__unbind_from_irq(info, info->irq);  in bind_ipi_to_irq()
1285 		bind_evtchn_to_cpu(info, cpu, true);  in bind_ipi_to_irq()
1286 		ret = info->irq;  in bind_ipi_to_irq()
1288 		info = info_for_irq(ret);  in bind_ipi_to_irq()
1289 		WARN_ON(info == NULL || info->type != IRQT_IPI);  in bind_ipi_to_irq()
1363 	struct irq_info *info;  in bind_virq_to_irq()  local
1371 		info = xen_allocate_irq_dynamic();  in bind_virq_to_irq()
1372 		if (!info)  in bind_virq_to_irq()
1376 			irq_set_chip_and_handler_name(info->irq, &xen_percpu_chip,  in bind_virq_to_irq()
1379 			irq_set_chip_and_handler_name(info->irq, &xen_dynamic_chip,  in bind_virq_to_irq()
1394 		ret = xen_irq_info_virq_setup(info, cpu, evtchn, virq);  in bind_virq_to_irq()
1396 			__unbind_from_irq(info, info->irq);  in bind_virq_to_irq()
1404 		bind_evtchn_to_cpu(info, cpu, percpu);  in bind_virq_to_irq()
1405 		ret = info->irq;  in bind_virq_to_irq()
1407 		info = info_for_irq(ret);  in bind_virq_to_irq()
1408 		WARN_ON(info == NULL || info->type != IRQT_VIRQ);  in bind_virq_to_irq()
1419 	struct irq_info *info;  in unbind_from_irq()  local
1422 	info = info_for_irq(irq);  in unbind_from_irq()
1423 	__unbind_from_irq(info, irq);  in unbind_from_irq()
1548 	struct irq_info *info = info_for_irq(irq);  in unbind_from_irqhandler()  local
1550 	if (WARN_ON(!info))  in unbind_from_irqhandler()
1576 	struct irq_info *info = evtchn_to_info(evtchn);  in evtchn_make_refcounted()  local
1578 	if (!info)  in evtchn_make_refcounted()
1581 	WARN_ON(info->refcnt != -1);  in evtchn_make_refcounted()
1583 	info->refcnt = 1;  in evtchn_make_refcounted()
1584 	info->is_static = is_static;  in evtchn_make_refcounted()
1592 	struct irq_info *info;  in evtchn_get()  local
1600 	info = evtchn_to_info(evtchn);  in evtchn_get()
1602 	if (!info)  in evtchn_get()
1606 	if (info->refcnt <= 0 || info->refcnt == SHRT_MAX)  in evtchn_get()
1609 	info->refcnt++;  in evtchn_get()
1620 	struct irq_info *info = evtchn_to_info(evtchn);  in evtchn_put()  local
1622 	if (WARN_ON(!info))  in evtchn_put()
1624 	unbind_from_irq(info->irq);  in evtchn_put()
1654 	struct irq_info *info = evtchn_to_info(port);  in handle_irq_for_port()  local
1657 	if (!info)  in handle_irq_for_port()
1682 	if (xchg_acquire(&info->is_active, 1))  in handle_irq_for_port()
1685 	dev = (info->type == IRQT_EVTCHN) ? info->u.interdomain : NULL;  in handle_irq_for_port()
1690 		info->eoi_cpu = smp_processor_id();  in handle_irq_for_port()
1691 		info->irq_epoch = __this_cpu_read(irq_epoch);  in handle_irq_for_port()
1692 		info->eoi_time = get_jiffies_64() + event_eoi_delay;  in handle_irq_for_port()
1695 	generic_handle_irq(info->irq);  in handle_irq_for_port()
1741 	struct irq_info *info = info_for_irq(irq);  in rebind_evtchn_irq()  local
1743 	if (WARN_ON(!info))  in rebind_evtchn_irq()
1756 	BUG_ON(info->type == IRQT_UNBOUND);  in rebind_evtchn_irq()
1758 	info->irq = irq;  in rebind_evtchn_irq()
1759 	(void)xen_irq_info_evtchn_setup(info, evtchn, NULL);  in rebind_evtchn_irq()
1763 	bind_evtchn_to_cpu(info, info->cpu, false);  in rebind_evtchn_irq()
1770 static int xen_rebind_evtchn_to_cpu(struct irq_info *info, unsigned int tcpu)  in xen_rebind_evtchn_to_cpu()  argument
1773 	evtchn_port_t evtchn = info ? info->evtchn : 0;  in xen_rebind_evtchn_to_cpu()
1789 	do_mask(info, EVT_MASK_REASON_TEMPORARY);  in xen_rebind_evtchn_to_cpu()
1797 		bind_evtchn_to_cpu(info, tcpu, false);  in xen_rebind_evtchn_to_cpu()
1799 	do_unmask(info, EVT_MASK_REASON_TEMPORARY);  in xen_rebind_evtchn_to_cpu()
1847 	struct irq_info *info = info_for_irq(data->irq);  in enable_dynirq()  local
1848 	evtchn_port_t evtchn = info ? info->evtchn : 0;  in enable_dynirq()
1851 		do_unmask(info, EVT_MASK_REASON_EXPLICIT);  in enable_dynirq()
1854 static void do_ack_dynirq(struct irq_info *info)  in do_ack_dynirq()  argument
1856 	evtchn_port_t evtchn = info->evtchn;  in do_ack_dynirq()
1859 		event_handler_exit(info);  in do_ack_dynirq()
1864 	struct irq_info *info = info_for_irq(data->irq);  in ack_dynirq()  local
1866 	if (info)  in ack_dynirq()
1867 		do_ack_dynirq(info);  in ack_dynirq()
1872 	struct irq_info *info = info_for_irq(data->irq);  in mask_ack_dynirq()  local
1874 	if (info) {  in mask_ack_dynirq()
1875 		do_disable_dynirq(info);  in mask_ack_dynirq()
1876 		do_ack_dynirq(info);  in mask_ack_dynirq()
1882 	struct irq_info *info = info_for_irq(data->irq);  in lateeoi_ack_dynirq()  local
1883 	evtchn_port_t evtchn = info ? info->evtchn : 0;  in lateeoi_ack_dynirq()
1886 		do_mask(info, EVT_MASK_REASON_EOI_PENDING);  in lateeoi_ack_dynirq()
1898 	struct irq_info *info = info_for_irq(data->irq);  in lateeoi_mask_ack_dynirq()  local
1899 	evtchn_port_t evtchn = info ? info->evtchn : 0;  in lateeoi_mask_ack_dynirq()
1902 		do_mask(info, EVT_MASK_REASON_EXPLICIT);  in lateeoi_mask_ack_dynirq()
1903 		event_handler_exit(info);  in lateeoi_mask_ack_dynirq()
1909 	struct irq_info *info = info_for_irq(data->irq);  in retrigger_dynirq()  local
1910 	evtchn_port_t evtchn = info ? info->evtchn : 0;  in retrigger_dynirq()
1915 	do_mask(info, EVT_MASK_REASON_TEMPORARY);  in retrigger_dynirq()
1917 	do_unmask(info, EVT_MASK_REASON_TEMPORARY);  in retrigger_dynirq()
1926 	struct irq_info *info;  in restore_pirqs()  local
1928 	list_for_each_entry(info, &xen_irq_list_head, list) {  in restore_pirqs()
1929 		if (info->type != IRQT_PIRQ)  in restore_pirqs()
1932 		pirq = info->u.pirq.pirq;  in restore_pirqs()
1933 		gsi = info->u.pirq.gsi;  in restore_pirqs()
1934 		irq = info->irq;  in restore_pirqs()
1950 			xen_free_irq(info);  in restore_pirqs()
1956 		__startup_pirq(info);  in restore_pirqs()
1964 	struct irq_info *info;  in restore_cpu_virqs()  local
1970 		info = info_for_irq(irq);  in restore_cpu_virqs()
1972 		BUG_ON(virq_from_irq(info) != virq);  in restore_cpu_virqs()
1983 		xen_irq_info_virq_setup(info, cpu, evtchn, virq);  in restore_cpu_virqs()
1985 		bind_evtchn_to_cpu(info, cpu, false);  in restore_cpu_virqs()
1993 	struct irq_info *info;  in restore_cpu_ipis()  local
1999 		info = info_for_irq(irq);  in restore_cpu_ipis()
2001 		BUG_ON(ipi_from_irq(info) != ipi);  in restore_cpu_ipis()
2011 		xen_irq_info_ipi_setup(info, cpu, evtchn, ipi);  in restore_cpu_ipis()
2013 		bind_evtchn_to_cpu(info, cpu, false);  in restore_cpu_ipis()
2020 	struct irq_info *info = info_for_irq(irq);  in xen_clear_irq_pending()  local
2021 	evtchn_port_t evtchn = info ? info->evtchn : 0;  in xen_clear_irq_pending()
2024 		event_handler_exit(info);  in xen_clear_irq_pending()
2067 	struct irq_info *info = info_for_irq(irq);  in xen_test_irq_shared()  local
2070 	if (WARN_ON(!info))  in xen_test_irq_shared()
2073 	irq_status.irq = info->u.pirq.pirq;  in xen_test_irq_shared()
2084 	struct irq_info *info;  in xen_irq_resume()  local
2090 	list_for_each_entry(info, &xen_irq_list_head, list) {  in xen_irq_resume()
2092 		info->evtchn = 0;  in xen_irq_resume()
2094 		channels_on_cpu_dec(info);  in xen_irq_resume()