61 static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
126 			     struct kvmppc_icp *icp)  in ics_check_resend()  argument
135 			icp_deliver_irq(xics, icp, state->number, true);  in ics_check_resend()
169 	struct kvmppc_icp *icp;  in kvmppc_xics_set_xive()  local
182 	icp = kvmppc_xics_find_server(kvm, server);  in kvmppc_xics_set_xive()
183 	if (!icp)  in kvmppc_xics_set_xive()
191 		icp_deliver_irq(xics, icp, irq, false);  in kvmppc_xics_set_xive()
225 	struct kvmppc_icp *icp;  in kvmppc_xics_int_on()  local
238 	icp = kvmppc_xics_find_server(kvm, state->server);  in kvmppc_xics_int_on()
239 	if (!icp)  in kvmppc_xics_int_on()
244 		icp_deliver_irq(xics, icp, irq, false);  in kvmppc_xics_int_on()
271 static inline bool icp_try_update(struct kvmppc_icp *icp,  in icp_try_update()  argument
282 	success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw;  in icp_try_update()
287 		 icp->server_num,  in icp_try_update()
309 		kvmppc_book3s_queue_irqprio(icp->vcpu,  in icp_try_update()
312 			kvmppc_fast_vcpu_kick(icp->vcpu);  in icp_try_update()
319 			     struct kvmppc_icp *icp)  in icp_check_resend()  argument
325 	for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) {  in icp_check_resend()
328 		if (!test_and_clear_bit(icsid, icp->resend_map))  in icp_check_resend()
332 		ics_check_resend(xics, ics, icp);  in icp_check_resend()
336 static bool icp_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority,  in icp_try_to_deliver()  argument
343 		 icp->server_num);  in icp_try_to_deliver()
346 		old_state = new_state = READ_ONCE(icp->state);  in icp_try_to_deliver()
372 	} while (!icp_try_update(icp, old_state, new_state, false));  in icp_try_to_deliver()
377 static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,  in icp_deliver_irq()  argument
415 	if (!icp || state->server != icp->server_num) {  in icp_deliver_irq()
416 		icp = kvmppc_xics_find_server(xics->kvm, state->server);  in icp_deliver_irq()
417 		if (!icp) {  in icp_deliver_irq()
468 	if (icp_try_to_deliver(icp, new_irq, state->priority, &reject)) {  in icp_deliver_irq()
491 		set_bit(ics->icsid, icp->resend_map);  in icp_deliver_irq()
500 		if (!icp->state.need_resend) {  in icp_deliver_irq()
513 static void icp_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,  in icp_down_cppr()  argument
549 		old_state = new_state = READ_ONCE(icp->state);  in icp_down_cppr()
575 	} while (!icp_try_update(icp, old_state, new_state, true));  in icp_down_cppr()
583 		icp_check_resend(xics, icp);  in icp_down_cppr()
589 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_h_xirr()  local
593 	kvmppc_book3s_dequeue_irqprio(icp->vcpu, BOOK3S_INTERRUPT_EXTERNAL);  in kvmppc_h_xirr()
603 		old_state = new_state = READ_ONCE(icp->state);  in kvmppc_h_xirr()
612 	} while (!icp_try_update(icp, old_state, new_state, true));  in kvmppc_h_xirr()
624 	struct kvmppc_icp *icp;  in kvmppc_h_ipi()  local
632 	icp = vcpu->arch.icp;  in kvmppc_h_ipi()
633 	local = icp->server_num == server;  in kvmppc_h_ipi()
635 		icp = kvmppc_xics_find_server(vcpu->kvm, server);  in kvmppc_h_ipi()
636 		if (!icp)  in kvmppc_h_ipi()
670 		old_state = new_state = READ_ONCE(icp->state);  in kvmppc_h_ipi()
691 	} while (!icp_try_update(icp, old_state, new_state, local));  in kvmppc_h_ipi()
695 		icp_deliver_irq(xics, icp, reject, false);  in kvmppc_h_ipi()
699 		icp_check_resend(xics, icp);  in kvmppc_h_ipi()
707 	struct kvmppc_icp *icp;  in kvmppc_h_ipoll()  local
709 	icp = vcpu->arch.icp;  in kvmppc_h_ipoll()
710 	if (icp->server_num != server) {  in kvmppc_h_ipoll()
711 		icp = kvmppc_xics_find_server(vcpu->kvm, server);  in kvmppc_h_ipoll()
712 		if (!icp)  in kvmppc_h_ipoll()
715 	state = READ_ONCE(icp->state);  in kvmppc_h_ipoll()
725 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_h_cppr()  local
737 	if (cppr > icp->state.cppr)  in kvmppc_h_cppr()
738 		icp_down_cppr(xics, icp, cppr);  in kvmppc_h_cppr()
739 	else if (cppr == icp->state.cppr)  in kvmppc_h_cppr()
753 	kvmppc_book3s_dequeue_irqprio(icp->vcpu, BOOK3S_INTERRUPT_EXTERNAL);  in kvmppc_h_cppr()
756 		old_state = new_state = READ_ONCE(icp->state);  in kvmppc_h_cppr()
767 	} while (!icp_try_update(icp, old_state, new_state, true));  in kvmppc_h_cppr()
774 		icp_deliver_irq(xics, icp, reject, false);  in kvmppc_h_cppr()
780 	struct kvmppc_icp *icp = vcpu->arch.icp;  in ics_eoi()  local
810 		icp_deliver_irq(xics, icp, irq, false);  in ics_eoi()
820 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_h_eoi()  local
839 	icp_down_cppr(xics, icp, xirr >> 24);  in kvmppc_h_eoi()
851 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_xics_rm_complete()  local
854 		 hcall, icp->rm_action, icp->rm_dbgstate.raw, icp->rm_dbgtgt);  in kvmppc_xics_rm_complete()
856 	if (icp->rm_action & XICS_RM_KICK_VCPU) {  in kvmppc_xics_rm_complete()
857 		icp->n_rm_kick_vcpu++;  in kvmppc_xics_rm_complete()
858 		kvmppc_fast_vcpu_kick(icp->rm_kick_target);  in kvmppc_xics_rm_complete()
860 	if (icp->rm_action & XICS_RM_CHECK_RESEND) {  in kvmppc_xics_rm_complete()
861 		icp->n_rm_check_resend++;  in kvmppc_xics_rm_complete()
862 		icp_check_resend(xics, icp->rm_resend_icp);  in kvmppc_xics_rm_complete()
864 	if (icp->rm_action & XICS_RM_NOTIFY_EOI) {  in kvmppc_xics_rm_complete()
865 		icp->n_rm_notify_eoi++;  in kvmppc_xics_rm_complete()
866 		kvm_notify_acked_irq(vcpu->kvm, 0, icp->rm_eoied_irq);  in kvmppc_xics_rm_complete()
869 	icp->rm_action = 0;  in kvmppc_xics_rm_complete()
882 	if (!xics || !vcpu->arch.icp)  in kvmppc_xics_hcall()
965 		struct kvmppc_icp *icp = vcpu->arch.icp;  in xics_debug_show()  local
968 		if (!icp)  in xics_debug_show()
971 		state.raw = READ_ONCE(icp->state.raw);  in xics_debug_show()
973 			   icp->server_num, state.xisr,  in xics_debug_show()
976 		t_rm_kick_vcpu += icp->n_rm_kick_vcpu;  in xics_debug_show()
977 		t_rm_notify_eoi += icp->n_rm_notify_eoi;  in xics_debug_show()
978 		t_rm_check_resend += icp->n_rm_check_resend;  in xics_debug_show()
979 		t_check_resend += icp->n_check_resend;  in xics_debug_show()
980 		t_reject += icp->n_reject;  in xics_debug_show()
1064 	struct kvmppc_icp *icp;  in kvmppc_xics_create_icp()  local
1072 	icp = kzalloc(sizeof(struct kvmppc_icp), GFP_KERNEL);  in kvmppc_xics_create_icp()
1073 	if (!icp)  in kvmppc_xics_create_icp()
1076 	icp->vcpu = vcpu;  in kvmppc_xics_create_icp()
1077 	icp->server_num = server_num;  in kvmppc_xics_create_icp()
1078 	icp->state.mfrr = MASKED;  in kvmppc_xics_create_icp()
1079 	icp->state.pending_pri = MASKED;  in kvmppc_xics_create_icp()
1080 	vcpu->arch.icp = icp;  in kvmppc_xics_create_icp()
1089 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_xics_get_icp()  local
1092 	if (!icp)  in kvmppc_xics_get_icp()
1094 	state = icp->state;  in kvmppc_xics_get_icp()
1103 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_xics_set_icp()  local
1112 	if (!icp || !xics)  in kvmppc_xics_set_icp()
1146 	kvmppc_book3s_dequeue_irqprio(icp->vcpu, BOOK3S_INTERRUPT_EXTERNAL);  in kvmppc_xics_set_icp()
1159 		old_state = READ_ONCE(icp->state);  in kvmppc_xics_set_icp()
1168 	} while (!icp_try_update(icp, old_state, new_state, false));  in kvmppc_xics_set_icp()
1171 		icp_check_resend(xics, icp);  in kvmppc_xics_set_icp()
1471 	if (!vcpu->arch.icp)  in kvmppc_xics_free_icp()
1473 	kfree(vcpu->arch.icp);  in kvmppc_xics_free_icp()
1474 	vcpu->arch.icp = NULL;  in kvmppc_xics_free_icp()