139 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_bus_wake_up()  local
141 	return mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev);  in ath12k_pci_bus_wake_up()
146 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_bus_release()  local
148 	mhi_device_put(ab_pci->mhi_ctrl->mhi_dev);  in ath12k_pci_bus_release()
161 static void ath12k_pci_select_window(struct ath12k_pci *ab_pci, u32 offset)  in ath12k_pci_select_window()  argument
163 	struct ath12k_base *ab = ab_pci->ab;  in ath12k_pci_select_window()
168 	lockdep_assert_held(&ab_pci->window_lock);  in ath12k_pci_select_window()
171 	static_window = ab_pci->register_window & WINDOW_STATIC_MASK;  in ath12k_pci_select_window()
174 	if (window != ab_pci->register_window) {  in ath12k_pci_select_window()
178 		ab_pci->register_window = window;  in ath12k_pci_select_window()
182 static void ath12k_pci_select_static_window(struct ath12k_pci *ab_pci)  in ath12k_pci_select_static_window()  argument
190 	spin_lock_bh(&ab_pci->window_lock);  in ath12k_pci_select_static_window()
191 	ab_pci->register_window = window;  in ath12k_pci_select_static_window()
192 	spin_unlock_bh(&ab_pci->window_lock);  in ath12k_pci_select_static_window()
194 	iowrite32(WINDOW_ENABLE_BIT | window, ab_pci->ab->mem + WINDOW_REG_ADDRESS);  in ath12k_pci_select_static_window()
373 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_ce_irq_enable()  local
379 	if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags))  in ath12k_pci_ce_irq_enable()
388 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_ce_irq_disable()  local
394 	if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags))  in ath12k_pci_ce_irq_disable()
459 	struct ath12k_pci *ab_pci = ath12k_pci_priv(irq_grp->ab);  in ath12k_pci_ext_grp_disable()  local
465 	if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags))  in ath12k_pci_ext_grp_disable()
491 	struct ath12k_pci *ab_pci = ath12k_pci_priv(irq_grp->ab);  in ath12k_pci_ext_grp_enable()  local
497 	if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags))  in ath12k_pci_ext_grp_enable()
564 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_ext_irq_config()  local
617 					  ab_pci->irq_flags,  in ath12k_pci_ext_irq_config()
641 static int ath12k_pci_set_irq_affinity_hint(struct ath12k_pci *ab_pci,  in ath12k_pci_set_irq_affinity_hint()  argument
644 	if (test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags))  in ath12k_pci_set_irq_affinity_hint()
647 	return irq_set_affinity_hint(ab_pci->pdev->irq, m);  in ath12k_pci_set_irq_affinity_hint()
652 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_config_irq()  local
681 				  ab_pci->irq_flags, irq_name[irq_idx],  in ath12k_pci_config_irq()
706 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_init_qmi_ce_config()  local
707 	struct pci_bus *bus = ab_pci->pdev->bus;  in ath12k_pci_init_qmi_ce_config()
717 		ab_pci->qmi_instance =  in ath12k_pci_init_qmi_ce_config()
720 		ab->qmi.service_ins_id += ab_pci->qmi_instance;  in ath12k_pci_init_qmi_ce_config()
737 static void ath12k_pci_msi_config(struct ath12k_pci *ab_pci, bool enable)  in ath12k_pci_msi_config()  argument
739 	struct pci_dev *dev = ab_pci->pdev;  in ath12k_pci_msi_config()
752 static void ath12k_pci_msi_enable(struct ath12k_pci *ab_pci)  in ath12k_pci_msi_enable()  argument
754 	ath12k_pci_msi_config(ab_pci, true);  in ath12k_pci_msi_enable()
757 static void ath12k_pci_msi_disable(struct ath12k_pci *ab_pci)  in ath12k_pci_msi_disable()  argument
759 	ath12k_pci_msi_config(ab_pci, false);  in ath12k_pci_msi_disable()
762 static int ath12k_pci_msi_alloc(struct ath12k_pci *ab_pci)  in ath12k_pci_msi_alloc()  argument
764 	struct ath12k_base *ab = ab_pci->ab;  in ath12k_pci_msi_alloc()
765 	const struct ath12k_msi_config *msi_config = ab_pci->msi_config;  in ath12k_pci_msi_alloc()
770 	num_vectors = pci_alloc_irq_vectors(ab_pci->pdev,  in ath12k_pci_msi_alloc()
776 		set_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags);  in ath12k_pci_msi_alloc()
777 		ab_pci->irq_flags = IRQF_SHARED;  in ath12k_pci_msi_alloc()
779 		num_vectors = pci_alloc_irq_vectors(ab_pci->pdev,  in ath12k_pci_msi_alloc()
787 		clear_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags);  in ath12k_pci_msi_alloc()
788 		ab_pci->msi_config = &msi_config_one_msi;  in ath12k_pci_msi_alloc()
789 		ab_pci->irq_flags = IRQF_SHARED | IRQF_NOBALANCING;  in ath12k_pci_msi_alloc()
795 	ath12k_pci_msi_disable(ab_pci);  in ath12k_pci_msi_alloc()
797 	msi_desc = irq_get_msi_desc(ab_pci->pdev->irq);  in ath12k_pci_msi_alloc()
804 	ab_pci->msi_ep_base_data = msi_desc->msg.data;  in ath12k_pci_msi_alloc()
806 		set_bit(ATH12K_PCI_FLAG_IS_MSI_64, &ab_pci->flags);  in ath12k_pci_msi_alloc()
808 	ath12k_dbg(ab, ATH12K_DBG_PCI, "msi base data is %d\n", ab_pci->msi_ep_base_data);  in ath12k_pci_msi_alloc()
813 	pci_free_irq_vectors(ab_pci->pdev);  in ath12k_pci_msi_alloc()
819 static void ath12k_pci_msi_free(struct ath12k_pci *ab_pci)  in ath12k_pci_msi_free()  argument
821 	pci_free_irq_vectors(ab_pci->pdev);  in ath12k_pci_msi_free()
824 static int ath12k_pci_config_msi_data(struct ath12k_pci *ab_pci)  in ath12k_pci_config_msi_data()  argument
828 	msi_desc = irq_get_msi_desc(ab_pci->pdev->irq);  in ath12k_pci_config_msi_data()
830 		ath12k_err(ab_pci->ab, "msi_desc is NULL!\n");  in ath12k_pci_config_msi_data()
831 		pci_free_irq_vectors(ab_pci->pdev);  in ath12k_pci_config_msi_data()
835 	ab_pci->msi_ep_base_data = msi_desc->msg.data;  in ath12k_pci_config_msi_data()
837 	ath12k_dbg(ab_pci->ab, ATH12K_DBG_PCI, "pci after request_irq msi_ep_base_data %d\n",  in ath12k_pci_config_msi_data()
838 		   ab_pci->msi_ep_base_data);  in ath12k_pci_config_msi_data()
843 static int ath12k_pci_claim(struct ath12k_pci *ab_pci, struct pci_dev *pdev)  in ath12k_pci_claim()  argument
845 	struct ath12k_base *ab = ab_pci->ab;  in ath12k_pci_claim()
850 	if (device_id != ab_pci->dev_id)  {  in ath12k_pci_claim()
852 			   device_id, ab_pci->dev_id);  in ath12k_pci_claim()
904 static void ath12k_pci_free_region(struct ath12k_pci *ab_pci)  in ath12k_pci_free_region()  argument
906 	struct ath12k_base *ab = ab_pci->ab;  in ath12k_pci_free_region()
907 	struct pci_dev *pci_dev = ab_pci->pdev;  in ath12k_pci_free_region()
916 static void ath12k_pci_aspm_disable(struct ath12k_pci *ab_pci)  in ath12k_pci_aspm_disable()  argument
918 	struct ath12k_base *ab = ab_pci->ab;  in ath12k_pci_aspm_disable()
920 	pcie_capability_read_word(ab_pci->pdev, PCI_EXP_LNKCTL,  in ath12k_pci_aspm_disable()
921 				  &ab_pci->link_ctl);  in ath12k_pci_aspm_disable()
924 		   ab_pci->link_ctl,  in ath12k_pci_aspm_disable()
925 		   u16_get_bits(ab_pci->link_ctl, PCI_EXP_LNKCTL_ASPM_L0S),  in ath12k_pci_aspm_disable()
926 		   u16_get_bits(ab_pci->link_ctl, PCI_EXP_LNKCTL_ASPM_L1));  in ath12k_pci_aspm_disable()
929 	pcie_capability_clear_word(ab_pci->pdev, PCI_EXP_LNKCTL,  in ath12k_pci_aspm_disable()
932 	set_bit(ATH12K_PCI_ASPM_RESTORE, &ab_pci->flags);  in ath12k_pci_aspm_disable()
937 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_update_qrtr_node_id()  local
949 	ath12k_pci_write32(ab, reg, ab_pci->qmi_instance);  in ath12k_pci_update_qrtr_node_id()
952 		   reg, ab_pci->qmi_instance, ath12k_pci_read32(ab, reg));  in ath12k_pci_update_qrtr_node_id()
955 static void ath12k_pci_aspm_restore(struct ath12k_pci *ab_pci)  in ath12k_pci_aspm_restore()  argument
957 	if (ab_pci->ab->hw_params->supports_aspm &&  in ath12k_pci_aspm_restore()
958 	    test_and_clear_bit(ATH12K_PCI_ASPM_RESTORE, &ab_pci->flags))  in ath12k_pci_aspm_restore()
959 		pcie_capability_clear_and_set_word(ab_pci->pdev, PCI_EXP_LNKCTL,  in ath12k_pci_aspm_restore()
961 						   ab_pci->link_ctl &  in ath12k_pci_aspm_restore()
1040 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_get_user_msi_assignment()  local
1041 	const struct ath12k_msi_config *msi_config = ab_pci->msi_config;  in ath12k_pci_get_user_msi_assignment()
1048 			*user_base_data = *base_vector + ab_pci->msi_ep_base_data;  in ath12k_pci_get_user_msi_assignment()
1067 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_get_msi_address()  local
1073 	if (test_bit(ATH12K_PCI_FLAG_IS_MSI_64, &ab_pci->flags)) {  in ath12k_pci_get_msi_address()
1154 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_start()  local
1156 	set_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags);  in ath12k_pci_start()
1158 	if (test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags))  in ath12k_pci_start()
1159 		ath12k_pci_aspm_restore(ab_pci);  in ath12k_pci_start()
1171 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_read32()  local
1178 	if (test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags) &&  in ath12k_pci_read32()
1179 	    offset >= ACCESS_ALWAYS_OFF && ab_pci->pci_ops->wakeup)  in ath12k_pci_read32()
1180 		ret = ab_pci->pci_ops->wakeup(ab);  in ath12k_pci_read32()
1191 			spin_lock_bh(&ab_pci->window_lock);  in ath12k_pci_read32()
1192 			ath12k_pci_select_window(ab_pci, offset);  in ath12k_pci_read32()
1202 			spin_unlock_bh(&ab_pci->window_lock);  in ath12k_pci_read32()
1209 	if (test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags) &&  in ath12k_pci_read32()
1210 	    offset >= ACCESS_ALWAYS_OFF && ab_pci->pci_ops->release &&  in ath12k_pci_read32()
1212 		ab_pci->pci_ops->release(ab);  in ath12k_pci_read32()
1218 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_write32()  local
1225 	if (test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags) &&  in ath12k_pci_write32()
1226 	    offset >= ACCESS_ALWAYS_OFF && ab_pci->pci_ops->wakeup)  in ath12k_pci_write32()
1227 		ret = ab_pci->pci_ops->wakeup(ab);  in ath12k_pci_write32()
1238 			spin_lock_bh(&ab_pci->window_lock);  in ath12k_pci_write32()
1239 			ath12k_pci_select_window(ab_pci, offset);  in ath12k_pci_write32()
1249 			spin_unlock_bh(&ab_pci->window_lock);  in ath12k_pci_write32()
1256 	if (test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags) &&  in ath12k_pci_write32()
1257 	    offset >= ACCESS_ALWAYS_OFF && ab_pci->pci_ops->release &&  in ath12k_pci_write32()
1259 		ab_pci->pci_ops->release(ab);  in ath12k_pci_write32()
1264 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_power_up()  local
1267 	ab_pci->register_window = 0;  in ath12k_pci_power_up()
1268 	clear_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags);  in ath12k_pci_power_up()
1269 	ath12k_pci_sw_reset(ab_pci->ab, true);  in ath12k_pci_power_up()
1274 	ath12k_pci_aspm_disable(ab_pci);  in ath12k_pci_power_up()
1276 	ath12k_pci_msi_enable(ab_pci);  in ath12k_pci_power_up()
1281 	ret = ath12k_mhi_start(ab_pci);  in ath12k_pci_power_up()
1288 		ath12k_pci_select_static_window(ab_pci);  in ath12k_pci_power_up()
1295 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_power_down()  local
1298 	ath12k_pci_aspm_restore(ab_pci);  in ath12k_pci_power_down()
1300 	ath12k_pci_force_wake(ab_pci->ab);  in ath12k_pci_power_down()
1301 	ath12k_pci_msi_disable(ab_pci);  in ath12k_pci_power_down()
1302 	ath12k_mhi_stop(ab_pci, is_suspend);  in ath12k_pci_power_down()
1303 	clear_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags);  in ath12k_pci_power_down()
1304 	ath12k_pci_sw_reset(ab_pci->ab, false);  in ath12k_pci_power_down()
1354 	struct ath12k_pci *ab_pci;  in ath12k_pci_probe()  local
1358 	ab = ath12k_core_alloc(&pdev->dev, sizeof(*ab_pci), ATH12K_BUS_PCI);  in ath12k_pci_probe()
1366 	ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_probe()
1367 	ab_pci->dev_id = pci_dev->device;  in ath12k_pci_probe()
1368 	ab_pci->ab = ab;  in ath12k_pci_probe()
1369 	ab_pci->pdev = pdev;  in ath12k_pci_probe()
1372 	spin_lock_init(&ab_pci->window_lock);  in ath12k_pci_probe()
1374 	ret = ath12k_pci_claim(ab_pci, pdev);  in ath12k_pci_probe()
1391 		ab_pci->msi_config = &ath12k_msi_config[0];  in ath12k_pci_probe()
1393 		ab_pci->pci_ops = &ath12k_pci_ops_qcn9274;  in ath12k_pci_probe()
1414 		ab_pci->msi_config = &ath12k_msi_config[0];  in ath12k_pci_probe()
1416 		ab_pci->pci_ops = &ath12k_pci_ops_wcn7850;  in ath12k_pci_probe()
1440 	ret = ath12k_pci_msi_alloc(ab_pci);  in ath12k_pci_probe()
1450 	ret = ath12k_pci_set_irq_affinity_hint(ab_pci, cpumask_of(0));  in ath12k_pci_probe()
1456 	ret = ath12k_mhi_register(ab_pci);  in ath12k_pci_probe()
1485 	ret = ath12k_pci_config_msi_data(ab_pci);  in ath12k_pci_probe()
1508 	ath12k_mhi_unregister(ab_pci);  in ath12k_pci_probe()
1511 	ath12k_pci_msi_free(ab_pci);  in ath12k_pci_probe()
1514 	ath12k_pci_set_irq_affinity_hint(ab_pci, NULL);  in ath12k_pci_probe()
1517 	ath12k_pci_free_region(ab_pci);  in ath12k_pci_probe()
1528 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_remove()  local
1530 	ath12k_pci_set_irq_affinity_hint(ab_pci, NULL);  in ath12k_pci_remove()
1544 	ath12k_mhi_unregister(ab_pci);  in ath12k_pci_remove()
1547 	ath12k_pci_msi_free(ab_pci);  in ath12k_pci_remove()
1548 	ath12k_pci_free_region(ab_pci);  in ath12k_pci_remove()
1558 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);  in ath12k_pci_shutdown()  local
1560 	ath12k_pci_set_irq_affinity_hint(ab_pci, NULL);  in ath12k_pci_shutdown()