3  * MediaTek PCIe host controller driver.
123 /* Time in ms needed to complete PCIe reset on EN7581 SoC */
130  * @power_up: pcie power_up callback
134 	int (*power_up)(struct mtk_gen3_pcie *pcie);
154  * struct mtk_gen3_pcie - PCIe port information
155  * @dev: pointer to PCIe device
161  * @clks: PCIe clocks
162  * @num_clks: PCIe clocks count for this port
163  * @irq: PCIe controller interrupt number
240 	struct mtk_gen3_pcie *pcie = bus->sysdata;  in mtk_pcie_config_tlp_header()  local
249 	writel_relaxed(val, pcie->base + PCIE_CFGNUM_REG);  in mtk_pcie_config_tlp_header()
255 	struct mtk_gen3_pcie *pcie = bus->sysdata;  in mtk_pcie_map_bus()  local
257 	return pcie->base + PCIE_CFG_OFFSET_ADDR + where;  in mtk_pcie_map_bus()
285 static int mtk_pcie_set_trans_table(struct mtk_gen3_pcie *pcie,  in mtk_pcie_set_trans_table()  argument
309 			dev_err(pcie->dev, "illegal table size %#llx\n",  in mtk_pcie_set_trans_table()
314 		table = pcie->base + PCIE_TRANS_TABLE_BASE_REG + *num * PCIE_ATR_TLB_SET_OFFSET;  in mtk_pcie_set_trans_table()
330 		dev_dbg(pcie->dev, "set %s trans window[%d]: cpu_addr = %#llx, pci_addr = %#llx, size = %#llx\n",  in mtk_pcie_set_trans_table()
341 		dev_warn(pcie->dev, "not enough translate table for addr: %#llx, limited to [%d]\n",  in mtk_pcie_set_trans_table()
347 static void mtk_pcie_enable_msi(struct mtk_gen3_pcie *pcie)  in mtk_pcie_enable_msi()  argument
353 		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];  in mtk_pcie_enable_msi()
355 		msi_set->base = pcie->base + PCIE_MSI_SET_BASE_REG +  in mtk_pcie_enable_msi()
357 		msi_set->msg_addr = pcie->reg_base + PCIE_MSI_SET_BASE_REG +  in mtk_pcie_enable_msi()
363 			       pcie->base + PCIE_MSI_SET_ADDR_HI_BASE +  in mtk_pcie_enable_msi()
367 	val = readl_relaxed(pcie->base + PCIE_MSI_SET_ENABLE_REG);  in mtk_pcie_enable_msi()
369 	writel_relaxed(val, pcie->base + PCIE_MSI_SET_ENABLE_REG);  in mtk_pcie_enable_msi()
371 	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_enable_msi()
373 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_enable_msi()
376 static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)  in mtk_pcie_startup_port()  argument
379 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in mtk_pcie_startup_port()
385 	val = readl_relaxed(pcie->base + PCIE_SETTING_REG);  in mtk_pcie_startup_port()
387 	writel_relaxed(val, pcie->base + PCIE_SETTING_REG);  in mtk_pcie_startup_port()
390 	val = readl_relaxed(pcie->base + PCIE_PCI_IDS_1);  in mtk_pcie_startup_port()
393 	writel_relaxed(val, pcie->base + PCIE_PCI_IDS_1);  in mtk_pcie_startup_port()
396 	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_startup_port()
398 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_startup_port()
401 	val = readl_relaxed(pcie->base + PCIE_MISC_CTRL_REG);  in mtk_pcie_startup_port()
403 	writel_relaxed(val, pcie->base + PCIE_MISC_CTRL_REG);  in mtk_pcie_startup_port()
406 	val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_startup_port()
408 	writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_startup_port()
411 	 * Described in PCIe CEM specification sections 2.2 (PERST# Signal)  in mtk_pcie_startup_port()
420 	writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_startup_port()
423 	err = readl_poll_timeout(pcie->base + PCIE_LINK_STATUS_REG, val,  in mtk_pcie_startup_port()
430 		val = readl_relaxed(pcie->base + PCIE_LTSSM_STATUS_REG);  in mtk_pcie_startup_port()
434 		dev_err(pcie->dev,  in mtk_pcie_startup_port()
435 			"PCIe link down, current LTSSM state: %s (%#x)\n",  in mtk_pcie_startup_port()
440 	mtk_pcie_enable_msi(pcie);  in mtk_pcie_startup_port()
442 	/* Set PCIe translation windows */  in mtk_pcie_startup_port()
459 		err = mtk_pcie_set_trans_table(pcie, cpu_addr, pci_addr, size,  in mtk_pcie_startup_port()
497 	struct mtk_gen3_pcie *pcie = data->domain->host_data;  in mtk_compose_msi_msg()  local
505 	dev_dbg(pcie->dev, "msi#%#lx address_hi %#x address_lo %#x data %d\n",  in mtk_compose_msi_msg()
522 	struct mtk_gen3_pcie *pcie = data->domain->host_data;  in mtk_msi_bottom_irq_mask()  local
528 	raw_spin_lock_irqsave(&pcie->irq_lock, flags);  in mtk_msi_bottom_irq_mask()
532 	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);  in mtk_msi_bottom_irq_mask()
538 	struct mtk_gen3_pcie *pcie = data->domain->host_data;  in mtk_msi_bottom_irq_unmask()  local
544 	raw_spin_lock_irqsave(&pcie->irq_lock, flags);  in mtk_msi_bottom_irq_unmask()
548 	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);  in mtk_msi_bottom_irq_unmask()
563 	struct mtk_gen3_pcie *pcie = domain->host_data;  in mtk_msi_bottom_domain_alloc()  local
567 	mutex_lock(&pcie->lock);  in mtk_msi_bottom_domain_alloc()
569 	hwirq = bitmap_find_free_region(pcie->msi_irq_in_use, PCIE_MSI_IRQS_NUM,  in mtk_msi_bottom_domain_alloc()
572 	mutex_unlock(&pcie->lock);  in mtk_msi_bottom_domain_alloc()
578 	msi_set = &pcie->msi_sets[set_idx];  in mtk_msi_bottom_domain_alloc()
591 	struct mtk_gen3_pcie *pcie = domain->host_data;  in mtk_msi_bottom_domain_free()  local
594 	mutex_lock(&pcie->lock);  in mtk_msi_bottom_domain_free()
596 	bitmap_release_region(pcie->msi_irq_in_use, data->hwirq,  in mtk_msi_bottom_domain_free()
599 	mutex_unlock(&pcie->lock);  in mtk_msi_bottom_domain_free()
611 	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);  in mtk_intx_mask()  local
615 	raw_spin_lock_irqsave(&pcie->irq_lock, flags);  in mtk_intx_mask()
616 	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_intx_mask()
618 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_intx_mask()
619 	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);  in mtk_intx_mask()
624 	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);  in mtk_intx_unmask()  local
628 	raw_spin_lock_irqsave(&pcie->irq_lock, flags);  in mtk_intx_unmask()
629 	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_intx_unmask()
631 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_intx_unmask()
632 	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);  in mtk_intx_unmask()
645 	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);  in mtk_intx_eoi()  local
649 	writel_relaxed(BIT(hwirq), pcie->base + PCIE_INT_STATUS_REG);  in mtk_intx_eoi()
672 static int mtk_pcie_init_irq_domains(struct mtk_gen3_pcie *pcie)  in mtk_pcie_init_irq_domains()  argument
674 	struct device *dev = pcie->dev;  in mtk_pcie_init_irq_domains()
678 	raw_spin_lock_init(&pcie->irq_lock);  in mtk_pcie_init_irq_domains()
687 	pcie->intx_domain = irq_domain_add_linear(intc_node, PCI_NUM_INTX,  in mtk_pcie_init_irq_domains()
688 						  &intx_domain_ops, pcie);  in mtk_pcie_init_irq_domains()
689 	if (!pcie->intx_domain) {  in mtk_pcie_init_irq_domains()
696 	mutex_init(&pcie->lock);  in mtk_pcie_init_irq_domains()
698 	pcie->msi_bottom_domain = irq_domain_add_linear(node, PCIE_MSI_IRQS_NUM,  in mtk_pcie_init_irq_domains()
699 				  &mtk_msi_bottom_domain_ops, pcie);  in mtk_pcie_init_irq_domains()
700 	if (!pcie->msi_bottom_domain) {  in mtk_pcie_init_irq_domains()
706 	pcie->msi_domain = pci_msi_create_irq_domain(dev->fwnode,  in mtk_pcie_init_irq_domains()
708 						     pcie->msi_bottom_domain);  in mtk_pcie_init_irq_domains()
709 	if (!pcie->msi_domain) {  in mtk_pcie_init_irq_domains()
719 	irq_domain_remove(pcie->msi_bottom_domain);  in mtk_pcie_init_irq_domains()
721 	irq_domain_remove(pcie->intx_domain);  in mtk_pcie_init_irq_domains()
727 static void mtk_pcie_irq_teardown(struct mtk_gen3_pcie *pcie)  in mtk_pcie_irq_teardown()  argument
729 	irq_set_chained_handler_and_data(pcie->irq, NULL, NULL);  in mtk_pcie_irq_teardown()
731 	if (pcie->intx_domain)  in mtk_pcie_irq_teardown()
732 		irq_domain_remove(pcie->intx_domain);  in mtk_pcie_irq_teardown()
734 	if (pcie->msi_domain)  in mtk_pcie_irq_teardown()
735 		irq_domain_remove(pcie->msi_domain);  in mtk_pcie_irq_teardown()
737 	if (pcie->msi_bottom_domain)  in mtk_pcie_irq_teardown()
738 		irq_domain_remove(pcie->msi_bottom_domain);  in mtk_pcie_irq_teardown()
740 	irq_dispose_mapping(pcie->irq);  in mtk_pcie_irq_teardown()
743 static void mtk_pcie_msi_handler(struct mtk_gen3_pcie *pcie, int set_idx)  in mtk_pcie_msi_handler()  argument
745 	struct mtk_msi_set *msi_set = &pcie->msi_sets[set_idx];  in mtk_pcie_msi_handler()
760 			generic_handle_domain_irq(pcie->msi_bottom_domain, hwirq);  in mtk_pcie_msi_handler()
767 	struct mtk_gen3_pcie *pcie = irq_desc_get_handler_data(desc);  in mtk_pcie_irq_handler()  local
774 	status = readl_relaxed(pcie->base + PCIE_INT_STATUS_REG);  in mtk_pcie_irq_handler()
777 		generic_handle_domain_irq(pcie->intx_domain,  in mtk_pcie_irq_handler()
783 		mtk_pcie_msi_handler(pcie, irq_bit - PCIE_MSI_SHIFT);  in mtk_pcie_irq_handler()
785 		writel_relaxed(BIT(irq_bit), pcie->base + PCIE_INT_STATUS_REG);  in mtk_pcie_irq_handler()
791 static int mtk_pcie_setup_irq(struct mtk_gen3_pcie *pcie)  in mtk_pcie_setup_irq()  argument
793 	struct device *dev = pcie->dev;  in mtk_pcie_setup_irq()
797 	err = mtk_pcie_init_irq_domains(pcie);  in mtk_pcie_setup_irq()
801 	pcie->irq = platform_get_irq(pdev, 0);  in mtk_pcie_setup_irq()
802 	if (pcie->irq < 0)  in mtk_pcie_setup_irq()
803 		return pcie->irq;  in mtk_pcie_setup_irq()
805 	irq_set_chained_handler_and_data(pcie->irq, mtk_pcie_irq_handler, pcie);  in mtk_pcie_setup_irq()
810 static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie)  in mtk_pcie_parse_port()  argument
812 	int i, ret, num_resets = pcie->soc->phy_resets.num_resets;  in mtk_pcie_parse_port()
813 	struct device *dev = pcie->dev;  in mtk_pcie_parse_port()
817 	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-mac");  in mtk_pcie_parse_port()
820 	pcie->base = devm_ioremap_resource(dev, regs);  in mtk_pcie_parse_port()
821 	if (IS_ERR(pcie->base)) {  in mtk_pcie_parse_port()
823 		return PTR_ERR(pcie->base);  in mtk_pcie_parse_port()
826 	pcie->reg_base = regs->start;  in mtk_pcie_parse_port()
829 		pcie->phy_resets[i].id = pcie->soc->phy_resets.id[i];  in mtk_pcie_parse_port()
831 	ret = devm_reset_control_bulk_get_optional_shared(dev, num_resets, pcie->phy_resets);  in mtk_pcie_parse_port()
837 	pcie->mac_reset = devm_reset_control_get_optional_exclusive(dev, "mac");  in mtk_pcie_parse_port()
838 	if (IS_ERR(pcie->mac_reset)) {  in mtk_pcie_parse_port()
839 		ret = PTR_ERR(pcie->mac_reset);  in mtk_pcie_parse_port()
846 	pcie->phy = devm_phy_optional_get(dev, "pcie-phy");  in mtk_pcie_parse_port()
847 	if (IS_ERR(pcie->phy)) {  in mtk_pcie_parse_port()
848 		ret = PTR_ERR(pcie->phy);  in mtk_pcie_parse_port()
855 	pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks);  in mtk_pcie_parse_port()
856 	if (pcie->num_clks < 0) {  in mtk_pcie_parse_port()
858 		return pcie->num_clks;  in mtk_pcie_parse_port()
864 static int mtk_pcie_en7581_power_up(struct mtk_gen3_pcie *pcie)  in mtk_pcie_en7581_power_up()  argument
866 	struct device *dev = pcie->dev;  in mtk_pcie_en7581_power_up()
876 	err = phy_init(pcie->phy);  in mtk_pcie_en7581_power_up()
882 	err = phy_power_on(pcie->phy);  in mtk_pcie_en7581_power_up()
888 	err = reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, pcie->phy_resets);  in mtk_pcie_en7581_power_up()
903 	err = clk_bulk_prepare(pcie->num_clks, pcie->clks);  in mtk_pcie_en7581_power_up()
913 	writel_relaxed(val, pcie->base + PCIE_EQ_PRESET_01_REG);  in mtk_pcie_en7581_power_up()
919 	writel_relaxed(val, pcie->base + PCIE_PIPE4_PIE8_REG);  in mtk_pcie_en7581_power_up()
921 	err = clk_bulk_enable(pcie->num_clks, pcie->clks);  in mtk_pcie_en7581_power_up()
930 	clk_bulk_unprepare(pcie->num_clks, pcie->clks);  in mtk_pcie_en7581_power_up()
934 	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets);  in mtk_pcie_en7581_power_up()
936 	phy_power_off(pcie->phy);  in mtk_pcie_en7581_power_up()
938 	phy_exit(pcie->phy);  in mtk_pcie_en7581_power_up()
943 static int mtk_pcie_power_up(struct mtk_gen3_pcie *pcie)  in mtk_pcie_power_up()  argument
945 	struct device *dev = pcie->dev;  in mtk_pcie_power_up()
949 	err = reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, pcie->phy_resets);  in mtk_pcie_power_up()
955 	err = phy_init(pcie->phy);  in mtk_pcie_power_up()
961 	err = phy_power_on(pcie->phy);  in mtk_pcie_power_up()
968 	reset_control_deassert(pcie->mac_reset);  in mtk_pcie_power_up()
973 	err = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks);  in mtk_pcie_power_up()
984 	reset_control_assert(pcie->mac_reset);  in mtk_pcie_power_up()
985 	phy_power_off(pcie->phy);  in mtk_pcie_power_up()
987 	phy_exit(pcie->phy);  in mtk_pcie_power_up()
989 	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets);  in mtk_pcie_power_up()
994 static void mtk_pcie_power_down(struct mtk_gen3_pcie *pcie)  in mtk_pcie_power_down()  argument
996 	clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks);  in mtk_pcie_power_down()
998 	pm_runtime_put_sync(pcie->dev);  in mtk_pcie_power_down()
999 	pm_runtime_disable(pcie->dev);  in mtk_pcie_power_down()
1000 	reset_control_assert(pcie->mac_reset);  in mtk_pcie_power_down()
1002 	phy_power_off(pcie->phy);  in mtk_pcie_power_down()
1003 	phy_exit(pcie->phy);  in mtk_pcie_power_down()
1004 	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets);  in mtk_pcie_power_down()
1007 static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie)  in mtk_pcie_setup()  argument
1011 	err = mtk_pcie_parse_port(pcie);  in mtk_pcie_setup()
1019 	reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, pcie->phy_resets);  in mtk_pcie_setup()
1024 	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets);  in mtk_pcie_setup()
1026 	reset_control_assert(pcie->mac_reset);  in mtk_pcie_setup()
1030 	err = pcie->soc->power_up(pcie);  in mtk_pcie_setup()
1035 	err = mtk_pcie_startup_port(pcie);  in mtk_pcie_setup()
1039 	err = mtk_pcie_setup_irq(pcie);  in mtk_pcie_setup()
1046 	mtk_pcie_power_down(pcie);  in mtk_pcie_setup()
1054 	struct mtk_gen3_pcie *pcie;  in mtk_pcie_probe()  local
1058 	host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));  in mtk_pcie_probe()
1062 	pcie = pci_host_bridge_priv(host);  in mtk_pcie_probe()
1064 	pcie->dev = dev;  in mtk_pcie_probe()
1065 	pcie->soc = device_get_match_data(dev);  in mtk_pcie_probe()
1066 	platform_set_drvdata(pdev, pcie);  in mtk_pcie_probe()
1068 	err = mtk_pcie_setup(pcie);  in mtk_pcie_probe()
1073 	host->sysdata = pcie;  in mtk_pcie_probe()
1077 		mtk_pcie_irq_teardown(pcie);  in mtk_pcie_probe()
1078 		mtk_pcie_power_down(pcie);  in mtk_pcie_probe()
1087 	struct mtk_gen3_pcie *pcie = platform_get_drvdata(pdev);  in mtk_pcie_remove()  local
1088 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in mtk_pcie_remove()
1095 	mtk_pcie_irq_teardown(pcie);  in mtk_pcie_remove()
1096 	mtk_pcie_power_down(pcie);  in mtk_pcie_remove()
1099 static void mtk_pcie_irq_save(struct mtk_gen3_pcie *pcie)  in mtk_pcie_irq_save()  argument
1103 	raw_spin_lock(&pcie->irq_lock);  in mtk_pcie_irq_save()
1105 	pcie->saved_irq_state = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_irq_save()
1108 		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];  in mtk_pcie_irq_save()
1114 	raw_spin_unlock(&pcie->irq_lock);  in mtk_pcie_irq_save()
1117 static void mtk_pcie_irq_restore(struct mtk_gen3_pcie *pcie)  in mtk_pcie_irq_restore()  argument
1121 	raw_spin_lock(&pcie->irq_lock);  in mtk_pcie_irq_restore()
1123 	writel_relaxed(pcie->saved_irq_state, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_irq_restore()
1126 		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];  in mtk_pcie_irq_restore()
1132 	raw_spin_unlock(&pcie->irq_lock);  in mtk_pcie_irq_restore()
1135 static int mtk_pcie_turn_off_link(struct mtk_gen3_pcie *pcie)  in mtk_pcie_turn_off_link()  argument
1139 	val = readl_relaxed(pcie->base + PCIE_ICMD_PM_REG);  in mtk_pcie_turn_off_link()
1141 	writel_relaxed(val, pcie->base + PCIE_ICMD_PM_REG);  in mtk_pcie_turn_off_link()
1144 	return readl_poll_timeout(pcie->base + PCIE_LTSSM_STATUS_REG, val,  in mtk_pcie_turn_off_link()
1152 	struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev);  in mtk_pcie_suspend_noirq()  local
1157 	err = mtk_pcie_turn_off_link(pcie);  in mtk_pcie_suspend_noirq()
1159 		dev_err(pcie->dev, "cannot enter L2 state\n");  in mtk_pcie_suspend_noirq()
1164 	val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_suspend_noirq()
1166 	writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_suspend_noirq()
1168 	dev_dbg(pcie->dev, "entered L2 states successfully");  in mtk_pcie_suspend_noirq()
1170 	mtk_pcie_irq_save(pcie);  in mtk_pcie_suspend_noirq()
1171 	mtk_pcie_power_down(pcie);  in mtk_pcie_suspend_noirq()
1178 	struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev);  in mtk_pcie_resume_noirq()  local
1181 	err = pcie->soc->power_up(pcie);  in mtk_pcie_resume_noirq()
1185 	err = mtk_pcie_startup_port(pcie);  in mtk_pcie_resume_noirq()
1187 		mtk_pcie_power_down(pcie);  in mtk_pcie_resume_noirq()
1191 	mtk_pcie_irq_restore(pcie);  in mtk_pcie_resume_noirq()
1220 	{ .compatible = "airoha,en7581-pcie", .data = &mtk_pcie_soc_en7581 },
1221 	{ .compatible = "mediatek,mt8192-pcie", .data = &mtk_pcie_soc_mt8192 },
1230 		.name = "mtk-pcie-gen3",
1237 MODULE_DESCRIPTION("MediaTek Gen3 PCIe host controller driver");