25 #include "cpsw.h"
34 int (*cpsw_slave_index)(struct cpsw_common *cpsw, struct cpsw_priv *priv);
36 void cpsw_intr_enable(struct cpsw_common *cpsw)  in cpsw_intr_enable()  argument
38 	writel_relaxed(0xFF, &cpsw->wr_regs->tx_en);  in cpsw_intr_enable()
39 	writel_relaxed(0xFF, &cpsw->wr_regs->rx_en);  in cpsw_intr_enable()
41 	cpdma_ctlr_int_ctrl(cpsw->dma, true);  in cpsw_intr_enable()
44 void cpsw_intr_disable(struct cpsw_common *cpsw)  in cpsw_intr_disable()  argument
46 	writel_relaxed(0, &cpsw->wr_regs->tx_en);  in cpsw_intr_disable()
47 	writel_relaxed(0, &cpsw->wr_regs->rx_en);  in cpsw_intr_disable()
49 	cpdma_ctlr_int_ctrl(cpsw->dma, false);  in cpsw_intr_disable()
88 	struct cpsw_common *cpsw = dev_id;  in cpsw_tx_interrupt()  local
90 	writel(0, &cpsw->wr_regs->tx_en);  in cpsw_tx_interrupt()
91 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);  in cpsw_tx_interrupt()
93 	if (cpsw->quirk_irq) {  in cpsw_tx_interrupt()
94 		disable_irq_nosync(cpsw->irqs_table[1]);  in cpsw_tx_interrupt()
95 		cpsw->tx_irq_disabled = true;  in cpsw_tx_interrupt()
98 	napi_schedule(&cpsw->napi_tx);  in cpsw_tx_interrupt()
104 	struct cpsw_common *cpsw = dev_id;  in cpsw_rx_interrupt()  local
106 	writel(0, &cpsw->wr_regs->rx_en);  in cpsw_rx_interrupt()
107 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);  in cpsw_rx_interrupt()
109 	if (cpsw->quirk_irq) {  in cpsw_rx_interrupt()
110 		disable_irq_nosync(cpsw->irqs_table[0]);  in cpsw_rx_interrupt()
111 		cpsw->rx_irq_disabled = true;  in cpsw_rx_interrupt()
114 	napi_schedule(&cpsw->napi_rx);  in cpsw_rx_interrupt()
120 	struct cpsw_common *cpsw = dev_id;  in cpsw_misc_interrupt()  local
122 	writel(0, &cpsw->wr_regs->misc_en);  in cpsw_misc_interrupt()
123 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_MISC);  in cpsw_misc_interrupt()
124 	cpts_misc_interrupt(cpsw->cpts);  in cpsw_misc_interrupt()
125 	writel(0x10, &cpsw->wr_regs->misc_en);  in cpsw_misc_interrupt()
132 	struct cpsw_common	*cpsw = napi_to_cpsw(napi_tx);  in cpsw_tx_mq_poll()  local
138 	ch_map = cpdma_ctrl_txchs_state(cpsw->dma);  in cpsw_tx_mq_poll()
143 		txv = &cpsw->txv[ch];  in cpsw_tx_mq_poll()
156 		writel(0xff, &cpsw->wr_regs->tx_en);  in cpsw_tx_mq_poll()
164 	struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);  in cpsw_tx_poll()  local
167 	num_tx = cpdma_chan_process(cpsw->txv[0].ch, budget);  in cpsw_tx_poll()
170 		writel(0xff, &cpsw->wr_regs->tx_en);  in cpsw_tx_poll()
171 		if (cpsw->tx_irq_disabled) {  in cpsw_tx_poll()
172 			cpsw->tx_irq_disabled = false;  in cpsw_tx_poll()
173 			enable_irq(cpsw->irqs_table[1]);  in cpsw_tx_poll()
182 	struct cpsw_common	*cpsw = napi_to_cpsw(napi_rx);  in cpsw_rx_mq_poll()  local
188 	ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);  in cpsw_rx_mq_poll()
193 		rxv = &cpsw->rxv[ch];  in cpsw_rx_mq_poll()
206 		writel(0xff, &cpsw->wr_regs->rx_en);  in cpsw_rx_mq_poll()
214 	struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);  in cpsw_rx_poll()  local
217 	num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget);  in cpsw_rx_poll()
220 		writel(0xff, &cpsw->wr_regs->rx_en);  in cpsw_rx_poll()
221 		if (cpsw->rx_irq_disabled) {  in cpsw_rx_poll()
222 			cpsw->rx_irq_disabled = false;  in cpsw_rx_poll()
223 			enable_irq(cpsw->irqs_table[0]);  in cpsw_rx_poll()
234 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_rx_vlan_encap()  local
256 	if (!cpsw_ale_get_vlan_p0_untag(cpsw->ale, vid)) {  in cpsw_rx_vlan_encap()
293 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_ndo_tx_timeout()  local
298 	cpsw_intr_disable(cpsw);  in cpsw_ndo_tx_timeout()
299 	for (ch = 0; ch < cpsw->tx_ch_num; ch++) {  in cpsw_ndo_tx_timeout()
300 		cpdma_chan_stop(cpsw->txv[ch].ch);  in cpsw_ndo_tx_timeout()
301 		cpdma_chan_start(cpsw->txv[ch].ch);  in cpsw_ndo_tx_timeout()
304 	cpsw_intr_enable(cpsw);  in cpsw_ndo_tx_timeout()
309 static int cpsw_get_common_speed(struct cpsw_common *cpsw)  in cpsw_get_common_speed()  argument
313 	for (i = 0, speed = 0; i < cpsw->data.slaves; i++)  in cpsw_get_common_speed()
314 		if (cpsw->slaves[i].phy && cpsw->slaves[i].phy->link)  in cpsw_get_common_speed()
315 			speed += cpsw->slaves[i].phy->speed;  in cpsw_get_common_speed()
320 int cpsw_need_resplit(struct cpsw_common *cpsw)  in cpsw_need_resplit()  argument
326 	speed = cpsw_get_common_speed(cpsw);  in cpsw_need_resplit()
327 	if (speed == cpsw->speed || !speed)  in cpsw_need_resplit()
330 	cpsw->speed = speed;  in cpsw_need_resplit()
332 	for (i = 0, rlim_ch_num = 0; i < cpsw->tx_ch_num; i++) {  in cpsw_need_resplit()
333 		ch_rate = cpdma_chan_get_rate(cpsw->txv[i].ch);  in cpsw_need_resplit()
341 	if (!rlim_ch_num || rlim_ch_num == cpsw->tx_ch_num)  in cpsw_need_resplit()
347 void cpsw_split_res(struct cpsw_common *cpsw)  in cpsw_split_res()  argument
350 	struct cpsw_vector *txv = cpsw->txv;  in cpsw_split_res()
356 	for (i = 0; i < cpsw->tx_ch_num; i++) {  in cpsw_split_res()
365 	if (cpsw->tx_ch_num == rlim_ch_num) {  in cpsw_split_res()
368 		ch_budget = NAPI_POLL_WEIGHT / cpsw->tx_ch_num;  in cpsw_split_res()
372 		max_rate = cpsw->speed * 1000;  in cpsw_split_res()
385 			    (cpsw->tx_ch_num - rlim_ch_num);  in cpsw_split_res()
387 			      (cpsw->tx_ch_num - rlim_ch_num);  in cpsw_split_res()
392 	for (i = 0; i < cpsw->tx_ch_num; i++) {  in cpsw_split_res()
406 			cpdma_chan_set_weight(cpsw->txv[i].ch, ch_weight);  in cpsw_split_res()
411 			cpdma_chan_set_weight(cpsw->txv[i].ch, 0);  in cpsw_split_res()
422 	ch_budget = budget / cpsw->rx_ch_num;  in cpsw_split_res()
423 	for (i = 0; i < cpsw->rx_ch_num; i++) {  in cpsw_split_res()
424 		cpsw->rxv[i].budget = ch_budget;  in cpsw_split_res()
429 		cpsw->rxv[0].budget += budget;  in cpsw_split_res()
432 int cpsw_init_common(struct cpsw_common *cpsw, void __iomem *ss_regs,  in cpsw_init_common()  argument
440 	struct device *dev = cpsw->dev;  in cpsw_init_common()
445 	data = &cpsw->data;  in cpsw_init_common()
446 	cpsw->rx_ch_num = 1;  in cpsw_init_common()
447 	cpsw->tx_ch_num = 1;  in cpsw_init_common()
449 	cpsw->version = readl(&cpsw->regs->id_ver);  in cpsw_init_common()
454 	switch (cpsw->version) {  in cpsw_init_common()
456 		cpsw->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;  in cpsw_init_common()
458 		cpsw->hw_stats	     = ss_regs + CPSW1_HW_STATS;  in cpsw_init_common()
470 		cpsw->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;  in cpsw_init_common()
472 		cpsw->hw_stats	     = ss_regs + CPSW2_HW_STATS;  in cpsw_init_common()
482 		dev_err(dev, "unknown version 0x%08x\n", cpsw->version);  in cpsw_init_common()
486 	for (i = 0; i < cpsw->data.slaves; i++) {  in cpsw_init_common()
487 		struct cpsw_slave *slave = &cpsw->slaves[i];  in cpsw_init_common()
488 		void __iomem		*regs = cpsw->regs;  in cpsw_init_common()
491 		slave->data	= &cpsw->data.slave_data[i];  in cpsw_init_common()
494 		slave->mac_sl = cpsw_sl_get("cpsw", dev, regs + sliver_offset);  in cpsw_init_common()
505 	ale_params.dev_id		= "cpsw";  in cpsw_init_common()
506 	ale_params.bus_freq		= cpsw->bus_freq_mhz * 1000000;  in cpsw_init_common()
508 	cpsw->ale = cpsw_ale_create(&ale_params);  in cpsw_init_common()
509 	if (IS_ERR(cpsw->ale)) {  in cpsw_init_common()
511 		return PTR_ERR(cpsw->ale);  in cpsw_init_common()
528 	dma_params.bus_freq_mhz		= cpsw->bus_freq_mhz;  in cpsw_init_common()
531 	cpsw->dma = cpdma_ctlr_create(&dma_params);  in cpsw_init_common()
532 	if (!cpsw->dma) {  in cpsw_init_common()
537 	cpts_node = of_get_child_by_name(cpsw->dev->of_node, "cpts");  in cpsw_init_common()
539 		cpts_node = cpsw->dev->of_node;  in cpsw_init_common()
541 	cpsw->cpts = cpts_create(cpsw->dev, cpts_regs, cpts_node,  in cpsw_init_common()
543 	if (IS_ERR(cpsw->cpts)) {  in cpsw_init_common()
544 		ret = PTR_ERR(cpsw->cpts);  in cpsw_init_common()
545 		cpdma_ctlr_destroy(cpsw->dma);  in cpsw_init_common()
556 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_hwtstamp_v1()  local
557 	struct cpsw_slave *slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_hwtstamp_v1()
580 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_hwtstamp_v2()  local
584 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_hwtstamp_v2()
587 	switch (cpsw->version) {  in cpsw_hwtstamp_v2()
613 	writel_relaxed(ETH_P_1588, &cpsw->regs->ts_ltype);  in cpsw_hwtstamp_v2()
614 	writel_relaxed(ETH_P_8021Q, &cpsw->regs->vlan_ltype);  in cpsw_hwtstamp_v2()
620 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_hwtstamp_set()  local
623 	if (cpsw->version != CPSW_VERSION_1 &&  in cpsw_hwtstamp_set()
624 	    cpsw->version != CPSW_VERSION_2 &&  in cpsw_hwtstamp_set()
625 	    cpsw->version != CPSW_VERSION_3)  in cpsw_hwtstamp_set()
662 	switch (cpsw->version) {  in cpsw_hwtstamp_set()
679 	struct cpsw_common *cpsw = ndev_to_cpsw(dev);  in cpsw_hwtstamp_get()  local
683 	if (cpsw->version != CPSW_VERSION_1 &&  in cpsw_hwtstamp_get()
684 	    cpsw->version != CPSW_VERSION_2 &&  in cpsw_hwtstamp_get()
685 	    cpsw->version != CPSW_VERSION_3)  in cpsw_hwtstamp_get()
709 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_ndo_ioctl()  local
710 	int slave_no = cpsw_slave_index(cpsw, priv);  in cpsw_ndo_ioctl()
716 	phy = cpsw->slaves[slave_no].phy;  in cpsw_ndo_ioctl()
736 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_ndo_set_tx_maxrate()  local
747 	min_rate = cpdma_chan_get_min_rate(cpsw->dma);  in cpsw_ndo_set_tx_maxrate()
754 	if (rate > cpsw->speed) {  in cpsw_ndo_set_tx_maxrate()
759 	ret = pm_runtime_resume_and_get(cpsw->dev);  in cpsw_ndo_set_tx_maxrate()
763 	ret = cpdma_chan_set_rate(cpsw->txv[queue].ch, ch_rate);  in cpsw_ndo_set_tx_maxrate()
764 	pm_runtime_put(cpsw->dev);  in cpsw_ndo_set_tx_maxrate()
770 	for (i = 0; i < cpsw->data.slaves; i++) {  in cpsw_ndo_set_tx_maxrate()
771 		slave = &cpsw->slaves[i];  in cpsw_ndo_set_tx_maxrate()
778 	cpsw_split_res(cpsw);  in cpsw_ndo_set_tx_maxrate()
792 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_shp_is_off()  local
796 	val = readl_relaxed(&cpsw->regs->ptype);  in cpsw_shp_is_off()
798 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_shp_is_off()
808 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_fifo_shp_on()  local
812 	val = readl_relaxed(&cpsw->regs->ptype);  in cpsw_fifo_shp_on()
814 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_fifo_shp_on()
819 	writel_relaxed(val, &cpsw->regs->ptype);  in cpsw_fifo_shp_on()
824 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_set_fifo_bw()  local
835 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_set_fifo_bw()
884 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_set_fifo_rlimit()  local
893 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_set_fifo_rlimit()
894 	tx_in_ctl_rg = cpsw->version == CPSW_VERSION_1 ?  in cpsw_set_fifo_rlimit()
936 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_set_cbs()  local
959 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_set_cbs()
973 	ret = pm_runtime_resume_and_get(cpsw->dev);  in cpsw_set_cbs()
988 	pm_runtime_put_sync(cpsw->dev);  in cpsw_set_cbs()
996 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_set_mqprio()  local
1009 	ret = pm_runtime_resume_and_get(cpsw->dev);  in cpsw_set_mqprio()
1036 	offset = cpsw->version == CPSW_VERSION_1 ?  in cpsw_set_mqprio()
1039 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];  in cpsw_set_mqprio()
1042 	pm_runtime_put_sync(cpsw->dev);  in cpsw_set_mqprio()
1082 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_mqprio_resume()  local
1096 	tx_prio_rg = cpsw->version == CPSW_VERSION_1 ?  in cpsw_mqprio_resume()
1104 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_fill_rx_channels()  local
1112 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {  in cpsw_fill_rx_channels()
1113 		pool = cpsw->page_pool[ch];  in cpsw_fill_rx_channels()
1114 		ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);  in cpsw_fill_rx_channels()
1127 			ret = cpdma_chan_idle_submit_mapped(cpsw->rxv[ch].ch,  in cpsw_fill_rx_channels()
1129 							    cpsw->rx_packet_max,  in cpsw_fill_rx_channels()
1147 static struct page_pool *cpsw_create_page_pool(struct cpsw_common *cpsw,  in cpsw_create_page_pool()  argument
1158 	pp_params.dev = cpsw->dev;  in cpsw_create_page_pool()
1162 		dev_err(cpsw->dev, "cannot create rx page pool\n");  in cpsw_create_page_pool()
1167 static int cpsw_create_rx_pool(struct cpsw_common *cpsw, int ch)  in cpsw_create_rx_pool()  argument
1172 	pool_size = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);  in cpsw_create_rx_pool()
1173 	pool = cpsw_create_page_pool(cpsw, pool_size);  in cpsw_create_rx_pool()
1177 		cpsw->page_pool[ch] = pool;  in cpsw_create_rx_pool()
1184 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_ndev_create_xdp_rxq()  local
1189 	pool = cpsw->page_pool[ch];  in cpsw_ndev_create_xdp_rxq()
1213 void cpsw_destroy_xdp_rxqs(struct cpsw_common *cpsw)  in cpsw_destroy_xdp_rxqs()  argument
1218 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {  in cpsw_destroy_xdp_rxqs()
1219 		for (i = 0; i < cpsw->data.slaves; i++) {  in cpsw_destroy_xdp_rxqs()
1220 			ndev = cpsw->slaves[i].ndev;  in cpsw_destroy_xdp_rxqs()
1227 		page_pool_destroy(cpsw->page_pool[ch]);  in cpsw_destroy_xdp_rxqs()
1228 		cpsw->page_pool[ch] = NULL;  in cpsw_destroy_xdp_rxqs()
1232 int cpsw_create_xdp_rxqs(struct cpsw_common *cpsw)  in cpsw_create_xdp_rxqs()  argument
1237 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {  in cpsw_create_xdp_rxqs()
1238 		ret = cpsw_create_rx_pool(cpsw, ch);  in cpsw_create_xdp_rxqs()
1245 		for (i = 0; i < cpsw->data.slaves; i++) {  in cpsw_create_xdp_rxqs()
1246 			ndev = cpsw->slaves[i].ndev;  in cpsw_create_xdp_rxqs()
1259 	cpsw_destroy_xdp_rxqs(cpsw);  in cpsw_create_xdp_rxqs()
1294 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_xdp_tx_frame()  local
1303 	txch = cpsw->txv[0].ch;  in cpsw_xdp_tx_frame()
1327 	struct cpsw_common *cpsw = priv->cpsw;  in cpsw_run_xdp()  local
1382 	page_pool_recycle_direct(cpsw->page_pool[ch], page);  in cpsw_run_xdp()
1423 	port_id = cpsw_slave_index(priv->cpsw, priv) + 1;  in cpsw_qos_clsflower_add_policer()
1427 		ret = cpsw_ale_rx_ratelimit_bc(priv->cpsw->ale, port_id, rate_pkt_ps);  in cpsw_qos_clsflower_add_policer()
1435 		ret = cpsw_ale_rx_ratelimit_mc(priv->cpsw->ale, port_id, rate_pkt_ps);  in cpsw_qos_clsflower_add_policer()
1509 	u32 port_id = cpsw_slave_index(priv->cpsw, priv) + 1;  in cpsw_qos_delete_clsflower()
1514 		cpsw_ale_rx_ratelimit_bc(priv->cpsw->ale, port_id, 0);  in cpsw_qos_delete_clsflower()
1520 		cpsw_ale_rx_ratelimit_mc(priv->cpsw->ale, port_id, 0);  in cpsw_qos_delete_clsflower()
1577 	u32 port_id = cpsw_slave_index(priv->cpsw, priv) + 1;  in cpsw_qos_clsflower_resume()
1580 		cpsw_ale_rx_ratelimit_bc(priv->cpsw->ale, port_id,  in cpsw_qos_clsflower_resume()
1584 		cpsw_ale_rx_ratelimit_mc(priv->cpsw->ale, port_id,  in cpsw_qos_clsflower_resume()