Lines Matching +full:route +full:- +full:ptp
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
32 #include <linux/dma/ti-cppi5.h>
33 #include <linux/dma/k3-udma-glue.h>
39 #include "am65-cpsw-nuss.h"
40 #include "am65-cpsw-switchdev.h"
41 #include "k3-cppi-desc-pool.h"
42 #include "am65-cpts.h"
125 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
175 writel(mac_hi, slave->port_base + AM65_CPSW_PORTN_REG_SA_H); in am65_cpsw_port_set_sl_mac()
176 writel(mac_lo, slave->port_base + AM65_CPSW_PORTN_REG_SA_L); in am65_cpsw_port_set_sl_mac()
181 cpsw_sl_reset(port->slave.mac_sl, 100); in am65_cpsw_sl_ctl_reset()
184 port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN); in am65_cpsw_sl_ctl_reset()
189 common->nuss_ver = readl(common->ss_base); in am65_cpsw_nuss_get_ver()
190 common->cpsw_ver = readl(common->cpsw_base); in am65_cpsw_nuss_get_ver()
191 dev_info(common->dev, in am65_cpsw_nuss_get_ver()
193 common->nuss_ver, in am65_cpsw_nuss_get_ver()
194 common->cpsw_ver, in am65_cpsw_nuss_get_ver()
195 common->port_num + 1, in am65_cpsw_nuss_get_ver()
196 common->pdata.quirks); in am65_cpsw_nuss_get_ver()
207 if (!common->is_emac_mode) in am65_cpsw_nuss_ndo_slave_add_vid()
213 ret = pm_runtime_resume_and_get(common->dev); in am65_cpsw_nuss_ndo_slave_add_vid()
217 port_mask = BIT(port->port_id) | ALE_PORT_HOST; in am65_cpsw_nuss_ndo_slave_add_vid()
220 dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid); in am65_cpsw_nuss_ndo_slave_add_vid()
221 ret = cpsw_ale_vlan_add_modify(common->ale, vid, port_mask, in am65_cpsw_nuss_ndo_slave_add_vid()
224 pm_runtime_put(common->dev); in am65_cpsw_nuss_ndo_slave_add_vid()
235 if (!common->is_emac_mode) in am65_cpsw_nuss_ndo_slave_kill_vid()
241 ret = pm_runtime_resume_and_get(common->dev); in am65_cpsw_nuss_ndo_slave_kill_vid()
245 dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid); in am65_cpsw_nuss_ndo_slave_kill_vid()
246 ret = cpsw_ale_del_vlan(common->ale, vid, in am65_cpsw_nuss_ndo_slave_kill_vid()
247 BIT(port->port_id) | ALE_PORT_HOST); in am65_cpsw_nuss_ndo_slave_kill_vid()
249 pm_runtime_put(common->dev); in am65_cpsw_nuss_ndo_slave_kill_vid()
256 struct am65_cpsw_common *common = port->common; in am65_cpsw_slave_set_promisc()
258 if (promisc && !common->is_emac_mode) { in am65_cpsw_slave_set_promisc()
259 dev_dbg(common->dev, "promisc mode requested in switch mode"); in am65_cpsw_slave_set_promisc()
265 cpsw_ale_control_set(common->ale, port->port_id, in am65_cpsw_slave_set_promisc()
267 dev_dbg(common->dev, "promisc enabled\n"); in am65_cpsw_slave_set_promisc()
270 cpsw_ale_control_set(common->ale, port->port_id, in am65_cpsw_slave_set_promisc()
272 dev_dbg(common->dev, "promisc disabled\n"); in am65_cpsw_slave_set_promisc()
283 promisc = !!(ndev->flags & IFF_PROMISC); in am65_cpsw_nuss_ndo_slave_set_rx_mode()
290 cpsw_ale_set_allmulti(common->ale, in am65_cpsw_nuss_ndo_slave_set_rx_mode()
291 ndev->flags & IFF_ALLMULTI, port->port_id); in am65_cpsw_nuss_ndo_slave_set_rx_mode()
295 cpsw_ale_flush_multicast(common->ale, port_mask, -1); in am65_cpsw_nuss_ndo_slave_set_rx_mode()
302 cpsw_ale_add_mcast(common->ale, ha->addr, in am65_cpsw_nuss_ndo_slave_set_rx_mode()
317 tx_chn = &common->tx_chns[txqueue]; in am65_cpsw_nuss_ndo_host_tx_timeout()
318 trans_start = READ_ONCE(netif_txq->trans_start); in am65_cpsw_nuss_ndo_host_tx_timeout()
323 jiffies_to_msecs(jiffies - trans_start), in am65_cpsw_nuss_ndo_host_tx_timeout()
325 k3_cppi_desc_pool_avail(tx_chn->desc_pool)); in am65_cpsw_nuss_ndo_host_tx_timeout()
337 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; in am65_cpsw_nuss_rx_push()
339 struct device *dev = common->dev; in am65_cpsw_nuss_rx_push()
344 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool); in am65_cpsw_nuss_rx_push()
347 return -ENOMEM; in am65_cpsw_nuss_rx_push()
349 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx); in am65_cpsw_nuss_rx_push()
351 buf_dma = dma_map_single(rx_chn->dma_dev, in am65_cpsw_nuss_rx_push()
354 if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) { in am65_cpsw_nuss_rx_push()
355 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); in am65_cpsw_nuss_rx_push()
357 return -EINVAL; in am65_cpsw_nuss_rx_push()
362 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma); in am65_cpsw_nuss_rx_push()
366 swdata->page = page; in am65_cpsw_nuss_rx_push()
367 swdata->flow_id = flow_idx; in am65_cpsw_nuss_rx_push()
369 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, flow_idx, in am65_cpsw_nuss_rx_push()
379 val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL); in am65_cpsw_nuss_set_p0_ptype()
381 if (common->pf_p0_rx_ptype_rrobin) { in am65_cpsw_nuss_set_p0_ptype()
393 writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP); in am65_cpsw_nuss_set_p0_ptype()
394 writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL); in am65_cpsw_nuss_set_p0_ptype()
404 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; in am65_cpsw_destroy_xdp_rxqs()
409 for (id = 0; id < common->rx_ch_num_flows; id++) { in am65_cpsw_destroy_xdp_rxqs()
410 flow = &rx_chn->flows[id]; in am65_cpsw_destroy_xdp_rxqs()
412 for (port = 0; port < common->port_num; port++) { in am65_cpsw_destroy_xdp_rxqs()
413 if (!common->ports[port].ndev) in am65_cpsw_destroy_xdp_rxqs()
416 rxq = &common->ports[port].xdp_rxq[id]; in am65_cpsw_destroy_xdp_rxqs()
422 if (flow->page_pool) { in am65_cpsw_destroy_xdp_rxqs()
423 page_pool_destroy(flow->page_pool); in am65_cpsw_destroy_xdp_rxqs()
424 flow->page_pool = NULL; in am65_cpsw_destroy_xdp_rxqs()
431 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; in am65_cpsw_create_xdp_rxqs()
436 .nid = dev_to_node(common->dev), in am65_cpsw_create_xdp_rxqs()
437 .dev = common->dev, in am65_cpsw_create_xdp_rxqs()
446 for (id = 0; id < common->rx_ch_num_flows; id++) { in am65_cpsw_create_xdp_rxqs()
447 flow = &rx_chn->flows[id]; in am65_cpsw_create_xdp_rxqs()
448 pp_params.napi = &flow->napi_rx; in am65_cpsw_create_xdp_rxqs()
455 flow->page_pool = pool; in am65_cpsw_create_xdp_rxqs()
460 for (port = 0; port < common->port_num; port++) { in am65_cpsw_create_xdp_rxqs()
461 if (!common->ports[port].ndev) in am65_cpsw_create_xdp_rxqs()
464 rxq = &common->ports[port].xdp_rxq[id]; in am65_cpsw_create_xdp_rxqs()
466 ret = xdp_rxq_info_reg(rxq, common->ports[port].ndev, in am65_cpsw_create_xdp_rxqs()
467 id, flow->napi_rx.napi_id); in am65_cpsw_create_xdp_rxqs()
492 return (desc - pool_addr) >> dsize_log2; in am65_cpsw_nuss_desc_idx()
501 desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc, in am65_cpsw_nuss_set_buf_type()
502 tx_chn->dsize_log2); in am65_cpsw_nuss_set_buf_type()
503 k3_cppi_desc_pool_desc_info_set(tx_chn->desc_pool, desc_idx, in am65_cpsw_nuss_set_buf_type()
513 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); in am65_cpsw_nuss_buf_type()
514 desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc_tx, in am65_cpsw_nuss_buf_type()
515 tx_chn->dsize_log2); in am65_cpsw_nuss_buf_type()
517 return (enum am65_cpsw_tx_buf_type)k3_cppi_desc_pool_desc_info(tx_chn->desc_pool, in am65_cpsw_nuss_buf_type()
525 page_pool_put_full_page(flow->page_pool, page, allow_direct); in am65_cpsw_put_page()
538 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); in am65_cpsw_nuss_rx_cleanup()
540 page = swdata->page; in am65_cpsw_nuss_rx_cleanup()
541 flow_id = swdata->flow_id; in am65_cpsw_nuss_rx_cleanup()
543 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); in am65_cpsw_nuss_rx_cleanup()
544 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); in am65_cpsw_nuss_rx_cleanup()
545 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); in am65_cpsw_nuss_rx_cleanup()
547 am65_cpsw_put_page(&rx_chn->flows[flow_id], page, false); in am65_cpsw_nuss_rx_cleanup()
561 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); in am65_cpsw_nuss_xmit_free()
563 dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE); in am65_cpsw_nuss_xmit_free()
566 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma); in am65_cpsw_nuss_xmit_free()
568 next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, in am65_cpsw_nuss_xmit_free()
571 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); in am65_cpsw_nuss_xmit_free()
573 dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len, in am65_cpsw_nuss_xmit_free()
577 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma); in am65_cpsw_nuss_xmit_free()
579 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); in am65_cpsw_nuss_xmit_free()
582 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc); in am65_cpsw_nuss_xmit_free()
592 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); in am65_cpsw_nuss_tx_cleanup()
613 skb->dev = ndev; in am65_cpsw_build_skb()
621 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; in am65_cpsw_nuss_common_open()
622 struct am65_cpsw_tx_chn *tx_chn = common->tx_chns; in am65_cpsw_nuss_common_open()
628 if (common->usage_count) in am65_cpsw_nuss_common_open()
634 common->cpsw_base + AM65_CPSW_REG_CTL); in am65_cpsw_nuss_common_open()
637 host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN); in am65_cpsw_nuss_common_open()
639 writel(common->rx_flow_id_base, in am65_cpsw_nuss_common_open()
640 host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET); in am65_cpsw_nuss_common_open()
642 host_p->port_base + AM65_CPSW_P0_REG_CTL); in am65_cpsw_nuss_common_open()
648 for (port_idx = 0; port_idx < common->port_num; port_idx++) { in am65_cpsw_nuss_common_open()
649 struct am65_cpsw_port *port = &common->ports[port_idx]; in am65_cpsw_nuss_common_open()
651 if (!port->disabled) in am65_cpsw_nuss_common_open()
652 val |= BIT(port->port_id); in am65_cpsw_nuss_common_open()
654 writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN); in am65_cpsw_nuss_common_open()
657 writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE); in am65_cpsw_nuss_common_open()
659 cpsw_ale_start(common->ale); in am65_cpsw_nuss_common_open()
662 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, in am65_cpsw_nuss_common_open()
664 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, in am65_cpsw_nuss_common_open()
667 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1); in am65_cpsw_nuss_common_open()
668 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, in am65_cpsw_nuss_common_open()
672 port_mask = GENMASK(common->port_num, 0) & in am65_cpsw_nuss_common_open()
673 ~common->disabled_ports_mask; in am65_cpsw_nuss_common_open()
675 cpsw_ale_add_vlan(common->ale, 0, port_mask, in am65_cpsw_nuss_common_open()
679 if (common->is_emac_mode) in am65_cpsw_nuss_common_open()
688 dev_err(common->dev, "Failed to create XDP rx queues\n"); in am65_cpsw_nuss_common_open()
692 for (flow_idx = 0; flow_idx < common->rx_ch_num_flows; flow_idx++) { in am65_cpsw_nuss_common_open()
693 flow = &rx_chn->flows[flow_idx]; in am65_cpsw_nuss_common_open()
695 page = page_pool_dev_alloc_pages(flow->page_pool); in am65_cpsw_nuss_common_open()
697 dev_err(common->dev, "cannot allocate page in flow %d\n", in am65_cpsw_nuss_common_open()
699 ret = -ENOMEM; in am65_cpsw_nuss_common_open()
705 dev_err(common->dev, in am65_cpsw_nuss_common_open()
714 ret = k3_udma_glue_enable_rx_chn(rx_chn->rx_chn); in am65_cpsw_nuss_common_open()
716 dev_err(common->dev, "couldn't enable rx chn: %d\n", ret); in am65_cpsw_nuss_common_open()
720 for (i = 0; i < common->rx_ch_num_flows ; i++) { in am65_cpsw_nuss_common_open()
721 napi_enable(&rx_chn->flows[i].napi_rx); in am65_cpsw_nuss_common_open()
722 if (rx_chn->flows[i].irq_disabled) { in am65_cpsw_nuss_common_open()
723 rx_chn->flows[i].irq_disabled = false; in am65_cpsw_nuss_common_open()
724 enable_irq(rx_chn->flows[i].irq); in am65_cpsw_nuss_common_open()
728 for (tx = 0; tx < common->tx_ch_num; tx++) { in am65_cpsw_nuss_common_open()
731 dev_err(common->dev, "couldn't enable tx chn %d: %d\n", in am65_cpsw_nuss_common_open()
733 tx--; in am65_cpsw_nuss_common_open()
739 dev_dbg(common->dev, "cpsw_nuss started\n"); in am65_cpsw_nuss_common_open()
746 tx--; in am65_cpsw_nuss_common_open()
749 for (flow_idx = 0; i < common->rx_ch_num_flows; flow_idx++) { in am65_cpsw_nuss_common_open()
750 flow = &rx_chn->flows[flow_idx]; in am65_cpsw_nuss_common_open()
751 if (!flow->irq_disabled) { in am65_cpsw_nuss_common_open()
752 disable_irq(flow->irq); in am65_cpsw_nuss_common_open()
753 flow->irq_disabled = true; in am65_cpsw_nuss_common_open()
755 napi_disable(&flow->napi_rx); in am65_cpsw_nuss_common_open()
758 k3_udma_glue_disable_rx_chn(rx_chn->rx_chn); in am65_cpsw_nuss_common_open()
761 for (i = 0; i < common->rx_ch_num_flows; i++) in am65_cpsw_nuss_common_open()
762 k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, i, rx_chn, in am65_cpsw_nuss_common_open()
772 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; in am65_cpsw_nuss_common_stop()
773 struct am65_cpsw_tx_chn *tx_chn = common->tx_chns; in am65_cpsw_nuss_common_stop()
776 if (common->usage_count != 1) in am65_cpsw_nuss_common_stop()
779 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, in am65_cpsw_nuss_common_stop()
783 atomic_set(&common->tdown_cnt, common->tx_ch_num); in am65_cpsw_nuss_common_stop()
786 reinit_completion(&common->tdown_complete); in am65_cpsw_nuss_common_stop()
788 for (i = 0; i < common->tx_ch_num; i++) in am65_cpsw_nuss_common_stop()
791 i = wait_for_completion_timeout(&common->tdown_complete, in am65_cpsw_nuss_common_stop()
794 dev_err(common->dev, "tx timeout\n"); in am65_cpsw_nuss_common_stop()
795 for (i = 0; i < common->tx_ch_num; i++) { in am65_cpsw_nuss_common_stop()
800 for (i = 0; i < common->tx_ch_num; i++) { in am65_cpsw_nuss_common_stop()
806 reinit_completion(&common->tdown_complete); in am65_cpsw_nuss_common_stop()
807 k3_udma_glue_tdown_rx_chn(rx_chn->rx_chn, true); in am65_cpsw_nuss_common_stop()
809 if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) { in am65_cpsw_nuss_common_stop()
810 i = wait_for_completion_timeout(&common->tdown_complete, msecs_to_jiffies(1000)); in am65_cpsw_nuss_common_stop()
812 dev_err(common->dev, "rx teardown timeout\n"); in am65_cpsw_nuss_common_stop()
815 for (i = common->rx_ch_num_flows - 1; i >= 0; i--) { in am65_cpsw_nuss_common_stop()
816 napi_disable(&rx_chn->flows[i].napi_rx); in am65_cpsw_nuss_common_stop()
817 hrtimer_cancel(&rx_chn->flows[i].rx_hrtimer); in am65_cpsw_nuss_common_stop()
818 k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, i, rx_chn, in am65_cpsw_nuss_common_stop()
822 k3_udma_glue_disable_rx_chn(rx_chn->rx_chn); in am65_cpsw_nuss_common_stop()
824 cpsw_ale_stop(common->ale); in am65_cpsw_nuss_common_stop()
826 writel(0, common->cpsw_base + AM65_CPSW_REG_CTL); in am65_cpsw_nuss_common_stop()
827 writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN); in am65_cpsw_nuss_common_stop()
831 dev_dbg(common->dev, "cpsw_nuss stopped\n"); in am65_cpsw_nuss_common_stop()
841 phylink_stop(port->slave.phylink); in am65_cpsw_nuss_ndo_slave_stop()
845 phylink_disconnect_phy(port->slave.phylink); in am65_cpsw_nuss_ndo_slave_stop()
851 common->usage_count--; in am65_cpsw_nuss_ndo_slave_stop()
852 pm_runtime_put(common->dev); in am65_cpsw_nuss_ndo_slave_stop()
863 return am65_cpsw_nuss_ndo_slave_add_vid(port->ndev, 0, vid); in cpsw_restore_vlans()
873 ret = pm_runtime_resume_and_get(common->dev); in am65_cpsw_nuss_ndo_slave_open()
878 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE); in am65_cpsw_nuss_ndo_slave_open()
879 cpsw_sl_wait_for_idle(port->slave.mac_sl, 100); in am65_cpsw_nuss_ndo_slave_open()
880 cpsw_sl_ctl_reset(port->slave.mac_sl); in am65_cpsw_nuss_ndo_slave_open()
883 cpsw_sl_reg_write(port->slave.mac_sl, CPSW_SL_SOFT_RESET, 1); in am65_cpsw_nuss_ndo_slave_open()
885 reg = cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_SOFT_RESET); in am65_cpsw_nuss_ndo_slave_open()
887 dev_err(common->dev, "soft RESET didn't complete\n"); in am65_cpsw_nuss_ndo_slave_open()
888 ret = -ETIMEDOUT; in am65_cpsw_nuss_ndo_slave_open()
893 ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num); in am65_cpsw_nuss_ndo_slave_open()
895 dev_err(common->dev, "cannot set real number of tx queues\n"); in am65_cpsw_nuss_ndo_slave_open()
899 ret = netif_set_real_num_rx_queues(ndev, common->rx_ch_num_flows); in am65_cpsw_nuss_ndo_slave_open()
901 dev_err(common->dev, "cannot set real number of rx queues\n"); in am65_cpsw_nuss_ndo_slave_open()
905 for (i = 0; i < common->tx_ch_num; i++) { in am65_cpsw_nuss_ndo_slave_open()
909 txq->tx_maxrate = common->tx_chns[i].rate_mbps; in am65_cpsw_nuss_ndo_slave_open()
916 common->usage_count++; in am65_cpsw_nuss_ndo_slave_open()
918 am65_cpsw_port_set_sl_mac(port, ndev->dev_addr); in am65_cpsw_nuss_ndo_slave_open()
920 if (common->is_emac_mode) in am65_cpsw_nuss_ndo_slave_open()
925 /* mac_sl should be configured via phy-link interface */ in am65_cpsw_nuss_ndo_slave_open()
928 ret = phylink_of_phy_connect(port->slave.phylink, port->slave.port_np, 0); in am65_cpsw_nuss_ndo_slave_open()
935 phylink_start(port->slave.phylink); in am65_cpsw_nuss_ndo_slave_open()
944 pm_runtime_put(common->dev); in am65_cpsw_nuss_ndo_slave_open()
958 u32 pkt_len = xdpf->len; in am65_cpsw_xdp_tx_frame()
962 host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); in am65_cpsw_xdp_tx_frame()
964 ndev->stats.tx_dropped++; in am65_cpsw_xdp_tx_frame()
970 dma_buf = dma_map_single(tx_chn->dma_dev, xdpf->data, in am65_cpsw_xdp_tx_frame()
972 if (unlikely(dma_mapping_error(tx_chn->dma_dev, dma_buf))) { in am65_cpsw_xdp_tx_frame()
973 ndev->stats.tx_dropped++; in am65_cpsw_xdp_tx_frame()
982 cppi5_desc_set_pktids(&host_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID); in am65_cpsw_xdp_tx_frame()
983 cppi5_desc_set_tags_ids(&host_desc->hdr, 0, port->port_id); in am65_cpsw_xdp_tx_frame()
985 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf); in am65_cpsw_xdp_tx_frame()
992 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id); in am65_cpsw_xdp_tx_frame()
995 dma_desc = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, host_desc); in am65_cpsw_xdp_tx_frame()
997 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc, in am65_cpsw_xdp_tx_frame()
1000 spin_lock_bh(&tx_chn->lock); in am65_cpsw_xdp_tx_frame()
1001 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc, in am65_cpsw_xdp_tx_frame()
1003 spin_unlock_bh(&tx_chn->lock); in am65_cpsw_xdp_tx_frame()
1008 ndev->stats.tx_errors++; in am65_cpsw_xdp_tx_frame()
1016 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &dma_buf); in am65_cpsw_xdp_tx_frame()
1017 dma_unmap_single(tx_chn->dma_dev, dma_buf, pkt_len, DMA_TO_DEVICE); in am65_cpsw_xdp_tx_frame()
1019 k3_cppi_desc_pool_free(tx_chn->desc_pool, host_desc); in am65_cpsw_xdp_tx_frame()
1028 struct am65_cpsw_common *common = flow->common; in am65_cpsw_run_xdp()
1030 struct net_device *ndev = port->ndev; in am65_cpsw_run_xdp()
1041 prog = READ_ONCE(port->xdp_prog); in am65_cpsw_run_xdp()
1047 *len = xdp->data_end - xdp->data; in am65_cpsw_run_xdp()
1050 stats = this_cpu_ptr(ndev_priv->stats); in am65_cpsw_run_xdp()
1057 tx_chn = &common->tx_chns[cpu % AM65_CPSW_MAX_QUEUES]; in am65_cpsw_run_xdp()
1058 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id); in am65_cpsw_run_xdp()
1071 u64_stats_update_begin(&stats->syncp); in am65_cpsw_run_xdp()
1072 stats->rx_bytes += *len; in am65_cpsw_run_xdp()
1073 stats->rx_packets++; in am65_cpsw_run_xdp()
1074 u64_stats_update_end(&stats->syncp); in am65_cpsw_run_xdp()
1081 u64_stats_update_begin(&stats->syncp); in am65_cpsw_run_xdp()
1082 stats->rx_bytes += *len; in am65_cpsw_run_xdp()
1083 stats->rx_packets++; in am65_cpsw_run_xdp()
1084 u64_stats_update_end(&stats->syncp); in am65_cpsw_run_xdp()
1095 ndev->stats.rx_dropped++; in am65_cpsw_run_xdp()
1098 page = virt_to_head_page(xdp->data); in am65_cpsw_run_xdp()
1105 /* RX psdata[2] word format - checksum information */
1117 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error in am65_cpsw_nuss_rx_csum()
1119 * bits - indicates IPv4/IPv6 packet in am65_cpsw_nuss_rx_csum()
1120 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet in am65_cpsw_nuss_rx_csum()
1126 if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) in am65_cpsw_nuss_rx_csum()
1134 skb->ip_summed = CHECKSUM_UNNECESSARY; in am65_cpsw_nuss_rx_csum()
1141 struct am65_cpsw_rx_chn *rx_chn = &flow->common->rx_chns; in am65_cpsw_nuss_rx_packets()
1143 struct am65_cpsw_common *common = flow->common; in am65_cpsw_nuss_rx_packets()
1147 struct device *dev = common->dev; in am65_cpsw_nuss_rx_packets()
1153 u32 flow_idx = flow->id; in am65_cpsw_nuss_rx_packets()
1161 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma); in am65_cpsw_nuss_rx_packets()
1163 if (ret != -ENODATA) in am65_cpsw_nuss_rx_packets()
1170 if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) in am65_cpsw_nuss_rx_packets()
1171 complete(&common->tdown_complete); in am65_cpsw_nuss_rx_packets()
1175 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); in am65_cpsw_nuss_rx_packets()
1180 page = swdata->page; in am65_cpsw_nuss_rx_packets()
1183 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); in am65_cpsw_nuss_rx_packets()
1185 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); in am65_cpsw_nuss_rx_packets()
1188 ndev = port->ndev; in am65_cpsw_nuss_rx_packets()
1193 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); in am65_cpsw_nuss_rx_packets()
1195 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); in am65_cpsw_nuss_rx_packets()
1204 if (port->xdp_prog) { in am65_cpsw_nuss_rx_packets()
1205 xdp_init_buff(&xdp, PAGE_SIZE, &port->xdp_rxq[flow->id]); in am65_cpsw_nuss_rx_packets()
1214 headroom = (xdp.data - xdp.data_hard_start) - skb_headroom(skb); in am65_cpsw_nuss_rx_packets()
1219 am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark); in am65_cpsw_nuss_rx_packets()
1221 if (port->rx_ts_enabled) in am65_cpsw_nuss_rx_packets()
1222 am65_cpts_rx_timestamp(common->cpts, skb); in am65_cpsw_nuss_rx_packets()
1224 skb->protocol = eth_type_trans(skb, ndev); in am65_cpsw_nuss_rx_packets()
1226 napi_gro_receive(&flow->napi_rx, skb); in am65_cpsw_nuss_rx_packets()
1228 stats = this_cpu_ptr(ndev_priv->stats); in am65_cpsw_nuss_rx_packets()
1230 u64_stats_update_begin(&stats->syncp); in am65_cpsw_nuss_rx_packets()
1231 stats->rx_packets++; in am65_cpsw_nuss_rx_packets()
1232 stats->rx_bytes += pkt_len; in am65_cpsw_nuss_rx_packets()
1233 u64_stats_update_end(&stats->syncp); in am65_cpsw_nuss_rx_packets()
1236 new_page = page_pool_dev_alloc_pages(flow->page_pool); in am65_cpsw_nuss_rx_packets()
1239 return -ENOMEM; in am65_cpsw_nuss_rx_packets()
1244 ndev->stats.rx_dropped++; in am65_cpsw_nuss_rx_packets()
1252 ndev->stats.rx_errors++; in am65_cpsw_nuss_rx_packets()
1253 ndev->stats.rx_dropped++; in am65_cpsw_nuss_rx_packets()
1265 enable_irq(flow->irq); in am65_cpsw_nuss_rx_timer_callback()
1272 struct am65_cpsw_common *common = flow->common; in am65_cpsw_nuss_rx_poll()
1281 while (cur_budget--) { in am65_cpsw_nuss_rx_poll()
1292 dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget); in am65_cpsw_nuss_rx_poll()
1295 if (flow->irq_disabled) { in am65_cpsw_nuss_rx_poll()
1296 flow->irq_disabled = false; in am65_cpsw_nuss_rx_poll()
1297 if (unlikely(flow->rx_pace_timeout)) { in am65_cpsw_nuss_rx_poll()
1298 hrtimer_start(&flow->rx_hrtimer, in am65_cpsw_nuss_rx_poll()
1299 ns_to_ktime(flow->rx_pace_timeout), in am65_cpsw_nuss_rx_poll()
1302 enable_irq(flow->irq); in am65_cpsw_nuss_rx_poll()
1321 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, in am65_cpsw_nuss_tx_compl_packet_skb()
1327 ndev = skb->dev; in am65_cpsw_nuss_tx_compl_packet_skb()
1329 am65_cpts_tx_timestamp(tx_chn->common->cpts, skb); in am65_cpsw_nuss_tx_compl_packet_skb()
1332 stats = this_cpu_ptr(ndev_priv->stats); in am65_cpsw_nuss_tx_compl_packet_skb()
1333 u64_stats_update_begin(&stats->syncp); in am65_cpsw_nuss_tx_compl_packet_skb()
1334 stats->tx_packets++; in am65_cpsw_nuss_tx_compl_packet_skb()
1335 stats->tx_bytes += skb->len; in am65_cpsw_nuss_tx_compl_packet_skb()
1336 u64_stats_update_end(&stats->syncp); in am65_cpsw_nuss_tx_compl_packet_skb()
1355 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); in am65_cpsw_nuss_tx_compl_packet_xdp()
1356 cppi5_desc_get_tags_ids(&desc_tx->hdr, NULL, &port_id); in am65_cpsw_nuss_tx_compl_packet_xdp()
1362 *ndev = port->ndev; in am65_cpsw_nuss_tx_compl_packet_xdp()
1365 stats = this_cpu_ptr(ndev_priv->stats); in am65_cpsw_nuss_tx_compl_packet_xdp()
1366 u64_stats_update_begin(&stats->syncp); in am65_cpsw_nuss_tx_compl_packet_xdp()
1367 stats->tx_packets++; in am65_cpsw_nuss_tx_compl_packet_xdp()
1368 stats->tx_bytes += xdpf->len; in am65_cpsw_nuss_tx_compl_packet_xdp()
1369 u64_stats_update_end(&stats->syncp); in am65_cpsw_nuss_tx_compl_packet_xdp()
1384 (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS)) in am65_cpsw_nuss_tx_wake()
1395 struct device *dev = common->dev; in am65_cpsw_nuss_tx_compl_packets()
1405 tx_chn = &common->tx_chns[chn]; in am65_cpsw_nuss_tx_compl_packets()
1408 spin_lock(&tx_chn->lock); in am65_cpsw_nuss_tx_compl_packets()
1409 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma); in am65_cpsw_nuss_tx_compl_packets()
1410 spin_unlock(&tx_chn->lock); in am65_cpsw_nuss_tx_compl_packets()
1411 if (res == -ENODATA) in am65_cpsw_nuss_tx_compl_packets()
1415 if (atomic_dec_and_test(&common->tdown_cnt)) in am65_cpsw_nuss_tx_compl_packets()
1416 complete(&common->tdown_complete); in am65_cpsw_nuss_tx_compl_packets()
1424 ndev = skb->dev; in am65_cpsw_nuss_tx_compl_packets()
1425 total_bytes = skb->len; in am65_cpsw_nuss_tx_compl_packets()
1430 total_bytes = xdpf->len; in am65_cpsw_nuss_tx_compl_packets()
1454 struct device *dev = common->dev; in am65_cpsw_nuss_tx_compl_packets_2g()
1464 tx_chn = &common->tx_chns[chn]; in am65_cpsw_nuss_tx_compl_packets_2g()
1467 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma); in am65_cpsw_nuss_tx_compl_packets_2g()
1468 if (res == -ENODATA) in am65_cpsw_nuss_tx_compl_packets_2g()
1472 if (atomic_dec_and_test(&common->tdown_cnt)) in am65_cpsw_nuss_tx_compl_packets_2g()
1473 complete(&common->tdown_complete); in am65_cpsw_nuss_tx_compl_packets_2g()
1481 ndev = skb->dev; in am65_cpsw_nuss_tx_compl_packets_2g()
1482 total_bytes += skb->len; in am65_cpsw_nuss_tx_compl_packets_2g()
1487 total_bytes += xdpf->len; in am65_cpsw_nuss_tx_compl_packets_2g()
1515 enable_irq(tx_chns->irq); in am65_cpsw_nuss_tx_timer_callback()
1525 if (AM65_CPSW_IS_CPSW2G(tx_chn->common)) in am65_cpsw_nuss_tx_poll()
1526 num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id, in am65_cpsw_nuss_tx_poll()
1529 num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, in am65_cpsw_nuss_tx_poll()
1530 tx_chn->id, budget, &tdown); in am65_cpsw_nuss_tx_poll()
1536 if (unlikely(tx_chn->tx_pace_timeout && !tdown)) { in am65_cpsw_nuss_tx_poll()
1537 hrtimer_start(&tx_chn->tx_hrtimer, in am65_cpsw_nuss_tx_poll()
1538 ns_to_ktime(tx_chn->tx_pace_timeout), in am65_cpsw_nuss_tx_poll()
1541 enable_irq(tx_chn->irq); in am65_cpsw_nuss_tx_poll()
1552 flow->irq_disabled = true; in am65_cpsw_nuss_rx_irq()
1554 napi_schedule(&flow->napi_rx); in am65_cpsw_nuss_rx_irq()
1564 napi_schedule(&tx_chn->napi_tx); in am65_cpsw_nuss_tx_irq()
1575 struct device *dev = common->dev; in am65_cpsw_nuss_ndo_slave_xmit()
1588 if (port->tx_ts_enabled) in am65_cpsw_nuss_ndo_slave_xmit()
1589 am65_cpts_prep_tx_timestamp(common->cpts, skb); in am65_cpsw_nuss_ndo_slave_xmit()
1594 tx_chn = &common->tx_chns[q_idx]; in am65_cpsw_nuss_ndo_slave_xmit()
1598 buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len, in am65_cpsw_nuss_ndo_slave_xmit()
1600 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) { in am65_cpsw_nuss_ndo_slave_xmit()
1602 ndev->stats.tx_errors++; in am65_cpsw_nuss_ndo_slave_xmit()
1606 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); in am65_cpsw_nuss_ndo_slave_xmit()
1609 dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len, in am65_cpsw_nuss_ndo_slave_xmit()
1619 cppi5_desc_set_pktids(&first_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID); in am65_cpsw_nuss_ndo_slave_xmit()
1621 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id); in am65_cpsw_nuss_ndo_slave_xmit()
1623 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); in am65_cpsw_nuss_ndo_slave_xmit()
1631 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { in am65_cpsw_nuss_ndo_slave_xmit()
1635 cs_offset = cs_start + skb->csum_offset; in am65_cpsw_nuss_ndo_slave_xmit()
1638 ((cs_start + 1) << 16) | (skb->len - cs_start); in am65_cpsw_nuss_ndo_slave_xmit()
1649 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in am65_cpsw_nuss_ndo_slave_xmit()
1650 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in am65_cpsw_nuss_ndo_slave_xmit()
1653 next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); in am65_cpsw_nuss_ndo_slave_xmit()
1662 buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size, in am65_cpsw_nuss_ndo_slave_xmit()
1664 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) { in am65_cpsw_nuss_ndo_slave_xmit()
1666 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); in am65_cpsw_nuss_ndo_slave_xmit()
1667 ndev->stats.tx_errors++; in am65_cpsw_nuss_ndo_slave_xmit()
1672 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); in am65_cpsw_nuss_ndo_slave_xmit()
1676 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, in am65_cpsw_nuss_ndo_slave_xmit()
1678 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma); in am65_cpsw_nuss_ndo_slave_xmit()
1684 WARN_ON(pkt_len != skb->len); in am65_cpsw_nuss_ndo_slave_xmit()
1693 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc); in am65_cpsw_nuss_ndo_slave_xmit()
1695 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma); in am65_cpsw_nuss_ndo_slave_xmit()
1697 spin_lock_bh(&tx_chn->lock); in am65_cpsw_nuss_ndo_slave_xmit()
1698 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma); in am65_cpsw_nuss_ndo_slave_xmit()
1699 spin_unlock_bh(&tx_chn->lock); in am65_cpsw_nuss_ndo_slave_xmit()
1705 ndev->stats.tx_errors++; in am65_cpsw_nuss_ndo_slave_xmit()
1709 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) { in am65_cpsw_nuss_ndo_slave_xmit()
1715 /* re-check for smp */ in am65_cpsw_nuss_ndo_slave_xmit()
1716 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= in am65_cpsw_nuss_ndo_slave_xmit()
1728 ndev->stats.tx_dropped++; in am65_cpsw_nuss_ndo_slave_xmit()
1751 ret = pm_runtime_resume_and_get(common->dev); in am65_cpsw_nuss_ndo_slave_set_mac_address()
1755 cpsw_ale_del_ucast(common->ale, ndev->dev_addr, in am65_cpsw_nuss_ndo_slave_set_mac_address()
1757 cpsw_ale_add_ucast(common->ale, sockaddr->sa_data, in am65_cpsw_nuss_ndo_slave_set_mac_address()
1763 pm_runtime_put(common->dev); in am65_cpsw_nuss_ndo_slave_set_mac_address()
1776 return -EOPNOTSUPP; in am65_cpsw_nuss_hwtstamp_set()
1778 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) in am65_cpsw_nuss_hwtstamp_set()
1779 return -EFAULT; in am65_cpsw_nuss_hwtstamp_set()
1787 return -ERANGE; in am65_cpsw_nuss_hwtstamp_set()
1792 port->rx_ts_enabled = false; in am65_cpsw_nuss_hwtstamp_set()
1803 port->rx_ts_enabled = true; in am65_cpsw_nuss_hwtstamp_set()
1809 return -EOPNOTSUPP; in am65_cpsw_nuss_hwtstamp_set()
1811 return -ERANGE; in am65_cpsw_nuss_hwtstamp_set()
1814 port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON); in am65_cpsw_nuss_hwtstamp_set()
1835 if (port->tx_ts_enabled) in am65_cpsw_nuss_hwtstamp_set()
1839 if (port->rx_ts_enabled) in am65_cpsw_nuss_hwtstamp_set()
1843 writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG); in am65_cpsw_nuss_hwtstamp_set()
1844 writel(ts_vlan_ltype, port->port_base + in am65_cpsw_nuss_hwtstamp_set()
1846 writel(ts_ctrl_ltype2, port->port_base + in am65_cpsw_nuss_hwtstamp_set()
1848 writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL); in am65_cpsw_nuss_hwtstamp_set()
1850 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; in am65_cpsw_nuss_hwtstamp_set()
1860 return -EOPNOTSUPP; in am65_cpsw_nuss_hwtstamp_get()
1863 cfg.tx_type = port->tx_ts_enabled ? in am65_cpsw_nuss_hwtstamp_get()
1865 cfg.rx_filter = port->rx_ts_enabled ? in am65_cpsw_nuss_hwtstamp_get()
1868 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; in am65_cpsw_nuss_hwtstamp_get()
1877 return -EINVAL; in am65_cpsw_nuss_ndo_slave_ioctl()
1886 return phylink_mii_ioctl(port->slave.phylink, req, cmd); in am65_cpsw_nuss_ndo_slave_ioctl()
1903 cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu); in am65_cpsw_nuss_ndo_get_stats()
1905 start = u64_stats_fetch_begin(&cpu_stats->syncp); in am65_cpsw_nuss_ndo_get_stats()
1906 rx_packets = cpu_stats->rx_packets; in am65_cpsw_nuss_ndo_get_stats()
1907 rx_bytes = cpu_stats->rx_bytes; in am65_cpsw_nuss_ndo_get_stats()
1908 tx_packets = cpu_stats->tx_packets; in am65_cpsw_nuss_ndo_get_stats()
1909 tx_bytes = cpu_stats->tx_bytes; in am65_cpsw_nuss_ndo_get_stats()
1910 } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); in am65_cpsw_nuss_ndo_get_stats()
1912 stats->rx_packets += rx_packets; in am65_cpsw_nuss_ndo_get_stats()
1913 stats->rx_bytes += rx_bytes; in am65_cpsw_nuss_ndo_get_stats()
1914 stats->tx_packets += tx_packets; in am65_cpsw_nuss_ndo_get_stats()
1915 stats->tx_bytes += tx_bytes; in am65_cpsw_nuss_ndo_get_stats()
1918 stats->rx_errors = dev->stats.rx_errors; in am65_cpsw_nuss_ndo_get_stats()
1919 stats->rx_dropped = dev->stats.rx_dropped; in am65_cpsw_nuss_ndo_get_stats()
1920 stats->tx_dropped = dev->stats.tx_dropped; in am65_cpsw_nuss_ndo_get_stats()
1933 old_prog = xchg(&port->xdp_prog, prog); in am65_cpsw_xdp_prog_setup()
1945 switch (bpf->command) { in am65_cpsw_ndo_bpf()
1947 return am65_cpsw_xdp_prog_setup(ndev, bpf->prog); in am65_cpsw_ndo_bpf()
1949 return -EINVAL; in am65_cpsw_ndo_bpf()
1962 tx_chn = &common->tx_chns[cpu % common->tx_ch_num]; in am65_cpsw_ndo_xdp_xmit()
1963 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id); in am65_cpsw_ndo_xdp_xmit()
2024 for (i = 0; i < common->port_num; i++) { in am65_cpsw_disable_serdes_phy()
2025 port = &common->ports[i]; in am65_cpsw_disable_serdes_phy()
2026 phy = port->slave.serdes_phy; in am65_cpsw_disable_serdes_phy()
2044 port->slave.serdes_phy = phy; in am65_cpsw_init_serdes_phy()
2063 struct am65_cpsw_common *common = port->common; in am65_cpsw_nuss_mac_config()
2065 if (common->pdata.extra_modes & BIT(state->interface)) { in am65_cpsw_nuss_mac_config()
2066 if (state->interface == PHY_INTERFACE_MODE_SGMII) { in am65_cpsw_nuss_mac_config()
2068 port->sgmii_base + AM65_CPSW_SGMII_MR_ADV_ABILITY_REG); in am65_cpsw_nuss_mac_config()
2069 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN); in am65_cpsw_nuss_mac_config()
2071 cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN); in am65_cpsw_nuss_mac_config()
2074 if (state->interface == PHY_INTERFACE_MODE_USXGMII) { in am65_cpsw_nuss_mac_config()
2075 cpsw_sl_ctl_set(port->slave.mac_sl, in am65_cpsw_nuss_mac_config()
2078 cpsw_sl_ctl_clr(port->slave.mac_sl, in am65_cpsw_nuss_mac_config()
2083 port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG); in am65_cpsw_nuss_mac_config()
2093 struct am65_cpsw_common *common = port->common; in am65_cpsw_nuss_mac_link_down()
2094 struct net_device *ndev = port->ndev; in am65_cpsw_nuss_mac_link_down()
2099 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_DISABLE); in am65_cpsw_nuss_mac_link_down()
2101 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE); in am65_cpsw_nuss_mac_link_down()
2103 tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100); in am65_cpsw_nuss_mac_link_down()
2104 dev_dbg(common->dev, "down msc_sl %08x tmo %d\n", in am65_cpsw_nuss_mac_link_down()
2105 cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS), tmo); in am65_cpsw_nuss_mac_link_down()
2114 cpsw_sl_ctl_clr(port->slave.mac_sl, mac_control); in am65_cpsw_nuss_mac_link_down()
2127 struct am65_cpsw_common *common = port->common; in am65_cpsw_nuss_mac_link_up()
2129 struct net_device *ndev = port->ndev; in am65_cpsw_nuss_mac_link_up()
2132 cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE); in am65_cpsw_nuss_mac_link_up()
2136 /* TODO: Verify whether in-band is necessary for 10 Mbps RGMII */ in am65_cpsw_nuss_mac_link_up()
2152 cpsw_sl_ctl_set(port->slave.mac_sl, mac_control); in am65_cpsw_nuss_mac_link_up()
2155 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_FORWARD); in am65_cpsw_nuss_mac_link_up()
2169 struct am65_cpsw_common *common = port->common; in am65_cpsw_nuss_slave_disable_unused()
2171 if (!port->disabled) in am65_cpsw_nuss_slave_disable_unused()
2174 cpsw_ale_control_set(common->ale, port->port_id, in am65_cpsw_nuss_slave_disable_unused()
2177 cpsw_sl_reset(port->slave.mac_sl, 100); in am65_cpsw_nuss_slave_disable_unused()
2178 cpsw_sl_ctl_reset(port->slave.mac_sl); in am65_cpsw_nuss_slave_disable_unused()
2186 for (i = 0; i < common->tx_ch_num; i++) { in am65_cpsw_nuss_free_tx_chns()
2187 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; in am65_cpsw_nuss_free_tx_chns()
2189 if (!IS_ERR_OR_NULL(tx_chn->desc_pool)) in am65_cpsw_nuss_free_tx_chns()
2190 k3_cppi_desc_pool_destroy(tx_chn->desc_pool); in am65_cpsw_nuss_free_tx_chns()
2192 if (!IS_ERR_OR_NULL(tx_chn->tx_chn)) in am65_cpsw_nuss_free_tx_chns()
2193 k3_udma_glue_release_tx_chn(tx_chn->tx_chn); in am65_cpsw_nuss_free_tx_chns()
2201 struct device *dev = common->dev; in am65_cpsw_nuss_remove_tx_chns()
2206 common->tx_ch_rate_msk = 0; in am65_cpsw_nuss_remove_tx_chns()
2207 for (i = 0; i < common->tx_ch_num; i++) { in am65_cpsw_nuss_remove_tx_chns()
2208 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; in am65_cpsw_nuss_remove_tx_chns()
2210 if (tx_chn->irq) in am65_cpsw_nuss_remove_tx_chns()
2211 devm_free_irq(dev, tx_chn->irq, tx_chn); in am65_cpsw_nuss_remove_tx_chns()
2213 netif_napi_del(&tx_chn->napi_tx); in am65_cpsw_nuss_remove_tx_chns()
2221 struct device *dev = common->dev; in am65_cpsw_nuss_ndev_add_tx_napi()
2224 for (i = 0; i < common->tx_ch_num; i++) { in am65_cpsw_nuss_ndev_add_tx_napi()
2225 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; in am65_cpsw_nuss_ndev_add_tx_napi()
2227 netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx, in am65_cpsw_nuss_ndev_add_tx_napi()
2229 hrtimer_init(&tx_chn->tx_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); in am65_cpsw_nuss_ndev_add_tx_napi()
2230 tx_chn->tx_hrtimer.function = &am65_cpsw_nuss_tx_timer_callback; in am65_cpsw_nuss_ndev_add_tx_napi()
2232 ret = devm_request_irq(dev, tx_chn->irq, in am65_cpsw_nuss_ndev_add_tx_napi()
2235 tx_chn->tx_chn_name, tx_chn); in am65_cpsw_nuss_ndev_add_tx_napi()
2238 tx_chn->id, tx_chn->irq, ret); in am65_cpsw_nuss_ndev_add_tx_napi()
2251 struct device *dev = common->dev; in am65_cpsw_nuss_init_tx_chns()
2269 for (i = 0; i < common->tx_ch_num; i++) { in am65_cpsw_nuss_init_tx_chns()
2270 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; in am65_cpsw_nuss_init_tx_chns()
2272 snprintf(tx_chn->tx_chn_name, in am65_cpsw_nuss_init_tx_chns()
2273 sizeof(tx_chn->tx_chn_name), "tx%d", i); in am65_cpsw_nuss_init_tx_chns()
2275 spin_lock_init(&tx_chn->lock); in am65_cpsw_nuss_init_tx_chns()
2276 tx_chn->common = common; in am65_cpsw_nuss_init_tx_chns()
2277 tx_chn->id = i; in am65_cpsw_nuss_init_tx_chns()
2278 tx_chn->descs_num = max_desc_num; in am65_cpsw_nuss_init_tx_chns()
2280 tx_chn->tx_chn = in am65_cpsw_nuss_init_tx_chns()
2282 tx_chn->tx_chn_name, in am65_cpsw_nuss_init_tx_chns()
2284 if (IS_ERR(tx_chn->tx_chn)) { in am65_cpsw_nuss_init_tx_chns()
2285 ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn), in am65_cpsw_nuss_init_tx_chns()
2289 tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn); in am65_cpsw_nuss_init_tx_chns()
2291 tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev, in am65_cpsw_nuss_init_tx_chns()
2292 tx_chn->descs_num, in am65_cpsw_nuss_init_tx_chns()
2294 tx_chn->tx_chn_name); in am65_cpsw_nuss_init_tx_chns()
2295 if (IS_ERR(tx_chn->desc_pool)) { in am65_cpsw_nuss_init_tx_chns()
2296 ret = PTR_ERR(tx_chn->desc_pool); in am65_cpsw_nuss_init_tx_chns()
2301 hdesc_size_out = k3_cppi_desc_pool_desc_size(tx_chn->desc_pool); in am65_cpsw_nuss_init_tx_chns()
2302 tx_chn->dsize_log2 = __fls(hdesc_size_out); in am65_cpsw_nuss_init_tx_chns()
2303 WARN_ON(hdesc_size_out != (1 << tx_chn->dsize_log2)); in am65_cpsw_nuss_init_tx_chns()
2305 tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn); in am65_cpsw_nuss_init_tx_chns()
2306 if (tx_chn->irq < 0) { in am65_cpsw_nuss_init_tx_chns()
2308 tx_chn->irq); in am65_cpsw_nuss_init_tx_chns()
2309 ret = tx_chn->irq; in am65_cpsw_nuss_init_tx_chns()
2313 snprintf(tx_chn->tx_chn_name, in am65_cpsw_nuss_init_tx_chns()
2314 sizeof(tx_chn->tx_chn_name), "%s-tx%d", in am65_cpsw_nuss_init_tx_chns()
2315 dev_name(dev), tx_chn->id); in am65_cpsw_nuss_init_tx_chns()
2339 rx_chn = &common->rx_chns; in am65_cpsw_nuss_free_rx_chns()
2341 if (!IS_ERR_OR_NULL(rx_chn->desc_pool)) in am65_cpsw_nuss_free_rx_chns()
2342 k3_cppi_desc_pool_destroy(rx_chn->desc_pool); in am65_cpsw_nuss_free_rx_chns()
2344 if (!IS_ERR_OR_NULL(rx_chn->rx_chn)) in am65_cpsw_nuss_free_rx_chns()
2345 k3_udma_glue_release_rx_chn(rx_chn->rx_chn); in am65_cpsw_nuss_free_rx_chns()
2350 struct device *dev = common->dev; in am65_cpsw_nuss_remove_rx_chns()
2355 rx_chn = &common->rx_chns; in am65_cpsw_nuss_remove_rx_chns()
2356 flows = rx_chn->flows; in am65_cpsw_nuss_remove_rx_chns()
2359 for (i = 0; i < common->rx_ch_num_flows; i++) { in am65_cpsw_nuss_remove_rx_chns()
2367 common->rx_flow_id_base = -1; in am65_cpsw_nuss_remove_rx_chns()
2372 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; in am65_cpsw_nuss_init_rx_chns()
2375 struct device *dev = common->dev; in am65_cpsw_nuss_init_rx_chns()
2385 rx_cfg.flow_id_num = common->rx_ch_num_flows; in am65_cpsw_nuss_init_rx_chns()
2386 rx_cfg.flow_id_base = common->rx_flow_id_base; in am65_cpsw_nuss_init_rx_chns()
2389 rx_chn->dev = dev; in am65_cpsw_nuss_init_rx_chns()
2390 rx_chn->descs_num = max_desc_num * rx_cfg.flow_id_num; in am65_cpsw_nuss_init_rx_chns()
2392 for (i = 0; i < common->rx_ch_num_flows; i++) { in am65_cpsw_nuss_init_rx_chns()
2393 flow = &rx_chn->flows[i]; in am65_cpsw_nuss_init_rx_chns()
2394 flow->page_pool = NULL; in am65_cpsw_nuss_init_rx_chns()
2397 rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg); in am65_cpsw_nuss_init_rx_chns()
2398 if (IS_ERR(rx_chn->rx_chn)) { in am65_cpsw_nuss_init_rx_chns()
2399 ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn), in am65_cpsw_nuss_init_rx_chns()
2403 rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn); in am65_cpsw_nuss_init_rx_chns()
2405 rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev, in am65_cpsw_nuss_init_rx_chns()
2406 rx_chn->descs_num, in am65_cpsw_nuss_init_rx_chns()
2408 if (IS_ERR(rx_chn->desc_pool)) { in am65_cpsw_nuss_init_rx_chns()
2409 ret = PTR_ERR(rx_chn->desc_pool); in am65_cpsw_nuss_init_rx_chns()
2414 hdesc_size_out = k3_cppi_desc_pool_desc_size(rx_chn->desc_pool); in am65_cpsw_nuss_init_rx_chns()
2415 rx_chn->dsize_log2 = __fls(hdesc_size_out); in am65_cpsw_nuss_init_rx_chns()
2416 WARN_ON(hdesc_size_out != (1 << rx_chn->dsize_log2)); in am65_cpsw_nuss_init_rx_chns()
2418 common->rx_flow_id_base = in am65_cpsw_nuss_init_rx_chns()
2419 k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn); in am65_cpsw_nuss_init_rx_chns()
2420 dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base); in am65_cpsw_nuss_init_rx_chns()
2441 flow = &rx_chn->flows[i]; in am65_cpsw_nuss_init_rx_chns()
2442 flow->id = i; in am65_cpsw_nuss_init_rx_chns()
2443 flow->common = common; in am65_cpsw_nuss_init_rx_chns()
2444 flow->irq = -EINVAL; in am65_cpsw_nuss_init_rx_chns()
2450 rx_flow_cfg.rxfdq_cfg.mode = common->pdata.fdqring_mode; in am65_cpsw_nuss_init_rx_chns()
2452 ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn, in am65_cpsw_nuss_init_rx_chns()
2460 k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn, in am65_cpsw_nuss_init_rx_chns()
2463 flow->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i); in am65_cpsw_nuss_init_rx_chns()
2464 if (flow->irq <= 0) { in am65_cpsw_nuss_init_rx_chns()
2466 flow->irq); in am65_cpsw_nuss_init_rx_chns()
2467 ret = flow->irq; in am65_cpsw_nuss_init_rx_chns()
2471 snprintf(flow->name, in am65_cpsw_nuss_init_rx_chns()
2472 sizeof(flow->name), "%s-rx%d", in am65_cpsw_nuss_init_rx_chns()
2474 netif_napi_add(common->dma_ndev, &flow->napi_rx, in am65_cpsw_nuss_init_rx_chns()
2476 hrtimer_init(&flow->rx_hrtimer, CLOCK_MONOTONIC, in am65_cpsw_nuss_init_rx_chns()
2478 flow->rx_hrtimer.function = &am65_cpsw_nuss_rx_timer_callback; in am65_cpsw_nuss_init_rx_chns()
2480 ret = devm_request_irq(dev, flow->irq, in am65_cpsw_nuss_init_rx_chns()
2483 flow->name, flow); in am65_cpsw_nuss_init_rx_chns()
2486 i, flow->irq, ret); in am65_cpsw_nuss_init_rx_chns()
2487 flow->irq = -EINVAL; in am65_cpsw_nuss_init_rx_chns()
2492 /* setup classifier to route priorities to flows */ in am65_cpsw_nuss_init_rx_chns()
2493 cpsw_ale_classifier_setup_default(common->ale, common->rx_ch_num_flows); in am65_cpsw_nuss_init_rx_chns()
2509 host_p->common = common; in am65_cpsw_nuss_init_host_p()
2510 host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE; in am65_cpsw_nuss_init_host_p()
2511 host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE; in am65_cpsw_nuss_init_host_p()
2523 syscon = syscon_regmap_lookup_by_phandle(of_node, "ti,syscon-efuse"); in am65_cpsw_am654_get_efuse_macid()
2525 if (PTR_ERR(syscon) == -ENODEV) in am65_cpsw_am654_get_efuse_macid()
2530 ret = of_property_read_u32_index(of_node, "ti,syscon-efuse", 1, in am65_cpsw_am654_get_efuse_macid()
2550 struct device *dev = common->dev; in am65_cpsw_init_cpts()
2558 node = of_get_child_by_name(dev->of_node, "cpts"); in am65_cpsw_init_cpts()
2561 return -ENOENT; in am65_cpsw_init_cpts()
2564 reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE; in am65_cpsw_init_cpts()
2573 common->cpts = cpts; in am65_cpsw_init_cpts()
2575 * K3 CPSWxG modules may completely lose context during ON->OFF in am65_cpsw_init_cpts()
2588 struct device *dev = common->dev; in am65_cpsw_nuss_init_slave_ports()
2591 node = of_get_child_by_name(dev->of_node, "ethernet-ports"); in am65_cpsw_nuss_init_slave_ports()
2593 return -ENOENT; in am65_cpsw_nuss_init_slave_ports()
2600 if (strcmp(port_np->name, "port")) in am65_cpsw_nuss_init_slave_ports()
2610 if (!port_id || port_id > common->port_num) { in am65_cpsw_nuss_init_slave_ports()
2612 port_np, port_id, port_np->name); in am65_cpsw_nuss_init_slave_ports()
2613 ret = -EINVAL; in am65_cpsw_nuss_init_slave_ports()
2618 port->port_id = port_id; in am65_cpsw_nuss_init_slave_ports()
2619 port->common = common; in am65_cpsw_nuss_init_slave_ports()
2620 port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE + in am65_cpsw_nuss_init_slave_ports()
2622 if (common->pdata.extra_modes) in am65_cpsw_nuss_init_slave_ports()
2623 port->sgmii_base = common->ss_base + AM65_CPSW_SGMII_BASE * (port_id); in am65_cpsw_nuss_init_slave_ports()
2624 port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE + in am65_cpsw_nuss_init_slave_ports()
2626 port->name = of_get_property(port_np, "label", NULL); in am65_cpsw_nuss_init_slave_ports()
2627 port->fetch_ram_base = in am65_cpsw_nuss_init_slave_ports()
2628 common->cpsw_base + AM65_CPSW_NU_FRAM_BASE + in am65_cpsw_nuss_init_slave_ports()
2629 (AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1)); in am65_cpsw_nuss_init_slave_ports()
2631 port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base); in am65_cpsw_nuss_init_slave_ports()
2632 if (IS_ERR(port->slave.mac_sl)) { in am65_cpsw_nuss_init_slave_ports()
2633 ret = PTR_ERR(port->slave.mac_sl); in am65_cpsw_nuss_init_slave_ports()
2637 port->disabled = !of_device_is_available(port_np); in am65_cpsw_nuss_init_slave_ports()
2638 if (port->disabled) { in am65_cpsw_nuss_init_slave_ports()
2639 common->disabled_ports_mask |= BIT(port->port_id); in am65_cpsw_nuss_init_slave_ports()
2643 port->slave.ifphy = devm_of_phy_get(dev, port_np, NULL); in am65_cpsw_nuss_init_slave_ports()
2644 if (IS_ERR(port->slave.ifphy)) { in am65_cpsw_nuss_init_slave_ports()
2645 ret = PTR_ERR(port->slave.ifphy); in am65_cpsw_nuss_init_slave_ports()
2656 port->slave.mac_only = in am65_cpsw_nuss_init_slave_ports()
2657 of_property_read_bool(port_np, "ti,mac-only"); in am65_cpsw_nuss_init_slave_ports()
2660 port->slave.port_np = port_np; in am65_cpsw_nuss_init_slave_ports()
2661 ret = of_get_phy_mode(port_np, &port->slave.phy_if); in am65_cpsw_nuss_init_slave_ports()
2663 dev_err(dev, "%pOF read phy-mode err %d\n", in am65_cpsw_nuss_init_slave_ports()
2668 ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET, port->slave.phy_if); in am65_cpsw_nuss_init_slave_ports()
2672 ret = of_get_mac_address(port_np, port->slave.mac_addr); in am65_cpsw_nuss_init_slave_ports()
2675 port->port_id, in am65_cpsw_nuss_init_slave_ports()
2676 port->slave.mac_addr); in am65_cpsw_nuss_init_slave_ports()
2677 if (!is_valid_ether_addr(port->slave.mac_addr)) { in am65_cpsw_nuss_init_slave_ports()
2678 eth_random_addr(port->slave.mac_addr); in am65_cpsw_nuss_init_slave_ports()
2684 writel(0, port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP); in am65_cpsw_nuss_init_slave_ports()
2689 if (!(~common->disabled_ports_mask & GENMASK(common->port_num, 1))) { in am65_cpsw_nuss_init_slave_ports()
2691 return -ENODEV; in am65_cpsw_nuss_init_slave_ports()
2714 for (i = 0; i < common->port_num; i++) { in am65_cpsw_nuss_phylink_cleanup()
2715 port = &common->ports[i]; in am65_cpsw_nuss_phylink_cleanup()
2716 if (port->slave.phylink) in am65_cpsw_nuss_phylink_cleanup()
2717 phylink_destroy(port->slave.phylink); in am65_cpsw_nuss_phylink_cleanup()
2725 struct device *dev = common->dev; in am65_cpsw_nuss_init_port_ndev()
2730 port = &common->ports[port_idx]; in am65_cpsw_nuss_init_port_ndev()
2732 if (port->disabled) in am65_cpsw_nuss_init_port_ndev()
2736 port->ndev = alloc_etherdev_mqs(sizeof(struct am65_cpsw_ndev_priv), in am65_cpsw_nuss_init_port_ndev()
2739 if (!port->ndev) { in am65_cpsw_nuss_init_port_ndev()
2741 port->port_id); in am65_cpsw_nuss_init_port_ndev()
2742 return -ENOMEM; in am65_cpsw_nuss_init_port_ndev()
2745 ndev_priv = netdev_priv(port->ndev); in am65_cpsw_nuss_init_port_ndev()
2746 ndev_priv->port = port; in am65_cpsw_nuss_init_port_ndev()
2747 ndev_priv->msg_enable = AM65_CPSW_DEBUG; in am65_cpsw_nuss_init_port_ndev()
2748 mutex_init(&ndev_priv->mm_lock); in am65_cpsw_nuss_init_port_ndev()
2749 port->qos.link_speed = SPEED_UNKNOWN; in am65_cpsw_nuss_init_port_ndev()
2750 SET_NETDEV_DEV(port->ndev, dev); in am65_cpsw_nuss_init_port_ndev()
2751 port->ndev->dev.of_node = port->slave.port_np; in am65_cpsw_nuss_init_port_ndev()
2753 eth_hw_addr_set(port->ndev, port->slave.mac_addr); in am65_cpsw_nuss_init_port_ndev()
2755 port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE; in am65_cpsw_nuss_init_port_ndev()
2756 port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE - in am65_cpsw_nuss_init_port_ndev()
2758 port->ndev->hw_features = NETIF_F_SG | in am65_cpsw_nuss_init_port_ndev()
2762 port->ndev->features = port->ndev->hw_features | in am65_cpsw_nuss_init_port_ndev()
2764 port->ndev->xdp_features = NETDEV_XDP_ACT_BASIC | in am65_cpsw_nuss_init_port_ndev()
2767 port->ndev->vlan_features |= NETIF_F_SG; in am65_cpsw_nuss_init_port_ndev()
2768 port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops; in am65_cpsw_nuss_init_port_ndev()
2769 port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave; in am65_cpsw_nuss_init_port_ndev()
2772 port->slave.phylink_config.dev = &port->ndev->dev; in am65_cpsw_nuss_init_port_ndev()
2773 port->slave.phylink_config.type = PHYLINK_NETDEV; in am65_cpsw_nuss_init_port_ndev()
2774 port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 | in am65_cpsw_nuss_init_port_ndev()
2776 port->slave.phylink_config.mac_managed_pm = true; /* MAC does PM */ in am65_cpsw_nuss_init_port_ndev()
2778 switch (port->slave.phy_if) { in am65_cpsw_nuss_init_port_ndev()
2783 phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces); in am65_cpsw_nuss_init_port_ndev()
2788 port->slave.phylink_config.supported_interfaces); in am65_cpsw_nuss_init_port_ndev()
2794 if (common->pdata.extra_modes & BIT(port->slave.phy_if)) { in am65_cpsw_nuss_init_port_ndev()
2795 __set_bit(port->slave.phy_if, in am65_cpsw_nuss_init_port_ndev()
2796 port->slave.phylink_config.supported_interfaces); in am65_cpsw_nuss_init_port_ndev()
2798 dev_err(dev, "selected phy-mode is not supported\n"); in am65_cpsw_nuss_init_port_ndev()
2799 return -EOPNOTSUPP; in am65_cpsw_nuss_init_port_ndev()
2804 dev_err(dev, "selected phy-mode is not supported\n"); in am65_cpsw_nuss_init_port_ndev()
2805 return -EOPNOTSUPP; in am65_cpsw_nuss_init_port_ndev()
2808 phylink = phylink_create(&port->slave.phylink_config, in am65_cpsw_nuss_init_port_ndev()
2809 of_fwnode_handle(port->slave.port_np), in am65_cpsw_nuss_init_port_ndev()
2810 port->slave.phy_if, in am65_cpsw_nuss_init_port_ndev()
2815 port->slave.phylink = phylink; in am65_cpsw_nuss_init_port_ndev()
2818 if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM) in am65_cpsw_nuss_init_port_ndev()
2819 port->ndev->features &= ~NETIF_F_HW_CSUM; in am65_cpsw_nuss_init_port_ndev()
2821 ndev_priv->stats = netdev_alloc_pcpu_stats(struct am65_cpsw_ndev_stats); in am65_cpsw_nuss_init_port_ndev()
2822 if (!ndev_priv->stats) in am65_cpsw_nuss_init_port_ndev()
2823 return -ENOMEM; in am65_cpsw_nuss_init_port_ndev()
2826 ndev_priv->stats); in am65_cpsw_nuss_init_port_ndev()
2830 port->xdp_prog = NULL; in am65_cpsw_nuss_init_port_ndev()
2832 if (!common->dma_ndev) in am65_cpsw_nuss_init_port_ndev()
2833 common->dma_ndev = port->ndev; in am65_cpsw_nuss_init_port_ndev()
2843 for (i = 0; i < common->port_num; i++) { in am65_cpsw_nuss_init_ndevs()
2857 for (i = 0; i < common->port_num; i++) { in am65_cpsw_nuss_cleanup_ndev()
2858 port = &common->ports[i]; in am65_cpsw_nuss_cleanup_ndev()
2859 if (!port->ndev) in am65_cpsw_nuss_cleanup_ndev()
2861 if (port->ndev->reg_state == NETREG_REGISTERED) in am65_cpsw_nuss_cleanup_ndev()
2862 unregister_netdev(port->ndev); in am65_cpsw_nuss_cleanup_ndev()
2863 free_netdev(port->ndev); in am65_cpsw_nuss_cleanup_ndev()
2864 port->ndev = NULL; in am65_cpsw_nuss_cleanup_ndev()
2873 if (common->br_members == (GENMASK(common->port_num, 1) & ~common->disabled_ports_mask)) in am65_cpsw_port_offload_fwd_mark_update()
2876 dev_dbg(common->dev, "set offload_fwd_mark %d\n", set_val); in am65_cpsw_port_offload_fwd_mark_update()
2878 for (i = 1; i <= common->port_num; i++) { in am65_cpsw_port_offload_fwd_mark_update()
2882 if (!port->ndev) in am65_cpsw_port_offload_fwd_mark_update()
2885 priv = am65_ndev_to_priv(port->ndev); in am65_cpsw_port_offload_fwd_mark_update()
2886 priv->offload_fwd_mark = set_val; in am65_cpsw_port_offload_fwd_mark_update()
2892 if (ndev->netdev_ops == &am65_cpsw_nuss_netdev_ops) { in am65_cpsw_port_dev_check()
2895 return !common->is_emac_mode; in am65_cpsw_port_dev_check()
2909 if (!common->br_members) { in am65_cpsw_netdevice_port_link()
2910 common->hw_bridge_dev = br_ndev; in am65_cpsw_netdevice_port_link()
2915 if (common->hw_bridge_dev != br_ndev) in am65_cpsw_netdevice_port_link()
2916 return -EOPNOTSUPP; in am65_cpsw_netdevice_port_link()
2924 common->br_members |= BIT(priv->port->port_id); in am65_cpsw_netdevice_port_link()
2938 common->br_members &= ~BIT(priv->port->port_id); in am65_cpsw_netdevice_port_unlink()
2942 if (!common->br_members) in am65_cpsw_netdevice_port_unlink()
2943 common->hw_bridge_dev = NULL; in am65_cpsw_netdevice_port_unlink()
2962 if (netif_is_bridge_master(info->upper_dev)) { in am65_cpsw_netdevice_event()
2963 if (info->linking) in am65_cpsw_netdevice_event()
2965 info->upper_dev, in am65_cpsw_netdevice_event()
2986 cpsw->am65_cpsw_netdevice_nb.notifier_call = &am65_cpsw_netdevice_event; in am65_cpsw_register_notifiers()
2987 ret = register_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb); in am65_cpsw_register_notifiers()
2989 dev_err(cpsw->dev, "can't register netdevice notifier\n"); in am65_cpsw_register_notifiers()
2995 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb); in am65_cpsw_register_notifiers()
3007 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb); in am65_cpsw_unregister_notifiers()
3014 cpsw_ale_add_mcast(cpsw->ale, eth_stp_addr, ALE_PORT_HOST, ALE_SUPER, 0, in am65_cpsw_init_stp_ale_entry()
3022 writel(common->default_vlan, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); in am65_cpsw_init_host_port_switch()
3026 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 1); in am65_cpsw_init_host_port_switch()
3027 dev_dbg(common->dev, "Set P0_UNI_FLOOD\n"); in am65_cpsw_init_host_port_switch()
3028 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 0); in am65_cpsw_init_host_port_switch()
3035 writel(0, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); in am65_cpsw_init_host_port_emac()
3037 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 0); in am65_cpsw_init_host_port_emac()
3038 dev_dbg(common->dev, "unset P0_UNI_FLOOD\n"); in am65_cpsw_init_host_port_emac()
3040 /* learning make no sense in multi-mac mode */ in am65_cpsw_init_host_port_emac()
3041 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 1); in am65_cpsw_init_host_port_emac()
3048 struct am65_cpsw_common *common = dl_priv->common; in am65_cpsw_dl_switch_mode_get()
3050 dev_dbg(common->dev, "%s id:%u\n", __func__, id); in am65_cpsw_dl_switch_mode_get()
3053 return -EOPNOTSUPP; in am65_cpsw_dl_switch_mode_get()
3055 ctx->val.vbool = !common->is_emac_mode; in am65_cpsw_dl_switch_mode_get()
3062 struct am65_cpsw_slave_data *slave = &port->slave; in am65_cpsw_init_port_emac_ale()
3063 struct am65_cpsw_common *common = port->common; in am65_cpsw_init_port_emac_ale()
3066 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); in am65_cpsw_init_port_emac_ale()
3068 if (slave->mac_only) in am65_cpsw_init_port_emac_ale()
3069 /* enable mac-only mode on port */ in am65_cpsw_init_port_emac_ale()
3070 cpsw_ale_control_set(common->ale, port->port_id, in am65_cpsw_init_port_emac_ale()
3073 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_NOLEARN, 1); in am65_cpsw_init_port_emac_ale()
3075 port_mask = BIT(port->port_id) | ALE_PORT_HOST; in am65_cpsw_init_port_emac_ale()
3077 cpsw_ale_add_ucast(common->ale, port->ndev->dev_addr, in am65_cpsw_init_port_emac_ale()
3078 HOST_PORT_NUM, ALE_SECURE, slave->port_vlan); in am65_cpsw_init_port_emac_ale()
3079 cpsw_ale_add_mcast(common->ale, port->ndev->broadcast, in am65_cpsw_init_port_emac_ale()
3080 port_mask, ALE_VLAN, slave->port_vlan, ALE_MCAST_FWD_2); in am65_cpsw_init_port_emac_ale()
3085 struct am65_cpsw_slave_data *slave = &port->slave; in am65_cpsw_init_port_switch_ale()
3086 struct am65_cpsw_common *cpsw = port->common; in am65_cpsw_init_port_switch_ale()
3089 cpsw_ale_control_set(cpsw->ale, port->port_id, in am65_cpsw_init_port_switch_ale()
3092 cpsw_ale_add_ucast(cpsw->ale, port->ndev->dev_addr, in am65_cpsw_init_port_switch_ale()
3094 slave->port_vlan); in am65_cpsw_init_port_switch_ale()
3096 port_mask = BIT(port->port_id) | ALE_PORT_HOST; in am65_cpsw_init_port_switch_ale()
3098 cpsw_ale_add_mcast(cpsw->ale, port->ndev->broadcast, in am65_cpsw_init_port_switch_ale()
3099 port_mask, ALE_VLAN, slave->port_vlan, in am65_cpsw_init_port_switch_ale()
3102 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); in am65_cpsw_init_port_switch_ale()
3104 cpsw_ale_control_set(cpsw->ale, port->port_id, in am65_cpsw_init_port_switch_ale()
3113 struct am65_cpsw_common *cpsw = dl_priv->common; in am65_cpsw_dl_switch_mode_set()
3114 bool switch_en = ctx->val.vbool; in am65_cpsw_dl_switch_mode_set()
3118 dev_dbg(cpsw->dev, "%s id:%u\n", __func__, id); in am65_cpsw_dl_switch_mode_set()
3121 return -EOPNOTSUPP; in am65_cpsw_dl_switch_mode_set()
3123 if (switch_en == !cpsw->is_emac_mode) in am65_cpsw_dl_switch_mode_set()
3126 if (!switch_en && cpsw->br_members) { in am65_cpsw_dl_switch_mode_set()
3127 dev_err(cpsw->dev, "Remove ports from bridge before disabling switch mode\n"); in am65_cpsw_dl_switch_mode_set()
3128 return -EINVAL; in am65_cpsw_dl_switch_mode_set()
3133 cpsw->is_emac_mode = !switch_en; in am65_cpsw_dl_switch_mode_set()
3135 for (i = 0; i < cpsw->port_num; i++) { in am65_cpsw_dl_switch_mode_set()
3136 struct net_device *sl_ndev = cpsw->ports[i].ndev; in am65_cpsw_dl_switch_mode_set()
3146 for (i = 0; i < cpsw->port_num; i++) { in am65_cpsw_dl_switch_mode_set()
3147 struct net_device *sl_ndev = cpsw->ports[i].ndev; in am65_cpsw_dl_switch_mode_set()
3155 slave->port_vlan = cpsw->default_vlan; in am65_cpsw_dl_switch_mode_set()
3157 slave->port_vlan = 0; in am65_cpsw_dl_switch_mode_set()
3163 cpsw_ale_control_set(cpsw->ale, 0, ALE_BYPASS, 1); in am65_cpsw_dl_switch_mode_set()
3165 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_CLEAR, 1); in am65_cpsw_dl_switch_mode_set()
3166 cpsw_ale_control_get(cpsw->ale, HOST_PORT_NUM, ALE_AGEOUT); in am65_cpsw_dl_switch_mode_set()
3169 dev_info(cpsw->dev, "Enable switch mode\n"); in am65_cpsw_dl_switch_mode_set()
3173 for (i = 0; i < cpsw->port_num; i++) { in am65_cpsw_dl_switch_mode_set()
3174 struct net_device *sl_ndev = cpsw->ports[i].ndev; in am65_cpsw_dl_switch_mode_set()
3183 slave->port_vlan = cpsw->default_vlan; in am65_cpsw_dl_switch_mode_set()
3190 dev_info(cpsw->dev, "Disable switch mode\n"); in am65_cpsw_dl_switch_mode_set()
3194 for (i = 0; i < cpsw->port_num; i++) { in am65_cpsw_dl_switch_mode_set()
3195 struct net_device *sl_ndev = cpsw->ports[i].ndev; in am65_cpsw_dl_switch_mode_set()
3202 port->slave.port_vlan = 0; in am65_cpsw_dl_switch_mode_set()
3207 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_BYPASS, 0); in am65_cpsw_dl_switch_mode_set()
3226 struct device *dev = common->dev; in am65_cpsw_nuss_register_devlink()
3232 common->devlink = in am65_cpsw_nuss_register_devlink()
3234 if (!common->devlink) in am65_cpsw_nuss_register_devlink()
3235 return -ENOMEM; in am65_cpsw_nuss_register_devlink()
3237 dl_priv = devlink_priv(common->devlink); in am65_cpsw_nuss_register_devlink()
3238 dl_priv->common = common; in am65_cpsw_nuss_register_devlink()
3245 ret = devlink_params_register(common->devlink, in am65_cpsw_nuss_register_devlink()
3254 for (i = 1; i <= common->port_num; i++) { in am65_cpsw_nuss_register_devlink()
3256 dl_port = &port->devlink_port; in am65_cpsw_nuss_register_devlink()
3258 if (port->ndev) in am65_cpsw_nuss_register_devlink()
3262 attrs.phys.port_number = port->port_id; in am65_cpsw_nuss_register_devlink()
3264 memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len); in am65_cpsw_nuss_register_devlink()
3267 ret = devlink_port_register(common->devlink, dl_port, port->port_id); in am65_cpsw_nuss_register_devlink()
3270 port->port_id, ret); in am65_cpsw_nuss_register_devlink()
3274 devlink_register(common->devlink); in am65_cpsw_nuss_register_devlink()
3278 for (i = i - 1; i >= 1; i--) { in am65_cpsw_nuss_register_devlink()
3280 dl_port = &port->devlink_port; in am65_cpsw_nuss_register_devlink()
3285 devlink_free(common->devlink); in am65_cpsw_nuss_register_devlink()
3295 devlink_unregister(common->devlink); in am65_cpsw_unregister_devlink()
3297 for (i = 1; i <= common->port_num; i++) { in am65_cpsw_unregister_devlink()
3299 dl_port = &port->devlink_port; in am65_cpsw_unregister_devlink()
3306 devlink_params_unregister(common->devlink, in am65_cpsw_unregister_devlink()
3310 devlink_free(common->devlink); in am65_cpsw_unregister_devlink()
3315 struct am65_cpsw_rx_chn *rx_chan = &common->rx_chns; in am65_cpsw_nuss_register_ndevs()
3316 struct am65_cpsw_tx_chn *tx_chan = common->tx_chns; in am65_cpsw_nuss_register_ndevs()
3317 struct device *dev = common->dev; in am65_cpsw_nuss_register_ndevs()
3333 for (i = 0; i < common->tx_ch_num; i++) { in am65_cpsw_nuss_register_ndevs()
3339 for (i = 0; i < common->rx_ch_num_flows; i++) in am65_cpsw_nuss_register_ndevs()
3340 k3_udma_glue_reset_rx_chn(rx_chan->rx_chn, i, in am65_cpsw_nuss_register_ndevs()
3344 k3_udma_glue_disable_rx_chn(rx_chan->rx_chn); in am65_cpsw_nuss_register_ndevs()
3350 for (i = 0; i < common->port_num; i++) { in am65_cpsw_nuss_register_ndevs()
3351 port = &common->ports[i]; in am65_cpsw_nuss_register_ndevs()
3353 if (!port->ndev) in am65_cpsw_nuss_register_ndevs()
3356 SET_NETDEV_DEVLINK_PORT(port->ndev, &port->devlink_port); in am65_cpsw_nuss_register_ndevs()
3358 ret = register_netdev(port->ndev); in am65_cpsw_nuss_register_ndevs()
3391 common->tx_ch_num = num_tx; in am65_cpsw_nuss_update_tx_rx_chns()
3392 common->rx_ch_num_flows = num_rx; in am65_cpsw_nuss_update_tx_rx_chns()
3420 .ale_dev_id = "am65x-cpsw2g",
3426 .ale_dev_id = "am65x-cpsw2g",
3432 .ale_dev_id = "am64-cpswxg",
3438 .ale_dev_id = "am64-cpswxg",
3445 .ale_dev_id = "am64-cpswxg",
3452 .ale_dev_id = "am64-cpswxg",
3459 { .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
3460 { .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
3461 { .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
3462 { .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata},
3463 { .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata},
3464 { .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata},
3474 if (soc && soc->data) { in am65_cpsw_nuss_apply_socinfo()
3475 const struct am65_cpsw_soc_pdata *socdata = soc->data; in am65_cpsw_nuss_apply_socinfo()
3478 common->pdata.quirks &= ~socdata->quirks_dis; in am65_cpsw_nuss_apply_socinfo()
3486 struct device *dev = &pdev->dev; in am65_cpsw_nuss_probe()
3497 return -ENOMEM; in am65_cpsw_nuss_probe()
3498 common->dev = dev; in am65_cpsw_nuss_probe()
3502 return -EINVAL; in am65_cpsw_nuss_probe()
3503 common->pdata = *(const struct am65_cpsw_pdata *)of_id->data; in am65_cpsw_nuss_probe()
3508 common->ss_base = devm_ioremap_resource(&pdev->dev, res); in am65_cpsw_nuss_probe()
3509 if (IS_ERR(common->ss_base)) in am65_cpsw_nuss_probe()
3510 return PTR_ERR(common->ss_base); in am65_cpsw_nuss_probe()
3511 common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE; in am65_cpsw_nuss_probe()
3513 id_temp = cpu_to_be64(res->start); in am65_cpsw_nuss_probe()
3514 memcpy(common->switch_id, &id_temp, sizeof(res->start)); in am65_cpsw_nuss_probe()
3516 node = of_get_child_by_name(dev->of_node, "ethernet-ports"); in am65_cpsw_nuss_probe()
3518 return -ENOENT; in am65_cpsw_nuss_probe()
3519 common->port_num = of_get_child_count(node); in am65_cpsw_nuss_probe()
3521 if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS) in am65_cpsw_nuss_probe()
3522 return -ENOENT; in am65_cpsw_nuss_probe()
3524 common->rx_flow_id_base = -1; in am65_cpsw_nuss_probe()
3525 init_completion(&common->tdown_complete); in am65_cpsw_nuss_probe()
3526 common->tx_ch_num = AM65_CPSW_DEFAULT_TX_CHNS; in am65_cpsw_nuss_probe()
3527 common->rx_ch_num_flows = AM65_CPSW_DEFAULT_RX_CHN_FLOWS; in am65_cpsw_nuss_probe()
3528 common->pf_p0_rx_ptype_rrobin = false; in am65_cpsw_nuss_probe()
3529 common->default_vlan = 1; in am65_cpsw_nuss_probe()
3531 common->ports = devm_kcalloc(dev, common->port_num, in am65_cpsw_nuss_probe()
3532 sizeof(*common->ports), in am65_cpsw_nuss_probe()
3534 if (!common->ports) in am65_cpsw_nuss_probe()
3535 return -ENOMEM; in am65_cpsw_nuss_probe()
3540 common->bus_freq = clk_get_rate(clk); in am65_cpsw_nuss_probe()
3549 node = of_get_child_by_name(dev->of_node, "mdio"); in am65_cpsw_nuss_probe()
3557 ret = -ENODEV; in am65_cpsw_nuss_probe()
3561 common->mdio_dev = &mdio_pdev->dev; in am65_cpsw_nuss_probe()
3578 ale_params.ale_ports = common->port_num + 1; in am65_cpsw_nuss_probe()
3579 ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE; in am65_cpsw_nuss_probe()
3580 ale_params.dev_id = common->pdata.ale_dev_id; in am65_cpsw_nuss_probe()
3581 ale_params.bus_freq = common->bus_freq; in am65_cpsw_nuss_probe()
3583 common->ale = cpsw_ale_create(&ale_params); in am65_cpsw_nuss_probe()
3584 if (IS_ERR(common->ale)) { in am65_cpsw_nuss_probe()
3586 ret = PTR_ERR(common->ale); in am65_cpsw_nuss_probe()
3590 ale_entries = common->ale->params.ale_entries; in am65_cpsw_nuss_probe()
3591 common->ale_context = devm_kzalloc(dev, in am65_cpsw_nuss_probe()
3599 for (i = 0; i < common->port_num; i++) in am65_cpsw_nuss_probe()
3600 am65_cpsw_nuss_slave_disable_unused(&common->ports[i]); in am65_cpsw_nuss_probe()
3604 common->is_emac_mode = true; in am65_cpsw_nuss_probe()
3620 am65_cpts_release(common->cpts); in am65_cpsw_nuss_probe()
3622 if (common->mdio_dev) in am65_cpsw_nuss_probe()
3623 of_platform_device_destroy(common->mdio_dev, NULL); in am65_cpsw_nuss_probe()
3632 struct device *dev = &pdev->dev; in am65_cpsw_nuss_remove()
3638 ret = pm_runtime_resume_and_get(&pdev->dev); in am65_cpsw_nuss_remove()
3643 dev_err(&pdev->dev, "Failed to resume device (%pe)\n", in am65_cpsw_nuss_remove()
3656 am65_cpts_release(common->cpts); in am65_cpsw_nuss_remove()
3659 if (common->mdio_dev) in am65_cpsw_nuss_remove()
3660 of_platform_device_destroy(common->mdio_dev, NULL); in am65_cpsw_nuss_remove()
3662 pm_runtime_put_sync(&pdev->dev); in am65_cpsw_nuss_remove()
3663 pm_runtime_disable(&pdev->dev); in am65_cpsw_nuss_remove()
3674 cpsw_ale_dump(common->ale, common->ale_context); in am65_cpsw_nuss_suspend()
3675 host_p->vid_context = readl(host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); in am65_cpsw_nuss_suspend()
3676 for (i = 0; i < common->port_num; i++) { in am65_cpsw_nuss_suspend()
3677 port = &common->ports[i]; in am65_cpsw_nuss_suspend()
3678 ndev = port->ndev; in am65_cpsw_nuss_suspend()
3683 port->vid_context = readl(port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); in am65_cpsw_nuss_suspend()
3696 am65_cpts_suspend(common->cpts); in am65_cpsw_nuss_suspend()
3720 for (i = 0; i < common->rx_ch_num_flows; i++) { in am65_cpsw_nuss_resume()
3721 if (common->rx_chns.flows[i].irq_disabled) in am65_cpsw_nuss_resume()
3722 disable_irq(common->rx_chns.flows[i].irq); in am65_cpsw_nuss_resume()
3725 am65_cpts_resume(common->cpts); in am65_cpsw_nuss_resume()
3727 for (i = 0; i < common->port_num; i++) { in am65_cpsw_nuss_resume()
3728 port = &common->ports[i]; in am65_cpsw_nuss_resume()
3729 ndev = port->ndev; in am65_cpsw_nuss_resume()
3745 writel(port->vid_context, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); in am65_cpsw_nuss_resume()
3748 writel(host_p->vid_context, host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); in am65_cpsw_nuss_resume()
3749 cpsw_ale_restore(common->ale, common->ale_context); in am65_cpsw_nuss_resume()