134 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue);
135 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue);
136 static void stmmac_reset_queues_param(struct stmmac_priv *priv);
137 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
138 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
139 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
150 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled)  in stmmac_bus_clks_config()  argument
155 		ret = clk_prepare_enable(priv->plat->stmmac_clk);  in stmmac_bus_clks_config()
158 		ret = clk_prepare_enable(priv->plat->pclk);  in stmmac_bus_clks_config()
160 			clk_disable_unprepare(priv->plat->stmmac_clk);  in stmmac_bus_clks_config()
163 		if (priv->plat->clks_config) {  in stmmac_bus_clks_config()
164 			ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled);  in stmmac_bus_clks_config()
166 				clk_disable_unprepare(priv->plat->stmmac_clk);  in stmmac_bus_clks_config()
167 				clk_disable_unprepare(priv->plat->pclk);  in stmmac_bus_clks_config()
172 		clk_disable_unprepare(priv->plat->stmmac_clk);  in stmmac_bus_clks_config()
173 		clk_disable_unprepare(priv->plat->pclk);  in stmmac_bus_clks_config()
174 		if (priv->plat->clks_config)  in stmmac_bus_clks_config()
175 			priv->plat->clks_config(priv->plat->bsp_priv, enabled);  in stmmac_bus_clks_config()
203 static void __stmmac_disable_all_queues(struct stmmac_priv *priv)  in __stmmac_disable_all_queues()  argument
205 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;  in __stmmac_disable_all_queues()
206 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;  in __stmmac_disable_all_queues()
211 		struct stmmac_channel *ch = &priv->channel[queue];  in __stmmac_disable_all_queues()
213 		if (stmmac_xdp_is_enabled(priv) &&  in __stmmac_disable_all_queues()
214 		    test_bit(queue, priv->af_xdp_zc_qps)) {  in __stmmac_disable_all_queues()
230 static void stmmac_disable_all_queues(struct stmmac_priv *priv)  in stmmac_disable_all_queues()  argument
232 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;  in stmmac_disable_all_queues()
238 		rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_disable_all_queues()
245 	__stmmac_disable_all_queues(priv);  in stmmac_disable_all_queues()
252 static void stmmac_enable_all_queues(struct stmmac_priv *priv)  in stmmac_enable_all_queues()  argument
254 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;  in stmmac_enable_all_queues()
255 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;  in stmmac_enable_all_queues()
260 		struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_enable_all_queues()
262 		if (stmmac_xdp_is_enabled(priv) &&  in stmmac_enable_all_queues()
263 		    test_bit(queue, priv->af_xdp_zc_qps)) {  in stmmac_enable_all_queues()
275 static void stmmac_service_event_schedule(struct stmmac_priv *priv)  in stmmac_service_event_schedule()  argument
277 	if (!test_bit(STMMAC_DOWN, &priv->state) &&  in stmmac_service_event_schedule()
278 	    !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))  in stmmac_service_event_schedule()
279 		queue_work(priv->wq, &priv->service_task);  in stmmac_service_event_schedule()
282 static void stmmac_global_err(struct stmmac_priv *priv)  in stmmac_global_err()  argument
284 	netif_carrier_off(priv->dev);  in stmmac_global_err()
285 	set_bit(STMMAC_RESET_REQUESTED, &priv->state);  in stmmac_global_err()
286 	stmmac_service_event_schedule(priv);  in stmmac_global_err()
301 static void stmmac_clk_csr_set(struct stmmac_priv *priv)  in stmmac_clk_csr_set()  argument
305 	clk_rate = clk_get_rate(priv->plat->stmmac_clk);  in stmmac_clk_csr_set()
314 	if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {  in stmmac_clk_csr_set()
316 			priv->clk_csr = STMMAC_CSR_20_35M;  in stmmac_clk_csr_set()
318 			priv->clk_csr = STMMAC_CSR_35_60M;  in stmmac_clk_csr_set()
320 			priv->clk_csr = STMMAC_CSR_60_100M;  in stmmac_clk_csr_set()
322 			priv->clk_csr = STMMAC_CSR_100_150M;  in stmmac_clk_csr_set()
324 			priv->clk_csr = STMMAC_CSR_150_250M;  in stmmac_clk_csr_set()
326 			priv->clk_csr = STMMAC_CSR_250_300M;  in stmmac_clk_csr_set()
329 	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) {  in stmmac_clk_csr_set()
331 			priv->clk_csr = 0x03;  in stmmac_clk_csr_set()
333 			priv->clk_csr = 0x02;  in stmmac_clk_csr_set()
335 			priv->clk_csr = 0x01;  in stmmac_clk_csr_set()
337 			priv->clk_csr = 0;  in stmmac_clk_csr_set()
340 	if (priv->plat->has_xgmac) {  in stmmac_clk_csr_set()
342 			priv->clk_csr = 0x5;  in stmmac_clk_csr_set()
344 			priv->clk_csr = 0x4;  in stmmac_clk_csr_set()
346 			priv->clk_csr = 0x3;  in stmmac_clk_csr_set()
348 			priv->clk_csr = 0x2;  in stmmac_clk_csr_set()
350 			priv->clk_csr = 0x1;  in stmmac_clk_csr_set()
352 			priv->clk_csr = 0x0;  in stmmac_clk_csr_set()
362 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)  in stmmac_tx_avail()  argument
364 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tx_avail()
370 		avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;  in stmmac_tx_avail()
380 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)  in stmmac_rx_dirty()  argument
382 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx_dirty()
388 		dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;  in stmmac_rx_dirty()
393 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en)  in stmmac_lpi_entry_timer_config()  argument
398 	priv->eee_sw_timer_en = en ? 0 : 1;  in stmmac_lpi_entry_timer_config()
399 	tx_lpi_timer  = en ? priv->tx_lpi_timer : 0;  in stmmac_lpi_entry_timer_config()
400 	stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer);  in stmmac_lpi_entry_timer_config()
409 static int stmmac_enable_eee_mode(struct stmmac_priv *priv)  in stmmac_enable_eee_mode()  argument
411 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_enable_eee_mode()
416 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_enable_eee_mode()
423 	if (!priv->tx_path_in_lpi_mode)  in stmmac_enable_eee_mode()
424 		stmmac_set_eee_mode(priv, priv->hw,  in stmmac_enable_eee_mode()
425 			priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING);  in stmmac_enable_eee_mode()
435 void stmmac_disable_eee_mode(struct stmmac_priv *priv)  in stmmac_disable_eee_mode()  argument
437 	if (!priv->eee_sw_timer_en) {  in stmmac_disable_eee_mode()
438 		stmmac_lpi_entry_timer_config(priv, 0);  in stmmac_disable_eee_mode()
442 	stmmac_reset_eee_mode(priv, priv->hw);  in stmmac_disable_eee_mode()
443 	del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_disable_eee_mode()
444 	priv->tx_path_in_lpi_mode = false;  in stmmac_disable_eee_mode()
456 	struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);  in stmmac_eee_ctrl_timer()  local
458 	if (stmmac_enable_eee_mode(priv))  in stmmac_eee_ctrl_timer()
459 		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));  in stmmac_eee_ctrl_timer()
470 bool stmmac_eee_init(struct stmmac_priv *priv)  in stmmac_eee_init()  argument
472 	int eee_tw_timer = priv->eee_tw_timer;  in stmmac_eee_init()
475 	if (!priv->dma_cap.eee)  in stmmac_eee_init()
478 	mutex_lock(&priv->lock);  in stmmac_eee_init()
481 	if (!priv->eee_active) {  in stmmac_eee_init()
482 		if (priv->eee_enabled) {  in stmmac_eee_init()
483 			netdev_dbg(priv->dev, "disable EEE\n");  in stmmac_eee_init()
484 			stmmac_lpi_entry_timer_config(priv, 0);  in stmmac_eee_init()
485 			del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_eee_init()
486 			stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer);  in stmmac_eee_init()
487 			if (priv->hw->xpcs)  in stmmac_eee_init()
488 				xpcs_config_eee(priv->hw->xpcs,  in stmmac_eee_init()
489 						priv->plat->mult_fact_100ns,  in stmmac_eee_init()
492 		mutex_unlock(&priv->lock);  in stmmac_eee_init()
496 	if (priv->eee_active && !priv->eee_enabled) {  in stmmac_eee_init()
497 		timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);  in stmmac_eee_init()
498 		stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,  in stmmac_eee_init()
500 		if (priv->hw->xpcs)  in stmmac_eee_init()
501 			xpcs_config_eee(priv->hw->xpcs,  in stmmac_eee_init()
502 					priv->plat->mult_fact_100ns,  in stmmac_eee_init()
506 	if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) {  in stmmac_eee_init()
507 		del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_eee_init()
508 		priv->tx_path_in_lpi_mode = false;  in stmmac_eee_init()
509 		stmmac_lpi_entry_timer_config(priv, 1);  in stmmac_eee_init()
511 		stmmac_lpi_entry_timer_config(priv, 0);  in stmmac_eee_init()
512 		mod_timer(&priv->eee_ctrl_timer,  in stmmac_eee_init()
513 			  STMMAC_LPI_T(priv->tx_lpi_timer));  in stmmac_eee_init()
516 	mutex_unlock(&priv->lock);  in stmmac_eee_init()
517 	netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");  in stmmac_eee_init()
529 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,  in stmmac_get_tx_hwtstamp()  argument
536 	if (!priv->hwts_tx_en)  in stmmac_get_tx_hwtstamp()
544 	if (stmmac_get_tx_timestamp_status(priv, p)) {  in stmmac_get_tx_hwtstamp()
545 		stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);  in stmmac_get_tx_hwtstamp()
547 	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {  in stmmac_get_tx_hwtstamp()
552 		ns -= priv->plat->cdc_error_adj;  in stmmac_get_tx_hwtstamp()
557 		netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);  in stmmac_get_tx_hwtstamp()
572 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,  in stmmac_get_rx_hwtstamp()  argument
579 	if (!priv->hwts_rx_en)  in stmmac_get_rx_hwtstamp()
582 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)  in stmmac_get_rx_hwtstamp()
586 	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {  in stmmac_get_rx_hwtstamp()
587 		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);  in stmmac_get_rx_hwtstamp()
589 		ns -= priv->plat->cdc_error_adj;  in stmmac_get_rx_hwtstamp()
591 		netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);  in stmmac_get_rx_hwtstamp()
596 		netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");  in stmmac_get_rx_hwtstamp()
613 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hwtstamp_set()  local
624 	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {  in stmmac_hwtstamp_set()
625 		netdev_alert(priv->dev, "No support for HW time stamping\n");  in stmmac_hwtstamp_set()
626 		priv->hwts_tx_en = 0;  in stmmac_hwtstamp_set()
627 		priv->hwts_rx_en = 0;  in stmmac_hwtstamp_set()
636 	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",  in stmmac_hwtstamp_set()
643 	if (priv->adv_ts) {  in stmmac_hwtstamp_set()
724 			if (priv->synopsys_id < DWMAC_CORE_4_10)  in stmmac_hwtstamp_set()
777 	priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);  in stmmac_hwtstamp_set()
778 	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;  in stmmac_hwtstamp_set()
780 	priv->systime_flags = STMMAC_HWTS_ACTIVE;  in stmmac_hwtstamp_set()
782 	if (priv->hwts_tx_en || priv->hwts_rx_en) {  in stmmac_hwtstamp_set()
783 		priv->systime_flags |= tstamp_all | ptp_v2 |  in stmmac_hwtstamp_set()
789 	stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags);  in stmmac_hwtstamp_set()
791 	memcpy(&priv->tstamp_config, &config, sizeof(config));  in stmmac_hwtstamp_set()
808 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hwtstamp_get()  local
809 	struct hwtstamp_config *config = &priv->tstamp_config;  in stmmac_hwtstamp_get()
811 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))  in stmmac_hwtstamp_get()
828 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags)  in stmmac_init_tstamp_counter()  argument
830 	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;  in stmmac_init_tstamp_counter()
835 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))  in stmmac_init_tstamp_counter()
838 	stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags);  in stmmac_init_tstamp_counter()
839 	priv->systime_flags = systime_flags;  in stmmac_init_tstamp_counter()
842 	stmmac_config_sub_second_increment(priv, priv->ptpaddr,  in stmmac_init_tstamp_counter()
843 					   priv->plat->clk_ptp_rate,  in stmmac_init_tstamp_counter()
848 	priv->sub_second_inc = sec_inc;  in stmmac_init_tstamp_counter()
856 	priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);  in stmmac_init_tstamp_counter()
857 	stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);  in stmmac_init_tstamp_counter()
863 	stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec);  in stmmac_init_tstamp_counter()
876 static int stmmac_init_ptp(struct stmmac_priv *priv)  in stmmac_init_ptp()  argument
878 	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;  in stmmac_init_ptp()
881 	if (priv->plat->ptp_clk_freq_config)  in stmmac_init_ptp()
882 		priv->plat->ptp_clk_freq_config(priv);  in stmmac_init_ptp()
884 	ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE);  in stmmac_init_ptp()
888 	priv->adv_ts = 0;  in stmmac_init_ptp()
890 	if (xmac && priv->dma_cap.atime_stamp)  in stmmac_init_ptp()
891 		priv->adv_ts = 1;  in stmmac_init_ptp()
893 	else if (priv->extend_desc && priv->dma_cap.atime_stamp)  in stmmac_init_ptp()
894 		priv->adv_ts = 1;  in stmmac_init_ptp()
896 	if (priv->dma_cap.time_stamp)  in stmmac_init_ptp()
897 		netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");  in stmmac_init_ptp()
899 	if (priv->adv_ts)  in stmmac_init_ptp()
900 		netdev_info(priv->dev,  in stmmac_init_ptp()
903 	priv->hwts_tx_en = 0;  in stmmac_init_ptp()
904 	priv->hwts_rx_en = 0;  in stmmac_init_ptp()
906 	if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY)  in stmmac_init_ptp()
907 		stmmac_hwtstamp_correct_latency(priv, priv);  in stmmac_init_ptp()
912 static void stmmac_release_ptp(struct stmmac_priv *priv)  in stmmac_release_ptp()  argument
914 	clk_disable_unprepare(priv->plat->clk_ptp_ref);  in stmmac_release_ptp()
915 	stmmac_ptp_unregister(priv);  in stmmac_release_ptp()
924 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)  in stmmac_mac_flow_ctrl()  argument
926 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_mac_flow_ctrl()
928 	stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,  in stmmac_mac_flow_ctrl()
929 			priv->pause, tx_cnt);  in stmmac_mac_flow_ctrl()
935 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_get_caps()  local
938 	stmmac_mac_update_caps(priv);  in stmmac_mac_get_caps()
940 	config->mac_capabilities = priv->hw->link.caps;  in stmmac_mac_get_caps()
942 	if (priv->plat->max_speed)  in stmmac_mac_get_caps()
943 		phylink_limit_mac_speed(config, priv->plat->max_speed);  in stmmac_mac_get_caps()
951 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_select_pcs()  local
954 	if (priv->plat->select_pcs) {  in stmmac_mac_select_pcs()
955 		pcs = priv->plat->select_pcs(priv, interface);  in stmmac_mac_select_pcs()
969 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up)  in stmmac_fpe_link_state_handle()  argument
971 	struct stmmac_fpe_cfg *fpe_cfg = &priv->fpe_cfg;  in stmmac_fpe_link_state_handle()
980 		stmmac_fpe_configure(priv, priv->ioaddr, fpe_cfg,  in stmmac_fpe_link_state_handle()
981 				     priv->plat->tx_queues_to_use,  in stmmac_fpe_link_state_handle()
982 				     priv->plat->rx_queues_to_use,  in stmmac_fpe_link_state_handle()
986 		stmmac_fpe_apply(priv);  in stmmac_fpe_link_state_handle()
989 		stmmac_fpe_configure(priv, priv->ioaddr, fpe_cfg,  in stmmac_fpe_link_state_handle()
990 				     priv->plat->tx_queues_to_use,  in stmmac_fpe_link_state_handle()
991 				     priv->plat->rx_queues_to_use,  in stmmac_fpe_link_state_handle()
1001 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_link_down()  local
1003 	stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_mac_link_down()
1004 	priv->eee_active = false;  in stmmac_mac_link_down()
1005 	priv->tx_lpi_enabled = false;  in stmmac_mac_link_down()
1006 	priv->eee_enabled = stmmac_eee_init(priv);  in stmmac_mac_link_down()
1007 	stmmac_set_eee_pls(priv, priv->hw, false);  in stmmac_mac_link_down()
1009 	if (priv->dma_cap.fpesel)  in stmmac_mac_link_down()
1010 		stmmac_fpe_link_state_handle(priv, false);  in stmmac_mac_link_down()
1019 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_link_up()  local
1022 	if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&  in stmmac_mac_link_up()
1023 	    priv->plat->serdes_powerup)  in stmmac_mac_link_up()
1024 		priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv);  in stmmac_mac_link_up()
1026 	old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG);  in stmmac_mac_link_up()
1027 	ctrl = old_ctrl & ~priv->hw->link.speed_mask;  in stmmac_mac_link_up()
1032 			ctrl |= priv->hw->link.xgmii.speed10000;  in stmmac_mac_link_up()
1035 			ctrl |= priv->hw->link.xgmii.speed5000;  in stmmac_mac_link_up()
1038 			ctrl |= priv->hw->link.xgmii.speed2500;  in stmmac_mac_link_up()
1046 			ctrl |= priv->hw->link.xlgmii.speed100000;  in stmmac_mac_link_up()
1049 			ctrl |= priv->hw->link.xlgmii.speed50000;  in stmmac_mac_link_up()
1052 			ctrl |= priv->hw->link.xlgmii.speed40000;  in stmmac_mac_link_up()
1055 			ctrl |= priv->hw->link.xlgmii.speed25000;  in stmmac_mac_link_up()
1058 			ctrl |= priv->hw->link.xgmii.speed10000;  in stmmac_mac_link_up()
1061 			ctrl |= priv->hw->link.speed2500;  in stmmac_mac_link_up()
1064 			ctrl |= priv->hw->link.speed1000;  in stmmac_mac_link_up()
1072 			ctrl |= priv->hw->link.speed2500;  in stmmac_mac_link_up()
1075 			ctrl |= priv->hw->link.speed1000;  in stmmac_mac_link_up()
1078 			ctrl |= priv->hw->link.speed100;  in stmmac_mac_link_up()
1081 			ctrl |= priv->hw->link.speed10;  in stmmac_mac_link_up()
1088 	priv->speed = speed;  in stmmac_mac_link_up()
1090 	if (priv->plat->fix_mac_speed)  in stmmac_mac_link_up()
1091 		priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode);  in stmmac_mac_link_up()
1094 		ctrl &= ~priv->hw->link.duplex;  in stmmac_mac_link_up()
1096 		ctrl |= priv->hw->link.duplex;  in stmmac_mac_link_up()
1100 		priv->flow_ctrl = FLOW_AUTO;  in stmmac_mac_link_up()
1102 		priv->flow_ctrl = FLOW_RX;  in stmmac_mac_link_up()
1104 		priv->flow_ctrl = FLOW_TX;  in stmmac_mac_link_up()
1106 		priv->flow_ctrl = FLOW_OFF;  in stmmac_mac_link_up()
1108 	stmmac_mac_flow_ctrl(priv, duplex);  in stmmac_mac_link_up()
1111 		writel(ctrl, priv->ioaddr + MAC_CTRL_REG);  in stmmac_mac_link_up()
1113 	stmmac_mac_set(priv, priv->ioaddr, true);  in stmmac_mac_link_up()
1114 	if (phy && priv->dma_cap.eee) {  in stmmac_mac_link_up()
1115 		priv->eee_active =  in stmmac_mac_link_up()
1116 			phy_init_eee(phy, !(priv->plat->flags &  in stmmac_mac_link_up()
1118 		priv->eee_enabled = stmmac_eee_init(priv);  in stmmac_mac_link_up()
1119 		priv->tx_lpi_enabled = priv->eee_enabled;  in stmmac_mac_link_up()
1120 		stmmac_set_eee_pls(priv, priv->hw, true);  in stmmac_mac_link_up()
1123 	if (priv->dma_cap.fpesel)  in stmmac_mac_link_up()
1124 		stmmac_fpe_link_state_handle(priv, true);  in stmmac_mac_link_up()
1126 	if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY)  in stmmac_mac_link_up()
1127 		stmmac_hwtstamp_correct_latency(priv, priv);  in stmmac_mac_link_up()
1145 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)  in stmmac_check_pcs_mode()  argument
1147 	int interface = priv->plat->mac_interface;  in stmmac_check_pcs_mode()
1149 	if (priv->dma_cap.pcs) {  in stmmac_check_pcs_mode()
1154 			netdev_dbg(priv->dev, "PCS RGMII support enabled\n");  in stmmac_check_pcs_mode()
1155 			priv->hw->pcs = STMMAC_PCS_RGMII;  in stmmac_check_pcs_mode()
1157 			netdev_dbg(priv->dev, "PCS SGMII support enabled\n");  in stmmac_check_pcs_mode()
1158 			priv->hw->pcs = STMMAC_PCS_SGMII;  in stmmac_check_pcs_mode()
1173 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_init_phy()  local
1178 	if (!phylink_expects_phy(priv->phylink))  in stmmac_init_phy()
1181 	fwnode = priv->plat->port_node;  in stmmac_init_phy()
1183 		fwnode = dev_fwnode(priv->device);  in stmmac_init_phy()
1194 		int addr = priv->plat->phy_addr;  in stmmac_init_phy()
1198 			netdev_err(priv->dev, "no phy found\n");  in stmmac_init_phy()
1202 		phydev = mdiobus_get_phy(priv->mii, addr);  in stmmac_init_phy()
1204 			netdev_err(priv->dev, "no phy at addr %d\n", addr);  in stmmac_init_phy()
1208 		ret = phylink_connect_phy(priv->phylink, phydev);  in stmmac_init_phy()
1211 		ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0);  in stmmac_init_phy()
1214 	if (!priv->plat->pmt) {  in stmmac_init_phy()
1217 		phylink_ethtool_get_wol(priv->phylink, &wol);  in stmmac_init_phy()
1218 		device_set_wakeup_capable(priv->device, !!wol.supported);  in stmmac_init_phy()
1219 		device_set_wakeup_enable(priv->device, !!wol.wolopts);  in stmmac_init_phy()
1225 static int stmmac_phy_setup(struct stmmac_priv *priv)  in stmmac_phy_setup()  argument
1228 	int mode = priv->plat->phy_interface;  in stmmac_phy_setup()
1232 	priv->phylink_config.dev = &priv->dev->dev;  in stmmac_phy_setup()
1233 	priv->phylink_config.type = PHYLINK_NETDEV;  in stmmac_phy_setup()
1234 	priv->phylink_config.mac_managed_pm = true;  in stmmac_phy_setup()
1237 	priv->phylink_config.mac_requires_rxc = true;  in stmmac_phy_setup()
1239 	mdio_bus_data = priv->plat->mdio_bus_data;  in stmmac_phy_setup()
1241 		priv->phylink_config.default_an_inband =  in stmmac_phy_setup()
1247 	__set_bit(mode, priv->phylink_config.supported_interfaces);  in stmmac_phy_setup()
1250 	if (priv->hw->xpcs)  in stmmac_phy_setup()
1251 		xpcs_get_interfaces(priv->hw->xpcs,  in stmmac_phy_setup()
1252 				    priv->phylink_config.supported_interfaces);  in stmmac_phy_setup()
1254 	fwnode = priv->plat->port_node;  in stmmac_phy_setup()
1256 		fwnode = dev_fwnode(priv->device);  in stmmac_phy_setup()
1258 	phylink = phylink_create(&priv->phylink_config, fwnode,  in stmmac_phy_setup()
1263 	priv->phylink = phylink;  in stmmac_phy_setup()
1267 static void stmmac_display_rx_rings(struct stmmac_priv *priv,  in stmmac_display_rx_rings()  argument
1270 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_display_rx_rings()
1281 		if (priv->extend_desc) {  in stmmac_display_rx_rings()
1290 		stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true,  in stmmac_display_rx_rings()
1295 static void stmmac_display_tx_rings(struct stmmac_priv *priv,  in stmmac_display_tx_rings()  argument
1298 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_display_tx_rings()
1309 		if (priv->extend_desc) {  in stmmac_display_tx_rings()
1320 		stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false,  in stmmac_display_tx_rings()
1325 static void stmmac_display_rings(struct stmmac_priv *priv,  in stmmac_display_rings()  argument
1329 	stmmac_display_rx_rings(priv, dma_conf);  in stmmac_display_rings()
1332 	stmmac_display_tx_rings(priv, dma_conf);  in stmmac_display_rings()
1361 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv,  in stmmac_clear_rx_descriptors()  argument
1370 		if (priv->extend_desc)  in stmmac_clear_rx_descriptors()
1371 			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,  in stmmac_clear_rx_descriptors()
1372 					priv->use_riwt, priv->mode,  in stmmac_clear_rx_descriptors()
1376 			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],  in stmmac_clear_rx_descriptors()
1377 					priv->use_riwt, priv->mode,  in stmmac_clear_rx_descriptors()
1390 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv,  in stmmac_clear_tx_descriptors()  argument
1402 		if (priv->extend_desc)  in stmmac_clear_tx_descriptors()
1409 		stmmac_init_tx_desc(priv, p, priv->mode, last);  in stmmac_clear_tx_descriptors()
1420 static void stmmac_clear_descriptors(struct stmmac_priv *priv,  in stmmac_clear_descriptors()  argument
1423 	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;  in stmmac_clear_descriptors()
1424 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;  in stmmac_clear_descriptors()
1429 		stmmac_clear_rx_descriptors(priv, dma_conf, queue);  in stmmac_clear_descriptors()
1433 		stmmac_clear_tx_descriptors(priv, dma_conf, queue);  in stmmac_clear_descriptors()
1447 static int stmmac_init_rx_buffers(struct stmmac_priv *priv,  in stmmac_init_rx_buffers()  argument
1456 	if (priv->dma_cap.host_dma_width <= 32)  in stmmac_init_rx_buffers()
1463 		buf->page_offset = stmmac_rx_offset(priv);  in stmmac_init_rx_buffers()
1466 	if (priv->sph && !buf->sec_page) {  in stmmac_init_rx_buffers()
1472 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);  in stmmac_init_rx_buffers()
1475 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);  in stmmac_init_rx_buffers()
1480 	stmmac_set_desc_addr(priv, p, buf->addr);  in stmmac_init_rx_buffers()
1482 		stmmac_init_desc3(priv, p);  in stmmac_init_rx_buffers()
1493 static void stmmac_free_rx_buffer(struct stmmac_priv *priv,  in stmmac_free_rx_buffer()  argument
1515 static void stmmac_free_tx_buffer(struct stmmac_priv *priv,  in stmmac_free_tx_buffer()  argument
1524 			dma_unmap_page(priv->device,  in stmmac_free_tx_buffer()
1529 			dma_unmap_single(priv->device,  in stmmac_free_tx_buffer()
1561 static void dma_free_rx_skbufs(struct stmmac_priv *priv,  in dma_free_rx_skbufs()  argument
1569 		stmmac_free_rx_buffer(priv, rx_q, i);  in dma_free_rx_skbufs()
1572 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv,  in stmmac_alloc_rx_buffers()  argument
1583 		if (priv->extend_desc)  in stmmac_alloc_rx_buffers()
1588 		ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags,  in stmmac_alloc_rx_buffers()
1605 static void dma_free_rx_xskbufs(struct stmmac_priv *priv,  in dma_free_rx_xskbufs()  argument
1623 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv,  in stmmac_alloc_rx_buffers_zc()  argument
1641 		if (priv->extend_desc)  in stmmac_alloc_rx_buffers_zc()
1653 		stmmac_set_desc_addr(priv, p, dma_addr);  in stmmac_alloc_rx_buffers_zc()
1660 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue)  in stmmac_get_xsk_pool()  argument
1662 	if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps))  in stmmac_get_xsk_pool()
1665 	return xsk_get_pool_from_qid(priv->dev, queue);  in stmmac_get_xsk_pool()
1678 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,  in __init_dma_rx_desc_rings()  argument
1685 	netif_dbg(priv, probe, priv->dev,  in __init_dma_rx_desc_rings()
1689 	stmmac_clear_rx_descriptors(priv, dma_conf, queue);  in __init_dma_rx_desc_rings()
1693 	rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);  in __init_dma_rx_desc_rings()
1699 		netdev_info(priv->dev,  in __init_dma_rx_desc_rings()
1707 		netdev_info(priv->dev,  in __init_dma_rx_desc_rings()
1716 		stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue);  in __init_dma_rx_desc_rings()
1718 		ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags);  in __init_dma_rx_desc_rings()
1724 	if (priv->mode == STMMAC_CHAIN_MODE) {  in __init_dma_rx_desc_rings()
1725 		if (priv->extend_desc)  in __init_dma_rx_desc_rings()
1726 			stmmac_mode_init(priv, rx_q->dma_erx,  in __init_dma_rx_desc_rings()
1730 			stmmac_mode_init(priv, rx_q->dma_rx,  in __init_dma_rx_desc_rings()
1742 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_rx_desc_rings()  local
1743 	u32 rx_count = priv->plat->rx_queues_to_use;  in init_dma_rx_desc_rings()
1748 	netif_dbg(priv, probe, priv->dev,  in init_dma_rx_desc_rings()
1752 		ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags);  in init_dma_rx_desc_rings()
1764 			dma_free_rx_xskbufs(priv, dma_conf, queue);  in init_dma_rx_desc_rings()
1766 			dma_free_rx_skbufs(priv, dma_conf, queue);  in init_dma_rx_desc_rings()
1786 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv,  in __init_dma_tx_desc_rings()  argument
1793 	netif_dbg(priv, probe, priv->dev,  in __init_dma_tx_desc_rings()
1798 	if (priv->mode == STMMAC_CHAIN_MODE) {  in __init_dma_tx_desc_rings()
1799 		if (priv->extend_desc)  in __init_dma_tx_desc_rings()
1800 			stmmac_mode_init(priv, tx_q->dma_etx,  in __init_dma_tx_desc_rings()
1804 			stmmac_mode_init(priv, tx_q->dma_tx,  in __init_dma_tx_desc_rings()
1809 	tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);  in __init_dma_tx_desc_rings()
1814 		if (priv->extend_desc)  in __init_dma_tx_desc_rings()
1821 		stmmac_clear_desc(priv, p);  in __init_dma_tx_desc_rings()
1836 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_tx_desc_rings()  local
1840 	tx_queue_cnt = priv->plat->tx_queues_to_use;  in init_dma_tx_desc_rings()
1843 		__init_dma_tx_desc_rings(priv, dma_conf, queue);  in init_dma_tx_desc_rings()
1861 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_desc_rings()  local
1870 	stmmac_clear_descriptors(priv, dma_conf);  in init_dma_desc_rings()
1872 	if (netif_msg_hw(priv))  in init_dma_desc_rings()
1873 		stmmac_display_rings(priv, dma_conf);  in init_dma_desc_rings()
1884 static void dma_free_tx_skbufs(struct stmmac_priv *priv,  in dma_free_tx_skbufs()  argument
1894 		stmmac_free_tx_buffer(priv, dma_conf, queue, i);  in dma_free_tx_skbufs()
1907 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)  in stmmac_free_tx_skbufs()  argument
1909 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;  in stmmac_free_tx_skbufs()
1913 		dma_free_tx_skbufs(priv, &priv->dma_conf, queue);  in stmmac_free_tx_skbufs()
1922 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,  in __free_dma_rx_desc_resources()  argument
1930 		dma_free_rx_xskbufs(priv, dma_conf, queue);  in __free_dma_rx_desc_resources()
1932 		dma_free_rx_skbufs(priv, dma_conf, queue);  in __free_dma_rx_desc_resources()
1938 	if (!priv->extend_desc)  in __free_dma_rx_desc_resources()
1939 		dma_free_coherent(priv->device, dma_conf->dma_rx_size *  in __free_dma_rx_desc_resources()
1943 		dma_free_coherent(priv->device, dma_conf->dma_rx_size *  in __free_dma_rx_desc_resources()
1955 static void free_dma_rx_desc_resources(struct stmmac_priv *priv,  in free_dma_rx_desc_resources()  argument
1958 	u32 rx_count = priv->plat->rx_queues_to_use;  in free_dma_rx_desc_resources()
1963 		__free_dma_rx_desc_resources(priv, dma_conf, queue);  in free_dma_rx_desc_resources()
1972 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,  in __free_dma_tx_desc_resources()  argument
1981 	dma_free_tx_skbufs(priv, dma_conf, queue);  in __free_dma_tx_desc_resources()
1983 	if (priv->extend_desc) {  in __free_dma_tx_desc_resources()
1996 	dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);  in __free_dma_tx_desc_resources()
2002 static void free_dma_tx_desc_resources(struct stmmac_priv *priv,  in free_dma_tx_desc_resources()  argument
2005 	u32 tx_count = priv->plat->tx_queues_to_use;  in free_dma_tx_desc_resources()
2010 		__free_dma_tx_desc_resources(priv, dma_conf, queue);  in free_dma_tx_desc_resources()
2023 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,  in __alloc_dma_rx_desc_resources()  argument
2028 	struct stmmac_channel *ch = &priv->channel[queue];  in __alloc_dma_rx_desc_resources()
2029 	bool xdp_prog = stmmac_xdp_is_enabled(priv);  in __alloc_dma_rx_desc_resources()
2036 	rx_q->priv_data = priv;  in __alloc_dma_rx_desc_resources()
2042 	pp_params.nid = dev_to_node(priv->device);  in __alloc_dma_rx_desc_resources()
2043 	pp_params.dev = priv->device;  in __alloc_dma_rx_desc_resources()
2045 	pp_params.offset = stmmac_rx_offset(priv);  in __alloc_dma_rx_desc_resources()
2061 	if (priv->extend_desc) {  in __alloc_dma_rx_desc_resources()
2062 		rx_q->dma_erx = dma_alloc_coherent(priv->device,  in __alloc_dma_rx_desc_resources()
2071 		rx_q->dma_rx = dma_alloc_coherent(priv->device,  in __alloc_dma_rx_desc_resources()
2080 	if (stmmac_xdp_is_enabled(priv) &&  in __alloc_dma_rx_desc_resources()
2081 	    test_bit(queue, priv->af_xdp_zc_qps))  in __alloc_dma_rx_desc_resources()
2086 	ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev,  in __alloc_dma_rx_desc_resources()
2090 		netdev_err(priv->dev, "Failed to register xdp rxq info\n");  in __alloc_dma_rx_desc_resources()
2097 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv,  in alloc_dma_rx_desc_resources()  argument
2100 	u32 rx_count = priv->plat->rx_queues_to_use;  in alloc_dma_rx_desc_resources()
2106 		ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue);  in alloc_dma_rx_desc_resources()
2114 	free_dma_rx_desc_resources(priv, dma_conf);  in alloc_dma_rx_desc_resources()
2129 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv,  in __alloc_dma_tx_desc_resources()  argument
2138 	tx_q->priv_data = priv;  in __alloc_dma_tx_desc_resources()
2152 	if (priv->extend_desc)  in __alloc_dma_tx_desc_resources()
2161 	addr = dma_alloc_coherent(priv->device, size,  in __alloc_dma_tx_desc_resources()
2166 	if (priv->extend_desc)  in __alloc_dma_tx_desc_resources()
2176 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv,  in alloc_dma_tx_desc_resources()  argument
2179 	u32 tx_count = priv->plat->tx_queues_to_use;  in alloc_dma_tx_desc_resources()
2185 		ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue);  in alloc_dma_tx_desc_resources()
2193 	free_dma_tx_desc_resources(priv, dma_conf);  in alloc_dma_tx_desc_resources()
2206 static int alloc_dma_desc_resources(struct stmmac_priv *priv,  in alloc_dma_desc_resources()  argument
2210 	int ret = alloc_dma_rx_desc_resources(priv, dma_conf);  in alloc_dma_desc_resources()
2215 	ret = alloc_dma_tx_desc_resources(priv, dma_conf);  in alloc_dma_desc_resources()
2225 static void free_dma_desc_resources(struct stmmac_priv *priv,  in free_dma_desc_resources()  argument
2229 	free_dma_tx_desc_resources(priv, dma_conf);  in free_dma_desc_resources()
2234 	free_dma_rx_desc_resources(priv, dma_conf);  in free_dma_desc_resources()
2242 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)  in stmmac_mac_enable_rx_queues()  argument
2244 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_mac_enable_rx_queues()
2249 		mode = priv->plat->rx_queues_cfg[queue].mode_to_use;  in stmmac_mac_enable_rx_queues()
2250 		stmmac_rx_queue_enable(priv, priv->hw, mode, queue);  in stmmac_mac_enable_rx_queues()
2261 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_start_rx_dma()  argument
2263 	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);  in stmmac_start_rx_dma()
2264 	stmmac_start_rx(priv, priv->ioaddr, chan);  in stmmac_start_rx_dma()
2274 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_start_tx_dma()  argument
2276 	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);  in stmmac_start_tx_dma()
2277 	stmmac_start_tx(priv, priv->ioaddr, chan);  in stmmac_start_tx_dma()
2287 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_stop_rx_dma()  argument
2289 	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);  in stmmac_stop_rx_dma()
2290 	stmmac_stop_rx(priv, priv->ioaddr, chan);  in stmmac_stop_rx_dma()
2300 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_stop_tx_dma()  argument
2302 	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);  in stmmac_stop_tx_dma()
2303 	stmmac_stop_tx(priv, priv->ioaddr, chan);  in stmmac_stop_tx_dma()
2306 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)  in stmmac_enable_all_dma_irq()  argument
2308 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_enable_all_dma_irq()
2309 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_enable_all_dma_irq()
2314 		struct stmmac_channel *ch = &priv->channel[chan];  in stmmac_enable_all_dma_irq()
2318 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);  in stmmac_enable_all_dma_irq()
2329 static void stmmac_start_all_dma(struct stmmac_priv *priv)  in stmmac_start_all_dma()  argument
2331 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_start_all_dma()
2332 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_start_all_dma()
2336 		stmmac_start_rx_dma(priv, chan);  in stmmac_start_all_dma()
2339 		stmmac_start_tx_dma(priv, chan);  in stmmac_start_all_dma()
2348 static void stmmac_stop_all_dma(struct stmmac_priv *priv)  in stmmac_stop_all_dma()  argument
2350 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_stop_all_dma()
2351 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_stop_all_dma()
2355 		stmmac_stop_rx_dma(priv, chan);  in stmmac_stop_all_dma()
2358 		stmmac_stop_tx_dma(priv, chan);  in stmmac_stop_all_dma()
2367 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)  in stmmac_dma_operation_mode()  argument
2369 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_dma_operation_mode()
2370 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_dma_operation_mode()
2371 	int rxfifosz = priv->plat->rx_fifo_size;  in stmmac_dma_operation_mode()
2372 	int txfifosz = priv->plat->tx_fifo_size;  in stmmac_dma_operation_mode()
2379 		rxfifosz = priv->dma_cap.rx_fifo_size;  in stmmac_dma_operation_mode()
2381 		txfifosz = priv->dma_cap.tx_fifo_size;  in stmmac_dma_operation_mode()
2384 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac) {  in stmmac_dma_operation_mode()
2389 	if (priv->plat->force_thresh_dma_mode) {  in stmmac_dma_operation_mode()
2392 	} else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {  in stmmac_dma_operation_mode()
2402 		priv->xstats.threshold = SF_DMA_MODE;  in stmmac_dma_operation_mode()
2410 		struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];  in stmmac_dma_operation_mode()
2413 		qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;  in stmmac_dma_operation_mode()
2415 		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,  in stmmac_dma_operation_mode()
2420 			stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_dma_operation_mode()
2424 			stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_dma_operation_mode()
2425 					      priv->dma_conf.dma_buf_sz,  in stmmac_dma_operation_mode()
2431 		qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;  in stmmac_dma_operation_mode()
2433 		stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,  in stmmac_dma_operation_mode()
2442 	stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc);  in stmmac_xsk_request_timestamp()
2449 	struct stmmac_priv *priv = tx_compl->priv;  in stmmac_xsk_fill_timestamp()  local
2454 	if (!priv->hwts_tx_en)  in stmmac_xsk_fill_timestamp()
2458 	if (stmmac_get_tx_timestamp_status(priv, desc)) {  in stmmac_xsk_fill_timestamp()
2459 		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);  in stmmac_xsk_fill_timestamp()
2461 	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {  in stmmac_xsk_fill_timestamp()
2466 		ns -= priv->plat->cdc_error_adj;  in stmmac_xsk_fill_timestamp()
2478 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)  in stmmac_xdp_xmit_zc()  argument
2480 	struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue);  in stmmac_xdp_xmit_zc()
2481 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_xdp_xmit_zc()
2482 	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_xdp_xmit_zc()
2493 	budget = min(budget, stmmac_tx_avail(priv, queue));  in stmmac_xdp_xmit_zc()
2504 		if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) ||  in stmmac_xdp_xmit_zc()
2505 		    !netif_carrier_ok(priv->dev)) {  in stmmac_xdp_xmit_zc()
2513 		if (priv->est && priv->est->enable &&  in stmmac_xdp_xmit_zc()
2514 		    priv->est->max_sdu[queue] &&  in stmmac_xdp_xmit_zc()
2515 		    xdp_desc.len > priv->est->max_sdu[queue]) {  in stmmac_xdp_xmit_zc()
2516 			priv->xstats.max_sdu_txq_drop[queue]++;  in stmmac_xdp_xmit_zc()
2520 		if (likely(priv->extend_desc))  in stmmac_xdp_xmit_zc()
2545 		stmmac_set_desc_addr(priv, tx_desc, dma_addr);  in stmmac_xdp_xmit_zc()
2549 		if (!priv->tx_coal_frames[queue])  in stmmac_xdp_xmit_zc()
2551 		else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)  in stmmac_xdp_xmit_zc()
2556 		meta_req.priv = priv;  in stmmac_xdp_xmit_zc()
2563 			stmmac_set_tx_ic(priv, tx_desc);  in stmmac_xdp_xmit_zc()
2567 		stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len,  in stmmac_xdp_xmit_zc()
2568 				       true, priv->mode, true, true,  in stmmac_xdp_xmit_zc()
2571 		stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);  in stmmac_xdp_xmit_zc()
2576 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);  in stmmac_xdp_xmit_zc()
2584 		stmmac_flush_tx_descriptors(priv, queue);  in stmmac_xdp_xmit_zc()
2596 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan)  in stmmac_bump_dma_threshold()  argument
2598 	if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) {  in stmmac_bump_dma_threshold()
2601 		if (priv->plat->force_thresh_dma_mode)  in stmmac_bump_dma_threshold()
2602 			stmmac_set_dma_operation_mode(priv, tc, tc, chan);  in stmmac_bump_dma_threshold()
2604 			stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE,  in stmmac_bump_dma_threshold()
2607 		priv->xstats.threshold = tc;  in stmmac_bump_dma_threshold()
2621 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,  in stmmac_tx_clean()  argument
2624 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tx_clean()
2625 	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_tx_clean()
2630 	__netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));  in stmmac_tx_clean()
2637 	while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) {  in stmmac_tx_clean()
2655 		if (priv->extend_desc)  in stmmac_tx_clean()
2662 		status = stmmac_tx_status(priv,	&priv->xstats, p, priv->ioaddr);  in stmmac_tx_clean()
2680 					stmmac_bump_dma_threshold(priv, queue);  in stmmac_tx_clean()
2685 				stmmac_get_tx_hwtstamp(priv, p, skb);  in stmmac_tx_clean()
2689 					.priv = priv,  in stmmac_tx_clean()
2702 				dma_unmap_page(priv->device,  in stmmac_tx_clean()
2707 				dma_unmap_single(priv->device,  in stmmac_tx_clean()
2716 		stmmac_clean_desc3(priv, tx_q, p);  in stmmac_tx_clean()
2745 		stmmac_release_tx_desc(priv, p, priv->mode);  in stmmac_tx_clean()
2747 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);  in stmmac_tx_clean()
2751 	netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),  in stmmac_tx_clean()
2754 	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,  in stmmac_tx_clean()
2756 	    stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) {  in stmmac_tx_clean()
2758 		netif_dbg(priv, tx_done, priv->dev,  in stmmac_tx_clean()
2760 		netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));  in stmmac_tx_clean()
2777 		work_done = stmmac_xdp_xmit_zc(priv, queue,  in stmmac_tx_clean()
2785 	if (priv->eee_enabled && !priv->tx_path_in_lpi_mode &&  in stmmac_tx_clean()
2786 	    priv->eee_sw_timer_en) {  in stmmac_tx_clean()
2787 		if (stmmac_enable_eee_mode(priv))  in stmmac_tx_clean()
2788 			mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));  in stmmac_tx_clean()
2801 	priv->xstats.tx_errors += tx_errors;  in stmmac_tx_clean()
2803 	__netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));  in stmmac_tx_clean()
2816 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)  in stmmac_tx_err()  argument
2818 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_tx_err()
2820 	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));  in stmmac_tx_err()
2822 	stmmac_stop_tx_dma(priv, chan);  in stmmac_tx_err()
2823 	dma_free_tx_skbufs(priv, &priv->dma_conf, chan);  in stmmac_tx_err()
2824 	stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan);  in stmmac_tx_err()
2825 	stmmac_reset_tx_queue(priv, chan);  in stmmac_tx_err()
2826 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_tx_err()
2828 	stmmac_start_tx_dma(priv, chan);  in stmmac_tx_err()
2830 	priv->xstats.tx_errors++;  in stmmac_tx_err()
2831 	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));  in stmmac_tx_err()
2844 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,  in stmmac_set_dma_operation_mode()  argument
2847 	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;  in stmmac_set_dma_operation_mode()
2848 	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;  in stmmac_set_dma_operation_mode()
2849 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_set_dma_operation_mode()
2850 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_set_dma_operation_mode()
2851 	int rxfifosz = priv->plat->rx_fifo_size;  in stmmac_set_dma_operation_mode()
2852 	int txfifosz = priv->plat->tx_fifo_size;  in stmmac_set_dma_operation_mode()
2855 		rxfifosz = priv->dma_cap.rx_fifo_size;  in stmmac_set_dma_operation_mode()
2857 		txfifosz = priv->dma_cap.tx_fifo_size;  in stmmac_set_dma_operation_mode()
2863 	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);  in stmmac_set_dma_operation_mode()
2864 	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);  in stmmac_set_dma_operation_mode()
2867 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)  in stmmac_safety_feat_interrupt()  argument
2871 	ret = stmmac_safety_feat_irq_status(priv, priv->dev,  in stmmac_safety_feat_interrupt()
2872 			priv->ioaddr, priv->dma_cap.asp, &priv->sstats);  in stmmac_safety_feat_interrupt()
2874 		stmmac_global_err(priv);  in stmmac_safety_feat_interrupt()
2881 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir)  in stmmac_napi_check()  argument
2883 	int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,  in stmmac_napi_check()
2884 						 &priv->xstats, chan, dir);  in stmmac_napi_check()
2885 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];  in stmmac_napi_check()
2886 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_napi_check()
2887 	struct stmmac_channel *ch = &priv->channel[chan];  in stmmac_napi_check()
2895 	if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {  in stmmac_napi_check()
2898 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);  in stmmac_napi_check()
2904 	if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {  in stmmac_napi_check()
2907 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);  in stmmac_napi_check()
2923 static void stmmac_dma_interrupt(struct stmmac_priv *priv)  in stmmac_dma_interrupt()  argument
2925 	u32 tx_channel_count = priv->plat->tx_queues_to_use;  in stmmac_dma_interrupt()
2926 	u32 rx_channel_count = priv->plat->rx_queues_to_use;  in stmmac_dma_interrupt()
2937 		status[chan] = stmmac_napi_check(priv, chan,  in stmmac_dma_interrupt()
2943 			stmmac_bump_dma_threshold(priv, chan);  in stmmac_dma_interrupt()
2945 			stmmac_tx_err(priv, chan);  in stmmac_dma_interrupt()
2955 static void stmmac_mmc_setup(struct stmmac_priv *priv)  in stmmac_mmc_setup()  argument
2960 	stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);  in stmmac_mmc_setup()
2962 	if (priv->dma_cap.rmon) {  in stmmac_mmc_setup()
2963 		stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);  in stmmac_mmc_setup()
2964 		memset(&priv->mmc, 0, sizeof(struct stmmac_counters));  in stmmac_mmc_setup()
2966 		netdev_info(priv->dev, "No MAC Management Counters available\n");  in stmmac_mmc_setup()
2978 static int stmmac_get_hw_features(struct stmmac_priv *priv)  in stmmac_get_hw_features()  argument
2980 	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;  in stmmac_get_hw_features()
2990 static void stmmac_check_ether_addr(struct stmmac_priv *priv)  in stmmac_check_ether_addr()  argument
2994 	if (!is_valid_ether_addr(priv->dev->dev_addr)) {  in stmmac_check_ether_addr()
2995 		stmmac_get_umac_addr(priv, priv->hw, addr, 0);  in stmmac_check_ether_addr()
2997 			eth_hw_addr_set(priv->dev, addr);  in stmmac_check_ether_addr()
2999 			eth_hw_addr_random(priv->dev);  in stmmac_check_ether_addr()
3000 		dev_info(priv->device, "device MAC address %pM\n",  in stmmac_check_ether_addr()
3001 			 priv->dev->dev_addr);  in stmmac_check_ether_addr()
3013 static int stmmac_init_dma_engine(struct stmmac_priv *priv)  in stmmac_init_dma_engine()  argument
3015 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_init_dma_engine()
3016 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_init_dma_engine()
3023 	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {  in stmmac_init_dma_engine()
3024 		dev_err(priv->device, "Invalid DMA configuration\n");  in stmmac_init_dma_engine()
3028 	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))  in stmmac_init_dma_engine()
3029 		priv->plat->dma_cfg->atds = 1;  in stmmac_init_dma_engine()
3031 	ret = stmmac_reset(priv, priv->ioaddr);  in stmmac_init_dma_engine()
3033 		dev_err(priv->device, "Failed to reset the dma\n");  in stmmac_init_dma_engine()
3038 	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg);  in stmmac_init_dma_engine()
3040 	if (priv->plat->axi)  in stmmac_init_dma_engine()
3041 		stmmac_axi(priv, priv->ioaddr, priv->plat->axi);  in stmmac_init_dma_engine()
3045 		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);  in stmmac_init_dma_engine()
3046 		stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);  in stmmac_init_dma_engine()
3051 		rx_q = &priv->dma_conf.rx_queue[chan];  in stmmac_init_dma_engine()
3053 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_init_dma_engine()
3059 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,  in stmmac_init_dma_engine()
3065 		tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_init_dma_engine()
3067 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_init_dma_engine()
3071 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,  in stmmac_init_dma_engine()
3078 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)  in stmmac_tx_timer_arm()  argument
3080 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tx_timer_arm()
3081 	u32 tx_coal_timer = priv->tx_coal_timer[queue];  in stmmac_tx_timer_arm()
3088 	ch = &priv->channel[tx_q->queue_index];  in stmmac_tx_timer_arm()
3112 	struct stmmac_priv *priv = tx_q->priv_data;  in stmmac_tx_timer()  local
3116 	ch = &priv->channel[tx_q->queue_index];  in stmmac_tx_timer()
3123 		stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);  in stmmac_tx_timer()
3139 static void stmmac_init_coalesce(struct stmmac_priv *priv)  in stmmac_init_coalesce()  argument
3141 	u32 tx_channel_count = priv->plat->tx_queues_to_use;  in stmmac_init_coalesce()
3142 	u32 rx_channel_count = priv->plat->rx_queues_to_use;  in stmmac_init_coalesce()
3146 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_init_coalesce()
3148 		priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES;  in stmmac_init_coalesce()
3149 		priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER;  in stmmac_init_coalesce()
3156 		priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES;  in stmmac_init_coalesce()
3159 static void stmmac_set_rings_length(struct stmmac_priv *priv)  in stmmac_set_rings_length()  argument
3161 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_set_rings_length()
3162 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_set_rings_length()
3167 		stmmac_set_tx_ring_len(priv, priv->ioaddr,  in stmmac_set_rings_length()
3168 				       (priv->dma_conf.dma_tx_size - 1), chan);  in stmmac_set_rings_length()
3172 		stmmac_set_rx_ring_len(priv, priv->ioaddr,  in stmmac_set_rings_length()
3173 				       (priv->dma_conf.dma_rx_size - 1), chan);  in stmmac_set_rings_length()
3181 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)  in stmmac_set_tx_queue_weight()  argument
3183 	u32 tx_queues_count = priv->plat->tx_queues_to_use;  in stmmac_set_tx_queue_weight()
3188 		weight = priv->plat->tx_queues_cfg[queue].weight;  in stmmac_set_tx_queue_weight()
3189 		stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);  in stmmac_set_tx_queue_weight()
3198 static void stmmac_configure_cbs(struct stmmac_priv *priv)  in stmmac_configure_cbs()  argument
3200 	u32 tx_queues_count = priv->plat->tx_queues_to_use;  in stmmac_configure_cbs()
3206 		mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;  in stmmac_configure_cbs()
3210 		stmmac_config_cbs(priv, priv->hw,  in stmmac_configure_cbs()
3211 				priv->plat->tx_queues_cfg[queue].send_slope,  in stmmac_configure_cbs()
3212 				priv->plat->tx_queues_cfg[queue].idle_slope,  in stmmac_configure_cbs()
3213 				priv->plat->tx_queues_cfg[queue].high_credit,  in stmmac_configure_cbs()
3214 				priv->plat->tx_queues_cfg[queue].low_credit,  in stmmac_configure_cbs()
3224 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)  in stmmac_rx_queue_dma_chan_map()  argument
3226 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_rx_queue_dma_chan_map()
3231 		chan = priv->plat->rx_queues_cfg[queue].chan;  in stmmac_rx_queue_dma_chan_map()
3232 		stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);  in stmmac_rx_queue_dma_chan_map()
3241 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)  in stmmac_mac_config_rx_queues_prio()  argument
3243 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_mac_config_rx_queues_prio()
3248 		if (!priv->plat->rx_queues_cfg[queue].use_prio)  in stmmac_mac_config_rx_queues_prio()
3251 		prio = priv->plat->rx_queues_cfg[queue].prio;  in stmmac_mac_config_rx_queues_prio()
3252 		stmmac_rx_queue_prio(priv, priv->hw, prio, queue);  in stmmac_mac_config_rx_queues_prio()
3261 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)  in stmmac_mac_config_tx_queues_prio()  argument
3263 	u32 tx_queues_count = priv->plat->tx_queues_to_use;  in stmmac_mac_config_tx_queues_prio()
3268 		if (!priv->plat->tx_queues_cfg[queue].use_prio)  in stmmac_mac_config_tx_queues_prio()
3271 		prio = priv->plat->tx_queues_cfg[queue].prio;  in stmmac_mac_config_tx_queues_prio()
3272 		stmmac_tx_queue_prio(priv, priv->hw, prio, queue);  in stmmac_mac_config_tx_queues_prio()
3281 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)  in stmmac_mac_config_rx_queues_routing()  argument
3283 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_mac_config_rx_queues_routing()
3289 		if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)  in stmmac_mac_config_rx_queues_routing()
3292 		packet = priv->plat->rx_queues_cfg[queue].pkt_route;  in stmmac_mac_config_rx_queues_routing()
3293 		stmmac_rx_queue_routing(priv, priv->hw, packet, queue);  in stmmac_mac_config_rx_queues_routing()
3297 static void stmmac_mac_config_rss(struct stmmac_priv *priv)  in stmmac_mac_config_rss()  argument
3299 	if (!priv->dma_cap.rssen || !priv->plat->rss_en) {  in stmmac_mac_config_rss()
3300 		priv->rss.enable = false;  in stmmac_mac_config_rss()
3304 	if (priv->dev->features & NETIF_F_RXHASH)  in stmmac_mac_config_rss()
3305 		priv->rss.enable = true;  in stmmac_mac_config_rss()
3307 		priv->rss.enable = false;  in stmmac_mac_config_rss()
3309 	stmmac_rss_configure(priv, priv->hw, &priv->rss,  in stmmac_mac_config_rss()
3310 			     priv->plat->rx_queues_to_use);  in stmmac_mac_config_rss()
3318 static void stmmac_mtl_configuration(struct stmmac_priv *priv)  in stmmac_mtl_configuration()  argument
3320 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_mtl_configuration()
3321 	u32 tx_queues_count = priv->plat->tx_queues_to_use;  in stmmac_mtl_configuration()
3324 		stmmac_set_tx_queue_weight(priv);  in stmmac_mtl_configuration()
3328 		stmmac_prog_mtl_rx_algorithms(priv, priv->hw,  in stmmac_mtl_configuration()
3329 				priv->plat->rx_sched_algorithm);  in stmmac_mtl_configuration()
3333 		stmmac_prog_mtl_tx_algorithms(priv, priv->hw,  in stmmac_mtl_configuration()
3334 				priv->plat->tx_sched_algorithm);  in stmmac_mtl_configuration()
3338 		stmmac_configure_cbs(priv);  in stmmac_mtl_configuration()
3341 	stmmac_rx_queue_dma_chan_map(priv);  in stmmac_mtl_configuration()
3344 	stmmac_mac_enable_rx_queues(priv);  in stmmac_mtl_configuration()
3348 		stmmac_mac_config_rx_queues_prio(priv);  in stmmac_mtl_configuration()
3352 		stmmac_mac_config_tx_queues_prio(priv);  in stmmac_mtl_configuration()
3356 		stmmac_mac_config_rx_queues_routing(priv);  in stmmac_mtl_configuration()
3360 		stmmac_mac_config_rss(priv);  in stmmac_mtl_configuration()
3363 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)  in stmmac_safety_feat_configuration()  argument
3365 	if (priv->dma_cap.asp) {  in stmmac_safety_feat_configuration()
3366 		netdev_info(priv->dev, "Enabling Safety Features\n");  in stmmac_safety_feat_configuration()
3367 		stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp,  in stmmac_safety_feat_configuration()
3368 					  priv->plat->safety_feat_cfg);  in stmmac_safety_feat_configuration()
3370 		netdev_info(priv->dev, "No Safety Features support found\n");  in stmmac_safety_feat_configuration()
3389 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hw_setup()  local
3390 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_hw_setup()
3391 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_hw_setup()
3397 	if (priv->hw->phylink_pcs)  in stmmac_hw_setup()
3398 		phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs);  in stmmac_hw_setup()
3401 	ret = stmmac_init_dma_engine(priv);  in stmmac_hw_setup()
3403 		netdev_err(priv->dev, "%s: DMA engine initialization failed\n",  in stmmac_hw_setup()
3409 	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);  in stmmac_hw_setup()
3412 	if (priv->hw->pcs) {  in stmmac_hw_setup()
3413 		int speed = priv->plat->mac_port_sel_speed;  in stmmac_hw_setup()
3417 			priv->hw->ps = speed;  in stmmac_hw_setup()
3419 			dev_warn(priv->device, "invalid port speed\n");  in stmmac_hw_setup()
3420 			priv->hw->ps = 0;  in stmmac_hw_setup()
3425 	stmmac_core_init(priv, priv->hw, dev);  in stmmac_hw_setup()
3428 	stmmac_mtl_configuration(priv);  in stmmac_hw_setup()
3431 	stmmac_safety_feat_configuration(priv);  in stmmac_hw_setup()
3433 	ret = stmmac_rx_ipc(priv, priv->hw);  in stmmac_hw_setup()
3435 		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");  in stmmac_hw_setup()
3436 		priv->plat->rx_coe = STMMAC_RX_COE_NONE;  in stmmac_hw_setup()
3437 		priv->hw->rx_csum = 0;  in stmmac_hw_setup()
3441 	stmmac_mac_set(priv, priv->ioaddr, true);  in stmmac_hw_setup()
3444 	stmmac_dma_operation_mode(priv);  in stmmac_hw_setup()
3446 	stmmac_mmc_setup(priv);  in stmmac_hw_setup()
3449 		ret = clk_prepare_enable(priv->plat->clk_ptp_ref);  in stmmac_hw_setup()
3451 			netdev_warn(priv->dev,  in stmmac_hw_setup()
3456 	ret = stmmac_init_ptp(priv);  in stmmac_hw_setup()
3458 		netdev_info(priv->dev, "PTP not supported by HW\n");  in stmmac_hw_setup()
3460 		netdev_warn(priv->dev, "PTP init failed\n");  in stmmac_hw_setup()
3462 		stmmac_ptp_register(priv);  in stmmac_hw_setup()
3464 	priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS;  in stmmac_hw_setup()
3467 	if (!priv->tx_lpi_timer)  in stmmac_hw_setup()
3468 		priv->tx_lpi_timer = eee_timer * 1000;  in stmmac_hw_setup()
3470 	if (priv->use_riwt) {  in stmmac_hw_setup()
3474 			if (!priv->rx_riwt[queue])  in stmmac_hw_setup()
3475 				priv->rx_riwt[queue] = DEF_DMA_RIWT;  in stmmac_hw_setup()
3477 			stmmac_rx_watchdog(priv, priv->ioaddr,  in stmmac_hw_setup()
3478 					   priv->rx_riwt[queue], queue);  in stmmac_hw_setup()
3482 	if (priv->hw->pcs)  in stmmac_hw_setup()
3483 		stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);  in stmmac_hw_setup()
3486 	stmmac_set_rings_length(priv);  in stmmac_hw_setup()
3489 	if (priv->tso) {  in stmmac_hw_setup()
3491 			struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_hw_setup()
3497 			stmmac_enable_tso(priv, priv->ioaddr, 1, chan);  in stmmac_hw_setup()
3502 	sph_en = (priv->hw->rx_csum > 0) && priv->sph;  in stmmac_hw_setup()
3504 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);  in stmmac_hw_setup()
3508 	if (priv->dma_cap.vlins)  in stmmac_hw_setup()
3509 		stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);  in stmmac_hw_setup()
3513 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_hw_setup()
3516 		stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);  in stmmac_hw_setup()
3520 	netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use);  in stmmac_hw_setup()
3521 	netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use);  in stmmac_hw_setup()
3524 	stmmac_start_all_dma(priv);  in stmmac_hw_setup()
3526 	stmmac_set_hw_vlan_mode(priv, priv->hw);  in stmmac_hw_setup()
3533 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hw_teardown()  local
3535 	clk_disable_unprepare(priv->plat->clk_ptp_ref);  in stmmac_hw_teardown()
3541 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_free_irq()  local
3546 		irq_idx = priv->plat->tx_queues_to_use;  in stmmac_free_irq()
3550 			if (priv->tx_irq[j] > 0) {  in stmmac_free_irq()
3551 				irq_set_affinity_hint(priv->tx_irq[j], NULL);  in stmmac_free_irq()
3552 				free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]);  in stmmac_free_irq()
3555 		irq_idx = priv->plat->rx_queues_to_use;  in stmmac_free_irq()
3559 			if (priv->rx_irq[j] > 0) {  in stmmac_free_irq()
3560 				irq_set_affinity_hint(priv->rx_irq[j], NULL);  in stmmac_free_irq()
3561 				free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]);  in stmmac_free_irq()
3565 		if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq)  in stmmac_free_irq()
3566 			free_irq(priv->sfty_ue_irq, dev);  in stmmac_free_irq()
3569 		if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq)  in stmmac_free_irq()
3570 			free_irq(priv->sfty_ce_irq, dev);  in stmmac_free_irq()
3573 		if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq)  in stmmac_free_irq()
3574 			free_irq(priv->lpi_irq, dev);  in stmmac_free_irq()
3577 		if (priv->wol_irq > 0 && priv->wol_irq != dev->irq)  in stmmac_free_irq()
3578 			free_irq(priv->wol_irq, dev);  in stmmac_free_irq()
3581 		if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq)  in stmmac_free_irq()
3582 			free_irq(priv->sfty_irq, dev);  in stmmac_free_irq()
3596 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq_multi_msi()  local
3605 	int_name = priv->int_name_mac;  in stmmac_request_irq_multi_msi()
3610 		netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3620 	priv->wol_irq_disabled = true;  in stmmac_request_irq_multi_msi()
3621 	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3622 		int_name = priv->int_name_wol;  in stmmac_request_irq_multi_msi()
3624 		ret = request_irq(priv->wol_irq,  in stmmac_request_irq_multi_msi()
3628 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3630 				   __func__, priv->wol_irq, ret);  in stmmac_request_irq_multi_msi()
3639 	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3640 		int_name = priv->int_name_lpi;  in stmmac_request_irq_multi_msi()
3642 		ret = request_irq(priv->lpi_irq,  in stmmac_request_irq_multi_msi()
3646 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3648 				   __func__, priv->lpi_irq, ret);  in stmmac_request_irq_multi_msi()
3657 	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3658 		int_name = priv->int_name_sfty;  in stmmac_request_irq_multi_msi()
3660 		ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt,  in stmmac_request_irq_multi_msi()
3663 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3665 				   __func__, priv->sfty_irq, ret);  in stmmac_request_irq_multi_msi()
3674 	if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3675 		int_name = priv->int_name_sfty_ce;  in stmmac_request_irq_multi_msi()
3677 		ret = request_irq(priv->sfty_ce_irq,  in stmmac_request_irq_multi_msi()
3681 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3683 				   __func__, priv->sfty_ce_irq, ret);  in stmmac_request_irq_multi_msi()
3692 	if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3693 		int_name = priv->int_name_sfty_ue;  in stmmac_request_irq_multi_msi()
3695 		ret = request_irq(priv->sfty_ue_irq,  in stmmac_request_irq_multi_msi()
3699 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3701 				   __func__, priv->sfty_ue_irq, ret);  in stmmac_request_irq_multi_msi()
3708 	for (i = 0; i < priv->plat->rx_queues_to_use; i++) {  in stmmac_request_irq_multi_msi()
3711 		if (priv->rx_irq[i] == 0)  in stmmac_request_irq_multi_msi()
3714 		int_name = priv->int_name_rx_irq[i];  in stmmac_request_irq_multi_msi()
3716 		ret = request_irq(priv->rx_irq[i],  in stmmac_request_irq_multi_msi()
3718 				  0, int_name, &priv->dma_conf.rx_queue[i]);  in stmmac_request_irq_multi_msi()
3720 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3722 				   __func__, i, priv->rx_irq[i], ret);  in stmmac_request_irq_multi_msi()
3729 		irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask);  in stmmac_request_irq_multi_msi()
3733 	for (i = 0; i < priv->plat->tx_queues_to_use; i++) {  in stmmac_request_irq_multi_msi()
3736 		if (priv->tx_irq[i] == 0)  in stmmac_request_irq_multi_msi()
3739 		int_name = priv->int_name_tx_irq[i];  in stmmac_request_irq_multi_msi()
3741 		ret = request_irq(priv->tx_irq[i],  in stmmac_request_irq_multi_msi()
3743 				  0, int_name, &priv->dma_conf.tx_queue[i]);  in stmmac_request_irq_multi_msi()
3745 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3747 				   __func__, i, priv->tx_irq[i], ret);  in stmmac_request_irq_multi_msi()
3754 		irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask);  in stmmac_request_irq_multi_msi()
3766 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq_single()  local
3773 		netdev_err(priv->dev,  in stmmac_request_irq_single()
3783 	priv->wol_irq_disabled = true;  in stmmac_request_irq_single()
3784 	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {  in stmmac_request_irq_single()
3785 		ret = request_irq(priv->wol_irq, stmmac_interrupt,  in stmmac_request_irq_single()
3788 			netdev_err(priv->dev,  in stmmac_request_irq_single()
3790 				   __func__, priv->wol_irq, ret);  in stmmac_request_irq_single()
3797 	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {  in stmmac_request_irq_single()
3798 		ret = request_irq(priv->lpi_irq, stmmac_interrupt,  in stmmac_request_irq_single()
3801 			netdev_err(priv->dev,  in stmmac_request_irq_single()
3803 				   __func__, priv->lpi_irq, ret);  in stmmac_request_irq_single()
3812 	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {  in stmmac_request_irq_single()
3813 		ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt,  in stmmac_request_irq_single()
3816 			netdev_err(priv->dev,  in stmmac_request_irq_single()
3818 				   __func__, priv->sfty_irq, ret);  in stmmac_request_irq_single()
3833 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq()  local
3837 	if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN)  in stmmac_request_irq()
3855 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu)  in stmmac_setup_dma_desc()  argument
3862 		netdev_err(priv->dev, "%s: DMA conf allocation failed\n",  in stmmac_setup_dma_desc()
3867 	bfsize = stmmac_set_16kib_bfsize(priv, mtu);  in stmmac_setup_dma_desc()
3878 	dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size;  in stmmac_setup_dma_desc()
3879 	dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size;  in stmmac_setup_dma_desc()
3887 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {  in stmmac_setup_dma_desc()
3889 		int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;  in stmmac_setup_dma_desc()
3895 	ret = alloc_dma_desc_resources(priv, dma_conf);  in stmmac_setup_dma_desc()
3897 		netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",  in stmmac_setup_dma_desc()
3902 	ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL);  in stmmac_setup_dma_desc()
3904 		netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",  in stmmac_setup_dma_desc()
3912 	free_dma_desc_resources(priv, dma_conf);  in stmmac_setup_dma_desc()
3931 	struct stmmac_priv *priv = netdev_priv(dev);  in __stmmac_open()  local
3932 	int mode = priv->plat->phy_interface;  in __stmmac_open()
3936 	ret = pm_runtime_resume_and_get(priv->device);  in __stmmac_open()
3940 	if ((!priv->hw->xpcs ||  in __stmmac_open()
3941 	     xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) {  in __stmmac_open()
3944 			netdev_err(priv->dev,  in __stmmac_open()
3951 	priv->rx_copybreak = STMMAC_RX_COPYBREAK;  in __stmmac_open()
3955 		if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN)  in __stmmac_open()
3956 			dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs;  in __stmmac_open()
3957 	memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf));  in __stmmac_open()
3959 	stmmac_reset_queues_param(priv);  in __stmmac_open()
3961 	if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&  in __stmmac_open()
3962 	    priv->plat->serdes_powerup) {  in __stmmac_open()
3963 		ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv);  in __stmmac_open()
3965 			netdev_err(priv->dev, "%s: Serdes powerup failed\n",  in __stmmac_open()
3973 		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);  in __stmmac_open()
3977 	stmmac_init_coalesce(priv);  in __stmmac_open()
3979 	phylink_start(priv->phylink);  in __stmmac_open()
3981 	phylink_speed_up(priv->phylink);  in __stmmac_open()
3987 	stmmac_enable_all_queues(priv);  in __stmmac_open()
3988 	netif_tx_start_all_queues(priv->dev);  in __stmmac_open()
3989 	stmmac_enable_all_dma_irq(priv);  in __stmmac_open()
3994 	phylink_stop(priv->phylink);  in __stmmac_open()
3996 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in __stmmac_open()
3997 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in __stmmac_open()
4001 	phylink_disconnect_phy(priv->phylink);  in __stmmac_open()
4003 	pm_runtime_put(priv->device);  in __stmmac_open()
4009 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_open()  local
4013 	dma_conf = stmmac_setup_dma_desc(priv, dev->mtu);  in stmmac_open()
4019 		free_dma_desc_resources(priv, dma_conf);  in stmmac_open()
4033 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_release()  local
4036 	if (device_may_wakeup(priv->device))  in stmmac_release()
4037 		phylink_speed_down(priv->phylink, false);  in stmmac_release()
4039 	phylink_stop(priv->phylink);  in stmmac_release()
4040 	phylink_disconnect_phy(priv->phylink);  in stmmac_release()
4042 	stmmac_disable_all_queues(priv);  in stmmac_release()
4044 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in stmmac_release()
4045 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in stmmac_release()
4052 	if (priv->eee_enabled) {  in stmmac_release()
4053 		priv->tx_path_in_lpi_mode = false;  in stmmac_release()
4054 		del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_release()
4058 	stmmac_stop_all_dma(priv);  in stmmac_release()
4061 	free_dma_desc_resources(priv, &priv->dma_conf);  in stmmac_release()
4064 	stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_release()
4067 	if (priv->plat->serdes_powerdown)  in stmmac_release()
4068 		priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv);  in stmmac_release()
4070 	stmmac_release_ptp(priv);  in stmmac_release()
4072 	if (priv->dma_cap.fpesel)  in stmmac_release()
4073 		timer_shutdown_sync(&priv->fpe_cfg.verify_timer);  in stmmac_release()
4075 	pm_runtime_put(priv->device);  in stmmac_release()
4080 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,  in stmmac_vlan_insert()  argument
4087 	if (!priv->dma_cap.vlins)  in stmmac_vlan_insert()
4103 	if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))  in stmmac_vlan_insert()
4106 	stmmac_set_tx_owner(priv, p);  in stmmac_vlan_insert()
4107 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);  in stmmac_vlan_insert()
4122 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,  in stmmac_tso_allocator()  argument
4125 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tso_allocator()
4136 						priv->dma_conf.dma_tx_size);  in stmmac_tso_allocator()
4145 		if (priv->dma_cap.addr64 <= 32)  in stmmac_tso_allocator()
4148 			stmmac_set_desc_addr(priv, desc, curr_addr);  in stmmac_tso_allocator()
4153 		stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,  in stmmac_tso_allocator()
4162 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)  in stmmac_flush_tx_descriptors()  argument
4164 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_flush_tx_descriptors()
4167 	if (likely(priv->extend_desc))  in stmmac_flush_tx_descriptors()
4181 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);  in stmmac_flush_tx_descriptors()
4214 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_tso_xmit()  local
4233 			priv->xstats.tx_dropped++;  in stmmac_tso_xmit()
4241 	tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tso_xmit()
4242 	txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_tso_xmit()
4255 	if (unlikely(stmmac_tx_avail(priv, queue) <  in stmmac_tso_xmit()
4258 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,  in stmmac_tso_xmit()
4261 			netdev_err(priv->dev,  in stmmac_tso_xmit()
4279 		stmmac_set_mss(priv, mss_desc, mss);  in stmmac_tso_xmit()
4282 						priv->dma_conf.dma_tx_size);  in stmmac_tso_xmit()
4286 	if (netif_msg_tx_queued(priv)) {  in stmmac_tso_xmit()
4303 	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),  in stmmac_tso_xmit()
4305 	if (dma_mapping_error(priv->device, des))  in stmmac_tso_xmit()
4308 	if (priv->dma_cap.addr64 <= 32) {  in stmmac_tso_xmit()
4318 		stmmac_set_desc_addr(priv, first, des);  in stmmac_tso_xmit()
4324 	stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);  in stmmac_tso_xmit()
4347 		des = skb_frag_dma_map(priv->device, frag, 0,  in stmmac_tso_xmit()
4350 		if (dma_mapping_error(priv->device, des))  in stmmac_tso_xmit()
4353 		stmmac_tso_allocator(priv, des, skb_frag_size(frag),  in stmmac_tso_xmit()
4372 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)  in stmmac_tso_xmit()
4374 	else if (!priv->tx_coal_frames[queue])  in stmmac_tso_xmit()
4376 	else if (tx_packets > priv->tx_coal_frames[queue])  in stmmac_tso_xmit()
4379 		  priv->tx_coal_frames[queue]) < tx_packets)  in stmmac_tso_xmit()
4391 		stmmac_set_tx_ic(priv, desc);  in stmmac_tso_xmit()
4399 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);  in stmmac_tso_xmit()
4401 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {  in stmmac_tso_xmit()
4402 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",  in stmmac_tso_xmit()
4404 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));  in stmmac_tso_xmit()
4415 	if (priv->sarc_type)  in stmmac_tso_xmit()
4416 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);  in stmmac_tso_xmit()
4421 		     priv->hwts_tx_en)) {  in stmmac_tso_xmit()
4424 		stmmac_enable_tx_timestamp(priv, first);  in stmmac_tso_xmit()
4428 	stmmac_prepare_tso_tx_desc(priv, first, 1,  in stmmac_tso_xmit()
4442 		stmmac_set_tx_owner(priv, mss_desc);  in stmmac_tso_xmit()
4445 	if (netif_msg_pktdata(priv)) {  in stmmac_tso_xmit()
4455 	stmmac_flush_tx_descriptors(priv, queue);  in stmmac_tso_xmit()
4456 	stmmac_tx_timer_arm(priv, queue);  in stmmac_tso_xmit()
4461 	dev_err(priv->device, "Tx dma map failed\n");  in stmmac_tso_xmit()
4463 	priv->xstats.tx_dropped++;  in stmmac_tso_xmit()
4500 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xmit()  local
4514 	tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_xmit()
4515 	txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_xmit()
4518 	if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en)  in stmmac_xmit()
4519 		stmmac_disable_eee_mode(priv);  in stmmac_xmit()
4522 	if (skb_is_gso(skb) && priv->tso) {  in stmmac_xmit()
4525 		if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))  in stmmac_xmit()
4529 	if (priv->est && priv->est->enable &&  in stmmac_xmit()
4530 	    priv->est->max_sdu[queue] &&  in stmmac_xmit()
4531 	    skb->len > priv->est->max_sdu[queue]){  in stmmac_xmit()
4532 		priv->xstats.max_sdu_txq_drop[queue]++;  in stmmac_xmit()
4536 	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {  in stmmac_xmit()
4538 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,  in stmmac_xmit()
4541 			netdev_err(priv->dev,  in stmmac_xmit()
4549 	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);  in stmmac_xmit()
4564 	    (priv->plat->tx_queues_cfg[queue].coe_unsupported ||  in stmmac_xmit()
4571 	if (likely(priv->extend_desc))  in stmmac_xmit()
4581 		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);  in stmmac_xmit()
4583 	enh_desc = priv->plat->enh_desc;  in stmmac_xmit()
4586 		is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);  in stmmac_xmit()
4589 		entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);  in stmmac_xmit()
4599 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);  in stmmac_xmit()
4602 		if (likely(priv->extend_desc))  in stmmac_xmit()
4609 		des = skb_frag_dma_map(priv->device, frag, 0, len,  in stmmac_xmit()
4611 		if (dma_mapping_error(priv->device, des))  in stmmac_xmit()
4616 		stmmac_set_desc_addr(priv, desc, des);  in stmmac_xmit()
4624 		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,  in stmmac_xmit()
4625 				priv->mode, 1, last_segment, skb->len);  in stmmac_xmit()
4640 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)  in stmmac_xmit()
4642 	else if (!priv->tx_coal_frames[queue])  in stmmac_xmit()
4644 	else if (tx_packets > priv->tx_coal_frames[queue])  in stmmac_xmit()
4647 		  priv->tx_coal_frames[queue]) < tx_packets)  in stmmac_xmit()
4653 		if (likely(priv->extend_desc))  in stmmac_xmit()
4661 		stmmac_set_tx_ic(priv, desc);  in stmmac_xmit()
4669 	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);  in stmmac_xmit()
4672 	if (netif_msg_pktdata(priv)) {  in stmmac_xmit()
4673 		netdev_dbg(priv->dev,  in stmmac_xmit()
4678 		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");  in stmmac_xmit()
4682 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {  in stmmac_xmit()
4683 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",  in stmmac_xmit()
4685 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));  in stmmac_xmit()
4694 	if (priv->sarc_type)  in stmmac_xmit()
4695 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);  in stmmac_xmit()
4706 		des = dma_map_single(priv->device, skb->data,  in stmmac_xmit()
4708 		if (dma_mapping_error(priv->device, des))  in stmmac_xmit()
4715 		stmmac_set_desc_addr(priv, first, des);  in stmmac_xmit()
4721 			     priv->hwts_tx_en)) {  in stmmac_xmit()
4724 			stmmac_enable_tx_timestamp(priv, first);  in stmmac_xmit()
4728 		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,  in stmmac_xmit()
4729 				csum_insertion, priv->mode, 0, last_segment,  in stmmac_xmit()
4737 		stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);  in stmmac_xmit()
4740 	stmmac_set_tx_owner(priv, first);  in stmmac_xmit()
4744 	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);  in stmmac_xmit()
4746 	stmmac_flush_tx_descriptors(priv, queue);  in stmmac_xmit()
4747 	stmmac_tx_timer_arm(priv, queue);  in stmmac_xmit()
4752 	netdev_err(priv->dev, "Tx DMA map failed\n");  in stmmac_xmit()
4755 	priv->xstats.tx_dropped++;  in stmmac_xmit()
4784 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)  in stmmac_rx_refill()  argument
4786 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx_refill()
4787 	int dirty = stmmac_rx_dirty(priv, queue);  in stmmac_rx_refill()
4791 	if (priv->dma_cap.host_dma_width <= 32)  in stmmac_rx_refill()
4799 		if (priv->extend_desc)  in stmmac_rx_refill()
4810 		if (priv->sph && !buf->sec_page) {  in stmmac_rx_refill()
4820 		stmmac_set_desc_addr(priv, p, buf->addr);  in stmmac_rx_refill()
4821 		if (priv->sph)  in stmmac_rx_refill()
4822 			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);  in stmmac_rx_refill()
4824 			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);  in stmmac_rx_refill()
4825 		stmmac_refill_desc3(priv, rx_q, p);  in stmmac_rx_refill()
4828 		rx_q->rx_count_frames += priv->rx_coal_frames[queue];  in stmmac_rx_refill()
4829 		if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])  in stmmac_rx_refill()
4832 		use_rx_wd = !priv->rx_coal_frames[queue];  in stmmac_rx_refill()
4834 		if (!priv->use_riwt)  in stmmac_rx_refill()
4838 		stmmac_set_rx_owner(priv, p, use_rx_wd);  in stmmac_rx_refill()
4840 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);  in stmmac_rx_refill()
4845 	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);  in stmmac_rx_refill()
4848 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,  in stmmac_rx_buf1_len()  argument
4853 	int coe = priv->hw->rx_csum;  in stmmac_rx_buf1_len()
4856 	if (priv->sph && len)  in stmmac_rx_buf1_len()
4860 	stmmac_get_rx_header_len(priv, p, &hlen);  in stmmac_rx_buf1_len()
4861 	if (priv->sph && hlen) {  in stmmac_rx_buf1_len()
4862 		priv->xstats.rx_split_hdr_pkt_n++;  in stmmac_rx_buf1_len()
4868 		return priv->dma_conf.dma_buf_sz;  in stmmac_rx_buf1_len()
4870 	plen = stmmac_get_rx_frame_len(priv, p, coe);  in stmmac_rx_buf1_len()
4873 	return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen);  in stmmac_rx_buf1_len()
4876 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,  in stmmac_rx_buf2_len()  argument
4880 	int coe = priv->hw->rx_csum;  in stmmac_rx_buf2_len()
4884 	if (!priv->sph)  in stmmac_rx_buf2_len()
4889 		return priv->dma_conf.dma_buf_sz;  in stmmac_rx_buf2_len()
4891 	plen = stmmac_get_rx_frame_len(priv, p, coe);  in stmmac_rx_buf2_len()
4897 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,  in stmmac_xdp_xmit_xdpf()  argument
4900 	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_xdp_xmit_xdpf()
4901 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_xdp_xmit_xdpf()
4907 	if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv))  in stmmac_xdp_xmit_xdpf()
4910 	if (priv->est && priv->est->enable &&  in stmmac_xdp_xmit_xdpf()
4911 	    priv->est->max_sdu[queue] &&  in stmmac_xdp_xmit_xdpf()
4912 	    xdpf->len > priv->est->max_sdu[queue]) {  in stmmac_xdp_xmit_xdpf()
4913 		priv->xstats.max_sdu_txq_drop[queue]++;  in stmmac_xdp_xmit_xdpf()
4917 	if (likely(priv->extend_desc))  in stmmac_xdp_xmit_xdpf()
4925 		dma_addr = dma_map_single(priv->device, xdpf->data,  in stmmac_xdp_xmit_xdpf()
4927 		if (dma_mapping_error(priv->device, dma_addr))  in stmmac_xdp_xmit_xdpf()
4936 		dma_sync_single_for_device(priv->device, dma_addr,  in stmmac_xdp_xmit_xdpf()
4950 	stmmac_set_desc_addr(priv, tx_desc, dma_addr);  in stmmac_xdp_xmit_xdpf()
4952 	stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len,  in stmmac_xdp_xmit_xdpf()
4953 			       true, priv->mode, true, true,  in stmmac_xdp_xmit_xdpf()
4958 	if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)  in stmmac_xdp_xmit_xdpf()
4965 		stmmac_set_tx_ic(priv, tx_desc);  in stmmac_xdp_xmit_xdpf()
4971 	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);  in stmmac_xdp_xmit_xdpf()
4973 	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);  in stmmac_xdp_xmit_xdpf()
4979 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv,  in stmmac_xdp_get_tx_queue()  argument
4987 	while (index >= priv->plat->tx_queues_to_use)  in stmmac_xdp_get_tx_queue()
4988 		index -= priv->plat->tx_queues_to_use;  in stmmac_xdp_get_tx_queue()
4993 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv,  in stmmac_xdp_xmit_back()  argument
5005 	queue = stmmac_xdp_get_tx_queue(priv, cpu);  in stmmac_xdp_xmit_back()
5006 	nq = netdev_get_tx_queue(priv->dev, queue);  in stmmac_xdp_xmit_back()
5012 	res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false);  in stmmac_xdp_xmit_back()
5014 		stmmac_flush_tx_descriptors(priv, queue);  in stmmac_xdp_xmit_back()
5021 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv,  in __stmmac_xdp_run_prog()  argument
5034 		res = stmmac_xdp_xmit_back(priv, xdp);  in __stmmac_xdp_run_prog()
5037 		if (xdp_do_redirect(priv->dev, xdp, prog) < 0)  in __stmmac_xdp_run_prog()
5043 		bpf_warn_invalid_xdp_action(priv->dev, prog, act);  in __stmmac_xdp_run_prog()
5046 		trace_xdp_exception(priv->dev, prog, act);  in __stmmac_xdp_run_prog()
5056 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv,  in stmmac_xdp_run_prog()  argument
5062 	prog = READ_ONCE(priv->xdp_prog);  in stmmac_xdp_run_prog()
5068 	res = __stmmac_xdp_run_prog(priv, prog, xdp);  in stmmac_xdp_run_prog()
5073 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv,  in stmmac_finalize_xdp_rx()  argument
5079 	queue = stmmac_xdp_get_tx_queue(priv, cpu);  in stmmac_finalize_xdp_rx()
5082 		stmmac_tx_timer_arm(priv, queue);  in stmmac_finalize_xdp_rx()
5108 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,  in stmmac_dispatch_skb_zc()  argument
5112 	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];  in stmmac_dispatch_skb_zc()
5113 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_dispatch_skb_zc()
5116 	int coe = priv->hw->rx_csum;  in stmmac_dispatch_skb_zc()
5122 		priv->xstats.rx_dropped++;  in stmmac_dispatch_skb_zc()
5126 	stmmac_get_rx_hwtstamp(priv, p, np, skb);  in stmmac_dispatch_skb_zc()
5127 	if (priv->hw->hw_vlan_en)  in stmmac_dispatch_skb_zc()
5129 		stmmac_rx_hw_vlan(priv, priv->hw, p, skb);  in stmmac_dispatch_skb_zc()
5132 		stmmac_rx_vlan(priv->dev, skb);  in stmmac_dispatch_skb_zc()
5133 	skb->protocol = eth_type_trans(skb, priv->dev);  in stmmac_dispatch_skb_zc()
5140 	if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))  in stmmac_dispatch_skb_zc()
5152 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)  in stmmac_rx_refill_zc()  argument
5154 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx_refill_zc()
5159 	budget = min(budget, stmmac_rx_dirty(priv, queue));  in stmmac_rx_refill_zc()
5174 		if (priv->extend_desc)  in stmmac_rx_refill_zc()
5180 		stmmac_set_desc_addr(priv, rx_desc, dma_addr);  in stmmac_rx_refill_zc()
5181 		stmmac_set_desc_sec_addr(priv, rx_desc, 0, false);  in stmmac_rx_refill_zc()
5182 		stmmac_refill_desc3(priv, rx_q, rx_desc);  in stmmac_rx_refill_zc()
5185 		rx_q->rx_count_frames += priv->rx_coal_frames[queue];  in stmmac_rx_refill_zc()
5186 		if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])  in stmmac_rx_refill_zc()
5189 		use_rx_wd = !priv->rx_coal_frames[queue];  in stmmac_rx_refill_zc()
5191 		if (!priv->use_riwt)  in stmmac_rx_refill_zc()
5195 		stmmac_set_rx_owner(priv, rx_desc, use_rx_wd);  in stmmac_rx_refill_zc()
5197 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);  in stmmac_rx_refill_zc()
5204 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);  in stmmac_rx_refill_zc()
5220 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)  in stmmac_rx_zc()  argument
5222 	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];  in stmmac_rx_zc()
5223 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx_zc()
5225 	int dirty = stmmac_rx_dirty(priv, queue);  in stmmac_rx_zc()
5234 	if (netif_msg_rx_status(priv)) {  in stmmac_rx_zc()
5237 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);  in stmmac_rx_zc()
5238 		if (priv->extend_desc) {  in stmmac_rx_zc()
5246 		stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,  in stmmac_rx_zc()
5276 				  !stmmac_rx_refill_zc(priv, queue, dirty);  in stmmac_rx_zc()
5280 		if (priv->extend_desc)  in stmmac_rx_zc()
5286 		status = stmmac_rx_status(priv, &priv->xstats, p);  in stmmac_rx_zc()
5293 						priv->dma_conf.dma_rx_size);  in stmmac_rx_zc()
5296 		if (priv->extend_desc)  in stmmac_rx_zc()
5307 		if (priv->extend_desc)  in stmmac_rx_zc()
5308 			stmmac_rx_extended_status(priv, &priv->xstats,  in stmmac_rx_zc()
5315 			if (!priv->hwts_rx_en)  in stmmac_rx_zc()
5336 		ctx->priv = priv;  in stmmac_rx_zc()
5341 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);  in stmmac_rx_zc()
5354 		prog = READ_ONCE(priv->xdp_prog);  in stmmac_rx_zc()
5355 		res = __stmmac_xdp_run_prog(priv, prog, buf->xdp);  in stmmac_rx_zc()
5359 			stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp);  in stmmac_rx_zc()
5383 	stmmac_finalize_xdp_rx(priv, xdp_status);  in stmmac_rx_zc()
5389 	priv->xstats.rx_dropped += rx_dropped;  in stmmac_rx_zc()
5390 	priv->xstats.rx_errors += rx_errors;  in stmmac_rx_zc()
5393 		if (failure || stmmac_rx_dirty(priv, queue) > 0)  in stmmac_rx_zc()
5412 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)  in stmmac_rx()  argument
5415 	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];  in stmmac_rx()
5416 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx()
5417 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_rx()
5419 	int status = 0, coe = priv->hw->rx_csum;  in stmmac_rx()
5429 	buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;  in stmmac_rx()
5430 	limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit);  in stmmac_rx()
5432 	if (netif_msg_rx_status(priv)) {  in stmmac_rx()
5435 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);  in stmmac_rx()
5436 		if (priv->extend_desc) {  in stmmac_rx()
5444 		stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,  in stmmac_rx()
5475 		if (priv->extend_desc)  in stmmac_rx()
5481 		status = stmmac_rx_status(priv, &priv->xstats, p);  in stmmac_rx()
5487 						priv->dma_conf.dma_rx_size);  in stmmac_rx()
5490 		if (priv->extend_desc)  in stmmac_rx()
5497 		if (priv->extend_desc)  in stmmac_rx()
5498 			stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry);  in stmmac_rx()
5503 			if (!priv->hwts_rx_en)  in stmmac_rx()
5522 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);  in stmmac_rx()
5524 		buf2_len = stmmac_rx_buf2_len(priv, p, status, len);  in stmmac_rx()
5541 			dma_sync_single_for_cpu(priv->device, buf->addr,  in stmmac_rx()
5551 			ctx.priv = priv;  in stmmac_rx()
5555 			skb = stmmac_xdp_run_prog(priv, &ctx.xdp);  in stmmac_rx()
5614 			dma_sync_single_for_cpu(priv->device, buf->addr,  in stmmac_rx()
5618 					priv->dma_conf.dma_buf_sz);  in stmmac_rx()
5626 			dma_sync_single_for_cpu(priv->device, buf->sec_addr,  in stmmac_rx()
5630 					priv->dma_conf.dma_buf_sz);  in stmmac_rx()
5645 		stmmac_get_rx_hwtstamp(priv, p, np, skb);  in stmmac_rx()
5647 		if (priv->hw->hw_vlan_en)  in stmmac_rx()
5649 			stmmac_rx_hw_vlan(priv, priv->hw, p, skb);  in stmmac_rx()
5652 			stmmac_rx_vlan(priv->dev, skb);  in stmmac_rx()
5654 		skb->protocol = eth_type_trans(skb, priv->dev);  in stmmac_rx()
5661 		if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))  in stmmac_rx()
5680 	stmmac_finalize_xdp_rx(priv, xdp_status);  in stmmac_rx()
5682 	stmmac_rx_refill(priv, queue);  in stmmac_rx()
5690 	priv->xstats.rx_dropped += rx_dropped;  in stmmac_rx()
5691 	priv->xstats.rx_errors += rx_errors;  in stmmac_rx()
5700 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_rx()  local
5705 	rxq_stats = &priv->xstats.rxq_stats[chan];  in stmmac_napi_poll_rx()
5710 	work_done = stmmac_rx(priv, budget, chan);  in stmmac_napi_poll_rx()
5715 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);  in stmmac_napi_poll_rx()
5726 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_tx()  local
5732 	txq_stats = &priv->xstats.txq_stats[chan];  in stmmac_napi_poll_tx()
5737 	work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets);  in stmmac_napi_poll_tx()
5744 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);  in stmmac_napi_poll_tx()
5750 		stmmac_tx_timer_arm(priv, chan);  in stmmac_napi_poll_tx()
5759 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_rxtx()  local
5766 	rxq_stats = &priv->xstats.rxq_stats[chan];  in stmmac_napi_poll_rxtx()
5771 	txq_stats = &priv->xstats.txq_stats[chan];  in stmmac_napi_poll_rxtx()
5776 	tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets);  in stmmac_napi_poll_rxtx()
5779 	rx_done = stmmac_rx_zc(priv, budget, chan);  in stmmac_napi_poll_rxtx()
5797 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);  in stmmac_napi_poll_rxtx()
5803 		stmmac_tx_timer_arm(priv, chan);  in stmmac_napi_poll_rxtx()
5819 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_tx_timeout()  local
5821 	stmmac_global_err(priv);  in stmmac_tx_timeout()
5835 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_set_rx_mode()  local
5837 	stmmac_set_filter(priv, priv->hw, dev);  in stmmac_set_rx_mode()
5853 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_change_mtu()  local
5854 	int txfifosz = priv->plat->tx_fifo_size;  in stmmac_change_mtu()
5860 		txfifosz = priv->dma_cap.tx_fifo_size;  in stmmac_change_mtu()
5862 	txfifosz /= priv->plat->tx_queues_to_use;  in stmmac_change_mtu()
5864 	if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) {  in stmmac_change_mtu()
5865 		netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n");  in stmmac_change_mtu()
5876 		netdev_dbg(priv->dev, "restarting interface to change its MTU\n");  in stmmac_change_mtu()
5878 		dma_conf = stmmac_setup_dma_desc(priv, mtu);  in stmmac_change_mtu()
5880 			netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n",  in stmmac_change_mtu()
5889 			free_dma_desc_resources(priv, dma_conf);  in stmmac_change_mtu()
5891 			netdev_err(priv->dev, "failed reopening the interface after MTU change\n");  in stmmac_change_mtu()
5909 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_fix_features()  local
5911 	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)  in stmmac_fix_features()
5914 	if (!priv->plat->tx_coe)  in stmmac_fix_features()
5922 	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))  in stmmac_fix_features()
5926 	if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) {  in stmmac_fix_features()
5928 			priv->tso = true;  in stmmac_fix_features()
5930 			priv->tso = false;  in stmmac_fix_features()
5939 	struct stmmac_priv *priv = netdev_priv(netdev);  in stmmac_set_features()  local
5943 		priv->hw->rx_csum = priv->plat->rx_coe;  in stmmac_set_features()
5945 		priv->hw->rx_csum = 0;  in stmmac_set_features()
5949 	stmmac_rx_ipc(priv, priv->hw);  in stmmac_set_features()
5951 	if (priv->sph_cap) {  in stmmac_set_features()
5952 		bool sph_en = (priv->hw->rx_csum > 0) && priv->sph;  in stmmac_set_features()
5955 		for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)  in stmmac_set_features()
5956 			stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);  in stmmac_set_features()
5960 		priv->hw->hw_vlan_en = true;  in stmmac_set_features()
5962 		priv->hw->hw_vlan_en = false;  in stmmac_set_features()
5964 	stmmac_set_hw_vlan_mode(priv, priv->hw);  in stmmac_set_features()
5969 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status)  in stmmac_fpe_event_status()  argument
5971 	struct stmmac_fpe_cfg *fpe_cfg = &priv->fpe_cfg;  in stmmac_fpe_event_status()
5981 		stmmac_fpe_send_mpacket(priv, priv->ioaddr, fpe_cfg,  in stmmac_fpe_event_status()
5998 static void stmmac_common_interrupt(struct stmmac_priv *priv)  in stmmac_common_interrupt()  argument
6000 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_common_interrupt()
6001 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_common_interrupt()
6006 	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;  in stmmac_common_interrupt()
6009 	if (priv->irq_wake)  in stmmac_common_interrupt()
6010 		pm_wakeup_event(priv->device, 0);  in stmmac_common_interrupt()
6012 	if (priv->dma_cap.estsel)  in stmmac_common_interrupt()
6013 		stmmac_est_irq_status(priv, priv, priv->dev,  in stmmac_common_interrupt()
6014 				      &priv->xstats, tx_cnt);  in stmmac_common_interrupt()
6016 	if (priv->dma_cap.fpesel) {  in stmmac_common_interrupt()
6017 		int status = stmmac_fpe_irq_status(priv, priv->ioaddr,  in stmmac_common_interrupt()
6018 						   priv->dev);  in stmmac_common_interrupt()
6020 		stmmac_fpe_event_status(priv, status);  in stmmac_common_interrupt()
6024 	if ((priv->plat->has_gmac) || xmac) {  in stmmac_common_interrupt()
6025 		int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);  in stmmac_common_interrupt()
6030 				priv->tx_path_in_lpi_mode = true;  in stmmac_common_interrupt()
6032 				priv->tx_path_in_lpi_mode = false;  in stmmac_common_interrupt()
6036 			stmmac_host_mtl_irq_status(priv, priv->hw, queue);  in stmmac_common_interrupt()
6039 		if (priv->hw->pcs &&  in stmmac_common_interrupt()
6040 		    !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) {  in stmmac_common_interrupt()
6041 			if (priv->xstats.pcs_link)  in stmmac_common_interrupt()
6042 				netif_carrier_on(priv->dev);  in stmmac_common_interrupt()
6044 				netif_carrier_off(priv->dev);  in stmmac_common_interrupt()
6047 		stmmac_timestamp_interrupt(priv, priv);  in stmmac_common_interrupt()
6065 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_interrupt()  local
6068 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_interrupt()
6072 	if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv))  in stmmac_interrupt()
6076 	stmmac_common_interrupt(priv);  in stmmac_interrupt()
6079 	stmmac_dma_interrupt(priv);  in stmmac_interrupt()
6087 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_mac_interrupt()  local
6090 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_mac_interrupt()
6094 	stmmac_common_interrupt(priv);  in stmmac_mac_interrupt()
6102 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_safety_interrupt()  local
6105 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_safety_interrupt()
6109 	stmmac_safety_feat_interrupt(priv);  in stmmac_safety_interrupt()
6119 	struct stmmac_priv *priv;  in stmmac_msi_intr_tx()  local
6123 	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);  in stmmac_msi_intr_tx()
6126 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_msi_intr_tx()
6129 	status = stmmac_napi_check(priv, chan, DMA_DIR_TX);  in stmmac_msi_intr_tx()
6133 		stmmac_bump_dma_threshold(priv, chan);  in stmmac_msi_intr_tx()
6135 		stmmac_tx_err(priv, chan);  in stmmac_msi_intr_tx()
6146 	struct stmmac_priv *priv;  in stmmac_msi_intr_rx()  local
6149 	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);  in stmmac_msi_intr_rx()
6152 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_msi_intr_rx()
6155 	stmmac_napi_check(priv, chan, DMA_DIR_RX);  in stmmac_msi_intr_rx()
6171 	struct stmmac_priv *priv = netdev_priv (dev);  in stmmac_ioctl()  local
6181 		ret = phylink_mii_ioctl(priv->phylink, rq, cmd);  in stmmac_ioctl()
6199 	struct stmmac_priv *priv = cb_priv;  in stmmac_setup_tc_block_cb()  local
6202 	if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))  in stmmac_setup_tc_block_cb()
6205 	__stmmac_disable_all_queues(priv);  in stmmac_setup_tc_block_cb()
6209 		ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);  in stmmac_setup_tc_block_cb()
6212 		ret = stmmac_tc_setup_cls(priv, priv, type_data);  in stmmac_setup_tc_block_cb()
6218 	stmmac_enable_all_queues(priv);  in stmmac_setup_tc_block_cb()
6227 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_setup_tc()  local
6231 		return stmmac_tc_query_caps(priv, priv, type_data);  in stmmac_setup_tc()
6233 		return stmmac_tc_setup_mqprio(priv, priv, type_data);  in stmmac_setup_tc()
6238 						  priv, priv, true);  in stmmac_setup_tc()
6240 		return stmmac_tc_setup_cbs(priv, priv, type_data);  in stmmac_setup_tc()
6242 		return stmmac_tc_setup_taprio(priv, priv, type_data);  in stmmac_setup_tc()
6244 		return stmmac_tc_setup_etf(priv, priv, type_data);  in stmmac_setup_tc()
6270 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_set_mac_address()  local
6273 	ret = pm_runtime_resume_and_get(priv->device);  in stmmac_set_mac_address()
6281 	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);  in stmmac_set_mac_address()
6284 	pm_runtime_put(priv->device);  in stmmac_set_mac_address()
6318 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_rings_status_show()  local
6319 	u32 rx_count = priv->plat->rx_queues_to_use;  in stmmac_rings_status_show()
6320 	u32 tx_count = priv->plat->tx_queues_to_use;  in stmmac_rings_status_show()
6327 		struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rings_status_show()
6331 		if (priv->extend_desc) {  in stmmac_rings_status_show()
6334 					   priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy);  in stmmac_rings_status_show()
6338 					   priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy);  in stmmac_rings_status_show()
6343 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_rings_status_show()
6347 		if (priv->extend_desc) {  in stmmac_rings_status_show()
6350 					   priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy);  in stmmac_rings_status_show()
6354 					   priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy);  in stmmac_rings_status_show()
6381 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_dma_cap_show()  local
6383 	if (!priv->hw_cap_support) {  in stmmac_dma_cap_show()
6393 		   (priv->dma_cap.mbps_10_100) ? "Y" : "N");  in stmmac_dma_cap_show()
6395 		   (priv->dma_cap.mbps_1000) ? "Y" : "N");  in stmmac_dma_cap_show()
6397 		   (priv->dma_cap.half_duplex) ? "Y" : "N");  in stmmac_dma_cap_show()
6398 	if (priv->plat->has_xgmac) {  in stmmac_dma_cap_show()
6401 			   priv->dma_cap.multi_addr);  in stmmac_dma_cap_show()
6404 			   (priv->dma_cap.hash_filter) ? "Y" : "N");  in stmmac_dma_cap_show()
6406 			   (priv->dma_cap.multi_addr) ? "Y" : "N");  in stmmac_dma_cap_show()
6409 		   (priv->dma_cap.pcs) ? "Y" : "N");  in stmmac_dma_cap_show()
6411 		   (priv->dma_cap.sma_mdio) ? "Y" : "N");  in stmmac_dma_cap_show()
6413 		   (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");  in stmmac_dma_cap_show()
6415 		   (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");  in stmmac_dma_cap_show()
6417 		   (priv->dma_cap.rmon) ? "Y" : "N");  in stmmac_dma_cap_show()
6419 		   (priv->dma_cap.time_stamp) ? "Y" : "N");  in stmmac_dma_cap_show()
6421 		   (priv->dma_cap.atime_stamp) ? "Y" : "N");  in stmmac_dma_cap_show()
6422 	if (priv->plat->has_xgmac)  in stmmac_dma_cap_show()
6424 			   dwxgmac_timestamp_source[priv->dma_cap.tssrc]);  in stmmac_dma_cap_show()
6426 		   (priv->dma_cap.eee) ? "Y" : "N");  in stmmac_dma_cap_show()
6427 	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");  in stmmac_dma_cap_show()
6429 		   (priv->dma_cap.tx_coe) ? "Y" : "N");  in stmmac_dma_cap_show()
6430 	if (priv->synopsys_id >= DWMAC_CORE_4_00 ||  in stmmac_dma_cap_show()
6431 	    priv->plat->has_xgmac) {  in stmmac_dma_cap_show()
6433 			   (priv->dma_cap.rx_coe) ? "Y" : "N");  in stmmac_dma_cap_show()
6436 			   (priv->dma_cap.rx_coe_type1) ? "Y" : "N");  in stmmac_dma_cap_show()
6438 			   (priv->dma_cap.rx_coe_type2) ? "Y" : "N");  in stmmac_dma_cap_show()
6440 			   (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");  in stmmac_dma_cap_show()
6443 		   priv->dma_cap.number_rx_channel);  in stmmac_dma_cap_show()
6445 		   priv->dma_cap.number_tx_channel);  in stmmac_dma_cap_show()
6447 		   priv->dma_cap.number_rx_queues);  in stmmac_dma_cap_show()
6449 		   priv->dma_cap.number_tx_queues);  in stmmac_dma_cap_show()
6451 		   (priv->dma_cap.enh_desc) ? "Y" : "N");  in stmmac_dma_cap_show()
6452 	seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);  in stmmac_dma_cap_show()
6453 	seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);  in stmmac_dma_cap_show()
6454 	seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ?  in stmmac_dma_cap_show()
6455 		   (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0);  in stmmac_dma_cap_show()
6456 	seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");  in stmmac_dma_cap_show()
6458 		   priv->dma_cap.pps_out_num);  in stmmac_dma_cap_show()
6460 		   dwxgmac_safety_feature_desc[priv->dma_cap.asp]);  in stmmac_dma_cap_show()
6462 		   priv->dma_cap.frpsel ? "Y" : "N");  in stmmac_dma_cap_show()
6464 		   priv->dma_cap.host_dma_width);  in stmmac_dma_cap_show()
6466 		   priv->dma_cap.rssen ? "Y" : "N");  in stmmac_dma_cap_show()
6468 		   priv->dma_cap.vlhash ? "Y" : "N");  in stmmac_dma_cap_show()
6470 		   priv->dma_cap.sphen ? "Y" : "N");  in stmmac_dma_cap_show()
6472 		   priv->dma_cap.vlins ? "Y" : "N");  in stmmac_dma_cap_show()
6474 		   priv->dma_cap.dvlan ? "Y" : "N");  in stmmac_dma_cap_show()
6476 		   priv->dma_cap.l3l4fnum);  in stmmac_dma_cap_show()
6478 		   priv->dma_cap.arpoffsel ? "Y" : "N");  in stmmac_dma_cap_show()
6480 		   priv->dma_cap.estsel ? "Y" : "N");  in stmmac_dma_cap_show()
6482 		   priv->dma_cap.fpesel ? "Y" : "N");  in stmmac_dma_cap_show()
6484 		   priv->dma_cap.tbssel ? "Y" : "N");  in stmmac_dma_cap_show()
6486 		   priv->dma_cap.tbs_ch_num);  in stmmac_dma_cap_show()
6488 		   priv->dma_cap.sgfsel ? "Y" : "N");  in stmmac_dma_cap_show()
6490 		   BIT(priv->dma_cap.ttsfd) >> 1);  in stmmac_dma_cap_show()
6492 		   priv->dma_cap.numtc);  in stmmac_dma_cap_show()
6494 		   priv->dma_cap.dcben ? "Y" : "N");  in stmmac_dma_cap_show()
6496 		   priv->dma_cap.advthword ? "Y" : "N");  in stmmac_dma_cap_show()
6498 		   priv->dma_cap.ptoen ? "Y" : "N");  in stmmac_dma_cap_show()
6500 		   priv->dma_cap.osten ? "Y" : "N");  in stmmac_dma_cap_show()
6502 		   priv->dma_cap.pfcen ? "Y" : "N");  in stmmac_dma_cap_show()
6504 		   BIT(priv->dma_cap.frpes) << 6);  in stmmac_dma_cap_show()
6506 		   BIT(priv->dma_cap.frpbs) << 6);  in stmmac_dma_cap_show()
6508 		   priv->dma_cap.frppipe_num);  in stmmac_dma_cap_show()
6510 		   priv->dma_cap.nrvf_num ?  in stmmac_dma_cap_show()
6511 		   (BIT(priv->dma_cap.nrvf_num) << 1) : 0);  in stmmac_dma_cap_show()
6513 		   priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0);  in stmmac_dma_cap_show()
6515 		   priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0);  in stmmac_dma_cap_show()
6517 		   priv->dma_cap.cbtisel ? "Y" : "N");  in stmmac_dma_cap_show()
6519 		   priv->dma_cap.aux_snapshot_n);  in stmmac_dma_cap_show()
6521 		   priv->dma_cap.pou_ost_en ? "Y" : "N");  in stmmac_dma_cap_show()
6523 		   priv->dma_cap.edma ? "Y" : "N");  in stmmac_dma_cap_show()
6525 		   priv->dma_cap.ediffc ? "Y" : "N");  in stmmac_dma_cap_show()
6527 		   priv->dma_cap.vxn ? "Y" : "N");  in stmmac_dma_cap_show()
6529 		   priv->dma_cap.dbgmem ? "Y" : "N");  in stmmac_dma_cap_show()
6531 		   priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0);  in stmmac_dma_cap_show()
6542 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_device_event()  local
6549 		if (priv->dbgfs_dir)  in stmmac_device_event()
6550 			priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,  in stmmac_device_event()
6551 							 priv->dbgfs_dir,  in stmmac_device_event()
6566 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_init_fs()  local
6571 	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);  in stmmac_init_fs()
6574 	debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,  in stmmac_init_fs()
6578 	debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,  in stmmac_init_fs()
6586 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_exit_fs()  local
6588 	debugfs_remove_recursive(priv->dbgfs_dir);  in stmmac_exit_fs()
6616 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)  in stmmac_vlan_update()  argument
6623 	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {  in stmmac_vlan_update()
6630 	if (!priv->dma_cap.vlhash) {  in stmmac_vlan_update()
6638 	return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);  in stmmac_vlan_update()
6643 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_vlan_rx_add_vid()  local
6647 	ret = pm_runtime_resume_and_get(priv->device);  in stmmac_vlan_rx_add_vid()
6654 	set_bit(vid, priv->active_vlans);  in stmmac_vlan_rx_add_vid()
6655 	ret = stmmac_vlan_update(priv, is_double);  in stmmac_vlan_rx_add_vid()
6657 		clear_bit(vid, priv->active_vlans);  in stmmac_vlan_rx_add_vid()
6661 	if (priv->hw->num_vlan) {  in stmmac_vlan_rx_add_vid()
6662 		ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);  in stmmac_vlan_rx_add_vid()
6667 	pm_runtime_put(priv->device);  in stmmac_vlan_rx_add_vid()
6674 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_vlan_rx_kill_vid()  local
6678 	ret = pm_runtime_resume_and_get(priv->device);  in stmmac_vlan_rx_kill_vid()
6685 	clear_bit(vid, priv->active_vlans);  in stmmac_vlan_rx_kill_vid()
6687 	if (priv->hw->num_vlan) {  in stmmac_vlan_rx_kill_vid()
6688 		ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);  in stmmac_vlan_rx_kill_vid()
6693 	ret = stmmac_vlan_update(priv, is_double);  in stmmac_vlan_rx_kill_vid()
6696 	pm_runtime_put(priv->device);  in stmmac_vlan_rx_kill_vid()
6703 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_bpf()  local
6707 		return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack);  in stmmac_bpf()
6709 		return stmmac_xdp_setup_pool(priv, bpf->xsk.pool,  in stmmac_bpf()
6719 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_xmit()  local
6725 	if (unlikely(test_bit(STMMAC_DOWN, &priv->state)))  in stmmac_xdp_xmit()
6731 	queue = stmmac_xdp_get_tx_queue(priv, cpu);  in stmmac_xdp_xmit()
6732 	nq = netdev_get_tx_queue(priv->dev, queue);  in stmmac_xdp_xmit()
6741 		res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true);  in stmmac_xdp_xmit()
6749 		stmmac_flush_tx_descriptors(priv, queue);  in stmmac_xdp_xmit()
6750 		stmmac_tx_timer_arm(priv, queue);  in stmmac_xdp_xmit()
6758 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_disable_rx_queue()  argument
6760 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_disable_rx_queue()
6764 	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0);  in stmmac_disable_rx_queue()
6767 	stmmac_stop_rx_dma(priv, queue);  in stmmac_disable_rx_queue()
6768 	__free_dma_rx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_disable_rx_queue()
6771 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_enable_rx_queue()  argument
6773 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_enable_rx_queue()
6774 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_enable_rx_queue()
6779 	ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_enable_rx_queue()
6781 		netdev_err(priv->dev, "Failed to alloc RX desc.\n");  in stmmac_enable_rx_queue()
6785 	ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL);  in stmmac_enable_rx_queue()
6787 		__free_dma_rx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_enable_rx_queue()
6788 		netdev_err(priv->dev, "Failed to init RX desc.\n");  in stmmac_enable_rx_queue()
6792 	stmmac_reset_rx_queue(priv, queue);  in stmmac_enable_rx_queue()
6793 	stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue);  in stmmac_enable_rx_queue()
6795 	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_enable_rx_queue()
6800 	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,  in stmmac_enable_rx_queue()
6805 		stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_enable_rx_queue()
6809 		stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_enable_rx_queue()
6810 				      priv->dma_conf.dma_buf_sz,  in stmmac_enable_rx_queue()
6814 	stmmac_start_rx_dma(priv, queue);  in stmmac_enable_rx_queue()
6817 	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0);  in stmmac_enable_rx_queue()
6821 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_disable_tx_queue()  argument
6823 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_disable_tx_queue()
6827 	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1);  in stmmac_disable_tx_queue()
6830 	stmmac_stop_tx_dma(priv, queue);  in stmmac_disable_tx_queue()
6831 	__free_dma_tx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_disable_tx_queue()
6834 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_enable_tx_queue()  argument
6836 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_enable_tx_queue()
6837 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_enable_tx_queue()
6841 	ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_enable_tx_queue()
6843 		netdev_err(priv->dev, "Failed to alloc TX desc.\n");  in stmmac_enable_tx_queue()
6847 	ret = __init_dma_tx_desc_rings(priv,  &priv->dma_conf, queue);  in stmmac_enable_tx_queue()
6849 		__free_dma_tx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_enable_tx_queue()
6850 		netdev_err(priv->dev, "Failed to init TX desc.\n");  in stmmac_enable_tx_queue()
6854 	stmmac_reset_tx_queue(priv, queue);  in stmmac_enable_tx_queue()
6855 	stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue);  in stmmac_enable_tx_queue()
6857 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_enable_tx_queue()
6861 		stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index);  in stmmac_enable_tx_queue()
6864 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,  in stmmac_enable_tx_queue()
6867 	stmmac_start_tx_dma(priv, queue);  in stmmac_enable_tx_queue()
6870 	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1);  in stmmac_enable_tx_queue()
6876 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_release()  local
6883 	stmmac_disable_all_queues(priv);  in stmmac_xdp_release()
6885 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in stmmac_xdp_release()
6886 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in stmmac_xdp_release()
6892 	stmmac_stop_all_dma(priv);  in stmmac_xdp_release()
6895 	free_dma_desc_resources(priv, &priv->dma_conf);  in stmmac_xdp_release()
6898 	stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_xdp_release()
6909 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_open()  local
6910 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_xdp_open()
6911 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_xdp_open()
6920 	ret = alloc_dma_desc_resources(priv, &priv->dma_conf);  in stmmac_xdp_open()
6927 	ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL);  in stmmac_xdp_open()
6934 	stmmac_reset_queues_param(priv);  in stmmac_xdp_open()
6938 		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);  in stmmac_xdp_open()
6939 		stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);  in stmmac_xdp_open()
6943 	sph_en = (priv->hw->rx_csum > 0) && priv->sph;  in stmmac_xdp_open()
6947 		rx_q = &priv->dma_conf.rx_queue[chan];  in stmmac_xdp_open()
6949 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_xdp_open()
6955 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,  in stmmac_xdp_open()
6960 			stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_xdp_open()
6964 			stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_xdp_open()
6965 					      priv->dma_conf.dma_buf_sz,  in stmmac_xdp_open()
6969 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);  in stmmac_xdp_open()
6974 		tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_xdp_open()
6976 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_xdp_open()
6980 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,  in stmmac_xdp_open()
6988 	stmmac_mac_set(priv, priv->ioaddr, true);  in stmmac_xdp_open()
6991 	stmmac_start_all_dma(priv);  in stmmac_xdp_open()
6998 	stmmac_enable_all_queues(priv);  in stmmac_xdp_open()
7001 	stmmac_enable_all_dma_irq(priv);  in stmmac_xdp_open()
7006 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in stmmac_xdp_open()
7007 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in stmmac_xdp_open()
7011 	free_dma_desc_resources(priv, &priv->dma_conf);  in stmmac_xdp_open()
7018 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xsk_wakeup()  local
7023 	if (test_bit(STMMAC_DOWN, &priv->state) ||  in stmmac_xsk_wakeup()
7024 	    !netif_carrier_ok(priv->dev))  in stmmac_xsk_wakeup()
7027 	if (!stmmac_xdp_is_enabled(priv))  in stmmac_xsk_wakeup()
7030 	if (queue >= priv->plat->rx_queues_to_use ||  in stmmac_xsk_wakeup()
7031 	    queue >= priv->plat->tx_queues_to_use)  in stmmac_xsk_wakeup()
7034 	rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_xsk_wakeup()
7035 	tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_xsk_wakeup()
7036 	ch = &priv->channel[queue];  in stmmac_xsk_wakeup()
7054 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_get_stats64()  local
7055 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_get_stats64()
7056 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_get_stats64()
7061 		struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q];  in stmmac_get_stats64()
7079 		struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q];  in stmmac_get_stats64()
7093 	stats->rx_dropped = priv->xstats.rx_dropped;  in stmmac_get_stats64()
7094 	stats->rx_errors = priv->xstats.rx_errors;  in stmmac_get_stats64()
7095 	stats->tx_dropped = priv->xstats.tx_dropped;  in stmmac_get_stats64()
7096 	stats->tx_errors = priv->xstats.tx_errors;  in stmmac_get_stats64()
7097 	stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier;  in stmmac_get_stats64()
7098 	stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision;  in stmmac_get_stats64()
7099 	stats->rx_length_errors = priv->xstats.rx_length;  in stmmac_get_stats64()
7100 	stats->rx_crc_errors = priv->xstats.rx_crc_errors;  in stmmac_get_stats64()
7101 	stats->rx_over_errors = priv->xstats.rx_overflow_cntr;  in stmmac_get_stats64()
7102 	stats->rx_missed_errors = priv->xstats.rx_missed_cntr;  in stmmac_get_stats64()
7126 static void stmmac_reset_subtask(struct stmmac_priv *priv)  in stmmac_reset_subtask()  argument
7128 	if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))  in stmmac_reset_subtask()
7130 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_reset_subtask()
7133 	netdev_err(priv->dev, "Reset adapter.\n");  in stmmac_reset_subtask()
7136 	netif_trans_update(priv->dev);  in stmmac_reset_subtask()
7137 	while (test_and_set_bit(STMMAC_RESETING, &priv->state))  in stmmac_reset_subtask()
7140 	set_bit(STMMAC_DOWN, &priv->state);  in stmmac_reset_subtask()
7141 	dev_close(priv->dev);  in stmmac_reset_subtask()
7142 	dev_open(priv->dev, NULL);  in stmmac_reset_subtask()
7143 	clear_bit(STMMAC_DOWN, &priv->state);  in stmmac_reset_subtask()
7144 	clear_bit(STMMAC_RESETING, &priv->state);  in stmmac_reset_subtask()
7150 	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,  in stmmac_service_task()  local
7153 	stmmac_reset_subtask(priv);  in stmmac_service_task()
7154 	clear_bit(STMMAC_SERVICE_SCHED, &priv->state);  in stmmac_service_task()
7165 static int stmmac_hw_init(struct stmmac_priv *priv)  in stmmac_hw_init()  argument
7170 	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I)  in stmmac_hw_init()
7172 	priv->chain_mode = chain_mode;  in stmmac_hw_init()
7175 	ret = stmmac_hwif_init(priv);  in stmmac_hw_init()
7180 	priv->hw_cap_support = stmmac_get_hw_features(priv);  in stmmac_hw_init()
7181 	if (priv->hw_cap_support) {  in stmmac_hw_init()
7182 		dev_info(priv->device, "DMA HW capability register supported\n");  in stmmac_hw_init()
7189 		priv->plat->enh_desc = priv->dma_cap.enh_desc;  in stmmac_hw_init()
7190 		priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up &&  in stmmac_hw_init()
7191 				!(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL);  in stmmac_hw_init()
7192 		priv->hw->pmt = priv->plat->pmt;  in stmmac_hw_init()
7193 		if (priv->dma_cap.hash_tb_sz) {  in stmmac_hw_init()
7194 			priv->hw->multicast_filter_bins =  in stmmac_hw_init()
7195 					(BIT(priv->dma_cap.hash_tb_sz) << 5);  in stmmac_hw_init()
7196 			priv->hw->mcast_bits_log2 =  in stmmac_hw_init()
7197 					ilog2(priv->hw->multicast_filter_bins);  in stmmac_hw_init()
7201 		if (priv->plat->force_thresh_dma_mode)  in stmmac_hw_init()
7202 			priv->plat->tx_coe = 0;  in stmmac_hw_init()
7204 			priv->plat->tx_coe = priv->dma_cap.tx_coe;  in stmmac_hw_init()
7207 		priv->plat->rx_coe = priv->dma_cap.rx_coe;  in stmmac_hw_init()
7209 		if (priv->dma_cap.rx_coe_type2)  in stmmac_hw_init()
7210 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;  in stmmac_hw_init()
7211 		else if (priv->dma_cap.rx_coe_type1)  in stmmac_hw_init()
7212 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;  in stmmac_hw_init()
7215 		dev_info(priv->device, "No HW DMA feature register supported\n");  in stmmac_hw_init()
7218 	if (priv->plat->rx_coe) {  in stmmac_hw_init()
7219 		priv->hw->rx_csum = priv->plat->rx_coe;  in stmmac_hw_init()
7220 		dev_info(priv->device, "RX Checksum Offload Engine supported\n");  in stmmac_hw_init()
7221 		if (priv->synopsys_id < DWMAC_CORE_4_00)  in stmmac_hw_init()
7222 			dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);  in stmmac_hw_init()
7224 	if (priv->plat->tx_coe)  in stmmac_hw_init()
7225 		dev_info(priv->device, "TX Checksum insertion supported\n");  in stmmac_hw_init()
7227 	if (priv->plat->pmt) {  in stmmac_hw_init()
7228 		dev_info(priv->device, "Wake-Up On Lan supported\n");  in stmmac_hw_init()
7229 		device_set_wakeup_capable(priv->device, 1);  in stmmac_hw_init()
7232 	if (priv->dma_cap.tsoen)  in stmmac_hw_init()
7233 		dev_info(priv->device, "TSO supported\n");  in stmmac_hw_init()
7235 	priv->hw->vlan_fail_q_en =  in stmmac_hw_init()
7236 		(priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN);  in stmmac_hw_init()
7237 	priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;  in stmmac_hw_init()
7240 	if (priv->hwif_quirks) {  in stmmac_hw_init()
7241 		ret = priv->hwif_quirks(priv);  in stmmac_hw_init()
7251 	if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||  in stmmac_hw_init()
7252 	    (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {  in stmmac_hw_init()
7253 		priv->use_riwt = 1;  in stmmac_hw_init()
7254 		dev_info(priv->device,  in stmmac_hw_init()
7263 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_napi_add()  local
7266 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);  in stmmac_napi_add()
7269 		struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_napi_add()
7271 		ch->priv_data = priv;  in stmmac_napi_add()
7275 		if (queue < priv->plat->rx_queues_to_use) {  in stmmac_napi_add()
7278 		if (queue < priv->plat->tx_queues_to_use) {  in stmmac_napi_add()
7282 		if (queue < priv->plat->rx_queues_to_use &&  in stmmac_napi_add()
7283 		    queue < priv->plat->tx_queues_to_use) {  in stmmac_napi_add()
7292 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_napi_del()  local
7295 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);  in stmmac_napi_del()
7298 		struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_napi_del()
7300 		if (queue < priv->plat->rx_queues_to_use)  in stmmac_napi_del()
7302 		if (queue < priv->plat->tx_queues_to_use)  in stmmac_napi_del()
7304 		if (queue < priv->plat->rx_queues_to_use &&  in stmmac_napi_del()
7305 		    queue < priv->plat->tx_queues_to_use) {  in stmmac_napi_del()
7313 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_reinit_queues()  local
7321 	priv->plat->rx_queues_to_use = rx_cnt;  in stmmac_reinit_queues()
7322 	priv->plat->tx_queues_to_use = tx_cnt;  in stmmac_reinit_queues()
7324 		for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)  in stmmac_reinit_queues()
7325 			priv->rss.table[i] = ethtool_rxfh_indir_default(i,  in stmmac_reinit_queues()
7338 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_reinit_ringparam()  local
7344 	priv->dma_conf.dma_rx_size = rx_size;  in stmmac_reinit_ringparam()
7345 	priv->dma_conf.dma_tx_size = tx_size;  in stmmac_reinit_ringparam()
7364 	struct stmmac_priv *priv = container_of(fpe_cfg, struct stmmac_priv,  in stmmac_fpe_verify_timer()  local
7375 			stmmac_fpe_send_mpacket(priv, priv->ioaddr,  in stmmac_fpe_verify_timer()
7386 		stmmac_fpe_configure(priv, priv->ioaddr, fpe_cfg,  in stmmac_fpe_verify_timer()
7387 				     priv->plat->tx_queues_to_use,  in stmmac_fpe_verify_timer()
7388 				     priv->plat->rx_queues_to_use,  in stmmac_fpe_verify_timer()
7415 void stmmac_fpe_apply(struct stmmac_priv *priv)  in stmmac_fpe_apply()  argument
7417 	struct stmmac_fpe_cfg *fpe_cfg = &priv->fpe_cfg;  in stmmac_fpe_apply()
7423 		stmmac_fpe_configure(priv, priv->ioaddr, fpe_cfg,  in stmmac_fpe_apply()
7424 				     priv->plat->tx_queues_to_use,  in stmmac_fpe_apply()
7425 				     priv->plat->rx_queues_to_use,  in stmmac_fpe_apply()
7432 		if (netif_running(priv->dev))  in stmmac_fpe_apply()
7441 	struct stmmac_priv *priv = ctx->priv;  in stmmac_xdp_rx_timestamp()  local
7446 	if (!priv->hwts_rx_en)  in stmmac_xdp_rx_timestamp()
7450 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)  in stmmac_xdp_rx_timestamp()
7454 	if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) {  in stmmac_xdp_rx_timestamp()
7455 		stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns);  in stmmac_xdp_rx_timestamp()
7456 		ns -= priv->plat->cdc_error_adj;  in stmmac_xdp_rx_timestamp()
7483 	struct stmmac_priv *priv;  in stmmac_dvr_probe()  local
7494 	priv = netdev_priv(ndev);  in stmmac_dvr_probe()
7495 	priv->device = device;  in stmmac_dvr_probe()
7496 	priv->dev = ndev;  in stmmac_dvr_probe()
7499 		u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp);  in stmmac_dvr_probe()
7501 		u64_stats_init(&priv->xstats.txq_stats[i].q_syncp);  in stmmac_dvr_probe()
7502 		u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp);  in stmmac_dvr_probe()
7505 	priv->xstats.pcpu_stats =  in stmmac_dvr_probe()
7507 	if (!priv->xstats.pcpu_stats)  in stmmac_dvr_probe()
7511 	priv->pause = pause;  in stmmac_dvr_probe()
7512 	priv->plat = plat_dat;  in stmmac_dvr_probe()
7513 	priv->ioaddr = res->addr;  in stmmac_dvr_probe()
7514 	priv->dev->base_addr = (unsigned long)res->addr;  in stmmac_dvr_probe()
7515 	priv->plat->dma_cfg->multi_msi_en =  in stmmac_dvr_probe()
7516 		(priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN);  in stmmac_dvr_probe()
7518 	priv->dev->irq = res->irq;  in stmmac_dvr_probe()
7519 	priv->wol_irq = res->wol_irq;  in stmmac_dvr_probe()
7520 	priv->lpi_irq = res->lpi_irq;  in stmmac_dvr_probe()
7521 	priv->sfty_irq = res->sfty_irq;  in stmmac_dvr_probe()
7522 	priv->sfty_ce_irq = res->sfty_ce_irq;  in stmmac_dvr_probe()
7523 	priv->sfty_ue_irq = res->sfty_ue_irq;  in stmmac_dvr_probe()
7525 		priv->rx_irq[i] = res->rx_irq[i];  in stmmac_dvr_probe()
7527 		priv->tx_irq[i] = res->tx_irq[i];  in stmmac_dvr_probe()
7530 		eth_hw_addr_set(priv->dev, res->mac);  in stmmac_dvr_probe()
7532 	dev_set_drvdata(device, priv->dev);  in stmmac_dvr_probe()
7537 	priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL);  in stmmac_dvr_probe()
7538 	if (!priv->af_xdp_zc_qps)  in stmmac_dvr_probe()
7542 	priv->wq = create_singlethread_workqueue("stmmac_wq");  in stmmac_dvr_probe()
7543 	if (!priv->wq) {  in stmmac_dvr_probe()
7544 		dev_err(priv->device, "failed to create workqueue\n");  in stmmac_dvr_probe()
7549 	INIT_WORK(&priv->service_task, stmmac_service_task);  in stmmac_dvr_probe()
7555 		priv->plat->phy_addr = phyaddr;  in stmmac_dvr_probe()
7557 	if (priv->plat->stmmac_rst) {  in stmmac_dvr_probe()
7558 		ret = reset_control_assert(priv->plat->stmmac_rst);  in stmmac_dvr_probe()
7559 		reset_control_deassert(priv->plat->stmmac_rst);  in stmmac_dvr_probe()
7564 			reset_control_reset(priv->plat->stmmac_rst);  in stmmac_dvr_probe()
7567 	ret = reset_control_deassert(priv->plat->stmmac_ahb_rst);  in stmmac_dvr_probe()
7569 		dev_err(priv->device, "unable to bring out of ahb reset: %pe\n",  in stmmac_dvr_probe()
7576 	ret = stmmac_hw_init(priv);  in stmmac_dvr_probe()
7582 	if (priv->synopsys_id < DWMAC_CORE_5_20)  in stmmac_dvr_probe()
7583 		priv->plat->dma_cfg->dche = false;  in stmmac_dvr_probe()
7585 	stmmac_check_ether_addr(priv);  in stmmac_dvr_probe()
7597 	ret = stmmac_tc_init(priv, priv);  in stmmac_dvr_probe()
7602 	if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) {  in stmmac_dvr_probe()
7604 		if (priv->plat->has_gmac4)  in stmmac_dvr_probe()
7606 		priv->tso = true;  in stmmac_dvr_probe()
7607 		dev_info(priv->device, "TSO feature enabled\n");  in stmmac_dvr_probe()
7610 	if (priv->dma_cap.sphen &&  in stmmac_dvr_probe()
7611 	    !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) {  in stmmac_dvr_probe()
7613 		priv->sph_cap = true;  in stmmac_dvr_probe()
7614 		priv->sph = priv->sph_cap;  in stmmac_dvr_probe()
7615 		dev_info(priv->device, "SPH feature enabled\n");  in stmmac_dvr_probe()
7623 	if (priv->plat->host_dma_width)  in stmmac_dvr_probe()
7624 		priv->dma_cap.host_dma_width = priv->plat->host_dma_width;  in stmmac_dvr_probe()
7626 		priv->dma_cap.host_dma_width = priv->dma_cap.addr64;  in stmmac_dvr_probe()
7628 	if (priv->dma_cap.host_dma_width) {  in stmmac_dvr_probe()
7630 				DMA_BIT_MASK(priv->dma_cap.host_dma_width));  in stmmac_dvr_probe()
7632 			dev_info(priv->device, "Using %d/%d bits DMA host/device width\n",  in stmmac_dvr_probe()
7633 				 priv->dma_cap.host_dma_width, priv->dma_cap.addr64);  in stmmac_dvr_probe()
7640 				priv->plat->dma_cfg->eame = true;  in stmmac_dvr_probe()
7644 				dev_err(priv->device, "Failed to set DMA Mask\n");  in stmmac_dvr_probe()
7648 			priv->dma_cap.host_dma_width = 32;  in stmmac_dvr_probe()
7657 	if (priv->plat->has_gmac4) {  in stmmac_dvr_probe()
7659 		priv->hw->hw_vlan_en = true;  in stmmac_dvr_probe()
7661 	if (priv->dma_cap.vlhash) {  in stmmac_dvr_probe()
7665 	if (priv->dma_cap.vlins) {  in stmmac_dvr_probe()
7667 		if (priv->dma_cap.dvlan)  in stmmac_dvr_probe()
7671 	priv->msg_enable = netif_msg_init(debug, default_msg_level);  in stmmac_dvr_probe()
7673 	priv->xstats.threshold = tc;  in stmmac_dvr_probe()
7676 	rxq = priv->plat->rx_queues_to_use;  in stmmac_dvr_probe()
7677 	netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));  in stmmac_dvr_probe()
7678 	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)  in stmmac_dvr_probe()
7679 		priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);  in stmmac_dvr_probe()
7681 	if (priv->dma_cap.rssen && priv->plat->rss_en)  in stmmac_dvr_probe()
7688 	if (priv->plat->has_xgmac)  in stmmac_dvr_probe()
7690 	else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))  in stmmac_dvr_probe()
7697 	if ((priv->plat->maxmtu < ndev->max_mtu) &&  in stmmac_dvr_probe()
7698 	    (priv->plat->maxmtu >= ndev->min_mtu))  in stmmac_dvr_probe()
7699 		ndev->max_mtu = priv->plat->maxmtu;  in stmmac_dvr_probe()
7700 	else if (priv->plat->maxmtu < ndev->min_mtu)  in stmmac_dvr_probe()
7701 		dev_warn(priv->device,  in stmmac_dvr_probe()
7703 			 __func__, priv->plat->maxmtu);  in stmmac_dvr_probe()
7706 		priv->flow_ctrl = FLOW_AUTO;	/* RX/TX pause on */  in stmmac_dvr_probe()
7713 	mutex_init(&priv->lock);  in stmmac_dvr_probe()
7715 	priv->fpe_cfg.verify_retries = STMMAC_FPE_MM_MAX_VERIFY_RETRIES;  in stmmac_dvr_probe()
7716 	priv->fpe_cfg.verify_time = STMMAC_FPE_MM_MAX_VERIFY_TIME_MS;  in stmmac_dvr_probe()
7717 	priv->fpe_cfg.status = ETHTOOL_MM_VERIFY_STATUS_DISABLED;  in stmmac_dvr_probe()
7718 	timer_setup(&priv->fpe_cfg.verify_timer, stmmac_fpe_verify_timer, 0);  in stmmac_dvr_probe()
7719 	spin_lock_init(&priv->fpe_cfg.lock);  in stmmac_dvr_probe()
7727 	if (priv->plat->clk_csr >= 0)  in stmmac_dvr_probe()
7728 		priv->clk_csr = priv->plat->clk_csr;  in stmmac_dvr_probe()
7730 		stmmac_clk_csr_set(priv);  in stmmac_dvr_probe()
7732 	stmmac_check_pcs_mode(priv);  in stmmac_dvr_probe()
7741 		dev_err_probe(priv->device, ret,  in stmmac_dvr_probe()
7743 			      priv->plat->bus_id);  in stmmac_dvr_probe()
7747 	if (priv->plat->speed_mode_2500)  in stmmac_dvr_probe()
7748 		priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv);  in stmmac_dvr_probe()
7754 	ret = stmmac_phy_setup(priv);  in stmmac_dvr_probe()
7762 		dev_err(priv->device, "%s: ERROR %i registering the device\n",  in stmmac_dvr_probe()
7771 	if (priv->plat->dump_debug_regs)  in stmmac_dvr_probe()
7772 		priv->plat->dump_debug_regs(priv->plat->bsp_priv);  in stmmac_dvr_probe()
7782 	phylink_destroy(priv->phylink);  in stmmac_dvr_probe()
7790 	destroy_workqueue(priv->wq);  in stmmac_dvr_probe()
7792 	bitmap_free(priv->af_xdp_zc_qps);  in stmmac_dvr_probe()
7807 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_dvr_remove()  local
7809 	netdev_info(priv->dev, "%s: removing driver", __func__);  in stmmac_dvr_remove()
7813 	stmmac_stop_all_dma(priv);  in stmmac_dvr_remove()
7814 	stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_dvr_remove()
7820 	phylink_destroy(priv->phylink);  in stmmac_dvr_remove()
7821 	if (priv->plat->stmmac_rst)  in stmmac_dvr_remove()
7822 		reset_control_assert(priv->plat->stmmac_rst);  in stmmac_dvr_remove()
7823 	reset_control_assert(priv->plat->stmmac_ahb_rst);  in stmmac_dvr_remove()
7828 	destroy_workqueue(priv->wq);  in stmmac_dvr_remove()
7829 	mutex_destroy(&priv->lock);  in stmmac_dvr_remove()
7830 	bitmap_free(priv->af_xdp_zc_qps);  in stmmac_dvr_remove()
7847 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_suspend()  local
7853 	mutex_lock(&priv->lock);  in stmmac_suspend()
7857 	stmmac_disable_all_queues(priv);  in stmmac_suspend()
7859 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in stmmac_suspend()
7860 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in stmmac_suspend()
7862 	if (priv->eee_enabled) {  in stmmac_suspend()
7863 		priv->tx_path_in_lpi_mode = false;  in stmmac_suspend()
7864 		del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_suspend()
7868 	stmmac_stop_all_dma(priv);  in stmmac_suspend()
7870 	if (priv->plat->serdes_powerdown)  in stmmac_suspend()
7871 		priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);  in stmmac_suspend()
7874 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {  in stmmac_suspend()
7875 		stmmac_pmt(priv, priv->hw, priv->wolopts);  in stmmac_suspend()
7876 		priv->irq_wake = 1;  in stmmac_suspend()
7878 		stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_suspend()
7879 		pinctrl_pm_select_sleep_state(priv->device);  in stmmac_suspend()
7882 	mutex_unlock(&priv->lock);  in stmmac_suspend()
7885 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {  in stmmac_suspend()
7886 		phylink_suspend(priv->phylink, true);  in stmmac_suspend()
7888 		if (device_may_wakeup(priv->device))  in stmmac_suspend()
7889 			phylink_speed_down(priv->phylink, false);  in stmmac_suspend()
7890 		phylink_suspend(priv->phylink, false);  in stmmac_suspend()
7894 	if (priv->dma_cap.fpesel)  in stmmac_suspend()
7895 		timer_shutdown_sync(&priv->fpe_cfg.verify_timer);  in stmmac_suspend()
7897 	priv->speed = SPEED_UNKNOWN;  in stmmac_suspend()
7902 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_reset_rx_queue()  argument
7904 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_reset_rx_queue()
7910 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_reset_tx_queue()  argument
7912 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_reset_tx_queue()
7918 	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));  in stmmac_reset_tx_queue()
7925 static void stmmac_reset_queues_param(struct stmmac_priv *priv)  in stmmac_reset_queues_param()  argument
7927 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_reset_queues_param()
7928 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_reset_queues_param()
7932 		stmmac_reset_rx_queue(priv, queue);  in stmmac_reset_queues_param()
7935 		stmmac_reset_tx_queue(priv, queue);  in stmmac_reset_queues_param()
7947 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_resume()  local
7959 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {  in stmmac_resume()
7960 		mutex_lock(&priv->lock);  in stmmac_resume()
7961 		stmmac_pmt(priv, priv->hw, 0);  in stmmac_resume()
7962 		mutex_unlock(&priv->lock);  in stmmac_resume()
7963 		priv->irq_wake = 0;  in stmmac_resume()
7965 		pinctrl_pm_select_default_state(priv->device);  in stmmac_resume()
7967 		if (priv->mii)  in stmmac_resume()
7968 			stmmac_mdio_reset(priv->mii);  in stmmac_resume()
7971 	if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&  in stmmac_resume()
7972 	    priv->plat->serdes_powerup) {  in stmmac_resume()
7973 		ret = priv->plat->serdes_powerup(ndev,  in stmmac_resume()
7974 						 priv->plat->bsp_priv);  in stmmac_resume()
7981 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {  in stmmac_resume()
7982 		phylink_resume(priv->phylink);  in stmmac_resume()
7984 		phylink_resume(priv->phylink);  in stmmac_resume()
7985 		if (device_may_wakeup(priv->device))  in stmmac_resume()
7986 			phylink_speed_up(priv->phylink);  in stmmac_resume()
7991 	mutex_lock(&priv->lock);  in stmmac_resume()
7993 	stmmac_reset_queues_param(priv);  in stmmac_resume()
7995 	stmmac_free_tx_skbufs(priv);  in stmmac_resume()
7996 	stmmac_clear_descriptors(priv, &priv->dma_conf);  in stmmac_resume()
7999 	stmmac_init_coalesce(priv);  in stmmac_resume()
8002 	stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);  in stmmac_resume()
8004 	stmmac_enable_all_queues(priv);  in stmmac_resume()
8005 	stmmac_enable_all_dma_irq(priv);  in stmmac_resume()
8007 	mutex_unlock(&priv->lock);  in stmmac_resume()