ethernet/sfc/net_driver.h

1 /* SPDX-License-Identifier: GPL-2.0-only */
4  * Copyright 2005-2006 Fen Systems Ltd.
5  * Copyright 2005-2013 Solarflare Communications Inc.
63 /* Checksum generation is a per-queue option in hardware, so each
64  * queue visible to the networking core is backed by two hardware TX
65  * queues. */
83 /* Size of an RX scatter buffer.  Small enough to pack 2 into a 4K page,
86 #define EFX_RX_USR_BUF_SIZE	(2048 - 256)
89  * of every buffer.  Otherwise, we just need to ensure 4-byte
98 /* Non-standard XDP_PACKET_HEADROOM and tailroom to satisfy XDP_REDIRECT and
111  * struct efx_buffer - A general-purpose DMA buffer
126  * struct efx_tx_buffer - buffer state for a TX descriptor
127  * @skb: When @flags & %EFX_TX_BUF_SKB, the associated socket buffer to be
130  *	member is the associated buffer to drop a page reference on.
131  * @option: When @flags & %EFX_TX_BUF_OPTION, an EF10-specific option
137  * @unmap_len: Length of this fragment to unmap
164  * struct efx_tx_queue - An Efx TX queue
166  * This is a ring buffer of TX fragments.
167  * Since the TX completion path always executes on the same
178  * @label: Label for TX completion events.
179  *	Is our index within @channel->tx_queue array.
180  * @type: configuration type of this TX queue.  A bitmask of %EFX_TXQ_TYPE_* flags.
181  * @tso_version: Version of TSO in use for this queue.
184  * @core_txq: The networking core TX queue structure
186  * @cb_page: Array of pages of copy buffers.  Carved up according to
187  *	%EFX_TX_CB_ORDER into %EFX_TX_CB_SIZE-sized chunks.
190  * @piobuf: PIO buffer region for this TX queue (shared with its partner).
192  * @piobuf_offset: Buffer offset to be specified in PIO descriptors
195  * @xdp_tx: Is this an XDP tx queue?
200  *	only get the up-to-date value of @write_count if this
201  *	variable indicates that the queue is empty.  This is to
202  *	avoid cache-line ping-pong between the xmit path and the
204  * @merge_events: Number of TX merged completion events
205  * @completed_timestamp_major: Top part of the most recent tx timestamp.
206  * @completed_timestamp_minor: Low part of the most recent tx timestamp.
208  *	This is the number of buffers that have been added to the
211  *	This is the number of buffers that have been added to the
217  *	Filled in iff @efx->type->option_descriptors; only used for PIO.
221  *	only get the up-to-date value of read_count if this
222  *	variable indicates that the queue is full.  This is to
223  *	avoid cache-line ping-pong between the xmit path and the
230  * @pushes: Number of times the TX push feature has been used
231  * @pio_packets: Number of times the TX PIO feature has been used
232  * @xmit_pending: Are any packets waiting to be pushed to the NIC
233  * @cb_packets: Number of times the TX copybreak feature has been used
234  * @notify_count: Count of notified descriptors to the NIC
237  *	@read_count bitwise-added to %EFX_EMPTY_COUNT_VALID; otherwise 0.
282 	/* Statistics to supplement MAC stats */
292 #define EFX_TX_CB_SIZE	(1 << EFX_TX_CB_ORDER) - NET_IP_ALIGN
295  * struct efx_rx_buffer - An Efx RX data buffer
321  * struct efx_rx_page_state - Page-based rx buffer state
324  * Used to facilitate sharing dma mappings between recycled rx buffers
325  * and those passed up to the kernel.
336  * struct efx_rx_queue - An Efx RX queue
346  * @grant_credits: Posted RX descriptors need to be granted to the MAE with
349  * @added_count: Number of buffers added to the receive queue.
350  * @notified_count: Number of buffers given to NIC (<= @added_count).
351  * @granted_count: Number of buffers granted to the MAE (<= @notified_count).
355  * @page_ring: The ring to store DMA mapped pages for reuse.
356  * @page_add: Counter to calculate the write pointer for the recycle ring.
357  * @page_remove: Counter to calculate the read pointer for the recycle ring.
360  *      the kernel still held a reference to them.
367  * @min_fill: RX descriptor minimum non-zero fill level.
371  * @slow_fill: Timer used to defer efx_nic_generate_fill_event().
372  * @grant_work: workitem used to grant credits to the MAE if @grant_credits
407 	/* Statistics to supplement MAC stats */
421  * struct efx_channel - An Efx channel
423  * A channel comprises an event queue, at least one TX queue, at least
432  * @irq: IRQ number (MSI and MSI-X only)
441  * @event_test_cpu: Last CPU to handle interrupt or test event for this channel
448  * @rfs_expire_index: next accelerated RFS filter ID to check for expiry
460  * @n_skbuff_leaks: Count of skbuffs leaked due to RX overrun
461  * @n_rx_nodesc_trunc: Number of RX packets truncated and then dropped due to
465  * @n_rx_xdp_drops: Count of RX packets intentionally dropped due to XDP
466  * @n_rx_xdp_bad_drops: Count of RX packets dropped due to XDP errors
467  * @n_rx_xdp_tx: Count of RX packets retransmitted due to XDP
468  * @n_rx_xdp_redirect: Count of RX packets redirected to a different NIC by XDP
471  * @rx_pkt_n_frags: Number of fragments in next packet to be delivered by
473  * @rx_pkt_index: Ring index of first buffer for next packet to be delivered
477  * @tx_queue: TX queues for this channel
550  * struct efx_msi_context - Context for each MSI
556  * safe for the IRQ handler to access.
565  * struct efx_channel_type - distinguishes traffic and extra channels
566  * @handle_no_channel: Handle failure to allocate an extra channel
567  * @pre_probe: Set up extra state prior to initialisation
573  * @copy: Copy the channel state prior to reallocation.  May be %NULL if
575  * @receive_skb: Handle an skb ready to be passed to netif_receive_skb()
576  * @receive_raw: Handle an RX buffer ready to be passed to __efx_rx_packet()
577  * @want_txqs: Determine whether this channel should have TX queues
578  *	created.  If %NULL, TX queues are not created.
581  * @want_pio: Flag for whether PIO buffers should be linked to this
582  *	channel's TX queues.
611 	STRING_TABLE_LOOKUP((efx)->loopback_mode, efx_loopback_mode)
614 	/* Be careful if altering to correct macro below */
620 #define EFX_INT_MODE_USE_MSI(x) (((x)->interrupt_mode) <= EFX_INT_MODE_MSI)
627 	STATE_DISABLED,		/* device disabled due to hardware errors */
675 /* Pseudo bit-mask flow control field */
681  * struct efx_link_state - Current state of the link
683  * @fd: Link is full-duplex
697 	return left->up == right->up && left->fd == right->fd &&  in efx_link_state_equal()
698 		left->fc == right->fc && left->speed == right->speed;  in efx_link_state_equal()
702  * enum efx_phy_mode - PHY operating mode flags
704  * @PHY_MODE_TX_DISABLED: on with TX disabled
705  * @PHY_MODE_LOW_POWER: set to low power through MDIO
723  * struct efx_hw_stat_desc - Description of a hardware statistic
726  * @dma_width: Width in bits (0 for non-DMA statistics)
727  * @offset: Offset within stats (ignored for non-DMA statistics)
740  * struct efx_rss_context_priv - driver private data for an RSS context
743  * @rx_hash_udp_4tuple: UDP 4-tuple hashing enabled
751  * struct efx_rss_context - an RSS context
752  * @priv: hardware-specific state
764  * is used to test if filter does or will exist.
766 #define EFX_ARFS_FILTER_ID_PENDING	-1
767 #define EFX_ARFS_FILTER_ID_ERROR	-2
768 #define EFX_ARFS_FILTER_ID_REMOVING	-3
770  * struct efx_arfs_rule - record of an ARFS filter and its IDs
772  * @spec: details of the filter (used as key for hash table).  Use efx->type to
773  *	determine which member to use.
774  * @rxq_index: channel to which the filter will steer traffic.
775  * @arfs_id: filter ID which was returned to ARFS
793  * struct efx_async_filter_insertion - Request to asynchronously insert a filter
794  * @net_dev: Reference to the netdevice
795  * @spec: The filter to insert
798  * @flow_id: Identifies the kernel-side flow for which this request was made
815 	EFX_XDP_TX_QUEUES_BORROWED	/* queues borrowed from net stack */
821  * struct efx_nic - an Efx NIC
839  * @vi_stride: step between per-VI registers / memory regions
843  * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues
845  * @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues
846  * @irq_rx_moderation_us: IRQ moderation time for RX event queues
850  * @tx_queue: TX DMA queues
851  * @rx_queue: RX DMA queues
854  * @extra_channel_types: Types of extra (non-traffic) channels that
858  * @xdp_tx_queues: Array of pointers to tx queues used for XDP transmit.
859  * @xdp_txq_queues_mode: XDP TX queues sharing strategy.
860  * @rxq_entries: Size of receive queues requested by user.
861  * @txq_entries: Size of transmit queues requested by user.
862  * @txq_stop_thresh: TX queue fill level at or above which we stop it.
863  * @txq_wake_thresh: TX queue fill level at or below which we wake it.
864  * @tx_dc_base: Base qword address in SRAM of TX queue descriptor caches
867  * @n_channels: Number of channels in use
868  * @n_rx_channels: Number of channels used for RX (= number of RX queues)
869  * @n_tx_channels: Number of channels used for TX
870  * @n_extra_tx_channels: Number of extra channels with TX queues
871  * @tx_queues_per_channel: number of TX queues probed on each channel
872  * @n_xdp_channels: Number of channels used for XDP TX
873  * @xdp_channel_offset: Offset of zeroth channel used for XPD TX.
874  * @xdp_tx_per_channel: Max number of TX queues on an XDP TX channel.
875  * @rx_ip_align: RX DMA address offset to have IP header aligned in
880  *	for use in sk_buff::truesize
887  *	(valid only if channel->sync_timestamps_enabled; always negative)
893  * @must_realloc_vis: Flag: VIs have yet to be reallocated after MC reboot
894  * @irq_soft_enabled: Are IRQs soft-enabled? If not, IRQ handler will
899  * @selftest_work: Work item for asynchronous self-test
900  * @mtd_list: List of MTDs attached to the NIC
902  * @mcdi: Management-Controller-to-Driver Interface state
907  *	efx_mac_work() with kernel interfaces. Safe to read under any
909  *	be held to modify it.
917  * @phy_data: PHY private data (including PHY-specific stats)
927  * @fc_disable: When non-zero flow control is disabled. Typically used to
933  * @loopback_selftest: Offline self-test private state
936  * @filter_state: Architecture-dependent filter table state
938  * @rps_slot_map: bitmap of in-flight entries in @rps_slot
944  * @active_queues: Count of RX and TX queues that haven't been flushed and drained.
945  * @rxq_flush_pending: Count of number of receive queues that need to be flushed.
948  *	completed (either success or failure). Not used when MCDI is used to
949  *	flush receive queues.
950  * @flush_wq: wait queue used by efx_nic_flush_queues() to wait for flush completions.
951  * @vf_count: Number of VFs intended to be enabled.
959  * @xdp_rxq_info_failed: Have any of the rx queues failed to initialise their
964  * @devlink: reference to devlink structure owned by this device
967  * @reg_base: Offset from the start of the bar to the function control window.
970  * @last_irq_cpu: Last CPU to handle a possible test interrupt.  This
971  *	field is used by efx_test_interrupts() to verify that an
975  * @n_rx_noskb_drops: Count of RX packets dropped due to failure to allocate an skb
1111 	 * so we want them next to each other.
1164  * struct efx_probe_data - State after hardware probe
1178 	return &probe_data->efx;  in efx_netdev_priv()
1183 	return efx->net_dev->reg_state == NETREG_REGISTERED;  in efx_dev_registered()
1188 	return efx->port_num;  in efx_port_num()
1206  * struct efx_nic_type - Efx device type definition
1216  * @map_reset_reason: Map ethtool reset reason to a reset method
1217  * @map_reset_flags: Map ethtool reset flags to a reset method, if possible
1223  * @fini_dmaq: Flush and finalise DMA queues (RX and TX queues)
1236  * @reconfigure_port: Push loopback/power/txdis changes to the MAC and PHY
1237  * @prepare_enable_fc_tx: Prepare MAC to enable pause frame TX (may be %NULL)
1239  *	to the hardware.  Serialised by the mac_lock.
1242  * @set_wol: Push WoL configuration to the NIC
1245  * @test_chip: Test registers. This is expected to reset the NIC.
1248  *	The SDU length may be any value from 0 up to the protocol-
1249  *	defined maximum, but its buffer will be padded to a multiple
1254  *	will be padded so it is safe to round up.
1261  * @irq_disable_non_ev: Disable non-event IRQs on the NIC.  Each event
1264  *	a pointer to the &struct efx_msi_context for the channel.
1266  *	is a pointer to the &struct efx_nic.
1267  * @tx_probe: Allocate resources for TX queue (and select TXQ type)
1268  * @tx_init: Initialise TX queue on the NIC
1269  * @tx_remove: Free resources for TX queue
1270  * @tx_write: Write TX descriptors and doorbell
1271  * @tx_enqueue: Add an SKB to TX queue
1272  * @rx_push_rss_config: Write RSS hash key and indirection table to the NIC
1275  *	user RSS context to the NIC
1289  * @ev_process: Process events for a queue, up to the given NAPI quota
1295  * @filter_update_rx_scatter: Update filters after change to rx scatter setting
1300  *	equal to the given priority and is not %EFX_FILTER_PRI_AUTO
1301  * @filter_count_rx_used: Get the number of filters in use at a given priority
1312  * @mtd_write: Write to an MTD partition
1313  * @mtd_sync: Wait for write-back to complete on MTD partition.  This
1316  * @ptp_write_host_time: Send host time to MC as part of sync protocol
1324  * @tso_versions: Returns mask of firmware-assisted TSO versions supported.
1326  * @udp_tnl_push_ports: Push the list of UDP tunnel ports to the NIC if required.
1328  * @print_additional_fwver: Dump NIC-specific additional FW version info
1332  * @txd_ptr_tbl_base: TX descriptor ring base address
1336  * @evq_rptr_tbl_base: Event queue read-pointer table base address
1342  * @can_rx_scatter: NIC is able to scatter packets to multiple buffers
1343  * @always_rx_scatter: NIC will always scatter packets to multiple buffers
1344  * @option_descriptors: NIC supports TX option descriptors
1539 	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_channels);  in efx_get_channel()
1540 	return efx->channel[index];  in efx_get_channel()
1545 	for (_channel = (_efx)->channel[0];				\
1547 	     _channel = (_channel->channel + 1 < (_efx)->n_channels) ?	\
1548 		     (_efx)->channel[_channel->channel + 1] : NULL)
1552 	for (_channel = (_efx)->channel[(_efx)->n_channels - 1];	\
1554 	     _channel = _channel->channel ?				\
1555 		     (_efx)->channel[_channel->channel - 1] : NULL)
1560 	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_tx_channels);  in efx_get_tx_channel()
1561 	return efx->channel[efx->tx_channel_offset + index];  in efx_get_tx_channel()
1567 	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_xdp_channels);  in efx_get_xdp_channel()
1568 	return efx->channel[efx->xdp_channel_offset + index];  in efx_get_xdp_channel()
1573 	return channel->channel - channel->efx->xdp_channel_offset <  in efx_channel_is_xdp_tx()
1574 	       channel->efx->n_xdp_channels;  in efx_channel_is_xdp_tx()
1579 	return channel && channel->channel >= channel->efx->tx_channel_offset;  in efx_channel_has_tx_queues()
1585 		return channel->efx->xdp_tx_per_channel;  in efx_channel_num_tx_queues()
1586 	return channel->efx->tx_queues_per_channel;  in efx_channel_num_tx_queues()
1593 	return channel->tx_queue_by_type[type];  in efx_channel_get_tx_queue()
1604 /* Iterate over all TX queues belonging to a channel */
1609 		for (_tx_queue = (_channel)->tx_queue;			\
1610 		     _tx_queue < (_channel)->tx_queue +			\
1616 	return channel->rx_queue.core_index >= 0;  in efx_channel_has_rx_queue()
1623 	return &channel->rx_queue;  in efx_channel_get_rx_queue()
1626 /* Iterate over all RX queues belonging to a channel */
1631 		for (_rx_queue = &(_channel)->rx_queue;			\
1643 	return efx_rx_queue_channel(rx_queue)->channel;  in efx_rx_queue_index()
1646 /* Returns a pointer to the specified receive buffer in the RX
1652 	return &rx_queue->buffer[index];  in efx_rx_buffer()
1658 	if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask)))  in efx_rx_buf_next()
1665  * EFX_MAX_FRAME_LEN - calculate maximum frame length
1668  * given MTU.  The frame length will be equal to the MTU plus a
1673  * The 10G MAC requires 8-byte alignment on the frame
1674  * length, so we round up to the nearest 8.
1676  * Re-clocking by the XGXS on RX can reduce an IPG to 32 bits (half an
1687 	return skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP;  in efx_xmit_with_hwtstamp()
1691 	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;  in efx_xmit_hwtstamp_pending()
1694 /* Get the max fill level of the TX queues on this channel */
1703 				 tx_queue->insert_count - tx_queue->read_count);  in efx_channel_tx_fill_level()
1717 				 tx_queue->insert_count - tx_queue->old_read_count);  in efx_channel_tx_old_fill_level()
1729 	const struct net_device *net_dev = efx->net_dev;  in efx_supported_features()
1731 	return net_dev->features | net_dev->hw_features;  in efx_supported_features()
1734 /* Get the current TX queue insert index. */
1738 	return tx_queue->insert_count & tx_queue->ptr_mask;  in efx_tx_queue_get_insert_index()
1741 /* Get a TX buffer. */
1745 	return &tx_queue->buffer[efx_tx_queue_get_insert_index(tx_queue)];  in __efx_tx_queue_get_insert_buffer()
1748 /* Get a TX buffer, checking it's not currently in use. */
1755 	EFX_WARN_ON_ONCE_PARANOID(buffer->len);  in efx_tx_queue_get_insert_buffer()
1756 	EFX_WARN_ON_ONCE_PARANOID(buffer->flags);  in efx_tx_queue_get_insert_buffer()
1757 	EFX_WARN_ON_ONCE_PARANOID(buffer->unmap_len);  in efx_tx_queue_get_insert_buffer()