1  /* SPDX-License-Identifier: GPL-2.0-or-later */
2  /*
3   * INET		An implementation of the TCP/IP protocol suite for the LINUX
4   *		operating system.  INET is implemented using the  BSD Socket
5   *		interface as the means of communication with the user level.
6   *
7   *		Definitions for the Interfaces handler.
8   *
9   * Version:	@(#)dev.h	1.0.10	08/12/93
10   *
11   * Authors:	Ross Biro
12   *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13   *		Corey Minyard <wf-rch!minyard@relay.EU.net>
14   *		Donald J. Becker, <becker@cesdis.gsfc.nasa.gov>
15   *		Alan Cox, <alan@lxorguk.ukuu.org.uk>
16   *		Bjorn Ekwall. <bj0rn@blox.se>
17   *              Pekka Riikonen <priikone@poseidon.pspt.fi>
18   *
19   *		Moved to /usr/include/linux for NET3
20   */
21  #ifndef _LINUX_NETDEVICE_H
22  #define _LINUX_NETDEVICE_H
23  
24  #include <linux/timer.h>
25  #include <linux/bug.h>
26  #include <linux/delay.h>
27  #include <linux/atomic.h>
28  #include <linux/prefetch.h>
29  #include <asm/cache.h>
30  #include <asm/byteorder.h>
31  #include <asm/local.h>
32  
33  #include <linux/percpu.h>
34  #include <linux/rculist.h>
35  #include <linux/workqueue.h>
36  #include <linux/dynamic_queue_limits.h>
37  
38  #include <net/net_namespace.h>
39  #ifdef CONFIG_DCB
40  #include <net/dcbnl.h>
41  #endif
42  #include <net/netprio_cgroup.h>
43  #include <linux/netdev_features.h>
44  #include <linux/neighbour.h>
45  #include <linux/netdevice_xmit.h>
46  #include <uapi/linux/netdevice.h>
47  #include <uapi/linux/if_bonding.h>
48  #include <uapi/linux/pkt_cls.h>
49  #include <uapi/linux/netdev.h>
50  #include <linux/hashtable.h>
51  #include <linux/rbtree.h>
52  #include <net/net_trackers.h>
53  #include <net/net_debug.h>
54  #include <net/dropreason-core.h>
55  
56  struct netpoll_info;
57  struct device;
58  struct ethtool_ops;
59  struct kernel_hwtstamp_config;
60  struct phy_device;
61  struct dsa_port;
62  struct ip_tunnel_parm_kern;
63  struct macsec_context;
64  struct macsec_ops;
65  struct netdev_name_node;
66  struct sd_flow_limit;
67  struct sfp_bus;
68  /* 802.11 specific */
69  struct wireless_dev;
70  /* 802.15.4 specific */
71  struct wpan_dev;
72  struct mpls_dev;
73  /* UDP Tunnel offloads */
74  struct udp_tunnel_info;
75  struct udp_tunnel_nic_info;
76  struct udp_tunnel_nic;
77  struct bpf_prog;
78  struct xdp_buff;
79  struct xdp_frame;
80  struct xdp_metadata_ops;
81  struct xdp_md;
82  struct ethtool_netdev_state;
83  struct phy_link_topology;
84  
85  typedef u32 xdp_features_t;
86  
87  void synchronize_net(void);
88  void netdev_set_default_ethtool_ops(struct net_device *dev,
89  				    const struct ethtool_ops *ops);
90  void netdev_sw_irq_coalesce_default_on(struct net_device *dev);
91  
92  /* Backlog congestion levels */
93  #define NET_RX_SUCCESS		0	/* keep 'em coming, baby */
94  #define NET_RX_DROP		1	/* packet dropped */
95  
96  #define MAX_NEST_DEV 8
97  
98  /*
99   * Transmit return codes: transmit return codes originate from three different
100   * namespaces:
101   *
102   * - qdisc return codes
103   * - driver transmit return codes
104   * - errno values
105   *
106   * Drivers are allowed to return any one of those in their hard_start_xmit()
107   * function. Real network devices commonly used with qdiscs should only return
108   * the driver transmit return codes though - when qdiscs are used, the actual
109   * transmission happens asynchronously, so the value is not propagated to
110   * higher layers. Virtual network devices transmit synchronously; in this case
111   * the driver transmit return codes are consumed by dev_queue_xmit(), and all
112   * others are propagated to higher layers.
113   */
114  
115  /* qdisc ->enqueue() return codes. */
116  #define NET_XMIT_SUCCESS	0x00
117  #define NET_XMIT_DROP		0x01	/* skb dropped			*/
118  #define NET_XMIT_CN		0x02	/* congestion notification	*/
119  #define NET_XMIT_MASK		0x0f	/* qdisc flags in net/sch_generic.h */
120  
121  /* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
122   * indicates that the device will soon be dropping packets, or already drops
123   * some packets of the same priority; prompting us to send less aggressively. */
124  #define net_xmit_eval(e)	((e) == NET_XMIT_CN ? 0 : (e))
125  #define net_xmit_errno(e)	((e) != NET_XMIT_CN ? -ENOBUFS : 0)
126  
127  /* Driver transmit return codes */
128  #define NETDEV_TX_MASK		0xf0
129  
130  enum netdev_tx {
131  	__NETDEV_TX_MIN	 = INT_MIN,	/* make sure enum is signed */
132  	NETDEV_TX_OK	 = 0x00,	/* driver took care of packet */
133  	NETDEV_TX_BUSY	 = 0x10,	/* driver tx path was busy*/
134  };
135  typedef enum netdev_tx netdev_tx_t;
136  
137  /*
138   * Current order: NETDEV_TX_MASK > NET_XMIT_MASK >= 0 is significant;
139   * hard_start_xmit() return < NET_XMIT_MASK means skb was consumed.
140   */
dev_xmit_complete(int rc)141  static inline bool dev_xmit_complete(int rc)
142  {
143  	/*
144  	 * Positive cases with an skb consumed by a driver:
145  	 * - successful transmission (rc == NETDEV_TX_OK)
146  	 * - error while transmitting (rc < 0)
147  	 * - error while queueing to a different device (rc & NET_XMIT_MASK)
148  	 */
149  	if (likely(rc < NET_XMIT_MASK))
150  		return true;
151  
152  	return false;
153  }
154  
155  /*
156   *	Compute the worst-case header length according to the protocols
157   *	used.
158   */
159  
160  #if defined(CONFIG_HYPERV_NET)
161  # define LL_MAX_HEADER 128
162  #elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
163  # if defined(CONFIG_MAC80211_MESH)
164  #  define LL_MAX_HEADER 128
165  # else
166  #  define LL_MAX_HEADER 96
167  # endif
168  #else
169  # define LL_MAX_HEADER 32
170  #endif
171  
172  #if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
173      !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
174  #define MAX_HEADER LL_MAX_HEADER
175  #else
176  #define MAX_HEADER (LL_MAX_HEADER + 48)
177  #endif
178  
179  /*
180   *	Old network device statistics. Fields are native words
181   *	(unsigned long) so they can be read and written atomically.
182   */
183  
184  #define NET_DEV_STAT(FIELD)			\
185  	union {					\
186  		unsigned long FIELD;		\
187  		atomic_long_t __##FIELD;	\
188  	}
189  
190  struct net_device_stats {
191  	NET_DEV_STAT(rx_packets);
192  	NET_DEV_STAT(tx_packets);
193  	NET_DEV_STAT(rx_bytes);
194  	NET_DEV_STAT(tx_bytes);
195  	NET_DEV_STAT(rx_errors);
196  	NET_DEV_STAT(tx_errors);
197  	NET_DEV_STAT(rx_dropped);
198  	NET_DEV_STAT(tx_dropped);
199  	NET_DEV_STAT(multicast);
200  	NET_DEV_STAT(collisions);
201  	NET_DEV_STAT(rx_length_errors);
202  	NET_DEV_STAT(rx_over_errors);
203  	NET_DEV_STAT(rx_crc_errors);
204  	NET_DEV_STAT(rx_frame_errors);
205  	NET_DEV_STAT(rx_fifo_errors);
206  	NET_DEV_STAT(rx_missed_errors);
207  	NET_DEV_STAT(tx_aborted_errors);
208  	NET_DEV_STAT(tx_carrier_errors);
209  	NET_DEV_STAT(tx_fifo_errors);
210  	NET_DEV_STAT(tx_heartbeat_errors);
211  	NET_DEV_STAT(tx_window_errors);
212  	NET_DEV_STAT(rx_compressed);
213  	NET_DEV_STAT(tx_compressed);
214  };
215  #undef NET_DEV_STAT
216  
217  /* per-cpu stats, allocated on demand.
218   * Try to fit them in a single cache line, for dev_get_stats() sake.
219   */
220  struct net_device_core_stats {
221  	unsigned long	rx_dropped;
222  	unsigned long	tx_dropped;
223  	unsigned long	rx_nohandler;
224  	unsigned long	rx_otherhost_dropped;
225  } __aligned(4 * sizeof(unsigned long));
226  
227  #include <linux/cache.h>
228  #include <linux/skbuff.h>
229  
230  struct neighbour;
231  struct neigh_parms;
232  struct sk_buff;
233  
234  struct netdev_hw_addr {
235  	struct list_head	list;
236  	struct rb_node		node;
237  	unsigned char		addr[MAX_ADDR_LEN];
238  	unsigned char		type;
239  #define NETDEV_HW_ADDR_T_LAN		1
240  #define NETDEV_HW_ADDR_T_SAN		2
241  #define NETDEV_HW_ADDR_T_UNICAST	3
242  #define NETDEV_HW_ADDR_T_MULTICAST	4
243  	bool			global_use;
244  	int			sync_cnt;
245  	int			refcount;
246  	int			synced;
247  	struct rcu_head		rcu_head;
248  };
249  
250  struct netdev_hw_addr_list {
251  	struct list_head	list;
252  	int			count;
253  
254  	/* Auxiliary tree for faster lookup on addition and deletion */
255  	struct rb_root		tree;
256  };
257  
258  #define netdev_hw_addr_list_count(l) ((l)->count)
259  #define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
260  #define netdev_hw_addr_list_for_each(ha, l) \
261  	list_for_each_entry(ha, &(l)->list, list)
262  
263  #define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
264  #define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
265  #define netdev_for_each_uc_addr(ha, dev) \
266  	netdev_hw_addr_list_for_each(ha, &(dev)->uc)
267  #define netdev_for_each_synced_uc_addr(_ha, _dev) \
268  	netdev_for_each_uc_addr((_ha), (_dev)) \
269  		if ((_ha)->sync_cnt)
270  
271  #define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
272  #define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
273  #define netdev_for_each_mc_addr(ha, dev) \
274  	netdev_hw_addr_list_for_each(ha, &(dev)->mc)
275  #define netdev_for_each_synced_mc_addr(_ha, _dev) \
276  	netdev_for_each_mc_addr((_ha), (_dev)) \
277  		if ((_ha)->sync_cnt)
278  
279  struct hh_cache {
280  	unsigned int	hh_len;
281  	seqlock_t	hh_lock;
282  
283  	/* cached hardware header; allow for machine alignment needs.        */
284  #define HH_DATA_MOD	16
285  #define HH_DATA_OFF(__len) \
286  	(HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
287  #define HH_DATA_ALIGN(__len) \
288  	(((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
289  	unsigned long	hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
290  };
291  
292  /* Reserve HH_DATA_MOD byte-aligned hard_header_len, but at least that much.
293   * Alternative is:
294   *   dev->hard_header_len ? (dev->hard_header_len +
295   *                           (HH_DATA_MOD - 1)) & ~(HH_DATA_MOD - 1) : 0
296   *
297   * We could use other alignment values, but we must maintain the
298   * relationship HH alignment <= LL alignment.
299   */
300  #define LL_RESERVED_SPACE(dev) \
301  	((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom)) \
302  	  & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
303  #define LL_RESERVED_SPACE_EXTRA(dev,extra) \
304  	((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom) + (extra)) \
305  	  & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
306  
307  struct header_ops {
308  	int	(*create) (struct sk_buff *skb, struct net_device *dev,
309  			   unsigned short type, const void *daddr,
310  			   const void *saddr, unsigned int len);
311  	int	(*parse)(const struct sk_buff *skb, unsigned char *haddr);
312  	int	(*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
313  	void	(*cache_update)(struct hh_cache *hh,
314  				const struct net_device *dev,
315  				const unsigned char *haddr);
316  	bool	(*validate)(const char *ll_header, unsigned int len);
317  	__be16	(*parse_protocol)(const struct sk_buff *skb);
318  };
319  
320  /* These flag bits are private to the generic network queueing
321   * layer; they may not be explicitly referenced by any other
322   * code.
323   */
324  
325  enum netdev_state_t {
326  	__LINK_STATE_START,
327  	__LINK_STATE_PRESENT,
328  	__LINK_STATE_NOCARRIER,
329  	__LINK_STATE_LINKWATCH_PENDING,
330  	__LINK_STATE_DORMANT,
331  	__LINK_STATE_TESTING,
332  };
333  
334  struct gro_list {
335  	struct list_head	list;
336  	int			count;
337  };
338  
339  /*
340   * size of gro hash buckets, must less than bit number of
341   * napi_struct::gro_bitmask
342   */
343  #define GRO_HASH_BUCKETS	8
344  
345  /*
346   * Structure for NAPI scheduling similar to tasklet but with weighting
347   */
348  struct napi_struct {
349  	/* The poll_list must only be managed by the entity which
350  	 * changes the state of the NAPI_STATE_SCHED bit.  This means
351  	 * whoever atomically sets that bit can add this napi_struct
352  	 * to the per-CPU poll_list, and whoever clears that bit
353  	 * can remove from the list right before clearing the bit.
354  	 */
355  	struct list_head	poll_list;
356  
357  	unsigned long		state;
358  	int			weight;
359  	u32			defer_hard_irqs_count;
360  	unsigned long		gro_bitmask;
361  	int			(*poll)(struct napi_struct *, int);
362  #ifdef CONFIG_NETPOLL
363  	/* CPU actively polling if netpoll is configured */
364  	int			poll_owner;
365  #endif
366  	/* CPU on which NAPI has been scheduled for processing */
367  	int			list_owner;
368  	struct net_device	*dev;
369  	struct gro_list		gro_hash[GRO_HASH_BUCKETS];
370  	struct sk_buff		*skb;
371  	struct list_head	rx_list; /* Pending GRO_NORMAL skbs */
372  	int			rx_count; /* length of rx_list */
373  	unsigned int		napi_id;
374  	struct hrtimer		timer;
375  	struct task_struct	*thread;
376  	/* control-path-only fields follow */
377  	struct list_head	dev_list;
378  	struct hlist_node	napi_hash_node;
379  	int			irq;
380  };
381  
382  enum {
383  	NAPI_STATE_SCHED,		/* Poll is scheduled */
384  	NAPI_STATE_MISSED,		/* reschedule a napi */
385  	NAPI_STATE_DISABLE,		/* Disable pending */
386  	NAPI_STATE_NPSVC,		/* Netpoll - don't dequeue from poll_list */
387  	NAPI_STATE_LISTED,		/* NAPI added to system lists */
388  	NAPI_STATE_NO_BUSY_POLL,	/* Do not add in napi_hash, no busy polling */
389  	NAPI_STATE_IN_BUSY_POLL,	/* sk_busy_loop() owns this NAPI */
390  	NAPI_STATE_PREFER_BUSY_POLL,	/* prefer busy-polling over softirq processing*/
391  	NAPI_STATE_THREADED,		/* The poll is performed inside its own thread*/
392  	NAPI_STATE_SCHED_THREADED,	/* Napi is currently scheduled in threaded mode */
393  };
394  
395  enum {
396  	NAPIF_STATE_SCHED		= BIT(NAPI_STATE_SCHED),
397  	NAPIF_STATE_MISSED		= BIT(NAPI_STATE_MISSED),
398  	NAPIF_STATE_DISABLE		= BIT(NAPI_STATE_DISABLE),
399  	NAPIF_STATE_NPSVC		= BIT(NAPI_STATE_NPSVC),
400  	NAPIF_STATE_LISTED		= BIT(NAPI_STATE_LISTED),
401  	NAPIF_STATE_NO_BUSY_POLL	= BIT(NAPI_STATE_NO_BUSY_POLL),
402  	NAPIF_STATE_IN_BUSY_POLL	= BIT(NAPI_STATE_IN_BUSY_POLL),
403  	NAPIF_STATE_PREFER_BUSY_POLL	= BIT(NAPI_STATE_PREFER_BUSY_POLL),
404  	NAPIF_STATE_THREADED		= BIT(NAPI_STATE_THREADED),
405  	NAPIF_STATE_SCHED_THREADED	= BIT(NAPI_STATE_SCHED_THREADED),
406  };
407  
408  enum gro_result {
409  	GRO_MERGED,
410  	GRO_MERGED_FREE,
411  	GRO_HELD,
412  	GRO_NORMAL,
413  	GRO_CONSUMED,
414  };
415  typedef enum gro_result gro_result_t;
416  
417  /*
418   * enum rx_handler_result - Possible return values for rx_handlers.
419   * @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
420   * further.
421   * @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
422   * case skb->dev was changed by rx_handler.
423   * @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
424   * @RX_HANDLER_PASS: Do nothing, pass the skb as if no rx_handler was called.
425   *
426   * rx_handlers are functions called from inside __netif_receive_skb(), to do
427   * special processing of the skb, prior to delivery to protocol handlers.
428   *
429   * Currently, a net_device can only have a single rx_handler registered. Trying
430   * to register a second rx_handler will return -EBUSY.
431   *
432   * To register a rx_handler on a net_device, use netdev_rx_handler_register().
433   * To unregister a rx_handler on a net_device, use
434   * netdev_rx_handler_unregister().
435   *
436   * Upon return, rx_handler is expected to tell __netif_receive_skb() what to
437   * do with the skb.
438   *
439   * If the rx_handler consumed the skb in some way, it should return
440   * RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
441   * the skb to be delivered in some other way.
442   *
443   * If the rx_handler changed skb->dev, to divert the skb to another
444   * net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
445   * new device will be called if it exists.
446   *
447   * If the rx_handler decides the skb should be ignored, it should return
448   * RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
449   * are registered on exact device (ptype->dev == skb->dev).
450   *
451   * If the rx_handler didn't change skb->dev, but wants the skb to be normally
452   * delivered, it should return RX_HANDLER_PASS.
453   *
454   * A device without a registered rx_handler will behave as if rx_handler
455   * returned RX_HANDLER_PASS.
456   */
457  
458  enum rx_handler_result {
459  	RX_HANDLER_CONSUMED,
460  	RX_HANDLER_ANOTHER,
461  	RX_HANDLER_EXACT,
462  	RX_HANDLER_PASS,
463  };
464  typedef enum rx_handler_result rx_handler_result_t;
465  typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
466  
467  void __napi_schedule(struct napi_struct *n);
468  void __napi_schedule_irqoff(struct napi_struct *n);
469  
napi_disable_pending(struct napi_struct * n)470  static inline bool napi_disable_pending(struct napi_struct *n)
471  {
472  	return test_bit(NAPI_STATE_DISABLE, &n->state);
473  }
474  
napi_prefer_busy_poll(struct napi_struct * n)475  static inline bool napi_prefer_busy_poll(struct napi_struct *n)
476  {
477  	return test_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
478  }
479  
480  /**
481   * napi_is_scheduled - test if NAPI is scheduled
482   * @n: NAPI context
483   *
484   * This check is "best-effort". With no locking implemented,
485   * a NAPI can be scheduled or terminate right after this check
486   * and produce not precise results.
487   *
488   * NAPI_STATE_SCHED is an internal state, napi_is_scheduled
489   * should not be used normally and napi_schedule should be
490   * used instead.
491   *
492   * Use only if the driver really needs to check if a NAPI
493   * is scheduled for example in the context of delayed timer
494   * that can be skipped if a NAPI is already scheduled.
495   *
496   * Return True if NAPI is scheduled, False otherwise.
497   */
napi_is_scheduled(struct napi_struct * n)498  static inline bool napi_is_scheduled(struct napi_struct *n)
499  {
500  	return test_bit(NAPI_STATE_SCHED, &n->state);
501  }
502  
503  bool napi_schedule_prep(struct napi_struct *n);
504  
505  /**
506   *	napi_schedule - schedule NAPI poll
507   *	@n: NAPI context
508   *
509   * Schedule NAPI poll routine to be called if it is not already
510   * running.
511   * Return true if we schedule a NAPI or false if not.
512   * Refer to napi_schedule_prep() for additional reason on why
513   * a NAPI might not be scheduled.
514   */
napi_schedule(struct napi_struct * n)515  static inline bool napi_schedule(struct napi_struct *n)
516  {
517  	if (napi_schedule_prep(n)) {
518  		__napi_schedule(n);
519  		return true;
520  	}
521  
522  	return false;
523  }
524  
525  /**
526   *	napi_schedule_irqoff - schedule NAPI poll
527   *	@n: NAPI context
528   *
529   * Variant of napi_schedule(), assuming hard irqs are masked.
530   */
napi_schedule_irqoff(struct napi_struct * n)531  static inline void napi_schedule_irqoff(struct napi_struct *n)
532  {
533  	if (napi_schedule_prep(n))
534  		__napi_schedule_irqoff(n);
535  }
536  
537  /**
538   * napi_complete_done - NAPI processing complete
539   * @n: NAPI context
540   * @work_done: number of packets processed
541   *
542   * Mark NAPI processing as complete. Should only be called if poll budget
543   * has not been completely consumed.
544   * Prefer over napi_complete().
545   * Return false if device should avoid rearming interrupts.
546   */
547  bool napi_complete_done(struct napi_struct *n, int work_done);
548  
napi_complete(struct napi_struct * n)549  static inline bool napi_complete(struct napi_struct *n)
550  {
551  	return napi_complete_done(n, 0);
552  }
553  
554  int dev_set_threaded(struct net_device *dev, bool threaded);
555  
556  /**
557   *	napi_disable - prevent NAPI from scheduling
558   *	@n: NAPI context
559   *
560   * Stop NAPI from being scheduled on this context.
561   * Waits till any outstanding processing completes.
562   */
563  void napi_disable(struct napi_struct *n);
564  
565  void napi_enable(struct napi_struct *n);
566  
567  /**
568   *	napi_synchronize - wait until NAPI is not running
569   *	@n: NAPI context
570   *
571   * Wait until NAPI is done being scheduled on this context.
572   * Waits till any outstanding processing completes but
573   * does not disable future activations.
574   */
napi_synchronize(const struct napi_struct * n)575  static inline void napi_synchronize(const struct napi_struct *n)
576  {
577  	if (IS_ENABLED(CONFIG_SMP))
578  		while (test_bit(NAPI_STATE_SCHED, &n->state))
579  			msleep(1);
580  	else
581  		barrier();
582  }
583  
584  /**
585   *	napi_if_scheduled_mark_missed - if napi is running, set the
586   *	NAPIF_STATE_MISSED
587   *	@n: NAPI context
588   *
589   * If napi is running, set the NAPIF_STATE_MISSED, and return true if
590   * NAPI is scheduled.
591   **/
napi_if_scheduled_mark_missed(struct napi_struct * n)592  static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
593  {
594  	unsigned long val, new;
595  
596  	val = READ_ONCE(n->state);
597  	do {
598  		if (val & NAPIF_STATE_DISABLE)
599  			return true;
600  
601  		if (!(val & NAPIF_STATE_SCHED))
602  			return false;
603  
604  		new = val | NAPIF_STATE_MISSED;
605  	} while (!try_cmpxchg(&n->state, &val, new));
606  
607  	return true;
608  }
609  
610  enum netdev_queue_state_t {
611  	__QUEUE_STATE_DRV_XOFF,
612  	__QUEUE_STATE_STACK_XOFF,
613  	__QUEUE_STATE_FROZEN,
614  };
615  
616  #define QUEUE_STATE_DRV_XOFF	(1 << __QUEUE_STATE_DRV_XOFF)
617  #define QUEUE_STATE_STACK_XOFF	(1 << __QUEUE_STATE_STACK_XOFF)
618  #define QUEUE_STATE_FROZEN	(1 << __QUEUE_STATE_FROZEN)
619  
620  #define QUEUE_STATE_ANY_XOFF	(QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
621  #define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
622  					QUEUE_STATE_FROZEN)
623  #define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
624  					QUEUE_STATE_FROZEN)
625  
626  /*
627   * __QUEUE_STATE_DRV_XOFF is used by drivers to stop the transmit queue.  The
628   * netif_tx_* functions below are used to manipulate this flag.  The
629   * __QUEUE_STATE_STACK_XOFF flag is used by the stack to stop the transmit
630   * queue independently.  The netif_xmit_*stopped functions below are called
631   * to check if the queue has been stopped by the driver or stack (either
632   * of the XOFF bits are set in the state).  Drivers should not need to call
633   * netif_xmit*stopped functions, they should only be using netif_tx_*.
634   */
635  
636  struct netdev_queue {
637  /*
638   * read-mostly part
639   */
640  	struct net_device	*dev;
641  	netdevice_tracker	dev_tracker;
642  
643  	struct Qdisc __rcu	*qdisc;
644  	struct Qdisc __rcu	*qdisc_sleeping;
645  #ifdef CONFIG_SYSFS
646  	struct kobject		kobj;
647  #endif
648  	unsigned long		tx_maxrate;
649  	/*
650  	 * Number of TX timeouts for this queue
651  	 * (/sys/class/net/DEV/Q/trans_timeout)
652  	 */
653  	atomic_long_t		trans_timeout;
654  
655  	/* Subordinate device that the queue has been assigned to */
656  	struct net_device	*sb_dev;
657  #ifdef CONFIG_XDP_SOCKETS
658  	struct xsk_buff_pool    *pool;
659  #endif
660  
661  /*
662   * write-mostly part
663   */
664  #ifdef CONFIG_BQL
665  	struct dql		dql;
666  #endif
667  	spinlock_t		_xmit_lock ____cacheline_aligned_in_smp;
668  	int			xmit_lock_owner;
669  	/*
670  	 * Time (in jiffies) of last Tx
671  	 */
672  	unsigned long		trans_start;
673  
674  	unsigned long		state;
675  
676  /*
677   * slow- / control-path part
678   */
679  	/* NAPI instance for the queue
680  	 * Readers and writers must hold RTNL
681  	 */
682  	struct napi_struct	*napi;
683  
684  #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
685  	int			numa_node;
686  #endif
687  } ____cacheline_aligned_in_smp;
688  
689  extern int sysctl_fb_tunnels_only_for_init_net;
690  extern int sysctl_devconf_inherit_init_net;
691  
692  /*
693   * sysctl_fb_tunnels_only_for_init_net == 0 : For all netns
694   *                                     == 1 : For initns only
695   *                                     == 2 : For none.
696   */
net_has_fallback_tunnels(const struct net * net)697  static inline bool net_has_fallback_tunnels(const struct net *net)
698  {
699  #if IS_ENABLED(CONFIG_SYSCTL)
700  	int fb_tunnels_only_for_init_net = READ_ONCE(sysctl_fb_tunnels_only_for_init_net);
701  
702  	return !fb_tunnels_only_for_init_net ||
703  		(net_eq(net, &init_net) && fb_tunnels_only_for_init_net == 1);
704  #else
705  	return true;
706  #endif
707  }
708  
net_inherit_devconf(void)709  static inline int net_inherit_devconf(void)
710  {
711  #if IS_ENABLED(CONFIG_SYSCTL)
712  	return READ_ONCE(sysctl_devconf_inherit_init_net);
713  #else
714  	return 0;
715  #endif
716  }
717  
netdev_queue_numa_node_read(const struct netdev_queue * q)718  static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
719  {
720  #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
721  	return q->numa_node;
722  #else
723  	return NUMA_NO_NODE;
724  #endif
725  }
726  
netdev_queue_numa_node_write(struct netdev_queue * q,int node)727  static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
728  {
729  #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
730  	q->numa_node = node;
731  #endif
732  }
733  
734  #ifdef CONFIG_RFS_ACCEL
735  bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
736  			 u16 filter_id);
737  #endif
738  
739  /* XPS map type and offset of the xps map within net_device->xps_maps[]. */
740  enum xps_map_type {
741  	XPS_CPUS = 0,
742  	XPS_RXQS,
743  	XPS_MAPS_MAX,
744  };
745  
746  #ifdef CONFIG_XPS
747  /*
748   * This structure holds an XPS map which can be of variable length.  The
749   * map is an array of queues.
750   */
751  struct xps_map {
752  	unsigned int len;
753  	unsigned int alloc_len;
754  	struct rcu_head rcu;
755  	u16 queues[];
756  };
757  #define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
758  #define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
759         - sizeof(struct xps_map)) / sizeof(u16))
760  
761  /*
762   * This structure holds all XPS maps for device.  Maps are indexed by CPU.
763   *
764   * We keep track of the number of cpus/rxqs used when the struct is allocated,
765   * in nr_ids. This will help not accessing out-of-bound memory.
766   *
767   * We keep track of the number of traffic classes used when the struct is
768   * allocated, in num_tc. This will be used to navigate the maps, to ensure we're
769   * not crossing its upper bound, as the original dev->num_tc can be updated in
770   * the meantime.
771   */
772  struct xps_dev_maps {
773  	struct rcu_head rcu;
774  	unsigned int nr_ids;
775  	s16 num_tc;
776  	struct xps_map __rcu *attr_map[]; /* Either CPUs map or RXQs map */
777  };
778  
779  #define XPS_CPU_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) +	\
780  	(nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
781  
782  #define XPS_RXQ_DEV_MAPS_SIZE(_tcs, _rxqs) (sizeof(struct xps_dev_maps) +\
783  	(_rxqs * (_tcs) * sizeof(struct xps_map *)))
784  
785  #endif /* CONFIG_XPS */
786  
787  #define TC_MAX_QUEUE	16
788  #define TC_BITMASK	15
789  /* HW offloaded queuing disciplines txq count and offset maps */
790  struct netdev_tc_txq {
791  	u16 count;
792  	u16 offset;
793  };
794  
795  #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
796  /*
797   * This structure is to hold information about the device
798   * configured to run FCoE protocol stack.
799   */
800  struct netdev_fcoe_hbainfo {
801  	char	manufacturer[64];
802  	char	serial_number[64];
803  	char	hardware_version[64];
804  	char	driver_version[64];
805  	char	optionrom_version[64];
806  	char	firmware_version[64];
807  	char	model[256];
808  	char	model_description[256];
809  };
810  #endif
811  
812  #define MAX_PHYS_ITEM_ID_LEN 32
813  
814  /* This structure holds a unique identifier to identify some
815   * physical item (port for example) used by a netdevice.
816   */
817  struct netdev_phys_item_id {
818  	unsigned char id[MAX_PHYS_ITEM_ID_LEN];
819  	unsigned char id_len;
820  };
821  
netdev_phys_item_id_same(struct netdev_phys_item_id * a,struct netdev_phys_item_id * b)822  static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
823  					    struct netdev_phys_item_id *b)
824  {
825  	return a->id_len == b->id_len &&
826  	       memcmp(a->id, b->id, a->id_len) == 0;
827  }
828  
829  typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
830  				       struct sk_buff *skb,
831  				       struct net_device *sb_dev);
832  
833  enum net_device_path_type {
834  	DEV_PATH_ETHERNET = 0,
835  	DEV_PATH_VLAN,
836  	DEV_PATH_BRIDGE,
837  	DEV_PATH_PPPOE,
838  	DEV_PATH_DSA,
839  	DEV_PATH_MTK_WDMA,
840  };
841  
842  struct net_device_path {
843  	enum net_device_path_type	type;
844  	const struct net_device		*dev;
845  	union {
846  		struct {
847  			u16		id;
848  			__be16		proto;
849  			u8		h_dest[ETH_ALEN];
850  		} encap;
851  		struct {
852  			enum {
853  				DEV_PATH_BR_VLAN_KEEP,
854  				DEV_PATH_BR_VLAN_TAG,
855  				DEV_PATH_BR_VLAN_UNTAG,
856  				DEV_PATH_BR_VLAN_UNTAG_HW,
857  			}		vlan_mode;
858  			u16		vlan_id;
859  			__be16		vlan_proto;
860  		} bridge;
861  		struct {
862  			int port;
863  			u16 proto;
864  		} dsa;
865  		struct {
866  			u8 wdma_idx;
867  			u8 queue;
868  			u16 wcid;
869  			u8 bss;
870  			u8 amsdu;
871  		} mtk_wdma;
872  	};
873  };
874  
875  #define NET_DEVICE_PATH_STACK_MAX	5
876  #define NET_DEVICE_PATH_VLAN_MAX	2
877  
878  struct net_device_path_stack {
879  	int			num_paths;
880  	struct net_device_path	path[NET_DEVICE_PATH_STACK_MAX];
881  };
882  
883  struct net_device_path_ctx {
884  	const struct net_device *dev;
885  	u8			daddr[ETH_ALEN];
886  
887  	int			num_vlans;
888  	struct {
889  		u16		id;
890  		__be16		proto;
891  	} vlan[NET_DEVICE_PATH_VLAN_MAX];
892  };
893  
894  enum tc_setup_type {
895  	TC_QUERY_CAPS,
896  	TC_SETUP_QDISC_MQPRIO,
897  	TC_SETUP_CLSU32,
898  	TC_SETUP_CLSFLOWER,
899  	TC_SETUP_CLSMATCHALL,
900  	TC_SETUP_CLSBPF,
901  	TC_SETUP_BLOCK,
902  	TC_SETUP_QDISC_CBS,
903  	TC_SETUP_QDISC_RED,
904  	TC_SETUP_QDISC_PRIO,
905  	TC_SETUP_QDISC_MQ,
906  	TC_SETUP_QDISC_ETF,
907  	TC_SETUP_ROOT_QDISC,
908  	TC_SETUP_QDISC_GRED,
909  	TC_SETUP_QDISC_TAPRIO,
910  	TC_SETUP_FT,
911  	TC_SETUP_QDISC_ETS,
912  	TC_SETUP_QDISC_TBF,
913  	TC_SETUP_QDISC_FIFO,
914  	TC_SETUP_QDISC_HTB,
915  	TC_SETUP_ACT,
916  };
917  
918  /* These structures hold the attributes of bpf state that are being passed
919   * to the netdevice through the bpf op.
920   */
921  enum bpf_netdev_command {
922  	/* Set or clear a bpf program used in the earliest stages of packet
923  	 * rx. The prog will have been loaded as BPF_PROG_TYPE_XDP. The callee
924  	 * is responsible for calling bpf_prog_put on any old progs that are
925  	 * stored. In case of error, the callee need not release the new prog
926  	 * reference, but on success it takes ownership and must bpf_prog_put
927  	 * when it is no longer used.
928  	 */
929  	XDP_SETUP_PROG,
930  	XDP_SETUP_PROG_HW,
931  	/* BPF program for offload callbacks, invoked at program load time. */
932  	BPF_OFFLOAD_MAP_ALLOC,
933  	BPF_OFFLOAD_MAP_FREE,
934  	XDP_SETUP_XSK_POOL,
935  };
936  
937  struct bpf_prog_offload_ops;
938  struct netlink_ext_ack;
939  struct xdp_umem;
940  struct xdp_dev_bulk_queue;
941  struct bpf_xdp_link;
942  
943  enum bpf_xdp_mode {
944  	XDP_MODE_SKB = 0,
945  	XDP_MODE_DRV = 1,
946  	XDP_MODE_HW = 2,
947  	__MAX_XDP_MODE
948  };
949  
950  struct bpf_xdp_entity {
951  	struct bpf_prog *prog;
952  	struct bpf_xdp_link *link;
953  };
954  
955  struct netdev_bpf {
956  	enum bpf_netdev_command command;
957  	union {
958  		/* XDP_SETUP_PROG */
959  		struct {
960  			u32 flags;
961  			struct bpf_prog *prog;
962  			struct netlink_ext_ack *extack;
963  		};
964  		/* BPF_OFFLOAD_MAP_ALLOC, BPF_OFFLOAD_MAP_FREE */
965  		struct {
966  			struct bpf_offloaded_map *offmap;
967  		};
968  		/* XDP_SETUP_XSK_POOL */
969  		struct {
970  			struct xsk_buff_pool *pool;
971  			u16 queue_id;
972  		} xsk;
973  	};
974  };
975  
976  /* Flags for ndo_xsk_wakeup. */
977  #define XDP_WAKEUP_RX (1 << 0)
978  #define XDP_WAKEUP_TX (1 << 1)
979  
980  #ifdef CONFIG_XFRM_OFFLOAD
981  struct xfrmdev_ops {
982  	int	(*xdo_dev_state_add) (struct xfrm_state *x, struct netlink_ext_ack *extack);
983  	void	(*xdo_dev_state_delete) (struct xfrm_state *x);
984  	void	(*xdo_dev_state_free) (struct xfrm_state *x);
985  	bool	(*xdo_dev_offload_ok) (struct sk_buff *skb,
986  				       struct xfrm_state *x);
987  	void	(*xdo_dev_state_advance_esn) (struct xfrm_state *x);
988  	void	(*xdo_dev_state_update_stats) (struct xfrm_state *x);
989  	int	(*xdo_dev_policy_add) (struct xfrm_policy *x, struct netlink_ext_ack *extack);
990  	void	(*xdo_dev_policy_delete) (struct xfrm_policy *x);
991  	void	(*xdo_dev_policy_free) (struct xfrm_policy *x);
992  };
993  #endif
994  
995  struct dev_ifalias {
996  	struct rcu_head rcuhead;
997  	char ifalias[];
998  };
999  
1000  struct devlink;
1001  struct tlsdev_ops;
1002  
1003  struct netdev_net_notifier {
1004  	struct list_head list;
1005  	struct notifier_block *nb;
1006  };
1007  
1008  /*
1009   * This structure defines the management hooks for network devices.
1010   * The following hooks can be defined; unless noted otherwise, they are
1011   * optional and can be filled with a null pointer.
1012   *
1013   * int (*ndo_init)(struct net_device *dev);
1014   *     This function is called once when a network device is registered.
1015   *     The network device can use this for any late stage initialization
1016   *     or semantic validation. It can fail with an error code which will
1017   *     be propagated back to register_netdev.
1018   *
1019   * void (*ndo_uninit)(struct net_device *dev);
1020   *     This function is called when device is unregistered or when registration
1021   *     fails. It is not called if init fails.
1022   *
1023   * int (*ndo_open)(struct net_device *dev);
1024   *     This function is called when a network device transitions to the up
1025   *     state.
1026   *
1027   * int (*ndo_stop)(struct net_device *dev);
1028   *     This function is called when a network device transitions to the down
1029   *     state.
1030   *
1031   * netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1032   *                               struct net_device *dev);
1033   *	Called when a packet needs to be transmitted.
1034   *	Returns NETDEV_TX_OK.  Can return NETDEV_TX_BUSY, but you should stop
1035   *	the queue before that can happen; it's for obsolete devices and weird
1036   *	corner cases, but the stack really does a non-trivial amount
1037   *	of useless work if you return NETDEV_TX_BUSY.
1038   *	Required; cannot be NULL.
1039   *
1040   * netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1041   *					   struct net_device *dev
1042   *					   netdev_features_t features);
1043   *	Called by core transmit path to determine if device is capable of
1044   *	performing offload operations on a given packet. This is to give
1045   *	the device an opportunity to implement any restrictions that cannot
1046   *	be otherwise expressed by feature flags. The check is called with
1047   *	the set of features that the stack has calculated and it returns
1048   *	those the driver believes to be appropriate.
1049   *
1050   * u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
1051   *                         struct net_device *sb_dev);
1052   *	Called to decide which queue to use when device supports multiple
1053   *	transmit queues.
1054   *
1055   * void (*ndo_change_rx_flags)(struct net_device *dev, int flags);
1056   *	This function is called to allow device receiver to make
1057   *	changes to configuration when multicast or promiscuous is enabled.
1058   *
1059   * void (*ndo_set_rx_mode)(struct net_device *dev);
1060   *	This function is called device changes address list filtering.
1061   *	If driver handles unicast address filtering, it should set
1062   *	IFF_UNICAST_FLT in its priv_flags.
1063   *
1064   * int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
1065   *	This function  is called when the Media Access Control address
1066   *	needs to be changed. If this interface is not defined, the
1067   *	MAC address can not be changed.
1068   *
1069   * int (*ndo_validate_addr)(struct net_device *dev);
1070   *	Test if Media Access Control address is valid for the device.
1071   *
1072   * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
1073   *	Old-style ioctl entry point. This is used internally by the
1074   *	appletalk and ieee802154 subsystems but is no longer called by
1075   *	the device ioctl handler.
1076   *
1077   * int (*ndo_siocbond)(struct net_device *dev, struct ifreq *ifr, int cmd);
1078   *	Used by the bonding driver for its device specific ioctls:
1079   *	SIOCBONDENSLAVE, SIOCBONDRELEASE, SIOCBONDSETHWADDR, SIOCBONDCHANGEACTIVE,
1080   *	SIOCBONDSLAVEINFOQUERY, and SIOCBONDINFOQUERY
1081   *
1082   * * int (*ndo_eth_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
1083   *	Called for ethernet specific ioctls: SIOCGMIIPHY, SIOCGMIIREG,
1084   *	SIOCSMIIREG, SIOCSHWTSTAMP and SIOCGHWTSTAMP.
1085   *
1086   * int (*ndo_set_config)(struct net_device *dev, struct ifmap *map);
1087   *	Used to set network devices bus interface parameters. This interface
1088   *	is retained for legacy reasons; new devices should use the bus
1089   *	interface (PCI) for low level management.
1090   *
1091   * int (*ndo_change_mtu)(struct net_device *dev, int new_mtu);
1092   *	Called when a user wants to change the Maximum Transfer Unit
1093   *	of a device.
1094   *
1095   * void (*ndo_tx_timeout)(struct net_device *dev, unsigned int txqueue);
1096   *	Callback used when the transmitter has not made any progress
1097   *	for dev->watchdog ticks.
1098   *
1099   * void (*ndo_get_stats64)(struct net_device *dev,
1100   *                         struct rtnl_link_stats64 *storage);
1101   * struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1102   *	Called when a user wants to get the network device usage
1103   *	statistics. Drivers must do one of the following:
1104   *	1. Define @ndo_get_stats64 to fill in a zero-initialised
1105   *	   rtnl_link_stats64 structure passed by the caller.
1106   *	2. Define @ndo_get_stats to update a net_device_stats structure
1107   *	   (which should normally be dev->stats) and return a pointer to
1108   *	   it. The structure may be changed asynchronously only if each
1109   *	   field is written atomically.
1110   *	3. Update dev->stats asynchronously and atomically, and define
1111   *	   neither operation.
1112   *
1113   * bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id)
1114   *	Return true if this device supports offload stats of this attr_id.
1115   *
1116   * int (*ndo_get_offload_stats)(int attr_id, const struct net_device *dev,
1117   *	void *attr_data)
1118   *	Get statistics for offload operations by attr_id. Write it into the
1119   *	attr_data pointer.
1120   *
1121   * int (*ndo_vlan_rx_add_vid)(struct net_device *dev, __be16 proto, u16 vid);
1122   *	If device supports VLAN filtering this function is called when a
1123   *	VLAN id is registered.
1124   *
1125   * int (*ndo_vlan_rx_kill_vid)(struct net_device *dev, __be16 proto, u16 vid);
1126   *	If device supports VLAN filtering this function is called when a
1127   *	VLAN id is unregistered.
1128   *
1129   * void (*ndo_poll_controller)(struct net_device *dev);
1130   *
1131   *	SR-IOV management functions.
1132   * int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
1133   * int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan,
1134   *			  u8 qos, __be16 proto);
1135   * int (*ndo_set_vf_rate)(struct net_device *dev, int vf, int min_tx_rate,
1136   *			  int max_tx_rate);
1137   * int (*ndo_set_vf_spoofchk)(struct net_device *dev, int vf, bool setting);
1138   * int (*ndo_set_vf_trust)(struct net_device *dev, int vf, bool setting);
1139   * int (*ndo_get_vf_config)(struct net_device *dev,
1140   *			    int vf, struct ifla_vf_info *ivf);
1141   * int (*ndo_set_vf_link_state)(struct net_device *dev, int vf, int link_state);
1142   * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
1143   *			  struct nlattr *port[]);
1144   *
1145   *      Enable or disable the VF ability to query its RSS Redirection Table and
1146   *      Hash Key. This is needed since on some devices VF share this information
1147   *      with PF and querying it may introduce a theoretical security risk.
1148   * int (*ndo_set_vf_rss_query_en)(struct net_device *dev, int vf, bool setting);
1149   * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
1150   * int (*ndo_setup_tc)(struct net_device *dev, enum tc_setup_type type,
1151   *		       void *type_data);
1152   *	Called to setup any 'tc' scheduler, classifier or action on @dev.
1153   *	This is always called from the stack with the rtnl lock held and netif
1154   *	tx queues stopped. This allows the netdevice to perform queue
1155   *	management safely.
1156   *
1157   *	Fiber Channel over Ethernet (FCoE) offload functions.
1158   * int (*ndo_fcoe_enable)(struct net_device *dev);
1159   *	Called when the FCoE protocol stack wants to start using LLD for FCoE
1160   *	so the underlying device can perform whatever needed configuration or
1161   *	initialization to support acceleration of FCoE traffic.
1162   *
1163   * int (*ndo_fcoe_disable)(struct net_device *dev);
1164   *	Called when the FCoE protocol stack wants to stop using LLD for FCoE
1165   *	so the underlying device can perform whatever needed clean-ups to
1166   *	stop supporting acceleration of FCoE traffic.
1167   *
1168   * int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
1169   *			     struct scatterlist *sgl, unsigned int sgc);
1170   *	Called when the FCoE Initiator wants to initialize an I/O that
1171   *	is a possible candidate for Direct Data Placement (DDP). The LLD can
1172   *	perform necessary setup and returns 1 to indicate the device is set up
1173   *	successfully to perform DDP on this I/O, otherwise this returns 0.
1174   *
1175   * int (*ndo_fcoe_ddp_done)(struct net_device *dev,  u16 xid);
1176   *	Called when the FCoE Initiator/Target is done with the DDPed I/O as
1177   *	indicated by the FC exchange id 'xid', so the underlying device can
1178   *	clean up and reuse resources for later DDP requests.
1179   *
1180   * int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
1181   *			      struct scatterlist *sgl, unsigned int sgc);
1182   *	Called when the FCoE Target wants to initialize an I/O that
1183   *	is a possible candidate for Direct Data Placement (DDP). The LLD can
1184   *	perform necessary setup and returns 1 to indicate the device is set up
1185   *	successfully to perform DDP on this I/O, otherwise this returns 0.
1186   *
1187   * int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1188   *			       struct netdev_fcoe_hbainfo *hbainfo);
1189   *	Called when the FCoE Protocol stack wants information on the underlying
1190   *	device. This information is utilized by the FCoE protocol stack to
1191   *	register attributes with Fiber Channel management service as per the
1192   *	FC-GS Fabric Device Management Information(FDMI) specification.
1193   *
1194   * int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
1195   *	Called when the underlying device wants to override default World Wide
1196   *	Name (WWN) generation mechanism in FCoE protocol stack to pass its own
1197   *	World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
1198   *	protocol stack to use.
1199   *
1200   *	RFS acceleration.
1201   * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
1202   *			    u16 rxq_index, u32 flow_id);
1203   *	Set hardware filter for RFS.  rxq_index is the target queue index;
1204   *	flow_id is a flow ID to be passed to rps_may_expire_flow() later.
1205   *	Return the filter ID on success, or a negative error code.
1206   *
1207   *	Slave management functions (for bridge, bonding, etc).
1208   * int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
1209   *	Called to make another netdev an underling.
1210   *
1211   * int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
1212   *	Called to release previously enslaved netdev.
1213   *
1214   * struct net_device *(*ndo_get_xmit_slave)(struct net_device *dev,
1215   *					    struct sk_buff *skb,
1216   *					    bool all_slaves);
1217   *	Get the xmit slave of master device. If all_slaves is true, function
1218   *	assume all the slaves can transmit.
1219   *
1220   *      Feature/offload setting functions.
1221   * netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1222   *		netdev_features_t features);
1223   *	Adjusts the requested feature flags according to device-specific
1224   *	constraints, and returns the resulting flags. Must not modify
1225   *	the device state.
1226   *
1227   * int (*ndo_set_features)(struct net_device *dev, netdev_features_t features);
1228   *	Called to update device configuration to new features. Passed
1229   *	feature set might be less than what was returned by ndo_fix_features()).
1230   *	Must return >0 or -errno if it changed dev->features itself.
1231   *
1232   * int (*ndo_fdb_add)(struct ndmsg *ndm, struct nlattr *tb[],
1233   *		      struct net_device *dev,
1234   *		      const unsigned char *addr, u16 vid, u16 flags,
1235   *		      struct netlink_ext_ack *extack);
1236   *	Adds an FDB entry to dev for addr.
1237   * int (*ndo_fdb_del)(struct ndmsg *ndm, struct nlattr *tb[],
1238   *		      struct net_device *dev,
1239   *		      const unsigned char *addr, u16 vid)
1240   *	Deletes the FDB entry from dev corresponding to addr.
1241   * int (*ndo_fdb_del_bulk)(struct nlmsghdr *nlh, struct net_device *dev,
1242   *			   struct netlink_ext_ack *extack);
1243   * int (*ndo_fdb_dump)(struct sk_buff *skb, struct netlink_callback *cb,
1244   *		       struct net_device *dev, struct net_device *filter_dev,
1245   *		       int *idx)
1246   *	Used to add FDB entries to dump requests. Implementers should add
1247   *	entries to skb and update idx with the number of entries.
1248   *
1249   * int (*ndo_mdb_add)(struct net_device *dev, struct nlattr *tb[],
1250   *		      u16 nlmsg_flags, struct netlink_ext_ack *extack);
1251   *	Adds an MDB entry to dev.
1252   * int (*ndo_mdb_del)(struct net_device *dev, struct nlattr *tb[],
1253   *		      struct netlink_ext_ack *extack);
1254   *	Deletes the MDB entry from dev.
1255   * int (*ndo_mdb_del_bulk)(struct net_device *dev, struct nlattr *tb[],
1256   *			   struct netlink_ext_ack *extack);
1257   *	Bulk deletes MDB entries from dev.
1258   * int (*ndo_mdb_dump)(struct net_device *dev, struct sk_buff *skb,
1259   *		       struct netlink_callback *cb);
1260   *	Dumps MDB entries from dev. The first argument (marker) in the netlink
1261   *	callback is used by core rtnetlink code.
1262   *
1263   * int (*ndo_bridge_setlink)(struct net_device *dev, struct nlmsghdr *nlh,
1264   *			     u16 flags, struct netlink_ext_ack *extack)
1265   * int (*ndo_bridge_getlink)(struct sk_buff *skb, u32 pid, u32 seq,
1266   *			     struct net_device *dev, u32 filter_mask,
1267   *			     int nlflags)
1268   * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
1269   *			     u16 flags);
1270   *
1271   * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
1272   *	Called to change device carrier. Soft-devices (like dummy, team, etc)
1273   *	which do not represent real hardware may define this to allow their
1274   *	userspace components to manage their virtual carrier state. Devices
1275   *	that determine carrier state from physical hardware properties (eg
1276   *	network cables) or protocol-dependent mechanisms (eg
1277   *	USB_CDC_NOTIFY_NETWORK_CONNECTION) should NOT implement this function.
1278   *
1279   * int (*ndo_get_phys_port_id)(struct net_device *dev,
1280   *			       struct netdev_phys_item_id *ppid);
1281   *	Called to get ID of physical port of this device. If driver does
1282   *	not implement this, it is assumed that the hw is not able to have
1283   *	multiple net devices on single physical port.
1284   *
1285   * int (*ndo_get_port_parent_id)(struct net_device *dev,
1286   *				 struct netdev_phys_item_id *ppid)
1287   *	Called to get the parent ID of the physical port of this device.
1288   *
1289   * void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1290   *				 struct net_device *dev)
1291   *	Called by upper layer devices to accelerate switching or other
1292   *	station functionality into hardware. 'pdev is the lowerdev
1293   *	to use for the offload and 'dev' is the net device that will
1294   *	back the offload. Returns a pointer to the private structure
1295   *	the upper layer will maintain.
1296   * void (*ndo_dfwd_del_station)(struct net_device *pdev, void *priv)
1297   *	Called by upper layer device to delete the station created
1298   *	by 'ndo_dfwd_add_station'. 'pdev' is the net device backing
1299   *	the station and priv is the structure returned by the add
1300   *	operation.
1301   * int (*ndo_set_tx_maxrate)(struct net_device *dev,
1302   *			     int queue_index, u32 maxrate);
1303   *	Called when a user wants to set a max-rate limitation of specific
1304   *	TX queue.
1305   * int (*ndo_get_iflink)(const struct net_device *dev);
1306   *	Called to get the iflink value of this device.
1307   * int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb);
1308   *	This function is used to get egress tunnel information for given skb.
1309   *	This is useful for retrieving outer tunnel header parameters while
1310   *	sampling packet.
1311   * void (*ndo_set_rx_headroom)(struct net_device *dev, int needed_headroom);
1312   *	This function is used to specify the headroom that the skb must
1313   *	consider when allocation skb during packet reception. Setting
1314   *	appropriate rx headroom value allows avoiding skb head copy on
1315   *	forward. Setting a negative value resets the rx headroom to the
1316   *	default value.
1317   * int (*ndo_bpf)(struct net_device *dev, struct netdev_bpf *bpf);
1318   *	This function is used to set or query state related to XDP on the
1319   *	netdevice and manage BPF offload. See definition of
1320   *	enum bpf_netdev_command for details.
1321   * int (*ndo_xdp_xmit)(struct net_device *dev, int n, struct xdp_frame **xdp,
1322   *			u32 flags);
1323   *	This function is used to submit @n XDP packets for transmit on a
1324   *	netdevice. Returns number of frames successfully transmitted, frames
1325   *	that got dropped are freed/returned via xdp_return_frame().
1326   *	Returns negative number, means general error invoking ndo, meaning
1327   *	no frames were xmit'ed and core-caller will free all frames.
1328   * struct net_device *(*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1329   *					        struct xdp_buff *xdp);
1330   *      Get the xmit slave of master device based on the xdp_buff.
1331   * int (*ndo_xsk_wakeup)(struct net_device *dev, u32 queue_id, u32 flags);
1332   *      This function is used to wake up the softirq, ksoftirqd or kthread
1333   *	responsible for sending and/or receiving packets on a specific
1334   *	queue id bound to an AF_XDP socket. The flags field specifies if
1335   *	only RX, only Tx, or both should be woken up using the flags
1336   *	XDP_WAKEUP_RX and XDP_WAKEUP_TX.
1337   * int (*ndo_tunnel_ctl)(struct net_device *dev, struct ip_tunnel_parm_kern *p,
1338   *			 int cmd);
1339   *	Add, change, delete or get information on an IPv4 tunnel.
1340   * struct net_device *(*ndo_get_peer_dev)(struct net_device *dev);
1341   *	If a device is paired with a peer device, return the peer instance.
1342   *	The caller must be under RCU read context.
1343   * int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx, struct net_device_path *path);
1344   *     Get the forwarding path to reach the real device from the HW destination address
1345   * ktime_t (*ndo_get_tstamp)(struct net_device *dev,
1346   *			     const struct skb_shared_hwtstamps *hwtstamps,
1347   *			     bool cycles);
1348   *	Get hardware timestamp based on normal/adjustable time or free running
1349   *	cycle counter. This function is required if physical clock supports a
1350   *	free running cycle counter.
1351   *
1352   * int (*ndo_hwtstamp_get)(struct net_device *dev,
1353   *			   struct kernel_hwtstamp_config *kernel_config);
1354   *	Get the currently configured hardware timestamping parameters for the
1355   *	NIC device.
1356   *
1357   * int (*ndo_hwtstamp_set)(struct net_device *dev,
1358   *			   struct kernel_hwtstamp_config *kernel_config,
1359   *			   struct netlink_ext_ack *extack);
1360   *	Change the hardware timestamping parameters for NIC device.
1361   */
1362  struct net_device_ops {
1363  	int			(*ndo_init)(struct net_device *dev);
1364  	void			(*ndo_uninit)(struct net_device *dev);
1365  	int			(*ndo_open)(struct net_device *dev);
1366  	int			(*ndo_stop)(struct net_device *dev);
1367  	netdev_tx_t		(*ndo_start_xmit)(struct sk_buff *skb,
1368  						  struct net_device *dev);
1369  	netdev_features_t	(*ndo_features_check)(struct sk_buff *skb,
1370  						      struct net_device *dev,
1371  						      netdev_features_t features);
1372  	u16			(*ndo_select_queue)(struct net_device *dev,
1373  						    struct sk_buff *skb,
1374  						    struct net_device *sb_dev);
1375  	void			(*ndo_change_rx_flags)(struct net_device *dev,
1376  						       int flags);
1377  	void			(*ndo_set_rx_mode)(struct net_device *dev);
1378  	int			(*ndo_set_mac_address)(struct net_device *dev,
1379  						       void *addr);
1380  	int			(*ndo_validate_addr)(struct net_device *dev);
1381  	int			(*ndo_do_ioctl)(struct net_device *dev,
1382  					        struct ifreq *ifr, int cmd);
1383  	int			(*ndo_eth_ioctl)(struct net_device *dev,
1384  						 struct ifreq *ifr, int cmd);
1385  	int			(*ndo_siocbond)(struct net_device *dev,
1386  						struct ifreq *ifr, int cmd);
1387  	int			(*ndo_siocwandev)(struct net_device *dev,
1388  						  struct if_settings *ifs);
1389  	int			(*ndo_siocdevprivate)(struct net_device *dev,
1390  						      struct ifreq *ifr,
1391  						      void __user *data, int cmd);
1392  	int			(*ndo_set_config)(struct net_device *dev,
1393  					          struct ifmap *map);
1394  	int			(*ndo_change_mtu)(struct net_device *dev,
1395  						  int new_mtu);
1396  	int			(*ndo_neigh_setup)(struct net_device *dev,
1397  						   struct neigh_parms *);
1398  	void			(*ndo_tx_timeout) (struct net_device *dev,
1399  						   unsigned int txqueue);
1400  
1401  	void			(*ndo_get_stats64)(struct net_device *dev,
1402  						   struct rtnl_link_stats64 *storage);
1403  	bool			(*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
1404  	int			(*ndo_get_offload_stats)(int attr_id,
1405  							 const struct net_device *dev,
1406  							 void *attr_data);
1407  	struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1408  
1409  	int			(*ndo_vlan_rx_add_vid)(struct net_device *dev,
1410  						       __be16 proto, u16 vid);
1411  	int			(*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1412  						        __be16 proto, u16 vid);
1413  #ifdef CONFIG_NET_POLL_CONTROLLER
1414  	void                    (*ndo_poll_controller)(struct net_device *dev);
1415  	int			(*ndo_netpoll_setup)(struct net_device *dev,
1416  						     struct netpoll_info *info);
1417  	void			(*ndo_netpoll_cleanup)(struct net_device *dev);
1418  #endif
1419  	int			(*ndo_set_vf_mac)(struct net_device *dev,
1420  						  int queue, u8 *mac);
1421  	int			(*ndo_set_vf_vlan)(struct net_device *dev,
1422  						   int queue, u16 vlan,
1423  						   u8 qos, __be16 proto);
1424  	int			(*ndo_set_vf_rate)(struct net_device *dev,
1425  						   int vf, int min_tx_rate,
1426  						   int max_tx_rate);
1427  	int			(*ndo_set_vf_spoofchk)(struct net_device *dev,
1428  						       int vf, bool setting);
1429  	int			(*ndo_set_vf_trust)(struct net_device *dev,
1430  						    int vf, bool setting);
1431  	int			(*ndo_get_vf_config)(struct net_device *dev,
1432  						     int vf,
1433  						     struct ifla_vf_info *ivf);
1434  	int			(*ndo_set_vf_link_state)(struct net_device *dev,
1435  							 int vf, int link_state);
1436  	int			(*ndo_get_vf_stats)(struct net_device *dev,
1437  						    int vf,
1438  						    struct ifla_vf_stats
1439  						    *vf_stats);
1440  	int			(*ndo_set_vf_port)(struct net_device *dev,
1441  						   int vf,
1442  						   struct nlattr *port[]);
1443  	int			(*ndo_get_vf_port)(struct net_device *dev,
1444  						   int vf, struct sk_buff *skb);
1445  	int			(*ndo_get_vf_guid)(struct net_device *dev,
1446  						   int vf,
1447  						   struct ifla_vf_guid *node_guid,
1448  						   struct ifla_vf_guid *port_guid);
1449  	int			(*ndo_set_vf_guid)(struct net_device *dev,
1450  						   int vf, u64 guid,
1451  						   int guid_type);
1452  	int			(*ndo_set_vf_rss_query_en)(
1453  						   struct net_device *dev,
1454  						   int vf, bool setting);
1455  	int			(*ndo_setup_tc)(struct net_device *dev,
1456  						enum tc_setup_type type,
1457  						void *type_data);
1458  #if IS_ENABLED(CONFIG_FCOE)
1459  	int			(*ndo_fcoe_enable)(struct net_device *dev);
1460  	int			(*ndo_fcoe_disable)(struct net_device *dev);
1461  	int			(*ndo_fcoe_ddp_setup)(struct net_device *dev,
1462  						      u16 xid,
1463  						      struct scatterlist *sgl,
1464  						      unsigned int sgc);
1465  	int			(*ndo_fcoe_ddp_done)(struct net_device *dev,
1466  						     u16 xid);
1467  	int			(*ndo_fcoe_ddp_target)(struct net_device *dev,
1468  						       u16 xid,
1469  						       struct scatterlist *sgl,
1470  						       unsigned int sgc);
1471  	int			(*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1472  							struct netdev_fcoe_hbainfo *hbainfo);
1473  #endif
1474  
1475  #if IS_ENABLED(CONFIG_LIBFCOE)
1476  #define NETDEV_FCOE_WWNN 0
1477  #define NETDEV_FCOE_WWPN 1
1478  	int			(*ndo_fcoe_get_wwn)(struct net_device *dev,
1479  						    u64 *wwn, int type);
1480  #endif
1481  
1482  #ifdef CONFIG_RFS_ACCEL
1483  	int			(*ndo_rx_flow_steer)(struct net_device *dev,
1484  						     const struct sk_buff *skb,
1485  						     u16 rxq_index,
1486  						     u32 flow_id);
1487  #endif
1488  	int			(*ndo_add_slave)(struct net_device *dev,
1489  						 struct net_device *slave_dev,
1490  						 struct netlink_ext_ack *extack);
1491  	int			(*ndo_del_slave)(struct net_device *dev,
1492  						 struct net_device *slave_dev);
1493  	struct net_device*	(*ndo_get_xmit_slave)(struct net_device *dev,
1494  						      struct sk_buff *skb,
1495  						      bool all_slaves);
1496  	struct net_device*	(*ndo_sk_get_lower_dev)(struct net_device *dev,
1497  							struct sock *sk);
1498  	netdev_features_t	(*ndo_fix_features)(struct net_device *dev,
1499  						    netdev_features_t features);
1500  	int			(*ndo_set_features)(struct net_device *dev,
1501  						    netdev_features_t features);
1502  	int			(*ndo_neigh_construct)(struct net_device *dev,
1503  						       struct neighbour *n);
1504  	void			(*ndo_neigh_destroy)(struct net_device *dev,
1505  						     struct neighbour *n);
1506  
1507  	int			(*ndo_fdb_add)(struct ndmsg *ndm,
1508  					       struct nlattr *tb[],
1509  					       struct net_device *dev,
1510  					       const unsigned char *addr,
1511  					       u16 vid,
1512  					       u16 flags,
1513  					       struct netlink_ext_ack *extack);
1514  	int			(*ndo_fdb_del)(struct ndmsg *ndm,
1515  					       struct nlattr *tb[],
1516  					       struct net_device *dev,
1517  					       const unsigned char *addr,
1518  					       u16 vid, struct netlink_ext_ack *extack);
1519  	int			(*ndo_fdb_del_bulk)(struct nlmsghdr *nlh,
1520  						    struct net_device *dev,
1521  						    struct netlink_ext_ack *extack);
1522  	int			(*ndo_fdb_dump)(struct sk_buff *skb,
1523  						struct netlink_callback *cb,
1524  						struct net_device *dev,
1525  						struct net_device *filter_dev,
1526  						int *idx);
1527  	int			(*ndo_fdb_get)(struct sk_buff *skb,
1528  					       struct nlattr *tb[],
1529  					       struct net_device *dev,
1530  					       const unsigned char *addr,
1531  					       u16 vid, u32 portid, u32 seq,
1532  					       struct netlink_ext_ack *extack);
1533  	int			(*ndo_mdb_add)(struct net_device *dev,
1534  					       struct nlattr *tb[],
1535  					       u16 nlmsg_flags,
1536  					       struct netlink_ext_ack *extack);
1537  	int			(*ndo_mdb_del)(struct net_device *dev,
1538  					       struct nlattr *tb[],
1539  					       struct netlink_ext_ack *extack);
1540  	int			(*ndo_mdb_del_bulk)(struct net_device *dev,
1541  						    struct nlattr *tb[],
1542  						    struct netlink_ext_ack *extack);
1543  	int			(*ndo_mdb_dump)(struct net_device *dev,
1544  						struct sk_buff *skb,
1545  						struct netlink_callback *cb);
1546  	int			(*ndo_mdb_get)(struct net_device *dev,
1547  					       struct nlattr *tb[], u32 portid,
1548  					       u32 seq,
1549  					       struct netlink_ext_ack *extack);
1550  	int			(*ndo_bridge_setlink)(struct net_device *dev,
1551  						      struct nlmsghdr *nlh,
1552  						      u16 flags,
1553  						      struct netlink_ext_ack *extack);
1554  	int			(*ndo_bridge_getlink)(struct sk_buff *skb,
1555  						      u32 pid, u32 seq,
1556  						      struct net_device *dev,
1557  						      u32 filter_mask,
1558  						      int nlflags);
1559  	int			(*ndo_bridge_dellink)(struct net_device *dev,
1560  						      struct nlmsghdr *nlh,
1561  						      u16 flags);
1562  	int			(*ndo_change_carrier)(struct net_device *dev,
1563  						      bool new_carrier);
1564  	int			(*ndo_get_phys_port_id)(struct net_device *dev,
1565  							struct netdev_phys_item_id *ppid);
1566  	int			(*ndo_get_port_parent_id)(struct net_device *dev,
1567  							  struct netdev_phys_item_id *ppid);
1568  	int			(*ndo_get_phys_port_name)(struct net_device *dev,
1569  							  char *name, size_t len);
1570  	void*			(*ndo_dfwd_add_station)(struct net_device *pdev,
1571  							struct net_device *dev);
1572  	void			(*ndo_dfwd_del_station)(struct net_device *pdev,
1573  							void *priv);
1574  
1575  	int			(*ndo_set_tx_maxrate)(struct net_device *dev,
1576  						      int queue_index,
1577  						      u32 maxrate);
1578  	int			(*ndo_get_iflink)(const struct net_device *dev);
1579  	int			(*ndo_fill_metadata_dst)(struct net_device *dev,
1580  						       struct sk_buff *skb);
1581  	void			(*ndo_set_rx_headroom)(struct net_device *dev,
1582  						       int needed_headroom);
1583  	int			(*ndo_bpf)(struct net_device *dev,
1584  					   struct netdev_bpf *bpf);
1585  	int			(*ndo_xdp_xmit)(struct net_device *dev, int n,
1586  						struct xdp_frame **xdp,
1587  						u32 flags);
1588  	struct net_device *	(*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1589  							  struct xdp_buff *xdp);
1590  	int			(*ndo_xsk_wakeup)(struct net_device *dev,
1591  						  u32 queue_id, u32 flags);
1592  	int			(*ndo_tunnel_ctl)(struct net_device *dev,
1593  						  struct ip_tunnel_parm_kern *p,
1594  						  int cmd);
1595  	struct net_device *	(*ndo_get_peer_dev)(struct net_device *dev);
1596  	int                     (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx,
1597                                                           struct net_device_path *path);
1598  	ktime_t			(*ndo_get_tstamp)(struct net_device *dev,
1599  						  const struct skb_shared_hwtstamps *hwtstamps,
1600  						  bool cycles);
1601  	int			(*ndo_hwtstamp_get)(struct net_device *dev,
1602  						    struct kernel_hwtstamp_config *kernel_config);
1603  	int			(*ndo_hwtstamp_set)(struct net_device *dev,
1604  						    struct kernel_hwtstamp_config *kernel_config,
1605  						    struct netlink_ext_ack *extack);
1606  };
1607  
1608  /**
1609   * enum netdev_priv_flags - &struct net_device priv_flags
1610   *
1611   * These are the &struct net_device, they are only set internally
1612   * by drivers and used in the kernel. These flags are invisible to
1613   * userspace; this means that the order of these flags can change
1614   * during any kernel release.
1615   *
1616   * You should add bitfield booleans after either net_device::priv_flags
1617   * (hotpath) or ::threaded (slowpath) instead of extending these flags.
1618   *
1619   * @IFF_802_1Q_VLAN: 802.1Q VLAN device
1620   * @IFF_EBRIDGE: Ethernet bridging device
1621   * @IFF_BONDING: bonding master or slave
1622   * @IFF_ISATAP: ISATAP interface (RFC4214)
1623   * @IFF_WAN_HDLC: WAN HDLC device
1624   * @IFF_XMIT_DST_RELEASE: dev_hard_start_xmit() is allowed to
1625   *	release skb->dst
1626   * @IFF_DONT_BRIDGE: disallow bridging this ether dev
1627   * @IFF_DISABLE_NETPOLL: disable netpoll at run-time
1628   * @IFF_MACVLAN_PORT: device used as macvlan port
1629   * @IFF_BRIDGE_PORT: device used as bridge port
1630   * @IFF_OVS_DATAPATH: device used as Open vSwitch datapath port
1631   * @IFF_TX_SKB_SHARING: The interface supports sharing skbs on transmit
1632   * @IFF_UNICAST_FLT: Supports unicast filtering
1633   * @IFF_TEAM_PORT: device used as team port
1634   * @IFF_SUPP_NOFCS: device supports sending custom FCS
1635   * @IFF_LIVE_ADDR_CHANGE: device supports hardware address
1636   *	change when it's running
1637   * @IFF_MACVLAN: Macvlan device
1638   * @IFF_XMIT_DST_RELEASE_PERM: IFF_XMIT_DST_RELEASE not taking into account
1639   *	underlying stacked devices
1640   * @IFF_L3MDEV_MASTER: device is an L3 master device
1641   * @IFF_NO_QUEUE: device can run without qdisc attached
1642   * @IFF_OPENVSWITCH: device is a Open vSwitch master
1643   * @IFF_L3MDEV_SLAVE: device is enslaved to an L3 master device
1644   * @IFF_TEAM: device is a team device
1645   * @IFF_RXFH_CONFIGURED: device has had Rx Flow indirection table configured
1646   * @IFF_PHONY_HEADROOM: the headroom value is controlled by an external
1647   *	entity (i.e. the master device for bridged veth)
1648   * @IFF_MACSEC: device is a MACsec device
1649   * @IFF_NO_RX_HANDLER: device doesn't support the rx_handler hook
1650   * @IFF_FAILOVER: device is a failover master device
1651   * @IFF_FAILOVER_SLAVE: device is lower dev of a failover master device
1652   * @IFF_L3MDEV_RX_HANDLER: only invoke the rx handler of L3 master device
1653   * @IFF_NO_ADDRCONF: prevent ipv6 addrconf
1654   * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with
1655   *	skb_headlen(skb) == 0 (data starts from frag0)
1656   */
1657  enum netdev_priv_flags {
1658  	IFF_802_1Q_VLAN			= 1<<0,
1659  	IFF_EBRIDGE			= 1<<1,
1660  	IFF_BONDING			= 1<<2,
1661  	IFF_ISATAP			= 1<<3,
1662  	IFF_WAN_HDLC			= 1<<4,
1663  	IFF_XMIT_DST_RELEASE		= 1<<5,
1664  	IFF_DONT_BRIDGE			= 1<<6,
1665  	IFF_DISABLE_NETPOLL		= 1<<7,
1666  	IFF_MACVLAN_PORT		= 1<<8,
1667  	IFF_BRIDGE_PORT			= 1<<9,
1668  	IFF_OVS_DATAPATH		= 1<<10,
1669  	IFF_TX_SKB_SHARING		= 1<<11,
1670  	IFF_UNICAST_FLT			= 1<<12,
1671  	IFF_TEAM_PORT			= 1<<13,
1672  	IFF_SUPP_NOFCS			= 1<<14,
1673  	IFF_LIVE_ADDR_CHANGE		= 1<<15,
1674  	IFF_MACVLAN			= 1<<16,
1675  	IFF_XMIT_DST_RELEASE_PERM	= 1<<17,
1676  	IFF_L3MDEV_MASTER		= 1<<18,
1677  	IFF_NO_QUEUE			= 1<<19,
1678  	IFF_OPENVSWITCH			= 1<<20,
1679  	IFF_L3MDEV_SLAVE		= 1<<21,
1680  	IFF_TEAM			= 1<<22,
1681  	IFF_RXFH_CONFIGURED		= 1<<23,
1682  	IFF_PHONY_HEADROOM		= 1<<24,
1683  	IFF_MACSEC			= 1<<25,
1684  	IFF_NO_RX_HANDLER		= 1<<26,
1685  	IFF_FAILOVER			= 1<<27,
1686  	IFF_FAILOVER_SLAVE		= 1<<28,
1687  	IFF_L3MDEV_RX_HANDLER		= 1<<29,
1688  	IFF_NO_ADDRCONF			= BIT_ULL(30),
1689  	IFF_TX_SKB_NO_LINEAR		= BIT_ULL(31),
1690  };
1691  
1692  /* Specifies the type of the struct net_device::ml_priv pointer */
1693  enum netdev_ml_priv_type {
1694  	ML_PRIV_NONE,
1695  	ML_PRIV_CAN,
1696  };
1697  
1698  enum netdev_stat_type {
1699  	NETDEV_PCPU_STAT_NONE,
1700  	NETDEV_PCPU_STAT_LSTATS, /* struct pcpu_lstats */
1701  	NETDEV_PCPU_STAT_TSTATS, /* struct pcpu_sw_netstats */
1702  	NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */
1703  };
1704  
1705  enum netdev_reg_state {
1706  	NETREG_UNINITIALIZED = 0,
1707  	NETREG_REGISTERED,	/* completed register_netdevice */
1708  	NETREG_UNREGISTERING,	/* called unregister_netdevice */
1709  	NETREG_UNREGISTERED,	/* completed unregister todo */
1710  	NETREG_RELEASED,	/* called free_netdev */
1711  	NETREG_DUMMY,		/* dummy device for NAPI poll */
1712  };
1713  
1714  /**
1715   *	struct net_device - The DEVICE structure.
1716   *
1717   *	Actually, this whole structure is a big mistake.  It mixes I/O
1718   *	data with strictly "high-level" data, and it has to know about
1719   *	almost every data structure used in the INET module.
1720   *
1721   *	@priv_flags:	flags invisible to userspace defined as bits, see
1722   *			enum netdev_priv_flags for the definitions
1723   *	@lltx:		device supports lockless Tx. Deprecated for real HW
1724   *			drivers. Mainly used by logical interfaces, such as
1725   *			bonding and tunnels
1726   *
1727   *	@name:	This is the first field of the "visible" part of this structure
1728   *		(i.e. as seen by users in the "Space.c" file).  It is the name
1729   *		of the interface.
1730   *
1731   *	@name_node:	Name hashlist node
1732   *	@ifalias:	SNMP alias
1733   *	@mem_end:	Shared memory end
1734   *	@mem_start:	Shared memory start
1735   *	@base_addr:	Device I/O address
1736   *	@irq:		Device IRQ number
1737   *
1738   *	@state:		Generic network queuing layer state, see netdev_state_t
1739   *	@dev_list:	The global list of network devices
1740   *	@napi_list:	List entry used for polling NAPI devices
1741   *	@unreg_list:	List entry  when we are unregistering the
1742   *			device; see the function unregister_netdev
1743   *	@close_list:	List entry used when we are closing the device
1744   *	@ptype_all:     Device-specific packet handlers for all protocols
1745   *	@ptype_specific: Device-specific, protocol-specific packet handlers
1746   *
1747   *	@adj_list:	Directly linked devices, like slaves for bonding
1748   *	@features:	Currently active device features
1749   *	@hw_features:	User-changeable features
1750   *
1751   *	@wanted_features:	User-requested features
1752   *	@vlan_features:		Mask of features inheritable by VLAN devices
1753   *
1754   *	@hw_enc_features:	Mask of features inherited by encapsulating devices
1755   *				This field indicates what encapsulation
1756   *				offloads the hardware is capable of doing,
1757   *				and drivers will need to set them appropriately.
1758   *
1759   *	@mpls_features:	Mask of features inheritable by MPLS
1760   *	@gso_partial_features: value(s) from NETIF_F_GSO\*
1761   *
1762   *	@ifindex:	interface index
1763   *	@group:		The group the device belongs to
1764   *
1765   *	@stats:		Statistics struct, which was left as a legacy, use
1766   *			rtnl_link_stats64 instead
1767   *
1768   *	@core_stats:	core networking counters,
1769   *			do not use this in drivers
1770   *	@carrier_up_count:	Number of times the carrier has been up
1771   *	@carrier_down_count:	Number of times the carrier has been down
1772   *
1773   *	@wireless_handlers:	List of functions to handle Wireless Extensions,
1774   *				instead of ioctl,
1775   *				see <net/iw_handler.h> for details.
1776   *	@wireless_data:	Instance data managed by the core of wireless extensions
1777   *
1778   *	@netdev_ops:	Includes several pointers to callbacks,
1779   *			if one wants to override the ndo_*() functions
1780   *	@xdp_metadata_ops:	Includes pointers to XDP metadata callbacks.
1781   *	@xsk_tx_metadata_ops:	Includes pointers to AF_XDP TX metadata callbacks.
1782   *	@ethtool_ops:	Management operations
1783   *	@l3mdev_ops:	Layer 3 master device operations
1784   *	@ndisc_ops:	Includes callbacks for different IPv6 neighbour
1785   *			discovery handling. Necessary for e.g. 6LoWPAN.
1786   *	@xfrmdev_ops:	Transformation offload operations
1787   *	@tlsdev_ops:	Transport Layer Security offload operations
1788   *	@header_ops:	Includes callbacks for creating,parsing,caching,etc
1789   *			of Layer 2 headers.
1790   *
1791   *	@flags:		Interface flags (a la BSD)
1792   *	@xdp_features:	XDP capability supported by the device
1793   *	@gflags:	Global flags ( kept as legacy )
1794   *	@priv_len:	Size of the ->priv flexible array
1795   *	@priv:		Flexible array containing private data
1796   *	@operstate:	RFC2863 operstate
1797   *	@link_mode:	Mapping policy to operstate
1798   *	@if_port:	Selectable AUI, TP, ...
1799   *	@dma:		DMA channel
1800   *	@mtu:		Interface MTU value
1801   *	@min_mtu:	Interface Minimum MTU value
1802   *	@max_mtu:	Interface Maximum MTU value
1803   *	@type:		Interface hardware type
1804   *	@hard_header_len: Maximum hardware header length.
1805   *	@min_header_len:  Minimum hardware header length
1806   *
1807   *	@needed_headroom: Extra headroom the hardware may need, but not in all
1808   *			  cases can this be guaranteed
1809   *	@needed_tailroom: Extra tailroom the hardware may need, but not in all
1810   *			  cases can this be guaranteed. Some cases also use
1811   *			  LL_MAX_HEADER instead to allocate the skb
1812   *
1813   *	interface address info:
1814   *
1815   * 	@perm_addr:		Permanent hw address
1816   * 	@addr_assign_type:	Hw address assignment type
1817   * 	@addr_len:		Hardware address length
1818   *	@upper_level:		Maximum depth level of upper devices.
1819   *	@lower_level:		Maximum depth level of lower devices.
1820   *	@neigh_priv_len:	Used in neigh_alloc()
1821   * 	@dev_id:		Used to differentiate devices that share
1822   * 				the same link layer address
1823   * 	@dev_port:		Used to differentiate devices that share
1824   * 				the same function
1825   *	@addr_list_lock:	XXX: need comments on this one
1826   *	@name_assign_type:	network interface name assignment type
1827   *	@uc_promisc:		Counter that indicates promiscuous mode
1828   *				has been enabled due to the need to listen to
1829   *				additional unicast addresses in a device that
1830   *				does not implement ndo_set_rx_mode()
1831   *	@uc:			unicast mac addresses
1832   *	@mc:			multicast mac addresses
1833   *	@dev_addrs:		list of device hw addresses
1834   *	@queues_kset:		Group of all Kobjects in the Tx and RX queues
1835   *	@promiscuity:		Number of times the NIC is told to work in
1836   *				promiscuous mode; if it becomes 0 the NIC will
1837   *				exit promiscuous mode
1838   *	@allmulti:		Counter, enables or disables allmulticast mode
1839   *
1840   *	@vlan_info:	VLAN info
1841   *	@dsa_ptr:	dsa specific data
1842   *	@tipc_ptr:	TIPC specific data
1843   *	@atalk_ptr:	AppleTalk link
1844   *	@ip_ptr:	IPv4 specific data
1845   *	@ip6_ptr:	IPv6 specific data
1846   *	@ax25_ptr:	AX.25 specific data
1847   *	@ieee80211_ptr:	IEEE 802.11 specific data, assign before registering
1848   *	@ieee802154_ptr: IEEE 802.15.4 low-rate Wireless Personal Area Network
1849   *			 device struct
1850   *	@mpls_ptr:	mpls_dev struct pointer
1851   *	@mctp_ptr:	MCTP specific data
1852   *
1853   *	@dev_addr:	Hw address (before bcast,
1854   *			because most packets are unicast)
1855   *
1856   *	@_rx:			Array of RX queues
1857   *	@num_rx_queues:		Number of RX queues
1858   *				allocated at register_netdev() time
1859   *	@real_num_rx_queues: 	Number of RX queues currently active in device
1860   *	@xdp_prog:		XDP sockets filter program pointer
1861   *	@gro_flush_timeout:	timeout for GRO layer in NAPI
1862   *	@napi_defer_hard_irqs:	If not zero, provides a counter that would
1863   *				allow to avoid NIC hard IRQ, on busy queues.
1864   *
1865   *	@rx_handler:		handler for received packets
1866   *	@rx_handler_data: 	XXX: need comments on this one
1867   *	@tcx_ingress:		BPF & clsact qdisc specific data for ingress processing
1868   *	@ingress_queue:		XXX: need comments on this one
1869   *	@nf_hooks_ingress:	netfilter hooks executed for ingress packets
1870   *	@broadcast:		hw bcast address
1871   *
1872   *	@rx_cpu_rmap:	CPU reverse-mapping for RX completion interrupts,
1873   *			indexed by RX queue number. Assigned by driver.
1874   *			This must only be set if the ndo_rx_flow_steer
1875   *			operation is defined
1876   *	@index_hlist:		Device index hash chain
1877   *
1878   *	@_tx:			Array of TX queues
1879   *	@num_tx_queues:		Number of TX queues allocated at alloc_netdev_mq() time
1880   *	@real_num_tx_queues: 	Number of TX queues currently active in device
1881   *	@qdisc:			Root qdisc from userspace point of view
1882   *	@tx_queue_len:		Max frames per queue allowed
1883   *	@tx_global_lock: 	XXX: need comments on this one
1884   *	@xdp_bulkq:		XDP device bulk queue
1885   *	@xps_maps:		all CPUs/RXQs maps for XPS device
1886   *
1887   *	@xps_maps:	XXX: need comments on this one
1888   *	@tcx_egress:		BPF & clsact qdisc specific data for egress processing
1889   *	@nf_hooks_egress:	netfilter hooks executed for egress packets
1890   *	@qdisc_hash:		qdisc hash table
1891   *	@watchdog_timeo:	Represents the timeout that is used by
1892   *				the watchdog (see dev_watchdog())
1893   *	@watchdog_timer:	List of timers
1894   *
1895   *	@proto_down_reason:	reason a netdev interface is held down
1896   *	@pcpu_refcnt:		Number of references to this device
1897   *	@dev_refcnt:		Number of references to this device
1898   *	@refcnt_tracker:	Tracker directory for tracked references to this device
1899   *	@todo_list:		Delayed register/unregister
1900   *	@link_watch_list:	XXX: need comments on this one
1901   *
1902   *	@reg_state:		Register/unregister state machine
1903   *	@dismantle:		Device is going to be freed
1904   *	@rtnl_link_state:	This enum represents the phases of creating
1905   *				a new link
1906   *
1907   *	@needs_free_netdev:	Should unregister perform free_netdev?
1908   *	@priv_destructor:	Called from unregister
1909   *	@npinfo:		XXX: need comments on this one
1910   * 	@nd_net:		Network namespace this network device is inside
1911   *
1912   * 	@ml_priv:	Mid-layer private
1913   *	@ml_priv_type:  Mid-layer private type
1914   *
1915   *	@pcpu_stat_type:	Type of device statistics which the core should
1916   *				allocate/free: none, lstats, tstats, dstats. none
1917   *				means the driver is handling statistics allocation/
1918   *				freeing internally.
1919   *	@lstats:		Loopback statistics: packets, bytes
1920   *	@tstats:		Tunnel statistics: RX/TX packets, RX/TX bytes
1921   *	@dstats:		Dummy statistics: RX/TX/drop packets, RX/TX bytes
1922   *
1923   *	@garp_port:	GARP
1924   *	@mrp_port:	MRP
1925   *
1926   *	@dm_private:	Drop monitor private
1927   *
1928   *	@dev:		Class/net/name entry
1929   *	@sysfs_groups:	Space for optional device, statistics and wireless
1930   *			sysfs groups
1931   *
1932   *	@sysfs_rx_queue_group:	Space for optional per-rx queue attributes
1933   *	@rtnl_link_ops:	Rtnl_link_ops
1934   *	@stat_ops:	Optional ops for queue-aware statistics
1935   *	@queue_mgmt_ops:	Optional ops for queue management
1936   *
1937   *	@gso_max_size:	Maximum size of generic segmentation offload
1938   *	@tso_max_size:	Device (as in HW) limit on the max TSO request size
1939   *	@gso_max_segs:	Maximum number of segments that can be passed to the
1940   *			NIC for GSO
1941   *	@tso_max_segs:	Device (as in HW) limit on the max TSO segment count
1942   * 	@gso_ipv4_max_size:	Maximum size of generic segmentation offload,
1943   * 				for IPv4.
1944   *
1945   *	@dcbnl_ops:	Data Center Bridging netlink ops
1946   *	@num_tc:	Number of traffic classes in the net device
1947   *	@tc_to_txq:	XXX: need comments on this one
1948   *	@prio_tc_map:	XXX: need comments on this one
1949   *
1950   *	@fcoe_ddp_xid:	Max exchange id for FCoE LRO by ddp
1951   *
1952   *	@priomap:	XXX: need comments on this one
1953   *	@link_topo:	Physical link topology tracking attached PHYs
1954   *	@phydev:	Physical device may attach itself
1955   *			for hardware timestamping
1956   *	@sfp_bus:	attached &struct sfp_bus structure.
1957   *
1958   *	@qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock
1959   *
1960   *	@proto_down:	protocol port state information can be sent to the
1961   *			switch driver and used to set the phys state of the
1962   *			switch port.
1963   *
1964   *	@threaded:	napi threaded mode is enabled
1965   *
1966   *	@see_all_hwtstamp_requests: device wants to see calls to
1967   *			ndo_hwtstamp_set() for all timestamp requests
1968   *			regardless of source, even if those aren't
1969   *			HWTSTAMP_SOURCE_NETDEV
1970   *	@change_proto_down: device supports setting carrier via IFLA_PROTO_DOWN
1971   *	@netns_local: interface can't change network namespaces
1972   *	@fcoe_mtu:	device supports maximum FCoE MTU, 2158 bytes
1973   *
1974   *	@net_notifier_list:	List of per-net netdev notifier block
1975   *				that follow this device when it is moved
1976   *				to another network namespace.
1977   *
1978   *	@macsec_ops:    MACsec offloading ops
1979   *
1980   *	@udp_tunnel_nic_info:	static structure describing the UDP tunnel
1981   *				offload capabilities of the device
1982   *	@udp_tunnel_nic:	UDP tunnel offload state
1983   *	@ethtool:	ethtool related state
1984   *	@xdp_state:		stores info on attached XDP BPF programs
1985   *
1986   *	@nested_level:	Used as a parameter of spin_lock_nested() of
1987   *			dev->addr_list_lock.
1988   *	@unlink_list:	As netif_addr_lock() can be called recursively,
1989   *			keep a list of interfaces to be deleted.
1990   *	@gro_max_size:	Maximum size of aggregated packet in generic
1991   *			receive offload (GRO)
1992   * 	@gro_ipv4_max_size:	Maximum size of aggregated packet in generic
1993   * 				receive offload (GRO), for IPv4.
1994   *	@xdp_zc_max_segs:	Maximum number of segments supported by AF_XDP
1995   *				zero copy driver
1996   *
1997   *	@dev_addr_shadow:	Copy of @dev_addr to catch direct writes.
1998   *	@linkwatch_dev_tracker:	refcount tracker used by linkwatch.
1999   *	@watchdog_dev_tracker:	refcount tracker used by watchdog.
2000   *	@dev_registered_tracker:	tracker for reference held while
2001   *					registered
2002   *	@offload_xstats_l3:	L3 HW stats for this netdevice.
2003   *
2004   *	@devlink_port:	Pointer to related devlink port structure.
2005   *			Assigned by a driver before netdev registration using
2006   *			SET_NETDEV_DEVLINK_PORT macro. This pointer is static
2007   *			during the time netdevice is registered.
2008   *
2009   *	@dpll_pin: Pointer to the SyncE source pin of a DPLL subsystem,
2010   *		   where the clock is recovered.
2011   *
2012   *	FIXME: cleanup struct net_device such that network protocol info
2013   *	moves out.
2014   */
2015  
2016  struct net_device {
2017  	/* Cacheline organization can be found documented in
2018  	 * Documentation/networking/net_cachelines/net_device.rst.
2019  	 * Please update the document when adding new fields.
2020  	 */
2021  
2022  	/* TX read-mostly hotpath */
2023  	__cacheline_group_begin(net_device_read_tx);
2024  	struct_group(priv_flags_fast,
2025  		unsigned long		priv_flags:32;
2026  		unsigned long		lltx:1;
2027  	);
2028  	const struct net_device_ops *netdev_ops;
2029  	const struct header_ops *header_ops;
2030  	struct netdev_queue	*_tx;
2031  	netdev_features_t	gso_partial_features;
2032  	unsigned int		real_num_tx_queues;
2033  	unsigned int		gso_max_size;
2034  	unsigned int		gso_ipv4_max_size;
2035  	u16			gso_max_segs;
2036  	s16			num_tc;
2037  	/* Note : dev->mtu is often read without holding a lock.
2038  	 * Writers usually hold RTNL.
2039  	 * It is recommended to use READ_ONCE() to annotate the reads,
2040  	 * and to use WRITE_ONCE() to annotate the writes.
2041  	 */
2042  	unsigned int		mtu;
2043  	unsigned short		needed_headroom;
2044  	struct netdev_tc_txq	tc_to_txq[TC_MAX_QUEUE];
2045  #ifdef CONFIG_XPS
2046  	struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX];
2047  #endif
2048  #ifdef CONFIG_NETFILTER_EGRESS
2049  	struct nf_hook_entries __rcu *nf_hooks_egress;
2050  #endif
2051  #ifdef CONFIG_NET_XGRESS
2052  	struct bpf_mprog_entry __rcu *tcx_egress;
2053  #endif
2054  	__cacheline_group_end(net_device_read_tx);
2055  
2056  	/* TXRX read-mostly hotpath */
2057  	__cacheline_group_begin(net_device_read_txrx);
2058  	union {
2059  		struct pcpu_lstats __percpu		*lstats;
2060  		struct pcpu_sw_netstats __percpu	*tstats;
2061  		struct pcpu_dstats __percpu		*dstats;
2062  	};
2063  	unsigned long		state;
2064  	unsigned int		flags;
2065  	unsigned short		hard_header_len;
2066  	netdev_features_t	features;
2067  	struct inet6_dev __rcu	*ip6_ptr;
2068  	__cacheline_group_end(net_device_read_txrx);
2069  
2070  	/* RX read-mostly hotpath */
2071  	__cacheline_group_begin(net_device_read_rx);
2072  	struct bpf_prog __rcu	*xdp_prog;
2073  	struct list_head	ptype_specific;
2074  	int			ifindex;
2075  	unsigned int		real_num_rx_queues;
2076  	struct netdev_rx_queue	*_rx;
2077  	unsigned long		gro_flush_timeout;
2078  	u32			napi_defer_hard_irqs;
2079  	unsigned int		gro_max_size;
2080  	unsigned int		gro_ipv4_max_size;
2081  	rx_handler_func_t __rcu	*rx_handler;
2082  	void __rcu		*rx_handler_data;
2083  	possible_net_t			nd_net;
2084  #ifdef CONFIG_NETPOLL
2085  	struct netpoll_info __rcu	*npinfo;
2086  #endif
2087  #ifdef CONFIG_NET_XGRESS
2088  	struct bpf_mprog_entry __rcu *tcx_ingress;
2089  #endif
2090  	__cacheline_group_end(net_device_read_rx);
2091  
2092  	char			name[IFNAMSIZ];
2093  	struct netdev_name_node	*name_node;
2094  	struct dev_ifalias	__rcu *ifalias;
2095  	/*
2096  	 *	I/O specific fields
2097  	 *	FIXME: Merge these and struct ifmap into one
2098  	 */
2099  	unsigned long		mem_end;
2100  	unsigned long		mem_start;
2101  	unsigned long		base_addr;
2102  
2103  	/*
2104  	 *	Some hardware also needs these fields (state,dev_list,
2105  	 *	napi_list,unreg_list,close_list) but they are not
2106  	 *	part of the usual set specified in Space.c.
2107  	 */
2108  
2109  
2110  	struct list_head	dev_list;
2111  	struct list_head	napi_list;
2112  	struct list_head	unreg_list;
2113  	struct list_head	close_list;
2114  	struct list_head	ptype_all;
2115  
2116  	struct {
2117  		struct list_head upper;
2118  		struct list_head lower;
2119  	} adj_list;
2120  
2121  	/* Read-mostly cache-line for fast-path access */
2122  	xdp_features_t		xdp_features;
2123  	const struct xdp_metadata_ops *xdp_metadata_ops;
2124  	const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops;
2125  	unsigned short		gflags;
2126  
2127  	unsigned short		needed_tailroom;
2128  
2129  	netdev_features_t	hw_features;
2130  	netdev_features_t	wanted_features;
2131  	netdev_features_t	vlan_features;
2132  	netdev_features_t	hw_enc_features;
2133  	netdev_features_t	mpls_features;
2134  
2135  	unsigned int		min_mtu;
2136  	unsigned int		max_mtu;
2137  	unsigned short		type;
2138  	unsigned char		min_header_len;
2139  	unsigned char		name_assign_type;
2140  
2141  	int			group;
2142  
2143  	struct net_device_stats	stats; /* not used by modern drivers */
2144  
2145  	struct net_device_core_stats __percpu *core_stats;
2146  
2147  	/* Stats to monitor link on/off, flapping */
2148  	atomic_t		carrier_up_count;
2149  	atomic_t		carrier_down_count;
2150  
2151  #ifdef CONFIG_WIRELESS_EXT
2152  	const struct iw_handler_def *wireless_handlers;
2153  	struct iw_public_data	*wireless_data;
2154  #endif
2155  	const struct ethtool_ops *ethtool_ops;
2156  #ifdef CONFIG_NET_L3_MASTER_DEV
2157  	const struct l3mdev_ops	*l3mdev_ops;
2158  #endif
2159  #if IS_ENABLED(CONFIG_IPV6)
2160  	const struct ndisc_ops *ndisc_ops;
2161  #endif
2162  
2163  #ifdef CONFIG_XFRM_OFFLOAD
2164  	const struct xfrmdev_ops *xfrmdev_ops;
2165  #endif
2166  
2167  #if IS_ENABLED(CONFIG_TLS_DEVICE)
2168  	const struct tlsdev_ops *tlsdev_ops;
2169  #endif
2170  
2171  	unsigned int		operstate;
2172  	unsigned char		link_mode;
2173  
2174  	unsigned char		if_port;
2175  	unsigned char		dma;
2176  
2177  	/* Interface address info. */
2178  	unsigned char		perm_addr[MAX_ADDR_LEN];
2179  	unsigned char		addr_assign_type;
2180  	unsigned char		addr_len;
2181  	unsigned char		upper_level;
2182  	unsigned char		lower_level;
2183  
2184  	unsigned short		neigh_priv_len;
2185  	unsigned short          dev_id;
2186  	unsigned short          dev_port;
2187  	int			irq;
2188  	u32			priv_len;
2189  
2190  	spinlock_t		addr_list_lock;
2191  
2192  	struct netdev_hw_addr_list	uc;
2193  	struct netdev_hw_addr_list	mc;
2194  	struct netdev_hw_addr_list	dev_addrs;
2195  
2196  #ifdef CONFIG_SYSFS
2197  	struct kset		*queues_kset;
2198  #endif
2199  #ifdef CONFIG_LOCKDEP
2200  	struct list_head	unlink_list;
2201  #endif
2202  	unsigned int		promiscuity;
2203  	unsigned int		allmulti;
2204  	bool			uc_promisc;
2205  #ifdef CONFIG_LOCKDEP
2206  	unsigned char		nested_level;
2207  #endif
2208  
2209  
2210  	/* Protocol-specific pointers */
2211  	struct in_device __rcu	*ip_ptr;
2212  #if IS_ENABLED(CONFIG_VLAN_8021Q)
2213  	struct vlan_info __rcu	*vlan_info;
2214  #endif
2215  #if IS_ENABLED(CONFIG_NET_DSA)
2216  	struct dsa_port		*dsa_ptr;
2217  #endif
2218  #if IS_ENABLED(CONFIG_TIPC)
2219  	struct tipc_bearer __rcu *tipc_ptr;
2220  #endif
2221  #if IS_ENABLED(CONFIG_ATALK)
2222  	void 			*atalk_ptr;
2223  #endif
2224  #if IS_ENABLED(CONFIG_AX25)
2225  	void			*ax25_ptr;
2226  #endif
2227  #if IS_ENABLED(CONFIG_CFG80211)
2228  	struct wireless_dev	*ieee80211_ptr;
2229  #endif
2230  #if IS_ENABLED(CONFIG_IEEE802154) || IS_ENABLED(CONFIG_6LOWPAN)
2231  	struct wpan_dev		*ieee802154_ptr;
2232  #endif
2233  #if IS_ENABLED(CONFIG_MPLS_ROUTING)
2234  	struct mpls_dev __rcu	*mpls_ptr;
2235  #endif
2236  #if IS_ENABLED(CONFIG_MCTP)
2237  	struct mctp_dev __rcu	*mctp_ptr;
2238  #endif
2239  
2240  /*
2241   * Cache lines mostly used on receive path (including eth_type_trans())
2242   */
2243  	/* Interface address info used in eth_type_trans() */
2244  	const unsigned char	*dev_addr;
2245  
2246  	unsigned int		num_rx_queues;
2247  #define GRO_LEGACY_MAX_SIZE	65536u
2248  /* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
2249   * and shinfo->gso_segs is a 16bit field.
2250   */
2251  #define GRO_MAX_SIZE		(8 * 65535u)
2252  	unsigned int		xdp_zc_max_segs;
2253  	struct netdev_queue __rcu *ingress_queue;
2254  #ifdef CONFIG_NETFILTER_INGRESS
2255  	struct nf_hook_entries __rcu *nf_hooks_ingress;
2256  #endif
2257  
2258  	unsigned char		broadcast[MAX_ADDR_LEN];
2259  #ifdef CONFIG_RFS_ACCEL
2260  	struct cpu_rmap		*rx_cpu_rmap;
2261  #endif
2262  	struct hlist_node	index_hlist;
2263  
2264  /*
2265   * Cache lines mostly used on transmit path
2266   */
2267  	unsigned int		num_tx_queues;
2268  	struct Qdisc __rcu	*qdisc;
2269  	unsigned int		tx_queue_len;
2270  	spinlock_t		tx_global_lock;
2271  
2272  	struct xdp_dev_bulk_queue __percpu *xdp_bulkq;
2273  
2274  #ifdef CONFIG_NET_SCHED
2275  	DECLARE_HASHTABLE	(qdisc_hash, 4);
2276  #endif
2277  	/* These may be needed for future network-power-down code. */
2278  	struct timer_list	watchdog_timer;
2279  	int			watchdog_timeo;
2280  
2281  	u32                     proto_down_reason;
2282  
2283  	struct list_head	todo_list;
2284  
2285  #ifdef CONFIG_PCPU_DEV_REFCNT
2286  	int __percpu		*pcpu_refcnt;
2287  #else
2288  	refcount_t		dev_refcnt;
2289  #endif
2290  	struct ref_tracker_dir	refcnt_tracker;
2291  
2292  	struct list_head	link_watch_list;
2293  
2294  	u8 reg_state;
2295  
2296  	bool dismantle;
2297  
2298  	enum {
2299  		RTNL_LINK_INITIALIZED,
2300  		RTNL_LINK_INITIALIZING,
2301  	} rtnl_link_state:16;
2302  
2303  	bool needs_free_netdev;
2304  	void (*priv_destructor)(struct net_device *dev);
2305  
2306  	/* mid-layer private */
2307  	void				*ml_priv;
2308  	enum netdev_ml_priv_type	ml_priv_type;
2309  
2310  	enum netdev_stat_type		pcpu_stat_type:8;
2311  
2312  #if IS_ENABLED(CONFIG_GARP)
2313  	struct garp_port __rcu	*garp_port;
2314  #endif
2315  #if IS_ENABLED(CONFIG_MRP)
2316  	struct mrp_port __rcu	*mrp_port;
2317  #endif
2318  #if IS_ENABLED(CONFIG_NET_DROP_MONITOR)
2319  	struct dm_hw_stat_delta __rcu *dm_private;
2320  #endif
2321  	struct device		dev;
2322  	const struct attribute_group *sysfs_groups[4];
2323  	const struct attribute_group *sysfs_rx_queue_group;
2324  
2325  	const struct rtnl_link_ops *rtnl_link_ops;
2326  
2327  	const struct netdev_stat_ops *stat_ops;
2328  
2329  	const struct netdev_queue_mgmt_ops *queue_mgmt_ops;
2330  
2331  	/* for setting kernel sock attribute on TCP connection setup */
2332  #define GSO_MAX_SEGS		65535u
2333  #define GSO_LEGACY_MAX_SIZE	65536u
2334  /* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
2335   * and shinfo->gso_segs is a 16bit field.
2336   */
2337  #define GSO_MAX_SIZE		(8 * GSO_MAX_SEGS)
2338  
2339  #define TSO_LEGACY_MAX_SIZE	65536
2340  #define TSO_MAX_SIZE		UINT_MAX
2341  	unsigned int		tso_max_size;
2342  #define TSO_MAX_SEGS		U16_MAX
2343  	u16			tso_max_segs;
2344  
2345  #ifdef CONFIG_DCB
2346  	const struct dcbnl_rtnl_ops *dcbnl_ops;
2347  #endif
2348  	u8			prio_tc_map[TC_BITMASK + 1];
2349  
2350  #if IS_ENABLED(CONFIG_FCOE)
2351  	unsigned int		fcoe_ddp_xid;
2352  #endif
2353  #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2354  	struct netprio_map __rcu *priomap;
2355  #endif
2356  	struct phy_link_topology	*link_topo;
2357  	struct phy_device	*phydev;
2358  	struct sfp_bus		*sfp_bus;
2359  	struct lock_class_key	*qdisc_tx_busylock;
2360  	bool			proto_down;
2361  	bool			threaded;
2362  
2363  	/* priv_flags_slow, ungrouped to save space */
2364  	unsigned long		see_all_hwtstamp_requests:1;
2365  	unsigned long		change_proto_down:1;
2366  	unsigned long		netns_local:1;
2367  	unsigned long		fcoe_mtu:1;
2368  
2369  	struct list_head	net_notifier_list;
2370  
2371  #if IS_ENABLED(CONFIG_MACSEC)
2372  	/* MACsec management functions */
2373  	const struct macsec_ops *macsec_ops;
2374  #endif
2375  	const struct udp_tunnel_nic_info	*udp_tunnel_nic_info;
2376  	struct udp_tunnel_nic	*udp_tunnel_nic;
2377  
2378  	struct ethtool_netdev_state *ethtool;
2379  
2380  	/* protected by rtnl_lock */
2381  	struct bpf_xdp_entity	xdp_state[__MAX_XDP_MODE];
2382  
2383  	u8 dev_addr_shadow[MAX_ADDR_LEN];
2384  	netdevice_tracker	linkwatch_dev_tracker;
2385  	netdevice_tracker	watchdog_dev_tracker;
2386  	netdevice_tracker	dev_registered_tracker;
2387  	struct rtnl_hw_stats64	*offload_xstats_l3;
2388  
2389  	struct devlink_port	*devlink_port;
2390  
2391  #if IS_ENABLED(CONFIG_DPLL)
2392  	struct dpll_pin	__rcu	*dpll_pin;
2393  #endif
2394  #if IS_ENABLED(CONFIG_PAGE_POOL)
2395  	/** @page_pools: page pools created for this netdevice */
2396  	struct hlist_head	page_pools;
2397  #endif
2398  
2399  	/** @irq_moder: dim parameters used if IS_ENABLED(CONFIG_DIMLIB). */
2400  	struct dim_irq_moder	*irq_moder;
2401  
2402  	u8			priv[] ____cacheline_aligned
2403  				       __counted_by(priv_len);
2404  } ____cacheline_aligned;
2405  #define to_net_dev(d) container_of(d, struct net_device, dev)
2406  
2407  /*
2408   * Driver should use this to assign devlink port instance to a netdevice
2409   * before it registers the netdevice. Therefore devlink_port is static
2410   * during the netdev lifetime after it is registered.
2411   */
2412  #define SET_NETDEV_DEVLINK_PORT(dev, port)			\
2413  ({								\
2414  	WARN_ON((dev)->reg_state != NETREG_UNINITIALIZED);	\
2415  	((dev)->devlink_port = (port));				\
2416  })
2417  
netif_elide_gro(const struct net_device * dev)2418  static inline bool netif_elide_gro(const struct net_device *dev)
2419  {
2420  	if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
2421  		return true;
2422  	return false;
2423  }
2424  
2425  #define	NETDEV_ALIGN		32
2426  
2427  static inline
netdev_get_prio_tc_map(const struct net_device * dev,u32 prio)2428  int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
2429  {
2430  	return dev->prio_tc_map[prio & TC_BITMASK];
2431  }
2432  
2433  static inline
netdev_set_prio_tc_map(struct net_device * dev,u8 prio,u8 tc)2434  int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
2435  {
2436  	if (tc >= dev->num_tc)
2437  		return -EINVAL;
2438  
2439  	dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
2440  	return 0;
2441  }
2442  
2443  int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
2444  void netdev_reset_tc(struct net_device *dev);
2445  int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
2446  int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
2447  
2448  static inline
netdev_get_num_tc(struct net_device * dev)2449  int netdev_get_num_tc(struct net_device *dev)
2450  {
2451  	return dev->num_tc;
2452  }
2453  
net_prefetch(void * p)2454  static inline void net_prefetch(void *p)
2455  {
2456  	prefetch(p);
2457  #if L1_CACHE_BYTES < 128
2458  	prefetch((u8 *)p + L1_CACHE_BYTES);
2459  #endif
2460  }
2461  
net_prefetchw(void * p)2462  static inline void net_prefetchw(void *p)
2463  {
2464  	prefetchw(p);
2465  #if L1_CACHE_BYTES < 128
2466  	prefetchw((u8 *)p + L1_CACHE_BYTES);
2467  #endif
2468  }
2469  
2470  void netdev_unbind_sb_channel(struct net_device *dev,
2471  			      struct net_device *sb_dev);
2472  int netdev_bind_sb_channel_queue(struct net_device *dev,
2473  				 struct net_device *sb_dev,
2474  				 u8 tc, u16 count, u16 offset);
2475  int netdev_set_sb_channel(struct net_device *dev, u16 channel);
netdev_get_sb_channel(struct net_device * dev)2476  static inline int netdev_get_sb_channel(struct net_device *dev)
2477  {
2478  	return max_t(int, -dev->num_tc, 0);
2479  }
2480  
2481  static inline
netdev_get_tx_queue(const struct net_device * dev,unsigned int index)2482  struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
2483  					 unsigned int index)
2484  {
2485  	DEBUG_NET_WARN_ON_ONCE(index >= dev->num_tx_queues);
2486  	return &dev->_tx[index];
2487  }
2488  
skb_get_tx_queue(const struct net_device * dev,const struct sk_buff * skb)2489  static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
2490  						    const struct sk_buff *skb)
2491  {
2492  	return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
2493  }
2494  
netdev_for_each_tx_queue(struct net_device * dev,void (* f)(struct net_device *,struct netdev_queue *,void *),void * arg)2495  static inline void netdev_for_each_tx_queue(struct net_device *dev,
2496  					    void (*f)(struct net_device *,
2497  						      struct netdev_queue *,
2498  						      void *),
2499  					    void *arg)
2500  {
2501  	unsigned int i;
2502  
2503  	for (i = 0; i < dev->num_tx_queues; i++)
2504  		f(dev, &dev->_tx[i], arg);
2505  }
2506  
2507  #define netdev_lockdep_set_classes(dev)				\
2508  {								\
2509  	static struct lock_class_key qdisc_tx_busylock_key;	\
2510  	static struct lock_class_key qdisc_xmit_lock_key;	\
2511  	static struct lock_class_key dev_addr_list_lock_key;	\
2512  	unsigned int i;						\
2513  								\
2514  	(dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key;	\
2515  	lockdep_set_class(&(dev)->addr_list_lock,		\
2516  			  &dev_addr_list_lock_key);		\
2517  	for (i = 0; i < (dev)->num_tx_queues; i++)		\
2518  		lockdep_set_class(&(dev)->_tx[i]._xmit_lock,	\
2519  				  &qdisc_xmit_lock_key);	\
2520  }
2521  
2522  u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
2523  		     struct net_device *sb_dev);
2524  struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
2525  					 struct sk_buff *skb,
2526  					 struct net_device *sb_dev);
2527  
2528  /* returns the headroom that the master device needs to take in account
2529   * when forwarding to this dev
2530   */
netdev_get_fwd_headroom(struct net_device * dev)2531  static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
2532  {
2533  	return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
2534  }
2535  
netdev_set_rx_headroom(struct net_device * dev,int new_hr)2536  static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
2537  {
2538  	if (dev->netdev_ops->ndo_set_rx_headroom)
2539  		dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
2540  }
2541  
2542  /* set the device rx headroom to the dev's default */
netdev_reset_rx_headroom(struct net_device * dev)2543  static inline void netdev_reset_rx_headroom(struct net_device *dev)
2544  {
2545  	netdev_set_rx_headroom(dev, -1);
2546  }
2547  
netdev_get_ml_priv(struct net_device * dev,enum netdev_ml_priv_type type)2548  static inline void *netdev_get_ml_priv(struct net_device *dev,
2549  				       enum netdev_ml_priv_type type)
2550  {
2551  	if (dev->ml_priv_type != type)
2552  		return NULL;
2553  
2554  	return dev->ml_priv;
2555  }
2556  
netdev_set_ml_priv(struct net_device * dev,void * ml_priv,enum netdev_ml_priv_type type)2557  static inline void netdev_set_ml_priv(struct net_device *dev,
2558  				      void *ml_priv,
2559  				      enum netdev_ml_priv_type type)
2560  {
2561  	WARN(dev->ml_priv_type && dev->ml_priv_type != type,
2562  	     "Overwriting already set ml_priv_type (%u) with different ml_priv_type (%u)!\n",
2563  	     dev->ml_priv_type, type);
2564  	WARN(!dev->ml_priv_type && dev->ml_priv,
2565  	     "Overwriting already set ml_priv and ml_priv_type is ML_PRIV_NONE!\n");
2566  
2567  	dev->ml_priv = ml_priv;
2568  	dev->ml_priv_type = type;
2569  }
2570  
2571  /*
2572   * Net namespace inlines
2573   */
2574  static inline
dev_net(const struct net_device * dev)2575  struct net *dev_net(const struct net_device *dev)
2576  {
2577  	return read_pnet(&dev->nd_net);
2578  }
2579  
2580  static inline
dev_net_set(struct net_device * dev,struct net * net)2581  void dev_net_set(struct net_device *dev, struct net *net)
2582  {
2583  	write_pnet(&dev->nd_net, net);
2584  }
2585  
2586  /**
2587   *	netdev_priv - access network device private data
2588   *	@dev: network device
2589   *
2590   * Get network device private data
2591   */
netdev_priv(const struct net_device * dev)2592  static inline void *netdev_priv(const struct net_device *dev)
2593  {
2594  	return (void *)dev->priv;
2595  }
2596  
2597  /* Set the sysfs physical device reference for the network logical device
2598   * if set prior to registration will cause a symlink during initialization.
2599   */
2600  #define SET_NETDEV_DEV(net, pdev)	((net)->dev.parent = (pdev))
2601  
2602  /* Set the sysfs device type for the network logical device to allow
2603   * fine-grained identification of different network device types. For
2604   * example Ethernet, Wireless LAN, Bluetooth, WiMAX etc.
2605   */
2606  #define SET_NETDEV_DEVTYPE(net, devtype)	((net)->dev.type = (devtype))
2607  
2608  void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
2609  			  enum netdev_queue_type type,
2610  			  struct napi_struct *napi);
2611  
netif_napi_set_irq(struct napi_struct * napi,int irq)2612  static inline void netif_napi_set_irq(struct napi_struct *napi, int irq)
2613  {
2614  	napi->irq = irq;
2615  }
2616  
2617  /* Default NAPI poll() weight
2618   * Device drivers are strongly advised to not use bigger value
2619   */
2620  #define NAPI_POLL_WEIGHT 64
2621  
2622  void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
2623  			   int (*poll)(struct napi_struct *, int), int weight);
2624  
2625  /**
2626   * netif_napi_add() - initialize a NAPI context
2627   * @dev:  network device
2628   * @napi: NAPI context
2629   * @poll: polling function
2630   *
2631   * netif_napi_add() must be used to initialize a NAPI context prior to calling
2632   * *any* of the other NAPI-related functions.
2633   */
2634  static inline void
netif_napi_add(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int))2635  netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2636  	       int (*poll)(struct napi_struct *, int))
2637  {
2638  	netif_napi_add_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2639  }
2640  
2641  static inline void
netif_napi_add_tx_weight(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int weight)2642  netif_napi_add_tx_weight(struct net_device *dev,
2643  			 struct napi_struct *napi,
2644  			 int (*poll)(struct napi_struct *, int),
2645  			 int weight)
2646  {
2647  	set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
2648  	netif_napi_add_weight(dev, napi, poll, weight);
2649  }
2650  
2651  /**
2652   * netif_napi_add_tx() - initialize a NAPI context to be used for Tx only
2653   * @dev:  network device
2654   * @napi: NAPI context
2655   * @poll: polling function
2656   *
2657   * This variant of netif_napi_add() should be used from drivers using NAPI
2658   * to exclusively poll a TX queue.
2659   * This will avoid we add it into napi_hash[], thus polluting this hash table.
2660   */
netif_napi_add_tx(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int))2661  static inline void netif_napi_add_tx(struct net_device *dev,
2662  				     struct napi_struct *napi,
2663  				     int (*poll)(struct napi_struct *, int))
2664  {
2665  	netif_napi_add_tx_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2666  }
2667  
2668  /**
2669   *  __netif_napi_del - remove a NAPI context
2670   *  @napi: NAPI context
2671   *
2672   * Warning: caller must observe RCU grace period before freeing memory
2673   * containing @napi. Drivers might want to call this helper to combine
2674   * all the needed RCU grace periods into a single one.
2675   */
2676  void __netif_napi_del(struct napi_struct *napi);
2677  
2678  /**
2679   *  netif_napi_del - remove a NAPI context
2680   *  @napi: NAPI context
2681   *
2682   *  netif_napi_del() removes a NAPI context from the network device NAPI list
2683   */
netif_napi_del(struct napi_struct * napi)2684  static inline void netif_napi_del(struct napi_struct *napi)
2685  {
2686  	__netif_napi_del(napi);
2687  	synchronize_net();
2688  }
2689  
2690  struct packet_type {
2691  	__be16			type;	/* This is really htons(ether_type). */
2692  	bool			ignore_outgoing;
2693  	struct net_device	*dev;	/* NULL is wildcarded here	     */
2694  	netdevice_tracker	dev_tracker;
2695  	int			(*func) (struct sk_buff *,
2696  					 struct net_device *,
2697  					 struct packet_type *,
2698  					 struct net_device *);
2699  	void			(*list_func) (struct list_head *,
2700  					      struct packet_type *,
2701  					      struct net_device *);
2702  	bool			(*id_match)(struct packet_type *ptype,
2703  					    struct sock *sk);
2704  	struct net		*af_packet_net;
2705  	void			*af_packet_priv;
2706  	struct list_head	list;
2707  };
2708  
2709  struct offload_callbacks {
2710  	struct sk_buff		*(*gso_segment)(struct sk_buff *skb,
2711  						netdev_features_t features);
2712  	struct sk_buff		*(*gro_receive)(struct list_head *head,
2713  						struct sk_buff *skb);
2714  	int			(*gro_complete)(struct sk_buff *skb, int nhoff);
2715  };
2716  
2717  struct packet_offload {
2718  	__be16			 type;	/* This is really htons(ether_type). */
2719  	u16			 priority;
2720  	struct offload_callbacks callbacks;
2721  	struct list_head	 list;
2722  };
2723  
2724  /* often modified stats are per-CPU, other are shared (netdev->stats) */
2725  struct pcpu_sw_netstats {
2726  	u64_stats_t		rx_packets;
2727  	u64_stats_t		rx_bytes;
2728  	u64_stats_t		tx_packets;
2729  	u64_stats_t		tx_bytes;
2730  	struct u64_stats_sync   syncp;
2731  } __aligned(4 * sizeof(u64));
2732  
2733  struct pcpu_dstats {
2734  	u64_stats_t		rx_packets;
2735  	u64_stats_t		rx_bytes;
2736  	u64_stats_t		rx_drops;
2737  	u64_stats_t		tx_packets;
2738  	u64_stats_t		tx_bytes;
2739  	u64_stats_t		tx_drops;
2740  	struct u64_stats_sync	syncp;
2741  } __aligned(8 * sizeof(u64));
2742  
2743  struct pcpu_lstats {
2744  	u64_stats_t packets;
2745  	u64_stats_t bytes;
2746  	struct u64_stats_sync syncp;
2747  } __aligned(2 * sizeof(u64));
2748  
2749  void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes);
2750  
dev_sw_netstats_rx_add(struct net_device * dev,unsigned int len)2751  static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int len)
2752  {
2753  	struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
2754  
2755  	u64_stats_update_begin(&tstats->syncp);
2756  	u64_stats_add(&tstats->rx_bytes, len);
2757  	u64_stats_inc(&tstats->rx_packets);
2758  	u64_stats_update_end(&tstats->syncp);
2759  }
2760  
dev_sw_netstats_tx_add(struct net_device * dev,unsigned int packets,unsigned int len)2761  static inline void dev_sw_netstats_tx_add(struct net_device *dev,
2762  					  unsigned int packets,
2763  					  unsigned int len)
2764  {
2765  	struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
2766  
2767  	u64_stats_update_begin(&tstats->syncp);
2768  	u64_stats_add(&tstats->tx_bytes, len);
2769  	u64_stats_add(&tstats->tx_packets, packets);
2770  	u64_stats_update_end(&tstats->syncp);
2771  }
2772  
dev_lstats_add(struct net_device * dev,unsigned int len)2773  static inline void dev_lstats_add(struct net_device *dev, unsigned int len)
2774  {
2775  	struct pcpu_lstats *lstats = this_cpu_ptr(dev->lstats);
2776  
2777  	u64_stats_update_begin(&lstats->syncp);
2778  	u64_stats_add(&lstats->bytes, len);
2779  	u64_stats_inc(&lstats->packets);
2780  	u64_stats_update_end(&lstats->syncp);
2781  }
2782  
2783  #define __netdev_alloc_pcpu_stats(type, gfp)				\
2784  ({									\
2785  	typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
2786  	if (pcpu_stats)	{						\
2787  		int __cpu;						\
2788  		for_each_possible_cpu(__cpu) {				\
2789  			typeof(type) *stat;				\
2790  			stat = per_cpu_ptr(pcpu_stats, __cpu);		\
2791  			u64_stats_init(&stat->syncp);			\
2792  		}							\
2793  	}								\
2794  	pcpu_stats;							\
2795  })
2796  
2797  #define netdev_alloc_pcpu_stats(type)					\
2798  	__netdev_alloc_pcpu_stats(type, GFP_KERNEL)
2799  
2800  #define devm_netdev_alloc_pcpu_stats(dev, type)				\
2801  ({									\
2802  	typeof(type) __percpu *pcpu_stats = devm_alloc_percpu(dev, type);\
2803  	if (pcpu_stats) {						\
2804  		int __cpu;						\
2805  		for_each_possible_cpu(__cpu) {				\
2806  			typeof(type) *stat;				\
2807  			stat = per_cpu_ptr(pcpu_stats, __cpu);		\
2808  			u64_stats_init(&stat->syncp);			\
2809  		}							\
2810  	}								\
2811  	pcpu_stats;							\
2812  })
2813  
2814  enum netdev_lag_tx_type {
2815  	NETDEV_LAG_TX_TYPE_UNKNOWN,
2816  	NETDEV_LAG_TX_TYPE_RANDOM,
2817  	NETDEV_LAG_TX_TYPE_BROADCAST,
2818  	NETDEV_LAG_TX_TYPE_ROUNDROBIN,
2819  	NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
2820  	NETDEV_LAG_TX_TYPE_HASH,
2821  };
2822  
2823  enum netdev_lag_hash {
2824  	NETDEV_LAG_HASH_NONE,
2825  	NETDEV_LAG_HASH_L2,
2826  	NETDEV_LAG_HASH_L34,
2827  	NETDEV_LAG_HASH_L23,
2828  	NETDEV_LAG_HASH_E23,
2829  	NETDEV_LAG_HASH_E34,
2830  	NETDEV_LAG_HASH_VLAN_SRCMAC,
2831  	NETDEV_LAG_HASH_UNKNOWN,
2832  };
2833  
2834  struct netdev_lag_upper_info {
2835  	enum netdev_lag_tx_type tx_type;
2836  	enum netdev_lag_hash hash_type;
2837  };
2838  
2839  struct netdev_lag_lower_state_info {
2840  	u8 link_up : 1,
2841  	   tx_enabled : 1;
2842  };
2843  
2844  #include <linux/notifier.h>
2845  
2846  /* netdevice notifier chain. Please remember to update netdev_cmd_to_name()
2847   * and the rtnetlink notification exclusion list in rtnetlink_event() when
2848   * adding new types.
2849   */
2850  enum netdev_cmd {
2851  	NETDEV_UP	= 1,	/* For now you can't veto a device up/down */
2852  	NETDEV_DOWN,
2853  	NETDEV_REBOOT,		/* Tell a protocol stack a network interface
2854  				   detected a hardware crash and restarted
2855  				   - we can use this eg to kick tcp sessions
2856  				   once done */
2857  	NETDEV_CHANGE,		/* Notify device state change */
2858  	NETDEV_REGISTER,
2859  	NETDEV_UNREGISTER,
2860  	NETDEV_CHANGEMTU,	/* notify after mtu change happened */
2861  	NETDEV_CHANGEADDR,	/* notify after the address change */
2862  	NETDEV_PRE_CHANGEADDR,	/* notify before the address change */
2863  	NETDEV_GOING_DOWN,
2864  	NETDEV_CHANGENAME,
2865  	NETDEV_FEAT_CHANGE,
2866  	NETDEV_BONDING_FAILOVER,
2867  	NETDEV_PRE_UP,
2868  	NETDEV_PRE_TYPE_CHANGE,
2869  	NETDEV_POST_TYPE_CHANGE,
2870  	NETDEV_POST_INIT,
2871  	NETDEV_PRE_UNINIT,
2872  	NETDEV_RELEASE,
2873  	NETDEV_NOTIFY_PEERS,
2874  	NETDEV_JOIN,
2875  	NETDEV_CHANGEUPPER,
2876  	NETDEV_RESEND_IGMP,
2877  	NETDEV_PRECHANGEMTU,	/* notify before mtu change happened */
2878  	NETDEV_CHANGEINFODATA,
2879  	NETDEV_BONDING_INFO,
2880  	NETDEV_PRECHANGEUPPER,
2881  	NETDEV_CHANGELOWERSTATE,
2882  	NETDEV_UDP_TUNNEL_PUSH_INFO,
2883  	NETDEV_UDP_TUNNEL_DROP_INFO,
2884  	NETDEV_CHANGE_TX_QUEUE_LEN,
2885  	NETDEV_CVLAN_FILTER_PUSH_INFO,
2886  	NETDEV_CVLAN_FILTER_DROP_INFO,
2887  	NETDEV_SVLAN_FILTER_PUSH_INFO,
2888  	NETDEV_SVLAN_FILTER_DROP_INFO,
2889  	NETDEV_OFFLOAD_XSTATS_ENABLE,
2890  	NETDEV_OFFLOAD_XSTATS_DISABLE,
2891  	NETDEV_OFFLOAD_XSTATS_REPORT_USED,
2892  	NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
2893  	NETDEV_XDP_FEAT_CHANGE,
2894  };
2895  const char *netdev_cmd_to_name(enum netdev_cmd cmd);
2896  
2897  int register_netdevice_notifier(struct notifier_block *nb);
2898  int unregister_netdevice_notifier(struct notifier_block *nb);
2899  int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb);
2900  int unregister_netdevice_notifier_net(struct net *net,
2901  				      struct notifier_block *nb);
2902  int register_netdevice_notifier_dev_net(struct net_device *dev,
2903  					struct notifier_block *nb,
2904  					struct netdev_net_notifier *nn);
2905  int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2906  					  struct notifier_block *nb,
2907  					  struct netdev_net_notifier *nn);
2908  
2909  struct netdev_notifier_info {
2910  	struct net_device	*dev;
2911  	struct netlink_ext_ack	*extack;
2912  };
2913  
2914  struct netdev_notifier_info_ext {
2915  	struct netdev_notifier_info info; /* must be first */
2916  	union {
2917  		u32 mtu;
2918  	} ext;
2919  };
2920  
2921  struct netdev_notifier_change_info {
2922  	struct netdev_notifier_info info; /* must be first */
2923  	unsigned int flags_changed;
2924  };
2925  
2926  struct netdev_notifier_changeupper_info {
2927  	struct netdev_notifier_info info; /* must be first */
2928  	struct net_device *upper_dev; /* new upper dev */
2929  	bool master; /* is upper dev master */
2930  	bool linking; /* is the notification for link or unlink */
2931  	void *upper_info; /* upper dev info */
2932  };
2933  
2934  struct netdev_notifier_changelowerstate_info {
2935  	struct netdev_notifier_info info; /* must be first */
2936  	void *lower_state_info; /* is lower dev state */
2937  };
2938  
2939  struct netdev_notifier_pre_changeaddr_info {
2940  	struct netdev_notifier_info info; /* must be first */
2941  	const unsigned char *dev_addr;
2942  };
2943  
2944  enum netdev_offload_xstats_type {
2945  	NETDEV_OFFLOAD_XSTATS_TYPE_L3 = 1,
2946  };
2947  
2948  struct netdev_notifier_offload_xstats_info {
2949  	struct netdev_notifier_info info; /* must be first */
2950  	enum netdev_offload_xstats_type type;
2951  
2952  	union {
2953  		/* NETDEV_OFFLOAD_XSTATS_REPORT_DELTA */
2954  		struct netdev_notifier_offload_xstats_rd *report_delta;
2955  		/* NETDEV_OFFLOAD_XSTATS_REPORT_USED */
2956  		struct netdev_notifier_offload_xstats_ru *report_used;
2957  	};
2958  };
2959  
2960  int netdev_offload_xstats_enable(struct net_device *dev,
2961  				 enum netdev_offload_xstats_type type,
2962  				 struct netlink_ext_ack *extack);
2963  int netdev_offload_xstats_disable(struct net_device *dev,
2964  				  enum netdev_offload_xstats_type type);
2965  bool netdev_offload_xstats_enabled(const struct net_device *dev,
2966  				   enum netdev_offload_xstats_type type);
2967  int netdev_offload_xstats_get(struct net_device *dev,
2968  			      enum netdev_offload_xstats_type type,
2969  			      struct rtnl_hw_stats64 *stats, bool *used,
2970  			      struct netlink_ext_ack *extack);
2971  void
2972  netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *rd,
2973  				   const struct rtnl_hw_stats64 *stats);
2974  void
2975  netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *ru);
2976  void netdev_offload_xstats_push_delta(struct net_device *dev,
2977  				      enum netdev_offload_xstats_type type,
2978  				      const struct rtnl_hw_stats64 *stats);
2979  
netdev_notifier_info_init(struct netdev_notifier_info * info,struct net_device * dev)2980  static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
2981  					     struct net_device *dev)
2982  {
2983  	info->dev = dev;
2984  	info->extack = NULL;
2985  }
2986  
2987  static inline struct net_device *
netdev_notifier_info_to_dev(const struct netdev_notifier_info * info)2988  netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
2989  {
2990  	return info->dev;
2991  }
2992  
2993  static inline struct netlink_ext_ack *
netdev_notifier_info_to_extack(const struct netdev_notifier_info * info)2994  netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
2995  {
2996  	return info->extack;
2997  }
2998  
2999  int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
3000  int call_netdevice_notifiers_info(unsigned long val,
3001  				  struct netdev_notifier_info *info);
3002  
3003  #define for_each_netdev(net, d)		\
3004  		list_for_each_entry(d, &(net)->dev_base_head, dev_list)
3005  #define for_each_netdev_reverse(net, d)	\
3006  		list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
3007  #define for_each_netdev_rcu(net, d)		\
3008  		list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
3009  #define for_each_netdev_safe(net, d, n)	\
3010  		list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
3011  #define for_each_netdev_continue(net, d)		\
3012  		list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
3013  #define for_each_netdev_continue_reverse(net, d)		\
3014  		list_for_each_entry_continue_reverse(d, &(net)->dev_base_head, \
3015  						     dev_list)
3016  #define for_each_netdev_continue_rcu(net, d)		\
3017  	list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
3018  #define for_each_netdev_in_bond_rcu(bond, slave)	\
3019  		for_each_netdev_rcu(&init_net, slave)	\
3020  			if (netdev_master_upper_dev_get_rcu(slave) == (bond))
3021  #define net_device_entry(lh)	list_entry(lh, struct net_device, dev_list)
3022  
3023  #define for_each_netdev_dump(net, d, ifindex)				\
3024  	for (; (d = xa_find(&(net)->dev_by_index, &ifindex,		\
3025  			    ULONG_MAX, XA_PRESENT)); ifindex++)
3026  
next_net_device(struct net_device * dev)3027  static inline struct net_device *next_net_device(struct net_device *dev)
3028  {
3029  	struct list_head *lh;
3030  	struct net *net;
3031  
3032  	net = dev_net(dev);
3033  	lh = dev->dev_list.next;
3034  	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3035  }
3036  
next_net_device_rcu(struct net_device * dev)3037  static inline struct net_device *next_net_device_rcu(struct net_device *dev)
3038  {
3039  	struct list_head *lh;
3040  	struct net *net;
3041  
3042  	net = dev_net(dev);
3043  	lh = rcu_dereference(list_next_rcu(&dev->dev_list));
3044  	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3045  }
3046  
first_net_device(struct net * net)3047  static inline struct net_device *first_net_device(struct net *net)
3048  {
3049  	return list_empty(&net->dev_base_head) ? NULL :
3050  		net_device_entry(net->dev_base_head.next);
3051  }
3052  
first_net_device_rcu(struct net * net)3053  static inline struct net_device *first_net_device_rcu(struct net *net)
3054  {
3055  	struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
3056  
3057  	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3058  }
3059  
3060  int netdev_boot_setup_check(struct net_device *dev);
3061  struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
3062  				       const char *hwaddr);
3063  struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
3064  void dev_add_pack(struct packet_type *pt);
3065  void dev_remove_pack(struct packet_type *pt);
3066  void __dev_remove_pack(struct packet_type *pt);
3067  void dev_add_offload(struct packet_offload *po);
3068  void dev_remove_offload(struct packet_offload *po);
3069  
3070  int dev_get_iflink(const struct net_device *dev);
3071  int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
3072  int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
3073  			  struct net_device_path_stack *stack);
3074  struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
3075  				      unsigned short mask);
3076  struct net_device *dev_get_by_name(struct net *net, const char *name);
3077  struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
3078  struct net_device *__dev_get_by_name(struct net *net, const char *name);
3079  bool netdev_name_in_use(struct net *net, const char *name);
3080  int dev_alloc_name(struct net_device *dev, const char *name);
3081  int dev_open(struct net_device *dev, struct netlink_ext_ack *extack);
3082  void dev_close(struct net_device *dev);
3083  void dev_close_many(struct list_head *head, bool unlink);
3084  void dev_disable_lro(struct net_device *dev);
3085  int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
3086  u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
3087  		     struct net_device *sb_dev);
3088  
3089  int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev);
3090  int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
3091  
dev_queue_xmit(struct sk_buff * skb)3092  static inline int dev_queue_xmit(struct sk_buff *skb)
3093  {
3094  	return __dev_queue_xmit(skb, NULL);
3095  }
3096  
dev_queue_xmit_accel(struct sk_buff * skb,struct net_device * sb_dev)3097  static inline int dev_queue_xmit_accel(struct sk_buff *skb,
3098  				       struct net_device *sb_dev)
3099  {
3100  	return __dev_queue_xmit(skb, sb_dev);
3101  }
3102  
dev_direct_xmit(struct sk_buff * skb,u16 queue_id)3103  static inline int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3104  {
3105  	int ret;
3106  
3107  	ret = __dev_direct_xmit(skb, queue_id);
3108  	if (!dev_xmit_complete(ret))
3109  		kfree_skb(skb);
3110  	return ret;
3111  }
3112  
3113  int register_netdevice(struct net_device *dev);
3114  void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
3115  void unregister_netdevice_many(struct list_head *head);
unregister_netdevice(struct net_device * dev)3116  static inline void unregister_netdevice(struct net_device *dev)
3117  {
3118  	unregister_netdevice_queue(dev, NULL);
3119  }
3120  
3121  int netdev_refcnt_read(const struct net_device *dev);
3122  void free_netdev(struct net_device *dev);
3123  void init_dummy_netdev(struct net_device *dev);
3124  
3125  struct net_device *netdev_get_xmit_slave(struct net_device *dev,
3126  					 struct sk_buff *skb,
3127  					 bool all_slaves);
3128  struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
3129  					    struct sock *sk);
3130  struct net_device *dev_get_by_index(struct net *net, int ifindex);
3131  struct net_device *__dev_get_by_index(struct net *net, int ifindex);
3132  struct net_device *netdev_get_by_index(struct net *net, int ifindex,
3133  				       netdevice_tracker *tracker, gfp_t gfp);
3134  struct net_device *netdev_get_by_name(struct net *net, const char *name,
3135  				      netdevice_tracker *tracker, gfp_t gfp);
3136  struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
3137  struct net_device *dev_get_by_napi_id(unsigned int napi_id);
3138  void netdev_copy_name(struct net_device *dev, char *name);
3139  
dev_hard_header(struct sk_buff * skb,struct net_device * dev,unsigned short type,const void * daddr,const void * saddr,unsigned int len)3140  static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
3141  				  unsigned short type,
3142  				  const void *daddr, const void *saddr,
3143  				  unsigned int len)
3144  {
3145  	if (!dev->header_ops || !dev->header_ops->create)
3146  		return 0;
3147  
3148  	return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
3149  }
3150  
dev_parse_header(const struct sk_buff * skb,unsigned char * haddr)3151  static inline int dev_parse_header(const struct sk_buff *skb,
3152  				   unsigned char *haddr)
3153  {
3154  	const struct net_device *dev = skb->dev;
3155  
3156  	if (!dev->header_ops || !dev->header_ops->parse)
3157  		return 0;
3158  	return dev->header_ops->parse(skb, haddr);
3159  }
3160  
dev_parse_header_protocol(const struct sk_buff * skb)3161  static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb)
3162  {
3163  	const struct net_device *dev = skb->dev;
3164  
3165  	if (!dev->header_ops || !dev->header_ops->parse_protocol)
3166  		return 0;
3167  	return dev->header_ops->parse_protocol(skb);
3168  }
3169  
3170  /* ll_header must have at least hard_header_len allocated */
dev_validate_header(const struct net_device * dev,char * ll_header,int len)3171  static inline bool dev_validate_header(const struct net_device *dev,
3172  				       char *ll_header, int len)
3173  {
3174  	if (likely(len >= dev->hard_header_len))
3175  		return true;
3176  	if (len < dev->min_header_len)
3177  		return false;
3178  
3179  	if (capable(CAP_SYS_RAWIO)) {
3180  		memset(ll_header + len, 0, dev->hard_header_len - len);
3181  		return true;
3182  	}
3183  
3184  	if (dev->header_ops && dev->header_ops->validate)
3185  		return dev->header_ops->validate(ll_header, len);
3186  
3187  	return false;
3188  }
3189  
dev_has_header(const struct net_device * dev)3190  static inline bool dev_has_header(const struct net_device *dev)
3191  {
3192  	return dev->header_ops && dev->header_ops->create;
3193  }
3194  
3195  /*
3196   * Incoming packets are placed on per-CPU queues
3197   */
3198  struct softnet_data {
3199  	struct list_head	poll_list;
3200  	struct sk_buff_head	process_queue;
3201  	local_lock_t		process_queue_bh_lock;
3202  
3203  	/* stats */
3204  	unsigned int		processed;
3205  	unsigned int		time_squeeze;
3206  #ifdef CONFIG_RPS
3207  	struct softnet_data	*rps_ipi_list;
3208  #endif
3209  
3210  	unsigned int		received_rps;
3211  	bool			in_net_rx_action;
3212  	bool			in_napi_threaded_poll;
3213  
3214  #ifdef CONFIG_NET_FLOW_LIMIT
3215  	struct sd_flow_limit __rcu *flow_limit;
3216  #endif
3217  	struct Qdisc		*output_queue;
3218  	struct Qdisc		**output_queue_tailp;
3219  	struct sk_buff		*completion_queue;
3220  #ifdef CONFIG_XFRM_OFFLOAD
3221  	struct sk_buff_head	xfrm_backlog;
3222  #endif
3223  	/* written and read only by owning cpu: */
3224  	struct netdev_xmit xmit;
3225  #ifdef CONFIG_RPS
3226  	/* input_queue_head should be written by cpu owning this struct,
3227  	 * and only read by other cpus. Worth using a cache line.
3228  	 */
3229  	unsigned int		input_queue_head ____cacheline_aligned_in_smp;
3230  
3231  	/* Elements below can be accessed between CPUs for RPS/RFS */
3232  	call_single_data_t	csd ____cacheline_aligned_in_smp;
3233  	struct softnet_data	*rps_ipi_next;
3234  	unsigned int		cpu;
3235  	unsigned int		input_queue_tail;
3236  #endif
3237  	struct sk_buff_head	input_pkt_queue;
3238  	struct napi_struct	backlog;
3239  
3240  	atomic_t		dropped ____cacheline_aligned_in_smp;
3241  
3242  	/* Another possibly contended cache line */
3243  	spinlock_t		defer_lock ____cacheline_aligned_in_smp;
3244  	int			defer_count;
3245  	int			defer_ipi_scheduled;
3246  	struct sk_buff		*defer_list;
3247  	call_single_data_t	defer_csd;
3248  };
3249  
3250  DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
3251  
3252  #ifndef CONFIG_PREEMPT_RT
dev_recursion_level(void)3253  static inline int dev_recursion_level(void)
3254  {
3255  	return this_cpu_read(softnet_data.xmit.recursion);
3256  }
3257  #else
dev_recursion_level(void)3258  static inline int dev_recursion_level(void)
3259  {
3260  	return current->net_xmit.recursion;
3261  }
3262  
3263  #endif
3264  
3265  void __netif_schedule(struct Qdisc *q);
3266  void netif_schedule_queue(struct netdev_queue *txq);
3267  
netif_tx_schedule_all(struct net_device * dev)3268  static inline void netif_tx_schedule_all(struct net_device *dev)
3269  {
3270  	unsigned int i;
3271  
3272  	for (i = 0; i < dev->num_tx_queues; i++)
3273  		netif_schedule_queue(netdev_get_tx_queue(dev, i));
3274  }
3275  
netif_tx_start_queue(struct netdev_queue * dev_queue)3276  static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
3277  {
3278  	clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3279  }
3280  
3281  /**
3282   *	netif_start_queue - allow transmit
3283   *	@dev: network device
3284   *
3285   *	Allow upper layers to call the device hard_start_xmit routine.
3286   */
netif_start_queue(struct net_device * dev)3287  static inline void netif_start_queue(struct net_device *dev)
3288  {
3289  	netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
3290  }
3291  
netif_tx_start_all_queues(struct net_device * dev)3292  static inline void netif_tx_start_all_queues(struct net_device *dev)
3293  {
3294  	unsigned int i;
3295  
3296  	for (i = 0; i < dev->num_tx_queues; i++) {
3297  		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3298  		netif_tx_start_queue(txq);
3299  	}
3300  }
3301  
3302  void netif_tx_wake_queue(struct netdev_queue *dev_queue);
3303  
3304  /**
3305   *	netif_wake_queue - restart transmit
3306   *	@dev: network device
3307   *
3308   *	Allow upper layers to call the device hard_start_xmit routine.
3309   *	Used for flow control when transmit resources are available.
3310   */
netif_wake_queue(struct net_device * dev)3311  static inline void netif_wake_queue(struct net_device *dev)
3312  {
3313  	netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
3314  }
3315  
netif_tx_wake_all_queues(struct net_device * dev)3316  static inline void netif_tx_wake_all_queues(struct net_device *dev)
3317  {
3318  	unsigned int i;
3319  
3320  	for (i = 0; i < dev->num_tx_queues; i++) {
3321  		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3322  		netif_tx_wake_queue(txq);
3323  	}
3324  }
3325  
netif_tx_stop_queue(struct netdev_queue * dev_queue)3326  static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
3327  {
3328  	/* Paired with READ_ONCE() from dev_watchdog() */
3329  	WRITE_ONCE(dev_queue->trans_start, jiffies);
3330  
3331  	/* This barrier is paired with smp_mb() from dev_watchdog() */
3332  	smp_mb__before_atomic();
3333  
3334  	/* Must be an atomic op see netif_txq_try_stop() */
3335  	set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3336  }
3337  
3338  /**
3339   *	netif_stop_queue - stop transmitted packets
3340   *	@dev: network device
3341   *
3342   *	Stop upper layers calling the device hard_start_xmit routine.
3343   *	Used for flow control when transmit resources are unavailable.
3344   */
netif_stop_queue(struct net_device * dev)3345  static inline void netif_stop_queue(struct net_device *dev)
3346  {
3347  	netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
3348  }
3349  
3350  void netif_tx_stop_all_queues(struct net_device *dev);
3351  
netif_tx_queue_stopped(const struct netdev_queue * dev_queue)3352  static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
3353  {
3354  	return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3355  }
3356  
3357  /**
3358   *	netif_queue_stopped - test if transmit queue is flowblocked
3359   *	@dev: network device
3360   *
3361   *	Test if transmit queue on device is currently unable to send.
3362   */
netif_queue_stopped(const struct net_device * dev)3363  static inline bool netif_queue_stopped(const struct net_device *dev)
3364  {
3365  	return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
3366  }
3367  
netif_xmit_stopped(const struct netdev_queue * dev_queue)3368  static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
3369  {
3370  	return dev_queue->state & QUEUE_STATE_ANY_XOFF;
3371  }
3372  
3373  static inline bool
netif_xmit_frozen_or_stopped(const struct netdev_queue * dev_queue)3374  netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
3375  {
3376  	return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
3377  }
3378  
3379  static inline bool
netif_xmit_frozen_or_drv_stopped(const struct netdev_queue * dev_queue)3380  netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
3381  {
3382  	return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
3383  }
3384  
3385  /**
3386   *	netdev_queue_set_dql_min_limit - set dql minimum limit
3387   *	@dev_queue: pointer to transmit queue
3388   *	@min_limit: dql minimum limit
3389   *
3390   * Forces xmit_more() to return true until the minimum threshold
3391   * defined by @min_limit is reached (or until the tx queue is
3392   * empty). Warning: to be use with care, misuse will impact the
3393   * latency.
3394   */
netdev_queue_set_dql_min_limit(struct netdev_queue * dev_queue,unsigned int min_limit)3395  static inline void netdev_queue_set_dql_min_limit(struct netdev_queue *dev_queue,
3396  						  unsigned int min_limit)
3397  {
3398  #ifdef CONFIG_BQL
3399  	dev_queue->dql.min_limit = min_limit;
3400  #endif
3401  }
3402  
netdev_queue_dql_avail(const struct netdev_queue * txq)3403  static inline int netdev_queue_dql_avail(const struct netdev_queue *txq)
3404  {
3405  #ifdef CONFIG_BQL
3406  	/* Non-BQL migrated drivers will return 0, too. */
3407  	return dql_avail(&txq->dql);
3408  #else
3409  	return 0;
3410  #endif
3411  }
3412  
3413  /**
3414   *	netdev_txq_bql_enqueue_prefetchw - prefetch bql data for write
3415   *	@dev_queue: pointer to transmit queue
3416   *
3417   * BQL enabled drivers might use this helper in their ndo_start_xmit(),
3418   * to give appropriate hint to the CPU.
3419   */
netdev_txq_bql_enqueue_prefetchw(struct netdev_queue * dev_queue)3420  static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
3421  {
3422  #ifdef CONFIG_BQL
3423  	prefetchw(&dev_queue->dql.num_queued);
3424  #endif
3425  }
3426  
3427  /**
3428   *	netdev_txq_bql_complete_prefetchw - prefetch bql data for write
3429   *	@dev_queue: pointer to transmit queue
3430   *
3431   * BQL enabled drivers might use this helper in their TX completion path,
3432   * to give appropriate hint to the CPU.
3433   */
netdev_txq_bql_complete_prefetchw(struct netdev_queue * dev_queue)3434  static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
3435  {
3436  #ifdef CONFIG_BQL
3437  	prefetchw(&dev_queue->dql.limit);
3438  #endif
3439  }
3440  
3441  /**
3442   *	netdev_tx_sent_queue - report the number of bytes queued to a given tx queue
3443   *	@dev_queue: network device queue
3444   *	@bytes: number of bytes queued to the device queue
3445   *
3446   *	Report the number of bytes queued for sending/completion to the network
3447   *	device hardware queue. @bytes should be a good approximation and should
3448   *	exactly match netdev_completed_queue() @bytes.
3449   *	This is typically called once per packet, from ndo_start_xmit().
3450   */
netdev_tx_sent_queue(struct netdev_queue * dev_queue,unsigned int bytes)3451  static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3452  					unsigned int bytes)
3453  {
3454  #ifdef CONFIG_BQL
3455  	dql_queued(&dev_queue->dql, bytes);
3456  
3457  	if (likely(dql_avail(&dev_queue->dql) >= 0))
3458  		return;
3459  
3460  	/* Paired with READ_ONCE() from dev_watchdog() */
3461  	WRITE_ONCE(dev_queue->trans_start, jiffies);
3462  
3463  	/* This barrier is paired with smp_mb() from dev_watchdog() */
3464  	smp_mb__before_atomic();
3465  
3466  	set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3467  
3468  	/*
3469  	 * The XOFF flag must be set before checking the dql_avail below,
3470  	 * because in netdev_tx_completed_queue we update the dql_completed
3471  	 * before checking the XOFF flag.
3472  	 */
3473  	smp_mb();
3474  
3475  	/* check again in case another CPU has just made room avail */
3476  	if (unlikely(dql_avail(&dev_queue->dql) >= 0))
3477  		clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3478  #endif
3479  }
3480  
3481  /* Variant of netdev_tx_sent_queue() for drivers that are aware
3482   * that they should not test BQL status themselves.
3483   * We do want to change __QUEUE_STATE_STACK_XOFF only for the last
3484   * skb of a batch.
3485   * Returns true if the doorbell must be used to kick the NIC.
3486   */
__netdev_tx_sent_queue(struct netdev_queue * dev_queue,unsigned int bytes,bool xmit_more)3487  static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3488  					  unsigned int bytes,
3489  					  bool xmit_more)
3490  {
3491  	if (xmit_more) {
3492  #ifdef CONFIG_BQL
3493  		dql_queued(&dev_queue->dql, bytes);
3494  #endif
3495  		return netif_tx_queue_stopped(dev_queue);
3496  	}
3497  	netdev_tx_sent_queue(dev_queue, bytes);
3498  	return true;
3499  }
3500  
3501  /**
3502   *	netdev_sent_queue - report the number of bytes queued to hardware
3503   *	@dev: network device
3504   *	@bytes: number of bytes queued to the hardware device queue
3505   *
3506   *	Report the number of bytes queued for sending/completion to the network
3507   *	device hardware queue#0. @bytes should be a good approximation and should
3508   *	exactly match netdev_completed_queue() @bytes.
3509   *	This is typically called once per packet, from ndo_start_xmit().
3510   */
netdev_sent_queue(struct net_device * dev,unsigned int bytes)3511  static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
3512  {
3513  	netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
3514  }
3515  
__netdev_sent_queue(struct net_device * dev,unsigned int bytes,bool xmit_more)3516  static inline bool __netdev_sent_queue(struct net_device *dev,
3517  				       unsigned int bytes,
3518  				       bool xmit_more)
3519  {
3520  	return __netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes,
3521  				      xmit_more);
3522  }
3523  
3524  /**
3525   *	netdev_tx_completed_queue - report number of packets/bytes at TX completion.
3526   *	@dev_queue: network device queue
3527   *	@pkts: number of packets (currently ignored)
3528   *	@bytes: number of bytes dequeued from the device queue
3529   *
3530   *	Must be called at most once per TX completion round (and not per
3531   *	individual packet), so that BQL can adjust its limits appropriately.
3532   */
netdev_tx_completed_queue(struct netdev_queue * dev_queue,unsigned int pkts,unsigned int bytes)3533  static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
3534  					     unsigned int pkts, unsigned int bytes)
3535  {
3536  #ifdef CONFIG_BQL
3537  	if (unlikely(!bytes))
3538  		return;
3539  
3540  	dql_completed(&dev_queue->dql, bytes);
3541  
3542  	/*
3543  	 * Without the memory barrier there is a small possibility that
3544  	 * netdev_tx_sent_queue will miss the update and cause the queue to
3545  	 * be stopped forever
3546  	 */
3547  	smp_mb(); /* NOTE: netdev_txq_completed_mb() assumes this exists */
3548  
3549  	if (unlikely(dql_avail(&dev_queue->dql) < 0))
3550  		return;
3551  
3552  	if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
3553  		netif_schedule_queue(dev_queue);
3554  #endif
3555  }
3556  
3557  /**
3558   * 	netdev_completed_queue - report bytes and packets completed by device
3559   * 	@dev: network device
3560   * 	@pkts: actual number of packets sent over the medium
3561   * 	@bytes: actual number of bytes sent over the medium
3562   *
3563   * 	Report the number of bytes and packets transmitted by the network device
3564   * 	hardware queue over the physical medium, @bytes must exactly match the
3565   * 	@bytes amount passed to netdev_sent_queue()
3566   */
netdev_completed_queue(struct net_device * dev,unsigned int pkts,unsigned int bytes)3567  static inline void netdev_completed_queue(struct net_device *dev,
3568  					  unsigned int pkts, unsigned int bytes)
3569  {
3570  	netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
3571  }
3572  
netdev_tx_reset_queue(struct netdev_queue * q)3573  static inline void netdev_tx_reset_queue(struct netdev_queue *q)
3574  {
3575  #ifdef CONFIG_BQL
3576  	clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
3577  	dql_reset(&q->dql);
3578  #endif
3579  }
3580  
3581  /**
3582   * netdev_tx_reset_subqueue - reset the BQL stats and state of a netdev queue
3583   * @dev: network device
3584   * @qid: stack index of the queue to reset
3585   */
netdev_tx_reset_subqueue(const struct net_device * dev,u32 qid)3586  static inline void netdev_tx_reset_subqueue(const struct net_device *dev,
3587  					    u32 qid)
3588  {
3589  	netdev_tx_reset_queue(netdev_get_tx_queue(dev, qid));
3590  }
3591  
3592  /**
3593   * 	netdev_reset_queue - reset the packets and bytes count of a network device
3594   * 	@dev_queue: network device
3595   *
3596   * 	Reset the bytes and packet count of a network device and clear the
3597   * 	software flow control OFF bit for this network device
3598   */
netdev_reset_queue(struct net_device * dev_queue)3599  static inline void netdev_reset_queue(struct net_device *dev_queue)
3600  {
3601  	netdev_tx_reset_subqueue(dev_queue, 0);
3602  }
3603  
3604  /**
3605   * 	netdev_cap_txqueue - check if selected tx queue exceeds device queues
3606   * 	@dev: network device
3607   * 	@queue_index: given tx queue index
3608   *
3609   * 	Returns 0 if given tx queue index >= number of device tx queues,
3610   * 	otherwise returns the originally passed tx queue index.
3611   */
netdev_cap_txqueue(struct net_device * dev,u16 queue_index)3612  static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
3613  {
3614  	if (unlikely(queue_index >= dev->real_num_tx_queues)) {
3615  		net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
3616  				     dev->name, queue_index,
3617  				     dev->real_num_tx_queues);
3618  		return 0;
3619  	}
3620  
3621  	return queue_index;
3622  }
3623  
3624  /**
3625   *	netif_running - test if up
3626   *	@dev: network device
3627   *
3628   *	Test if the device has been brought up.
3629   */
netif_running(const struct net_device * dev)3630  static inline bool netif_running(const struct net_device *dev)
3631  {
3632  	return test_bit(__LINK_STATE_START, &dev->state);
3633  }
3634  
3635  /*
3636   * Routines to manage the subqueues on a device.  We only need start,
3637   * stop, and a check if it's stopped.  All other device management is
3638   * done at the overall netdevice level.
3639   * Also test the device if we're multiqueue.
3640   */
3641  
3642  /**
3643   *	netif_start_subqueue - allow sending packets on subqueue
3644   *	@dev: network device
3645   *	@queue_index: sub queue index
3646   *
3647   * Start individual transmit queue of a device with multiple transmit queues.
3648   */
netif_start_subqueue(struct net_device * dev,u16 queue_index)3649  static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
3650  {
3651  	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3652  
3653  	netif_tx_start_queue(txq);
3654  }
3655  
3656  /**
3657   *	netif_stop_subqueue - stop sending packets on subqueue
3658   *	@dev: network device
3659   *	@queue_index: sub queue index
3660   *
3661   * Stop individual transmit queue of a device with multiple transmit queues.
3662   */
netif_stop_subqueue(struct net_device * dev,u16 queue_index)3663  static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
3664  {
3665  	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3666  	netif_tx_stop_queue(txq);
3667  }
3668  
3669  /**
3670   *	__netif_subqueue_stopped - test status of subqueue
3671   *	@dev: network device
3672   *	@queue_index: sub queue index
3673   *
3674   * Check individual transmit queue of a device with multiple transmit queues.
3675   */
__netif_subqueue_stopped(const struct net_device * dev,u16 queue_index)3676  static inline bool __netif_subqueue_stopped(const struct net_device *dev,
3677  					    u16 queue_index)
3678  {
3679  	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3680  
3681  	return netif_tx_queue_stopped(txq);
3682  }
3683  
3684  /**
3685   *	netif_subqueue_stopped - test status of subqueue
3686   *	@dev: network device
3687   *	@skb: sub queue buffer pointer
3688   *
3689   * Check individual transmit queue of a device with multiple transmit queues.
3690   */
netif_subqueue_stopped(const struct net_device * dev,struct sk_buff * skb)3691  static inline bool netif_subqueue_stopped(const struct net_device *dev,
3692  					  struct sk_buff *skb)
3693  {
3694  	return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
3695  }
3696  
3697  /**
3698   *	netif_wake_subqueue - allow sending packets on subqueue
3699   *	@dev: network device
3700   *	@queue_index: sub queue index
3701   *
3702   * Resume individual transmit queue of a device with multiple transmit queues.
3703   */
netif_wake_subqueue(struct net_device * dev,u16 queue_index)3704  static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
3705  {
3706  	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3707  
3708  	netif_tx_wake_queue(txq);
3709  }
3710  
3711  #ifdef CONFIG_XPS
3712  int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
3713  			u16 index);
3714  int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
3715  			  u16 index, enum xps_map_type type);
3716  
3717  /**
3718   *	netif_attr_test_mask - Test a CPU or Rx queue set in a mask
3719   *	@j: CPU/Rx queue index
3720   *	@mask: bitmask of all cpus/rx queues
3721   *	@nr_bits: number of bits in the bitmask
3722   *
3723   * Test if a CPU or Rx queue index is set in a mask of all CPU/Rx queues.
3724   */
netif_attr_test_mask(unsigned long j,const unsigned long * mask,unsigned int nr_bits)3725  static inline bool netif_attr_test_mask(unsigned long j,
3726  					const unsigned long *mask,
3727  					unsigned int nr_bits)
3728  {
3729  	cpu_max_bits_warn(j, nr_bits);
3730  	return test_bit(j, mask);
3731  }
3732  
3733  /**
3734   *	netif_attr_test_online - Test for online CPU/Rx queue
3735   *	@j: CPU/Rx queue index
3736   *	@online_mask: bitmask for CPUs/Rx queues that are online
3737   *	@nr_bits: number of bits in the bitmask
3738   *
3739   * Returns true if a CPU/Rx queue is online.
3740   */
netif_attr_test_online(unsigned long j,const unsigned long * online_mask,unsigned int nr_bits)3741  static inline bool netif_attr_test_online(unsigned long j,
3742  					  const unsigned long *online_mask,
3743  					  unsigned int nr_bits)
3744  {
3745  	cpu_max_bits_warn(j, nr_bits);
3746  
3747  	if (online_mask)
3748  		return test_bit(j, online_mask);
3749  
3750  	return (j < nr_bits);
3751  }
3752  
3753  /**
3754   *	netif_attrmask_next - get the next CPU/Rx queue in a cpu/Rx queues mask
3755   *	@n: CPU/Rx queue index
3756   *	@srcp: the cpumask/Rx queue mask pointer
3757   *	@nr_bits: number of bits in the bitmask
3758   *
3759   * Returns >= nr_bits if no further CPUs/Rx queues set.
3760   */
netif_attrmask_next(int n,const unsigned long * srcp,unsigned int nr_bits)3761  static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp,
3762  					       unsigned int nr_bits)
3763  {
3764  	/* -1 is a legal arg here. */
3765  	if (n != -1)
3766  		cpu_max_bits_warn(n, nr_bits);
3767  
3768  	if (srcp)
3769  		return find_next_bit(srcp, nr_bits, n + 1);
3770  
3771  	return n + 1;
3772  }
3773  
3774  /**
3775   *	netif_attrmask_next_and - get the next CPU/Rx queue in \*src1p & \*src2p
3776   *	@n: CPU/Rx queue index
3777   *	@src1p: the first CPUs/Rx queues mask pointer
3778   *	@src2p: the second CPUs/Rx queues mask pointer
3779   *	@nr_bits: number of bits in the bitmask
3780   *
3781   * Returns >= nr_bits if no further CPUs/Rx queues set in both.
3782   */
netif_attrmask_next_and(int n,const unsigned long * src1p,const unsigned long * src2p,unsigned int nr_bits)3783  static inline int netif_attrmask_next_and(int n, const unsigned long *src1p,
3784  					  const unsigned long *src2p,
3785  					  unsigned int nr_bits)
3786  {
3787  	/* -1 is a legal arg here. */
3788  	if (n != -1)
3789  		cpu_max_bits_warn(n, nr_bits);
3790  
3791  	if (src1p && src2p)
3792  		return find_next_and_bit(src1p, src2p, nr_bits, n + 1);
3793  	else if (src1p)
3794  		return find_next_bit(src1p, nr_bits, n + 1);
3795  	else if (src2p)
3796  		return find_next_bit(src2p, nr_bits, n + 1);
3797  
3798  	return n + 1;
3799  }
3800  #else
netif_set_xps_queue(struct net_device * dev,const struct cpumask * mask,u16 index)3801  static inline int netif_set_xps_queue(struct net_device *dev,
3802  				      const struct cpumask *mask,
3803  				      u16 index)
3804  {
3805  	return 0;
3806  }
3807  
__netif_set_xps_queue(struct net_device * dev,const unsigned long * mask,u16 index,enum xps_map_type type)3808  static inline int __netif_set_xps_queue(struct net_device *dev,
3809  					const unsigned long *mask,
3810  					u16 index, enum xps_map_type type)
3811  {
3812  	return 0;
3813  }
3814  #endif
3815  
3816  /**
3817   *	netif_is_multiqueue - test if device has multiple transmit queues
3818   *	@dev: network device
3819   *
3820   * Check if device has multiple transmit queues
3821   */
netif_is_multiqueue(const struct net_device * dev)3822  static inline bool netif_is_multiqueue(const struct net_device *dev)
3823  {
3824  	return dev->num_tx_queues > 1;
3825  }
3826  
3827  int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
3828  
3829  #ifdef CONFIG_SYSFS
3830  int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
3831  #else
netif_set_real_num_rx_queues(struct net_device * dev,unsigned int rxqs)3832  static inline int netif_set_real_num_rx_queues(struct net_device *dev,
3833  						unsigned int rxqs)
3834  {
3835  	dev->real_num_rx_queues = rxqs;
3836  	return 0;
3837  }
3838  #endif
3839  int netif_set_real_num_queues(struct net_device *dev,
3840  			      unsigned int txq, unsigned int rxq);
3841  
3842  int netif_get_num_default_rss_queues(void);
3843  
3844  void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason);
3845  void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason);
3846  
3847  /*
3848   * It is not allowed to call kfree_skb() or consume_skb() from hardware
3849   * interrupt context or with hardware interrupts being disabled.
3850   * (in_hardirq() || irqs_disabled())
3851   *
3852   * We provide four helpers that can be used in following contexts :
3853   *
3854   * dev_kfree_skb_irq(skb) when caller drops a packet from irq context,
3855   *  replacing kfree_skb(skb)
3856   *
3857   * dev_consume_skb_irq(skb) when caller consumes a packet from irq context.
3858   *  Typically used in place of consume_skb(skb) in TX completion path
3859   *
3860   * dev_kfree_skb_any(skb) when caller doesn't know its current irq context,
3861   *  replacing kfree_skb(skb)
3862   *
3863   * dev_consume_skb_any(skb) when caller doesn't know its current irq context,
3864   *  and consumed a packet. Used in place of consume_skb(skb)
3865   */
dev_kfree_skb_irq(struct sk_buff * skb)3866  static inline void dev_kfree_skb_irq(struct sk_buff *skb)
3867  {
3868  	dev_kfree_skb_irq_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
3869  }
3870  
dev_consume_skb_irq(struct sk_buff * skb)3871  static inline void dev_consume_skb_irq(struct sk_buff *skb)
3872  {
3873  	dev_kfree_skb_irq_reason(skb, SKB_CONSUMED);
3874  }
3875  
dev_kfree_skb_any(struct sk_buff * skb)3876  static inline void dev_kfree_skb_any(struct sk_buff *skb)
3877  {
3878  	dev_kfree_skb_any_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
3879  }
3880  
dev_consume_skb_any(struct sk_buff * skb)3881  static inline void dev_consume_skb_any(struct sk_buff *skb)
3882  {
3883  	dev_kfree_skb_any_reason(skb, SKB_CONSUMED);
3884  }
3885  
3886  u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
3887  			     struct bpf_prog *xdp_prog);
3888  void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
3889  int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb);
3890  int netif_rx(struct sk_buff *skb);
3891  int __netif_rx(struct sk_buff *skb);
3892  
3893  int netif_receive_skb(struct sk_buff *skb);
3894  int netif_receive_skb_core(struct sk_buff *skb);
3895  void netif_receive_skb_list_internal(struct list_head *head);
3896  void netif_receive_skb_list(struct list_head *head);
3897  gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
3898  void napi_gro_flush(struct napi_struct *napi, bool flush_old);
3899  struct sk_buff *napi_get_frags(struct napi_struct *napi);
3900  void napi_get_frags_check(struct napi_struct *napi);
3901  gro_result_t napi_gro_frags(struct napi_struct *napi);
3902  
napi_free_frags(struct napi_struct * napi)3903  static inline void napi_free_frags(struct napi_struct *napi)
3904  {
3905  	kfree_skb(napi->skb);
3906  	napi->skb = NULL;
3907  }
3908  
3909  bool netdev_is_rx_handler_busy(struct net_device *dev);
3910  int netdev_rx_handler_register(struct net_device *dev,
3911  			       rx_handler_func_t *rx_handler,
3912  			       void *rx_handler_data);
3913  void netdev_rx_handler_unregister(struct net_device *dev);
3914  
3915  bool dev_valid_name(const char *name);
is_socket_ioctl_cmd(unsigned int cmd)3916  static inline bool is_socket_ioctl_cmd(unsigned int cmd)
3917  {
3918  	return _IOC_TYPE(cmd) == SOCK_IOC_TYPE;
3919  }
3920  int get_user_ifreq(struct ifreq *ifr, void __user **ifrdata, void __user *arg);
3921  int put_user_ifreq(struct ifreq *ifr, void __user *arg);
3922  int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
3923  		void __user *data, bool *need_copyout);
3924  int dev_ifconf(struct net *net, struct ifconf __user *ifc);
3925  int generic_hwtstamp_get_lower(struct net_device *dev,
3926  			       struct kernel_hwtstamp_config *kernel_cfg);
3927  int generic_hwtstamp_set_lower(struct net_device *dev,
3928  			       struct kernel_hwtstamp_config *kernel_cfg,
3929  			       struct netlink_ext_ack *extack);
3930  int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata);
3931  unsigned int dev_get_flags(const struct net_device *);
3932  int __dev_change_flags(struct net_device *dev, unsigned int flags,
3933  		       struct netlink_ext_ack *extack);
3934  int dev_change_flags(struct net_device *dev, unsigned int flags,
3935  		     struct netlink_ext_ack *extack);
3936  int dev_set_alias(struct net_device *, const char *, size_t);
3937  int dev_get_alias(const struct net_device *, char *, size_t);
3938  int __dev_change_net_namespace(struct net_device *dev, struct net *net,
3939  			       const char *pat, int new_ifindex);
3940  static inline
dev_change_net_namespace(struct net_device * dev,struct net * net,const char * pat)3941  int dev_change_net_namespace(struct net_device *dev, struct net *net,
3942  			     const char *pat)
3943  {
3944  	return __dev_change_net_namespace(dev, net, pat, 0);
3945  }
3946  int __dev_set_mtu(struct net_device *, int);
3947  int dev_set_mtu(struct net_device *, int);
3948  int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
3949  			      struct netlink_ext_ack *extack);
3950  int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
3951  			struct netlink_ext_ack *extack);
3952  int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
3953  			     struct netlink_ext_ack *extack);
3954  int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name);
3955  int dev_get_port_parent_id(struct net_device *dev,
3956  			   struct netdev_phys_item_id *ppid, bool recurse);
3957  bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b);
3958  
3959  struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
3960  struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
3961  				    struct netdev_queue *txq, int *ret);
3962  
3963  int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
3964  u8 dev_xdp_prog_count(struct net_device *dev);
3965  int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);
3966  u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);
3967  
3968  u32 dev_get_min_mp_channel_count(const struct net_device *dev);
3969  
3970  int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3971  int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3972  int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
3973  bool is_skb_forwardable(const struct net_device *dev,
3974  			const struct sk_buff *skb);
3975  
__is_skb_forwardable(const struct net_device * dev,const struct sk_buff * skb,const bool check_mtu)3976  static __always_inline bool __is_skb_forwardable(const struct net_device *dev,
3977  						 const struct sk_buff *skb,
3978  						 const bool check_mtu)
3979  {
3980  	const u32 vlan_hdr_len = 4; /* VLAN_HLEN */
3981  	unsigned int len;
3982  
3983  	if (!(dev->flags & IFF_UP))
3984  		return false;
3985  
3986  	if (!check_mtu)
3987  		return true;
3988  
3989  	len = dev->mtu + dev->hard_header_len + vlan_hdr_len;
3990  	if (skb->len <= len)
3991  		return true;
3992  
3993  	/* if TSO is enabled, we don't care about the length as the packet
3994  	 * could be forwarded without being segmented before
3995  	 */
3996  	if (skb_is_gso(skb))
3997  		return true;
3998  
3999  	return false;
4000  }
4001  
4002  void netdev_core_stats_inc(struct net_device *dev, u32 offset);
4003  
4004  #define DEV_CORE_STATS_INC(FIELD)						\
4005  static inline void dev_core_stats_##FIELD##_inc(struct net_device *dev)		\
4006  {										\
4007  	netdev_core_stats_inc(dev,						\
4008  			offsetof(struct net_device_core_stats, FIELD));		\
4009  }
4010  DEV_CORE_STATS_INC(rx_dropped)
DEV_CORE_STATS_INC(tx_dropped)4011  DEV_CORE_STATS_INC(tx_dropped)
4012  DEV_CORE_STATS_INC(rx_nohandler)
4013  DEV_CORE_STATS_INC(rx_otherhost_dropped)
4014  #undef DEV_CORE_STATS_INC
4015  
4016  static __always_inline int ____dev_forward_skb(struct net_device *dev,
4017  					       struct sk_buff *skb,
4018  					       const bool check_mtu)
4019  {
4020  	if (skb_orphan_frags(skb, GFP_ATOMIC) ||
4021  	    unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) {
4022  		dev_core_stats_rx_dropped_inc(dev);
4023  		kfree_skb(skb);
4024  		return NET_RX_DROP;
4025  	}
4026  
4027  	skb_scrub_packet(skb, !net_eq(dev_net(dev), dev_net(skb->dev)));
4028  	skb->priority = 0;
4029  	return 0;
4030  }
4031  
4032  bool dev_nit_active(struct net_device *dev);
4033  void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
4034  
__dev_put(struct net_device * dev)4035  static inline void __dev_put(struct net_device *dev)
4036  {
4037  	if (dev) {
4038  #ifdef CONFIG_PCPU_DEV_REFCNT
4039  		this_cpu_dec(*dev->pcpu_refcnt);
4040  #else
4041  		refcount_dec(&dev->dev_refcnt);
4042  #endif
4043  	}
4044  }
4045  
__dev_hold(struct net_device * dev)4046  static inline void __dev_hold(struct net_device *dev)
4047  {
4048  	if (dev) {
4049  #ifdef CONFIG_PCPU_DEV_REFCNT
4050  		this_cpu_inc(*dev->pcpu_refcnt);
4051  #else
4052  		refcount_inc(&dev->dev_refcnt);
4053  #endif
4054  	}
4055  }
4056  
__netdev_tracker_alloc(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4057  static inline void __netdev_tracker_alloc(struct net_device *dev,
4058  					  netdevice_tracker *tracker,
4059  					  gfp_t gfp)
4060  {
4061  #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4062  	ref_tracker_alloc(&dev->refcnt_tracker, tracker, gfp);
4063  #endif
4064  }
4065  
4066  /* netdev_tracker_alloc() can upgrade a prior untracked reference
4067   * taken by dev_get_by_name()/dev_get_by_index() to a tracked one.
4068   */
netdev_tracker_alloc(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4069  static inline void netdev_tracker_alloc(struct net_device *dev,
4070  					netdevice_tracker *tracker, gfp_t gfp)
4071  {
4072  #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4073  	refcount_dec(&dev->refcnt_tracker.no_tracker);
4074  	__netdev_tracker_alloc(dev, tracker, gfp);
4075  #endif
4076  }
4077  
netdev_tracker_free(struct net_device * dev,netdevice_tracker * tracker)4078  static inline void netdev_tracker_free(struct net_device *dev,
4079  				       netdevice_tracker *tracker)
4080  {
4081  #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4082  	ref_tracker_free(&dev->refcnt_tracker, tracker);
4083  #endif
4084  }
4085  
netdev_hold(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4086  static inline void netdev_hold(struct net_device *dev,
4087  			       netdevice_tracker *tracker, gfp_t gfp)
4088  {
4089  	if (dev) {
4090  		__dev_hold(dev);
4091  		__netdev_tracker_alloc(dev, tracker, gfp);
4092  	}
4093  }
4094  
netdev_put(struct net_device * dev,netdevice_tracker * tracker)4095  static inline void netdev_put(struct net_device *dev,
4096  			      netdevice_tracker *tracker)
4097  {
4098  	if (dev) {
4099  		netdev_tracker_free(dev, tracker);
4100  		__dev_put(dev);
4101  	}
4102  }
4103  
4104  /**
4105   *	dev_hold - get reference to device
4106   *	@dev: network device
4107   *
4108   * Hold reference to device to keep it from being freed.
4109   * Try using netdev_hold() instead.
4110   */
dev_hold(struct net_device * dev)4111  static inline void dev_hold(struct net_device *dev)
4112  {
4113  	netdev_hold(dev, NULL, GFP_ATOMIC);
4114  }
4115  
4116  /**
4117   *	dev_put - release reference to device
4118   *	@dev: network device
4119   *
4120   * Release reference to device to allow it to be freed.
4121   * Try using netdev_put() instead.
4122   */
dev_put(struct net_device * dev)4123  static inline void dev_put(struct net_device *dev)
4124  {
4125  	netdev_put(dev, NULL);
4126  }
4127  
DEFINE_FREE(dev_put,struct net_device *,if (_T)dev_put (_T))4128  DEFINE_FREE(dev_put, struct net_device *, if (_T) dev_put(_T))
4129  
4130  static inline void netdev_ref_replace(struct net_device *odev,
4131  				      struct net_device *ndev,
4132  				      netdevice_tracker *tracker,
4133  				      gfp_t gfp)
4134  {
4135  	if (odev)
4136  		netdev_tracker_free(odev, tracker);
4137  
4138  	__dev_hold(ndev);
4139  	__dev_put(odev);
4140  
4141  	if (ndev)
4142  		__netdev_tracker_alloc(ndev, tracker, gfp);
4143  }
4144  
4145  /* Carrier loss detection, dial on demand. The functions netif_carrier_on
4146   * and _off may be called from IRQ context, but it is caller
4147   * who is responsible for serialization of these calls.
4148   *
4149   * The name carrier is inappropriate, these functions should really be
4150   * called netif_lowerlayer_*() because they represent the state of any
4151   * kind of lower layer not just hardware media.
4152   */
4153  void linkwatch_fire_event(struct net_device *dev);
4154  
4155  /**
4156   * linkwatch_sync_dev - sync linkwatch for the given device
4157   * @dev: network device to sync linkwatch for
4158   *
4159   * Sync linkwatch for the given device, removing it from the
4160   * pending work list (if queued).
4161   */
4162  void linkwatch_sync_dev(struct net_device *dev);
4163  
4164  /**
4165   *	netif_carrier_ok - test if carrier present
4166   *	@dev: network device
4167   *
4168   * Check if carrier is present on device
4169   */
netif_carrier_ok(const struct net_device * dev)4170  static inline bool netif_carrier_ok(const struct net_device *dev)
4171  {
4172  	return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
4173  }
4174  
4175  unsigned long dev_trans_start(struct net_device *dev);
4176  
4177  void __netdev_watchdog_up(struct net_device *dev);
4178  
4179  void netif_carrier_on(struct net_device *dev);
4180  void netif_carrier_off(struct net_device *dev);
4181  void netif_carrier_event(struct net_device *dev);
4182  
4183  /**
4184   *	netif_dormant_on - mark device as dormant.
4185   *	@dev: network device
4186   *
4187   * Mark device as dormant (as per RFC2863).
4188   *
4189   * The dormant state indicates that the relevant interface is not
4190   * actually in a condition to pass packets (i.e., it is not 'up') but is
4191   * in a "pending" state, waiting for some external event.  For "on-
4192   * demand" interfaces, this new state identifies the situation where the
4193   * interface is waiting for events to place it in the up state.
4194   */
netif_dormant_on(struct net_device * dev)4195  static inline void netif_dormant_on(struct net_device *dev)
4196  {
4197  	if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
4198  		linkwatch_fire_event(dev);
4199  }
4200  
4201  /**
4202   *	netif_dormant_off - set device as not dormant.
4203   *	@dev: network device
4204   *
4205   * Device is not in dormant state.
4206   */
netif_dormant_off(struct net_device * dev)4207  static inline void netif_dormant_off(struct net_device *dev)
4208  {
4209  	if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
4210  		linkwatch_fire_event(dev);
4211  }
4212  
4213  /**
4214   *	netif_dormant - test if device is dormant
4215   *	@dev: network device
4216   *
4217   * Check if device is dormant.
4218   */
netif_dormant(const struct net_device * dev)4219  static inline bool netif_dormant(const struct net_device *dev)
4220  {
4221  	return test_bit(__LINK_STATE_DORMANT, &dev->state);
4222  }
4223  
4224  
4225  /**
4226   *	netif_testing_on - mark device as under test.
4227   *	@dev: network device
4228   *
4229   * Mark device as under test (as per RFC2863).
4230   *
4231   * The testing state indicates that some test(s) must be performed on
4232   * the interface. After completion, of the test, the interface state
4233   * will change to up, dormant, or down, as appropriate.
4234   */
netif_testing_on(struct net_device * dev)4235  static inline void netif_testing_on(struct net_device *dev)
4236  {
4237  	if (!test_and_set_bit(__LINK_STATE_TESTING, &dev->state))
4238  		linkwatch_fire_event(dev);
4239  }
4240  
4241  /**
4242   *	netif_testing_off - set device as not under test.
4243   *	@dev: network device
4244   *
4245   * Device is not in testing state.
4246   */
netif_testing_off(struct net_device * dev)4247  static inline void netif_testing_off(struct net_device *dev)
4248  {
4249  	if (test_and_clear_bit(__LINK_STATE_TESTING, &dev->state))
4250  		linkwatch_fire_event(dev);
4251  }
4252  
4253  /**
4254   *	netif_testing - test if device is under test
4255   *	@dev: network device
4256   *
4257   * Check if device is under test
4258   */
netif_testing(const struct net_device * dev)4259  static inline bool netif_testing(const struct net_device *dev)
4260  {
4261  	return test_bit(__LINK_STATE_TESTING, &dev->state);
4262  }
4263  
4264  
4265  /**
4266   *	netif_oper_up - test if device is operational
4267   *	@dev: network device
4268   *
4269   * Check if carrier is operational
4270   */
netif_oper_up(const struct net_device * dev)4271  static inline bool netif_oper_up(const struct net_device *dev)
4272  {
4273  	unsigned int operstate = READ_ONCE(dev->operstate);
4274  
4275  	return	operstate == IF_OPER_UP ||
4276  		operstate == IF_OPER_UNKNOWN /* backward compat */;
4277  }
4278  
4279  /**
4280   *	netif_device_present - is device available or removed
4281   *	@dev: network device
4282   *
4283   * Check if device has not been removed from system.
4284   */
netif_device_present(const struct net_device * dev)4285  static inline bool netif_device_present(const struct net_device *dev)
4286  {
4287  	return test_bit(__LINK_STATE_PRESENT, &dev->state);
4288  }
4289  
4290  void netif_device_detach(struct net_device *dev);
4291  
4292  void netif_device_attach(struct net_device *dev);
4293  
4294  /*
4295   * Network interface message level settings
4296   */
4297  
4298  enum {
4299  	NETIF_MSG_DRV_BIT,
4300  	NETIF_MSG_PROBE_BIT,
4301  	NETIF_MSG_LINK_BIT,
4302  	NETIF_MSG_TIMER_BIT,
4303  	NETIF_MSG_IFDOWN_BIT,
4304  	NETIF_MSG_IFUP_BIT,
4305  	NETIF_MSG_RX_ERR_BIT,
4306  	NETIF_MSG_TX_ERR_BIT,
4307  	NETIF_MSG_TX_QUEUED_BIT,
4308  	NETIF_MSG_INTR_BIT,
4309  	NETIF_MSG_TX_DONE_BIT,
4310  	NETIF_MSG_RX_STATUS_BIT,
4311  	NETIF_MSG_PKTDATA_BIT,
4312  	NETIF_MSG_HW_BIT,
4313  	NETIF_MSG_WOL_BIT,
4314  
4315  	/* When you add a new bit above, update netif_msg_class_names array
4316  	 * in net/ethtool/common.c
4317  	 */
4318  	NETIF_MSG_CLASS_COUNT,
4319  };
4320  /* Both ethtool_ops interface and internal driver implementation use u32 */
4321  static_assert(NETIF_MSG_CLASS_COUNT <= 32);
4322  
4323  #define __NETIF_MSG_BIT(bit)	((u32)1 << (bit))
4324  #define __NETIF_MSG(name)	__NETIF_MSG_BIT(NETIF_MSG_ ## name ## _BIT)
4325  
4326  #define NETIF_MSG_DRV		__NETIF_MSG(DRV)
4327  #define NETIF_MSG_PROBE		__NETIF_MSG(PROBE)
4328  #define NETIF_MSG_LINK		__NETIF_MSG(LINK)
4329  #define NETIF_MSG_TIMER		__NETIF_MSG(TIMER)
4330  #define NETIF_MSG_IFDOWN	__NETIF_MSG(IFDOWN)
4331  #define NETIF_MSG_IFUP		__NETIF_MSG(IFUP)
4332  #define NETIF_MSG_RX_ERR	__NETIF_MSG(RX_ERR)
4333  #define NETIF_MSG_TX_ERR	__NETIF_MSG(TX_ERR)
4334  #define NETIF_MSG_TX_QUEUED	__NETIF_MSG(TX_QUEUED)
4335  #define NETIF_MSG_INTR		__NETIF_MSG(INTR)
4336  #define NETIF_MSG_TX_DONE	__NETIF_MSG(TX_DONE)
4337  #define NETIF_MSG_RX_STATUS	__NETIF_MSG(RX_STATUS)
4338  #define NETIF_MSG_PKTDATA	__NETIF_MSG(PKTDATA)
4339  #define NETIF_MSG_HW		__NETIF_MSG(HW)
4340  #define NETIF_MSG_WOL		__NETIF_MSG(WOL)
4341  
4342  #define netif_msg_drv(p)	((p)->msg_enable & NETIF_MSG_DRV)
4343  #define netif_msg_probe(p)	((p)->msg_enable & NETIF_MSG_PROBE)
4344  #define netif_msg_link(p)	((p)->msg_enable & NETIF_MSG_LINK)
4345  #define netif_msg_timer(p)	((p)->msg_enable & NETIF_MSG_TIMER)
4346  #define netif_msg_ifdown(p)	((p)->msg_enable & NETIF_MSG_IFDOWN)
4347  #define netif_msg_ifup(p)	((p)->msg_enable & NETIF_MSG_IFUP)
4348  #define netif_msg_rx_err(p)	((p)->msg_enable & NETIF_MSG_RX_ERR)
4349  #define netif_msg_tx_err(p)	((p)->msg_enable & NETIF_MSG_TX_ERR)
4350  #define netif_msg_tx_queued(p)	((p)->msg_enable & NETIF_MSG_TX_QUEUED)
4351  #define netif_msg_intr(p)	((p)->msg_enable & NETIF_MSG_INTR)
4352  #define netif_msg_tx_done(p)	((p)->msg_enable & NETIF_MSG_TX_DONE)
4353  #define netif_msg_rx_status(p)	((p)->msg_enable & NETIF_MSG_RX_STATUS)
4354  #define netif_msg_pktdata(p)	((p)->msg_enable & NETIF_MSG_PKTDATA)
4355  #define netif_msg_hw(p)		((p)->msg_enable & NETIF_MSG_HW)
4356  #define netif_msg_wol(p)	((p)->msg_enable & NETIF_MSG_WOL)
4357  
netif_msg_init(int debug_value,int default_msg_enable_bits)4358  static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
4359  {
4360  	/* use default */
4361  	if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
4362  		return default_msg_enable_bits;
4363  	if (debug_value == 0)	/* no output */
4364  		return 0;
4365  	/* set low N bits */
4366  	return (1U << debug_value) - 1;
4367  }
4368  
__netif_tx_lock(struct netdev_queue * txq,int cpu)4369  static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
4370  {
4371  	spin_lock(&txq->_xmit_lock);
4372  	/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4373  	WRITE_ONCE(txq->xmit_lock_owner, cpu);
4374  }
4375  
__netif_tx_acquire(struct netdev_queue * txq)4376  static inline bool __netif_tx_acquire(struct netdev_queue *txq)
4377  {
4378  	__acquire(&txq->_xmit_lock);
4379  	return true;
4380  }
4381  
__netif_tx_release(struct netdev_queue * txq)4382  static inline void __netif_tx_release(struct netdev_queue *txq)
4383  {
4384  	__release(&txq->_xmit_lock);
4385  }
4386  
__netif_tx_lock_bh(struct netdev_queue * txq)4387  static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
4388  {
4389  	spin_lock_bh(&txq->_xmit_lock);
4390  	/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4391  	WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4392  }
4393  
__netif_tx_trylock(struct netdev_queue * txq)4394  static inline bool __netif_tx_trylock(struct netdev_queue *txq)
4395  {
4396  	bool ok = spin_trylock(&txq->_xmit_lock);
4397  
4398  	if (likely(ok)) {
4399  		/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4400  		WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4401  	}
4402  	return ok;
4403  }
4404  
__netif_tx_unlock(struct netdev_queue * txq)4405  static inline void __netif_tx_unlock(struct netdev_queue *txq)
4406  {
4407  	/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4408  	WRITE_ONCE(txq->xmit_lock_owner, -1);
4409  	spin_unlock(&txq->_xmit_lock);
4410  }
4411  
__netif_tx_unlock_bh(struct netdev_queue * txq)4412  static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
4413  {
4414  	/* Pairs with READ_ONCE() in __dev_queue_xmit() */
4415  	WRITE_ONCE(txq->xmit_lock_owner, -1);
4416  	spin_unlock_bh(&txq->_xmit_lock);
4417  }
4418  
4419  /*
4420   * txq->trans_start can be read locklessly from dev_watchdog()
4421   */
txq_trans_update(struct netdev_queue * txq)4422  static inline void txq_trans_update(struct netdev_queue *txq)
4423  {
4424  	if (txq->xmit_lock_owner != -1)
4425  		WRITE_ONCE(txq->trans_start, jiffies);
4426  }
4427  
txq_trans_cond_update(struct netdev_queue * txq)4428  static inline void txq_trans_cond_update(struct netdev_queue *txq)
4429  {
4430  	unsigned long now = jiffies;
4431  
4432  	if (READ_ONCE(txq->trans_start) != now)
4433  		WRITE_ONCE(txq->trans_start, now);
4434  }
4435  
4436  /* legacy drivers only, netdev_start_xmit() sets txq->trans_start */
netif_trans_update(struct net_device * dev)4437  static inline void netif_trans_update(struct net_device *dev)
4438  {
4439  	struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
4440  
4441  	txq_trans_cond_update(txq);
4442  }
4443  
4444  /**
4445   *	netif_tx_lock - grab network device transmit lock
4446   *	@dev: network device
4447   *
4448   * Get network device transmit lock
4449   */
4450  void netif_tx_lock(struct net_device *dev);
4451  
netif_tx_lock_bh(struct net_device * dev)4452  static inline void netif_tx_lock_bh(struct net_device *dev)
4453  {
4454  	local_bh_disable();
4455  	netif_tx_lock(dev);
4456  }
4457  
4458  void netif_tx_unlock(struct net_device *dev);
4459  
netif_tx_unlock_bh(struct net_device * dev)4460  static inline void netif_tx_unlock_bh(struct net_device *dev)
4461  {
4462  	netif_tx_unlock(dev);
4463  	local_bh_enable();
4464  }
4465  
4466  #define HARD_TX_LOCK(dev, txq, cpu) {			\
4467  	if (!(dev)->lltx) {				\
4468  		__netif_tx_lock(txq, cpu);		\
4469  	} else {					\
4470  		__netif_tx_acquire(txq);		\
4471  	}						\
4472  }
4473  
4474  #define HARD_TX_TRYLOCK(dev, txq)			\
4475  	(!(dev)->lltx ?					\
4476  		__netif_tx_trylock(txq) :		\
4477  		__netif_tx_acquire(txq))
4478  
4479  #define HARD_TX_UNLOCK(dev, txq) {			\
4480  	if (!(dev)->lltx) {				\
4481  		__netif_tx_unlock(txq);			\
4482  	} else {					\
4483  		__netif_tx_release(txq);		\
4484  	}						\
4485  }
4486  
netif_tx_disable(struct net_device * dev)4487  static inline void netif_tx_disable(struct net_device *dev)
4488  {
4489  	unsigned int i;
4490  	int cpu;
4491  
4492  	local_bh_disable();
4493  	cpu = smp_processor_id();
4494  	spin_lock(&dev->tx_global_lock);
4495  	for (i = 0; i < dev->num_tx_queues; i++) {
4496  		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
4497  
4498  		__netif_tx_lock(txq, cpu);
4499  		netif_tx_stop_queue(txq);
4500  		__netif_tx_unlock(txq);
4501  	}
4502  	spin_unlock(&dev->tx_global_lock);
4503  	local_bh_enable();
4504  }
4505  
netif_addr_lock(struct net_device * dev)4506  static inline void netif_addr_lock(struct net_device *dev)
4507  {
4508  	unsigned char nest_level = 0;
4509  
4510  #ifdef CONFIG_LOCKDEP
4511  	nest_level = dev->nested_level;
4512  #endif
4513  	spin_lock_nested(&dev->addr_list_lock, nest_level);
4514  }
4515  
netif_addr_lock_bh(struct net_device * dev)4516  static inline void netif_addr_lock_bh(struct net_device *dev)
4517  {
4518  	unsigned char nest_level = 0;
4519  
4520  #ifdef CONFIG_LOCKDEP
4521  	nest_level = dev->nested_level;
4522  #endif
4523  	local_bh_disable();
4524  	spin_lock_nested(&dev->addr_list_lock, nest_level);
4525  }
4526  
netif_addr_unlock(struct net_device * dev)4527  static inline void netif_addr_unlock(struct net_device *dev)
4528  {
4529  	spin_unlock(&dev->addr_list_lock);
4530  }
4531  
netif_addr_unlock_bh(struct net_device * dev)4532  static inline void netif_addr_unlock_bh(struct net_device *dev)
4533  {
4534  	spin_unlock_bh(&dev->addr_list_lock);
4535  }
4536  
4537  /*
4538   * dev_addrs walker. Should be used only for read access. Call with
4539   * rcu_read_lock held.
4540   */
4541  #define for_each_dev_addr(dev, ha) \
4542  		list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
4543  
4544  /* These functions live elsewhere (drivers/net/net_init.c, but related) */
4545  
4546  void ether_setup(struct net_device *dev);
4547  
4548  /* Allocate dummy net_device */
4549  struct net_device *alloc_netdev_dummy(int sizeof_priv);
4550  
4551  /* Support for loadable net-drivers */
4552  struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
4553  				    unsigned char name_assign_type,
4554  				    void (*setup)(struct net_device *),
4555  				    unsigned int txqs, unsigned int rxqs);
4556  #define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
4557  	alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
4558  
4559  #define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
4560  	alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
4561  			 count)
4562  
4563  int register_netdev(struct net_device *dev);
4564  void unregister_netdev(struct net_device *dev);
4565  
4566  int devm_register_netdev(struct device *dev, struct net_device *ndev);
4567  
4568  /* General hardware address lists handling functions */
4569  int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
4570  		   struct netdev_hw_addr_list *from_list, int addr_len);
4571  void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
4572  		      struct netdev_hw_addr_list *from_list, int addr_len);
4573  int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
4574  		       struct net_device *dev,
4575  		       int (*sync)(struct net_device *, const unsigned char *),
4576  		       int (*unsync)(struct net_device *,
4577  				     const unsigned char *));
4578  int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
4579  			   struct net_device *dev,
4580  			   int (*sync)(struct net_device *,
4581  				       const unsigned char *, int),
4582  			   int (*unsync)(struct net_device *,
4583  					 const unsigned char *, int));
4584  void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
4585  			      struct net_device *dev,
4586  			      int (*unsync)(struct net_device *,
4587  					    const unsigned char *, int));
4588  void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
4589  			  struct net_device *dev,
4590  			  int (*unsync)(struct net_device *,
4591  					const unsigned char *));
4592  void __hw_addr_init(struct netdev_hw_addr_list *list);
4593  
4594  /* Functions used for device addresses handling */
4595  void dev_addr_mod(struct net_device *dev, unsigned int offset,
4596  		  const void *addr, size_t len);
4597  
4598  static inline void
__dev_addr_set(struct net_device * dev,const void * addr,size_t len)4599  __dev_addr_set(struct net_device *dev, const void *addr, size_t len)
4600  {
4601  	dev_addr_mod(dev, 0, addr, len);
4602  }
4603  
dev_addr_set(struct net_device * dev,const u8 * addr)4604  static inline void dev_addr_set(struct net_device *dev, const u8 *addr)
4605  {
4606  	__dev_addr_set(dev, addr, dev->addr_len);
4607  }
4608  
4609  int dev_addr_add(struct net_device *dev, const unsigned char *addr,
4610  		 unsigned char addr_type);
4611  int dev_addr_del(struct net_device *dev, const unsigned char *addr,
4612  		 unsigned char addr_type);
4613  
4614  /* Functions used for unicast addresses handling */
4615  int dev_uc_add(struct net_device *dev, const unsigned char *addr);
4616  int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
4617  int dev_uc_del(struct net_device *dev, const unsigned char *addr);
4618  int dev_uc_sync(struct net_device *to, struct net_device *from);
4619  int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
4620  void dev_uc_unsync(struct net_device *to, struct net_device *from);
4621  void dev_uc_flush(struct net_device *dev);
4622  void dev_uc_init(struct net_device *dev);
4623  
4624  /**
4625   *  __dev_uc_sync - Synchronize device's unicast list
4626   *  @dev:  device to sync
4627   *  @sync: function to call if address should be added
4628   *  @unsync: function to call if address should be removed
4629   *
4630   *  Add newly added addresses to the interface, and release
4631   *  addresses that have been deleted.
4632   */
__dev_uc_sync(struct net_device * dev,int (* sync)(struct net_device *,const unsigned char *),int (* unsync)(struct net_device *,const unsigned char *))4633  static inline int __dev_uc_sync(struct net_device *dev,
4634  				int (*sync)(struct net_device *,
4635  					    const unsigned char *),
4636  				int (*unsync)(struct net_device *,
4637  					      const unsigned char *))
4638  {
4639  	return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
4640  }
4641  
4642  /**
4643   *  __dev_uc_unsync - Remove synchronized addresses from device
4644   *  @dev:  device to sync
4645   *  @unsync: function to call if address should be removed
4646   *
4647   *  Remove all addresses that were added to the device by dev_uc_sync().
4648   */
__dev_uc_unsync(struct net_device * dev,int (* unsync)(struct net_device *,const unsigned char *))4649  static inline void __dev_uc_unsync(struct net_device *dev,
4650  				   int (*unsync)(struct net_device *,
4651  						 const unsigned char *))
4652  {
4653  	__hw_addr_unsync_dev(&dev->uc, dev, unsync);
4654  }
4655  
4656  /* Functions used for multicast addresses handling */
4657  int dev_mc_add(struct net_device *dev, const unsigned char *addr);
4658  int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
4659  int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
4660  int dev_mc_del(struct net_device *dev, const unsigned char *addr);
4661  int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
4662  int dev_mc_sync(struct net_device *to, struct net_device *from);
4663  int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
4664  void dev_mc_unsync(struct net_device *to, struct net_device *from);
4665  void dev_mc_flush(struct net_device *dev);
4666  void dev_mc_init(struct net_device *dev);
4667  
4668  /**
4669   *  __dev_mc_sync - Synchronize device's multicast list
4670   *  @dev:  device to sync
4671   *  @sync: function to call if address should be added
4672   *  @unsync: function to call if address should be removed
4673   *
4674   *  Add newly added addresses to the interface, and release
4675   *  addresses that have been deleted.
4676   */
__dev_mc_sync(struct net_device * dev,int (* sync)(struct net_device *,const unsigned char *),int (* unsync)(struct net_device *,const unsigned char *))4677  static inline int __dev_mc_sync(struct net_device *dev,
4678  				int (*sync)(struct net_device *,
4679  					    const unsigned char *),
4680  				int (*unsync)(struct net_device *,
4681  					      const unsigned char *))
4682  {
4683  	return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
4684  }
4685  
4686  /**
4687   *  __dev_mc_unsync - Remove synchronized addresses from device
4688   *  @dev:  device to sync
4689   *  @unsync: function to call if address should be removed
4690   *
4691   *  Remove all addresses that were added to the device by dev_mc_sync().
4692   */
__dev_mc_unsync(struct net_device * dev,int (* unsync)(struct net_device *,const unsigned char *))4693  static inline void __dev_mc_unsync(struct net_device *dev,
4694  				   int (*unsync)(struct net_device *,
4695  						 const unsigned char *))
4696  {
4697  	__hw_addr_unsync_dev(&dev->mc, dev, unsync);
4698  }
4699  
4700  /* Functions used for secondary unicast and multicast support */
4701  void dev_set_rx_mode(struct net_device *dev);
4702  int dev_set_promiscuity(struct net_device *dev, int inc);
4703  int dev_set_allmulti(struct net_device *dev, int inc);
4704  void netdev_state_change(struct net_device *dev);
4705  void __netdev_notify_peers(struct net_device *dev);
4706  void netdev_notify_peers(struct net_device *dev);
4707  void netdev_features_change(struct net_device *dev);
4708  /* Load a device via the kmod */
4709  void dev_load(struct net *net, const char *name);
4710  struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
4711  					struct rtnl_link_stats64 *storage);
4712  void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
4713  			     const struct net_device_stats *netdev_stats);
4714  void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
4715  			   const struct pcpu_sw_netstats __percpu *netstats);
4716  void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
4717  
4718  enum {
4719  	NESTED_SYNC_IMM_BIT,
4720  	NESTED_SYNC_TODO_BIT,
4721  };
4722  
4723  #define __NESTED_SYNC_BIT(bit)	((u32)1 << (bit))
4724  #define __NESTED_SYNC(name)	__NESTED_SYNC_BIT(NESTED_SYNC_ ## name ## _BIT)
4725  
4726  #define NESTED_SYNC_IMM		__NESTED_SYNC(IMM)
4727  #define NESTED_SYNC_TODO	__NESTED_SYNC(TODO)
4728  
4729  struct netdev_nested_priv {
4730  	unsigned char flags;
4731  	void *data;
4732  };
4733  
4734  bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
4735  struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4736  						     struct list_head **iter);
4737  
4738  /* iterate through upper list, must be called under RCU read lock */
4739  #define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
4740  	for (iter = &(dev)->adj_list.upper, \
4741  	     updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
4742  	     updev; \
4743  	     updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
4744  
4745  int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
4746  				  int (*fn)(struct net_device *upper_dev,
4747  					    struct netdev_nested_priv *priv),
4748  				  struct netdev_nested_priv *priv);
4749  
4750  bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
4751  				  struct net_device *upper_dev);
4752  
4753  bool netdev_has_any_upper_dev(struct net_device *dev);
4754  
4755  void *netdev_lower_get_next_private(struct net_device *dev,
4756  				    struct list_head **iter);
4757  void *netdev_lower_get_next_private_rcu(struct net_device *dev,
4758  					struct list_head **iter);
4759  
4760  #define netdev_for_each_lower_private(dev, priv, iter) \
4761  	for (iter = (dev)->adj_list.lower.next, \
4762  	     priv = netdev_lower_get_next_private(dev, &(iter)); \
4763  	     priv; \
4764  	     priv = netdev_lower_get_next_private(dev, &(iter)))
4765  
4766  #define netdev_for_each_lower_private_rcu(dev, priv, iter) \
4767  	for (iter = &(dev)->adj_list.lower, \
4768  	     priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
4769  	     priv; \
4770  	     priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
4771  
4772  void *netdev_lower_get_next(struct net_device *dev,
4773  				struct list_head **iter);
4774  
4775  #define netdev_for_each_lower_dev(dev, ldev, iter) \
4776  	for (iter = (dev)->adj_list.lower.next, \
4777  	     ldev = netdev_lower_get_next(dev, &(iter)); \
4778  	     ldev; \
4779  	     ldev = netdev_lower_get_next(dev, &(iter)))
4780  
4781  struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
4782  					     struct list_head **iter);
4783  int netdev_walk_all_lower_dev(struct net_device *dev,
4784  			      int (*fn)(struct net_device *lower_dev,
4785  					struct netdev_nested_priv *priv),
4786  			      struct netdev_nested_priv *priv);
4787  int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
4788  				  int (*fn)(struct net_device *lower_dev,
4789  					    struct netdev_nested_priv *priv),
4790  				  struct netdev_nested_priv *priv);
4791  
4792  void *netdev_adjacent_get_private(struct list_head *adj_list);
4793  void *netdev_lower_get_first_private_rcu(struct net_device *dev);
4794  struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
4795  struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
4796  int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
4797  			  struct netlink_ext_ack *extack);
4798  int netdev_master_upper_dev_link(struct net_device *dev,
4799  				 struct net_device *upper_dev,
4800  				 void *upper_priv, void *upper_info,
4801  				 struct netlink_ext_ack *extack);
4802  void netdev_upper_dev_unlink(struct net_device *dev,
4803  			     struct net_device *upper_dev);
4804  int netdev_adjacent_change_prepare(struct net_device *old_dev,
4805  				   struct net_device *new_dev,
4806  				   struct net_device *dev,
4807  				   struct netlink_ext_ack *extack);
4808  void netdev_adjacent_change_commit(struct net_device *old_dev,
4809  				   struct net_device *new_dev,
4810  				   struct net_device *dev);
4811  void netdev_adjacent_change_abort(struct net_device *old_dev,
4812  				  struct net_device *new_dev,
4813  				  struct net_device *dev);
4814  void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
4815  void *netdev_lower_dev_get_private(struct net_device *dev,
4816  				   struct net_device *lower_dev);
4817  void netdev_lower_state_changed(struct net_device *lower_dev,
4818  				void *lower_state_info);
4819  
4820  /* RSS keys are 40 or 52 bytes long */
4821  #define NETDEV_RSS_KEY_LEN 52
4822  extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
4823  void netdev_rss_key_fill(void *buffer, size_t len);
4824  
4825  int skb_checksum_help(struct sk_buff *skb);
4826  int skb_crc32c_csum_help(struct sk_buff *skb);
4827  int skb_csum_hwoffload_help(struct sk_buff *skb,
4828  			    const netdev_features_t features);
4829  
4830  struct netdev_bonding_info {
4831  	ifslave	slave;
4832  	ifbond	master;
4833  };
4834  
4835  struct netdev_notifier_bonding_info {
4836  	struct netdev_notifier_info info; /* must be first */
4837  	struct netdev_bonding_info  bonding_info;
4838  };
4839  
4840  void netdev_bonding_info_change(struct net_device *dev,
4841  				struct netdev_bonding_info *bonding_info);
4842  
4843  #if IS_ENABLED(CONFIG_ETHTOOL_NETLINK)
4844  void ethtool_notify(struct net_device *dev, unsigned int cmd, const void *data);
4845  #else
ethtool_notify(struct net_device * dev,unsigned int cmd,const void * data)4846  static inline void ethtool_notify(struct net_device *dev, unsigned int cmd,
4847  				  const void *data)
4848  {
4849  }
4850  #endif
4851  
4852  __be16 skb_network_protocol(struct sk_buff *skb, int *depth);
4853  
can_checksum_protocol(netdev_features_t features,__be16 protocol)4854  static inline bool can_checksum_protocol(netdev_features_t features,
4855  					 __be16 protocol)
4856  {
4857  	if (protocol == htons(ETH_P_FCOE))
4858  		return !!(features & NETIF_F_FCOE_CRC);
4859  
4860  	/* Assume this is an IP checksum (not SCTP CRC) */
4861  
4862  	if (features & NETIF_F_HW_CSUM) {
4863  		/* Can checksum everything */
4864  		return true;
4865  	}
4866  
4867  	switch (protocol) {
4868  	case htons(ETH_P_IP):
4869  		return !!(features & NETIF_F_IP_CSUM);
4870  	case htons(ETH_P_IPV6):
4871  		return !!(features & NETIF_F_IPV6_CSUM);
4872  	default:
4873  		return false;
4874  	}
4875  }
4876  
4877  #ifdef CONFIG_BUG
4878  void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb);
4879  #else
netdev_rx_csum_fault(struct net_device * dev,struct sk_buff * skb)4880  static inline void netdev_rx_csum_fault(struct net_device *dev,
4881  					struct sk_buff *skb)
4882  {
4883  }
4884  #endif
4885  /* rx skb timestamps */
4886  void net_enable_timestamp(void);
4887  void net_disable_timestamp(void);
4888  
netdev_get_tstamp(struct net_device * dev,const struct skb_shared_hwtstamps * hwtstamps,bool cycles)4889  static inline ktime_t netdev_get_tstamp(struct net_device *dev,
4890  					const struct skb_shared_hwtstamps *hwtstamps,
4891  					bool cycles)
4892  {
4893  	const struct net_device_ops *ops = dev->netdev_ops;
4894  
4895  	if (ops->ndo_get_tstamp)
4896  		return ops->ndo_get_tstamp(dev, hwtstamps, cycles);
4897  
4898  	return hwtstamps->hwtstamp;
4899  }
4900  
4901  #ifndef CONFIG_PREEMPT_RT
netdev_xmit_set_more(bool more)4902  static inline void netdev_xmit_set_more(bool more)
4903  {
4904  	__this_cpu_write(softnet_data.xmit.more, more);
4905  }
4906  
netdev_xmit_more(void)4907  static inline bool netdev_xmit_more(void)
4908  {
4909  	return __this_cpu_read(softnet_data.xmit.more);
4910  }
4911  #else
netdev_xmit_set_more(bool more)4912  static inline void netdev_xmit_set_more(bool more)
4913  {
4914  	current->net_xmit.more = more;
4915  }
4916  
netdev_xmit_more(void)4917  static inline bool netdev_xmit_more(void)
4918  {
4919  	return current->net_xmit.more;
4920  }
4921  #endif
4922  
__netdev_start_xmit(const struct net_device_ops * ops,struct sk_buff * skb,struct net_device * dev,bool more)4923  static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
4924  					      struct sk_buff *skb, struct net_device *dev,
4925  					      bool more)
4926  {
4927  	netdev_xmit_set_more(more);
4928  	return ops->ndo_start_xmit(skb, dev);
4929  }
4930  
netdev_start_xmit(struct sk_buff * skb,struct net_device * dev,struct netdev_queue * txq,bool more)4931  static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
4932  					    struct netdev_queue *txq, bool more)
4933  {
4934  	const struct net_device_ops *ops = dev->netdev_ops;
4935  	netdev_tx_t rc;
4936  
4937  	rc = __netdev_start_xmit(ops, skb, dev, more);
4938  	if (rc == NETDEV_TX_OK)
4939  		txq_trans_update(txq);
4940  
4941  	return rc;
4942  }
4943  
4944  int netdev_class_create_file_ns(const struct class_attribute *class_attr,
4945  				const void *ns);
4946  void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
4947  				 const void *ns);
4948  
4949  extern const struct kobj_ns_type_operations net_ns_type_operations;
4950  
4951  const char *netdev_drivername(const struct net_device *dev);
4952  
netdev_intersect_features(netdev_features_t f1,netdev_features_t f2)4953  static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
4954  							  netdev_features_t f2)
4955  {
4956  	if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
4957  		if (f1 & NETIF_F_HW_CSUM)
4958  			f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4959  		else
4960  			f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4961  	}
4962  
4963  	return f1 & f2;
4964  }
4965  
netdev_get_wanted_features(struct net_device * dev)4966  static inline netdev_features_t netdev_get_wanted_features(
4967  	struct net_device *dev)
4968  {
4969  	return (dev->features & ~dev->hw_features) | dev->wanted_features;
4970  }
4971  netdev_features_t netdev_increment_features(netdev_features_t all,
4972  	netdev_features_t one, netdev_features_t mask);
4973  
4974  /* Allow TSO being used on stacked device :
4975   * Performing the GSO segmentation before last device
4976   * is a performance improvement.
4977   */
netdev_add_tso_features(netdev_features_t features,netdev_features_t mask)4978  static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
4979  							netdev_features_t mask)
4980  {
4981  	return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
4982  }
4983  
4984  int __netdev_update_features(struct net_device *dev);
4985  void netdev_update_features(struct net_device *dev);
4986  void netdev_change_features(struct net_device *dev);
4987  
4988  void netif_stacked_transfer_operstate(const struct net_device *rootdev,
4989  					struct net_device *dev);
4990  
4991  netdev_features_t passthru_features_check(struct sk_buff *skb,
4992  					  struct net_device *dev,
4993  					  netdev_features_t features);
4994  netdev_features_t netif_skb_features(struct sk_buff *skb);
4995  void skb_warn_bad_offload(const struct sk_buff *skb);
4996  
net_gso_ok(netdev_features_t features,int gso_type)4997  static inline bool net_gso_ok(netdev_features_t features, int gso_type)
4998  {
4999  	netdev_features_t feature = (netdev_features_t)gso_type << NETIF_F_GSO_SHIFT;
5000  
5001  	/* check flags correspondence */
5002  	BUILD_BUG_ON(SKB_GSO_TCPV4   != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
5003  	BUILD_BUG_ON(SKB_GSO_DODGY   != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
5004  	BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
5005  	BUILD_BUG_ON(SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
5006  	BUILD_BUG_ON(SKB_GSO_TCPV6   != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
5007  	BUILD_BUG_ON(SKB_GSO_FCOE    != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
5008  	BUILD_BUG_ON(SKB_GSO_GRE     != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
5009  	BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
5010  	BUILD_BUG_ON(SKB_GSO_IPXIP4  != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
5011  	BUILD_BUG_ON(SKB_GSO_IPXIP6  != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
5012  	BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
5013  	BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
5014  	BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
5015  	BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
5016  	BUILD_BUG_ON(SKB_GSO_SCTP    != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
5017  	BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
5018  	BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
5019  	BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
5020  	BUILD_BUG_ON(SKB_GSO_FRAGLIST != (NETIF_F_GSO_FRAGLIST >> NETIF_F_GSO_SHIFT));
5021  
5022  	return (features & feature) == feature;
5023  }
5024  
skb_gso_ok(struct sk_buff * skb,netdev_features_t features)5025  static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
5026  {
5027  	return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
5028  	       (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
5029  }
5030  
netif_needs_gso(struct sk_buff * skb,netdev_features_t features)5031  static inline bool netif_needs_gso(struct sk_buff *skb,
5032  				   netdev_features_t features)
5033  {
5034  	return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
5035  		unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
5036  			 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
5037  }
5038  
5039  void netif_set_tso_max_size(struct net_device *dev, unsigned int size);
5040  void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs);
5041  void netif_inherit_tso_max(struct net_device *to,
5042  			   const struct net_device *from);
5043  
5044  static inline unsigned int
netif_get_gro_max_size(const struct net_device * dev,const struct sk_buff * skb)5045  netif_get_gro_max_size(const struct net_device *dev, const struct sk_buff *skb)
5046  {
5047  	/* pairs with WRITE_ONCE() in netif_set_gro(_ipv4)_max_size() */
5048  	return skb->protocol == htons(ETH_P_IPV6) ?
5049  	       READ_ONCE(dev->gro_max_size) :
5050  	       READ_ONCE(dev->gro_ipv4_max_size);
5051  }
5052  
5053  static inline unsigned int
netif_get_gso_max_size(const struct net_device * dev,const struct sk_buff * skb)5054  netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb)
5055  {
5056  	/* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */
5057  	return skb->protocol == htons(ETH_P_IPV6) ?
5058  	       READ_ONCE(dev->gso_max_size) :
5059  	       READ_ONCE(dev->gso_ipv4_max_size);
5060  }
5061  
netif_is_macsec(const struct net_device * dev)5062  static inline bool netif_is_macsec(const struct net_device *dev)
5063  {
5064  	return dev->priv_flags & IFF_MACSEC;
5065  }
5066  
netif_is_macvlan(const struct net_device * dev)5067  static inline bool netif_is_macvlan(const struct net_device *dev)
5068  {
5069  	return dev->priv_flags & IFF_MACVLAN;
5070  }
5071  
netif_is_macvlan_port(const struct net_device * dev)5072  static inline bool netif_is_macvlan_port(const struct net_device *dev)
5073  {
5074  	return dev->priv_flags & IFF_MACVLAN_PORT;
5075  }
5076  
netif_is_bond_master(const struct net_device * dev)5077  static inline bool netif_is_bond_master(const struct net_device *dev)
5078  {
5079  	return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
5080  }
5081  
netif_is_bond_slave(const struct net_device * dev)5082  static inline bool netif_is_bond_slave(const struct net_device *dev)
5083  {
5084  	return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
5085  }
5086  
netif_supports_nofcs(struct net_device * dev)5087  static inline bool netif_supports_nofcs(struct net_device *dev)
5088  {
5089  	return dev->priv_flags & IFF_SUPP_NOFCS;
5090  }
5091  
netif_has_l3_rx_handler(const struct net_device * dev)5092  static inline bool netif_has_l3_rx_handler(const struct net_device *dev)
5093  {
5094  	return dev->priv_flags & IFF_L3MDEV_RX_HANDLER;
5095  }
5096  
netif_is_l3_master(const struct net_device * dev)5097  static inline bool netif_is_l3_master(const struct net_device *dev)
5098  {
5099  	return dev->priv_flags & IFF_L3MDEV_MASTER;
5100  }
5101  
netif_is_l3_slave(const struct net_device * dev)5102  static inline bool netif_is_l3_slave(const struct net_device *dev)
5103  {
5104  	return dev->priv_flags & IFF_L3MDEV_SLAVE;
5105  }
5106  
dev_sdif(const struct net_device * dev)5107  static inline int dev_sdif(const struct net_device *dev)
5108  {
5109  #ifdef CONFIG_NET_L3_MASTER_DEV
5110  	if (netif_is_l3_slave(dev))
5111  		return dev->ifindex;
5112  #endif
5113  	return 0;
5114  }
5115  
netif_is_bridge_master(const struct net_device * dev)5116  static inline bool netif_is_bridge_master(const struct net_device *dev)
5117  {
5118  	return dev->priv_flags & IFF_EBRIDGE;
5119  }
5120  
netif_is_bridge_port(const struct net_device * dev)5121  static inline bool netif_is_bridge_port(const struct net_device *dev)
5122  {
5123  	return dev->priv_flags & IFF_BRIDGE_PORT;
5124  }
5125  
netif_is_ovs_master(const struct net_device * dev)5126  static inline bool netif_is_ovs_master(const struct net_device *dev)
5127  {
5128  	return dev->priv_flags & IFF_OPENVSWITCH;
5129  }
5130  
netif_is_ovs_port(const struct net_device * dev)5131  static inline bool netif_is_ovs_port(const struct net_device *dev)
5132  {
5133  	return dev->priv_flags & IFF_OVS_DATAPATH;
5134  }
5135  
netif_is_any_bridge_master(const struct net_device * dev)5136  static inline bool netif_is_any_bridge_master(const struct net_device *dev)
5137  {
5138  	return netif_is_bridge_master(dev) || netif_is_ovs_master(dev);
5139  }
5140  
netif_is_any_bridge_port(const struct net_device * dev)5141  static inline bool netif_is_any_bridge_port(const struct net_device *dev)
5142  {
5143  	return netif_is_bridge_port(dev) || netif_is_ovs_port(dev);
5144  }
5145  
netif_is_team_master(const struct net_device * dev)5146  static inline bool netif_is_team_master(const struct net_device *dev)
5147  {
5148  	return dev->priv_flags & IFF_TEAM;
5149  }
5150  
netif_is_team_port(const struct net_device * dev)5151  static inline bool netif_is_team_port(const struct net_device *dev)
5152  {
5153  	return dev->priv_flags & IFF_TEAM_PORT;
5154  }
5155  
netif_is_lag_master(const struct net_device * dev)5156  static inline bool netif_is_lag_master(const struct net_device *dev)
5157  {
5158  	return netif_is_bond_master(dev) || netif_is_team_master(dev);
5159  }
5160  
netif_is_lag_port(const struct net_device * dev)5161  static inline bool netif_is_lag_port(const struct net_device *dev)
5162  {
5163  	return netif_is_bond_slave(dev) || netif_is_team_port(dev);
5164  }
5165  
netif_is_rxfh_configured(const struct net_device * dev)5166  static inline bool netif_is_rxfh_configured(const struct net_device *dev)
5167  {
5168  	return dev->priv_flags & IFF_RXFH_CONFIGURED;
5169  }
5170  
netif_is_failover(const struct net_device * dev)5171  static inline bool netif_is_failover(const struct net_device *dev)
5172  {
5173  	return dev->priv_flags & IFF_FAILOVER;
5174  }
5175  
netif_is_failover_slave(const struct net_device * dev)5176  static inline bool netif_is_failover_slave(const struct net_device *dev)
5177  {
5178  	return dev->priv_flags & IFF_FAILOVER_SLAVE;
5179  }
5180  
5181  /* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */
netif_keep_dst(struct net_device * dev)5182  static inline void netif_keep_dst(struct net_device *dev)
5183  {
5184  	dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
5185  }
5186  
5187  /* return true if dev can't cope with mtu frames that need vlan tag insertion */
netif_reduces_vlan_mtu(struct net_device * dev)5188  static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
5189  {
5190  	/* TODO: reserve and use an additional IFF bit, if we get more users */
5191  	return netif_is_macsec(dev);
5192  }
5193  
5194  extern struct pernet_operations __net_initdata loopback_net_ops;
5195  
5196  /* Logging, debugging and troubleshooting/diagnostic helpers. */
5197  
5198  /* netdev_printk helpers, similar to dev_printk */
5199  
netdev_name(const struct net_device * dev)5200  static inline const char *netdev_name(const struct net_device *dev)
5201  {
5202  	if (!dev->name[0] || strchr(dev->name, '%'))
5203  		return "(unnamed net_device)";
5204  	return dev->name;
5205  }
5206  
netdev_reg_state(const struct net_device * dev)5207  static inline const char *netdev_reg_state(const struct net_device *dev)
5208  {
5209  	u8 reg_state = READ_ONCE(dev->reg_state);
5210  
5211  	switch (reg_state) {
5212  	case NETREG_UNINITIALIZED: return " (uninitialized)";
5213  	case NETREG_REGISTERED: return "";
5214  	case NETREG_UNREGISTERING: return " (unregistering)";
5215  	case NETREG_UNREGISTERED: return " (unregistered)";
5216  	case NETREG_RELEASED: return " (released)";
5217  	case NETREG_DUMMY: return " (dummy)";
5218  	}
5219  
5220  	WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, reg_state);
5221  	return " (unknown)";
5222  }
5223  
5224  #define MODULE_ALIAS_NETDEV(device) \
5225  	MODULE_ALIAS("netdev-" device)
5226  
5227  /*
5228   * netdev_WARN() acts like dev_printk(), but with the key difference
5229   * of using a WARN/WARN_ON to get the message out, including the
5230   * file/line information and a backtrace.
5231   */
5232  #define netdev_WARN(dev, format, args...)			\
5233  	WARN(1, "netdevice: %s%s: " format, netdev_name(dev),	\
5234  	     netdev_reg_state(dev), ##args)
5235  
5236  #define netdev_WARN_ONCE(dev, format, args...)				\
5237  	WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev),	\
5238  		  netdev_reg_state(dev), ##args)
5239  
5240  /*
5241   *	The list of packet types we will receive (as opposed to discard)
5242   *	and the routines to invoke.
5243   *
5244   *	Why 16. Because with 16 the only overlap we get on a hash of the
5245   *	low nibble of the protocol value is RARP/SNAP/X.25.
5246   *
5247   *		0800	IP
5248   *		0001	802.3
5249   *		0002	AX.25
5250   *		0004	802.2
5251   *		8035	RARP
5252   *		0005	SNAP
5253   *		0805	X.25
5254   *		0806	ARP
5255   *		8137	IPX
5256   *		0009	Localtalk
5257   *		86DD	IPv6
5258   */
5259  #define PTYPE_HASH_SIZE	(16)
5260  #define PTYPE_HASH_MASK	(PTYPE_HASH_SIZE - 1)
5261  
5262  extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
5263  
5264  extern struct net_device *blackhole_netdev;
5265  
5266  /* Note: Avoid these macros in fast path, prefer per-cpu or per-queue counters. */
5267  #define DEV_STATS_INC(DEV, FIELD) atomic_long_inc(&(DEV)->stats.__##FIELD)
5268  #define DEV_STATS_ADD(DEV, FIELD, VAL) 	\
5269  		atomic_long_add((VAL), &(DEV)->stats.__##FIELD)
5270  #define DEV_STATS_READ(DEV, FIELD) atomic_long_read(&(DEV)->stats.__##FIELD)
5271  
5272  #endif	/* _LINUX_NETDEVICE_H */
5273