1  // SPDX-License-Identifier: GPL-2.0-only
2  /*
3   * INET		An implementation of the TCP/IP protocol suite for the LINUX
4   *		operating system.  INET is implemented using the  BSD Socket
5   *		interface as the means of communication with the user level.
6   *
7   *		Implementation of the Transmission Control Protocol(TCP).
8   *
9   * Authors:	Ross Biro
10   *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11   *		Mark Evans, <evansmp@uhura.aston.ac.uk>
12   *		Corey Minyard <wf-rch!minyard@relay.EU.net>
13   *		Florian La Roche, <flla@stud.uni-sb.de>
14   *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15   *		Linus Torvalds, <torvalds@cs.helsinki.fi>
16   *		Alan Cox, <gw4pts@gw4pts.ampr.org>
17   *		Matthew Dillon, <dillon@apollo.west.oic.com>
18   *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19   *		Jorge Cwik, <jorge@laser.satlink.net>
20   */
21  
22  /*
23   * Changes:	Pedro Roque	:	Retransmit queue handled by TCP.
24   *				:	Fragmentation on mtu decrease
25   *				:	Segment collapse on retransmit
26   *				:	AF independence
27   *
28   *		Linus Torvalds	:	send_delayed_ack
29   *		David S. Miller	:	Charge memory using the right skb
30   *					during syn/ack processing.
31   *		David S. Miller :	Output engine completely rewritten.
32   *		Andrea Arcangeli:	SYNACK carry ts_recent in tsecr.
33   *		Cacophonix Gaul :	draft-minshall-nagle-01
34   *		J Hadi Salim	:	ECN support
35   *
36   */
37  
38  #define pr_fmt(fmt) "TCP: " fmt
39  
40  #include <net/tcp.h>
41  #include <net/mptcp.h>
42  #include <net/proto_memory.h>
43  
44  #include <linux/compiler.h>
45  #include <linux/gfp.h>
46  #include <linux/module.h>
47  #include <linux/static_key.h>
48  #include <linux/skbuff_ref.h>
49  
50  #include <trace/events/tcp.h>
51  
52  /* Refresh clocks of a TCP socket,
53   * ensuring monotically increasing values.
54   */
tcp_mstamp_refresh(struct tcp_sock * tp)55  void tcp_mstamp_refresh(struct tcp_sock *tp)
56  {
57  	u64 val = tcp_clock_ns();
58  
59  	tp->tcp_clock_cache = val;
60  	tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);
61  }
62  
63  static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
64  			   int push_one, gfp_t gfp);
65  
66  /* Account for new data that has been sent to the network. */
tcp_event_new_data_sent(struct sock * sk,struct sk_buff * skb)67  static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
68  {
69  	struct inet_connection_sock *icsk = inet_csk(sk);
70  	struct tcp_sock *tp = tcp_sk(sk);
71  	unsigned int prior_packets = tp->packets_out;
72  
73  	WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(skb)->end_seq);
74  
75  	__skb_unlink(skb, &sk->sk_write_queue);
76  	tcp_rbtree_insert(&sk->tcp_rtx_queue, skb);
77  
78  	if (tp->highest_sack == NULL)
79  		tp->highest_sack = skb;
80  
81  	tp->packets_out += tcp_skb_pcount(skb);
82  	if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
83  		tcp_rearm_rto(sk);
84  
85  	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
86  		      tcp_skb_pcount(skb));
87  	tcp_check_space(sk);
88  }
89  
90  /* SND.NXT, if window was not shrunk or the amount of shrunk was less than one
91   * window scaling factor due to loss of precision.
92   * If window has been shrunk, what should we make? It is not clear at all.
93   * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
94   * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
95   * invalid. OK, let's make this for now:
96   */
tcp_acceptable_seq(const struct sock * sk)97  static inline __u32 tcp_acceptable_seq(const struct sock *sk)
98  {
99  	const struct tcp_sock *tp = tcp_sk(sk);
100  
101  	if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||
102  	    (tp->rx_opt.wscale_ok &&
103  	     ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))
104  		return tp->snd_nxt;
105  	else
106  		return tcp_wnd_end(tp);
107  }
108  
109  /* Calculate mss to advertise in SYN segment.
110   * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
111   *
112   * 1. It is independent of path mtu.
113   * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
114   * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
115   *    attached devices, because some buggy hosts are confused by
116   *    large MSS.
117   * 4. We do not make 3, we advertise MSS, calculated from first
118   *    hop device mtu, but allow to raise it to ip_rt_min_advmss.
119   *    This may be overridden via information stored in routing table.
120   * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
121   *    probably even Jumbo".
122   */
tcp_advertise_mss(struct sock * sk)123  static __u16 tcp_advertise_mss(struct sock *sk)
124  {
125  	struct tcp_sock *tp = tcp_sk(sk);
126  	const struct dst_entry *dst = __sk_dst_get(sk);
127  	int mss = tp->advmss;
128  
129  	if (dst) {
130  		unsigned int metric = dst_metric_advmss(dst);
131  
132  		if (metric < mss) {
133  			mss = metric;
134  			tp->advmss = mss;
135  		}
136  	}
137  
138  	return (__u16)mss;
139  }
140  
141  /* RFC2861. Reset CWND after idle period longer RTO to "restart window".
142   * This is the first part of cwnd validation mechanism.
143   */
tcp_cwnd_restart(struct sock * sk,s32 delta)144  void tcp_cwnd_restart(struct sock *sk, s32 delta)
145  {
146  	struct tcp_sock *tp = tcp_sk(sk);
147  	u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
148  	u32 cwnd = tcp_snd_cwnd(tp);
149  
150  	tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
151  
152  	tp->snd_ssthresh = tcp_current_ssthresh(sk);
153  	restart_cwnd = min(restart_cwnd, cwnd);
154  
155  	while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
156  		cwnd >>= 1;
157  	tcp_snd_cwnd_set(tp, max(cwnd, restart_cwnd));
158  	tp->snd_cwnd_stamp = tcp_jiffies32;
159  	tp->snd_cwnd_used = 0;
160  }
161  
162  /* Congestion state accounting after a packet has been sent. */
tcp_event_data_sent(struct tcp_sock * tp,struct sock * sk)163  static void tcp_event_data_sent(struct tcp_sock *tp,
164  				struct sock *sk)
165  {
166  	struct inet_connection_sock *icsk = inet_csk(sk);
167  	const u32 now = tcp_jiffies32;
168  
169  	if (tcp_packets_in_flight(tp) == 0)
170  		tcp_ca_event(sk, CA_EVENT_TX_START);
171  
172  	tp->lsndtime = now;
173  
174  	/* If it is a reply for ato after last received
175  	 * packet, increase pingpong count.
176  	 */
177  	if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
178  		inet_csk_inc_pingpong_cnt(sk);
179  }
180  
181  /* Account for an ACK we sent. */
tcp_event_ack_sent(struct sock * sk,u32 rcv_nxt)182  static inline void tcp_event_ack_sent(struct sock *sk, u32 rcv_nxt)
183  {
184  	struct tcp_sock *tp = tcp_sk(sk);
185  
186  	if (unlikely(tp->compressed_ack)) {
187  		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
188  			      tp->compressed_ack);
189  		tp->compressed_ack = 0;
190  		if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
191  			__sock_put(sk);
192  	}
193  
194  	if (unlikely(rcv_nxt != tp->rcv_nxt))
195  		return;  /* Special ACK sent by DCTCP to reflect ECN */
196  	tcp_dec_quickack_mode(sk);
197  	inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
198  }
199  
200  /* Determine a window scaling and initial window to offer.
201   * Based on the assumption that the given amount of space
202   * will be offered. Store the results in the tp structure.
203   * NOTE: for smooth operation initial space offering should
204   * be a multiple of mss if possible. We assume here that mss >= 1.
205   * This MUST be enforced by all callers.
206   */
tcp_select_initial_window(const struct sock * sk,int __space,__u32 mss,__u32 * rcv_wnd,__u32 * __window_clamp,int wscale_ok,__u8 * rcv_wscale,__u32 init_rcv_wnd)207  void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
208  			       __u32 *rcv_wnd, __u32 *__window_clamp,
209  			       int wscale_ok, __u8 *rcv_wscale,
210  			       __u32 init_rcv_wnd)
211  {
212  	unsigned int space = (__space < 0 ? 0 : __space);
213  	u32 window_clamp = READ_ONCE(*__window_clamp);
214  
215  	/* If no clamp set the clamp to the max possible scaled window */
216  	if (window_clamp == 0)
217  		window_clamp = (U16_MAX << TCP_MAX_WSCALE);
218  	space = min(window_clamp, space);
219  
220  	/* Quantize space offering to a multiple of mss if possible. */
221  	if (space > mss)
222  		space = rounddown(space, mss);
223  
224  	/* NOTE: offering an initial window larger than 32767
225  	 * will break some buggy TCP stacks. If the admin tells us
226  	 * it is likely we could be speaking with such a buggy stack
227  	 * we will truncate our initial window offering to 32K-1
228  	 * unless the remote has sent us a window scaling option,
229  	 * which we interpret as a sign the remote TCP is not
230  	 * misinterpreting the window field as a signed quantity.
231  	 */
232  	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows))
233  		(*rcv_wnd) = min(space, MAX_TCP_WINDOW);
234  	else
235  		(*rcv_wnd) = space;
236  
237  	if (init_rcv_wnd)
238  		*rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
239  
240  	*rcv_wscale = 0;
241  	if (wscale_ok) {
242  		/* Set window scaling on max possible window */
243  		space = max_t(u32, space, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]));
244  		space = max_t(u32, space, READ_ONCE(sysctl_rmem_max));
245  		space = min_t(u32, space, window_clamp);
246  		*rcv_wscale = clamp_t(int, ilog2(space) - 15,
247  				      0, TCP_MAX_WSCALE);
248  	}
249  	/* Set the clamp no higher than max representable value */
250  	WRITE_ONCE(*__window_clamp,
251  		   min_t(__u32, U16_MAX << (*rcv_wscale), window_clamp));
252  }
253  EXPORT_SYMBOL(tcp_select_initial_window);
254  
255  /* Chose a new window to advertise, update state in tcp_sock for the
256   * socket, and return result with RFC1323 scaling applied.  The return
257   * value can be stuffed directly into th->window for an outgoing
258   * frame.
259   */
tcp_select_window(struct sock * sk)260  static u16 tcp_select_window(struct sock *sk)
261  {
262  	struct tcp_sock *tp = tcp_sk(sk);
263  	struct net *net = sock_net(sk);
264  	u32 old_win = tp->rcv_wnd;
265  	u32 cur_win, new_win;
266  
267  	/* Make the window 0 if we failed to queue the data because we
268  	 * are out of memory. The window is temporary, so we don't store
269  	 * it on the socket.
270  	 */
271  	if (unlikely(inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOMEM))
272  		return 0;
273  
274  	cur_win = tcp_receive_window(tp);
275  	new_win = __tcp_select_window(sk);
276  	if (new_win < cur_win) {
277  		/* Danger Will Robinson!
278  		 * Don't update rcv_wup/rcv_wnd here or else
279  		 * we will not be able to advertise a zero
280  		 * window in time.  --DaveM
281  		 *
282  		 * Relax Will Robinson.
283  		 */
284  		if (!READ_ONCE(net->ipv4.sysctl_tcp_shrink_window) || !tp->rx_opt.rcv_wscale) {
285  			/* Never shrink the offered window */
286  			if (new_win == 0)
287  				NET_INC_STATS(net, LINUX_MIB_TCPWANTZEROWINDOWADV);
288  			new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
289  		}
290  	}
291  
292  	tp->rcv_wnd = new_win;
293  	tp->rcv_wup = tp->rcv_nxt;
294  
295  	/* Make sure we do not exceed the maximum possible
296  	 * scaled window.
297  	 */
298  	if (!tp->rx_opt.rcv_wscale &&
299  	    READ_ONCE(net->ipv4.sysctl_tcp_workaround_signed_windows))
300  		new_win = min(new_win, MAX_TCP_WINDOW);
301  	else
302  		new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
303  
304  	/* RFC1323 scaling applied */
305  	new_win >>= tp->rx_opt.rcv_wscale;
306  
307  	/* If we advertise zero window, disable fast path. */
308  	if (new_win == 0) {
309  		tp->pred_flags = 0;
310  		if (old_win)
311  			NET_INC_STATS(net, LINUX_MIB_TCPTOZEROWINDOWADV);
312  	} else if (old_win == 0) {
313  		NET_INC_STATS(net, LINUX_MIB_TCPFROMZEROWINDOWADV);
314  	}
315  
316  	return new_win;
317  }
318  
319  /* Packet ECN state for a SYN-ACK */
tcp_ecn_send_synack(struct sock * sk,struct sk_buff * skb)320  static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
321  {
322  	const struct tcp_sock *tp = tcp_sk(sk);
323  
324  	TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
325  	if (!(tp->ecn_flags & TCP_ECN_OK))
326  		TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
327  	else if (tcp_ca_needs_ecn(sk) ||
328  		 tcp_bpf_ca_needs_ecn(sk))
329  		INET_ECN_xmit(sk);
330  }
331  
332  /* Packet ECN state for a SYN.  */
tcp_ecn_send_syn(struct sock * sk,struct sk_buff * skb)333  static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
334  {
335  	struct tcp_sock *tp = tcp_sk(sk);
336  	bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk);
337  	bool use_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn) == 1 ||
338  		tcp_ca_needs_ecn(sk) || bpf_needs_ecn;
339  
340  	if (!use_ecn) {
341  		const struct dst_entry *dst = __sk_dst_get(sk);
342  
343  		if (dst && dst_feature(dst, RTAX_FEATURE_ECN))
344  			use_ecn = true;
345  	}
346  
347  	tp->ecn_flags = 0;
348  
349  	if (use_ecn) {
350  		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
351  		tp->ecn_flags = TCP_ECN_OK;
352  		if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn)
353  			INET_ECN_xmit(sk);
354  	}
355  }
356  
tcp_ecn_clear_syn(struct sock * sk,struct sk_buff * skb)357  static void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb)
358  {
359  	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback))
360  		/* tp->ecn_flags are cleared at a later point in time when
361  		 * SYN ACK is ultimatively being received.
362  		 */
363  		TCP_SKB_CB(skb)->tcp_flags &= ~(TCPHDR_ECE | TCPHDR_CWR);
364  }
365  
366  static void
tcp_ecn_make_synack(const struct request_sock * req,struct tcphdr * th)367  tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th)
368  {
369  	if (inet_rsk(req)->ecn_ok)
370  		th->ece = 1;
371  }
372  
373  /* Set up ECN state for a packet on a ESTABLISHED socket that is about to
374   * be sent.
375   */
tcp_ecn_send(struct sock * sk,struct sk_buff * skb,struct tcphdr * th,int tcp_header_len)376  static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
377  			 struct tcphdr *th, int tcp_header_len)
378  {
379  	struct tcp_sock *tp = tcp_sk(sk);
380  
381  	if (tp->ecn_flags & TCP_ECN_OK) {
382  		/* Not-retransmitted data segment: set ECT and inject CWR. */
383  		if (skb->len != tcp_header_len &&
384  		    !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
385  			INET_ECN_xmit(sk);
386  			if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
387  				tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
388  				th->cwr = 1;
389  				skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
390  			}
391  		} else if (!tcp_ca_needs_ecn(sk)) {
392  			/* ACK or retransmitted segment: clear ECT|CE */
393  			INET_ECN_dontxmit(sk);
394  		}
395  		if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
396  			th->ece = 1;
397  	}
398  }
399  
400  /* Constructs common control bits of non-data skb. If SYN/FIN is present,
401   * auto increment end seqno.
402   */
tcp_init_nondata_skb(struct sk_buff * skb,u32 seq,u8 flags)403  static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
404  {
405  	skb->ip_summed = CHECKSUM_PARTIAL;
406  
407  	TCP_SKB_CB(skb)->tcp_flags = flags;
408  
409  	tcp_skb_pcount_set(skb, 1);
410  
411  	TCP_SKB_CB(skb)->seq = seq;
412  	if (flags & (TCPHDR_SYN | TCPHDR_FIN))
413  		seq++;
414  	TCP_SKB_CB(skb)->end_seq = seq;
415  }
416  
tcp_urg_mode(const struct tcp_sock * tp)417  static inline bool tcp_urg_mode(const struct tcp_sock *tp)
418  {
419  	return tp->snd_una != tp->snd_up;
420  }
421  
422  #define OPTION_SACK_ADVERTISE	BIT(0)
423  #define OPTION_TS		BIT(1)
424  #define OPTION_MD5		BIT(2)
425  #define OPTION_WSCALE		BIT(3)
426  #define OPTION_FAST_OPEN_COOKIE	BIT(8)
427  #define OPTION_SMC		BIT(9)
428  #define OPTION_MPTCP		BIT(10)
429  #define OPTION_AO		BIT(11)
430  
smc_options_write(__be32 * ptr,u16 * options)431  static void smc_options_write(__be32 *ptr, u16 *options)
432  {
433  #if IS_ENABLED(CONFIG_SMC)
434  	if (static_branch_unlikely(&tcp_have_smc)) {
435  		if (unlikely(OPTION_SMC & *options)) {
436  			*ptr++ = htonl((TCPOPT_NOP  << 24) |
437  				       (TCPOPT_NOP  << 16) |
438  				       (TCPOPT_EXP <<  8) |
439  				       (TCPOLEN_EXP_SMC_BASE));
440  			*ptr++ = htonl(TCPOPT_SMC_MAGIC);
441  		}
442  	}
443  #endif
444  }
445  
446  struct tcp_out_options {
447  	u16 options;		/* bit field of OPTION_* */
448  	u16 mss;		/* 0 to disable */
449  	u8 ws;			/* window scale, 0 to disable */
450  	u8 num_sack_blocks;	/* number of SACK blocks to include */
451  	u8 hash_size;		/* bytes in hash_location */
452  	u8 bpf_opt_len;		/* length of BPF hdr option */
453  	__u8 *hash_location;	/* temporary pointer, overloaded */
454  	__u32 tsval, tsecr;	/* need to include OPTION_TS */
455  	struct tcp_fastopen_cookie *fastopen_cookie;	/* Fast open cookie */
456  	struct mptcp_out_options mptcp;
457  };
458  
mptcp_options_write(struct tcphdr * th,__be32 * ptr,struct tcp_sock * tp,struct tcp_out_options * opts)459  static void mptcp_options_write(struct tcphdr *th, __be32 *ptr,
460  				struct tcp_sock *tp,
461  				struct tcp_out_options *opts)
462  {
463  #if IS_ENABLED(CONFIG_MPTCP)
464  	if (unlikely(OPTION_MPTCP & opts->options))
465  		mptcp_write_options(th, ptr, tp, &opts->mptcp);
466  #endif
467  }
468  
469  #ifdef CONFIG_CGROUP_BPF
bpf_skops_write_hdr_opt_arg0(struct sk_buff * skb,enum tcp_synack_type synack_type)470  static int bpf_skops_write_hdr_opt_arg0(struct sk_buff *skb,
471  					enum tcp_synack_type synack_type)
472  {
473  	if (unlikely(!skb))
474  		return BPF_WRITE_HDR_TCP_CURRENT_MSS;
475  
476  	if (unlikely(synack_type == TCP_SYNACK_COOKIE))
477  		return BPF_WRITE_HDR_TCP_SYNACK_COOKIE;
478  
479  	return 0;
480  }
481  
482  /* req, syn_skb and synack_type are used when writing synack */
bpf_skops_hdr_opt_len(struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct sk_buff * syn_skb,enum tcp_synack_type synack_type,struct tcp_out_options * opts,unsigned int * remaining)483  static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
484  				  struct request_sock *req,
485  				  struct sk_buff *syn_skb,
486  				  enum tcp_synack_type synack_type,
487  				  struct tcp_out_options *opts,
488  				  unsigned int *remaining)
489  {
490  	struct bpf_sock_ops_kern sock_ops;
491  	int err;
492  
493  	if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
494  					   BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) ||
495  	    !*remaining)
496  		return;
497  
498  	/* *remaining has already been aligned to 4 bytes, so *remaining >= 4 */
499  
500  	/* init sock_ops */
501  	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
502  
503  	sock_ops.op = BPF_SOCK_OPS_HDR_OPT_LEN_CB;
504  
505  	if (req) {
506  		/* The listen "sk" cannot be passed here because
507  		 * it is not locked.  It would not make too much
508  		 * sense to do bpf_setsockopt(listen_sk) based
509  		 * on individual connection request also.
510  		 *
511  		 * Thus, "req" is passed here and the cgroup-bpf-progs
512  		 * of the listen "sk" will be run.
513  		 *
514  		 * "req" is also used here for fastopen even the "sk" here is
515  		 * a fullsock "child" sk.  It is to keep the behavior
516  		 * consistent between fastopen and non-fastopen on
517  		 * the bpf programming side.
518  		 */
519  		sock_ops.sk = (struct sock *)req;
520  		sock_ops.syn_skb = syn_skb;
521  	} else {
522  		sock_owned_by_me(sk);
523  
524  		sock_ops.is_fullsock = 1;
525  		sock_ops.sk = sk;
526  	}
527  
528  	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
529  	sock_ops.remaining_opt_len = *remaining;
530  	/* tcp_current_mss() does not pass a skb */
531  	if (skb)
532  		bpf_skops_init_skb(&sock_ops, skb, 0);
533  
534  	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
535  
536  	if (err || sock_ops.remaining_opt_len == *remaining)
537  		return;
538  
539  	opts->bpf_opt_len = *remaining - sock_ops.remaining_opt_len;
540  	/* round up to 4 bytes */
541  	opts->bpf_opt_len = (opts->bpf_opt_len + 3) & ~3;
542  
543  	*remaining -= opts->bpf_opt_len;
544  }
545  
bpf_skops_write_hdr_opt(struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct sk_buff * syn_skb,enum tcp_synack_type synack_type,struct tcp_out_options * opts)546  static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
547  				    struct request_sock *req,
548  				    struct sk_buff *syn_skb,
549  				    enum tcp_synack_type synack_type,
550  				    struct tcp_out_options *opts)
551  {
552  	u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len;
553  	struct bpf_sock_ops_kern sock_ops;
554  	int err;
555  
556  	if (likely(!max_opt_len))
557  		return;
558  
559  	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
560  
561  	sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
562  
563  	if (req) {
564  		sock_ops.sk = (struct sock *)req;
565  		sock_ops.syn_skb = syn_skb;
566  	} else {
567  		sock_owned_by_me(sk);
568  
569  		sock_ops.is_fullsock = 1;
570  		sock_ops.sk = sk;
571  	}
572  
573  	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
574  	sock_ops.remaining_opt_len = max_opt_len;
575  	first_opt_off = tcp_hdrlen(skb) - max_opt_len;
576  	bpf_skops_init_skb(&sock_ops, skb, first_opt_off);
577  
578  	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
579  
580  	if (err)
581  		nr_written = 0;
582  	else
583  		nr_written = max_opt_len - sock_ops.remaining_opt_len;
584  
585  	if (nr_written < max_opt_len)
586  		memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP,
587  		       max_opt_len - nr_written);
588  }
589  #else
bpf_skops_hdr_opt_len(struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct sk_buff * syn_skb,enum tcp_synack_type synack_type,struct tcp_out_options * opts,unsigned int * remaining)590  static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
591  				  struct request_sock *req,
592  				  struct sk_buff *syn_skb,
593  				  enum tcp_synack_type synack_type,
594  				  struct tcp_out_options *opts,
595  				  unsigned int *remaining)
596  {
597  }
598  
bpf_skops_write_hdr_opt(struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct sk_buff * syn_skb,enum tcp_synack_type synack_type,struct tcp_out_options * opts)599  static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
600  				    struct request_sock *req,
601  				    struct sk_buff *syn_skb,
602  				    enum tcp_synack_type synack_type,
603  				    struct tcp_out_options *opts)
604  {
605  }
606  #endif
607  
process_tcp_ao_options(struct tcp_sock * tp,const struct tcp_request_sock * tcprsk,struct tcp_out_options * opts,struct tcp_key * key,__be32 * ptr)608  static __be32 *process_tcp_ao_options(struct tcp_sock *tp,
609  				      const struct tcp_request_sock *tcprsk,
610  				      struct tcp_out_options *opts,
611  				      struct tcp_key *key, __be32 *ptr)
612  {
613  #ifdef CONFIG_TCP_AO
614  	u8 maclen = tcp_ao_maclen(key->ao_key);
615  
616  	if (tcprsk) {
617  		u8 aolen = maclen + sizeof(struct tcp_ao_hdr);
618  
619  		*ptr++ = htonl((TCPOPT_AO << 24) | (aolen << 16) |
620  			       (tcprsk->ao_keyid << 8) |
621  			       (tcprsk->ao_rcv_next));
622  	} else {
623  		struct tcp_ao_key *rnext_key;
624  		struct tcp_ao_info *ao_info;
625  
626  		ao_info = rcu_dereference_check(tp->ao_info,
627  			lockdep_sock_is_held(&tp->inet_conn.icsk_inet.sk));
628  		rnext_key = READ_ONCE(ao_info->rnext_key);
629  		if (WARN_ON_ONCE(!rnext_key))
630  			return ptr;
631  		*ptr++ = htonl((TCPOPT_AO << 24) |
632  			       (tcp_ao_len(key->ao_key) << 16) |
633  			       (key->ao_key->sndid << 8) |
634  			       (rnext_key->rcvid));
635  	}
636  	opts->hash_location = (__u8 *)ptr;
637  	ptr += maclen / sizeof(*ptr);
638  	if (unlikely(maclen % sizeof(*ptr))) {
639  		memset(ptr, TCPOPT_NOP, sizeof(*ptr));
640  		ptr++;
641  	}
642  #endif
643  	return ptr;
644  }
645  
646  /* Write previously computed TCP options to the packet.
647   *
648   * Beware: Something in the Internet is very sensitive to the ordering of
649   * TCP options, we learned this through the hard way, so be careful here.
650   * Luckily we can at least blame others for their non-compliance but from
651   * inter-operability perspective it seems that we're somewhat stuck with
652   * the ordering which we have been using if we want to keep working with
653   * those broken things (not that it currently hurts anybody as there isn't
654   * particular reason why the ordering would need to be changed).
655   *
656   * At least SACK_PERM as the first option is known to lead to a disaster
657   * (but it may well be that other scenarios fail similarly).
658   */
tcp_options_write(struct tcphdr * th,struct tcp_sock * tp,const struct tcp_request_sock * tcprsk,struct tcp_out_options * opts,struct tcp_key * key)659  static void tcp_options_write(struct tcphdr *th, struct tcp_sock *tp,
660  			      const struct tcp_request_sock *tcprsk,
661  			      struct tcp_out_options *opts,
662  			      struct tcp_key *key)
663  {
664  	__be32 *ptr = (__be32 *)(th + 1);
665  	u16 options = opts->options;	/* mungable copy */
666  
667  	if (tcp_key_is_md5(key)) {
668  		*ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
669  			       (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
670  		/* overload cookie hash location */
671  		opts->hash_location = (__u8 *)ptr;
672  		ptr += 4;
673  	} else if (tcp_key_is_ao(key)) {
674  		ptr = process_tcp_ao_options(tp, tcprsk, opts, key, ptr);
675  	}
676  	if (unlikely(opts->mss)) {
677  		*ptr++ = htonl((TCPOPT_MSS << 24) |
678  			       (TCPOLEN_MSS << 16) |
679  			       opts->mss);
680  	}
681  
682  	if (likely(OPTION_TS & options)) {
683  		if (unlikely(OPTION_SACK_ADVERTISE & options)) {
684  			*ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
685  				       (TCPOLEN_SACK_PERM << 16) |
686  				       (TCPOPT_TIMESTAMP << 8) |
687  				       TCPOLEN_TIMESTAMP);
688  			options &= ~OPTION_SACK_ADVERTISE;
689  		} else {
690  			*ptr++ = htonl((TCPOPT_NOP << 24) |
691  				       (TCPOPT_NOP << 16) |
692  				       (TCPOPT_TIMESTAMP << 8) |
693  				       TCPOLEN_TIMESTAMP);
694  		}
695  		*ptr++ = htonl(opts->tsval);
696  		*ptr++ = htonl(opts->tsecr);
697  	}
698  
699  	if (unlikely(OPTION_SACK_ADVERTISE & options)) {
700  		*ptr++ = htonl((TCPOPT_NOP << 24) |
701  			       (TCPOPT_NOP << 16) |
702  			       (TCPOPT_SACK_PERM << 8) |
703  			       TCPOLEN_SACK_PERM);
704  	}
705  
706  	if (unlikely(OPTION_WSCALE & options)) {
707  		*ptr++ = htonl((TCPOPT_NOP << 24) |
708  			       (TCPOPT_WINDOW << 16) |
709  			       (TCPOLEN_WINDOW << 8) |
710  			       opts->ws);
711  	}
712  
713  	if (unlikely(opts->num_sack_blocks)) {
714  		struct tcp_sack_block *sp = tp->rx_opt.dsack ?
715  			tp->duplicate_sack : tp->selective_acks;
716  		int this_sack;
717  
718  		*ptr++ = htonl((TCPOPT_NOP  << 24) |
719  			       (TCPOPT_NOP  << 16) |
720  			       (TCPOPT_SACK <<  8) |
721  			       (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
722  						     TCPOLEN_SACK_PERBLOCK)));
723  
724  		for (this_sack = 0; this_sack < opts->num_sack_blocks;
725  		     ++this_sack) {
726  			*ptr++ = htonl(sp[this_sack].start_seq);
727  			*ptr++ = htonl(sp[this_sack].end_seq);
728  		}
729  
730  		tp->rx_opt.dsack = 0;
731  	}
732  
733  	if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
734  		struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
735  		u8 *p = (u8 *)ptr;
736  		u32 len; /* Fast Open option length */
737  
738  		if (foc->exp) {
739  			len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
740  			*ptr = htonl((TCPOPT_EXP << 24) | (len << 16) |
741  				     TCPOPT_FASTOPEN_MAGIC);
742  			p += TCPOLEN_EXP_FASTOPEN_BASE;
743  		} else {
744  			len = TCPOLEN_FASTOPEN_BASE + foc->len;
745  			*p++ = TCPOPT_FASTOPEN;
746  			*p++ = len;
747  		}
748  
749  		memcpy(p, foc->val, foc->len);
750  		if ((len & 3) == 2) {
751  			p[foc->len] = TCPOPT_NOP;
752  			p[foc->len + 1] = TCPOPT_NOP;
753  		}
754  		ptr += (len + 3) >> 2;
755  	}
756  
757  	smc_options_write(ptr, &options);
758  
759  	mptcp_options_write(th, ptr, tp, opts);
760  }
761  
smc_set_option(const struct tcp_sock * tp,struct tcp_out_options * opts,unsigned int * remaining)762  static void smc_set_option(const struct tcp_sock *tp,
763  			   struct tcp_out_options *opts,
764  			   unsigned int *remaining)
765  {
766  #if IS_ENABLED(CONFIG_SMC)
767  	if (static_branch_unlikely(&tcp_have_smc)) {
768  		if (tp->syn_smc) {
769  			if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
770  				opts->options |= OPTION_SMC;
771  				*remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
772  			}
773  		}
774  	}
775  #endif
776  }
777  
smc_set_option_cond(const struct tcp_sock * tp,const struct inet_request_sock * ireq,struct tcp_out_options * opts,unsigned int * remaining)778  static void smc_set_option_cond(const struct tcp_sock *tp,
779  				const struct inet_request_sock *ireq,
780  				struct tcp_out_options *opts,
781  				unsigned int *remaining)
782  {
783  #if IS_ENABLED(CONFIG_SMC)
784  	if (static_branch_unlikely(&tcp_have_smc)) {
785  		if (tp->syn_smc && ireq->smc_ok) {
786  			if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
787  				opts->options |= OPTION_SMC;
788  				*remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
789  			}
790  		}
791  	}
792  #endif
793  }
794  
mptcp_set_option_cond(const struct request_sock * req,struct tcp_out_options * opts,unsigned int * remaining)795  static void mptcp_set_option_cond(const struct request_sock *req,
796  				  struct tcp_out_options *opts,
797  				  unsigned int *remaining)
798  {
799  	if (rsk_is_mptcp(req)) {
800  		unsigned int size;
801  
802  		if (mptcp_synack_options(req, &size, &opts->mptcp)) {
803  			if (*remaining >= size) {
804  				opts->options |= OPTION_MPTCP;
805  				*remaining -= size;
806  			}
807  		}
808  	}
809  }
810  
811  /* Compute TCP options for SYN packets. This is not the final
812   * network wire format yet.
813   */
tcp_syn_options(struct sock * sk,struct sk_buff * skb,struct tcp_out_options * opts,struct tcp_key * key)814  static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
815  				struct tcp_out_options *opts,
816  				struct tcp_key *key)
817  {
818  	struct tcp_sock *tp = tcp_sk(sk);
819  	unsigned int remaining = MAX_TCP_OPTION_SPACE;
820  	struct tcp_fastopen_request *fastopen = tp->fastopen_req;
821  	bool timestamps;
822  
823  	/* Better than switch (key.type) as it has static branches */
824  	if (tcp_key_is_md5(key)) {
825  		timestamps = false;
826  		opts->options |= OPTION_MD5;
827  		remaining -= TCPOLEN_MD5SIG_ALIGNED;
828  	} else {
829  		timestamps = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps);
830  		if (tcp_key_is_ao(key)) {
831  			opts->options |= OPTION_AO;
832  			remaining -= tcp_ao_len_aligned(key->ao_key);
833  		}
834  	}
835  
836  	/* We always get an MSS option.  The option bytes which will be seen in
837  	 * normal data packets should timestamps be used, must be in the MSS
838  	 * advertised.  But we subtract them from tp->mss_cache so that
839  	 * calculations in tcp_sendmsg are simpler etc.  So account for this
840  	 * fact here if necessary.  If we don't do this correctly, as a
841  	 * receiver we won't recognize data packets as being full sized when we
842  	 * should, and thus we won't abide by the delayed ACK rules correctly.
843  	 * SACKs don't matter, we never delay an ACK when we have any of those
844  	 * going out.  */
845  	opts->mss = tcp_advertise_mss(sk);
846  	remaining -= TCPOLEN_MSS_ALIGNED;
847  
848  	if (likely(timestamps)) {
849  		opts->options |= OPTION_TS;
850  		opts->tsval = tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb) + tp->tsoffset;
851  		opts->tsecr = tp->rx_opt.ts_recent;
852  		remaining -= TCPOLEN_TSTAMP_ALIGNED;
853  	}
854  	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling))) {
855  		opts->ws = tp->rx_opt.rcv_wscale;
856  		opts->options |= OPTION_WSCALE;
857  		remaining -= TCPOLEN_WSCALE_ALIGNED;
858  	}
859  	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_sack))) {
860  		opts->options |= OPTION_SACK_ADVERTISE;
861  		if (unlikely(!(OPTION_TS & opts->options)))
862  			remaining -= TCPOLEN_SACKPERM_ALIGNED;
863  	}
864  
865  	if (fastopen && fastopen->cookie.len >= 0) {
866  		u32 need = fastopen->cookie.len;
867  
868  		need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE :
869  					       TCPOLEN_FASTOPEN_BASE;
870  		need = (need + 3) & ~3U;  /* Align to 32 bits */
871  		if (remaining >= need) {
872  			opts->options |= OPTION_FAST_OPEN_COOKIE;
873  			opts->fastopen_cookie = &fastopen->cookie;
874  			remaining -= need;
875  			tp->syn_fastopen = 1;
876  			tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0;
877  		}
878  	}
879  
880  	smc_set_option(tp, opts, &remaining);
881  
882  	if (sk_is_mptcp(sk)) {
883  		unsigned int size;
884  
885  		if (mptcp_syn_options(sk, skb, &size, &opts->mptcp)) {
886  			opts->options |= OPTION_MPTCP;
887  			remaining -= size;
888  		}
889  	}
890  
891  	bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
892  
893  	return MAX_TCP_OPTION_SPACE - remaining;
894  }
895  
896  /* Set up TCP options for SYN-ACKs. */
tcp_synack_options(const struct sock * sk,struct request_sock * req,unsigned int mss,struct sk_buff * skb,struct tcp_out_options * opts,const struct tcp_key * key,struct tcp_fastopen_cookie * foc,enum tcp_synack_type synack_type,struct sk_buff * syn_skb)897  static unsigned int tcp_synack_options(const struct sock *sk,
898  				       struct request_sock *req,
899  				       unsigned int mss, struct sk_buff *skb,
900  				       struct tcp_out_options *opts,
901  				       const struct tcp_key *key,
902  				       struct tcp_fastopen_cookie *foc,
903  				       enum tcp_synack_type synack_type,
904  				       struct sk_buff *syn_skb)
905  {
906  	struct inet_request_sock *ireq = inet_rsk(req);
907  	unsigned int remaining = MAX_TCP_OPTION_SPACE;
908  
909  	if (tcp_key_is_md5(key)) {
910  		opts->options |= OPTION_MD5;
911  		remaining -= TCPOLEN_MD5SIG_ALIGNED;
912  
913  		/* We can't fit any SACK blocks in a packet with MD5 + TS
914  		 * options. There was discussion about disabling SACK
915  		 * rather than TS in order to fit in better with old,
916  		 * buggy kernels, but that was deemed to be unnecessary.
917  		 */
918  		if (synack_type != TCP_SYNACK_COOKIE)
919  			ireq->tstamp_ok &= !ireq->sack_ok;
920  	} else if (tcp_key_is_ao(key)) {
921  		opts->options |= OPTION_AO;
922  		remaining -= tcp_ao_len_aligned(key->ao_key);
923  		ireq->tstamp_ok &= !ireq->sack_ok;
924  	}
925  
926  	/* We always send an MSS option. */
927  	opts->mss = mss;
928  	remaining -= TCPOLEN_MSS_ALIGNED;
929  
930  	if (likely(ireq->wscale_ok)) {
931  		opts->ws = ireq->rcv_wscale;
932  		opts->options |= OPTION_WSCALE;
933  		remaining -= TCPOLEN_WSCALE_ALIGNED;
934  	}
935  	if (likely(ireq->tstamp_ok)) {
936  		opts->options |= OPTION_TS;
937  		opts->tsval = tcp_skb_timestamp_ts(tcp_rsk(req)->req_usec_ts, skb) +
938  			      tcp_rsk(req)->ts_off;
939  		opts->tsecr = READ_ONCE(req->ts_recent);
940  		remaining -= TCPOLEN_TSTAMP_ALIGNED;
941  	}
942  	if (likely(ireq->sack_ok)) {
943  		opts->options |= OPTION_SACK_ADVERTISE;
944  		if (unlikely(!ireq->tstamp_ok))
945  			remaining -= TCPOLEN_SACKPERM_ALIGNED;
946  	}
947  	if (foc != NULL && foc->len >= 0) {
948  		u32 need = foc->len;
949  
950  		need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE :
951  				   TCPOLEN_FASTOPEN_BASE;
952  		need = (need + 3) & ~3U;  /* Align to 32 bits */
953  		if (remaining >= need) {
954  			opts->options |= OPTION_FAST_OPEN_COOKIE;
955  			opts->fastopen_cookie = foc;
956  			remaining -= need;
957  		}
958  	}
959  
960  	mptcp_set_option_cond(req, opts, &remaining);
961  
962  	smc_set_option_cond(tcp_sk(sk), ireq, opts, &remaining);
963  
964  	bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
965  			      synack_type, opts, &remaining);
966  
967  	return MAX_TCP_OPTION_SPACE - remaining;
968  }
969  
970  /* Compute TCP options for ESTABLISHED sockets. This is not the
971   * final wire format yet.
972   */
tcp_established_options(struct sock * sk,struct sk_buff * skb,struct tcp_out_options * opts,struct tcp_key * key)973  static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
974  					struct tcp_out_options *opts,
975  					struct tcp_key *key)
976  {
977  	struct tcp_sock *tp = tcp_sk(sk);
978  	unsigned int size = 0;
979  	unsigned int eff_sacks;
980  
981  	opts->options = 0;
982  
983  	/* Better than switch (key.type) as it has static branches */
984  	if (tcp_key_is_md5(key)) {
985  		opts->options |= OPTION_MD5;
986  		size += TCPOLEN_MD5SIG_ALIGNED;
987  	} else if (tcp_key_is_ao(key)) {
988  		opts->options |= OPTION_AO;
989  		size += tcp_ao_len_aligned(key->ao_key);
990  	}
991  
992  	if (likely(tp->rx_opt.tstamp_ok)) {
993  		opts->options |= OPTION_TS;
994  		opts->tsval = skb ? tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb) +
995  				tp->tsoffset : 0;
996  		opts->tsecr = tp->rx_opt.ts_recent;
997  		size += TCPOLEN_TSTAMP_ALIGNED;
998  	}
999  
1000  	/* MPTCP options have precedence over SACK for the limited TCP
1001  	 * option space because a MPTCP connection would be forced to
1002  	 * fall back to regular TCP if a required multipath option is
1003  	 * missing. SACK still gets a chance to use whatever space is
1004  	 * left.
1005  	 */
1006  	if (sk_is_mptcp(sk)) {
1007  		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
1008  		unsigned int opt_size = 0;
1009  
1010  		if (mptcp_established_options(sk, skb, &opt_size, remaining,
1011  					      &opts->mptcp)) {
1012  			opts->options |= OPTION_MPTCP;
1013  			size += opt_size;
1014  		}
1015  	}
1016  
1017  	eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
1018  	if (unlikely(eff_sacks)) {
1019  		const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
1020  		if (unlikely(remaining < TCPOLEN_SACK_BASE_ALIGNED +
1021  					 TCPOLEN_SACK_PERBLOCK))
1022  			return size;
1023  
1024  		opts->num_sack_blocks =
1025  			min_t(unsigned int, eff_sacks,
1026  			      (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
1027  			      TCPOLEN_SACK_PERBLOCK);
1028  
1029  		size += TCPOLEN_SACK_BASE_ALIGNED +
1030  			opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
1031  	}
1032  
1033  	if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp,
1034  					    BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) {
1035  		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
1036  
1037  		bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
1038  
1039  		size = MAX_TCP_OPTION_SPACE - remaining;
1040  	}
1041  
1042  	return size;
1043  }
1044  
1045  
1046  /* TCP SMALL QUEUES (TSQ)
1047   *
1048   * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
1049   * to reduce RTT and bufferbloat.
1050   * We do this using a special skb destructor (tcp_wfree).
1051   *
1052   * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
1053   * needs to be reallocated in a driver.
1054   * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc
1055   *
1056   * Since transmit from skb destructor is forbidden, we use a tasklet
1057   * to process all sockets that eventually need to send more skbs.
1058   * We use one tasklet per cpu, with its own queue of sockets.
1059   */
1060  struct tsq_tasklet {
1061  	struct tasklet_struct	tasklet;
1062  	struct list_head	head; /* queue of tcp sockets */
1063  };
1064  static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
1065  
tcp_tsq_write(struct sock * sk)1066  static void tcp_tsq_write(struct sock *sk)
1067  {
1068  	if ((1 << sk->sk_state) &
1069  	    (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
1070  	     TCPF_CLOSE_WAIT  | TCPF_LAST_ACK)) {
1071  		struct tcp_sock *tp = tcp_sk(sk);
1072  
1073  		if (tp->lost_out > tp->retrans_out &&
1074  		    tcp_snd_cwnd(tp) > tcp_packets_in_flight(tp)) {
1075  			tcp_mstamp_refresh(tp);
1076  			tcp_xmit_retransmit_queue(sk);
1077  		}
1078  
1079  		tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle,
1080  			       0, GFP_ATOMIC);
1081  	}
1082  }
1083  
tcp_tsq_handler(struct sock * sk)1084  static void tcp_tsq_handler(struct sock *sk)
1085  {
1086  	bh_lock_sock(sk);
1087  	if (!sock_owned_by_user(sk))
1088  		tcp_tsq_write(sk);
1089  	else if (!test_and_set_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
1090  		sock_hold(sk);
1091  	bh_unlock_sock(sk);
1092  }
1093  /*
1094   * One tasklet per cpu tries to send more skbs.
1095   * We run in tasklet context but need to disable irqs when
1096   * transferring tsq->head because tcp_wfree() might
1097   * interrupt us (non NAPI drivers)
1098   */
tcp_tasklet_func(struct tasklet_struct * t)1099  static void tcp_tasklet_func(struct tasklet_struct *t)
1100  {
1101  	struct tsq_tasklet *tsq = from_tasklet(tsq,  t, tasklet);
1102  	LIST_HEAD(list);
1103  	unsigned long flags;
1104  	struct list_head *q, *n;
1105  	struct tcp_sock *tp;
1106  	struct sock *sk;
1107  
1108  	local_irq_save(flags);
1109  	list_splice_init(&tsq->head, &list);
1110  	local_irq_restore(flags);
1111  
1112  	list_for_each_safe(q, n, &list) {
1113  		tp = list_entry(q, struct tcp_sock, tsq_node);
1114  		list_del(&tp->tsq_node);
1115  
1116  		sk = (struct sock *)tp;
1117  		smp_mb__before_atomic();
1118  		clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
1119  
1120  		tcp_tsq_handler(sk);
1121  		sk_free(sk);
1122  	}
1123  }
1124  
1125  #define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED |		\
1126  			  TCPF_WRITE_TIMER_DEFERRED |	\
1127  			  TCPF_DELACK_TIMER_DEFERRED |	\
1128  			  TCPF_MTU_REDUCED_DEFERRED |	\
1129  			  TCPF_ACK_DEFERRED)
1130  /**
1131   * tcp_release_cb - tcp release_sock() callback
1132   * @sk: socket
1133   *
1134   * called from release_sock() to perform protocol dependent
1135   * actions before socket release.
1136   */
tcp_release_cb(struct sock * sk)1137  void tcp_release_cb(struct sock *sk)
1138  {
1139  	unsigned long flags = smp_load_acquire(&sk->sk_tsq_flags);
1140  	unsigned long nflags;
1141  
1142  	/* perform an atomic operation only if at least one flag is set */
1143  	do {
1144  		if (!(flags & TCP_DEFERRED_ALL))
1145  			return;
1146  		nflags = flags & ~TCP_DEFERRED_ALL;
1147  	} while (!try_cmpxchg(&sk->sk_tsq_flags, &flags, nflags));
1148  
1149  	if (flags & TCPF_TSQ_DEFERRED) {
1150  		tcp_tsq_write(sk);
1151  		__sock_put(sk);
1152  	}
1153  
1154  	if (flags & TCPF_WRITE_TIMER_DEFERRED) {
1155  		tcp_write_timer_handler(sk);
1156  		__sock_put(sk);
1157  	}
1158  	if (flags & TCPF_DELACK_TIMER_DEFERRED) {
1159  		tcp_delack_timer_handler(sk);
1160  		__sock_put(sk);
1161  	}
1162  	if (flags & TCPF_MTU_REDUCED_DEFERRED) {
1163  		inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
1164  		__sock_put(sk);
1165  	}
1166  	if ((flags & TCPF_ACK_DEFERRED) && inet_csk_ack_scheduled(sk))
1167  		tcp_send_ack(sk);
1168  }
1169  EXPORT_SYMBOL(tcp_release_cb);
1170  
tcp_tasklet_init(void)1171  void __init tcp_tasklet_init(void)
1172  {
1173  	int i;
1174  
1175  	for_each_possible_cpu(i) {
1176  		struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
1177  
1178  		INIT_LIST_HEAD(&tsq->head);
1179  		tasklet_setup(&tsq->tasklet, tcp_tasklet_func);
1180  	}
1181  }
1182  
1183  /*
1184   * Write buffer destructor automatically called from kfree_skb.
1185   * We can't xmit new skbs from this context, as we might already
1186   * hold qdisc lock.
1187   */
tcp_wfree(struct sk_buff * skb)1188  void tcp_wfree(struct sk_buff *skb)
1189  {
1190  	struct sock *sk = skb->sk;
1191  	struct tcp_sock *tp = tcp_sk(sk);
1192  	unsigned long flags, nval, oval;
1193  	struct tsq_tasklet *tsq;
1194  	bool empty;
1195  
1196  	/* Keep one reference on sk_wmem_alloc.
1197  	 * Will be released by sk_free() from here or tcp_tasklet_func()
1198  	 */
1199  	WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc));
1200  
1201  	/* If this softirq is serviced by ksoftirqd, we are likely under stress.
1202  	 * Wait until our queues (qdisc + devices) are drained.
1203  	 * This gives :
1204  	 * - less callbacks to tcp_write_xmit(), reducing stress (batches)
1205  	 * - chance for incoming ACK (processed by another cpu maybe)
1206  	 *   to migrate this flow (skb->ooo_okay will be eventually set)
1207  	 */
1208  	if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
1209  		goto out;
1210  
1211  	oval = smp_load_acquire(&sk->sk_tsq_flags);
1212  	do {
1213  		if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
1214  			goto out;
1215  
1216  		nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED;
1217  	} while (!try_cmpxchg(&sk->sk_tsq_flags, &oval, nval));
1218  
1219  	/* queue this socket to tasklet queue */
1220  	local_irq_save(flags);
1221  	tsq = this_cpu_ptr(&tsq_tasklet);
1222  	empty = list_empty(&tsq->head);
1223  	list_add(&tp->tsq_node, &tsq->head);
1224  	if (empty)
1225  		tasklet_schedule(&tsq->tasklet);
1226  	local_irq_restore(flags);
1227  	return;
1228  out:
1229  	sk_free(sk);
1230  }
1231  
1232  /* Note: Called under soft irq.
1233   * We can call TCP stack right away, unless socket is owned by user.
1234   */
tcp_pace_kick(struct hrtimer * timer)1235  enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer)
1236  {
1237  	struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer);
1238  	struct sock *sk = (struct sock *)tp;
1239  
1240  	tcp_tsq_handler(sk);
1241  	sock_put(sk);
1242  
1243  	return HRTIMER_NORESTART;
1244  }
1245  
tcp_update_skb_after_send(struct sock * sk,struct sk_buff * skb,u64 prior_wstamp)1246  static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb,
1247  				      u64 prior_wstamp)
1248  {
1249  	struct tcp_sock *tp = tcp_sk(sk);
1250  
1251  	if (sk->sk_pacing_status != SK_PACING_NONE) {
1252  		unsigned long rate = READ_ONCE(sk->sk_pacing_rate);
1253  
1254  		/* Original sch_fq does not pace first 10 MSS
1255  		 * Note that tp->data_segs_out overflows after 2^32 packets,
1256  		 * this is a minor annoyance.
1257  		 */
1258  		if (rate != ~0UL && rate && tp->data_segs_out >= 10) {
1259  			u64 len_ns = div64_ul((u64)skb->len * NSEC_PER_SEC, rate);
1260  			u64 credit = tp->tcp_wstamp_ns - prior_wstamp;
1261  
1262  			/* take into account OS jitter */
1263  			len_ns -= min_t(u64, len_ns / 2, credit);
1264  			tp->tcp_wstamp_ns += len_ns;
1265  		}
1266  	}
1267  	list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
1268  }
1269  
1270  INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
1271  INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
1272  INDIRECT_CALLABLE_DECLARE(void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb));
1273  
1274  /* This routine actually transmits TCP packets queued in by
1275   * tcp_do_sendmsg().  This is used by both the initial
1276   * transmission and possible later retransmissions.
1277   * All SKB's seen here are completely headerless.  It is our
1278   * job to build the TCP header, and pass the packet down to
1279   * IP so it can do the same plus pass the packet off to the
1280   * device.
1281   *
1282   * We are working here with either a clone of the original
1283   * SKB, or a fresh unique copy made by the retransmit engine.
1284   */
__tcp_transmit_skb(struct sock * sk,struct sk_buff * skb,int clone_it,gfp_t gfp_mask,u32 rcv_nxt)1285  static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
1286  			      int clone_it, gfp_t gfp_mask, u32 rcv_nxt)
1287  {
1288  	const struct inet_connection_sock *icsk = inet_csk(sk);
1289  	struct inet_sock *inet;
1290  	struct tcp_sock *tp;
1291  	struct tcp_skb_cb *tcb;
1292  	struct tcp_out_options opts;
1293  	unsigned int tcp_options_size, tcp_header_size;
1294  	struct sk_buff *oskb = NULL;
1295  	struct tcp_key key;
1296  	struct tcphdr *th;
1297  	u64 prior_wstamp;
1298  	int err;
1299  
1300  	BUG_ON(!skb || !tcp_skb_pcount(skb));
1301  	tp = tcp_sk(sk);
1302  	prior_wstamp = tp->tcp_wstamp_ns;
1303  	tp->tcp_wstamp_ns = max(tp->tcp_wstamp_ns, tp->tcp_clock_cache);
1304  	skb_set_delivery_time(skb, tp->tcp_wstamp_ns, SKB_CLOCK_MONOTONIC);
1305  	if (clone_it) {
1306  		oskb = skb;
1307  
1308  		tcp_skb_tsorted_save(oskb) {
1309  			if (unlikely(skb_cloned(oskb)))
1310  				skb = pskb_copy(oskb, gfp_mask);
1311  			else
1312  				skb = skb_clone(oskb, gfp_mask);
1313  		} tcp_skb_tsorted_restore(oskb);
1314  
1315  		if (unlikely(!skb))
1316  			return -ENOBUFS;
1317  		/* retransmit skbs might have a non zero value in skb->dev
1318  		 * because skb->dev is aliased with skb->rbnode.rb_left
1319  		 */
1320  		skb->dev = NULL;
1321  	}
1322  
1323  	inet = inet_sk(sk);
1324  	tcb = TCP_SKB_CB(skb);
1325  	memset(&opts, 0, sizeof(opts));
1326  
1327  	tcp_get_current_key(sk, &key);
1328  	if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
1329  		tcp_options_size = tcp_syn_options(sk, skb, &opts, &key);
1330  	} else {
1331  		tcp_options_size = tcp_established_options(sk, skb, &opts, &key);
1332  		/* Force a PSH flag on all (GSO) packets to expedite GRO flush
1333  		 * at receiver : This slightly improve GRO performance.
1334  		 * Note that we do not force the PSH flag for non GSO packets,
1335  		 * because they might be sent under high congestion events,
1336  		 * and in this case it is better to delay the delivery of 1-MSS
1337  		 * packets and thus the corresponding ACK packet that would
1338  		 * release the following packet.
1339  		 */
1340  		if (tcp_skb_pcount(skb) > 1)
1341  			tcb->tcp_flags |= TCPHDR_PSH;
1342  	}
1343  	tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
1344  
1345  	/* We set skb->ooo_okay to one if this packet can select
1346  	 * a different TX queue than prior packets of this flow,
1347  	 * to avoid self inflicted reorders.
1348  	 * The 'other' queue decision is based on current cpu number
1349  	 * if XPS is enabled, or sk->sk_txhash otherwise.
1350  	 * We can switch to another (and better) queue if:
1351  	 * 1) No packet with payload is in qdisc/device queues.
1352  	 *    Delays in TX completion can defeat the test
1353  	 *    even if packets were already sent.
1354  	 * 2) Or rtx queue is empty.
1355  	 *    This mitigates above case if ACK packets for
1356  	 *    all prior packets were already processed.
1357  	 */
1358  	skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1) ||
1359  			tcp_rtx_queue_empty(sk);
1360  
1361  	/* If we had to use memory reserve to allocate this skb,
1362  	 * this might cause drops if packet is looped back :
1363  	 * Other socket might not have SOCK_MEMALLOC.
1364  	 * Packets not looped back do not care about pfmemalloc.
1365  	 */
1366  	skb->pfmemalloc = 0;
1367  
1368  	skb_push(skb, tcp_header_size);
1369  	skb_reset_transport_header(skb);
1370  
1371  	skb_orphan(skb);
1372  	skb->sk = sk;
1373  	skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
1374  	refcount_add(skb->truesize, &sk->sk_wmem_alloc);
1375  
1376  	skb_set_dst_pending_confirm(skb, READ_ONCE(sk->sk_dst_pending_confirm));
1377  
1378  	/* Build TCP header and checksum it. */
1379  	th = (struct tcphdr *)skb->data;
1380  	th->source		= inet->inet_sport;
1381  	th->dest		= inet->inet_dport;
1382  	th->seq			= htonl(tcb->seq);
1383  	th->ack_seq		= htonl(rcv_nxt);
1384  	*(((__be16 *)th) + 6)	= htons(((tcp_header_size >> 2) << 12) |
1385  					tcb->tcp_flags);
1386  
1387  	th->check		= 0;
1388  	th->urg_ptr		= 0;
1389  
1390  	/* The urg_mode check is necessary during a below snd_una win probe */
1391  	if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
1392  		if (before(tp->snd_up, tcb->seq + 0x10000)) {
1393  			th->urg_ptr = htons(tp->snd_up - tcb->seq);
1394  			th->urg = 1;
1395  		} else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
1396  			th->urg_ptr = htons(0xFFFF);
1397  			th->urg = 1;
1398  		}
1399  	}
1400  
1401  	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
1402  	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
1403  		th->window      = htons(tcp_select_window(sk));
1404  		tcp_ecn_send(sk, skb, th, tcp_header_size);
1405  	} else {
1406  		/* RFC1323: The window in SYN & SYN/ACK segments
1407  		 * is never scaled.
1408  		 */
1409  		th->window	= htons(min(tp->rcv_wnd, 65535U));
1410  	}
1411  
1412  	tcp_options_write(th, tp, NULL, &opts, &key);
1413  
1414  	if (tcp_key_is_md5(&key)) {
1415  #ifdef CONFIG_TCP_MD5SIG
1416  		/* Calculate the MD5 hash, as we have all we need now */
1417  		sk_gso_disable(sk);
1418  		tp->af_specific->calc_md5_hash(opts.hash_location,
1419  					       key.md5_key, sk, skb);
1420  #endif
1421  	} else if (tcp_key_is_ao(&key)) {
1422  		int err;
1423  
1424  		err = tcp_ao_transmit_skb(sk, skb, key.ao_key, th,
1425  					  opts.hash_location);
1426  		if (err) {
1427  			kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
1428  			return -ENOMEM;
1429  		}
1430  	}
1431  
1432  	/* BPF prog is the last one writing header option */
1433  	bpf_skops_write_hdr_opt(sk, skb, NULL, NULL, 0, &opts);
1434  
1435  	INDIRECT_CALL_INET(icsk->icsk_af_ops->send_check,
1436  			   tcp_v6_send_check, tcp_v4_send_check,
1437  			   sk, skb);
1438  
1439  	if (likely(tcb->tcp_flags & TCPHDR_ACK))
1440  		tcp_event_ack_sent(sk, rcv_nxt);
1441  
1442  	if (skb->len != tcp_header_size) {
1443  		tcp_event_data_sent(tp, sk);
1444  		tp->data_segs_out += tcp_skb_pcount(skb);
1445  		tp->bytes_sent += skb->len - tcp_header_size;
1446  	}
1447  
1448  	if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
1449  		TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
1450  			      tcp_skb_pcount(skb));
1451  
1452  	tp->segs_out += tcp_skb_pcount(skb);
1453  	skb_set_hash_from_sk(skb, sk);
1454  	/* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */
1455  	skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
1456  	skb_shinfo(skb)->gso_size = tcp_skb_mss(skb);
1457  
1458  	/* Leave earliest departure time in skb->tstamp (skb->skb_mstamp_ns) */
1459  
1460  	/* Cleanup our debris for IP stacks */
1461  	memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
1462  			       sizeof(struct inet6_skb_parm)));
1463  
1464  	tcp_add_tx_delay(skb, tp);
1465  
1466  	err = INDIRECT_CALL_INET(icsk->icsk_af_ops->queue_xmit,
1467  				 inet6_csk_xmit, ip_queue_xmit,
1468  				 sk, skb, &inet->cork.fl);
1469  
1470  	if (unlikely(err > 0)) {
1471  		tcp_enter_cwr(sk);
1472  		err = net_xmit_eval(err);
1473  	}
1474  	if (!err && oskb) {
1475  		tcp_update_skb_after_send(sk, oskb, prior_wstamp);
1476  		tcp_rate_skb_sent(sk, oskb);
1477  	}
1478  	return err;
1479  }
1480  
tcp_transmit_skb(struct sock * sk,struct sk_buff * skb,int clone_it,gfp_t gfp_mask)1481  static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
1482  			    gfp_t gfp_mask)
1483  {
1484  	return __tcp_transmit_skb(sk, skb, clone_it, gfp_mask,
1485  				  tcp_sk(sk)->rcv_nxt);
1486  }
1487  
1488  /* This routine just queues the buffer for sending.
1489   *
1490   * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
1491   * otherwise socket can stall.
1492   */
tcp_queue_skb(struct sock * sk,struct sk_buff * skb)1493  static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
1494  {
1495  	struct tcp_sock *tp = tcp_sk(sk);
1496  
1497  	/* Advance write_seq and place onto the write_queue. */
1498  	WRITE_ONCE(tp->write_seq, TCP_SKB_CB(skb)->end_seq);
1499  	__skb_header_release(skb);
1500  	tcp_add_write_queue_tail(sk, skb);
1501  	sk_wmem_queued_add(sk, skb->truesize);
1502  	sk_mem_charge(sk, skb->truesize);
1503  }
1504  
1505  /* Initialize TSO segments for a packet. */
tcp_set_skb_tso_segs(struct sk_buff * skb,unsigned int mss_now)1506  static int tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now)
1507  {
1508  	int tso_segs;
1509  
1510  	if (skb->len <= mss_now) {
1511  		/* Avoid the costly divide in the normal
1512  		 * non-TSO case.
1513  		 */
1514  		TCP_SKB_CB(skb)->tcp_gso_size = 0;
1515  		tcp_skb_pcount_set(skb, 1);
1516  		return 1;
1517  	}
1518  	TCP_SKB_CB(skb)->tcp_gso_size = mss_now;
1519  	tso_segs = DIV_ROUND_UP(skb->len, mss_now);
1520  	tcp_skb_pcount_set(skb, tso_segs);
1521  	return tso_segs;
1522  }
1523  
1524  /* Pcount in the middle of the write queue got changed, we need to do various
1525   * tweaks to fix counters
1526   */
tcp_adjust_pcount(struct sock * sk,const struct sk_buff * skb,int decr)1527  static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
1528  {
1529  	struct tcp_sock *tp = tcp_sk(sk);
1530  
1531  	tp->packets_out -= decr;
1532  
1533  	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1534  		tp->sacked_out -= decr;
1535  	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1536  		tp->retrans_out -= decr;
1537  	if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1538  		tp->lost_out -= decr;
1539  
1540  	/* Reno case is special. Sigh... */
1541  	if (tcp_is_reno(tp) && decr > 0)
1542  		tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1543  
1544  	if (tp->lost_skb_hint &&
1545  	    before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
1546  	    (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
1547  		tp->lost_cnt_hint -= decr;
1548  
1549  	tcp_verify_left_out(tp);
1550  }
1551  
tcp_has_tx_tstamp(const struct sk_buff * skb)1552  static bool tcp_has_tx_tstamp(const struct sk_buff *skb)
1553  {
1554  	return TCP_SKB_CB(skb)->txstamp_ack ||
1555  		(skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP);
1556  }
1557  
tcp_fragment_tstamp(struct sk_buff * skb,struct sk_buff * skb2)1558  static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2)
1559  {
1560  	struct skb_shared_info *shinfo = skb_shinfo(skb);
1561  
1562  	if (unlikely(tcp_has_tx_tstamp(skb)) &&
1563  	    !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) {
1564  		struct skb_shared_info *shinfo2 = skb_shinfo(skb2);
1565  		u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP;
1566  
1567  		shinfo->tx_flags &= ~tsflags;
1568  		shinfo2->tx_flags |= tsflags;
1569  		swap(shinfo->tskey, shinfo2->tskey);
1570  		TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack;
1571  		TCP_SKB_CB(skb)->txstamp_ack = 0;
1572  	}
1573  }
1574  
tcp_skb_fragment_eor(struct sk_buff * skb,struct sk_buff * skb2)1575  static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2)
1576  {
1577  	TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor;
1578  	TCP_SKB_CB(skb)->eor = 0;
1579  }
1580  
1581  /* Insert buff after skb on the write or rtx queue of sk.  */
tcp_insert_write_queue_after(struct sk_buff * skb,struct sk_buff * buff,struct sock * sk,enum tcp_queue tcp_queue)1582  static void tcp_insert_write_queue_after(struct sk_buff *skb,
1583  					 struct sk_buff *buff,
1584  					 struct sock *sk,
1585  					 enum tcp_queue tcp_queue)
1586  {
1587  	if (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE)
1588  		__skb_queue_after(&sk->sk_write_queue, skb, buff);
1589  	else
1590  		tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
1591  }
1592  
1593  /* Function to create two new TCP segments.  Shrinks the given segment
1594   * to the specified size and appends a new segment with the rest of the
1595   * packet to the list.  This won't be called frequently, I hope.
1596   * Remember, these are still headerless SKBs at this point.
1597   */
tcp_fragment(struct sock * sk,enum tcp_queue tcp_queue,struct sk_buff * skb,u32 len,unsigned int mss_now,gfp_t gfp)1598  int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
1599  		 struct sk_buff *skb, u32 len,
1600  		 unsigned int mss_now, gfp_t gfp)
1601  {
1602  	struct tcp_sock *tp = tcp_sk(sk);
1603  	struct sk_buff *buff;
1604  	int old_factor;
1605  	long limit;
1606  	int nlen;
1607  	u8 flags;
1608  
1609  	if (WARN_ON(len > skb->len))
1610  		return -EINVAL;
1611  
1612  	DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));
1613  
1614  	/* tcp_sendmsg() can overshoot sk_wmem_queued by one full size skb.
1615  	 * We need some allowance to not penalize applications setting small
1616  	 * SO_SNDBUF values.
1617  	 * Also allow first and last skb in retransmit queue to be split.
1618  	 */
1619  	limit = sk->sk_sndbuf + 2 * SKB_TRUESIZE(GSO_LEGACY_MAX_SIZE);
1620  	if (unlikely((sk->sk_wmem_queued >> 1) > limit &&
1621  		     tcp_queue != TCP_FRAG_IN_WRITE_QUEUE &&
1622  		     skb != tcp_rtx_queue_head(sk) &&
1623  		     skb != tcp_rtx_queue_tail(sk))) {
1624  		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPWQUEUETOOBIG);
1625  		return -ENOMEM;
1626  	}
1627  
1628  	if (skb_unclone_keeptruesize(skb, gfp))
1629  		return -ENOMEM;
1630  
1631  	/* Get a new skb... force flag on. */
1632  	buff = tcp_stream_alloc_skb(sk, gfp, true);
1633  	if (!buff)
1634  		return -ENOMEM; /* We'll just try again later. */
1635  	skb_copy_decrypted(buff, skb);
1636  	mptcp_skb_ext_copy(buff, skb);
1637  
1638  	sk_wmem_queued_add(sk, buff->truesize);
1639  	sk_mem_charge(sk, buff->truesize);
1640  	nlen = skb->len - len;
1641  	buff->truesize += nlen;
1642  	skb->truesize -= nlen;
1643  
1644  	/* Correct the sequence numbers. */
1645  	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1646  	TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1647  	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1648  
1649  	/* PSH and FIN should only be set in the second packet. */
1650  	flags = TCP_SKB_CB(skb)->tcp_flags;
1651  	TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1652  	TCP_SKB_CB(buff)->tcp_flags = flags;
1653  	TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1654  	tcp_skb_fragment_eor(skb, buff);
1655  
1656  	skb_split(skb, buff, len);
1657  
1658  	skb_set_delivery_time(buff, skb->tstamp, SKB_CLOCK_MONOTONIC);
1659  	tcp_fragment_tstamp(skb, buff);
1660  
1661  	old_factor = tcp_skb_pcount(skb);
1662  
1663  	/* Fix up tso_factor for both original and new SKB.  */
1664  	tcp_set_skb_tso_segs(skb, mss_now);
1665  	tcp_set_skb_tso_segs(buff, mss_now);
1666  
1667  	/* Update delivered info for the new segment */
1668  	TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx;
1669  
1670  	/* If this packet has been sent out already, we must
1671  	 * adjust the various packet counters.
1672  	 */
1673  	if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
1674  		int diff = old_factor - tcp_skb_pcount(skb) -
1675  			tcp_skb_pcount(buff);
1676  
1677  		if (diff)
1678  			tcp_adjust_pcount(sk, skb, diff);
1679  	}
1680  
1681  	/* Link BUFF into the send queue. */
1682  	__skb_header_release(buff);
1683  	tcp_insert_write_queue_after(skb, buff, sk, tcp_queue);
1684  	if (tcp_queue == TCP_FRAG_IN_RTX_QUEUE)
1685  		list_add(&buff->tcp_tsorted_anchor, &skb->tcp_tsorted_anchor);
1686  
1687  	return 0;
1688  }
1689  
1690  /* This is similar to __pskb_pull_tail(). The difference is that pulled
1691   * data is not copied, but immediately discarded.
1692   */
__pskb_trim_head(struct sk_buff * skb,int len)1693  static int __pskb_trim_head(struct sk_buff *skb, int len)
1694  {
1695  	struct skb_shared_info *shinfo;
1696  	int i, k, eat;
1697  
1698  	DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));
1699  	eat = len;
1700  	k = 0;
1701  	shinfo = skb_shinfo(skb);
1702  	for (i = 0; i < shinfo->nr_frags; i++) {
1703  		int size = skb_frag_size(&shinfo->frags[i]);
1704  
1705  		if (size <= eat) {
1706  			skb_frag_unref(skb, i);
1707  			eat -= size;
1708  		} else {
1709  			shinfo->frags[k] = shinfo->frags[i];
1710  			if (eat) {
1711  				skb_frag_off_add(&shinfo->frags[k], eat);
1712  				skb_frag_size_sub(&shinfo->frags[k], eat);
1713  				eat = 0;
1714  			}
1715  			k++;
1716  		}
1717  	}
1718  	shinfo->nr_frags = k;
1719  
1720  	skb->data_len -= len;
1721  	skb->len = skb->data_len;
1722  	return len;
1723  }
1724  
1725  /* Remove acked data from a packet in the transmit queue. */
tcp_trim_head(struct sock * sk,struct sk_buff * skb,u32 len)1726  int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1727  {
1728  	u32 delta_truesize;
1729  
1730  	if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
1731  		return -ENOMEM;
1732  
1733  	delta_truesize = __pskb_trim_head(skb, len);
1734  
1735  	TCP_SKB_CB(skb)->seq += len;
1736  
1737  	skb->truesize	   -= delta_truesize;
1738  	sk_wmem_queued_add(sk, -delta_truesize);
1739  	if (!skb_zcopy_pure(skb))
1740  		sk_mem_uncharge(sk, delta_truesize);
1741  
1742  	/* Any change of skb->len requires recalculation of tso factor. */
1743  	if (tcp_skb_pcount(skb) > 1)
1744  		tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb));
1745  
1746  	return 0;
1747  }
1748  
1749  /* Calculate MSS not accounting any TCP options.  */
__tcp_mtu_to_mss(struct sock * sk,int pmtu)1750  static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
1751  {
1752  	const struct tcp_sock *tp = tcp_sk(sk);
1753  	const struct inet_connection_sock *icsk = inet_csk(sk);
1754  	int mss_now;
1755  
1756  	/* Calculate base mss without TCP options:
1757  	   It is MMS_S - sizeof(tcphdr) of rfc1122
1758  	 */
1759  	mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1760  
1761  	/* Clamp it (mss_clamp does not include tcp options) */
1762  	if (mss_now > tp->rx_opt.mss_clamp)
1763  		mss_now = tp->rx_opt.mss_clamp;
1764  
1765  	/* Now subtract optional transport overhead */
1766  	mss_now -= icsk->icsk_ext_hdr_len;
1767  
1768  	/* Then reserve room for full set of TCP options and 8 bytes of data */
1769  	mss_now = max(mss_now,
1770  		      READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_snd_mss));
1771  	return mss_now;
1772  }
1773  
1774  /* Calculate MSS. Not accounting for SACKs here.  */
tcp_mtu_to_mss(struct sock * sk,int pmtu)1775  int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1776  {
1777  	/* Subtract TCP options size, not including SACKs */
1778  	return __tcp_mtu_to_mss(sk, pmtu) -
1779  	       (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1780  }
1781  EXPORT_SYMBOL(tcp_mtu_to_mss);
1782  
1783  /* Inverse of above */
tcp_mss_to_mtu(struct sock * sk,int mss)1784  int tcp_mss_to_mtu(struct sock *sk, int mss)
1785  {
1786  	const struct tcp_sock *tp = tcp_sk(sk);
1787  	const struct inet_connection_sock *icsk = inet_csk(sk);
1788  
1789  	return mss +
1790  	      tp->tcp_header_len +
1791  	      icsk->icsk_ext_hdr_len +
1792  	      icsk->icsk_af_ops->net_header_len;
1793  }
1794  EXPORT_SYMBOL(tcp_mss_to_mtu);
1795  
1796  /* MTU probing init per socket */
tcp_mtup_init(struct sock * sk)1797  void tcp_mtup_init(struct sock *sk)
1798  {
1799  	struct tcp_sock *tp = tcp_sk(sk);
1800  	struct inet_connection_sock *icsk = inet_csk(sk);
1801  	struct net *net = sock_net(sk);
1802  
1803  	icsk->icsk_mtup.enabled = READ_ONCE(net->ipv4.sysctl_tcp_mtu_probing) > 1;
1804  	icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
1805  			       icsk->icsk_af_ops->net_header_len;
1806  	icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, READ_ONCE(net->ipv4.sysctl_tcp_base_mss));
1807  	icsk->icsk_mtup.probe_size = 0;
1808  	if (icsk->icsk_mtup.enabled)
1809  		icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
1810  }
1811  EXPORT_SYMBOL(tcp_mtup_init);
1812  
1813  /* This function synchronize snd mss to current pmtu/exthdr set.
1814  
1815     tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
1816     for TCP options, but includes only bare TCP header.
1817  
1818     tp->rx_opt.mss_clamp is mss negotiated at connection setup.
1819     It is minimum of user_mss and mss received with SYN.
1820     It also does not include TCP options.
1821  
1822     inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
1823  
1824     tp->mss_cache is current effective sending mss, including
1825     all tcp options except for SACKs. It is evaluated,
1826     taking into account current pmtu, but never exceeds
1827     tp->rx_opt.mss_clamp.
1828  
1829     NOTE1. rfc1122 clearly states that advertised MSS
1830     DOES NOT include either tcp or ip options.
1831  
1832     NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1833     are READ ONLY outside this function.		--ANK (980731)
1834   */
tcp_sync_mss(struct sock * sk,u32 pmtu)1835  unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1836  {
1837  	struct tcp_sock *tp = tcp_sk(sk);
1838  	struct inet_connection_sock *icsk = inet_csk(sk);
1839  	int mss_now;
1840  
1841  	if (icsk->icsk_mtup.search_high > pmtu)
1842  		icsk->icsk_mtup.search_high = pmtu;
1843  
1844  	mss_now = tcp_mtu_to_mss(sk, pmtu);
1845  	mss_now = tcp_bound_to_half_wnd(tp, mss_now);
1846  
1847  	/* And store cached results */
1848  	icsk->icsk_pmtu_cookie = pmtu;
1849  	if (icsk->icsk_mtup.enabled)
1850  		mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
1851  	tp->mss_cache = mss_now;
1852  
1853  	return mss_now;
1854  }
1855  EXPORT_SYMBOL(tcp_sync_mss);
1856  
1857  /* Compute the current effective MSS, taking SACKs and IP options,
1858   * and even PMTU discovery events into account.
1859   */
tcp_current_mss(struct sock * sk)1860  unsigned int tcp_current_mss(struct sock *sk)
1861  {
1862  	const struct tcp_sock *tp = tcp_sk(sk);
1863  	const struct dst_entry *dst = __sk_dst_get(sk);
1864  	u32 mss_now;
1865  	unsigned int header_len;
1866  	struct tcp_out_options opts;
1867  	struct tcp_key key;
1868  
1869  	mss_now = tp->mss_cache;
1870  
1871  	if (dst) {
1872  		u32 mtu = dst_mtu(dst);
1873  		if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
1874  			mss_now = tcp_sync_mss(sk, mtu);
1875  	}
1876  	tcp_get_current_key(sk, &key);
1877  	header_len = tcp_established_options(sk, NULL, &opts, &key) +
1878  		     sizeof(struct tcphdr);
1879  	/* The mss_cache is sized based on tp->tcp_header_len, which assumes
1880  	 * some common options. If this is an odd packet (because we have SACK
1881  	 * blocks etc) then our calculated header_len will be different, and
1882  	 * we have to adjust mss_now correspondingly */
1883  	if (header_len != tp->tcp_header_len) {
1884  		int delta = (int) header_len - tp->tcp_header_len;
1885  		mss_now -= delta;
1886  	}
1887  
1888  	return mss_now;
1889  }
1890  
1891  /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto.
1892   * As additional protections, we do not touch cwnd in retransmission phases,
1893   * and if application hit its sndbuf limit recently.
1894   */
tcp_cwnd_application_limited(struct sock * sk)1895  static void tcp_cwnd_application_limited(struct sock *sk)
1896  {
1897  	struct tcp_sock *tp = tcp_sk(sk);
1898  
1899  	if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
1900  	    sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1901  		/* Limited by application or receiver window. */
1902  		u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
1903  		u32 win_used = max(tp->snd_cwnd_used, init_win);
1904  		if (win_used < tcp_snd_cwnd(tp)) {
1905  			tp->snd_ssthresh = tcp_current_ssthresh(sk);
1906  			tcp_snd_cwnd_set(tp, (tcp_snd_cwnd(tp) + win_used) >> 1);
1907  		}
1908  		tp->snd_cwnd_used = 0;
1909  	}
1910  	tp->snd_cwnd_stamp = tcp_jiffies32;
1911  }
1912  
tcp_cwnd_validate(struct sock * sk,bool is_cwnd_limited)1913  static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
1914  {
1915  	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1916  	struct tcp_sock *tp = tcp_sk(sk);
1917  
1918  	/* Track the strongest available signal of the degree to which the cwnd
1919  	 * is fully utilized. If cwnd-limited then remember that fact for the
1920  	 * current window. If not cwnd-limited then track the maximum number of
1921  	 * outstanding packets in the current window. (If cwnd-limited then we
1922  	 * chose to not update tp->max_packets_out to avoid an extra else
1923  	 * clause with no functional impact.)
1924  	 */
1925  	if (!before(tp->snd_una, tp->cwnd_usage_seq) ||
1926  	    is_cwnd_limited ||
1927  	    (!tp->is_cwnd_limited &&
1928  	     tp->packets_out > tp->max_packets_out)) {
1929  		tp->is_cwnd_limited = is_cwnd_limited;
1930  		tp->max_packets_out = tp->packets_out;
1931  		tp->cwnd_usage_seq = tp->snd_nxt;
1932  	}
1933  
1934  	if (tcp_is_cwnd_limited(sk)) {
1935  		/* Network is feed fully. */
1936  		tp->snd_cwnd_used = 0;
1937  		tp->snd_cwnd_stamp = tcp_jiffies32;
1938  	} else {
1939  		/* Network starves. */
1940  		if (tp->packets_out > tp->snd_cwnd_used)
1941  			tp->snd_cwnd_used = tp->packets_out;
1942  
1943  		if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) &&
1944  		    (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto &&
1945  		    !ca_ops->cong_control)
1946  			tcp_cwnd_application_limited(sk);
1947  
1948  		/* The following conditions together indicate the starvation
1949  		 * is caused by insufficient sender buffer:
1950  		 * 1) just sent some data (see tcp_write_xmit)
1951  		 * 2) not cwnd limited (this else condition)
1952  		 * 3) no more data to send (tcp_write_queue_empty())
1953  		 * 4) application is hitting buffer limit (SOCK_NOSPACE)
1954  		 */
1955  		if (tcp_write_queue_empty(sk) && sk->sk_socket &&
1956  		    test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) &&
1957  		    (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
1958  			tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED);
1959  	}
1960  }
1961  
1962  /* Minshall's variant of the Nagle send check. */
tcp_minshall_check(const struct tcp_sock * tp)1963  static bool tcp_minshall_check(const struct tcp_sock *tp)
1964  {
1965  	return after(tp->snd_sml, tp->snd_una) &&
1966  		!after(tp->snd_sml, tp->snd_nxt);
1967  }
1968  
1969  /* Update snd_sml if this skb is under mss
1970   * Note that a TSO packet might end with a sub-mss segment
1971   * The test is really :
1972   * if ((skb->len % mss) != 0)
1973   *        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1974   * But we can avoid doing the divide again given we already have
1975   *  skb_pcount = skb->len / mss_now
1976   */
tcp_minshall_update(struct tcp_sock * tp,unsigned int mss_now,const struct sk_buff * skb)1977  static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
1978  				const struct sk_buff *skb)
1979  {
1980  	if (skb->len < tcp_skb_pcount(skb) * mss_now)
1981  		tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1982  }
1983  
1984  /* Return false, if packet can be sent now without violation Nagle's rules:
1985   * 1. It is full sized. (provided by caller in %partial bool)
1986   * 2. Or it contains FIN. (already checked by caller)
1987   * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
1988   * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
1989   *    With Minshall's modification: all sent small packets are ACKed.
1990   */
tcp_nagle_check(bool partial,const struct tcp_sock * tp,int nonagle)1991  static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
1992  			    int nonagle)
1993  {
1994  	return partial &&
1995  		((nonagle & TCP_NAGLE_CORK) ||
1996  		 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
1997  }
1998  
1999  /* Return how many segs we'd like on a TSO packet,
2000   * depending on current pacing rate, and how close the peer is.
2001   *
2002   * Rationale is:
2003   * - For close peers, we rather send bigger packets to reduce
2004   *   cpu costs, because occasional losses will be repaired fast.
2005   * - For long distance/rtt flows, we would like to get ACK clocking
2006   *   with 1 ACK per ms.
2007   *
2008   * Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting
2009   * in bigger TSO bursts. We we cut the RTT-based allowance in half
2010   * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance
2011   * is below 1500 bytes after 6 * ~500 usec = 3ms.
2012   */
tcp_tso_autosize(const struct sock * sk,unsigned int mss_now,int min_tso_segs)2013  static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
2014  			    int min_tso_segs)
2015  {
2016  	unsigned long bytes;
2017  	u32 r;
2018  
2019  	bytes = READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift);
2020  
2021  	r = tcp_min_rtt(tcp_sk(sk)) >> READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_rtt_log);
2022  	if (r < BITS_PER_TYPE(sk->sk_gso_max_size))
2023  		bytes += sk->sk_gso_max_size >> r;
2024  
2025  	bytes = min_t(unsigned long, bytes, sk->sk_gso_max_size);
2026  
2027  	return max_t(u32, bytes / mss_now, min_tso_segs);
2028  }
2029  
2030  /* Return the number of segments we want in the skb we are transmitting.
2031   * See if congestion control module wants to decide; otherwise, autosize.
2032   */
tcp_tso_segs(struct sock * sk,unsigned int mss_now)2033  static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
2034  {
2035  	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
2036  	u32 min_tso, tso_segs;
2037  
2038  	min_tso = ca_ops->min_tso_segs ?
2039  			ca_ops->min_tso_segs(sk) :
2040  			READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs);
2041  
2042  	tso_segs = tcp_tso_autosize(sk, mss_now, min_tso);
2043  	return min_t(u32, tso_segs, sk->sk_gso_max_segs);
2044  }
2045  
2046  /* Returns the portion of skb which can be sent right away */
tcp_mss_split_point(const struct sock * sk,const struct sk_buff * skb,unsigned int mss_now,unsigned int max_segs,int nonagle)2047  static unsigned int tcp_mss_split_point(const struct sock *sk,
2048  					const struct sk_buff *skb,
2049  					unsigned int mss_now,
2050  					unsigned int max_segs,
2051  					int nonagle)
2052  {
2053  	const struct tcp_sock *tp = tcp_sk(sk);
2054  	u32 partial, needed, window, max_len;
2055  
2056  	window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
2057  	max_len = mss_now * max_segs;
2058  
2059  	if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
2060  		return max_len;
2061  
2062  	needed = min(skb->len, window);
2063  
2064  	if (max_len <= needed)
2065  		return max_len;
2066  
2067  	partial = needed % mss_now;
2068  	/* If last segment is not a full MSS, check if Nagle rules allow us
2069  	 * to include this last segment in this skb.
2070  	 * Otherwise, we'll split the skb at last MSS boundary
2071  	 */
2072  	if (tcp_nagle_check(partial != 0, tp, nonagle))
2073  		return needed - partial;
2074  
2075  	return needed;
2076  }
2077  
2078  /* Can at least one segment of SKB be sent right now, according to the
2079   * congestion window rules?  If so, return how many segments are allowed.
2080   */
tcp_cwnd_test(const struct tcp_sock * tp)2081  static u32 tcp_cwnd_test(const struct tcp_sock *tp)
2082  {
2083  	u32 in_flight, cwnd, halfcwnd;
2084  
2085  	in_flight = tcp_packets_in_flight(tp);
2086  	cwnd = tcp_snd_cwnd(tp);
2087  	if (in_flight >= cwnd)
2088  		return 0;
2089  
2090  	/* For better scheduling, ensure we have at least
2091  	 * 2 GSO packets in flight.
2092  	 */
2093  	halfcwnd = max(cwnd >> 1, 1U);
2094  	return min(halfcwnd, cwnd - in_flight);
2095  }
2096  
2097  /* Initialize TSO state of a skb.
2098   * This must be invoked the first time we consider transmitting
2099   * SKB onto the wire.
2100   */
tcp_init_tso_segs(struct sk_buff * skb,unsigned int mss_now)2101  static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now)
2102  {
2103  	int tso_segs = tcp_skb_pcount(skb);
2104  
2105  	if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now))
2106  		return tcp_set_skb_tso_segs(skb, mss_now);
2107  
2108  	return tso_segs;
2109  }
2110  
2111  
2112  /* Return true if the Nagle test allows this packet to be
2113   * sent now.
2114   */
tcp_nagle_test(const struct tcp_sock * tp,const struct sk_buff * skb,unsigned int cur_mss,int nonagle)2115  static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
2116  				  unsigned int cur_mss, int nonagle)
2117  {
2118  	/* Nagle rule does not apply to frames, which sit in the middle of the
2119  	 * write_queue (they have no chances to get new data).
2120  	 *
2121  	 * This is implemented in the callers, where they modify the 'nonagle'
2122  	 * argument based upon the location of SKB in the send queue.
2123  	 */
2124  	if (nonagle & TCP_NAGLE_PUSH)
2125  		return true;
2126  
2127  	/* Don't use the nagle rule for urgent data (or for the final FIN). */
2128  	if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
2129  		return true;
2130  
2131  	if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
2132  		return true;
2133  
2134  	return false;
2135  }
2136  
2137  /* Does at least the first segment of SKB fit into the send window? */
tcp_snd_wnd_test(const struct tcp_sock * tp,const struct sk_buff * skb,unsigned int cur_mss)2138  static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
2139  			     const struct sk_buff *skb,
2140  			     unsigned int cur_mss)
2141  {
2142  	u32 end_seq = TCP_SKB_CB(skb)->end_seq;
2143  
2144  	if (skb->len > cur_mss)
2145  		end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
2146  
2147  	return !after(end_seq, tcp_wnd_end(tp));
2148  }
2149  
2150  /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
2151   * which is put after SKB on the list.  It is very much like
2152   * tcp_fragment() except that it may make several kinds of assumptions
2153   * in order to speed up the splitting operation.  In particular, we
2154   * know that all the data is in scatter-gather pages, and that the
2155   * packet has never been sent out before (and thus is not cloned).
2156   */
tso_fragment(struct sock * sk,struct sk_buff * skb,unsigned int len,unsigned int mss_now,gfp_t gfp)2157  static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
2158  			unsigned int mss_now, gfp_t gfp)
2159  {
2160  	int nlen = skb->len - len;
2161  	struct sk_buff *buff;
2162  	u8 flags;
2163  
2164  	/* All of a TSO frame must be composed of paged data.  */
2165  	DEBUG_NET_WARN_ON_ONCE(skb->len != skb->data_len);
2166  
2167  	buff = tcp_stream_alloc_skb(sk, gfp, true);
2168  	if (unlikely(!buff))
2169  		return -ENOMEM;
2170  	skb_copy_decrypted(buff, skb);
2171  	mptcp_skb_ext_copy(buff, skb);
2172  
2173  	sk_wmem_queued_add(sk, buff->truesize);
2174  	sk_mem_charge(sk, buff->truesize);
2175  	buff->truesize += nlen;
2176  	skb->truesize -= nlen;
2177  
2178  	/* Correct the sequence numbers. */
2179  	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
2180  	TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
2181  	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
2182  
2183  	/* PSH and FIN should only be set in the second packet. */
2184  	flags = TCP_SKB_CB(skb)->tcp_flags;
2185  	TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
2186  	TCP_SKB_CB(buff)->tcp_flags = flags;
2187  
2188  	tcp_skb_fragment_eor(skb, buff);
2189  
2190  	skb_split(skb, buff, len);
2191  	tcp_fragment_tstamp(skb, buff);
2192  
2193  	/* Fix up tso_factor for both original and new SKB.  */
2194  	tcp_set_skb_tso_segs(skb, mss_now);
2195  	tcp_set_skb_tso_segs(buff, mss_now);
2196  
2197  	/* Link BUFF into the send queue. */
2198  	__skb_header_release(buff);
2199  	tcp_insert_write_queue_after(skb, buff, sk, TCP_FRAG_IN_WRITE_QUEUE);
2200  
2201  	return 0;
2202  }
2203  
2204  /* Try to defer sending, if possible, in order to minimize the amount
2205   * of TSO splitting we do.  View it as a kind of TSO Nagle test.
2206   *
2207   * This algorithm is from John Heffner.
2208   */
tcp_tso_should_defer(struct sock * sk,struct sk_buff * skb,bool * is_cwnd_limited,bool * is_rwnd_limited,u32 max_segs)2209  static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
2210  				 bool *is_cwnd_limited,
2211  				 bool *is_rwnd_limited,
2212  				 u32 max_segs)
2213  {
2214  	const struct inet_connection_sock *icsk = inet_csk(sk);
2215  	u32 send_win, cong_win, limit, in_flight;
2216  	struct tcp_sock *tp = tcp_sk(sk);
2217  	struct sk_buff *head;
2218  	int win_divisor;
2219  	s64 delta;
2220  
2221  	if (icsk->icsk_ca_state >= TCP_CA_Recovery)
2222  		goto send_now;
2223  
2224  	/* Avoid bursty behavior by allowing defer
2225  	 * only if the last write was recent (1 ms).
2226  	 * Note that tp->tcp_wstamp_ns can be in the future if we have
2227  	 * packets waiting in a qdisc or device for EDT delivery.
2228  	 */
2229  	delta = tp->tcp_clock_cache - tp->tcp_wstamp_ns - NSEC_PER_MSEC;
2230  	if (delta > 0)
2231  		goto send_now;
2232  
2233  	in_flight = tcp_packets_in_flight(tp);
2234  
2235  	BUG_ON(tcp_skb_pcount(skb) <= 1);
2236  	BUG_ON(tcp_snd_cwnd(tp) <= in_flight);
2237  
2238  	send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
2239  
2240  	/* From in_flight test above, we know that cwnd > in_flight.  */
2241  	cong_win = (tcp_snd_cwnd(tp) - in_flight) * tp->mss_cache;
2242  
2243  	limit = min(send_win, cong_win);
2244  
2245  	/* If a full-sized TSO skb can be sent, do it. */
2246  	if (limit >= max_segs * tp->mss_cache)
2247  		goto send_now;
2248  
2249  	/* Middle in queue won't get any more data, full sendable already? */
2250  	if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
2251  		goto send_now;
2252  
2253  	win_divisor = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_win_divisor);
2254  	if (win_divisor) {
2255  		u32 chunk = min(tp->snd_wnd, tcp_snd_cwnd(tp) * tp->mss_cache);
2256  
2257  		/* If at least some fraction of a window is available,
2258  		 * just use it.
2259  		 */
2260  		chunk /= win_divisor;
2261  		if (limit >= chunk)
2262  			goto send_now;
2263  	} else {
2264  		/* Different approach, try not to defer past a single
2265  		 * ACK.  Receiver should ACK every other full sized
2266  		 * frame, so if we have space for more than 3 frames
2267  		 * then send now.
2268  		 */
2269  		if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
2270  			goto send_now;
2271  	}
2272  
2273  	/* TODO : use tsorted_sent_queue ? */
2274  	head = tcp_rtx_queue_head(sk);
2275  	if (!head)
2276  		goto send_now;
2277  	delta = tp->tcp_clock_cache - head->tstamp;
2278  	/* If next ACK is likely to come too late (half srtt), do not defer */
2279  	if ((s64)(delta - (u64)NSEC_PER_USEC * (tp->srtt_us >> 4)) < 0)
2280  		goto send_now;
2281  
2282  	/* Ok, it looks like it is advisable to defer.
2283  	 * Three cases are tracked :
2284  	 * 1) We are cwnd-limited
2285  	 * 2) We are rwnd-limited
2286  	 * 3) We are application limited.
2287  	 */
2288  	if (cong_win < send_win) {
2289  		if (cong_win <= skb->len) {
2290  			*is_cwnd_limited = true;
2291  			return true;
2292  		}
2293  	} else {
2294  		if (send_win <= skb->len) {
2295  			*is_rwnd_limited = true;
2296  			return true;
2297  		}
2298  	}
2299  
2300  	/* If this packet won't get more data, do not wait. */
2301  	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) ||
2302  	    TCP_SKB_CB(skb)->eor)
2303  		goto send_now;
2304  
2305  	return true;
2306  
2307  send_now:
2308  	return false;
2309  }
2310  
tcp_mtu_check_reprobe(struct sock * sk)2311  static inline void tcp_mtu_check_reprobe(struct sock *sk)
2312  {
2313  	struct inet_connection_sock *icsk = inet_csk(sk);
2314  	struct tcp_sock *tp = tcp_sk(sk);
2315  	struct net *net = sock_net(sk);
2316  	u32 interval;
2317  	s32 delta;
2318  
2319  	interval = READ_ONCE(net->ipv4.sysctl_tcp_probe_interval);
2320  	delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp;
2321  	if (unlikely(delta >= interval * HZ)) {
2322  		int mss = tcp_current_mss(sk);
2323  
2324  		/* Update current search range */
2325  		icsk->icsk_mtup.probe_size = 0;
2326  		icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp +
2327  			sizeof(struct tcphdr) +
2328  			icsk->icsk_af_ops->net_header_len;
2329  		icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss);
2330  
2331  		/* Update probe time stamp */
2332  		icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
2333  	}
2334  }
2335  
tcp_can_coalesce_send_queue_head(struct sock * sk,int len)2336  static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
2337  {
2338  	struct sk_buff *skb, *next;
2339  
2340  	skb = tcp_send_head(sk);
2341  	tcp_for_write_queue_from_safe(skb, next, sk) {
2342  		if (len <= skb->len)
2343  			break;
2344  
2345  		if (tcp_has_tx_tstamp(skb) || !tcp_skb_can_collapse(skb, next))
2346  			return false;
2347  
2348  		len -= skb->len;
2349  	}
2350  
2351  	return true;
2352  }
2353  
tcp_clone_payload(struct sock * sk,struct sk_buff * to,int probe_size)2354  static int tcp_clone_payload(struct sock *sk, struct sk_buff *to,
2355  			     int probe_size)
2356  {
2357  	skb_frag_t *lastfrag = NULL, *fragto = skb_shinfo(to)->frags;
2358  	int i, todo, len = 0, nr_frags = 0;
2359  	const struct sk_buff *skb;
2360  
2361  	if (!sk_wmem_schedule(sk, to->truesize + probe_size))
2362  		return -ENOMEM;
2363  
2364  	skb_queue_walk(&sk->sk_write_queue, skb) {
2365  		const skb_frag_t *fragfrom = skb_shinfo(skb)->frags;
2366  
2367  		if (skb_headlen(skb))
2368  			return -EINVAL;
2369  
2370  		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, fragfrom++) {
2371  			if (len >= probe_size)
2372  				goto commit;
2373  			todo = min_t(int, skb_frag_size(fragfrom),
2374  				     probe_size - len);
2375  			len += todo;
2376  			if (lastfrag &&
2377  			    skb_frag_page(fragfrom) == skb_frag_page(lastfrag) &&
2378  			    skb_frag_off(fragfrom) == skb_frag_off(lastfrag) +
2379  						      skb_frag_size(lastfrag)) {
2380  				skb_frag_size_add(lastfrag, todo);
2381  				continue;
2382  			}
2383  			if (unlikely(nr_frags == MAX_SKB_FRAGS))
2384  				return -E2BIG;
2385  			skb_frag_page_copy(fragto, fragfrom);
2386  			skb_frag_off_copy(fragto, fragfrom);
2387  			skb_frag_size_set(fragto, todo);
2388  			nr_frags++;
2389  			lastfrag = fragto++;
2390  		}
2391  	}
2392  commit:
2393  	WARN_ON_ONCE(len != probe_size);
2394  	for (i = 0; i < nr_frags; i++)
2395  		skb_frag_ref(to, i);
2396  
2397  	skb_shinfo(to)->nr_frags = nr_frags;
2398  	to->truesize += probe_size;
2399  	to->len += probe_size;
2400  	to->data_len += probe_size;
2401  	__skb_header_release(to);
2402  	return 0;
2403  }
2404  
2405  /* tcp_mtu_probe() and tcp_grow_skb() can both eat an skb (src) if
2406   * all its payload was moved to another one (dst).
2407   * Make sure to transfer tcp_flags, eor, and tstamp.
2408   */
tcp_eat_one_skb(struct sock * sk,struct sk_buff * dst,struct sk_buff * src)2409  static void tcp_eat_one_skb(struct sock *sk,
2410  			    struct sk_buff *dst,
2411  			    struct sk_buff *src)
2412  {
2413  	TCP_SKB_CB(dst)->tcp_flags |= TCP_SKB_CB(src)->tcp_flags;
2414  	TCP_SKB_CB(dst)->eor = TCP_SKB_CB(src)->eor;
2415  	tcp_skb_collapse_tstamp(dst, src);
2416  	tcp_unlink_write_queue(src, sk);
2417  	tcp_wmem_free_skb(sk, src);
2418  }
2419  
2420  /* Create a new MTU probe if we are ready.
2421   * MTU probe is regularly attempting to increase the path MTU by
2422   * deliberately sending larger packets.  This discovers routing
2423   * changes resulting in larger path MTUs.
2424   *
2425   * Returns 0 if we should wait to probe (no cwnd available),
2426   *         1 if a probe was sent,
2427   *         -1 otherwise
2428   */
tcp_mtu_probe(struct sock * sk)2429  static int tcp_mtu_probe(struct sock *sk)
2430  {
2431  	struct inet_connection_sock *icsk = inet_csk(sk);
2432  	struct tcp_sock *tp = tcp_sk(sk);
2433  	struct sk_buff *skb, *nskb, *next;
2434  	struct net *net = sock_net(sk);
2435  	int probe_size;
2436  	int size_needed;
2437  	int copy, len;
2438  	int mss_now;
2439  	int interval;
2440  
2441  	/* Not currently probing/verifying,
2442  	 * not in recovery,
2443  	 * have enough cwnd, and
2444  	 * not SACKing (the variable headers throw things off)
2445  	 */
2446  	if (likely(!icsk->icsk_mtup.enabled ||
2447  		   icsk->icsk_mtup.probe_size ||
2448  		   inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
2449  		   tcp_snd_cwnd(tp) < 11 ||
2450  		   tp->rx_opt.num_sacks || tp->rx_opt.dsack))
2451  		return -1;
2452  
2453  	/* Use binary search for probe_size between tcp_mss_base,
2454  	 * and current mss_clamp. if (search_high - search_low)
2455  	 * smaller than a threshold, backoff from probing.
2456  	 */
2457  	mss_now = tcp_current_mss(sk);
2458  	probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high +
2459  				    icsk->icsk_mtup.search_low) >> 1);
2460  	size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
2461  	interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low;
2462  	/* When misfortune happens, we are reprobing actively,
2463  	 * and then reprobe timer has expired. We stick with current
2464  	 * probing process by not resetting search range to its orignal.
2465  	 */
2466  	if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) ||
2467  	    interval < READ_ONCE(net->ipv4.sysctl_tcp_probe_threshold)) {
2468  		/* Check whether enough time has elaplased for
2469  		 * another round of probing.
2470  		 */
2471  		tcp_mtu_check_reprobe(sk);
2472  		return -1;
2473  	}
2474  
2475  	/* Have enough data in the send queue to probe? */
2476  	if (tp->write_seq - tp->snd_nxt < size_needed)
2477  		return -1;
2478  
2479  	if (tp->snd_wnd < size_needed)
2480  		return -1;
2481  	if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
2482  		return 0;
2483  
2484  	/* Do we need to wait to drain cwnd? With none in flight, don't stall */
2485  	if (tcp_packets_in_flight(tp) + 2 > tcp_snd_cwnd(tp)) {
2486  		if (!tcp_packets_in_flight(tp))
2487  			return -1;
2488  		else
2489  			return 0;
2490  	}
2491  
2492  	if (!tcp_can_coalesce_send_queue_head(sk, probe_size))
2493  		return -1;
2494  
2495  	/* We're allowed to probe.  Build it now. */
2496  	nskb = tcp_stream_alloc_skb(sk, GFP_ATOMIC, false);
2497  	if (!nskb)
2498  		return -1;
2499  
2500  	/* build the payload, and be prepared to abort if this fails. */
2501  	if (tcp_clone_payload(sk, nskb, probe_size)) {
2502  		tcp_skb_tsorted_anchor_cleanup(nskb);
2503  		consume_skb(nskb);
2504  		return -1;
2505  	}
2506  	sk_wmem_queued_add(sk, nskb->truesize);
2507  	sk_mem_charge(sk, nskb->truesize);
2508  
2509  	skb = tcp_send_head(sk);
2510  	skb_copy_decrypted(nskb, skb);
2511  	mptcp_skb_ext_copy(nskb, skb);
2512  
2513  	TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
2514  	TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
2515  	TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
2516  
2517  	tcp_insert_write_queue_before(nskb, skb, sk);
2518  	tcp_highest_sack_replace(sk, skb, nskb);
2519  
2520  	len = 0;
2521  	tcp_for_write_queue_from_safe(skb, next, sk) {
2522  		copy = min_t(int, skb->len, probe_size - len);
2523  
2524  		if (skb->len <= copy) {
2525  			tcp_eat_one_skb(sk, nskb, skb);
2526  		} else {
2527  			TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
2528  						   ~(TCPHDR_FIN|TCPHDR_PSH);
2529  			__pskb_trim_head(skb, copy);
2530  			tcp_set_skb_tso_segs(skb, mss_now);
2531  			TCP_SKB_CB(skb)->seq += copy;
2532  		}
2533  
2534  		len += copy;
2535  
2536  		if (len >= probe_size)
2537  			break;
2538  	}
2539  	tcp_init_tso_segs(nskb, nskb->len);
2540  
2541  	/* We're ready to send.  If this fails, the probe will
2542  	 * be resegmented into mss-sized pieces by tcp_write_xmit().
2543  	 */
2544  	if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
2545  		/* Decrement cwnd here because we are sending
2546  		 * effectively two packets. */
2547  		tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1);
2548  		tcp_event_new_data_sent(sk, nskb);
2549  
2550  		icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
2551  		tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
2552  		tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
2553  
2554  		return 1;
2555  	}
2556  
2557  	return -1;
2558  }
2559  
tcp_pacing_check(struct sock * sk)2560  static bool tcp_pacing_check(struct sock *sk)
2561  {
2562  	struct tcp_sock *tp = tcp_sk(sk);
2563  
2564  	if (!tcp_needs_internal_pacing(sk))
2565  		return false;
2566  
2567  	if (tp->tcp_wstamp_ns <= tp->tcp_clock_cache)
2568  		return false;
2569  
2570  	if (!hrtimer_is_queued(&tp->pacing_timer)) {
2571  		hrtimer_start(&tp->pacing_timer,
2572  			      ns_to_ktime(tp->tcp_wstamp_ns),
2573  			      HRTIMER_MODE_ABS_PINNED_SOFT);
2574  		sock_hold(sk);
2575  	}
2576  	return true;
2577  }
2578  
tcp_rtx_queue_empty_or_single_skb(const struct sock * sk)2579  static bool tcp_rtx_queue_empty_or_single_skb(const struct sock *sk)
2580  {
2581  	const struct rb_node *node = sk->tcp_rtx_queue.rb_node;
2582  
2583  	/* No skb in the rtx queue. */
2584  	if (!node)
2585  		return true;
2586  
2587  	/* Only one skb in rtx queue. */
2588  	return !node->rb_left && !node->rb_right;
2589  }
2590  
2591  /* TCP Small Queues :
2592   * Control number of packets in qdisc/devices to two packets / or ~1 ms.
2593   * (These limits are doubled for retransmits)
2594   * This allows for :
2595   *  - better RTT estimation and ACK scheduling
2596   *  - faster recovery
2597   *  - high rates
2598   * Alas, some drivers / subsystems require a fair amount
2599   * of queued bytes to ensure line rate.
2600   * One example is wifi aggregation (802.11 AMPDU)
2601   */
tcp_small_queue_check(struct sock * sk,const struct sk_buff * skb,unsigned int factor)2602  static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
2603  				  unsigned int factor)
2604  {
2605  	unsigned long limit;
2606  
2607  	limit = max_t(unsigned long,
2608  		      2 * skb->truesize,
2609  		      READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift));
2610  	if (sk->sk_pacing_status == SK_PACING_NONE)
2611  		limit = min_t(unsigned long, limit,
2612  			      READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes));
2613  	limit <<= factor;
2614  
2615  	if (static_branch_unlikely(&tcp_tx_delay_enabled) &&
2616  	    tcp_sk(sk)->tcp_tx_delay) {
2617  		u64 extra_bytes = (u64)READ_ONCE(sk->sk_pacing_rate) *
2618  				  tcp_sk(sk)->tcp_tx_delay;
2619  
2620  		/* TSQ is based on skb truesize sum (sk_wmem_alloc), so we
2621  		 * approximate our needs assuming an ~100% skb->truesize overhead.
2622  		 * USEC_PER_SEC is approximated by 2^20.
2623  		 * do_div(extra_bytes, USEC_PER_SEC/2) is replaced by a right shift.
2624  		 */
2625  		extra_bytes >>= (20 - 1);
2626  		limit += extra_bytes;
2627  	}
2628  	if (refcount_read(&sk->sk_wmem_alloc) > limit) {
2629  		/* Always send skb if rtx queue is empty or has one skb.
2630  		 * No need to wait for TX completion to call us back,
2631  		 * after softirq/tasklet schedule.
2632  		 * This helps when TX completions are delayed too much.
2633  		 */
2634  		if (tcp_rtx_queue_empty_or_single_skb(sk))
2635  			return false;
2636  
2637  		set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2638  		/* It is possible TX completion already happened
2639  		 * before we set TSQ_THROTTLED, so we must
2640  		 * test again the condition.
2641  		 */
2642  		smp_mb__after_atomic();
2643  		if (refcount_read(&sk->sk_wmem_alloc) > limit)
2644  			return true;
2645  	}
2646  	return false;
2647  }
2648  
tcp_chrono_set(struct tcp_sock * tp,const enum tcp_chrono new)2649  static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
2650  {
2651  	const u32 now = tcp_jiffies32;
2652  	enum tcp_chrono old = tp->chrono_type;
2653  
2654  	if (old > TCP_CHRONO_UNSPEC)
2655  		tp->chrono_stat[old - 1] += now - tp->chrono_start;
2656  	tp->chrono_start = now;
2657  	tp->chrono_type = new;
2658  }
2659  
tcp_chrono_start(struct sock * sk,const enum tcp_chrono type)2660  void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
2661  {
2662  	struct tcp_sock *tp = tcp_sk(sk);
2663  
2664  	/* If there are multiple conditions worthy of tracking in a
2665  	 * chronograph then the highest priority enum takes precedence
2666  	 * over the other conditions. So that if something "more interesting"
2667  	 * starts happening, stop the previous chrono and start a new one.
2668  	 */
2669  	if (type > tp->chrono_type)
2670  		tcp_chrono_set(tp, type);
2671  }
2672  
tcp_chrono_stop(struct sock * sk,const enum tcp_chrono type)2673  void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type)
2674  {
2675  	struct tcp_sock *tp = tcp_sk(sk);
2676  
2677  
2678  	/* There are multiple conditions worthy of tracking in a
2679  	 * chronograph, so that the highest priority enum takes
2680  	 * precedence over the other conditions (see tcp_chrono_start).
2681  	 * If a condition stops, we only stop chrono tracking if
2682  	 * it's the "most interesting" or current chrono we are
2683  	 * tracking and starts busy chrono if we have pending data.
2684  	 */
2685  	if (tcp_rtx_and_write_queues_empty(sk))
2686  		tcp_chrono_set(tp, TCP_CHRONO_UNSPEC);
2687  	else if (type == tp->chrono_type)
2688  		tcp_chrono_set(tp, TCP_CHRONO_BUSY);
2689  }
2690  
2691  /* First skb in the write queue is smaller than ideal packet size.
2692   * Check if we can move payload from the second skb in the queue.
2693   */
tcp_grow_skb(struct sock * sk,struct sk_buff * skb,int amount)2694  static void tcp_grow_skb(struct sock *sk, struct sk_buff *skb, int amount)
2695  {
2696  	struct sk_buff *next_skb = skb->next;
2697  	unsigned int nlen;
2698  
2699  	if (tcp_skb_is_last(sk, skb))
2700  		return;
2701  
2702  	if (!tcp_skb_can_collapse(skb, next_skb))
2703  		return;
2704  
2705  	nlen = min_t(u32, amount, next_skb->len);
2706  	if (!nlen || !skb_shift(skb, next_skb, nlen))
2707  		return;
2708  
2709  	TCP_SKB_CB(skb)->end_seq += nlen;
2710  	TCP_SKB_CB(next_skb)->seq += nlen;
2711  
2712  	if (!next_skb->len) {
2713  		/* In case FIN is set, we need to update end_seq */
2714  		TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
2715  
2716  		tcp_eat_one_skb(sk, skb, next_skb);
2717  	}
2718  }
2719  
2720  /* This routine writes packets to the network.  It advances the
2721   * send_head.  This happens as incoming acks open up the remote
2722   * window for us.
2723   *
2724   * LARGESEND note: !tcp_urg_mode is overkill, only frames between
2725   * snd_up-64k-mss .. snd_up cannot be large. However, taking into
2726   * account rare use of URG, this is not a big flaw.
2727   *
2728   * Send at most one packet when push_one > 0. Temporarily ignore
2729   * cwnd limit to force at most one packet out when push_one == 2.
2730  
2731   * Returns true, if no segments are in flight and we have queued segments,
2732   * but cannot send anything now because of SWS or another problem.
2733   */
tcp_write_xmit(struct sock * sk,unsigned int mss_now,int nonagle,int push_one,gfp_t gfp)2734  static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
2735  			   int push_one, gfp_t gfp)
2736  {
2737  	struct tcp_sock *tp = tcp_sk(sk);
2738  	struct sk_buff *skb;
2739  	unsigned int tso_segs, sent_pkts;
2740  	u32 cwnd_quota, max_segs;
2741  	int result;
2742  	bool is_cwnd_limited = false, is_rwnd_limited = false;
2743  
2744  	sent_pkts = 0;
2745  
2746  	tcp_mstamp_refresh(tp);
2747  	if (!push_one) {
2748  		/* Do MTU probing. */
2749  		result = tcp_mtu_probe(sk);
2750  		if (!result) {
2751  			return false;
2752  		} else if (result > 0) {
2753  			sent_pkts = 1;
2754  		}
2755  	}
2756  
2757  	max_segs = tcp_tso_segs(sk, mss_now);
2758  	while ((skb = tcp_send_head(sk))) {
2759  		unsigned int limit;
2760  		int missing_bytes;
2761  
2762  		if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {
2763  			/* "skb_mstamp_ns" is used as a start point for the retransmit timer */
2764  			tp->tcp_wstamp_ns = tp->tcp_clock_cache;
2765  			skb_set_delivery_time(skb, tp->tcp_wstamp_ns, SKB_CLOCK_MONOTONIC);
2766  			list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
2767  			tcp_init_tso_segs(skb, mss_now);
2768  			goto repair; /* Skip network transmission */
2769  		}
2770  
2771  		if (tcp_pacing_check(sk))
2772  			break;
2773  
2774  		cwnd_quota = tcp_cwnd_test(tp);
2775  		if (!cwnd_quota) {
2776  			if (push_one == 2)
2777  				/* Force out a loss probe pkt. */
2778  				cwnd_quota = 1;
2779  			else
2780  				break;
2781  		}
2782  		cwnd_quota = min(cwnd_quota, max_segs);
2783  		missing_bytes = cwnd_quota * mss_now - skb->len;
2784  		if (missing_bytes > 0)
2785  			tcp_grow_skb(sk, skb, missing_bytes);
2786  
2787  		tso_segs = tcp_set_skb_tso_segs(skb, mss_now);
2788  
2789  		if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) {
2790  			is_rwnd_limited = true;
2791  			break;
2792  		}
2793  
2794  		if (tso_segs == 1) {
2795  			if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
2796  						     (tcp_skb_is_last(sk, skb) ?
2797  						      nonagle : TCP_NAGLE_PUSH))))
2798  				break;
2799  		} else {
2800  			if (!push_one &&
2801  			    tcp_tso_should_defer(sk, skb, &is_cwnd_limited,
2802  						 &is_rwnd_limited, max_segs))
2803  				break;
2804  		}
2805  
2806  		limit = mss_now;
2807  		if (tso_segs > 1 && !tcp_urg_mode(tp))
2808  			limit = tcp_mss_split_point(sk, skb, mss_now,
2809  						    cwnd_quota,
2810  						    nonagle);
2811  
2812  		if (skb->len > limit &&
2813  		    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
2814  			break;
2815  
2816  		if (tcp_small_queue_check(sk, skb, 0))
2817  			break;
2818  
2819  		/* Argh, we hit an empty skb(), presumably a thread
2820  		 * is sleeping in sendmsg()/sk_stream_wait_memory().
2821  		 * We do not want to send a pure-ack packet and have
2822  		 * a strange looking rtx queue with empty packet(s).
2823  		 */
2824  		if (TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq)
2825  			break;
2826  
2827  		if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
2828  			break;
2829  
2830  repair:
2831  		/* Advance the send_head.  This one is sent out.
2832  		 * This call will increment packets_out.
2833  		 */
2834  		tcp_event_new_data_sent(sk, skb);
2835  
2836  		tcp_minshall_update(tp, mss_now, skb);
2837  		sent_pkts += tcp_skb_pcount(skb);
2838  
2839  		if (push_one)
2840  			break;
2841  	}
2842  
2843  	if (is_rwnd_limited)
2844  		tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED);
2845  	else
2846  		tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED);
2847  
2848  	is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp));
2849  	if (likely(sent_pkts || is_cwnd_limited))
2850  		tcp_cwnd_validate(sk, is_cwnd_limited);
2851  
2852  	if (likely(sent_pkts)) {
2853  		if (tcp_in_cwnd_reduction(sk))
2854  			tp->prr_out += sent_pkts;
2855  
2856  		/* Send one loss probe per tail loss episode. */
2857  		if (push_one != 2)
2858  			tcp_schedule_loss_probe(sk, false);
2859  		return false;
2860  	}
2861  	return !tp->packets_out && !tcp_write_queue_empty(sk);
2862  }
2863  
tcp_schedule_loss_probe(struct sock * sk,bool advancing_rto)2864  bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto)
2865  {
2866  	struct inet_connection_sock *icsk = inet_csk(sk);
2867  	struct tcp_sock *tp = tcp_sk(sk);
2868  	u32 timeout, timeout_us, rto_delta_us;
2869  	int early_retrans;
2870  
2871  	/* Don't do any loss probe on a Fast Open connection before 3WHS
2872  	 * finishes.
2873  	 */
2874  	if (rcu_access_pointer(tp->fastopen_rsk))
2875  		return false;
2876  
2877  	early_retrans = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_early_retrans);
2878  	/* Schedule a loss probe in 2*RTT for SACK capable connections
2879  	 * not in loss recovery, that are either limited by cwnd or application.
2880  	 */
2881  	if ((early_retrans != 3 && early_retrans != 4) ||
2882  	    !tp->packets_out || !tcp_is_sack(tp) ||
2883  	    (icsk->icsk_ca_state != TCP_CA_Open &&
2884  	     icsk->icsk_ca_state != TCP_CA_CWR))
2885  		return false;
2886  
2887  	/* Probe timeout is 2*rtt. Add minimum RTO to account
2888  	 * for delayed ack when there's one outstanding packet. If no RTT
2889  	 * sample is available then probe after TCP_TIMEOUT_INIT.
2890  	 */
2891  	if (tp->srtt_us) {
2892  		timeout_us = tp->srtt_us >> 2;
2893  		if (tp->packets_out == 1)
2894  			timeout_us += tcp_rto_min_us(sk);
2895  		else
2896  			timeout_us += TCP_TIMEOUT_MIN_US;
2897  		timeout = usecs_to_jiffies(timeout_us);
2898  	} else {
2899  		timeout = TCP_TIMEOUT_INIT;
2900  	}
2901  
2902  	/* If the RTO formula yields an earlier time, then use that time. */
2903  	rto_delta_us = advancing_rto ?
2904  			jiffies_to_usecs(inet_csk(sk)->icsk_rto) :
2905  			tcp_rto_delta_us(sk);  /* How far in future is RTO? */
2906  	if (rto_delta_us > 0)
2907  		timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us));
2908  
2909  	tcp_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout, TCP_RTO_MAX);
2910  	return true;
2911  }
2912  
2913  /* Thanks to skb fast clones, we can detect if a prior transmit of
2914   * a packet is still in a qdisc or driver queue.
2915   * In this case, there is very little point doing a retransmit !
2916   */
skb_still_in_host_queue(struct sock * sk,const struct sk_buff * skb)2917  static bool skb_still_in_host_queue(struct sock *sk,
2918  				    const struct sk_buff *skb)
2919  {
2920  	if (unlikely(skb_fclone_busy(sk, skb))) {
2921  		set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2922  		smp_mb__after_atomic();
2923  		if (skb_fclone_busy(sk, skb)) {
2924  			NET_INC_STATS(sock_net(sk),
2925  				      LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
2926  			return true;
2927  		}
2928  	}
2929  	return false;
2930  }
2931  
2932  /* When probe timeout (PTO) fires, try send a new segment if possible, else
2933   * retransmit the last segment.
2934   */
tcp_send_loss_probe(struct sock * sk)2935  void tcp_send_loss_probe(struct sock *sk)
2936  {
2937  	struct tcp_sock *tp = tcp_sk(sk);
2938  	struct sk_buff *skb;
2939  	int pcount;
2940  	int mss = tcp_current_mss(sk);
2941  
2942  	/* At most one outstanding TLP */
2943  	if (tp->tlp_high_seq)
2944  		goto rearm_timer;
2945  
2946  	tp->tlp_retrans = 0;
2947  	skb = tcp_send_head(sk);
2948  	if (skb && tcp_snd_wnd_test(tp, skb, mss)) {
2949  		pcount = tp->packets_out;
2950  		tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
2951  		if (tp->packets_out > pcount)
2952  			goto probe_sent;
2953  		goto rearm_timer;
2954  	}
2955  	skb = skb_rb_last(&sk->tcp_rtx_queue);
2956  	if (unlikely(!skb)) {
2957  		WARN_ONCE(tp->packets_out,
2958  			  "invalid inflight: %u state %u cwnd %u mss %d\n",
2959  			  tp->packets_out, sk->sk_state, tcp_snd_cwnd(tp), mss);
2960  		inet_csk(sk)->icsk_pending = 0;
2961  		return;
2962  	}
2963  
2964  	if (skb_still_in_host_queue(sk, skb))
2965  		goto rearm_timer;
2966  
2967  	pcount = tcp_skb_pcount(skb);
2968  	if (WARN_ON(!pcount))
2969  		goto rearm_timer;
2970  
2971  	if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
2972  		if (unlikely(tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
2973  					  (pcount - 1) * mss, mss,
2974  					  GFP_ATOMIC)))
2975  			goto rearm_timer;
2976  		skb = skb_rb_next(skb);
2977  	}
2978  
2979  	if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
2980  		goto rearm_timer;
2981  
2982  	if (__tcp_retransmit_skb(sk, skb, 1))
2983  		goto rearm_timer;
2984  
2985  	tp->tlp_retrans = 1;
2986  
2987  probe_sent:
2988  	/* Record snd_nxt for loss detection. */
2989  	tp->tlp_high_seq = tp->snd_nxt;
2990  
2991  	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES);
2992  	/* Reset s.t. tcp_rearm_rto will restart timer from now */
2993  	inet_csk(sk)->icsk_pending = 0;
2994  rearm_timer:
2995  	tcp_rearm_rto(sk);
2996  }
2997  
2998  /* Push out any pending frames which were held back due to
2999   * TCP_CORK or attempt at coalescing tiny packets.
3000   * The socket must be locked by the caller.
3001   */
__tcp_push_pending_frames(struct sock * sk,unsigned int cur_mss,int nonagle)3002  void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
3003  			       int nonagle)
3004  {
3005  	/* If we are closed, the bytes will have to remain here.
3006  	 * In time closedown will finish, we empty the write queue and
3007  	 * all will be happy.
3008  	 */
3009  	if (unlikely(sk->sk_state == TCP_CLOSE))
3010  		return;
3011  
3012  	if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
3013  			   sk_gfp_mask(sk, GFP_ATOMIC)))
3014  		tcp_check_probe_timer(sk);
3015  }
3016  
3017  /* Send _single_ skb sitting at the send head. This function requires
3018   * true push pending frames to setup probe timer etc.
3019   */
tcp_push_one(struct sock * sk,unsigned int mss_now)3020  void tcp_push_one(struct sock *sk, unsigned int mss_now)
3021  {
3022  	struct sk_buff *skb = tcp_send_head(sk);
3023  
3024  	BUG_ON(!skb || skb->len < mss_now);
3025  
3026  	tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
3027  }
3028  
3029  /* This function returns the amount that we can raise the
3030   * usable window based on the following constraints
3031   *
3032   * 1. The window can never be shrunk once it is offered (RFC 793)
3033   * 2. We limit memory per socket
3034   *
3035   * RFC 1122:
3036   * "the suggested [SWS] avoidance algorithm for the receiver is to keep
3037   *  RECV.NEXT + RCV.WIN fixed until:
3038   *  RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
3039   *
3040   * i.e. don't raise the right edge of the window until you can raise
3041   * it at least MSS bytes.
3042   *
3043   * Unfortunately, the recommended algorithm breaks header prediction,
3044   * since header prediction assumes th->window stays fixed.
3045   *
3046   * Strictly speaking, keeping th->window fixed violates the receiver
3047   * side SWS prevention criteria. The problem is that under this rule
3048   * a stream of single byte packets will cause the right side of the
3049   * window to always advance by a single byte.
3050   *
3051   * Of course, if the sender implements sender side SWS prevention
3052   * then this will not be a problem.
3053   *
3054   * BSD seems to make the following compromise:
3055   *
3056   *	If the free space is less than the 1/4 of the maximum
3057   *	space available and the free space is less than 1/2 mss,
3058   *	then set the window to 0.
3059   *	[ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
3060   *	Otherwise, just prevent the window from shrinking
3061   *	and from being larger than the largest representable value.
3062   *
3063   * This prevents incremental opening of the window in the regime
3064   * where TCP is limited by the speed of the reader side taking
3065   * data out of the TCP receive queue. It does nothing about
3066   * those cases where the window is constrained on the sender side
3067   * because the pipeline is full.
3068   *
3069   * BSD also seems to "accidentally" limit itself to windows that are a
3070   * multiple of MSS, at least until the free space gets quite small.
3071   * This would appear to be a side effect of the mbuf implementation.
3072   * Combining these two algorithms results in the observed behavior
3073   * of having a fixed window size at almost all times.
3074   *
3075   * Below we obtain similar behavior by forcing the offered window to
3076   * a multiple of the mss when it is feasible to do so.
3077   *
3078   * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
3079   * Regular options like TIMESTAMP are taken into account.
3080   */
__tcp_select_window(struct sock * sk)3081  u32 __tcp_select_window(struct sock *sk)
3082  {
3083  	struct inet_connection_sock *icsk = inet_csk(sk);
3084  	struct tcp_sock *tp = tcp_sk(sk);
3085  	struct net *net = sock_net(sk);
3086  	/* MSS for the peer's data.  Previous versions used mss_clamp
3087  	 * here.  I don't know if the value based on our guesses
3088  	 * of peer's MSS is better for the performance.  It's more correct
3089  	 * but may be worse for the performance because of rcv_mss
3090  	 * fluctuations.  --SAW  1998/11/1
3091  	 */
3092  	int mss = icsk->icsk_ack.rcv_mss;
3093  	int free_space = tcp_space(sk);
3094  	int allowed_space = tcp_full_space(sk);
3095  	int full_space, window;
3096  
3097  	if (sk_is_mptcp(sk))
3098  		mptcp_space(sk, &free_space, &allowed_space);
3099  
3100  	full_space = min_t(int, tp->window_clamp, allowed_space);
3101  
3102  	if (unlikely(mss > full_space)) {
3103  		mss = full_space;
3104  		if (mss <= 0)
3105  			return 0;
3106  	}
3107  
3108  	/* Only allow window shrink if the sysctl is enabled and we have
3109  	 * a non-zero scaling factor in effect.
3110  	 */
3111  	if (READ_ONCE(net->ipv4.sysctl_tcp_shrink_window) && tp->rx_opt.rcv_wscale)
3112  		goto shrink_window_allowed;
3113  
3114  	/* do not allow window to shrink */
3115  
3116  	if (free_space < (full_space >> 1)) {
3117  		icsk->icsk_ack.quick = 0;
3118  
3119  		if (tcp_under_memory_pressure(sk))
3120  			tcp_adjust_rcv_ssthresh(sk);
3121  
3122  		/* free_space might become our new window, make sure we don't
3123  		 * increase it due to wscale.
3124  		 */
3125  		free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
3126  
3127  		/* if free space is less than mss estimate, or is below 1/16th
3128  		 * of the maximum allowed, try to move to zero-window, else
3129  		 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and
3130  		 * new incoming data is dropped due to memory limits.
3131  		 * With large window, mss test triggers way too late in order
3132  		 * to announce zero window in time before rmem limit kicks in.
3133  		 */
3134  		if (free_space < (allowed_space >> 4) || free_space < mss)
3135  			return 0;
3136  	}
3137  
3138  	if (free_space > tp->rcv_ssthresh)
3139  		free_space = tp->rcv_ssthresh;
3140  
3141  	/* Don't do rounding if we are using window scaling, since the
3142  	 * scaled window will not line up with the MSS boundary anyway.
3143  	 */
3144  	if (tp->rx_opt.rcv_wscale) {
3145  		window = free_space;
3146  
3147  		/* Advertise enough space so that it won't get scaled away.
3148  		 * Import case: prevent zero window announcement if
3149  		 * 1<<rcv_wscale > mss.
3150  		 */
3151  		window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale));
3152  	} else {
3153  		window = tp->rcv_wnd;
3154  		/* Get the largest window that is a nice multiple of mss.
3155  		 * Window clamp already applied above.
3156  		 * If our current window offering is within 1 mss of the
3157  		 * free space we just keep it. This prevents the divide
3158  		 * and multiply from happening most of the time.
3159  		 * We also don't do any window rounding when the free space
3160  		 * is too small.
3161  		 */
3162  		if (window <= free_space - mss || window > free_space)
3163  			window = rounddown(free_space, mss);
3164  		else if (mss == full_space &&
3165  			 free_space > window + (full_space >> 1))
3166  			window = free_space;
3167  	}
3168  
3169  	return window;
3170  
3171  shrink_window_allowed:
3172  	/* new window should always be an exact multiple of scaling factor */
3173  	free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
3174  
3175  	if (free_space < (full_space >> 1)) {
3176  		icsk->icsk_ack.quick = 0;
3177  
3178  		if (tcp_under_memory_pressure(sk))
3179  			tcp_adjust_rcv_ssthresh(sk);
3180  
3181  		/* if free space is too low, return a zero window */
3182  		if (free_space < (allowed_space >> 4) || free_space < mss ||
3183  			free_space < (1 << tp->rx_opt.rcv_wscale))
3184  			return 0;
3185  	}
3186  
3187  	if (free_space > tp->rcv_ssthresh) {
3188  		free_space = tp->rcv_ssthresh;
3189  		/* new window should always be an exact multiple of scaling factor
3190  		 *
3191  		 * For this case, we ALIGN "up" (increase free_space) because
3192  		 * we know free_space is not zero here, it has been reduced from
3193  		 * the memory-based limit, and rcv_ssthresh is not a hard limit
3194  		 * (unlike sk_rcvbuf).
3195  		 */
3196  		free_space = ALIGN(free_space, (1 << tp->rx_opt.rcv_wscale));
3197  	}
3198  
3199  	return free_space;
3200  }
3201  
tcp_skb_collapse_tstamp(struct sk_buff * skb,const struct sk_buff * next_skb)3202  void tcp_skb_collapse_tstamp(struct sk_buff *skb,
3203  			     const struct sk_buff *next_skb)
3204  {
3205  	if (unlikely(tcp_has_tx_tstamp(next_skb))) {
3206  		const struct skb_shared_info *next_shinfo =
3207  			skb_shinfo(next_skb);
3208  		struct skb_shared_info *shinfo = skb_shinfo(skb);
3209  
3210  		shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP;
3211  		shinfo->tskey = next_shinfo->tskey;
3212  		TCP_SKB_CB(skb)->txstamp_ack |=
3213  			TCP_SKB_CB(next_skb)->txstamp_ack;
3214  	}
3215  }
3216  
3217  /* Collapses two adjacent SKB's during retransmission. */
tcp_collapse_retrans(struct sock * sk,struct sk_buff * skb)3218  static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
3219  {
3220  	struct tcp_sock *tp = tcp_sk(sk);
3221  	struct sk_buff *next_skb = skb_rb_next(skb);
3222  	int next_skb_size;
3223  
3224  	next_skb_size = next_skb->len;
3225  
3226  	BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
3227  
3228  	if (next_skb_size && !tcp_skb_shift(skb, next_skb, 1, next_skb_size))
3229  		return false;
3230  
3231  	tcp_highest_sack_replace(sk, next_skb, skb);
3232  
3233  	/* Update sequence range on original skb. */
3234  	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
3235  
3236  	/* Merge over control information. This moves PSH/FIN etc. over */
3237  	TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
3238  
3239  	/* All done, get rid of second SKB and account for it so
3240  	 * packet counting does not break.
3241  	 */
3242  	TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
3243  	TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor;
3244  
3245  	/* changed transmit queue under us so clear hints */
3246  	tcp_clear_retrans_hints_partial(tp);
3247  	if (next_skb == tp->retransmit_skb_hint)
3248  		tp->retransmit_skb_hint = skb;
3249  
3250  	tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
3251  
3252  	tcp_skb_collapse_tstamp(skb, next_skb);
3253  
3254  	tcp_rtx_queue_unlink_and_free(next_skb, sk);
3255  	return true;
3256  }
3257  
3258  /* Check if coalescing SKBs is legal. */
tcp_can_collapse(const struct sock * sk,const struct sk_buff * skb)3259  static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
3260  {
3261  	if (tcp_skb_pcount(skb) > 1)
3262  		return false;
3263  	if (skb_cloned(skb))
3264  		return false;
3265  	if (!skb_frags_readable(skb))
3266  		return false;
3267  	/* Some heuristics for collapsing over SACK'd could be invented */
3268  	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
3269  		return false;
3270  
3271  	return true;
3272  }
3273  
3274  /* Collapse packets in the retransmit queue to make to create
3275   * less packets on the wire. This is only done on retransmission.
3276   */
tcp_retrans_try_collapse(struct sock * sk,struct sk_buff * to,int space)3277  static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
3278  				     int space)
3279  {
3280  	struct tcp_sock *tp = tcp_sk(sk);
3281  	struct sk_buff *skb = to, *tmp;
3282  	bool first = true;
3283  
3284  	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse))
3285  		return;
3286  	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3287  		return;
3288  
3289  	skb_rbtree_walk_from_safe(skb, tmp) {
3290  		if (!tcp_can_collapse(sk, skb))
3291  			break;
3292  
3293  		if (!tcp_skb_can_collapse(to, skb))
3294  			break;
3295  
3296  		space -= skb->len;
3297  
3298  		if (first) {
3299  			first = false;
3300  			continue;
3301  		}
3302  
3303  		if (space < 0)
3304  			break;
3305  
3306  		if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
3307  			break;
3308  
3309  		if (!tcp_collapse_retrans(sk, to))
3310  			break;
3311  	}
3312  }
3313  
3314  /* This retransmits one SKB.  Policy decisions and retransmit queue
3315   * state updates are done by the caller.  Returns non-zero if an
3316   * error occurred which prevented the send.
3317   */
__tcp_retransmit_skb(struct sock * sk,struct sk_buff * skb,int segs)3318  int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
3319  {
3320  	struct inet_connection_sock *icsk = inet_csk(sk);
3321  	struct tcp_sock *tp = tcp_sk(sk);
3322  	unsigned int cur_mss;
3323  	int diff, len, err;
3324  	int avail_wnd;
3325  
3326  	/* Inconclusive MTU probe */
3327  	if (icsk->icsk_mtup.probe_size)
3328  		icsk->icsk_mtup.probe_size = 0;
3329  
3330  	if (skb_still_in_host_queue(sk, skb))
3331  		return -EBUSY;
3332  
3333  start:
3334  	if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
3335  		if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3336  			TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN;
3337  			TCP_SKB_CB(skb)->seq++;
3338  			goto start;
3339  		}
3340  		if (unlikely(before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))) {
3341  			WARN_ON_ONCE(1);
3342  			return -EINVAL;
3343  		}
3344  		if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
3345  			return -ENOMEM;
3346  	}
3347  
3348  	if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
3349  		return -EHOSTUNREACH; /* Routing failure or similar. */
3350  
3351  	cur_mss = tcp_current_mss(sk);
3352  	avail_wnd = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
3353  
3354  	/* If receiver has shrunk his window, and skb is out of
3355  	 * new window, do not retransmit it. The exception is the
3356  	 * case, when window is shrunk to zero. In this case
3357  	 * our retransmit of one segment serves as a zero window probe.
3358  	 */
3359  	if (avail_wnd <= 0) {
3360  		if (TCP_SKB_CB(skb)->seq != tp->snd_una)
3361  			return -EAGAIN;
3362  		avail_wnd = cur_mss;
3363  	}
3364  
3365  	len = cur_mss * segs;
3366  	if (len > avail_wnd) {
3367  		len = rounddown(avail_wnd, cur_mss);
3368  		if (!len)
3369  			len = avail_wnd;
3370  	}
3371  	if (skb->len > len) {
3372  		if (tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb, len,
3373  				 cur_mss, GFP_ATOMIC))
3374  			return -ENOMEM; /* We'll try again later. */
3375  	} else {
3376  		if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
3377  			return -ENOMEM;
3378  
3379  		diff = tcp_skb_pcount(skb);
3380  		tcp_set_skb_tso_segs(skb, cur_mss);
3381  		diff -= tcp_skb_pcount(skb);
3382  		if (diff)
3383  			tcp_adjust_pcount(sk, skb, diff);
3384  		avail_wnd = min_t(int, avail_wnd, cur_mss);
3385  		if (skb->len < avail_wnd)
3386  			tcp_retrans_try_collapse(sk, skb, avail_wnd);
3387  	}
3388  
3389  	/* RFC3168, section 6.1.1.1. ECN fallback */
3390  	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN)
3391  		tcp_ecn_clear_syn(sk, skb);
3392  
3393  	/* Update global and local TCP statistics. */
3394  	segs = tcp_skb_pcount(skb);
3395  	TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs);
3396  	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3397  		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
3398  	tp->total_retrans += segs;
3399  	tp->bytes_retrans += skb->len;
3400  
3401  	/* make sure skb->data is aligned on arches that require it
3402  	 * and check if ack-trimming & collapsing extended the headroom
3403  	 * beyond what csum_start can cover.
3404  	 */
3405  	if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
3406  		     skb_headroom(skb) >= 0xFFFF)) {
3407  		struct sk_buff *nskb;
3408  
3409  		tcp_skb_tsorted_save(skb) {
3410  			nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
3411  			if (nskb) {
3412  				nskb->dev = NULL;
3413  				err = tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC);
3414  			} else {
3415  				err = -ENOBUFS;
3416  			}
3417  		} tcp_skb_tsorted_restore(skb);
3418  
3419  		if (!err) {
3420  			tcp_update_skb_after_send(sk, skb, tp->tcp_wstamp_ns);
3421  			tcp_rate_skb_sent(sk, skb);
3422  		}
3423  	} else {
3424  		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3425  	}
3426  
3427  	if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG))
3428  		tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB,
3429  				  TCP_SKB_CB(skb)->seq, segs, err);
3430  
3431  	if (likely(!err)) {
3432  		trace_tcp_retransmit_skb(sk, skb);
3433  	} else if (err != -EBUSY) {
3434  		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs);
3435  	}
3436  
3437  	/* To avoid taking spuriously low RTT samples based on a timestamp
3438  	 * for a transmit that never happened, always mark EVER_RETRANS
3439  	 */
3440  	TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS;
3441  
3442  	return err;
3443  }
3444  
tcp_retransmit_skb(struct sock * sk,struct sk_buff * skb,int segs)3445  int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
3446  {
3447  	struct tcp_sock *tp = tcp_sk(sk);
3448  	int err = __tcp_retransmit_skb(sk, skb, segs);
3449  
3450  	if (err == 0) {
3451  #if FASTRETRANS_DEBUG > 0
3452  		if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
3453  			net_dbg_ratelimited("retrans_out leaked\n");
3454  		}
3455  #endif
3456  		TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
3457  		tp->retrans_out += tcp_skb_pcount(skb);
3458  	}
3459  
3460  	/* Save stamp of the first (attempted) retransmit. */
3461  	if (!tp->retrans_stamp)
3462  		tp->retrans_stamp = tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb);
3463  
3464  	if (tp->undo_retrans < 0)
3465  		tp->undo_retrans = 0;
3466  	tp->undo_retrans += tcp_skb_pcount(skb);
3467  	return err;
3468  }
3469  
3470  /* This gets called after a retransmit timeout, and the initially
3471   * retransmitted data is acknowledged.  It tries to continue
3472   * resending the rest of the retransmit queue, until either
3473   * we've sent it all or the congestion window limit is reached.
3474   */
tcp_xmit_retransmit_queue(struct sock * sk)3475  void tcp_xmit_retransmit_queue(struct sock *sk)
3476  {
3477  	const struct inet_connection_sock *icsk = inet_csk(sk);
3478  	struct sk_buff *skb, *rtx_head, *hole = NULL;
3479  	struct tcp_sock *tp = tcp_sk(sk);
3480  	bool rearm_timer = false;
3481  	u32 max_segs;
3482  	int mib_idx;
3483  
3484  	if (!tp->packets_out)
3485  		return;
3486  
3487  	rtx_head = tcp_rtx_queue_head(sk);
3488  	skb = tp->retransmit_skb_hint ?: rtx_head;
3489  	max_segs = tcp_tso_segs(sk, tcp_current_mss(sk));
3490  	skb_rbtree_walk_from(skb) {
3491  		__u8 sacked;
3492  		int segs;
3493  
3494  		if (tcp_pacing_check(sk))
3495  			break;
3496  
3497  		/* we could do better than to assign each time */
3498  		if (!hole)
3499  			tp->retransmit_skb_hint = skb;
3500  
3501  		segs = tcp_snd_cwnd(tp) - tcp_packets_in_flight(tp);
3502  		if (segs <= 0)
3503  			break;
3504  		sacked = TCP_SKB_CB(skb)->sacked;
3505  		/* In case tcp_shift_skb_data() have aggregated large skbs,
3506  		 * we need to make sure not sending too bigs TSO packets
3507  		 */
3508  		segs = min_t(int, segs, max_segs);
3509  
3510  		if (tp->retrans_out >= tp->lost_out) {
3511  			break;
3512  		} else if (!(sacked & TCPCB_LOST)) {
3513  			if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
3514  				hole = skb;
3515  			continue;
3516  
3517  		} else {
3518  			if (icsk->icsk_ca_state != TCP_CA_Loss)
3519  				mib_idx = LINUX_MIB_TCPFASTRETRANS;
3520  			else
3521  				mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
3522  		}
3523  
3524  		if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
3525  			continue;
3526  
3527  		if (tcp_small_queue_check(sk, skb, 1))
3528  			break;
3529  
3530  		if (tcp_retransmit_skb(sk, skb, segs))
3531  			break;
3532  
3533  		NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb));
3534  
3535  		if (tcp_in_cwnd_reduction(sk))
3536  			tp->prr_out += tcp_skb_pcount(skb);
3537  
3538  		if (skb == rtx_head &&
3539  		    icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT)
3540  			rearm_timer = true;
3541  
3542  	}
3543  	if (rearm_timer)
3544  		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3545  				     inet_csk(sk)->icsk_rto,
3546  				     TCP_RTO_MAX);
3547  }
3548  
3549  /* We allow to exceed memory limits for FIN packets to expedite
3550   * connection tear down and (memory) recovery.
3551   * Otherwise tcp_send_fin() could be tempted to either delay FIN
3552   * or even be forced to close flow without any FIN.
3553   * In general, we want to allow one skb per socket to avoid hangs
3554   * with edge trigger epoll()
3555   */
sk_forced_mem_schedule(struct sock * sk,int size)3556  void sk_forced_mem_schedule(struct sock *sk, int size)
3557  {
3558  	int delta, amt;
3559  
3560  	delta = size - sk->sk_forward_alloc;
3561  	if (delta <= 0)
3562  		return;
3563  	amt = sk_mem_pages(delta);
3564  	sk_forward_alloc_add(sk, amt << PAGE_SHIFT);
3565  	sk_memory_allocated_add(sk, amt);
3566  
3567  	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
3568  		mem_cgroup_charge_skmem(sk->sk_memcg, amt,
3569  					gfp_memcg_charge() | __GFP_NOFAIL);
3570  }
3571  
3572  /* Send a FIN. The caller locks the socket for us.
3573   * We should try to send a FIN packet really hard, but eventually give up.
3574   */
tcp_send_fin(struct sock * sk)3575  void tcp_send_fin(struct sock *sk)
3576  {
3577  	struct sk_buff *skb, *tskb, *tail = tcp_write_queue_tail(sk);
3578  	struct tcp_sock *tp = tcp_sk(sk);
3579  
3580  	/* Optimization, tack on the FIN if we have one skb in write queue and
3581  	 * this skb was not yet sent, or we are under memory pressure.
3582  	 * Note: in the latter case, FIN packet will be sent after a timeout,
3583  	 * as TCP stack thinks it has already been transmitted.
3584  	 */
3585  	tskb = tail;
3586  	if (!tskb && tcp_under_memory_pressure(sk))
3587  		tskb = skb_rb_last(&sk->tcp_rtx_queue);
3588  
3589  	if (tskb) {
3590  		TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
3591  		TCP_SKB_CB(tskb)->end_seq++;
3592  		tp->write_seq++;
3593  		if (!tail) {
3594  			/* This means tskb was already sent.
3595  			 * Pretend we included the FIN on previous transmit.
3596  			 * We need to set tp->snd_nxt to the value it would have
3597  			 * if FIN had been sent. This is because retransmit path
3598  			 * does not change tp->snd_nxt.
3599  			 */
3600  			WRITE_ONCE(tp->snd_nxt, tp->snd_nxt + 1);
3601  			return;
3602  		}
3603  	} else {
3604  		skb = alloc_skb_fclone(MAX_TCP_HEADER,
3605  				       sk_gfp_mask(sk, GFP_ATOMIC |
3606  						       __GFP_NOWARN));
3607  		if (unlikely(!skb))
3608  			return;
3609  
3610  		INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
3611  		skb_reserve(skb, MAX_TCP_HEADER);
3612  		sk_forced_mem_schedule(sk, skb->truesize);
3613  		/* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
3614  		tcp_init_nondata_skb(skb, tp->write_seq,
3615  				     TCPHDR_ACK | TCPHDR_FIN);
3616  		tcp_queue_skb(sk, skb);
3617  	}
3618  	__tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
3619  }
3620  
3621  /* We get here when a process closes a file descriptor (either due to
3622   * an explicit close() or as a byproduct of exit()'ing) and there
3623   * was unread data in the receive queue.  This behavior is recommended
3624   * by RFC 2525, section 2.17.  -DaveM
3625   */
tcp_send_active_reset(struct sock * sk,gfp_t priority,enum sk_rst_reason reason)3626  void tcp_send_active_reset(struct sock *sk, gfp_t priority,
3627  			   enum sk_rst_reason reason)
3628  {
3629  	struct sk_buff *skb;
3630  
3631  	TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
3632  
3633  	/* NOTE: No TCP options attached and we never retransmit this. */
3634  	skb = alloc_skb(MAX_TCP_HEADER, priority);
3635  	if (!skb) {
3636  		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3637  		return;
3638  	}
3639  
3640  	/* Reserve space for headers and prepare control bits. */
3641  	skb_reserve(skb, MAX_TCP_HEADER);
3642  	tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
3643  			     TCPHDR_ACK | TCPHDR_RST);
3644  	tcp_mstamp_refresh(tcp_sk(sk));
3645  	/* Send it off. */
3646  	if (tcp_transmit_skb(sk, skb, 0, priority))
3647  		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3648  
3649  	/* skb of trace_tcp_send_reset() keeps the skb that caused RST,
3650  	 * skb here is different to the troublesome skb, so use NULL
3651  	 */
3652  	trace_tcp_send_reset(sk, NULL, reason);
3653  }
3654  
3655  /* Send a crossed SYN-ACK during socket establishment.
3656   * WARNING: This routine must only be called when we have already sent
3657   * a SYN packet that crossed the incoming SYN that caused this routine
3658   * to get called. If this assumption fails then the initial rcv_wnd
3659   * and rcv_wscale values will not be correct.
3660   */
tcp_send_synack(struct sock * sk)3661  int tcp_send_synack(struct sock *sk)
3662  {
3663  	struct sk_buff *skb;
3664  
3665  	skb = tcp_rtx_queue_head(sk);
3666  	if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3667  		pr_err("%s: wrong queue state\n", __func__);
3668  		return -EFAULT;
3669  	}
3670  	if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
3671  		if (skb_cloned(skb)) {
3672  			struct sk_buff *nskb;
3673  
3674  			tcp_skb_tsorted_save(skb) {
3675  				nskb = skb_copy(skb, GFP_ATOMIC);
3676  			} tcp_skb_tsorted_restore(skb);
3677  			if (!nskb)
3678  				return -ENOMEM;
3679  			INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor);
3680  			tcp_highest_sack_replace(sk, skb, nskb);
3681  			tcp_rtx_queue_unlink_and_free(skb, sk);
3682  			__skb_header_release(nskb);
3683  			tcp_rbtree_insert(&sk->tcp_rtx_queue, nskb);
3684  			sk_wmem_queued_add(sk, nskb->truesize);
3685  			sk_mem_charge(sk, nskb->truesize);
3686  			skb = nskb;
3687  		}
3688  
3689  		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
3690  		tcp_ecn_send_synack(sk, skb);
3691  	}
3692  	return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3693  }
3694  
3695  /**
3696   * tcp_make_synack - Allocate one skb and build a SYNACK packet.
3697   * @sk: listener socket
3698   * @dst: dst entry attached to the SYNACK. It is consumed and caller
3699   *       should not use it again.
3700   * @req: request_sock pointer
3701   * @foc: cookie for tcp fast open
3702   * @synack_type: Type of synack to prepare
3703   * @syn_skb: SYN packet just received.  It could be NULL for rtx case.
3704   */
tcp_make_synack(const struct sock * sk,struct dst_entry * dst,struct request_sock * req,struct tcp_fastopen_cookie * foc,enum tcp_synack_type synack_type,struct sk_buff * syn_skb)3705  struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
3706  				struct request_sock *req,
3707  				struct tcp_fastopen_cookie *foc,
3708  				enum tcp_synack_type synack_type,
3709  				struct sk_buff *syn_skb)
3710  {
3711  	struct inet_request_sock *ireq = inet_rsk(req);
3712  	const struct tcp_sock *tp = tcp_sk(sk);
3713  	struct tcp_out_options opts;
3714  	struct tcp_key key = {};
3715  	struct sk_buff *skb;
3716  	int tcp_header_size;
3717  	struct tcphdr *th;
3718  	int mss;
3719  	u64 now;
3720  
3721  	skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
3722  	if (unlikely(!skb)) {
3723  		dst_release(dst);
3724  		return NULL;
3725  	}
3726  	/* Reserve space for headers. */
3727  	skb_reserve(skb, MAX_TCP_HEADER);
3728  
3729  	switch (synack_type) {
3730  	case TCP_SYNACK_NORMAL:
3731  		skb_set_owner_w(skb, req_to_sk(req));
3732  		break;
3733  	case TCP_SYNACK_COOKIE:
3734  		/* Under synflood, we do not attach skb to a socket,
3735  		 * to avoid false sharing.
3736  		 */
3737  		break;
3738  	case TCP_SYNACK_FASTOPEN:
3739  		/* sk is a const pointer, because we want to express multiple
3740  		 * cpu might call us concurrently.
3741  		 * sk->sk_wmem_alloc in an atomic, we can promote to rw.
3742  		 */
3743  		skb_set_owner_w(skb, (struct sock *)sk);
3744  		break;
3745  	}
3746  	skb_dst_set(skb, dst);
3747  
3748  	mss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3749  
3750  	memset(&opts, 0, sizeof(opts));
3751  	now = tcp_clock_ns();
3752  #ifdef CONFIG_SYN_COOKIES
3753  	if (unlikely(synack_type == TCP_SYNACK_COOKIE && ireq->tstamp_ok))
3754  		skb_set_delivery_time(skb, cookie_init_timestamp(req, now),
3755  				      SKB_CLOCK_MONOTONIC);
3756  	else
3757  #endif
3758  	{
3759  		skb_set_delivery_time(skb, now, SKB_CLOCK_MONOTONIC);
3760  		if (!tcp_rsk(req)->snt_synack) /* Timestamp first SYNACK */
3761  			tcp_rsk(req)->snt_synack = tcp_skb_timestamp_us(skb);
3762  	}
3763  
3764  #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
3765  	rcu_read_lock();
3766  #endif
3767  	if (tcp_rsk_used_ao(req)) {
3768  #ifdef CONFIG_TCP_AO
3769  		struct tcp_ao_key *ao_key = NULL;
3770  		u8 keyid = tcp_rsk(req)->ao_keyid;
3771  		u8 rnext = tcp_rsk(req)->ao_rcv_next;
3772  
3773  		ao_key = tcp_sk(sk)->af_specific->ao_lookup(sk, req_to_sk(req),
3774  							    keyid, -1);
3775  		/* If there is no matching key - avoid sending anything,
3776  		 * especially usigned segments. It could try harder and lookup
3777  		 * for another peer-matching key, but the peer has requested
3778  		 * ao_keyid (RFC5925 RNextKeyID), so let's keep it simple here.
3779  		 */
3780  		if (unlikely(!ao_key)) {
3781  			trace_tcp_ao_synack_no_key(sk, keyid, rnext);
3782  			rcu_read_unlock();
3783  			kfree_skb(skb);
3784  			net_warn_ratelimited("TCP-AO: the keyid %u from SYN packet is not present - not sending SYNACK\n",
3785  					     keyid);
3786  			return NULL;
3787  		}
3788  		key.ao_key = ao_key;
3789  		key.type = TCP_KEY_AO;
3790  #endif
3791  	} else {
3792  #ifdef CONFIG_TCP_MD5SIG
3793  		key.md5_key = tcp_rsk(req)->af_specific->req_md5_lookup(sk,
3794  					req_to_sk(req));
3795  		if (key.md5_key)
3796  			key.type = TCP_KEY_MD5;
3797  #endif
3798  	}
3799  	skb_set_hash(skb, READ_ONCE(tcp_rsk(req)->txhash), PKT_HASH_TYPE_L4);
3800  	/* bpf program will be interested in the tcp_flags */
3801  	TCP_SKB_CB(skb)->tcp_flags = TCPHDR_SYN | TCPHDR_ACK;
3802  	tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts,
3803  					     &key, foc, synack_type, syn_skb)
3804  					+ sizeof(*th);
3805  
3806  	skb_push(skb, tcp_header_size);
3807  	skb_reset_transport_header(skb);
3808  
3809  	th = (struct tcphdr *)skb->data;
3810  	memset(th, 0, sizeof(struct tcphdr));
3811  	th->syn = 1;
3812  	th->ack = 1;
3813  	tcp_ecn_make_synack(req, th);
3814  	th->source = htons(ireq->ir_num);
3815  	th->dest = ireq->ir_rmt_port;
3816  	skb->mark = ireq->ir_mark;
3817  	skb->ip_summed = CHECKSUM_PARTIAL;
3818  	th->seq = htonl(tcp_rsk(req)->snt_isn);
3819  	/* XXX data is queued and acked as is. No buffer/window check */
3820  	th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
3821  
3822  	/* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
3823  	th->window = htons(min(req->rsk_rcv_wnd, 65535U));
3824  	tcp_options_write(th, NULL, tcp_rsk(req), &opts, &key);
3825  	th->doff = (tcp_header_size >> 2);
3826  	TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
3827  
3828  	/* Okay, we have all we need - do the md5 hash if needed */
3829  	if (tcp_key_is_md5(&key)) {
3830  #ifdef CONFIG_TCP_MD5SIG
3831  		tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
3832  					key.md5_key, req_to_sk(req), skb);
3833  #endif
3834  	} else if (tcp_key_is_ao(&key)) {
3835  #ifdef CONFIG_TCP_AO
3836  		tcp_rsk(req)->af_specific->ao_synack_hash(opts.hash_location,
3837  					key.ao_key, req, skb,
3838  					opts.hash_location - (u8 *)th, 0);
3839  #endif
3840  	}
3841  #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
3842  	rcu_read_unlock();
3843  #endif
3844  
3845  	bpf_skops_write_hdr_opt((struct sock *)sk, skb, req, syn_skb,
3846  				synack_type, &opts);
3847  
3848  	skb_set_delivery_time(skb, now, SKB_CLOCK_MONOTONIC);
3849  	tcp_add_tx_delay(skb, tp);
3850  
3851  	return skb;
3852  }
3853  EXPORT_SYMBOL(tcp_make_synack);
3854  
tcp_ca_dst_init(struct sock * sk,const struct dst_entry * dst)3855  static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst)
3856  {
3857  	struct inet_connection_sock *icsk = inet_csk(sk);
3858  	const struct tcp_congestion_ops *ca;
3859  	u32 ca_key = dst_metric(dst, RTAX_CC_ALGO);
3860  
3861  	if (ca_key == TCP_CA_UNSPEC)
3862  		return;
3863  
3864  	rcu_read_lock();
3865  	ca = tcp_ca_find_key(ca_key);
3866  	if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
3867  		bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
3868  		icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
3869  		icsk->icsk_ca_ops = ca;
3870  	}
3871  	rcu_read_unlock();
3872  }
3873  
3874  /* Do all connect socket setups that can be done AF independent. */
tcp_connect_init(struct sock * sk)3875  static void tcp_connect_init(struct sock *sk)
3876  {
3877  	const struct dst_entry *dst = __sk_dst_get(sk);
3878  	struct tcp_sock *tp = tcp_sk(sk);
3879  	__u8 rcv_wscale;
3880  	u32 rcv_wnd;
3881  
3882  	/* We'll fix this up when we get a response from the other end.
3883  	 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
3884  	 */
3885  	tp->tcp_header_len = sizeof(struct tcphdr);
3886  	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps))
3887  		tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
3888  
3889  	tcp_ao_connect_init(sk);
3890  
3891  	/* If user gave his TCP_MAXSEG, record it to clamp */
3892  	if (tp->rx_opt.user_mss)
3893  		tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
3894  	tp->max_window = 0;
3895  	tcp_mtup_init(sk);
3896  	tcp_sync_mss(sk, dst_mtu(dst));
3897  
3898  	tcp_ca_dst_init(sk, dst);
3899  
3900  	if (!tp->window_clamp)
3901  		WRITE_ONCE(tp->window_clamp, dst_metric(dst, RTAX_WINDOW));
3902  	tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3903  
3904  	tcp_initialize_rcv_mss(sk);
3905  
3906  	/* limit the window selection if the user enforce a smaller rx buffer */
3907  	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
3908  	    (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
3909  		WRITE_ONCE(tp->window_clamp, tcp_full_space(sk));
3910  
3911  	rcv_wnd = tcp_rwnd_init_bpf(sk);
3912  	if (rcv_wnd == 0)
3913  		rcv_wnd = dst_metric(dst, RTAX_INITRWND);
3914  
3915  	tcp_select_initial_window(sk, tcp_full_space(sk),
3916  				  tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
3917  				  &tp->rcv_wnd,
3918  				  &tp->window_clamp,
3919  				  READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling),
3920  				  &rcv_wscale,
3921  				  rcv_wnd);
3922  
3923  	tp->rx_opt.rcv_wscale = rcv_wscale;
3924  	tp->rcv_ssthresh = tp->rcv_wnd;
3925  
3926  	WRITE_ONCE(sk->sk_err, 0);
3927  	sock_reset_flag(sk, SOCK_DONE);
3928  	tp->snd_wnd = 0;
3929  	tcp_init_wl(tp, 0);
3930  	tcp_write_queue_purge(sk);
3931  	tp->snd_una = tp->write_seq;
3932  	tp->snd_sml = tp->write_seq;
3933  	tp->snd_up = tp->write_seq;
3934  	WRITE_ONCE(tp->snd_nxt, tp->write_seq);
3935  
3936  	if (likely(!tp->repair))
3937  		tp->rcv_nxt = 0;
3938  	else
3939  		tp->rcv_tstamp = tcp_jiffies32;
3940  	tp->rcv_wup = tp->rcv_nxt;
3941  	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
3942  
3943  	inet_csk(sk)->icsk_rto = tcp_timeout_init(sk);
3944  	inet_csk(sk)->icsk_retransmits = 0;
3945  	tcp_clear_retrans(tp);
3946  }
3947  
tcp_connect_queue_skb(struct sock * sk,struct sk_buff * skb)3948  static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
3949  {
3950  	struct tcp_sock *tp = tcp_sk(sk);
3951  	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
3952  
3953  	tcb->end_seq += skb->len;
3954  	__skb_header_release(skb);
3955  	sk_wmem_queued_add(sk, skb->truesize);
3956  	sk_mem_charge(sk, skb->truesize);
3957  	WRITE_ONCE(tp->write_seq, tcb->end_seq);
3958  	tp->packets_out += tcp_skb_pcount(skb);
3959  }
3960  
3961  /* Build and send a SYN with data and (cached) Fast Open cookie. However,
3962   * queue a data-only packet after the regular SYN, such that regular SYNs
3963   * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
3964   * only the SYN sequence, the data are retransmitted in the first ACK.
3965   * If cookie is not cached or other error occurs, falls back to send a
3966   * regular SYN with Fast Open cookie request option.
3967   */
tcp_send_syn_data(struct sock * sk,struct sk_buff * syn)3968  static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
3969  {
3970  	struct inet_connection_sock *icsk = inet_csk(sk);
3971  	struct tcp_sock *tp = tcp_sk(sk);
3972  	struct tcp_fastopen_request *fo = tp->fastopen_req;
3973  	struct page_frag *pfrag = sk_page_frag(sk);
3974  	struct sk_buff *syn_data;
3975  	int space, err = 0;
3976  
3977  	tp->rx_opt.mss_clamp = tp->advmss;  /* If MSS is not cached */
3978  	if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))
3979  		goto fallback;
3980  
3981  	/* MSS for SYN-data is based on cached MSS and bounded by PMTU and
3982  	 * user-MSS. Reserve maximum option space for middleboxes that add
3983  	 * private TCP options. The cost is reduced data space in SYN :(
3984  	 */
3985  	tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);
3986  	/* Sync mss_cache after updating the mss_clamp */
3987  	tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
3988  
3989  	space = __tcp_mtu_to_mss(sk, icsk->icsk_pmtu_cookie) -
3990  		MAX_TCP_OPTION_SPACE;
3991  
3992  	space = min_t(size_t, space, fo->size);
3993  
3994  	if (space &&
3995  	    !skb_page_frag_refill(min_t(size_t, space, PAGE_SIZE),
3996  				  pfrag, sk->sk_allocation))
3997  		goto fallback;
3998  	syn_data = tcp_stream_alloc_skb(sk, sk->sk_allocation, false);
3999  	if (!syn_data)
4000  		goto fallback;
4001  	memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
4002  	if (space) {
4003  		space = min_t(size_t, space, pfrag->size - pfrag->offset);
4004  		space = tcp_wmem_schedule(sk, space);
4005  	}
4006  	if (space) {
4007  		space = copy_page_from_iter(pfrag->page, pfrag->offset,
4008  					    space, &fo->data->msg_iter);
4009  		if (unlikely(!space)) {
4010  			tcp_skb_tsorted_anchor_cleanup(syn_data);
4011  			kfree_skb(syn_data);
4012  			goto fallback;
4013  		}
4014  		skb_fill_page_desc(syn_data, 0, pfrag->page,
4015  				   pfrag->offset, space);
4016  		page_ref_inc(pfrag->page);
4017  		pfrag->offset += space;
4018  		skb_len_add(syn_data, space);
4019  		skb_zcopy_set(syn_data, fo->uarg, NULL);
4020  	}
4021  	/* No more data pending in inet_wait_for_connect() */
4022  	if (space == fo->size)
4023  		fo->data = NULL;
4024  	fo->copied = space;
4025  
4026  	tcp_connect_queue_skb(sk, syn_data);
4027  	if (syn_data->len)
4028  		tcp_chrono_start(sk, TCP_CHRONO_BUSY);
4029  
4030  	err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
4031  
4032  	skb_set_delivery_time(syn, syn_data->skb_mstamp_ns, SKB_CLOCK_MONOTONIC);
4033  
4034  	/* Now full SYN+DATA was cloned and sent (or not),
4035  	 * remove the SYN from the original skb (syn_data)
4036  	 * we keep in write queue in case of a retransmit, as we
4037  	 * also have the SYN packet (with no data) in the same queue.
4038  	 */
4039  	TCP_SKB_CB(syn_data)->seq++;
4040  	TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
4041  	if (!err) {
4042  		tp->syn_data = (fo->copied > 0);
4043  		tcp_rbtree_insert(&sk->tcp_rtx_queue, syn_data);
4044  		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
4045  		goto done;
4046  	}
4047  
4048  	/* data was not sent, put it in write_queue */
4049  	__skb_queue_tail(&sk->sk_write_queue, syn_data);
4050  	tp->packets_out -= tcp_skb_pcount(syn_data);
4051  
4052  fallback:
4053  	/* Send a regular SYN with Fast Open cookie request option */
4054  	if (fo->cookie.len > 0)
4055  		fo->cookie.len = 0;
4056  	err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
4057  	if (err)
4058  		tp->syn_fastopen = 0;
4059  done:
4060  	fo->cookie.len = -1;  /* Exclude Fast Open option for SYN retries */
4061  	return err;
4062  }
4063  
4064  /* Build a SYN and send it off. */
tcp_connect(struct sock * sk)4065  int tcp_connect(struct sock *sk)
4066  {
4067  	struct tcp_sock *tp = tcp_sk(sk);
4068  	struct sk_buff *buff;
4069  	int err;
4070  
4071  	tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL);
4072  
4073  #if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO)
4074  	/* Has to be checked late, after setting daddr/saddr/ops.
4075  	 * Return error if the peer has both a md5 and a tcp-ao key
4076  	 * configured as this is ambiguous.
4077  	 */
4078  	if (unlikely(rcu_dereference_protected(tp->md5sig_info,
4079  					       lockdep_sock_is_held(sk)))) {
4080  		bool needs_ao = !!tp->af_specific->ao_lookup(sk, sk, -1, -1);
4081  		bool needs_md5 = !!tp->af_specific->md5_lookup(sk, sk);
4082  		struct tcp_ao_info *ao_info;
4083  
4084  		ao_info = rcu_dereference_check(tp->ao_info,
4085  						lockdep_sock_is_held(sk));
4086  		if (ao_info) {
4087  			/* This is an extra check: tcp_ao_required() in
4088  			 * tcp_v{4,6}_parse_md5_keys() should prevent adding
4089  			 * md5 keys on ao_required socket.
4090  			 */
4091  			needs_ao |= ao_info->ao_required;
4092  			WARN_ON_ONCE(ao_info->ao_required && needs_md5);
4093  		}
4094  		if (needs_md5 && needs_ao)
4095  			return -EKEYREJECTED;
4096  
4097  		/* If we have a matching md5 key and no matching tcp-ao key
4098  		 * then free up ao_info if allocated.
4099  		 */
4100  		if (needs_md5) {
4101  			tcp_ao_destroy_sock(sk, false);
4102  		} else if (needs_ao) {
4103  			tcp_clear_md5_list(sk);
4104  			kfree(rcu_replace_pointer(tp->md5sig_info, NULL,
4105  						  lockdep_sock_is_held(sk)));
4106  		}
4107  	}
4108  #endif
4109  #ifdef CONFIG_TCP_AO
4110  	if (unlikely(rcu_dereference_protected(tp->ao_info,
4111  					       lockdep_sock_is_held(sk)))) {
4112  		/* Don't allow connecting if ao is configured but no
4113  		 * matching key is found.
4114  		 */
4115  		if (!tp->af_specific->ao_lookup(sk, sk, -1, -1))
4116  			return -EKEYREJECTED;
4117  	}
4118  #endif
4119  
4120  	if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
4121  		return -EHOSTUNREACH; /* Routing failure or similar. */
4122  
4123  	tcp_connect_init(sk);
4124  
4125  	if (unlikely(tp->repair)) {
4126  		tcp_finish_connect(sk, NULL);
4127  		return 0;
4128  	}
4129  
4130  	buff = tcp_stream_alloc_skb(sk, sk->sk_allocation, true);
4131  	if (unlikely(!buff))
4132  		return -ENOBUFS;
4133  
4134  	tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
4135  	tcp_mstamp_refresh(tp);
4136  	tp->retrans_stamp = tcp_time_stamp_ts(tp);
4137  	tcp_connect_queue_skb(sk, buff);
4138  	tcp_ecn_send_syn(sk, buff);
4139  	tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
4140  
4141  	/* Send off SYN; include data in Fast Open. */
4142  	err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
4143  	      tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
4144  	if (err == -ECONNREFUSED)
4145  		return err;
4146  
4147  	/* We change tp->snd_nxt after the tcp_transmit_skb() call
4148  	 * in order to make this packet get counted in tcpOutSegs.
4149  	 */
4150  	WRITE_ONCE(tp->snd_nxt, tp->write_seq);
4151  	tp->pushed_seq = tp->write_seq;
4152  	buff = tcp_send_head(sk);
4153  	if (unlikely(buff)) {
4154  		WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(buff)->seq);
4155  		tp->pushed_seq	= TCP_SKB_CB(buff)->seq;
4156  	}
4157  	TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
4158  
4159  	/* Timer for repeating the SYN until an answer. */
4160  	inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
4161  				  inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
4162  	return 0;
4163  }
4164  EXPORT_SYMBOL(tcp_connect);
4165  
tcp_delack_max(const struct sock * sk)4166  u32 tcp_delack_max(const struct sock *sk)
4167  {
4168  	u32 delack_from_rto_min = max(tcp_rto_min(sk), 2) - 1;
4169  
4170  	return min(inet_csk(sk)->icsk_delack_max, delack_from_rto_min);
4171  }
4172  
4173  /* Send out a delayed ack, the caller does the policy checking
4174   * to see if we should even be here.  See tcp_input.c:tcp_ack_snd_check()
4175   * for details.
4176   */
tcp_send_delayed_ack(struct sock * sk)4177  void tcp_send_delayed_ack(struct sock *sk)
4178  {
4179  	struct inet_connection_sock *icsk = inet_csk(sk);
4180  	int ato = icsk->icsk_ack.ato;
4181  	unsigned long timeout;
4182  
4183  	if (ato > TCP_DELACK_MIN) {
4184  		const struct tcp_sock *tp = tcp_sk(sk);
4185  		int max_ato = HZ / 2;
4186  
4187  		if (inet_csk_in_pingpong_mode(sk) ||
4188  		    (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
4189  			max_ato = TCP_DELACK_MAX;
4190  
4191  		/* Slow path, intersegment interval is "high". */
4192  
4193  		/* If some rtt estimate is known, use it to bound delayed ack.
4194  		 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
4195  		 * directly.
4196  		 */
4197  		if (tp->srtt_us) {
4198  			int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
4199  					TCP_DELACK_MIN);
4200  
4201  			if (rtt < max_ato)
4202  				max_ato = rtt;
4203  		}
4204  
4205  		ato = min(ato, max_ato);
4206  	}
4207  
4208  	ato = min_t(u32, ato, tcp_delack_max(sk));
4209  
4210  	/* Stay within the limit we were given */
4211  	timeout = jiffies + ato;
4212  
4213  	/* Use new timeout only if there wasn't a older one earlier. */
4214  	if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
4215  		/* If delack timer is about to expire, send ACK now. */
4216  		if (time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
4217  			tcp_send_ack(sk);
4218  			return;
4219  		}
4220  
4221  		if (!time_before(timeout, icsk->icsk_ack.timeout))
4222  			timeout = icsk->icsk_ack.timeout;
4223  	}
4224  	icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
4225  	icsk->icsk_ack.timeout = timeout;
4226  	sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
4227  }
4228  
4229  /* This routine sends an ack and also updates the window. */
__tcp_send_ack(struct sock * sk,u32 rcv_nxt)4230  void __tcp_send_ack(struct sock *sk, u32 rcv_nxt)
4231  {
4232  	struct sk_buff *buff;
4233  
4234  	/* If we have been reset, we may not send again. */
4235  	if (sk->sk_state == TCP_CLOSE)
4236  		return;
4237  
4238  	/* We are not putting this on the write queue, so
4239  	 * tcp_transmit_skb() will set the ownership to this
4240  	 * sock.
4241  	 */
4242  	buff = alloc_skb(MAX_TCP_HEADER,
4243  			 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
4244  	if (unlikely(!buff)) {
4245  		struct inet_connection_sock *icsk = inet_csk(sk);
4246  		unsigned long delay;
4247  
4248  		delay = TCP_DELACK_MAX << icsk->icsk_ack.retry;
4249  		if (delay < TCP_RTO_MAX)
4250  			icsk->icsk_ack.retry++;
4251  		inet_csk_schedule_ack(sk);
4252  		icsk->icsk_ack.ato = TCP_ATO_MIN;
4253  		inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, delay, TCP_RTO_MAX);
4254  		return;
4255  	}
4256  
4257  	/* Reserve space for headers and prepare control bits. */
4258  	skb_reserve(buff, MAX_TCP_HEADER);
4259  	tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
4260  
4261  	/* We do not want pure acks influencing TCP Small Queues or fq/pacing
4262  	 * too much.
4263  	 * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
4264  	 */
4265  	skb_set_tcp_pure_ack(buff);
4266  
4267  	/* Send it off, this clears delayed acks for us. */
4268  	__tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0, rcv_nxt);
4269  }
4270  EXPORT_SYMBOL_GPL(__tcp_send_ack);
4271  
tcp_send_ack(struct sock * sk)4272  void tcp_send_ack(struct sock *sk)
4273  {
4274  	__tcp_send_ack(sk, tcp_sk(sk)->rcv_nxt);
4275  }
4276  
4277  /* This routine sends a packet with an out of date sequence
4278   * number. It assumes the other end will try to ack it.
4279   *
4280   * Question: what should we make while urgent mode?
4281   * 4.4BSD forces sending single byte of data. We cannot send
4282   * out of window data, because we have SND.NXT==SND.MAX...
4283   *
4284   * Current solution: to send TWO zero-length segments in urgent mode:
4285   * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
4286   * out-of-date with SND.UNA-1 to probe window.
4287   */
tcp_xmit_probe_skb(struct sock * sk,int urgent,int mib)4288  static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib)
4289  {
4290  	struct tcp_sock *tp = tcp_sk(sk);
4291  	struct sk_buff *skb;
4292  
4293  	/* We don't queue it, tcp_transmit_skb() sets ownership. */
4294  	skb = alloc_skb(MAX_TCP_HEADER,
4295  			sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
4296  	if (!skb)
4297  		return -1;
4298  
4299  	/* Reserve space for headers and set control bits. */
4300  	skb_reserve(skb, MAX_TCP_HEADER);
4301  	/* Use a previous sequence.  This should cause the other
4302  	 * end to send an ack.  Don't queue or clone SKB, just
4303  	 * send it.
4304  	 */
4305  	tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
4306  	NET_INC_STATS(sock_net(sk), mib);
4307  	return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0);
4308  }
4309  
4310  /* Called from setsockopt( ... TCP_REPAIR ) */
tcp_send_window_probe(struct sock * sk)4311  void tcp_send_window_probe(struct sock *sk)
4312  {
4313  	if (sk->sk_state == TCP_ESTABLISHED) {
4314  		tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
4315  		tcp_mstamp_refresh(tcp_sk(sk));
4316  		tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE);
4317  	}
4318  }
4319  
4320  /* Initiate keepalive or window probe from timer. */
tcp_write_wakeup(struct sock * sk,int mib)4321  int tcp_write_wakeup(struct sock *sk, int mib)
4322  {
4323  	struct tcp_sock *tp = tcp_sk(sk);
4324  	struct sk_buff *skb;
4325  
4326  	if (sk->sk_state == TCP_CLOSE)
4327  		return -1;
4328  
4329  	skb = tcp_send_head(sk);
4330  	if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
4331  		int err;
4332  		unsigned int mss = tcp_current_mss(sk);
4333  		unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
4334  
4335  		if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
4336  			tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
4337  
4338  		/* We are probing the opening of a window
4339  		 * but the window size is != 0
4340  		 * must have been a result SWS avoidance ( sender )
4341  		 */
4342  		if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
4343  		    skb->len > mss) {
4344  			seg_size = min(seg_size, mss);
4345  			TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
4346  			if (tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
4347  					 skb, seg_size, mss, GFP_ATOMIC))
4348  				return -1;
4349  		} else if (!tcp_skb_pcount(skb))
4350  			tcp_set_skb_tso_segs(skb, mss);
4351  
4352  		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
4353  		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
4354  		if (!err)
4355  			tcp_event_new_data_sent(sk, skb);
4356  		return err;
4357  	} else {
4358  		if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
4359  			tcp_xmit_probe_skb(sk, 1, mib);
4360  		return tcp_xmit_probe_skb(sk, 0, mib);
4361  	}
4362  }
4363  
4364  /* A window probe timeout has occurred.  If window is not closed send
4365   * a partial packet else a zero probe.
4366   */
tcp_send_probe0(struct sock * sk)4367  void tcp_send_probe0(struct sock *sk)
4368  {
4369  	struct inet_connection_sock *icsk = inet_csk(sk);
4370  	struct tcp_sock *tp = tcp_sk(sk);
4371  	struct net *net = sock_net(sk);
4372  	unsigned long timeout;
4373  	int err;
4374  
4375  	err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE);
4376  
4377  	if (tp->packets_out || tcp_write_queue_empty(sk)) {
4378  		/* Cancel probe timer, if it is not required. */
4379  		icsk->icsk_probes_out = 0;
4380  		icsk->icsk_backoff = 0;
4381  		icsk->icsk_probes_tstamp = 0;
4382  		return;
4383  	}
4384  
4385  	icsk->icsk_probes_out++;
4386  	if (err <= 0) {
4387  		if (icsk->icsk_backoff < READ_ONCE(net->ipv4.sysctl_tcp_retries2))
4388  			icsk->icsk_backoff++;
4389  		timeout = tcp_probe0_when(sk, TCP_RTO_MAX);
4390  	} else {
4391  		/* If packet was not sent due to local congestion,
4392  		 * Let senders fight for local resources conservatively.
4393  		 */
4394  		timeout = TCP_RESOURCE_PROBE_INTERVAL;
4395  	}
4396  
4397  	timeout = tcp_clamp_probe0_to_user_timeout(sk, timeout);
4398  	tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, timeout, TCP_RTO_MAX);
4399  }
4400  
tcp_rtx_synack(const struct sock * sk,struct request_sock * req)4401  int tcp_rtx_synack(const struct sock *sk, struct request_sock *req)
4402  {
4403  	const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific;
4404  	struct flowi fl;
4405  	int res;
4406  
4407  	/* Paired with WRITE_ONCE() in sock_setsockopt() */
4408  	if (READ_ONCE(sk->sk_txrehash) == SOCK_TXREHASH_ENABLED)
4409  		WRITE_ONCE(tcp_rsk(req)->txhash, net_tx_rndhash());
4410  	res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL,
4411  				  NULL);
4412  	if (!res) {
4413  		TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
4414  		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
4415  		if (unlikely(tcp_passive_fastopen(sk))) {
4416  			/* sk has const attribute because listeners are lockless.
4417  			 * However in this case, we are dealing with a passive fastopen
4418  			 * socket thus we can change total_retrans value.
4419  			 */
4420  			tcp_sk_rw(sk)->total_retrans++;
4421  		}
4422  		trace_tcp_retransmit_synack(sk, req);
4423  	}
4424  	return res;
4425  }
4426  EXPORT_SYMBOL(tcp_rtx_synack);
4427