30 static void mptcp_subflow_ops_undo_override(struct sock *ssk);
429 void mptcp_subflow_reset(struct sock *ssk)  in mptcp_subflow_reset()  argument
431 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_reset()
437 	if (ssk->sk_state == TCP_CLOSE)  in mptcp_subflow_reset()
443 	mptcp_send_active_reset_reason(ssk);  in mptcp_subflow_reset()
444 	tcp_done(ssk);  in mptcp_subflow_reset()
460 	struct sock *ssk = msk->first;  in __mptcp_sync_state()  local
462 	subflow = mptcp_subflow_ctx(ssk);  in __mptcp_sync_state()
463 	__mptcp_propagate_sndbuf(sk, ssk);  in __mptcp_sync_state()
465 		mptcp_rcv_space_init(msk, ssk);  in __mptcp_sync_state()
499 static void mptcp_propagate_state(struct sock *sk, struct sock *ssk,  in mptcp_propagate_state()  argument
511 		WRITE_ONCE(msk->wnd_end, subflow->idsn + 1 + tcp_sk(ssk)->snd_wnd);  in mptcp_propagate_state()
516 		__mptcp_sync_state(sk, ssk->sk_state);  in mptcp_propagate_state()
518 		msk->pending_state = ssk->sk_state;  in mptcp_propagate_state()
781 void mptcp_subflow_drop_ctx(struct sock *ssk)  in mptcp_subflow_drop_ctx()  argument
783 	struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);  in mptcp_subflow_drop_ctx()
788 	list_del(&mptcp_subflow_ctx(ssk)->node);  in mptcp_subflow_drop_ctx()
789 	if (inet_csk(ssk)->icsk_ulp_ops) {  in mptcp_subflow_drop_ctx()
790 		subflow_ulp_fallback(ssk, ctx);  in mptcp_subflow_drop_ctx()
983 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb)  in skb_is_fully_mapped()  argument
985 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in skb_is_fully_mapped()
988 	skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq;  in skb_is_fully_mapped()
998 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb)  in validate_mapping()  argument
1000 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in validate_mapping()
1001 	u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;  in validate_mapping()
1019 static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *skb,  in validate_data_csum()  argument
1022 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in validate_data_csum()
1041 		seq = tcp_sk(ssk)->copied_seq + subflow->map_csum_len;  in validate_data_csum()
1061 		if (skb_queue_is_last(&ssk->sk_receive_queue, skb)) {  in validate_data_csum()
1066 			if (unlikely(ssk->sk_state == TCP_CLOSE))  in validate_data_csum()
1067 				while ((skb = skb_peek(&ssk->sk_receive_queue)))  in validate_data_csum()
1068 					sk_eat_skb(ssk, skb);  in validate_data_csum()
1090 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR);  in validate_data_csum()
1098 static enum mapping_status get_mapping_status(struct sock *ssk,  in get_mapping_status()  argument
1101 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in get_mapping_status()
1108 	skb = skb_peek(&ssk->sk_receive_queue);  in get_mapping_status()
1112 	if (mptcp_check_fallback(ssk))  in get_mapping_status()
1127 			sk_eat_skb(ssk, skb);  in get_mapping_status()
1142 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX);  in get_mapping_status()
1202 		if (skb_is_fully_mapped(ssk, skb)) {  in get_mapping_status()
1203 			MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH);  in get_mapping_status()
1234 	if (!validate_mapping(ssk, skb)) {  in get_mapping_status()
1235 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSTCPMISMATCH);  in get_mapping_status()
1242 	return validate_data_csum(ssk, skb, csum_reqd);  in get_mapping_status()
1245 static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb,  in mptcp_subflow_discard_data()  argument
1248 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_discard_data()
1250 	struct tcp_sock *tp = tcp_sk(ssk);  in mptcp_subflow_discard_data()
1262 	MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA);  in mptcp_subflow_discard_data()
1263 	tcp_sk(ssk)->copied_seq += incr;  in mptcp_subflow_discard_data()
1266 	if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq))  in mptcp_subflow_discard_data()
1267 		sk_eat_skb(ssk, skb);  in mptcp_subflow_discard_data()
1273 static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk)  in subflow_sched_work_if_closed()  argument
1277 	if (likely(ssk->sk_state != TCP_CLOSE &&  in subflow_sched_work_if_closed()
1278 		   (ssk->sk_state != TCP_CLOSE_WAIT ||  in subflow_sched_work_if_closed()
1282 	if (skb_queue_empty(&ssk->sk_receive_queue) &&  in subflow_sched_work_if_closed()
1299 static void mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk)  in mptcp_subflow_fail()  argument
1301 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_fail()
1305 	if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first)))  in mptcp_subflow_fail()
1321 	tcp_send_ack(ssk);  in mptcp_subflow_fail()
1326 static bool subflow_check_data_avail(struct sock *ssk)  in subflow_check_data_avail()  argument
1328 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in subflow_check_data_avail()
1333 	if (!skb_peek(&ssk->sk_receive_queue))  in subflow_check_data_avail()
1343 		status = get_mapping_status(ssk, msk);  in subflow_check_data_avail()
1344 		trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue));  in subflow_check_data_avail()
1352 		skb = skb_peek(&ssk->sk_receive_queue);  in subflow_check_data_avail()
1364 			mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq);  in subflow_check_data_avail()
1374 	subflow_sched_work_if_closed(msk, ssk);  in subflow_check_data_avail()
1389 			mptcp_subflow_fail(msk, ssk);  in subflow_check_data_avail()
1402 			WRITE_ONCE(ssk->sk_err, EBADMSG);  in subflow_check_data_avail()
1403 			tcp_set_state(ssk, TCP_CLOSE);  in subflow_check_data_avail()
1404 			while ((skb = skb_peek(&ssk->sk_receive_queue)))  in subflow_check_data_avail()
1405 				sk_eat_skb(ssk, skb);  in subflow_check_data_avail()
1406 			mptcp_send_active_reset_reason(ssk);  in subflow_check_data_avail()
1411 		mptcp_do_fallback(ssk);  in subflow_check_data_avail()
1414 	skb = skb_peek(&ssk->sk_receive_queue);  in subflow_check_data_avail()
1418 	subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;  in subflow_check_data_avail()
1450 void mptcp_space(const struct sock *ssk, int *space, int *full_space)  in mptcp_space()  argument
1452 	const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_space()
1459 static void subflow_error_report(struct sock *ssk)  in subflow_error_report()  argument
1461 	struct sock *sk = mptcp_subflow_ctx(ssk)->conn;  in subflow_error_report()
1516 static void subflow_write_space(struct sock *ssk)  in subflow_write_space()  argument
1518 	struct sock *sk = mptcp_subflow_ctx(ssk)->conn;  in subflow_write_space()
1520 	mptcp_propagate_sndbuf(sk, ssk);  in subflow_write_space()
1595 	struct sock *ssk;  in __mptcp_subflow_connect()  local
1611 	ssk = sf->sk;  in __mptcp_subflow_connect()
1612 	subflow = mptcp_subflow_ctx(ssk);  in __mptcp_subflow_connect()
1635 	mptcp_info2sockaddr(&local->addr, &addr, ssk->sk_family);  in __mptcp_subflow_connect()
1642 	ssk->sk_bound_dev_if = local->ifindex;  in __mptcp_subflow_connect()
1659 	mptcp_info2sockaddr(remote, &addr, ssk->sk_family);  in __mptcp_subflow_connect()
1661 	sock_hold(ssk);  in __mptcp_subflow_connect()
1674 	mptcp_sock_graft(ssk, sk->sk_socket);  in __mptcp_subflow_connect()
1720 static void mptcp_subflow_ops_override(struct sock *ssk)  in mptcp_subflow_ops_override()  argument
1723 	if (ssk->sk_prot == &tcpv6_prot)  in mptcp_subflow_ops_override()
1724 		ssk->sk_prot = &tcpv6_prot_override;  in mptcp_subflow_ops_override()
1727 		ssk->sk_prot = &tcp_prot_override;  in mptcp_subflow_ops_override()
1730 static void mptcp_subflow_ops_undo_override(struct sock *ssk)  in mptcp_subflow_ops_undo_override()  argument
1733 	if (ssk->sk_prot == &tcpv6_prot_override)  in mptcp_subflow_ops_undo_override()
1734 		ssk->sk_prot = &tcpv6_prot;  in mptcp_subflow_ops_undo_override()
1737 		ssk->sk_prot = &tcp_prot;  in mptcp_subflow_ops_undo_override()
1886 	struct sock *sk, *ssk;  in mptcp_subflow_queue_clean()  local
1908 		ssk = req->sk;  in mptcp_subflow_queue_clean()
1909 		if (!sk_is_mptcp(ssk))  in mptcp_subflow_queue_clean()
1912 		subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_queue_clean()
1992 static void subflow_ulp_release(struct sock *ssk)  in subflow_ulp_release()  argument
1994 	struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);  in subflow_ulp_release()
2018 	mptcp_subflow_ops_undo_override(ssk);  in subflow_ulp_release()
2080 static void tcp_release_cb_override(struct sock *ssk)  in tcp_release_cb_override()  argument
2082 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in tcp_release_cb_override()
2091 		mptcp_subflow_process_delegated(ssk, status);  in tcp_release_cb_override()
2093 	tcp_release_cb(ssk);  in tcp_release_cb_override()
2096 static int tcp_abort_override(struct sock *ssk, int err)  in tcp_abort_override()  argument
2101 	if (inet_sk_state_load(ssk) == TCP_LISTEN)  in tcp_abort_override()
2104 	return tcp_abort(ssk, err);  in tcp_abort_override()