412 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,  in sk_msg_recvmsg()  argument
420 	msg_rx = sk_psock_peek_msg(psock);  in sk_msg_recvmsg()
471 			msg_rx = sk_psock_next_msg(psock, msg_rx);  in sk_msg_recvmsg()
479 			msg_rx = sk_psock_dequeue_msg(psock);  in sk_msg_recvmsg()
482 		msg_rx = sk_psock_peek_msg(psock);  in sk_msg_recvmsg()
491 	struct sk_psock *psock;  in sk_msg_is_readable()  local
495 	psock = sk_psock(sk);  in sk_msg_is_readable()
496 	if (likely(psock))  in sk_msg_is_readable()
497 		empty = list_empty(&psock->ingress_msg);  in sk_msg_is_readable()
528 					struct sk_psock *psock,  in sk_psock_skb_ingress_enqueue()  argument
555 	sk_psock_queue_msg(psock, msg);  in sk_psock_skb_ingress_enqueue()
556 	sk_psock_data_ready(sk, psock);  in sk_psock_skb_ingress_enqueue()
560 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
563 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_skb_ingress()  argument
566 	struct sock *sk = psock->sk;  in sk_psock_skb_ingress()
575 		return sk_psock_skb_ingress_self(psock, skb, off, len);  in sk_psock_skb_ingress()
587 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);  in sk_psock_skb_ingress()
597 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_skb_ingress_self()  argument
601 	struct sock *sk = psock->sk;  in sk_psock_skb_ingress_self()
607 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);  in sk_psock_skb_ingress_self()
613 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_handle_skb()  argument
619 		if (!sock_writeable(psock->sk))  in sk_psock_handle_skb()
621 		return skb_send_sock(psock->sk, skb, off, len);  in sk_psock_handle_skb()
624 	err = sk_psock_skb_ingress(psock, skb, off, len);  in sk_psock_handle_skb()
630 static void sk_psock_skb_state(struct sk_psock *psock,  in sk_psock_skb_state()  argument
634 	spin_lock_bh(&psock->ingress_lock);  in sk_psock_skb_state()
635 	if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {  in sk_psock_skb_state()
639 	spin_unlock_bh(&psock->ingress_lock);  in sk_psock_skb_state()
645 	struct sk_psock *psock = container_of(dwork, struct sk_psock, work);  in sk_psock_backlog()  local
646 	struct sk_psock_work_state *state = &psock->work_state;  in sk_psock_backlog()
652 	mutex_lock(&psock->work_mutex);  in sk_psock_backlog()
658 	while ((skb = skb_peek(&psock->ingress_skb))) {  in sk_psock_backlog()
671 			if (!sock_flag(psock->sk, SOCK_DEAD))  in sk_psock_backlog()
672 				ret = sk_psock_handle_skb(psock, skb, off,  in sk_psock_backlog()
676 					sk_psock_skb_state(psock, state, len, off);  in sk_psock_backlog()
681 					if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_backlog()
682 						schedule_delayed_work(&psock->work, 1);  in sk_psock_backlog()
686 				sk_psock_report_error(psock, ret ? -ret : EPIPE);  in sk_psock_backlog()
687 				sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_backlog()
694 		skb = skb_dequeue(&psock->ingress_skb);  in sk_psock_backlog()
698 	mutex_unlock(&psock->work_mutex);  in sk_psock_backlog()
703 	struct sk_psock *psock;  in sk_psock_init()  local
709 		psock = ERR_PTR(-EINVAL);  in sk_psock_init()
714 		psock = ERR_PTR(-EBUSY);  in sk_psock_init()
718 	psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);  in sk_psock_init()
719 	if (!psock) {  in sk_psock_init()
720 		psock = ERR_PTR(-ENOMEM);  in sk_psock_init()
725 	psock->sk = sk;  in sk_psock_init()
726 	psock->eval = __SK_NONE;  in sk_psock_init()
727 	psock->sk_proto = prot;  in sk_psock_init()
728 	psock->saved_unhash = prot->unhash;  in sk_psock_init()
729 	psock->saved_destroy = prot->destroy;  in sk_psock_init()
730 	psock->saved_close = prot->close;  in sk_psock_init()
731 	psock->saved_write_space = sk->sk_write_space;  in sk_psock_init()
733 	INIT_LIST_HEAD(&psock->link);  in sk_psock_init()
734 	spin_lock_init(&psock->link_lock);  in sk_psock_init()
736 	INIT_DELAYED_WORK(&psock->work, sk_psock_backlog);  in sk_psock_init()
737 	mutex_init(&psock->work_mutex);  in sk_psock_init()
738 	INIT_LIST_HEAD(&psock->ingress_msg);  in sk_psock_init()
739 	spin_lock_init(&psock->ingress_lock);  in sk_psock_init()
740 	skb_queue_head_init(&psock->ingress_skb);  in sk_psock_init()
742 	sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_init()
743 	refcount_set(&psock->refcnt, 1);  in sk_psock_init()
745 	__rcu_assign_sk_user_data_with_flags(sk, psock,  in sk_psock_init()
752 	return psock;  in sk_psock_init()
756 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)  in sk_psock_link_pop()  argument
760 	spin_lock_bh(&psock->link_lock);  in sk_psock_link_pop()
761 	link = list_first_entry_or_null(&psock->link, struct sk_psock_link,  in sk_psock_link_pop()
765 	spin_unlock_bh(&psock->link_lock);  in sk_psock_link_pop()
769 static void __sk_psock_purge_ingress_msg(struct sk_psock *psock)  in __sk_psock_purge_ingress_msg()  argument
773 	list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {  in __sk_psock_purge_ingress_msg()
775 		sk_msg_free(psock->sk, msg);  in __sk_psock_purge_ingress_msg()
780 static void __sk_psock_zap_ingress(struct sk_psock *psock)  in __sk_psock_zap_ingress()  argument
784 	while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {  in __sk_psock_zap_ingress()
786 		sock_drop(psock->sk, skb);  in __sk_psock_zap_ingress()
788 	__sk_psock_purge_ingress_msg(psock);  in __sk_psock_zap_ingress()
791 static void sk_psock_link_destroy(struct sk_psock *psock)  in sk_psock_link_destroy()  argument
795 	list_for_each_entry_safe(link, tmp, &psock->link, list) {  in sk_psock_link_destroy()
801 void sk_psock_stop(struct sk_psock *psock)  in sk_psock_stop()  argument
803 	spin_lock_bh(&psock->ingress_lock);  in sk_psock_stop()
804 	sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_stop()
805 	sk_psock_cork_free(psock);  in sk_psock_stop()
806 	spin_unlock_bh(&psock->ingress_lock);  in sk_psock_stop()
809 static void sk_psock_done_strp(struct sk_psock *psock);
813 	struct sk_psock *psock = container_of(to_rcu_work(work),  in sk_psock_destroy()  local
817 	sk_psock_done_strp(psock);  in sk_psock_destroy()
819 	cancel_delayed_work_sync(&psock->work);  in sk_psock_destroy()
820 	__sk_psock_zap_ingress(psock);  in sk_psock_destroy()
821 	mutex_destroy(&psock->work_mutex);  in sk_psock_destroy()
823 	psock_progs_drop(&psock->progs);  in sk_psock_destroy()
825 	sk_psock_link_destroy(psock);  in sk_psock_destroy()
826 	sk_psock_cork_free(psock);  in sk_psock_destroy()
828 	if (psock->sk_redir)  in sk_psock_destroy()
829 		sock_put(psock->sk_redir);  in sk_psock_destroy()
830 	if (psock->sk_pair)  in sk_psock_destroy()
831 		sock_put(psock->sk_pair);  in sk_psock_destroy()
832 	sock_put(psock->sk);  in sk_psock_destroy()
833 	kfree(psock);  in sk_psock_destroy()
836 void sk_psock_drop(struct sock *sk, struct sk_psock *psock)  in sk_psock_drop()  argument
839 	sk_psock_restore_proto(sk, psock);  in sk_psock_drop()
841 	if (psock->progs.stream_parser)  in sk_psock_drop()
842 		sk_psock_stop_strp(sk, psock);  in sk_psock_drop()
843 	else if (psock->progs.stream_verdict || psock->progs.skb_verdict)  in sk_psock_drop()
844 		sk_psock_stop_verdict(sk, psock);  in sk_psock_drop()
847 	sk_psock_stop(psock);  in sk_psock_drop()
849 	INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);  in sk_psock_drop()
850 	queue_rcu_work(system_wq, &psock->rwork);  in sk_psock_drop()
867 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,  in sk_psock_msg_verdict()  argument
874 	prog = READ_ONCE(psock->progs.msg_parser);  in sk_psock_msg_verdict()
884 	psock->apply_bytes = msg->apply_bytes;  in sk_psock_msg_verdict()
886 		if (psock->sk_redir) {  in sk_psock_msg_verdict()
887 			sock_put(psock->sk_redir);  in sk_psock_msg_verdict()
888 			psock->sk_redir = NULL;  in sk_psock_msg_verdict()
894 		psock->redir_ingress = sk_msg_to_ingress(msg);  in sk_psock_msg_verdict()
895 		psock->sk_redir = msg->sk_redir;  in sk_psock_msg_verdict()
896 		sock_hold(psock->sk_redir);  in sk_psock_msg_verdict()
956 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)  in sk_psock_tls_strp_read()  argument
962 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_tls_strp_read()
964 		skb->sk = psock->sk;  in sk_psock_tls_strp_read()
971 	sk_psock_tls_verdict_apply(skb, psock, ret);  in sk_psock_tls_strp_read()
977 static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_verdict_apply()  argument
987 		sk_other = psock->sk;  in sk_psock_verdict_apply()
989 		    !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_verdict_apply()
1000 		if (skb_queue_empty(&psock->ingress_skb)) {  in sk_psock_verdict_apply()
1009 			err = sk_psock_skb_ingress_self(psock, skb, off, len);  in sk_psock_verdict_apply()
1012 			spin_lock_bh(&psock->ingress_lock);  in sk_psock_verdict_apply()
1013 			if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {  in sk_psock_verdict_apply()
1014 				skb_queue_tail(&psock->ingress_skb, skb);  in sk_psock_verdict_apply()
1015 				schedule_delayed_work(&psock->work, 0);  in sk_psock_verdict_apply()
1018 			spin_unlock_bh(&psock->ingress_lock);  in sk_psock_verdict_apply()
1024 		tcp_eat_skb(psock->sk, skb);  in sk_psock_verdict_apply()
1025 		err = sk_psock_skb_redirect(psock, skb);  in sk_psock_verdict_apply()
1031 		tcp_eat_skb(psock->sk, skb);  in sk_psock_verdict_apply()
1032 		sock_drop(psock->sk, skb);  in sk_psock_verdict_apply()
1040 	struct sk_psock *psock;  in sk_psock_write_space()  local
1044 	psock = sk_psock(sk);  in sk_psock_write_space()
1045 	if (likely(psock)) {  in sk_psock_write_space()
1046 		if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_write_space()
1047 			schedule_delayed_work(&psock->work, 0);  in sk_psock_write_space()
1048 		write_space = psock->saved_write_space;  in sk_psock_write_space()
1058 	struct sk_psock *psock;  in sk_psock_strp_read()  local
1065 	psock = sk_psock(sk);  in sk_psock_strp_read()
1066 	if (unlikely(!psock)) {  in sk_psock_strp_read()
1070 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_strp_read()
1080 	sk_psock_verdict_apply(psock, skb, ret);  in sk_psock_strp_read()
1092 	struct sk_psock *psock = container_of(strp, struct sk_psock, strp);  in sk_psock_strp_parse()  local
1097 	prog = READ_ONCE(psock->progs.stream_parser);  in sk_psock_strp_parse()
1099 		skb->sk = psock->sk;  in sk_psock_strp_parse()
1110 	struct sk_psock *psock;  in sk_psock_strp_data_ready()  local
1115 	psock = sk_psock(sk);  in sk_psock_strp_data_ready()
1116 	if (likely(psock)) {  in sk_psock_strp_data_ready()
1118 			psock->saved_data_ready(sk);  in sk_psock_strp_data_ready()
1121 			strp_data_ready(&psock->strp);  in sk_psock_strp_data_ready()
1128 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_init_strp()  argument
1138 	ret = strp_init(&psock->strp, sk, &cb);  in sk_psock_init_strp()
1140 		sk_psock_set_state(psock, SK_PSOCK_RX_STRP_ENABLED);  in sk_psock_init_strp()
1145 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_start_strp()  argument
1147 	if (psock->saved_data_ready)  in sk_psock_start_strp()
1150 	psock->saved_data_ready = sk->sk_data_ready;  in sk_psock_start_strp()
1155 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_stop_strp()  argument
1157 	psock_set_prog(&psock->progs.stream_parser, NULL);  in sk_psock_stop_strp()
1159 	if (!psock->saved_data_ready)  in sk_psock_stop_strp()
1162 	sk->sk_data_ready = psock->saved_data_ready;  in sk_psock_stop_strp()
1163 	psock->saved_data_ready = NULL;  in sk_psock_stop_strp()
1164 	strp_stop(&psock->strp);  in sk_psock_stop_strp()
1167 static void sk_psock_done_strp(struct sk_psock *psock)  in sk_psock_done_strp()  argument
1170 	if (sk_psock_test_state(psock, SK_PSOCK_RX_STRP_ENABLED))  in sk_psock_done_strp()
1171 		strp_done(&psock->strp);  in sk_psock_done_strp()
1174 static void sk_psock_done_strp(struct sk_psock *psock)  in sk_psock_done_strp()  argument
1181 	struct sk_psock *psock;  in sk_psock_verdict_recv()  local
1187 	psock = sk_psock(sk);  in sk_psock_verdict_recv()
1188 	if (unlikely(!psock)) {  in sk_psock_verdict_recv()
1194 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_verdict_recv()
1196 		prog = READ_ONCE(psock->progs.skb_verdict);  in sk_psock_verdict_recv()
1203 	ret = sk_psock_verdict_apply(psock, skb, ret);  in sk_psock_verdict_recv()
1226 		struct sk_psock *psock;  in sk_psock_verdict_data_ready()  local
1229 		psock = sk_psock(sk);  in sk_psock_verdict_data_ready()
1230 		if (psock)  in sk_psock_verdict_data_ready()
1231 			sk_psock_data_ready(sk, psock);  in sk_psock_verdict_data_ready()
1236 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)  in sk_psock_start_verdict()  argument
1238 	if (psock->saved_data_ready)  in sk_psock_start_verdict()
1241 	psock->saved_data_ready = sk->sk_data_ready;  in sk_psock_start_verdict()
1246 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)  in sk_psock_stop_verdict()  argument
1248 	psock_set_prog(&psock->progs.stream_verdict, NULL);  in sk_psock_stop_verdict()
1249 	psock_set_prog(&psock->progs.skb_verdict, NULL);  in sk_psock_stop_verdict()
1251 	if (!psock->saved_data_ready)  in sk_psock_stop_verdict()
1254 	sk->sk_data_ready = psock->saved_data_ready;  in sk_psock_stop_verdict()
1255 	psock->saved_data_ready = NULL;  in sk_psock_stop_verdict()