Lines Matching refs:psock
32 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, in bpf_tcp_ingress() argument
77 sk_psock_queue_msg(psock, tmp); in bpf_tcp_ingress()
78 sk_psock_data_ready(sk, psock); in bpf_tcp_ingress()
164 struct sk_psock *psock = sk_psock_get(sk); in tcp_bpf_sendmsg_redir() local
167 if (unlikely(!psock)) in tcp_bpf_sendmsg_redir()
170 ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes) : in tcp_bpf_sendmsg_redir()
172 sk_psock_put(sk, psock); in tcp_bpf_sendmsg_redir()
178 static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock, in tcp_msg_wait_data() argument
193 !list_empty(&psock->ingress_msg) || in tcp_msg_wait_data()
200 static bool is_next_msg_fin(struct sk_psock *psock) in is_next_msg_fin() argument
206 msg_rx = sk_psock_peek_msg(psock); in is_next_msg_fin()
225 struct sk_psock *psock; in tcp_bpf_recvmsg_parser() local
236 psock = sk_psock_get(sk); in tcp_bpf_recvmsg_parser()
237 if (unlikely(!psock)) in tcp_bpf_recvmsg_parser()
263 copied = sk_msg_recvmsg(sk, psock, msg, len, flags); in tcp_bpf_recvmsg_parser()
270 bool is_fin = is_next_msg_fin(psock); in tcp_bpf_recvmsg_parser()
310 data = tcp_msg_wait_data(sk, psock, timeo); in tcp_bpf_recvmsg_parser()
315 if (data && !sk_psock_queue_empty(psock)) in tcp_bpf_recvmsg_parser()
328 sk_psock_put(sk, psock); in tcp_bpf_recvmsg_parser()
335 struct sk_psock *psock; in tcp_bpf_recvmsg() local
344 psock = sk_psock_get(sk); in tcp_bpf_recvmsg()
345 if (unlikely(!psock)) in tcp_bpf_recvmsg()
348 sk_psock_queue_empty(psock)) { in tcp_bpf_recvmsg()
349 sk_psock_put(sk, psock); in tcp_bpf_recvmsg()
354 copied = sk_msg_recvmsg(sk, psock, msg, len, flags); in tcp_bpf_recvmsg()
360 data = tcp_msg_wait_data(sk, psock, timeo); in tcp_bpf_recvmsg()
366 if (!sk_psock_queue_empty(psock)) in tcp_bpf_recvmsg()
369 sk_psock_put(sk, psock); in tcp_bpf_recvmsg()
378 sk_psock_put(sk, psock); in tcp_bpf_recvmsg()
382 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, in tcp_bpf_send_verdict() argument
392 if (psock->eval == __SK_NONE) { in tcp_bpf_send_verdict()
399 psock->eval = sk_psock_msg_verdict(sk, psock, msg); in tcp_bpf_send_verdict()
405 psock->cork_bytes = msg->cork_bytes - msg->sg.size; in tcp_bpf_send_verdict()
406 if (!psock->cork) { in tcp_bpf_send_verdict()
407 psock->cork = kzalloc(sizeof(*psock->cork), in tcp_bpf_send_verdict()
409 if (!psock->cork) in tcp_bpf_send_verdict()
412 memcpy(psock->cork, msg, sizeof(*msg)); in tcp_bpf_send_verdict()
417 if (psock->apply_bytes && psock->apply_bytes < tosend) in tcp_bpf_send_verdict()
418 tosend = psock->apply_bytes; in tcp_bpf_send_verdict()
421 switch (psock->eval) { in tcp_bpf_send_verdict()
428 sk_msg_apply_bytes(psock, tosend); in tcp_bpf_send_verdict()
431 redir_ingress = psock->redir_ingress; in tcp_bpf_send_verdict()
432 sk_redir = psock->sk_redir; in tcp_bpf_send_verdict()
433 sk_msg_apply_bytes(psock, tosend); in tcp_bpf_send_verdict()
434 if (!psock->apply_bytes) { in tcp_bpf_send_verdict()
436 eval = psock->eval; in tcp_bpf_send_verdict()
437 psock->eval = __SK_NONE; in tcp_bpf_send_verdict()
438 psock->sk_redir = NULL; in tcp_bpf_send_verdict()
440 if (psock->cork) { in tcp_bpf_send_verdict()
442 psock->cork = NULL; in tcp_bpf_send_verdict()
472 sk_msg_apply_bytes(psock, tosend); in tcp_bpf_send_verdict()
478 if (!psock->apply_bytes) { in tcp_bpf_send_verdict()
479 psock->eval = __SK_NONE; in tcp_bpf_send_verdict()
480 if (psock->sk_redir) { in tcp_bpf_send_verdict()
481 sock_put(psock->sk_redir); in tcp_bpf_send_verdict()
482 psock->sk_redir = NULL; in tcp_bpf_send_verdict()
500 struct sk_psock *psock; in tcp_bpf_sendmsg() local
508 psock = sk_psock_get(sk); in tcp_bpf_sendmsg()
509 if (unlikely(!psock)) in tcp_bpf_sendmsg()
526 if (psock->cork) { in tcp_bpf_sendmsg()
527 msg_tx = psock->cork; in tcp_bpf_sendmsg()
550 if (psock->cork_bytes) { in tcp_bpf_sendmsg()
551 if (size > psock->cork_bytes) in tcp_bpf_sendmsg()
552 psock->cork_bytes = 0; in tcp_bpf_sendmsg()
554 psock->cork_bytes -= size; in tcp_bpf_sendmsg()
555 if (psock->cork_bytes && !enospc) in tcp_bpf_sendmsg()
558 psock->eval = __SK_NONE; in tcp_bpf_sendmsg()
559 psock->cork_bytes = 0; in tcp_bpf_sendmsg()
562 err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags); in tcp_bpf_sendmsg()
571 if (msg_tx && msg_tx != psock->cork) in tcp_bpf_sendmsg()
580 sk_psock_put(sk, psock); in tcp_bpf_sendmsg()
650 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) in tcp_bpf_update_proto() argument
653 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; in tcp_bpf_update_proto()
655 if (psock->progs.stream_verdict || psock->progs.skb_verdict) { in tcp_bpf_update_proto()
666 WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash); in tcp_bpf_update_proto()
667 tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space); in tcp_bpf_update_proto()
669 sk->sk_write_space = psock->saved_write_space; in tcp_bpf_update_proto()
671 sock_replace_proto(sk, psock->sk_proto); in tcp_bpf_update_proto()
677 if (tcp_bpf_assert_proto_ops(psock->sk_proto)) in tcp_bpf_update_proto()
680 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto); in tcp_bpf_update_proto()