Lines Matching refs:psock
55 static void kcm_abort_tx_psock(struct kcm_psock *psock, int err, in kcm_abort_tx_psock() argument
58 struct sock *csk = psock->sk; in kcm_abort_tx_psock()
59 struct kcm_mux *mux = psock->mux; in kcm_abort_tx_psock()
65 if (psock->tx_stopped) { in kcm_abort_tx_psock()
70 psock->tx_stopped = 1; in kcm_abort_tx_psock()
71 KCM_STATS_INCR(psock->stats.tx_aborts); in kcm_abort_tx_psock()
73 if (!psock->tx_kcm) { in kcm_abort_tx_psock()
75 list_del(&psock->psock_avail_list); in kcm_abort_tx_psock()
84 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_abort_tx_psock()
95 struct kcm_psock *psock) in kcm_update_rx_mux_stats() argument
98 psock->strp.stats.bytes - in kcm_update_rx_mux_stats()
99 psock->saved_rx_bytes); in kcm_update_rx_mux_stats()
101 psock->strp.stats.msgs - psock->saved_rx_msgs; in kcm_update_rx_mux_stats()
102 psock->saved_rx_msgs = psock->strp.stats.msgs; in kcm_update_rx_mux_stats()
103 psock->saved_rx_bytes = psock->strp.stats.bytes; in kcm_update_rx_mux_stats()
107 struct kcm_psock *psock) in kcm_update_tx_mux_stats() argument
110 psock->stats.tx_bytes - psock->saved_tx_bytes); in kcm_update_tx_mux_stats()
112 psock->stats.tx_msgs - psock->saved_tx_msgs; in kcm_update_tx_mux_stats()
113 psock->saved_tx_msgs = psock->stats.tx_msgs; in kcm_update_tx_mux_stats()
114 psock->saved_tx_bytes = psock->stats.tx_bytes; in kcm_update_tx_mux_stats()
126 struct kcm_psock *psock; in kcm_rcv_ready() local
142 psock = list_first_entry(&mux->psocks_ready, struct kcm_psock, in kcm_rcv_ready()
145 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) { in kcm_rcv_ready()
154 list_del(&psock->psock_ready_list); in kcm_rcv_ready()
155 psock->ready_rx_msg = NULL; in kcm_rcv_ready()
159 strp_unpause(&psock->strp); in kcm_rcv_ready()
160 strp_check_rcv(&psock->strp); in kcm_rcv_ready()
254 static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock, in reserve_rx_kcm() argument
257 struct kcm_mux *mux = psock->mux; in reserve_rx_kcm()
260 WARN_ON(psock->ready_rx_msg); in reserve_rx_kcm()
262 if (psock->rx_kcm) in reserve_rx_kcm()
263 return psock->rx_kcm; in reserve_rx_kcm()
267 if (psock->rx_kcm) { in reserve_rx_kcm()
269 return psock->rx_kcm; in reserve_rx_kcm()
272 kcm_update_rx_mux_stats(mux, psock); in reserve_rx_kcm()
275 psock->ready_rx_msg = head; in reserve_rx_kcm()
276 strp_pause(&psock->strp); in reserve_rx_kcm()
277 list_add_tail(&psock->psock_ready_list, in reserve_rx_kcm()
289 psock->rx_kcm = kcm; in reserve_rx_kcm()
291 WRITE_ONCE(kcm->rx_psock, psock); in reserve_rx_kcm()
306 static void unreserve_rx_kcm(struct kcm_psock *psock, in unreserve_rx_kcm() argument
309 struct kcm_sock *kcm = psock->rx_kcm; in unreserve_rx_kcm()
310 struct kcm_mux *mux = psock->mux; in unreserve_rx_kcm()
317 psock->rx_kcm = NULL; in unreserve_rx_kcm()
351 struct kcm_psock *psock; in psock_data_ready() local
357 psock = (struct kcm_psock *)sk->sk_user_data; in psock_data_ready()
358 if (likely(psock)) in psock_data_ready()
359 strp_data_ready(&psock->strp); in psock_data_ready()
367 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_rcv_strparser() local
371 kcm = reserve_rx_kcm(psock, skb); in kcm_rcv_strparser()
381 unreserve_rx_kcm(psock, false); in kcm_rcv_strparser()
388 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_parse_func_strparser() local
389 struct bpf_prog *prog = psock->bpf_prog; in kcm_parse_func_strparser()
398 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_read_sock_done() local
400 unreserve_rx_kcm(psock, true); in kcm_read_sock_done()
417 struct kcm_psock *psock; in psock_write_space() local
423 psock = (struct kcm_psock *)sk->sk_user_data; in psock_write_space()
424 if (unlikely(!psock)) in psock_write_space()
426 mux = psock->mux; in psock_write_space()
431 kcm = psock->tx_kcm; in psock_write_space()
446 struct kcm_psock *psock; in reserve_psock() local
448 psock = kcm->tx_psock; in reserve_psock()
452 if (psock) { in reserve_psock()
454 if (unlikely(psock->tx_stopped)) in reserve_psock()
465 psock = kcm->tx_psock; in reserve_psock()
466 if (unlikely(psock)) { in reserve_psock()
473 psock = list_first_entry(&mux->psocks_avail, in reserve_psock()
476 list_del(&psock->psock_avail_list); in reserve_psock()
481 kcm->tx_psock = psock; in reserve_psock()
482 psock->tx_kcm = kcm; in reserve_psock()
483 KCM_STATS_INCR(psock->stats.reserved); in reserve_psock()
492 return psock; in reserve_psock()
496 static void psock_now_avail(struct kcm_psock *psock) in psock_now_avail() argument
498 struct kcm_mux *mux = psock->mux; in psock_now_avail()
502 list_add_tail(&psock->psock_avail_list, in psock_now_avail()
510 psock->tx_kcm = kcm; in psock_now_avail()
517 kcm->tx_psock = psock; in psock_now_avail()
518 KCM_STATS_INCR(psock->stats.reserved); in psock_now_avail()
526 struct kcm_psock *psock; in unreserve_psock() local
531 psock = kcm->tx_psock; in unreserve_psock()
533 if (WARN_ON(!psock)) { in unreserve_psock()
540 kcm_update_tx_mux_stats(mux, psock); in unreserve_psock()
545 psock->tx_kcm = NULL; in unreserve_psock()
546 KCM_STATS_INCR(psock->stats.unreserved); in unreserve_psock()
548 if (unlikely(psock->tx_stopped)) { in unreserve_psock()
549 if (psock->done) { in unreserve_psock()
551 list_del(&psock->psock_list); in unreserve_psock()
553 sock_put(psock->sk); in unreserve_psock()
554 fput(psock->sk->sk_socket->file); in unreserve_psock()
555 kmem_cache_free(kcm_psockp, psock); in unreserve_psock()
565 psock_now_avail(psock); in unreserve_psock()
586 struct kcm_psock *psock; in kcm_write_msgs() local
591 psock = kcm->tx_psock; in kcm_write_msgs()
592 if (unlikely(psock && psock->tx_stopped)) { in kcm_write_msgs()
615 psock = reserve_psock(kcm); in kcm_write_msgs()
616 if (!psock) in kcm_write_msgs()
623 if (WARN_ON(!psock)) { in kcm_write_msgs()
646 ret = sock_sendmsg(psock->sk->sk_socket, &msg); in kcm_write_msgs()
662 kcm_abort_tx_psock(psock, ret ? -ret : EPIPE, in kcm_write_msgs()
665 psock = NULL; in kcm_write_msgs()
675 KCM_STATS_ADD(psock->stats.tx_bytes, ret); in kcm_write_msgs()
695 KCM_STATS_INCR(psock->stats.tx_msgs); in kcm_write_msgs()
701 if (psock) in kcm_write_msgs()
1223 struct kcm_psock *psock = NULL, *tpsock; in kcm_attach() local
1252 psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL); in kcm_attach()
1253 if (!psock) { in kcm_attach()
1258 psock->mux = mux; in kcm_attach()
1259 psock->sk = csk; in kcm_attach()
1260 psock->bpf_prog = prog; in kcm_attach()
1269 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1274 err = strp_init(&psock->strp, csk, &cb); in kcm_attach()
1277 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1281 psock->save_data_ready = csk->sk_data_ready; in kcm_attach()
1282 psock->save_write_space = csk->sk_write_space; in kcm_attach()
1283 psock->save_state_change = csk->sk_state_change; in kcm_attach()
1284 csk->sk_user_data = psock; in kcm_attach()
1303 list_add(&psock->psock_list, head); in kcm_attach()
1304 psock->index = index; in kcm_attach()
1308 psock_now_avail(psock); in kcm_attach()
1312 strp_check_rcv(&psock->strp); in kcm_attach()
1350 static void kcm_unattach(struct kcm_psock *psock) in kcm_unattach() argument
1352 struct sock *csk = psock->sk; in kcm_unattach()
1353 struct kcm_mux *mux = psock->mux; in kcm_unattach()
1362 csk->sk_data_ready = psock->save_data_ready; in kcm_unattach()
1363 csk->sk_write_space = psock->save_write_space; in kcm_unattach()
1364 csk->sk_state_change = psock->save_state_change; in kcm_unattach()
1365 strp_stop(&psock->strp); in kcm_unattach()
1367 if (WARN_ON(psock->rx_kcm)) { in kcm_unattach()
1378 if (psock->ready_rx_msg) { in kcm_unattach()
1379 list_del(&psock->psock_ready_list); in kcm_unattach()
1380 kfree_skb(psock->ready_rx_msg); in kcm_unattach()
1381 psock->ready_rx_msg = NULL; in kcm_unattach()
1391 strp_done(&psock->strp); in kcm_unattach()
1394 bpf_prog_put(psock->bpf_prog); in kcm_unattach()
1398 aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats); in kcm_unattach()
1399 save_strp_stats(&psock->strp, &mux->aggregate_strp_stats); in kcm_unattach()
1403 if (psock->tx_kcm) { in kcm_unattach()
1415 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_unattach()
1418 if (!psock->tx_kcm) { in kcm_unattach()
1422 psock->done = 1; in kcm_unattach()
1428 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_unattach()
1432 if (!psock->tx_stopped) in kcm_unattach()
1433 list_del(&psock->psock_avail_list); in kcm_unattach()
1434 list_del(&psock->psock_list); in kcm_unattach()
1440 kmem_cache_free(kcm_psockp, psock); in kcm_unattach()
1450 struct kcm_psock *psock; in kcm_unattach_ioctl() local
1469 list_for_each_entry(psock, &mux->psocks, psock_list) { in kcm_unattach_ioctl()
1470 if (psock->sk != csk) in kcm_unattach_ioctl()
1475 if (psock->unattaching || WARN_ON(psock->done)) { in kcm_unattach_ioctl()
1480 psock->unattaching = 1; in kcm_unattach_ioctl()
1485 kcm_unattach(psock); in kcm_unattach_ioctl()
1598 struct kcm_psock *psock, *tmp_psock; in release_mux() local
1601 list_for_each_entry_safe(psock, tmp_psock, in release_mux()
1603 if (!WARN_ON(psock->unattaching)) in release_mux()
1604 kcm_unattach(psock); in release_mux()
1681 struct kcm_psock *psock; in kcm_release() local
1723 psock = kcm->tx_psock; in kcm_release()
1724 if (psock) { in kcm_release()
1729 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_release()