Lines Matching +full:sock +full:- +full:priv
1 // SPDX-License-Identifier: GPL-2.0
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
18 #include <net/sock.h>
19 #include <trace/events/sock.h>
32 static void smc_rx_wake_up(struct sock *sk) in smc_rx_wake_up()
41 wq = rcu_dereference(sk->sk_wq); in smc_rx_wake_up()
43 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | in smc_rx_wake_up()
46 if ((sk->sk_shutdown == SHUTDOWN_MASK) || in smc_rx_wake_up()
47 (sk->sk_state == SMC_CLOSED)) in smc_rx_wake_up()
62 struct smc_connection *conn = &smc->conn; in smc_rx_update_consumer()
63 struct sock *sk = &smc->sk; in smc_rx_update_consumer()
67 smc_curs_add(conn->rmb_desc->len, &cons, len); in smc_rx_update_consumer()
70 if (conn->urg_state == SMC_URG_VALID || conn->urg_rx_skip_pend) { in smc_rx_update_consumer()
71 diff = smc_curs_comp(conn->rmb_desc->len, &cons, in smc_rx_update_consumer()
72 &conn->urg_curs); in smc_rx_update_consumer()
77 conn->urg_state = SMC_URG_READ; in smc_rx_update_consumer()
83 smc_curs_add(conn->rmb_desc->len, &cons, 1); in smc_rx_update_consumer()
84 conn->urg_rx_skip_pend = false; in smc_rx_update_consumer()
85 } else if (diff < -1) in smc_rx_update_consumer()
87 conn->urg_state = SMC_URG_READ; in smc_rx_update_consumer()
91 smc_curs_copy(&conn->local_tx_ctrl.cons, &cons, conn); in smc_rx_update_consumer()
102 struct smc_connection *conn = &smc->conn; in smc_rx_update_cons()
105 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn); in smc_rx_update_cons()
117 struct smc_spd_priv *priv = (struct smc_spd_priv *)buf->private; in smc_rx_pipe_buf_release() local
118 struct smc_sock *smc = priv->smc; in smc_rx_pipe_buf_release()
120 struct sock *sk = &smc->sk; in smc_rx_pipe_buf_release()
122 if (sk->sk_state == SMC_CLOSED || in smc_rx_pipe_buf_release()
123 sk->sk_state == SMC_PEERFINCLOSEWAIT || in smc_rx_pipe_buf_release()
124 sk->sk_state == SMC_APPFINCLOSEWAIT) in smc_rx_pipe_buf_release()
126 conn = &smc->conn; in smc_rx_pipe_buf_release()
128 smc_rx_update_cons(smc, priv->len); in smc_rx_pipe_buf_release()
130 if (atomic_sub_and_test(priv->len, &conn->splice_pending)) in smc_rx_pipe_buf_release()
133 kfree(priv); in smc_rx_pipe_buf_release()
134 put_page(buf->page); in smc_rx_pipe_buf_release()
146 put_page(spd->pages[i]); in smc_rx_spd_release()
152 struct smc_link_group *lgr = smc->conn.lgr; in smc_rx_splice()
156 struct smc_spd_priv **priv; in smc_rx_splice() local
161 nr_pages = !lgr->is_smcd && smc->conn.rmb_desc->is_vm ? in smc_rx_splice()
170 priv = kcalloc(nr_pages, sizeof(*priv), GFP_KERNEL); in smc_rx_splice()
171 if (!priv) in smc_rx_splice()
174 priv[i] = kzalloc(sizeof(**priv), GFP_KERNEL); in smc_rx_splice()
175 if (!priv[i]) in smc_rx_splice()
179 if (lgr->is_smcd || in smc_rx_splice()
180 (!lgr->is_smcd && !smc->conn.rmb_desc->is_vm)) { in smc_rx_splice()
182 priv[0]->len = len; in smc_rx_splice()
183 priv[0]->smc = smc; in smc_rx_splice()
184 partial[0].offset = src - (char *)smc->conn.rmb_desc->cpu_addr; in smc_rx_splice()
186 partial[0].private = (unsigned long)priv[0]; in smc_rx_splice()
187 pages[0] = smc->conn.rmb_desc->pages; in smc_rx_splice()
193 size = min_t(int, PAGE_SIZE - offset, left); in smc_rx_splice()
194 priv[i]->len = size; in smc_rx_splice()
195 priv[i]->smc = smc; in smc_rx_splice()
199 partial[i].private = (unsigned long)priv[i]; in smc_rx_splice()
201 left -= size; in smc_rx_splice()
214 sock_hold(&smc->sk); in smc_rx_splice()
215 if (!lgr->is_smcd && smc->conn.rmb_desc->is_vm) { in smc_rx_splice()
219 get_page(smc->conn.rmb_desc->pages); in smc_rx_splice()
221 atomic_add(bytes, &smc->conn.splice_pending); in smc_rx_splice()
223 kfree(priv); in smc_rx_splice()
230 for (i = (i - 1); i >= 0; i--) in smc_rx_splice()
231 kfree(priv[i]); in smc_rx_splice()
232 kfree(priv); in smc_rx_splice()
238 return -ENOMEM; in smc_rx_splice()
243 return atomic_read(&conn->bytes_to_rcv) && in smc_rx_data_available_and_no_splice_pend()
244 !atomic_read(&conn->splice_pending); in smc_rx_data_available_and_no_splice_pend()
259 struct smc_connection *conn = &smc->conn; in smc_rx_wait()
261 &conn->local_tx_ctrl.conn_state_flags; in smc_rx_wait()
262 struct sock *sk = &smc->sk; in smc_rx_wait()
270 READ_ONCE(sk->sk_err) || in smc_rx_wait()
271 cflags->peer_conn_abort || in smc_rx_wait()
272 READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN || in smc_rx_wait()
273 conn->killed || in smc_rx_wait()
284 struct smc_connection *conn = &smc->conn; in smc_rx_recv_urg()
286 struct sock *sk = &smc->sk; in smc_rx_recv_urg()
290 !(conn->urg_state == SMC_URG_VALID) || in smc_rx_recv_urg()
291 conn->urg_state == SMC_URG_READ) in smc_rx_recv_urg()
292 return -EINVAL; in smc_rx_recv_urg()
295 if (conn->urg_state == SMC_URG_VALID) { in smc_rx_recv_urg()
297 smc->conn.urg_state = SMC_URG_READ; in smc_rx_recv_urg()
298 msg->msg_flags |= MSG_OOB; in smc_rx_recv_urg()
301 rc = memcpy_to_msg(msg, &conn->urg_rx_byte, 1); in smc_rx_recv_urg()
303 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn); in smc_rx_recv_urg()
304 if (smc_curs_diff(conn->rmb_desc->len, &cons, in smc_rx_recv_urg()
305 &conn->urg_curs) > 1) in smc_rx_recv_urg()
306 conn->urg_rx_skip_pend = true; in smc_rx_recv_urg()
308 * skipping the urgent byte in non-inline case in smc_rx_recv_urg()
313 msg->msg_flags |= MSG_TRUNC; in smc_rx_recv_urg()
316 return rc ? -EFAULT : len; in smc_rx_recv_urg()
319 if (sk->sk_state == SMC_CLOSED || sk->sk_shutdown & RCV_SHUTDOWN) in smc_rx_recv_urg()
322 return -EAGAIN; in smc_rx_recv_urg()
327 struct smc_connection *conn = &smc->conn; in smc_rx_recvmsg_data_available()
331 else if (conn->urg_state == SMC_URG_VALID) in smc_rx_recvmsg_data_available()
332 /* we received a single urgent Byte - skip */ in smc_rx_recvmsg_data_available()
337 /* smc_rx_recvmsg - receive data from RMBE
339 * @pipe: copy data to pipe if set - indicates splice() call
349 struct smc_connection *conn = &smc->conn; in smc_rx_recvmsg()
354 struct sock *sk; in smc_rx_recvmsg()
361 return -EINVAL; /* future work for sk.sk_family == AF_SMC */ in smc_rx_recvmsg()
363 sk = &smc->sk; in smc_rx_recvmsg()
364 if (sk->sk_state == SMC_LISTEN) in smc_rx_recvmsg()
365 return -ENOTCONN; in smc_rx_recvmsg()
371 readable = atomic_read(&conn->bytes_to_rcv); in smc_rx_recvmsg()
372 if (readable >= conn->rmb_desc->len) in smc_rx_recvmsg()
373 SMC_STAT_RMB_RX_FULL(smc, !conn->lnk); in smc_rx_recvmsg()
376 SMC_STAT_RMB_RX_SIZE_SMALL(smc, !conn->lnk); in smc_rx_recvmsg()
378 rcvbuf_base = conn->rx_off + conn->rmb_desc->cpu_addr; in smc_rx_recvmsg()
384 if (conn->killed) in smc_rx_recvmsg()
390 if (sk->sk_shutdown & RCV_SHUTDOWN) { in smc_rx_recvmsg()
400 if (sk->sk_err || in smc_rx_recvmsg()
401 sk->sk_state == SMC_CLOSED || in smc_rx_recvmsg()
406 if (sk->sk_err) { in smc_rx_recvmsg()
410 if (sk->sk_state == SMC_CLOSED) { in smc_rx_recvmsg()
415 read_done = -ENOTCONN; in smc_rx_recvmsg()
421 return -EAGAIN; in smc_rx_recvmsg()
436 readable = atomic_read(&conn->bytes_to_rcv); in smc_rx_recvmsg()
437 splbytes = atomic_read(&conn->splice_pending); in smc_rx_recvmsg()
447 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn); in smc_rx_recvmsg()
450 smc_curs_add(conn->rmb_desc->len, &cons, splbytes); in smc_rx_recvmsg()
451 if (conn->urg_state == SMC_URG_VALID && in smc_rx_recvmsg()
452 sock_flag(&smc->sk, SOCK_URGINLINE) && in smc_rx_recvmsg()
454 readable--; /* always stop at urgent Byte */ in smc_rx_recvmsg()
459 chunk_len = min_t(size_t, copylen, conn->rmb_desc->len - in smc_rx_recvmsg()
477 read_done = -EFAULT; in smc_rx_recvmsg()
481 read_remaining -= chunk_len; in smc_rx_recvmsg()
487 chunk_len = copylen - chunk_len; /* remainder */ in smc_rx_recvmsg()
496 atomic_sub(copylen, &conn->bytes_to_rcv); in smc_rx_recvmsg()
497 /* guarantee 0 <= bytes_to_rcv <= rmb_desc->len */ in smc_rx_recvmsg()
512 smc->sk.sk_data_ready = smc_rx_wake_up; in smc_rx_init()
513 atomic_set(&smc->conn.splice_pending, 0); in smc_rx_init()
514 smc->conn.urg_state = SMC_URG_READ; in smc_rx_init()