202 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_mshot_prep_retry()  local
205 	sr->done_io = 0;  in io_mshot_prep_retry()
206 	sr->len = 0; /* get from the provided buffer */  in io_mshot_prep_retry()
207 	req->buf_index = sr->buf_group;  in io_mshot_prep_retry()
215 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_compat_msg_copy_hdr()  local
228 	if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))  in io_compat_msg_copy_hdr()
236 			sr->len = iov->iov_len = 0;  in io_compat_msg_copy_hdr()
247 			sr->len = clen;  in io_compat_msg_copy_hdr()
265 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_msg_copy_hdr()  local
277 	if (!user_access_begin(sr->umsg, sizeof(*sr->umsg)))  in io_msg_copy_hdr()
281 	unsafe_get_user(msg->msg_name, &sr->umsg->msg_name, ua_end);  in io_msg_copy_hdr()
282 	unsafe_get_user(msg->msg_namelen, &sr->umsg->msg_namelen, ua_end);  in io_msg_copy_hdr()
283 	unsafe_get_user(msg->msg_iov, &sr->umsg->msg_iov, ua_end);  in io_msg_copy_hdr()
284 	unsafe_get_user(msg->msg_iovlen, &sr->umsg->msg_iovlen, ua_end);  in io_msg_copy_hdr()
285 	unsafe_get_user(msg->msg_control, &sr->umsg->msg_control, ua_end);  in io_msg_copy_hdr()
286 	unsafe_get_user(msg->msg_controllen, &sr->umsg->msg_controllen, ua_end);  in io_msg_copy_hdr()
291 			sr->len = iov->iov_len = 0;  in io_msg_copy_hdr()
302 			sr->len = iov->iov_len;  in io_msg_copy_hdr()
322 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendmsg_copy_hdr()  local
348 	sr->msg_control = iomsg->msg.msg_control_user;  in io_sendmsg_copy_hdr()
361 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_send_setup()  local
371 	if (sr->addr) {  in io_send_setup()
372 		ret = move_addr_to_kernel(sr->addr, sr->addr_len, &kmsg->addr);  in io_send_setup()
376 		kmsg->msg.msg_namelen = sr->addr_len;  in io_send_setup()
379 		ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,  in io_send_setup()
407 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendmsg_prep()  local
409 	sr->done_io = 0;  in io_sendmsg_prep()
414 		sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));  in io_sendmsg_prep()
415 		sr->addr_len = READ_ONCE(sqe->addr_len);  in io_sendmsg_prep()
420 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_sendmsg_prep()
421 	sr->len = READ_ONCE(sqe->len);  in io_sendmsg_prep()
422 	sr->flags = READ_ONCE(sqe->ioprio);  in io_sendmsg_prep()
423 	if (sr->flags & ~SENDMSG_FLAGS)  in io_sendmsg_prep()
425 	sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;  in io_sendmsg_prep()
426 	if (sr->msg_flags & MSG_DONTWAIT)  in io_sendmsg_prep()
428 	if (sr->flags & IORING_RECVSEND_BUNDLE) {  in io_sendmsg_prep()
433 		sr->msg_flags |= MSG_WAITALL;  in io_sendmsg_prep()
434 		sr->buf_group = req->buf_index;  in io_sendmsg_prep()
440 		sr->msg_flags |= MSG_CMSG_COMPAT;  in io_sendmsg_prep()
495 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_send_finish()  local
499 	if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {  in io_send_finish()
527 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendmsg()  local
539 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_sendmsg()
542 	flags = sr->msg_flags;  in io_sendmsg()
548 	kmsg->msg.msg_control_user = sr->msg_control;  in io_sendmsg()
558 			sr->done_io += ret;  in io_sendmsg()
568 		ret += sr->done_io;  in io_sendmsg()
569 	else if (sr->done_io)  in io_sendmsg()
570 		ret = sr->done_io;  in io_sendmsg()
577 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_send()  local
589 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_send()
592 	flags = sr->msg_flags;  in io_send()
600 			.max_len = min_not_zero(sr->len, INT_MAX),  in io_send()
610 		if (!(sr->flags & IORING_RECVSEND_BUNDLE))  in io_send()
624 		sr->len = arg.out_len;  in io_send()
627 			sr->buf = arg.iovs[0].iov_base;  in io_send()
628 			ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,  in io_send()
643 	if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE)  in io_send()
654 			sr->len -= ret;  in io_send()
655 			sr->buf += ret;  in io_send()
656 			sr->done_io += ret;  in io_send()
665 		ret += sr->done_io;  in io_send()
666 	else if (sr->done_io)  in io_send()
667 		ret = sr->done_io;  in io_send()
740 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recvmsg_prep_setup()  local
758 			ret = import_ubuf(ITER_DEST, sr->buf, sr->len,  in io_recvmsg_prep_setup()
777 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recvmsg_prep()  local
779 	sr->done_io = 0;  in io_recvmsg_prep()
784 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_recvmsg_prep()
785 	sr->len = READ_ONCE(sqe->len);  in io_recvmsg_prep()
786 	sr->flags = READ_ONCE(sqe->ioprio);  in io_recvmsg_prep()
787 	if (sr->flags & ~RECVMSG_FLAGS)  in io_recvmsg_prep()
789 	sr->msg_flags = READ_ONCE(sqe->msg_flags);  in io_recvmsg_prep()
790 	if (sr->msg_flags & MSG_DONTWAIT)  in io_recvmsg_prep()
792 	if (sr->msg_flags & MSG_ERRQUEUE)  in io_recvmsg_prep()
803 		sr->buf_group = req->buf_index;  in io_recvmsg_prep()
806 	if (sr->flags & IORING_RECV_MULTISHOT) {  in io_recvmsg_prep()
809 		if (sr->msg_flags & MSG_WAITALL)  in io_recvmsg_prep()
811 		if (req->opcode == IORING_OP_RECV && sr->len)  in io_recvmsg_prep()
815 	if (sr->flags & IORING_RECVSEND_BUNDLE) {  in io_recvmsg_prep()
822 		sr->msg_flags |= MSG_CMSG_COMPAT;  in io_recvmsg_prep()
824 	sr->nr_multishot_loops = 0;  in io_recvmsg_prep()
838 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recv_finish()  local
844 	if (sr->flags & IORING_RECVSEND_BUNDLE) {  in io_recv_finish()
865 			if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY)  in io_recv_finish()
868 			sr->nr_multishot_loops = 0;  in io_recv_finish()
891 				     struct io_sr_msg *sr, void __user **buf,  in io_recvmsg_prep_multishot()  argument
909 	sr->buf = *buf; /* stash for later copy */  in io_recvmsg_prep_multishot()
976 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recvmsg()  local
989 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_recvmsg()
992 	flags = sr->msg_flags;  in io_recvmsg()
999 		size_t len = sr->len;  in io_recvmsg()
1006 			ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);  in io_recvmsg()
1019 		ret = io_recvmsg_multishot(sock, sr, kmsg, flags,  in io_recvmsg()
1026 		ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,  in io_recvmsg()
1039 			sr->done_io += ret;  in io_recvmsg()
1051 		ret += sr->done_io;  in io_recvmsg()
1052 	else if (sr->done_io)  in io_recvmsg()
1053 		ret = sr->done_io;  in io_recvmsg()
1066 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recv_buf_select()  local
1075 	    sr->flags & IORING_RECVSEND_BUNDLE) {  in io_recv_buf_select()
1089 			arg.max_len = min_not_zero(sr->len, kmsg->msg.msg_inq);  in io_recv_buf_select()
1097 			sr->buf = arg.iovs[0].iov_base;  in io_recv_buf_select()
1098 			sr->len = arg.iovs[0].iov_len;  in io_recv_buf_select()
1111 		*len = sr->len;  in io_recv_buf_select()
1115 		sr->buf = buf;  in io_recv_buf_select()
1116 		sr->len = *len;  in io_recv_buf_select()
1118 		ret = import_ubuf(ITER_DEST, sr->buf, sr->len,  in io_recv_buf_select()
1129 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recv()  local
1135 	size_t len = sr->len;  in io_recv()
1139 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_recv()
1146 	flags = sr->msg_flags;  in io_recv()
1157 		sr->buf = NULL;  in io_recv()
1177 			sr->len -= ret;  in io_recv()
1178 			sr->buf += ret;  in io_recv()
1179 			sr->done_io += ret;  in io_recv()
1193 		ret += sr->done_io;  in io_recv()
1194 	else if (sr->done_io)  in io_recv()
1195 		ret = sr->done_io;  in io_recv()
1344 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_send_zc_import()  local
1347 	if (sr->flags & IORING_RECVSEND_FIXED_BUF) {  in io_send_zc_import()
1349 					(u64)(uintptr_t)sr->buf, sr->len);  in io_send_zc_import()
1354 		ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);  in io_send_zc_import()
1357 		ret = io_notif_account_mem(sr->notif, sr->len);  in io_send_zc_import()
1436 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendmsg_zc()  local
1449 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_sendmsg_zc()
1452 	flags = sr->msg_flags;  in io_sendmsg_zc()
1458 	kmsg->msg.msg_control_user = sr->msg_control;  in io_sendmsg_zc()
1459 	kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;  in io_sendmsg_zc()
1468 			sr->done_io += ret;  in io_sendmsg_zc()
1478 		ret += sr->done_io;  in io_sendmsg_zc()
1479 	else if (sr->done_io)  in io_sendmsg_zc()
1480 		ret = sr->done_io;  in io_sendmsg_zc()
1487 		io_notif_flush(sr->notif);  in io_sendmsg_zc()
1496 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendrecv_fail()  local
1498 	if (sr->done_io)  in io_sendrecv_fail()
1499 		req->cqe.res = sr->done_io;  in io_sendrecv_fail()