19 static struct uring_cache *io_uring_async_get(struct io_kiocb *req)  in io_uring_async_get()  argument
21 	struct io_ring_ctx *ctx = req->ctx;  in io_uring_async_get()
26 		req->flags |= REQ_F_ASYNC_DATA;  in io_uring_async_get()
27 		req->async_data = cache;  in io_uring_async_get()
30 	if (!io_alloc_async_data(req))  in io_uring_async_get()
31 		return req->async_data;  in io_uring_async_get()
35 static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)  in io_req_uring_cleanup()  argument
37 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_req_uring_cleanup()
38 	struct uring_cache *cache = req->async_data;  in io_req_uring_cleanup()
42 	if (io_alloc_cache_put(&req->ctx->uring_cache, cache)) {  in io_req_uring_cleanup()
44 		req->async_data = NULL;  in io_req_uring_cleanup()
45 		req->flags &= ~REQ_F_ASYNC_DATA;  in io_req_uring_cleanup()
53 	struct io_kiocb *req;  in io_uring_try_cancel_uring_cmd()  local
58 	hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,  in io_uring_try_cancel_uring_cmd()
60 		struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,  in io_uring_try_cancel_uring_cmd()
62 		struct file *file = req->file;  in io_uring_try_cancel_uring_cmd()
64 		if (!cancel_all && req->task != task)  in io_uring_try_cancel_uring_cmd()
69 			if (!req_has_async_data(req))  in io_uring_try_cancel_uring_cmd()
83 	struct io_kiocb *req = cmd_to_io_kiocb(cmd);  in io_uring_cmd_del_cancelable()  local
84 	struct io_ring_ctx *ctx = req->ctx;  in io_uring_cmd_del_cancelable()
91 	hlist_del(&req->hash_node);  in io_uring_cmd_del_cancelable()
107 	struct io_kiocb *req = cmd_to_io_kiocb(cmd);  in io_uring_cmd_mark_cancelable()  local
108 	struct io_ring_ctx *ctx = req->ctx;  in io_uring_cmd_mark_cancelable()
113 		hlist_add_head(&req->hash_node, &ctx->cancelable_uring_cmd);  in io_uring_cmd_mark_cancelable()
119 static void io_uring_cmd_work(struct io_kiocb *req, struct io_tw_state *ts)  in io_uring_cmd_work()  argument
121 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_uring_cmd_work()
131 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in __io_uring_cmd_do_in_task()  local
134 	req->io_task_work.func = io_uring_cmd_work;  in __io_uring_cmd_do_in_task()
135 	__io_req_task_work_add(req, flags);  in __io_uring_cmd_do_in_task()
139 static inline void io_req_set_cqe32_extra(struct io_kiocb *req,  in io_req_set_cqe32_extra()  argument
142 	req->big_cqe.extra1 = extra1;  in io_req_set_cqe32_extra()
143 	req->big_cqe.extra2 = extra2;  in io_req_set_cqe32_extra()
153 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in io_uring_cmd_done()  local
158 		req_set_fail(req);  in io_uring_cmd_done()
160 	io_req_set_res(req, ret, 0);  in io_uring_cmd_done()
161 	if (req->ctx->flags & IORING_SETUP_CQE32)  in io_uring_cmd_done()
162 		io_req_set_cqe32_extra(req, res2, 0);  in io_uring_cmd_done()
163 	io_req_uring_cleanup(req, issue_flags);  in io_uring_cmd_done()
164 	if (req->ctx->flags & IORING_SETUP_IOPOLL) {  in io_uring_cmd_done()
166 		smp_store_release(&req->iopoll_completed, 1);  in io_uring_cmd_done()
170 		io_req_complete_defer(req);  in io_uring_cmd_done()
172 		req->io_task_work.func = io_req_task_complete;  in io_uring_cmd_done()
173 		io_req_task_work_add(req);  in io_uring_cmd_done()
178 static int io_uring_cmd_prep_setup(struct io_kiocb *req,  in io_uring_cmd_prep_setup()  argument
181 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_uring_cmd_prep_setup()
184 	cache = io_uring_async_get(req);  in io_uring_cmd_prep_setup()
188 	if (!(req->flags & REQ_F_FORCE_ASYNC)) {  in io_uring_cmd_prep_setup()
194 	memcpy(req->async_data, sqe, uring_sqe_size(req->ctx));  in io_uring_cmd_prep_setup()
195 	ioucmd->sqe = req->async_data;  in io_uring_cmd_prep_setup()
199 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_uring_cmd_prep()  argument
201 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_uring_cmd_prep()
211 		struct io_ring_ctx *ctx = req->ctx;  in io_uring_cmd_prep()
214 		req->buf_index = READ_ONCE(sqe->buf_index);  in io_uring_cmd_prep()
215 		if (unlikely(req->buf_index >= ctx->nr_user_bufs))  in io_uring_cmd_prep()
217 		index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);  in io_uring_cmd_prep()
218 		req->imu = ctx->user_bufs[index];  in io_uring_cmd_prep()
219 		io_req_set_rsrc_node(req, ctx, 0);  in io_uring_cmd_prep()
223 	return io_uring_cmd_prep_setup(req, sqe);  in io_uring_cmd_prep()
226 int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)  in io_uring_cmd()  argument
228 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_uring_cmd()
229 	struct io_ring_ctx *ctx = req->ctx;  in io_uring_cmd()
230 	struct file *file = req->file;  in io_uring_cmd()
250 		req->iopoll_completed = 0;  in io_uring_cmd()
255 		struct uring_cache *cache = req->async_data;  in io_uring_cmd()
258 			memcpy(cache, ioucmd->sqe, uring_sqe_size(req->ctx));  in io_uring_cmd()
265 		req_set_fail(req);  in io_uring_cmd()
266 	io_req_uring_cleanup(req, issue_flags);  in io_uring_cmd()
267 	io_req_set_res(req, ret, 0);  in io_uring_cmd()
274 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in io_uring_cmd_import_fixed()  local
276 	return io_import_fixed(rw, iter, req->imu, ubuf, len);  in io_uring_cmd_import_fixed()
282 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in io_uring_cmd_issue_blocking()  local
284 	io_req_queue_iowq(req);  in io_uring_cmd_issue_blocking()