38 static inline bool io_is_timeout_noseq(struct io_kiocb *req)  in io_is_timeout_noseq()  argument
40 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_is_timeout_noseq()
41 	struct io_timeout_data *data = req->async_data;  in io_is_timeout_noseq()
46 static inline void io_put_req(struct io_kiocb *req)  in io_put_req()  argument
48 	if (req_ref_put_and_test(req)) {  in io_put_req()
49 		io_queue_next(req);  in io_put_req()
50 		io_free_req(req);  in io_put_req()
68 static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)  in io_timeout_complete()  argument
70 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_complete()
71 	struct io_timeout_data *data = req->async_data;  in io_timeout_complete()
72 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_complete()
75 		if (io_req_post_cqe(req, -ETIME, IORING_CQE_F_MORE)) {  in io_timeout_complete()
86 	io_req_task_complete(req, ts);  in io_timeout_complete()
89 static bool io_kill_timeout(struct io_kiocb *req, int status)  in io_kill_timeout()  argument
90 	__must_hold(&req->ctx->timeout_lock)  in io_kill_timeout()
92 	struct io_timeout_data *io = req->async_data;  in io_kill_timeout()
95 		struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_kill_timeout()
98 			req_set_fail(req);  in io_kill_timeout()
99 		atomic_set(&req->ctx->cq_timeouts,  in io_kill_timeout()
100 			atomic_read(&req->ctx->cq_timeouts) + 1);  in io_kill_timeout()
102 		io_req_queue_tw_complete(req, status);  in io_kill_timeout()
117 		struct io_kiocb *req = cmd_to_io_kiocb(timeout);  in io_flush_timeouts()  local
120 		if (io_is_timeout_noseq(req))  in io_flush_timeouts()
135 		io_kill_timeout(req, 0);  in io_flush_timeouts()
157 static void io_fail_links(struct io_kiocb *req)  in io_fail_links()  argument
158 	__must_hold(&req->ctx->completion_lock)  in io_fail_links()
160 	struct io_kiocb *link = req->link;  in io_fail_links()
161 	bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;  in io_fail_links()
171 		trace_io_uring_fail_link(req, link);  in io_fail_links()
175 	link = req->link;  in io_fail_links()
178 	req->link = NULL;  in io_fail_links()
181 static inline void io_remove_next_linked(struct io_kiocb *req)  in io_remove_next_linked()  argument
183 	struct io_kiocb *nxt = req->link;  in io_remove_next_linked()
185 	req->link = nxt->link;  in io_remove_next_linked()
189 void io_disarm_next(struct io_kiocb *req)  in io_disarm_next()  argument
190 	__must_hold(&req->ctx->completion_lock)  in io_disarm_next()
194 	if (req->flags & REQ_F_ARM_LTIMEOUT) {  in io_disarm_next()
195 		link = req->link;  in io_disarm_next()
196 		req->flags &= ~REQ_F_ARM_LTIMEOUT;  in io_disarm_next()
198 			io_remove_next_linked(req);  in io_disarm_next()
201 	} else if (req->flags & REQ_F_LINK_TIMEOUT) {  in io_disarm_next()
202 		struct io_ring_ctx *ctx = req->ctx;  in io_disarm_next()
205 		link = io_disarm_linked_timeout(req);  in io_disarm_next()
210 	if (unlikely((req->flags & REQ_F_FAIL) &&  in io_disarm_next()
211 		     !(req->flags & REQ_F_HARDLINK)))  in io_disarm_next()
212 		io_fail_links(req);  in io_disarm_next()
215 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,  in __io_disarm_linked_timeout()  argument
217 	__must_hold(&req->ctx->completion_lock)  in __io_disarm_linked_timeout()
218 	__must_hold(&req->ctx->timeout_lock)  in __io_disarm_linked_timeout()
223 	io_remove_next_linked(req);  in __io_disarm_linked_timeout()
237 	struct io_kiocb *req = data->req;  in io_timeout_fn()  local
238 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_fn()
239 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_fn()
244 	atomic_set(&req->ctx->cq_timeouts,  in io_timeout_fn()
245 		atomic_read(&req->ctx->cq_timeouts) + 1);  in io_timeout_fn()
249 		req_set_fail(req);  in io_timeout_fn()
251 	io_req_set_res(req, -ETIME, 0);  in io_timeout_fn()
252 	req->io_task_work.func = io_timeout_complete;  in io_timeout_fn()
253 	io_req_task_work_add(req);  in io_timeout_fn()
263 	struct io_kiocb *req = NULL;  in io_timeout_extract()  local
269 			req = tmp;  in io_timeout_extract()
273 	if (!req)  in io_timeout_extract()
276 	io = req->async_data;  in io_timeout_extract()
279 	timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_extract()
281 	return req;  in io_timeout_extract()
287 	struct io_kiocb *req;  in io_timeout_cancel()  local
290 	req = io_timeout_extract(ctx, cd);  in io_timeout_cancel()
293 	if (IS_ERR(req))  in io_timeout_cancel()
294 		return PTR_ERR(req);  in io_timeout_cancel()
295 	io_req_task_queue_fail(req, -ECANCELED);  in io_timeout_cancel()
299 static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts)  in io_req_task_link_timeout()  argument
301 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_req_task_link_timeout()
306 		if (!(req->task->flags & PF_EXITING)) {  in io_req_task_link_timeout()
308 				.ctx		= req->ctx,  in io_req_task_link_timeout()
312 			ret = io_try_cancel(req->task->io_uring, &cd, 0);  in io_req_task_link_timeout()
314 		io_req_set_res(req, ret ?: -ETIME, 0);  in io_req_task_link_timeout()
315 		io_req_task_complete(req, ts);  in io_req_task_link_timeout()
318 		io_req_set_res(req, -ETIME, 0);  in io_req_task_link_timeout()
319 		io_req_task_complete(req, ts);  in io_req_task_link_timeout()
327 	struct io_kiocb *prev, *req = data->req;  in io_link_timeout_fn()  local
328 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_link_timeout_fn()
329 	struct io_ring_ctx *ctx = req->ctx;  in io_link_timeout_fn()
349 	req->io_task_work.func = io_req_task_link_timeout;  in io_link_timeout_fn()
350 	io_req_task_work_add(req);  in io_link_timeout_fn()
376 	struct io_kiocb *req = NULL;  in io_linked_timeout_update()  local
382 			req = tmp;  in io_linked_timeout_update()
386 	if (!req)  in io_linked_timeout_update()
389 	io = req->async_data;  in io_linked_timeout_update()
403 	struct io_kiocb *req = io_timeout_extract(ctx, &cd);  in io_timeout_update()  local
404 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_update()
407 	if (IS_ERR(req))  in io_timeout_update()
408 		return PTR_ERR(req);  in io_timeout_update()
411 	data = req->async_data;  in io_timeout_update()
419 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_timeout_remove_prep()  argument
421 	struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);  in io_timeout_remove_prep()
423 	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))  in io_timeout_remove_prep()
459 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout_remove()  argument
461 	struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);  in io_timeout_remove()
462 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_remove()
483 		req_set_fail(req);  in io_timeout_remove()
484 	io_req_set_res(req, ret, 0);  in io_timeout_remove()
488 static int __io_timeout_prep(struct io_kiocb *req,  in __io_timeout_prep()  argument
492 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in __io_timeout_prep()
515 	if (unlikely(off && !req->ctx->off_timeout_used))  in __io_timeout_prep()
516 		req->ctx->off_timeout_used = true;  in __io_timeout_prep()
525 	if (WARN_ON_ONCE(req_has_async_data(req)))  in __io_timeout_prep()
527 	if (io_alloc_async_data(req))  in __io_timeout_prep()
530 	data = req->async_data;  in __io_timeout_prep()
531 	data->req = req;  in __io_timeout_prep()
544 		struct io_submit_link *link = &req->ctx->submit_state.link;  in __io_timeout_prep()
556 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_timeout_prep()  argument
558 	return __io_timeout_prep(req, sqe, false);  in io_timeout_prep()
561 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_link_timeout_prep()  argument
563 	return __io_timeout_prep(req, sqe, true);  in io_link_timeout_prep()
566 int io_timeout(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout()  argument
568 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout()
569 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout()
570 	struct io_timeout_data *data = req->async_data;  in io_timeout()
581 	if (io_is_timeout_noseq(req)) {  in io_timeout()
617 void io_queue_linked_timeout(struct io_kiocb *req)  in io_queue_linked_timeout()  argument
619 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_queue_linked_timeout()
620 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_linked_timeout()
628 		struct io_timeout_data *data = req->async_data;  in io_queue_linked_timeout()
637 	io_put_req(req);  in io_queue_linked_timeout()
644 	struct io_kiocb *req;  in io_match_task()  local
651 	io_for_each_link(req, head) {  in io_match_task()
652 		if (req->flags & REQ_F_INFLIGHT)  in io_match_task()
672 		struct io_kiocb *req = cmd_to_io_kiocb(timeout);  in io_kill_timeouts()  local
674 		if (io_match_task(req, tsk, cancel_all) &&  in io_kill_timeouts()
675 		    io_kill_timeout(req, -ECANCELED))  in io_kill_timeouts()