Lines Matching full:req
36 struct io_kiocb *req; member
73 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req) in io_poll_get_ownership_slowpath() argument
82 v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs); in io_poll_get_ownership_slowpath()
85 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); in io_poll_get_ownership_slowpath()
94 static inline bool io_poll_get_ownership(struct io_kiocb *req) in io_poll_get_ownership() argument
96 if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS)) in io_poll_get_ownership()
97 return io_poll_get_ownership_slowpath(req); in io_poll_get_ownership()
98 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); in io_poll_get_ownership()
101 static void io_poll_mark_cancelled(struct io_kiocb *req) in io_poll_mark_cancelled() argument
103 atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs); in io_poll_mark_cancelled()
106 static struct io_poll *io_poll_get_double(struct io_kiocb *req) in io_poll_get_double() argument
109 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_get_double()
110 return req->async_data; in io_poll_get_double()
111 return req->apoll->double_poll; in io_poll_get_double()
114 static struct io_poll *io_poll_get_single(struct io_kiocb *req) in io_poll_get_single() argument
116 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_get_single()
117 return io_kiocb_to_cmd(req, struct io_poll); in io_poll_get_single()
118 return &req->apoll->poll; in io_poll_get_single()
121 static void io_poll_req_insert(struct io_kiocb *req) in io_poll_req_insert() argument
123 struct io_hash_table *table = &req->ctx->cancel_table; in io_poll_req_insert()
124 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_insert()
128 hlist_add_head(&req->hash_node, &hb->list); in io_poll_req_insert()
132 static void io_poll_req_delete(struct io_kiocb *req, struct io_ring_ctx *ctx) in io_poll_req_delete() argument
134 struct io_hash_table *table = &req->ctx->cancel_table; in io_poll_req_delete()
135 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_delete()
139 hash_del(&req->hash_node); in io_poll_req_delete()
143 static void io_poll_req_insert_locked(struct io_kiocb *req) in io_poll_req_insert_locked() argument
145 struct io_hash_table *table = &req->ctx->cancel_table_locked; in io_poll_req_insert_locked()
146 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_insert_locked()
148 lockdep_assert_held(&req->ctx->uring_lock); in io_poll_req_insert_locked()
150 hlist_add_head(&req->hash_node, &table->hbs[index].list); in io_poll_req_insert_locked()
153 static void io_poll_tw_hash_eject(struct io_kiocb *req, struct io_tw_state *ts) in io_poll_tw_hash_eject() argument
155 struct io_ring_ctx *ctx = req->ctx; in io_poll_tw_hash_eject()
157 if (req->flags & REQ_F_HASH_LOCKED) { in io_poll_tw_hash_eject()
165 hash_del(&req->hash_node); in io_poll_tw_hash_eject()
166 req->flags &= ~REQ_F_HASH_LOCKED; in io_poll_tw_hash_eject()
168 io_poll_req_delete(req, ctx); in io_poll_tw_hash_eject()
194 static void io_poll_remove_entries(struct io_kiocb *req) in io_poll_remove_entries() argument
200 if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL))) in io_poll_remove_entries()
219 if (req->flags & REQ_F_SINGLE_POLL) in io_poll_remove_entries()
220 io_poll_remove_entry(io_poll_get_single(req)); in io_poll_remove_entries()
221 if (req->flags & REQ_F_DOUBLE_POLL) in io_poll_remove_entries()
222 io_poll_remove_entry(io_poll_get_double(req)); in io_poll_remove_entries()
234 static void __io_poll_execute(struct io_kiocb *req, int mask) in __io_poll_execute() argument
238 io_req_set_res(req, mask, 0); in __io_poll_execute()
239 req->io_task_work.func = io_poll_task_func; in __io_poll_execute()
241 trace_io_uring_task_add(req, mask); in __io_poll_execute()
243 if (!(req->flags & REQ_F_POLL_NO_LAZY)) in __io_poll_execute()
245 __io_req_task_work_add(req, flags); in __io_poll_execute()
248 static inline void io_poll_execute(struct io_kiocb *req, int res) in io_poll_execute() argument
250 if (io_poll_get_ownership(req)) in io_poll_execute()
251 __io_poll_execute(req, res); in io_poll_execute()
261 * req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot
262 * poll and that the result is stored in req->cqe.
264 static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts) in io_poll_check_events() argument
268 /* req->task == current here, checking PF_EXITING is safe */ in io_poll_check_events()
269 if (unlikely(req->task->flags & PF_EXITING)) in io_poll_check_events()
273 v = atomic_read(&req->poll_refs); in io_poll_check_events()
287 req->cqe.res = 0; in io_poll_check_events()
290 req->cqe.res = 0; in io_poll_check_events()
296 atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); in io_poll_check_events()
302 if (!req->cqe.res) { in io_poll_check_events()
303 struct poll_table_struct pt = { ._key = req->apoll_events }; in io_poll_check_events()
304 req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events; in io_poll_check_events()
311 if (unlikely(!req->cqe.res)) { in io_poll_check_events()
313 if (!(req->apoll_events & EPOLLONESHOT)) in io_poll_check_events()
318 if (req->apoll_events & EPOLLONESHOT) in io_poll_check_events()
322 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { in io_poll_check_events()
323 __poll_t mask = mangle_poll(req->cqe.res & in io_poll_check_events()
324 req->apoll_events); in io_poll_check_events()
326 if (!io_req_post_cqe(req, mask, IORING_CQE_F_MORE)) { in io_poll_check_events()
327 io_req_set_res(req, mask, 0); in io_poll_check_events()
331 int ret = io_poll_issue(req, ts); in io_poll_check_events()
341 req->cqe.res = 0; in io_poll_check_events()
348 } while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK); in io_poll_check_events()
350 io_napi_add(req); in io_poll_check_events()
354 void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts) in io_poll_task_func() argument
358 ret = io_poll_check_events(req, ts); in io_poll_task_func()
362 __io_poll_execute(req, 0); in io_poll_task_func()
365 io_poll_remove_entries(req); in io_poll_task_func()
366 io_poll_tw_hash_eject(req, ts); in io_poll_task_func()
368 if (req->opcode == IORING_OP_POLL_ADD) { in io_poll_task_func()
372 poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_task_func()
373 req->cqe.res = mangle_poll(req->cqe.res & poll->events); in io_poll_task_func()
375 io_req_task_submit(req, ts); in io_poll_task_func()
378 req->cqe.res = ret; in io_poll_task_func()
379 req_set_fail(req); in io_poll_task_func()
382 io_req_set_res(req, req->cqe.res, 0); in io_poll_task_func()
383 io_req_task_complete(req, ts); in io_poll_task_func()
385 io_tw_lock(req->ctx, ts); in io_poll_task_func()
388 io_req_task_complete(req, ts); in io_poll_task_func()
390 io_req_task_submit(req, ts); in io_poll_task_func()
392 io_req_defer_failed(req, ret); in io_poll_task_func()
396 static void io_poll_cancel_req(struct io_kiocb *req) in io_poll_cancel_req() argument
398 io_poll_mark_cancelled(req); in io_poll_cancel_req()
400 io_poll_execute(req, 0); in io_poll_cancel_req()
405 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) in io_pollfree_wake() argument
407 io_poll_mark_cancelled(req); in io_pollfree_wake()
409 io_poll_execute(req, 0); in io_pollfree_wake()
422 * as req->head is NULL'ed out, the request can be in io_pollfree_wake()
433 struct io_kiocb *req = wqe_to_req(wait); in io_poll_wake() local
438 return io_pollfree_wake(req, poll); in io_poll_wake()
444 if (io_poll_get_ownership(req)) { in io_poll_wake()
458 req->flags &= ~REQ_F_DOUBLE_POLL; in io_poll_wake()
460 req->flags &= ~REQ_F_SINGLE_POLL; in io_poll_wake()
462 __io_poll_execute(req, mask); in io_poll_wake()
468 static bool io_poll_double_prepare(struct io_kiocb *req) in io_poll_double_prepare() argument
471 struct io_poll *poll = io_poll_get_single(req); in io_poll_double_prepare()
477 * poll arm might not hold ownership and so race for req->flags with in io_poll_double_prepare()
484 req->flags |= REQ_F_DOUBLE_POLL; in io_poll_double_prepare()
485 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_double_prepare()
486 req->flags |= REQ_F_ASYNC_DATA; in io_poll_double_prepare()
497 struct io_kiocb *req = pt->req; in __io_queue_proc() local
498 unsigned long wqe_private = (unsigned long) req; in __io_queue_proc()
528 if (!io_poll_double_prepare(req)) { in __io_queue_proc()
536 req->flags |= REQ_F_SINGLE_POLL; in __io_queue_proc()
554 struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll); in io_poll_queue_proc()
557 (struct io_poll **) &pt->req->async_data); in io_poll_queue_proc()
560 static bool io_poll_can_finish_inline(struct io_kiocb *req, in io_poll_can_finish_inline() argument
563 return pt->owning || io_poll_get_ownership(req); in io_poll_can_finish_inline()
566 static void io_poll_add_hash(struct io_kiocb *req) in io_poll_add_hash() argument
568 if (req->flags & REQ_F_HASH_LOCKED) in io_poll_add_hash()
569 io_poll_req_insert_locked(req); in io_poll_add_hash()
571 io_poll_req_insert(req); in io_poll_add_hash()
580 static int __io_arm_poll_handler(struct io_kiocb *req, in __io_arm_poll_handler() argument
585 INIT_HLIST_NODE(&req->hash_node); in __io_arm_poll_handler()
587 poll->file = req->file; in __io_arm_poll_handler()
588 req->apoll_events = poll->events; in __io_arm_poll_handler()
591 ipt->req = req; in __io_arm_poll_handler()
606 atomic_set(&req->poll_refs, (int)ipt->owning); in __io_arm_poll_handler()
610 req->flags &= ~REQ_F_HASH_LOCKED; in __io_arm_poll_handler()
621 req->flags |= REQ_F_POLL_NO_LAZY; in __io_arm_poll_handler()
623 mask = vfs_poll(req->file, &ipt->pt) & poll->events; in __io_arm_poll_handler()
626 io_poll_remove_entries(req); in __io_arm_poll_handler()
628 if (!io_poll_can_finish_inline(req, ipt)) { in __io_arm_poll_handler()
629 io_poll_mark_cancelled(req); in __io_arm_poll_handler()
640 if (!io_poll_can_finish_inline(req, ipt)) { in __io_arm_poll_handler()
641 io_poll_add_hash(req); in __io_arm_poll_handler()
644 io_poll_remove_entries(req); in __io_arm_poll_handler()
646 /* no one else has access to the req, forget about the ref */ in __io_arm_poll_handler()
650 io_poll_add_hash(req); in __io_arm_poll_handler()
653 io_poll_can_finish_inline(req, ipt)) { in __io_arm_poll_handler()
654 __io_poll_execute(req, mask); in __io_arm_poll_handler()
657 io_napi_add(req); in __io_arm_poll_handler()
664 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) in __io_arm_poll_handler()
665 __io_poll_execute(req, 0); in __io_arm_poll_handler()
674 struct async_poll *apoll = pt->req->apoll; in io_async_queue_proc()
687 static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req, in io_req_alloc_apoll() argument
690 struct io_ring_ctx *ctx = req->ctx; in io_req_alloc_apoll()
693 if (req->flags & REQ_F_POLLED) { in io_req_alloc_apoll()
694 apoll = req->apoll; in io_req_alloc_apoll()
709 req->apoll = apoll; in io_req_alloc_apoll()
715 int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) in io_arm_poll_handler() argument
717 const struct io_issue_def *def = &io_issue_defs[req->opcode]; in io_arm_poll_handler()
727 req->flags |= REQ_F_HASH_LOCKED; in io_arm_poll_handler()
731 if (!io_file_can_poll(req)) in io_arm_poll_handler()
733 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) in io_arm_poll_handler()
740 if (req->flags & REQ_F_CLEAR_POLLIN) in io_arm_poll_handler()
748 apoll = io_req_alloc_apoll(req, issue_flags); in io_arm_poll_handler()
751 req->flags &= ~(REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL); in io_arm_poll_handler()
752 req->flags |= REQ_F_POLLED; in io_arm_poll_handler()
755 io_kbuf_recycle(req, issue_flags); in io_arm_poll_handler()
757 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); in io_arm_poll_handler()
760 trace_io_uring_poll_arm(req, mask, apoll->poll.events); in io_arm_poll_handler()
770 struct io_kiocb *req; in io_poll_remove_all_table() local
778 hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { in io_poll_remove_all_table()
779 if (io_match_task_safe(req, tsk, cancel_all)) { in io_poll_remove_all_table()
780 hlist_del_init(&req->hash_node); in io_poll_remove_all_table()
781 io_poll_cancel_req(req); in io_poll_remove_all_table()
809 struct io_kiocb *req; in io_poll_find() local
816 hlist_for_each_entry(req, &hb->list, hash_node) { in io_poll_find()
817 if (cd->data != req->cqe.user_data) in io_poll_find()
819 if (poll_only && req->opcode != IORING_OP_POLL_ADD) in io_poll_find()
822 if (io_cancel_match_sequence(req, cd->seq)) in io_poll_find()
826 return req; in io_poll_find()
838 struct io_kiocb *req; in io_poll_file_find() local
847 hlist_for_each_entry(req, &hb->list, hash_node) { in io_poll_file_find()
848 if (io_cancel_req_match(req, cd)) { in io_poll_file_find()
850 return req; in io_poll_file_find()
858 static int io_poll_disarm(struct io_kiocb *req) in io_poll_disarm() argument
860 if (!req) in io_poll_disarm()
862 if (!io_poll_get_ownership(req)) in io_poll_disarm()
864 io_poll_remove_entries(req); in io_poll_disarm()
865 hash_del(&req->hash_node); in io_poll_disarm()
873 struct io_kiocb *req; in __io_poll_cancel() local
877 req = io_poll_file_find(ctx, cd, table, &bucket); in __io_poll_cancel()
879 req = io_poll_find(ctx, false, cd, table, &bucket); in __io_poll_cancel()
881 if (req) in __io_poll_cancel()
882 io_poll_cancel_req(req); in __io_poll_cancel()
885 return req ? 0 : -ENOENT; in __io_poll_cancel()
920 int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_poll_remove_prep() argument
922 struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update); in io_poll_remove_prep()
950 int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_poll_add_prep() argument
952 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_add_prep()
960 if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP)) in io_poll_add_prep()
967 int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) in io_poll_add() argument
969 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); in io_poll_add()
979 if (req->ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_SINGLE_ISSUER)) in io_poll_add()
980 req->flags |= REQ_F_HASH_LOCKED; in io_poll_add()
982 ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags); in io_poll_add()
984 io_req_set_res(req, ipt.result_mask, 0); in io_poll_add()
990 int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) in io_poll_remove() argument
992 struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update); in io_poll_remove()
993 struct io_ring_ctx *ctx = req->ctx; in io_poll_remove()
1051 req_set_fail(req); in io_poll_remove()
1055 io_req_set_res(req, ret, 0); in io_poll_remove()