31 	struct io_kiocb	*req;  member
47 static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)  in __io_futex_complete()  argument
49 	req->async_data = NULL;  in __io_futex_complete()
50 	hlist_del_init(&req->hash_node);  in __io_futex_complete()
51 	io_req_task_complete(req, ts);  in __io_futex_complete()
54 static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)  in io_futex_complete()  argument
56 	struct io_futex_data *ifd = req->async_data;  in io_futex_complete()
57 	struct io_ring_ctx *ctx = req->ctx;  in io_futex_complete()
62 	__io_futex_complete(req, ts);  in io_futex_complete()
65 static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)  in io_futexv_complete()  argument
67 	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);  in io_futexv_complete()
68 	struct futex_vector *futexv = req->async_data;  in io_futexv_complete()
70 	io_tw_lock(req->ctx, ts);  in io_futexv_complete()
77 			io_req_set_res(req, res, 0);  in io_futexv_complete()
80 	kfree(req->async_data);  in io_futexv_complete()
81 	req->flags &= ~REQ_F_ASYNC_DATA;  in io_futexv_complete()
82 	__io_futex_complete(req, ts);  in io_futexv_complete()
93 static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)  in __io_futex_cancel()  argument
96 	if (req->opcode == IORING_OP_FUTEX_WAIT) {  in __io_futex_cancel()
97 		struct io_futex_data *ifd = req->async_data;  in __io_futex_cancel()
101 		req->io_task_work.func = io_futex_complete;  in __io_futex_cancel()
103 		struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);  in __io_futex_cancel()
107 		req->io_task_work.func = io_futexv_complete;  in __io_futex_cancel()
110 	hlist_del_init(&req->hash_node);  in __io_futex_cancel()
111 	io_req_set_res(req, -ECANCELED, 0);  in __io_futex_cancel()
112 	io_req_task_work_add(req);  in __io_futex_cancel()
120 	struct io_kiocb *req;  in io_futex_cancel()  local
127 	hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {  in io_futex_cancel()
128 		if (req->cqe.user_data != cd->data &&  in io_futex_cancel()
131 		if (__io_futex_cancel(ctx, req))  in io_futex_cancel()
148 	struct io_kiocb *req;  in io_futex_remove_all()  local
153 	hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {  in io_futex_remove_all()
154 		if (!io_match_task_safe(req, task, cancel_all))  in io_futex_remove_all()
156 		hlist_del_init(&req->hash_node);  in io_futex_remove_all()
157 		__io_futex_cancel(ctx, req);  in io_futex_remove_all()
164 int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_futex_prep()  argument
166 	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);  in io_futex_prep()
194 	struct io_kiocb *req = q->wake_data;  in io_futex_wakev_fn()  local
195 	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);  in io_futex_wakev_fn()
202 	io_req_set_res(req, 0, 0);  in io_futex_wakev_fn()
203 	req->io_task_work.func = io_futexv_complete;  in io_futex_wakev_fn()
204 	io_req_task_work_add(req);  in io_futex_wakev_fn()
207 int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_futexv_prep()  argument
209 	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);  in io_futexv_prep()
228 				io_futex_wakev_fn, req);  in io_futexv_prep()
236 	req->flags |= REQ_F_ASYNC_DATA;  in io_futexv_prep()
237 	req->async_data = futexv;  in io_futexv_prep()
244 	struct io_kiocb *req = ifd->req;  in io_futex_wake_fn()  local
249 	io_req_set_res(req, 0, 0);  in io_futex_wake_fn()
250 	req->io_task_work.func = io_futex_complete;  in io_futex_wake_fn()
251 	io_req_task_work_add(req);  in io_futex_wake_fn()
265 int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)  in io_futexv_wait()  argument
267 	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);  in io_futexv_wait()
268 	struct futex_vector *futexv = req->async_data;  in io_futexv_wait()
269 	struct io_ring_ctx *ctx = req->ctx;  in io_futexv_wait()
281 		req_set_fail(req);  in io_futexv_wait()
282 		io_req_set_res(req, ret, 0);  in io_futexv_wait()
284 		req->async_data = NULL;  in io_futexv_wait()
285 		req->flags &= ~REQ_F_ASYNC_DATA;  in io_futexv_wait()
309 		hlist_add_head(&req->hash_node, &ctx->futex_list);  in io_futexv_wait()
313 			io_req_set_res(req, woken, 0);  in io_futexv_wait()
320 int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags)  in io_futex_wait()  argument
322 	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);  in io_futex_wait()
323 	struct io_ring_ctx *ctx = req->ctx;  in io_futex_wait()
340 	req->async_data = ifd;  in io_futex_wait()
344 	ifd->req = req;  in io_futex_wait()
349 		hlist_add_head(&req->hash_node, &ctx->futex_list);  in io_futex_wait()
360 		req_set_fail(req);  in io_futex_wait()
361 	io_req_set_res(req, ret, 0);  in io_futex_wait()
366 int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags)  in io_futex_wake()  argument
368 	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);  in io_futex_wake()
378 		req_set_fail(req);  in io_futex_wake()
379 	io_req_set_res(req, ret, 0);  in io_futex_wake()