| /linux-6.12.1/io_uring/ |
| D | napi.h | 12 void io_napi_init(struct io_ring_ctx *ctx);
13 void io_napi_free(struct io_ring_ctx *ctx);
15 int io_register_napi(struct io_ring_ctx *ctx, void __user *arg);
16 int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg);
18 void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock);
20 void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq);
21 int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx);
23 static inline bool io_napi(struct io_ring_ctx *ctx) in io_napi()
28 static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, in io_napi_busy_loop()
44 struct io_ring_ctx *ctx = req->ctx; in io_napi_add()
[all …]
|
| D | io_uring.h | 40 struct io_ring_ctx *ctx;
57 struct io_ring_ctx *ctx = iowq->ctx; in io_should_wake()
68 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow);
69 int io_run_task_work_sig(struct io_ring_ctx *ctx);
71 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
72 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
74 void __io_commit_cqring_flush(struct io_ring_ctx *ctx);
81 void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx,
93 struct io_ring_ctx *ctx);
100 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
[all …]
|
| D | rsrc.h | 26 struct io_ring_ctx *ctx;
35 struct io_ring_ctx *ctx;
62 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
63 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
70 int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg);
71 void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
72 int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
73 int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
75 void __io_sqe_files_unregister(struct io_ring_ctx *ctx);
76 int io_sqe_files_unregister(struct io_ring_ctx *ctx);
[all …]
|
| D | futex.h | 12 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
14 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct task_struct *task,
16 bool io_futex_cache_init(struct io_ring_ctx *ctx);
17 void io_futex_cache_free(struct io_ring_ctx *ctx);
19 static inline int io_futex_cancel(struct io_ring_ctx *ctx, in io_futex_cancel()
25 static inline bool io_futex_remove_all(struct io_ring_ctx *ctx, in io_futex_remove_all()
30 static inline bool io_futex_cache_init(struct io_ring_ctx *ctx) in io_futex_cache_init()
34 static inline void io_futex_cache_free(struct io_ring_ctx *ctx) in io_futex_cache_free()
|
| D | tctx.h | 6 struct io_ring_ctx *ctx;
10 struct io_ring_ctx *ctx);
12 int __io_uring_add_tctx_node(struct io_ring_ctx *ctx);
13 int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx);
17 int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg,
19 int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg,
25 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) in io_uring_add_tctx_node()
|
| D | eventfd.h | 2 struct io_ring_ctx;
3 int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
5 int io_eventfd_unregister(struct io_ring_ctx *ctx);
7 void io_eventfd_flush_signal(struct io_ring_ctx *ctx);
8 void io_eventfd_signal(struct io_ring_ctx *ctx);
|
| D | napi.c | 41 void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock) in __io_napi_add()
88 static void __io_napi_remove_stale(struct io_ring_ctx *ctx) in __io_napi_remove_stale()
104 static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale) in io_napi_remove_stale()
139 static bool __io_napi_do_busy_loop(struct io_ring_ctx *ctx, in __io_napi_do_busy_loop()
160 static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, in io_napi_blocking_busy_loop()
188 void io_napi_init(struct io_ring_ctx *ctx) in io_napi_init()
204 void io_napi_free(struct io_ring_ctx *ctx) in io_napi_free()
224 int io_register_napi(struct io_ring_ctx *ctx, void __user *arg) in io_register_napi()
256 int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg) in io_unregister_napi()
279 void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) in __io_napi_busy_loop()
[all …]
|
| D | msg_ring.c | 36 static void io_double_unlock_ctx(struct io_ring_ctx *octx) in io_double_unlock_ctx()
41 static int io_double_lock_ctx(struct io_ring_ctx *octx, in io_double_lock_ctx()
69 static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) in io_msg_need_remote()
76 struct io_ring_ctx *ctx = req->ctx; in io_msg_tw_complete()
89 static int io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req, in io_msg_remote_post()
106 static struct io_kiocb *io_msg_get_kiocb(struct io_ring_ctx *ctx) in io_msg_get_kiocb()
121 struct io_ring_ctx *target_ctx = req->file->private_data; in io_msg_data_remote()
139 struct io_ring_ctx *target_ctx = req->file->private_data; in io_msg_ring_data()
172 struct io_ring_ctx *ctx = req->ctx; in io_msg_grab_file()
189 struct io_ring_ctx *target_ctx = req->file->private_data; in io_msg_install_complete()
[all …]
|
| D | io_uring.c | 146 static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
179 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) in __io_cqring_events()
184 static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx) in __io_cqring_events_user()
215 struct io_ring_ctx *ctx = head->ctx; in io_match_task_safe()
233 static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx) in io_req_add_to_cache()
240 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); in io_ring_ctx_ref_free()
247 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, in io_fallback_req_func()
276 static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) in io_ring_ctx_alloc()
278 struct io_ring_ctx *ctx; in io_ring_ctx_alloc()
370 static void io_account_cq_overflow(struct io_ring_ctx *ctx) in io_account_cq_overflow()
[all …]
|
| D | sqpoll.h | 24 int io_sq_offload_create(struct io_ring_ctx *ctx, struct io_uring_params *p);
25 void io_sq_thread_finish(struct io_ring_ctx *ctx);
30 void io_sqpoll_wait_sq(struct io_ring_ctx *ctx);
31 int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx, cpumask_var_t mask);
|
| D | rsrc.c | 29 static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
30 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
64 static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) in io_unaccount_mem()
73 static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) in io_account_mem()
113 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot) in io_buffer_unmap()
150 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node) in io_rsrc_node_destroy()
159 struct io_ring_ctx *ctx = node->ctx; in io_rsrc_node_ref_zero()
177 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx) in io_rsrc_node_alloc()
195 struct io_ring_ctx *ctx) in io_rsrc_ref_quiesce()
294 __cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, int type, in io_rsrc_data_alloc()
[all …]
|
| D | filetable.h | 13 int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file,
15 int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset);
17 int io_register_file_alloc_range(struct io_ring_ctx *ctx,
69 static inline void io_reset_alloc_hint(struct io_ring_ctx *ctx) in io_reset_alloc_hint()
74 static inline void io_file_table_set_alloc_range(struct io_ring_ctx *ctx, in io_file_table_set_alloc_range()
|
| D | filetable.c | 16 static int io_file_bitmap_get(struct io_ring_ctx *ctx) in io_file_bitmap_get()
63 static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, in io_install_fixed_file()
96 int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file, in __io_fixed_fd_install()
123 struct io_ring_ctx *ctx = req->ctx; in io_fixed_fd_install()
135 int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) in io_fixed_fd_remove()
160 int io_register_file_alloc_range(struct io_ring_ctx *ctx, in io_register_file_alloc_range()
|
| D | kbuf.c | 34 static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx, in io_buffer_get_list()
42 static int io_buffer_add_list(struct io_ring_ctx *ctx, in io_buffer_add_list()
57 struct io_ring_ctx *ctx = req->ctx; in io_kbuf_recycle_legacy()
88 struct io_ring_ctx *ctx = req->ctx; in __io_put_kbuf()
177 struct io_ring_ctx *ctx = req->ctx; in io_buffer_select()
290 struct io_ring_ctx *ctx = req->ctx; in io_buffers_select()
322 struct io_ring_ctx *ctx = req->ctx; in io_buffers_peek()
343 static int __io_remove_buffers(struct io_ring_ctx *ctx, in __io_remove_buffers()
387 void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) in io_put_bl()
395 void io_destroy_buffers(struct io_ring_ctx *ctx) in io_destroy_buffers()
[all …]
|
| D | kbuf.h | 71 void io_destroy_buffers(struct io_ring_ctx *ctx);
79 int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
80 int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
81 int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg);
87 void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl);
88 struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx,
|
| D | futex.c | 36 bool io_futex_cache_init(struct io_ring_ctx *ctx) in io_futex_cache_init()
42 void io_futex_cache_free(struct io_ring_ctx *ctx) in io_futex_cache_free()
57 struct io_ring_ctx *ctx = req->ctx; in io_futex_complete()
93 static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req) in __io_futex_cancel()
116 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_futex_cancel()
144 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct task_struct *task, in io_futex_remove_all()
254 static struct io_futex_data *io_alloc_ifd(struct io_ring_ctx *ctx) in io_alloc_ifd()
269 struct io_ring_ctx *ctx = req->ctx; in io_futexv_wait()
323 struct io_ring_ctx *ctx = req->ctx; in io_futex_wait()
|
| D | register.c | 35 static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, in io_probe()
74 int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) in io_unregister_personality()
88 static int io_register_personality(struct io_ring_ctx *ctx) in io_register_personality()
105 static __cold int io_register_restrictions(struct io_ring_ctx *ctx, in io_register_restrictions()
175 static int io_register_enable_rings(struct io_ring_ctx *ctx) in io_register_enable_rings()
199 static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, in __io_register_iowq_aff()
215 static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, in io_register_iowq_aff()
247 static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) in io_unregister_iowq_aff()
252 static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, in io_register_iowq_max_workers()
338 static int io_register_clock(struct io_ring_ctx *ctx, in io_register_clock()
[all …]
|
| D | timeout.h | 24 __cold void io_flush_timeouts(struct io_ring_ctx *ctx);
26 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd);
27 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
|
| D | timeout.c | 72 struct io_ring_ctx *ctx = req->ctx; in io_timeout_complete()
108 __cold void io_flush_timeouts(struct io_ring_ctx *ctx) in io_flush_timeouts()
202 struct io_ring_ctx *ctx = req->ctx; in io_disarm_next()
239 struct io_ring_ctx *ctx = req->ctx; in io_timeout_fn()
257 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, in io_timeout_extract()
284 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) in io_timeout_cancel()
329 struct io_ring_ctx *ctx = req->ctx; in io_link_timeout_fn()
370 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, in io_linked_timeout_update()
398 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, in io_timeout_update()
462 struct io_ring_ctx *ctx = req->ctx; in io_timeout_remove()
[all …]
|
| D | eventfd.c | 44 void io_eventfd_signal(struct io_ring_ctx *ctx) in io_eventfd_signal()
84 void io_eventfd_flush_signal(struct io_ring_ctx *ctx) in io_eventfd_flush_signal()
107 int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, in io_eventfd_register()
146 int io_eventfd_unregister(struct io_ring_ctx *ctx) in io_eventfd_unregister()
|
| D | register.h | 5 int io_eventfd_unregister(struct io_ring_ctx *ctx);
6 int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id);
|
| D | sqpoll.c | 82 struct io_ring_ctx *ctx; in io_sqd_update_thread_idle()
90 void io_sq_thread_finish(struct io_ring_ctx *ctx) in io_sq_thread_finish()
107 struct io_ring_ctx *ctx_attach; in io_attach_sq_data()
170 static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) in __io_sq_thread()
271 struct io_ring_ctx *ctx; in io_sq_thread()
395 void io_sqpoll_wait_sq(struct io_ring_ctx *ctx) in io_sqpoll_wait_sq()
412 __cold int io_sq_offload_create(struct io_ring_ctx *ctx, in io_sq_offload_create()
513 __cold int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx, in io_sqpoll_wq_cpu_affinity()
|
| D | tctx.c | 15 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx, in io_init_wq_offload()
61 struct io_ring_ctx *ctx) in io_uring_alloc_task_context()
94 int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) in __io_uring_add_tctx_node()
136 int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx) in __io_uring_add_tctx_node_from_submit()
253 int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg, in io_ringfd_register()
312 int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg, in io_ringfd_unregister()
|
| D | waitid.h | 12 int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
14 bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct task_struct *task,
|
| D | uring_cmd.c | 21 struct io_ring_ctx *ctx = req->ctx; in io_uring_async_get()
49 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, in io_uring_try_cancel_uring_cmd()
84 struct io_ring_ctx *ctx = req->ctx; in io_uring_cmd_del_cancelable()
108 struct io_ring_ctx *ctx = req->ctx; in io_uring_cmd_mark_cancelable()
211 struct io_ring_ctx *ctx = req->ctx; in io_uring_cmd_prep()
229 struct io_ring_ctx *ctx = req->ctx; in io_uring_cmd()
|