| /io_uring/ |
| A D | io_uring.h | 44 struct io_ring_ctx *ctx; member
61 struct io_ring_ctx *ctx = iowq->ctx; in io_should_wake() local
150 static inline bool io_is_compat(struct io_ring_ctx *ctx) in io_is_compat()
291 static inline void io_commit_cqring(struct io_ring_ctx *ctx) in io_commit_cqring()
310 static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) in io_poll_wq_wake()
315 static inline void io_cqring_wake(struct io_ring_ctx *ctx) in io_cqring_wake()
326 static inline bool io_sqring_full(struct io_ring_ctx *ctx) in io_sqring_full()
465 static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) in io_allowed_run_tw()
495 static inline size_t uring_sqe_size(struct io_ring_ctx *ctx) in uring_sqe_size()
513 static inline ktime_t io_get_time(struct io_ring_ctx *ctx) in io_get_time()
[all …]
|
| A D | napi.h | 23 static inline bool io_napi(struct io_ring_ctx *ctx) in io_napi()
28 static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, in io_napi_busy_loop()
44 struct io_ring_ctx *ctx = req->ctx; in io_napi_add() local
57 static inline void io_napi_init(struct io_ring_ctx *ctx) in io_napi_init()
60 static inline void io_napi_free(struct io_ring_ctx *ctx) in io_napi_free()
63 static inline int io_register_napi(struct io_ring_ctx *ctx, void __user *arg) in io_register_napi()
67 static inline int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg) in io_unregister_napi()
71 static inline bool io_napi(struct io_ring_ctx *ctx) in io_napi()
78 static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, in io_napi_busy_loop()
82 static inline int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) in io_napi_sqpoll_busy_poll()
|
| A D | napi.c | 41 int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id) in __io_napi_add_id()
105 static void __io_napi_remove_stale(struct io_ring_ctx *ctx) in __io_napi_remove_stale()
163 static bool static_tracking_do_busy_loop(struct io_ring_ctx *ctx, in static_tracking_do_busy_loop()
176 dynamic_tracking_do_busy_loop(struct io_ring_ctx *ctx, in dynamic_tracking_do_busy_loop()
195 __io_napi_do_busy_loop(struct io_ring_ctx *ctx, in __io_napi_do_busy_loop()
204 static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, in io_napi_blocking_busy_loop()
237 void io_napi_init(struct io_ring_ctx *ctx) in io_napi_init()
254 void io_napi_free(struct io_ring_ctx *ctx) in io_napi_free()
266 static int io_napi_register_napi(struct io_ring_ctx *ctx, in io_napi_register_napi()
291 int io_register_napi(struct io_ring_ctx *ctx, void __user *arg) in io_register_napi()
[all …]
|
| A D | filetable.c | 16 static int io_file_bitmap_get(struct io_ring_ctx *ctx) in io_file_bitmap_get()
39 bool io_alloc_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table, in io_alloc_file_tables()
51 void io_free_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table) in io_free_file_tables()
58 static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, in io_install_fixed_file()
83 int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file, in __io_fixed_fd_install()
110 struct io_ring_ctx *ctx = req->ctx; in io_fixed_fd_install() local
122 int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) in io_fixed_fd_remove()
139 int io_register_file_alloc_range(struct io_ring_ctx *ctx, in io_register_file_alloc_range()
|
| A D | io_uring.c | 218 struct io_ring_ctx *ctx = head->ctx; in io_match_task_safe() local
300 struct io_ring_ctx *ctx; in io_ring_ctx_alloc() local
436 struct io_ring_ctx *ctx = req->ctx; in io_prep_async_work() local
891 struct io_ring_ctx *ctx = req->ctx; in io_req_post_cqe() local
923 struct io_ring_ctx *ctx = req->ctx; in io_req_post_cqe32() local
944 struct io_ring_ctx *ctx = req->ctx; in io_req_complete_post() local
1088 struct io_ring_ctx *ctx = NULL; in io_handle_tw_list() local
2906 struct io_ring_ctx *ctx; member
3202 struct io_ring_ctx *ctx; in io_uring_cancel_generic() local
3402 struct io_ring_ctx *ctx; in SYSCALL_DEFINE6() local
[all …]
|
| A D | register.c | 38 static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, in io_probe()
77 int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) in io_unregister_personality()
91 static int io_register_personality(struct io_ring_ctx *ctx) in io_register_personality()
180 static int io_register_enable_rings(struct io_ring_ctx *ctx) in io_register_enable_rings()
204 static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, in __io_register_iowq_aff()
220 static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, in io_register_iowq_aff()
252 static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) in io_unregister_iowq_aff()
346 static int io_register_clock(struct io_ring_ctx *ctx, in io_register_clock()
383 static void io_register_free_rings(struct io_ring_ctx *ctx, in io_register_free_rings()
391 #define swap_old(ctx, o, n, field) \ argument
[all …]
|
| A D | futex.h | 19 static inline int io_futex_cancel(struct io_ring_ctx *ctx, in io_futex_cancel()
25 static inline bool io_futex_remove_all(struct io_ring_ctx *ctx, in io_futex_remove_all()
30 static inline bool io_futex_cache_init(struct io_ring_ctx *ctx) in io_futex_cache_init()
34 static inline void io_futex_cache_free(struct io_ring_ctx *ctx) in io_futex_cache_free()
|
| A D | memmap.c | 91 void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr) in io_free_region()
134 static int io_region_pin_pages(struct io_ring_ctx *ctx, in io_region_pin_pages()
153 static int io_region_allocate_pages(struct io_ring_ctx *ctx, in io_region_allocate_pages()
188 int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, in io_create_region()
258 static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, in io_mmap_get_region()
283 static void *io_region_validate_mmap(struct io_ring_ctx *ctx, in io_region_validate_mmap()
299 struct io_ring_ctx *ctx = file->private_data; in io_uring_validate_mmap_request() local
310 static int io_region_mmap(struct io_ring_ctx *ctx, in io_region_mmap()
323 struct io_ring_ctx *ctx = file->private_data; in io_uring_mmap() local
351 struct io_ring_ctx *ctx = filp->private_data; in io_uring_get_unmapped_area() local
[all …]
|
| A D | kbuf.c | 78 static int io_buffer_add_list(struct io_ring_ctx *ctx, in io_buffer_add_list()
102 struct io_ring_ctx *ctx = req->ctx; in io_kbuf_recycle_legacy() local
198 struct io_ring_ctx *ctx = req->ctx; in io_buffer_select() local
304 struct io_ring_ctx *ctx = req->ctx; in io_buffers_select() local
336 struct io_ring_ctx *ctx = req->ctx; in io_buffers_peek() local
385 static int io_remove_buffers_legacy(struct io_ring_ctx *ctx, in io_remove_buffers_legacy()
416 void io_destroy_buffers(struct io_ring_ctx *ctx) in io_destroy_buffers()
562 struct io_ring_ctx *ctx = req->ctx; in io_manage_buffers_legacy() local
577 int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) in io_register_pbuf_ring()
658 int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) in io_unregister_pbuf_ring()
[all …]
|
| A D | rsrc.c | 166 bool io_rsrc_cache_init(struct io_ring_ctx *ctx) in io_rsrc_cache_init()
180 void io_rsrc_cache_free(struct io_ring_ctx *ctx) in io_rsrc_cache_free()
198 __cold void io_rsrc_data_free(struct io_ring_ctx *ctx, in io_rsrc_data_free()
223 static int __io_sqe_files_update(struct io_ring_ctx *ctx, in __io_sqe_files_update()
288 static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, in __io_sqe_buffers_update()
485 struct io_ring_ctx *ctx = req->ctx; in io_files_update() local
531 int io_sqe_files_unregister(struct io_ring_ctx *ctx) in io_sqe_files_unregister()
611 int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) in io_sqe_buffers_unregister()
940 struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; in io_buffer_register_bvec() local
1001 struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; in io_buffer_unregister_bvec() local
[all …]
|
| A D | sqpoll.c | 92 struct io_ring_ctx *ctx; in io_sqd_update_thread_idle() local
100 void io_sq_thread_finish(struct io_ring_ctx *ctx) in io_sq_thread_finish()
172 static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) in __io_sq_thread()
273 struct io_ring_ctx *ctx; in io_sq_thread() local
403 void io_sqpoll_wait_sq(struct io_ring_ctx *ctx) in io_sqpoll_wait_sq()
420 __cold int io_sq_offload_create(struct io_ring_ctx *ctx, in io_sq_offload_create()
519 __cold int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx, in io_sqpoll_wq_cpu_affinity()
|
| A D | fdinfo.c | 19 static __cold void common_tracking_show_fdinfo(struct io_ring_ctx *ctx, in common_tracking_show_fdinfo()
32 static __cold void napi_show_fdinfo(struct io_ring_ctx *ctx, in napi_show_fdinfo()
52 static inline void napi_show_fdinfo(struct io_ring_ctx *ctx, in napi_show_fdinfo()
58 static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) in __io_uring_show_fdinfo()
224 struct io_ring_ctx *ctx = file->private_data; in io_uring_show_fdinfo() local
|
| A D | tctx.c | 15 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx, in io_init_wq_offload()
70 struct io_ring_ctx *ctx) in io_uring_alloc_task_context()
104 int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) in __io_uring_add_tctx_node()
146 int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx) in __io_uring_add_tctx_node_from_submit()
263 int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg, in io_ringfd_register()
322 int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg, in io_ringfd_unregister()
|
| A D | timeout.c | 75 struct io_ring_ctx *ctx = req->ctx; in io_timeout_complete() local
125 __cold void io_flush_timeouts(struct io_ring_ctx *ctx) in io_flush_timeouts()
221 struct io_ring_ctx *ctx = req->ctx; in io_disarm_next() local
260 struct io_ring_ctx *ctx = req->ctx; in io_timeout_fn() local
278 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, in io_timeout_extract()
305 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) in io_timeout_cancel()
352 struct io_ring_ctx *ctx = req->ctx; in io_link_timeout_fn() local
420 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, in io_timeout_update()
485 struct io_ring_ctx *ctx = req->ctx; in io_timeout_remove() local
594 struct io_ring_ctx *ctx = req->ctx; in io_timeout() local
[all …]
|
| A D | futex.c | 33 bool io_futex_cache_init(struct io_ring_ctx *ctx) in io_futex_cache_init()
39 void io_futex_cache_free(struct io_ring_ctx *ctx) in io_futex_cache_free()
53 struct io_ring_ctx *ctx = req->ctx; in io_futex_complete() local
111 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_futex_cancel()
117 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_futex_remove_all()
221 struct io_ring_ctx *ctx = req->ctx; in io_futexv_wait() local
275 struct io_ring_ctx *ctx = req->ctx; in io_futex_wait() local
|
| A D | cancel.c | 106 struct io_ring_ctx *ctx = cd->ctx; in io_try_cancel() local
170 struct io_ring_ctx *ctx = cd->ctx; in __io_async_cancel() local
238 struct io_ring_ctx *ctx = cd->ctx; in __io_sync_cancel() local
256 int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) in io_sync_cancel()
345 bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_cancel_remove_all()
366 int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_cancel_remove()
|
| A D | tctx.h | 6 struct io_ring_ctx *ctx; member
25 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) in io_uring_add_tctx_node()
|
| A D | msg_ring.c | 76 struct io_ring_ctx *ctx = req->ctx; in io_msg_tw_complete() local
89 static int io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req, in io_msg_remote_post()
107 static struct io_kiocb *io_msg_get_kiocb(struct io_ring_ctx *ctx) in io_msg_get_kiocb()
179 struct io_ring_ctx *ctx = req->ctx; in io_msg_grab_file() local
243 struct io_ring_ctx *ctx = req->file->private_data; in io_msg_fd_remote() local
261 struct io_ring_ctx *ctx = req->ctx; in io_msg_send_fd() local
|
| A D | zcrx.h | 43 struct io_ring_ctx *ctx; member
72 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx, in io_register_zcrx_ifq()
77 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) in io_unregister_zcrx_ifqs()
80 static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx) in io_shutdown_zcrx_ifqs()
89 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, in io_zcrx_get_region()
|
| A D | eventfd.c | 75 void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) in io_eventfd_signal()
115 int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, in io_eventfd_register()
154 int io_eventfd_unregister(struct io_ring_ctx *ctx) in io_eventfd_unregister()
|
| A D | waitid.c | 154 int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_waitid_cancel()
160 bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_waitid_remove_all()
187 struct io_ring_ctx *ctx = req->ctx; in io_waitid_cb() local
269 struct io_ring_ctx *ctx = req->ctx; in io_waitid() local
|
| A D | poll.c | 530 struct io_ring_ctx *ctx = req->ctx; in io_poll_add_hash() local
648 struct io_ring_ctx *ctx = req->ctx; in io_req_alloc_apoll() local
726 __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_poll_remove_all()
751 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, in io_poll_find()
772 static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, in io_poll_file_find()
801 static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) in __io_poll_cancel()
818 int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_poll_cancel()
912 struct io_ring_ctx *ctx = req->ctx; in io_poll_remove() local
|
| A D | uring_cmd.c | 49 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, in io_uring_try_cancel_uring_cmd()
81 struct io_ring_ctx *ctx = req->ctx; in io_uring_cmd_del_cancelable() local
105 struct io_ring_ctx *ctx = req->ctx; in io_uring_cmd_mark_cancelable() local
224 struct io_ring_ctx *ctx = req->ctx; in io_uring_cmd() local
|
| A D | cancel.h | 8 struct io_ring_ctx *ctx; member
|
| A D | rsrc.h | 100 static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node) in io_put_rsrc_node()
107 static inline bool io_reset_rsrc_node(struct io_ring_ctx *ctx, in io_reset_rsrc_node()
|