| /io_uring/ |
| A D | timeout.c | 82 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); in io_timeout_complete()
434 data->ts = *ts; in io_timeout_update()
438 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), mode); in io_timeout_update()
551 if (!data) in __io_timeout_prep()
553 data->req = req; in __io_timeout_prep()
554 data->flags = flags; in __io_timeout_prep()
559 if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0) in __io_timeout_prep()
576 hrtimer_setup(&data->timer, io_timeout_fn, io_timeout_get_clock(data), data->mode); in __io_timeout_prep()
636 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); in io_timeout()
654 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), in io_queue_linked_timeout()
[all …]
|
| A D | filetable.c | 42 if (io_rsrc_data_alloc(&table->data, nr_files)) in io_alloc_file_tables()
47 io_rsrc_data_free(ctx, &table->data); in io_alloc_file_tables()
53 io_rsrc_data_free(ctx, &table->data); in io_free_file_tables()
66 if (!ctx->file_table.data.nr) in io_install_fixed_file()
68 if (slot_index >= ctx->file_table.data.nr) in io_install_fixed_file()
78 ctx->file_table.data.nodes[slot_index] = node; in io_install_fixed_file()
126 if (unlikely(!ctx->file_table.data.nr)) in io_fixed_fd_remove()
128 if (offset >= ctx->file_table.data.nr) in io_fixed_fd_remove()
131 node = io_rsrc_node_lookup(&ctx->file_table.data, offset); in io_fixed_fd_remove()
134 io_reset_rsrc_node(ctx, &ctx->file_table.data, offset); in io_fixed_fd_remove()
[all …]
|
| A D | rsrc.c | 201 if (!data->nr) in io_rsrc_data_free()
203 while (data->nr--) { in io_rsrc_data_free()
204 if (data->nodes[data->nr]) in io_rsrc_data_free()
205 io_put_rsrc_node(ctx, data->nodes[data->nr]); in io_rsrc_data_free()
207 kvfree(data->nodes); in io_rsrc_data_free()
208 data->nodes = NULL; in io_rsrc_data_free()
209 data->nr = 0; in io_rsrc_data_free()
216 if (data->nodes) { in io_rsrc_data_alloc()
217 data->nr = nr; in io_rsrc_data_alloc()
490 up2.data = up->arg; in io_files_update()
[all …]
|
| A D | rsrc.h | 59 void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data);
60 int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);
90 struct io_imu_folio_data *data);
92 static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data, in io_rsrc_node_lookup() argument
95 if (index < data->nr) in io_rsrc_node_lookup()
96 return data->nodes[array_index_nospec(index, data->nr)]; in io_rsrc_node_lookup()
108 struct io_rsrc_data *data, int index) in io_reset_rsrc_node() argument
110 struct io_rsrc_node *node = data->nodes[index]; in io_reset_rsrc_node()
115 data->nodes[index] = NULL; in io_reset_rsrc_node()
|
| A D | io-wq.c | 139 void *data; member
227 return worker == data; in io_task_worker_match()
930 void *data) in io_acct_for_each_worker() argument
939 ret = func(worker, data); in io_acct_for_each_worker()
951 void *data) in io_wq_for_each_worker() argument
1000 return work == data; in io_wq_work_match_item()
1009 .data = work, in io_wq_enqueue()
1177 .data = data, in io_wq_cancel_cb()
1241 wq->hash = data->hash; in io_wq_create()
1277 io_wq_put_hash(data->hash); in io_wq_create()
[all …]
|
| A D | tctx.c | 19 struct io_wq_data data; in io_init_wq_offload() local
36 data.hash = hash; in io_init_wq_offload()
37 data.task = task; in io_init_wq_offload()
42 return io_wq_create(concurrency, &data); in io_init_wq_offload()
306 ret = io_ring_add_registered_fd(tctx, reg.data, start, end); in io_ringfd_register()
340 if (reg.resv || reg.data || reg.offset >= IO_RINGFD_REG_MAX) { in io_ringfd_unregister()
|
| A D | cancel.c | 57 if (match_user_data && req->cqe.user_data != cd->data) in io_cancel_req_match()
68 static bool io_cancel_cb(struct io_wq_work *work, void *data) in io_cancel_cb() argument
71 struct io_cancel_data *cd = data; in io_cancel_cb()
203 .data = cancel->addr, in io_async_cancel()
245 node = io_rsrc_node_lookup(&ctx->file_table.data, fd); in __io_sync_cancel()
280 cd.data = sc.addr; in io_sync_cancel()
|
| A D | io-wq.h | 41 struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
65 void *data, bool cancel_all);
|
| A D | fdinfo.c | 171 seq_printf(m, "UserFiles:\t%u\n", ctx->file_table.data.nr); in __io_uring_show_fdinfo()
172 for (i = 0; i < ctx->file_table.data.nr; i++) { in __io_uring_show_fdinfo()
175 if (ctx->file_table.data.nodes[i]) in __io_uring_show_fdinfo()
176 f = io_slot_file(ctx->file_table.data.nodes[i]); in __io_uring_show_fdinfo()
|
| A D | cancel.h | 10 u64 data; member
|
| A D | napi.c | 144 static bool io_napi_busy_loop_should_end(void *data, in io_napi_busy_loop_should_end() argument
147 struct io_wait_queue *iowq = data; in io_napi_busy_loop_should_end()
|
| A D | splice.c | 70 node = io_rsrc_node_lookup(&ctx->file_table.data, sp->splice_fd_in); in io_splice_get_file()
|
| A D | sqpoll.c | 269 static int io_sq_thread(void *data) in io_sq_thread() argument
272 struct io_sq_data *sqd = data; in io_sq_thread()
|
| A D | poll.c | 755 u32 index = hash_long(cd->data, ctx->cancel_table.hash_bits); in io_poll_find()
759 if (cd->data != req->cqe.user_data) in io_poll_find()
913 struct io_cancel_data cd = { .ctx = ctx, .data = poll_update->old_user_data, }; in io_poll_remove()
|
| A D | zcrx.c | 1082 struct io_zcrx_args *args = desc->arg.data; in io_zcrx_recv_skb()
1107 copied = io_zcrx_copy_chunk(req, ifq, virt_to_page(skb->data), in io_zcrx_recv_skb()
1108 offset_in_page(skb->data) + offset, in io_zcrx_recv_skb()
1193 .arg.data = &args, in io_zcrx_tcp_recvmsg()
|
| A D | io_uring.c | 165 .data = &sysctl_io_uring_disabled,
174 .data = &sysctl_io_uring_group,
1953 node = io_rsrc_node_lookup(&ctx->file_table.data, fd); in io_file_get_fixed()
2100 memset(&req->cmd.data, 0, sizeof(req->cmd.data)); in io_init_fail_req()
2926 static __cold bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) in io_cancel_ctx_cb() argument
2930 return req->ctx == data; in io_cancel_ctx_cb()
3071 static bool io_cancel_task_cb(struct io_wq_work *work, void *data) in io_cancel_task_cb() argument
3074 struct io_task_cancel *cancel = data; in io_cancel_task_cb()
3930 .useroffset = offsetof(struct io_kiocb, cmd.data), in io_uring_init()
3931 .usersize = sizeof_field(struct io_kiocb, cmd.data), in io_uring_init()
|
| A D | msg_ring.c | 184 node = io_rsrc_node_lookup(&ctx->file_table.data, msg->src_fd); in io_msg_grab_file()
|
| A D | register.c | 931 trace_io_uring_register(ctx, opcode, ctx->file_table.data.nr, in SYSCALL_DEFINE4()
|