Lines Matching refs:dio
40 closure_type(dio, struct dio_read, cl); in CLOSURE_CALLBACK()
42 dio->req->ki_complete(dio->req, dio->ret); in CLOSURE_CALLBACK()
43 bio_check_or_release(&dio->rbio.bio, dio->should_dirty); in CLOSURE_CALLBACK()
48 struct dio_read *dio = bio->bi_private; in bch2_direct_IO_read_endio() local
51 dio->ret = blk_status_to_errno(bio->bi_status); in bch2_direct_IO_read_endio()
53 closure_put(&dio->cl); in bch2_direct_IO_read_endio()
58 struct dio_read *dio = bio->bi_private; in bch2_direct_IO_read_split_endio() local
59 bool should_dirty = dio->should_dirty; in bch2_direct_IO_read_split_endio()
71 struct dio_read *dio; in bch2_direct_IO_read() local
103 dio = container_of(bio, struct dio_read, rbio.bio); in bch2_direct_IO_read()
104 closure_init(&dio->cl, NULL); in bch2_direct_IO_read()
111 set_closure_fn(&dio->cl, bch2_dio_read_complete, NULL); in bch2_direct_IO_read()
112 atomic_set(&dio->cl.remaining, in bch2_direct_IO_read()
117 atomic_set(&dio->cl.remaining, in bch2_direct_IO_read()
119 dio->cl.closure_get_happened = true; in bch2_direct_IO_read()
122 dio->req = req; in bch2_direct_IO_read()
123 dio->ret = ret; in bch2_direct_IO_read()
129 dio->should_dirty = iter_is_iovec(iter); in bch2_direct_IO_read()
142 bio->bi_private = dio; in bch2_direct_IO_read()
154 if (dio->should_dirty) in bch2_direct_IO_read()
158 closure_get(&dio->cl); in bch2_direct_IO_read()
166 closure_sync(&dio->cl); in bch2_direct_IO_read()
167 closure_debug_destroy(&dio->cl); in bch2_direct_IO_read()
168 ret = dio->ret; in bch2_direct_IO_read()
169 bio_check_or_release(&dio->rbio.bio, dio->should_dirty); in bch2_direct_IO_read()
279 static noinline bool bch2_dio_write_check_allocated(struct dio_write *dio) in bch2_dio_write_check_allocated() argument
281 struct bch_fs *c = dio->op.c; in bch2_dio_write_check_allocated()
282 struct bch_inode_info *inode = dio->inode; in bch2_dio_write_check_allocated()
283 struct bio *bio = &dio->op.wbio.bio; in bch2_dio_write_check_allocated()
286 dio->op.pos.offset, bio_sectors(bio), in bch2_dio_write_check_allocated()
287 dio->op.opts.data_replicas, in bch2_dio_write_check_allocated()
288 dio->op.opts.compression != 0); in bch2_dio_write_check_allocated()
292 static __always_inline long bch2_dio_write_done(struct dio_write *dio);
300 static noinline int bch2_dio_write_copy_iov(struct dio_write *dio) in bch2_dio_write_copy_iov() argument
302 struct iovec *iov = dio->inline_vecs; in bch2_dio_write_copy_iov()
307 if (iter_is_ubuf(&dio->iter)) in bch2_dio_write_copy_iov()
314 if (!iter_is_iovec(&dio->iter)) in bch2_dio_write_copy_iov()
317 if (dio->iter.nr_segs > ARRAY_SIZE(dio->inline_vecs)) { in bch2_dio_write_copy_iov()
318 dio->iov = iov = kmalloc_array(dio->iter.nr_segs, sizeof(*iov), in bch2_dio_write_copy_iov()
324 memcpy(iov, dio->iter.__iov, dio->iter.nr_segs * sizeof(*iov)); in bch2_dio_write_copy_iov()
325 dio->iter.__iov = iov; in bch2_dio_write_copy_iov()
331 closure_type(dio, struct dio_write, op.cl); in CLOSURE_CALLBACK()
332 struct bch_fs *c = dio->op.c; in CLOSURE_CALLBACK()
336 dio->op.error = bch2_journal_error(&c->journal); in CLOSURE_CALLBACK()
338 bch2_dio_write_done(dio); in CLOSURE_CALLBACK()
341 static noinline void bch2_dio_write_flush(struct dio_write *dio) in bch2_dio_write_flush() argument
343 struct bch_fs *c = dio->op.c; in bch2_dio_write_flush()
347 dio->flush = 0; in bch2_dio_write_flush()
349 closure_init(&dio->op.cl, NULL); in bch2_dio_write_flush()
351 if (!dio->op.error) { in bch2_dio_write_flush()
352 ret = bch2_inode_find_by_inum(c, inode_inum(dio->inode), &inode); in bch2_dio_write_flush()
354 dio->op.error = ret; in bch2_dio_write_flush()
357 &dio->op.cl); in bch2_dio_write_flush()
358 bch2_inode_flush_nocow_writes_async(c, dio->inode, &dio->op.cl); in bch2_dio_write_flush()
362 if (dio->sync) { in bch2_dio_write_flush()
363 closure_sync(&dio->op.cl); in bch2_dio_write_flush()
364 closure_debug_destroy(&dio->op.cl); in bch2_dio_write_flush()
366 continue_at(&dio->op.cl, bch2_dio_write_flush_done, NULL); in bch2_dio_write_flush()
370 static __always_inline long bch2_dio_write_done(struct dio_write *dio) in bch2_dio_write_done() argument
372 struct bch_fs *c = dio->op.c; in bch2_dio_write_done()
373 struct kiocb *req = dio->req; in bch2_dio_write_done()
374 struct bch_inode_info *inode = dio->inode; in bch2_dio_write_done()
375 bool sync = dio->sync; in bch2_dio_write_done()
378 if (unlikely(dio->flush)) { in bch2_dio_write_done()
379 bch2_dio_write_flush(dio); in bch2_dio_write_done()
386 kfree(dio->iov); in bch2_dio_write_done()
388 ret = dio->op.error ?: ((long) dio->written << 9); in bch2_dio_write_done()
389 bio_put(&dio->op.wbio.bio); in bch2_dio_write_done()
406 static __always_inline void bch2_dio_write_end(struct dio_write *dio) in bch2_dio_write_end() argument
408 struct bch_fs *c = dio->op.c; in bch2_dio_write_end()
409 struct kiocb *req = dio->req; in bch2_dio_write_end()
410 struct bch_inode_info *inode = dio->inode; in bch2_dio_write_end()
411 struct bio *bio = &dio->op.wbio.bio; in bch2_dio_write_end()
413 req->ki_pos += (u64) dio->op.written << 9; in bch2_dio_write_end()
414 dio->written += dio->op.written; in bch2_dio_write_end()
416 if (dio->extending) { in bch2_dio_write_end()
423 if (dio->op.i_sectors_delta || dio->quota_res.sectors) { in bch2_dio_write_end()
425 __bch2_i_sectors_acct(c, inode, &dio->quota_res, dio->op.i_sectors_delta); in bch2_dio_write_end()
426 __bch2_quota_reservation_put(c, inode, &dio->quota_res); in bch2_dio_write_end()
432 if (unlikely(dio->op.error)) in bch2_dio_write_end()
436 static __always_inline long bch2_dio_write_loop(struct dio_write *dio) in bch2_dio_write_loop() argument
438 struct bch_fs *c = dio->op.c; in bch2_dio_write_loop()
439 struct kiocb *req = dio->req; in bch2_dio_write_loop()
440 struct address_space *mapping = dio->mapping; in bch2_dio_write_loop()
441 struct bch_inode_info *inode = dio->inode; in bch2_dio_write_loop()
443 struct bio *bio = &dio->op.wbio.bio; in bch2_dio_write_loop()
445 bool sync = dio->sync, dropped_locks; in bch2_dio_write_loop()
451 iter_count = dio->iter.count; in bch2_dio_write_loop()
456 ret = bio_iov_iter_get_pages(bio, &dio->iter); in bch2_dio_write_loop()
486 iov_iter_revert(&dio->iter, unaligned); in bch2_dio_write_loop()
497 bch2_write_op_init(&dio->op, c, opts); in bch2_dio_write_loop()
498 dio->op.end_io = sync in bch2_dio_write_loop()
501 dio->op.target = dio->op.opts.foreground_target; in bch2_dio_write_loop()
502 dio->op.write_point = writepoint_hashed((unsigned long) current); in bch2_dio_write_loop()
503 dio->op.nr_replicas = dio->op.opts.data_replicas; in bch2_dio_write_loop()
504 dio->op.subvol = inode->ei_inum.subvol; in bch2_dio_write_loop()
505 dio->op.pos = POS(inode->v.i_ino, (u64) req->ki_pos >> 9); in bch2_dio_write_loop()
506 dio->op.devs_need_flush = &inode->ei_devs_need_flush; in bch2_dio_write_loop()
509 dio->op.flags |= BCH_WRITE_SYNC; in bch2_dio_write_loop()
510 dio->op.flags |= BCH_WRITE_CHECK_ENOSPC; in bch2_dio_write_loop()
512 ret = bch2_quota_reservation_add(c, inode, &dio->quota_res, in bch2_dio_write_loop()
517 ret = bch2_disk_reservation_get(c, &dio->op.res, bio_sectors(bio), in bch2_dio_write_loop()
518 dio->op.opts.data_replicas, 0); in bch2_dio_write_loop()
520 !bch2_dio_write_check_allocated(dio)) in bch2_dio_write_loop()
525 if (unlikely(dio->iter.count) && in bch2_dio_write_loop()
526 !dio->sync && in bch2_dio_write_loop()
527 !dio->loop && in bch2_dio_write_loop()
528 bch2_dio_write_copy_iov(dio)) in bch2_dio_write_loop()
529 dio->sync = sync = true; in bch2_dio_write_loop()
531 dio->loop = true; in bch2_dio_write_loop()
532 closure_call(&dio->op.cl, bch2_write, NULL, NULL); in bch2_dio_write_loop()
537 bch2_dio_write_end(dio); in bch2_dio_write_loop()
539 if (likely(!dio->iter.count) || dio->op.error) in bch2_dio_write_loop()
545 return bch2_dio_write_done(dio); in bch2_dio_write_loop()
547 dio->op.error = ret; in bch2_dio_write_loop()
551 bch2_quota_reservation_put(c, inode, &dio->quota_res); in bch2_dio_write_loop()
555 static noinline __cold void bch2_dio_write_continue(struct dio_write *dio) in bch2_dio_write_continue() argument
557 struct mm_struct *mm = dio->mm; in bch2_dio_write_continue()
559 bio_reset(&dio->op.wbio.bio, NULL, REQ_OP_WRITE); in bch2_dio_write_continue()
563 bch2_dio_write_loop(dio); in bch2_dio_write_continue()
570 struct dio_write *dio = container_of(op, struct dio_write, op); in bch2_dio_write_loop_async() local
572 bch2_dio_write_end(dio); in bch2_dio_write_loop_async()
574 if (likely(!dio->iter.count) || dio->op.error) in bch2_dio_write_loop_async()
575 bch2_dio_write_done(dio); in bch2_dio_write_loop_async()
577 bch2_dio_write_continue(dio); in bch2_dio_write_loop_async()
586 struct dio_write *dio; in bch2_direct_write() local
632 dio = container_of(bio, struct dio_write, op.wbio.bio); in bch2_direct_write()
633 dio->req = req; in bch2_direct_write()
634 dio->mapping = mapping; in bch2_direct_write()
635 dio->inode = inode; in bch2_direct_write()
636 dio->mm = current->mm; in bch2_direct_write()
637 dio->iov = NULL; in bch2_direct_write()
638 dio->loop = false; in bch2_direct_write()
639 dio->extending = extending; in bch2_direct_write()
640 dio->sync = is_sync_kiocb(req) || extending; in bch2_direct_write()
641 dio->flush = iocb_is_dsync(req) && !c->opts.journal_flush_disabled; in bch2_direct_write()
642 dio->quota_res.sectors = 0; in bch2_direct_write()
643 dio->written = 0; in bch2_direct_write()
644 dio->iter = *iter; in bch2_direct_write()
645 dio->op.c = c; in bch2_direct_write()
655 ret = bch2_dio_write_loop(dio); in bch2_direct_write()