Lines Matching refs:dio
55 struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf) in iomap_dio_alloc_bio() argument
57 if (dio->dops && dio->dops->bio_set) in iomap_dio_alloc_bio()
59 GFP_KERNEL, dio->dops->bio_set); in iomap_dio_alloc_bio()
64 struct iomap_dio *dio, struct bio *bio, loff_t pos) in iomap_dio_submit_bio() argument
66 atomic_inc(&dio->ref); in iomap_dio_submit_bio()
69 if ((dio->iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(dio->iocb)) { in iomap_dio_submit_bio()
70 bio_set_polled(bio, dio->iocb); in iomap_dio_submit_bio()
71 dio->submit.poll_bio = bio; in iomap_dio_submit_bio()
74 if (dio->dops && dio->dops->submit_io) in iomap_dio_submit_bio()
75 dio->dops->submit_io(iter, bio, pos); in iomap_dio_submit_bio()
80 ssize_t iomap_dio_complete(struct iomap_dio *dio) in iomap_dio_complete() argument
82 const struct iomap_dio_ops *dops = dio->dops; in iomap_dio_complete()
83 struct kiocb *iocb = dio->iocb; in iomap_dio_complete()
86 ssize_t ret = dio->error; in iomap_dio_complete()
89 ret = dops->end_io(iocb, dio->size, ret, dio->flags); in iomap_dio_complete()
92 ret = dio->size; in iomap_dio_complete()
94 if (offset + ret > dio->i_size && in iomap_dio_complete()
95 !(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_complete()
96 ret = dio->i_size - offset; in iomap_dio_complete()
112 if (!dio->error && dio->size && in iomap_dio_complete()
113 (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) { in iomap_dio_complete()
117 (offset + dio->size - 1) >> PAGE_SHIFT); in iomap_dio_complete()
127 if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC)) in iomap_dio_complete()
131 ret += dio->done_before; in iomap_dio_complete()
133 kfree(dio); in iomap_dio_complete()
141 struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work); in iomap_dio_complete_work() local
142 struct kiocb *iocb = dio->iocb; in iomap_dio_complete_work()
144 iocb->ki_complete(iocb, iomap_dio_complete(dio)); in iomap_dio_complete_work()
152 static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret) in iomap_dio_set_error() argument
154 cmpxchg(&dio->error, 0, ret); in iomap_dio_set_error()
159 struct iomap_dio *dio = bio->bi_private; in iomap_dio_bio_end_io() local
160 bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY); in iomap_dio_bio_end_io()
163 iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status)); in iomap_dio_bio_end_io()
165 if (atomic_dec_and_test(&dio->ref)) { in iomap_dio_bio_end_io()
166 if (dio->wait_for_completion) { in iomap_dio_bio_end_io()
167 struct task_struct *waiter = dio->submit.waiter; in iomap_dio_bio_end_io()
168 WRITE_ONCE(dio->submit.waiter, NULL); in iomap_dio_bio_end_io()
170 } else if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_bio_end_io()
171 struct inode *inode = file_inode(dio->iocb->ki_filp); in iomap_dio_bio_end_io()
173 WRITE_ONCE(dio->iocb->private, NULL); in iomap_dio_bio_end_io()
174 INIT_WORK(&dio->aio.work, iomap_dio_complete_work); in iomap_dio_bio_end_io()
175 queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work); in iomap_dio_bio_end_io()
177 WRITE_ONCE(dio->iocb->private, NULL); in iomap_dio_bio_end_io()
178 iomap_dio_complete_work(&dio->aio.work); in iomap_dio_bio_end_io()
191 static void iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio, in iomap_dio_zero() argument
194 struct inode *inode = file_inode(dio->iocb->ki_filp); in iomap_dio_zero()
198 bio = iomap_dio_alloc_bio(iter, dio, 1, REQ_OP_WRITE | REQ_SYNC | REQ_IDLE); in iomap_dio_zero()
202 bio->bi_private = dio; in iomap_dio_zero()
207 iomap_dio_submit_bio(iter, dio, bio, pos); in iomap_dio_zero()
215 static inline blk_opf_t iomap_dio_bio_opflags(struct iomap_dio *dio, in iomap_dio_bio_opflags() argument
220 if (!(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_bio_opflags()
227 dio->flags &= ~IOMAP_DIO_WRITE_FUA; in iomap_dio_bio_opflags()
233 struct iomap_dio *dio) in iomap_dio_bio_iter() argument
249 !bdev_iter_is_aligned(iomap->bdev, dio->submit.iter)) in iomap_dio_bio_iter()
253 dio->flags |= IOMAP_DIO_UNWRITTEN; in iomap_dio_bio_iter()
258 dio->flags |= IOMAP_DIO_COW; in iomap_dio_bio_iter()
271 (dio->flags & IOMAP_DIO_WRITE_FUA) && bdev_fua(iomap->bdev)) in iomap_dio_bio_iter()
280 orig_count = iov_iter_count(dio->submit.iter); in iomap_dio_bio_iter()
281 iov_iter_truncate(dio->submit.iter, length); in iomap_dio_bio_iter()
283 if (!iov_iter_count(dio->submit.iter)) in iomap_dio_bio_iter()
290 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) in iomap_dio_bio_iter()
291 dio->iocb->ki_flags &= ~IOCB_HIPRI; in iomap_dio_bio_iter()
297 iomap_dio_zero(iter, dio, pos - pad, pad); in iomap_dio_bio_iter()
305 bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua); in iomap_dio_bio_iter()
307 nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS); in iomap_dio_bio_iter()
310 if (dio->error) { in iomap_dio_bio_iter()
311 iov_iter_revert(dio->submit.iter, copied); in iomap_dio_bio_iter()
316 bio = iomap_dio_alloc_bio(iter, dio, nr_pages, bio_opf); in iomap_dio_bio_iter()
320 bio->bi_ioprio = dio->iocb->ki_ioprio; in iomap_dio_bio_iter()
321 bio->bi_private = dio; in iomap_dio_bio_iter()
324 ret = bio_iov_iter_get_pages(bio, dio->submit.iter); in iomap_dio_bio_iter()
337 if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_bio_iter()
340 if (dio->flags & IOMAP_DIO_DIRTY) in iomap_dio_bio_iter()
344 dio->size += n; in iomap_dio_bio_iter()
347 nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, in iomap_dio_bio_iter()
353 dio->iocb->ki_flags &= ~IOCB_HIPRI; in iomap_dio_bio_iter()
354 iomap_dio_submit_bio(iter, dio, bio, pos); in iomap_dio_bio_iter()
366 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) { in iomap_dio_bio_iter()
370 iomap_dio_zero(iter, dio, pos, fs_block_size - pad); in iomap_dio_bio_iter()
374 iov_iter_reexpand(dio->submit.iter, orig_count - copied); in iomap_dio_bio_iter()
381 struct iomap_dio *dio) in iomap_dio_hole_iter() argument
383 loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter); in iomap_dio_hole_iter()
385 dio->size += length; in iomap_dio_hole_iter()
392 struct iomap_dio *dio) in iomap_dio_inline_iter() argument
395 struct iov_iter *iter = dio->submit.iter; in iomap_dio_inline_iter()
404 if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_inline_iter()
418 dio->size += copied; in iomap_dio_inline_iter()
425 struct iomap_dio *dio) in iomap_dio_iter() argument
429 if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_iter()
431 return iomap_dio_hole_iter(iter, dio); in iomap_dio_iter()
433 if (!(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_iter()
434 return iomap_dio_hole_iter(iter, dio); in iomap_dio_iter()
435 return iomap_dio_bio_iter(iter, dio); in iomap_dio_iter()
437 return iomap_dio_bio_iter(iter, dio); in iomap_dio_iter()
439 return iomap_dio_inline_iter(iter, dio); in iomap_dio_iter()
448 dio->iocb->ki_filp, current->comm); in iomap_dio_iter()
494 struct iomap_dio *dio; in __iomap_dio_rw() local
499 dio = kmalloc(sizeof(*dio), GFP_KERNEL); in __iomap_dio_rw()
500 if (!dio) in __iomap_dio_rw()
503 dio->iocb = iocb; in __iomap_dio_rw()
504 atomic_set(&dio->ref, 1); in __iomap_dio_rw()
505 dio->size = 0; in __iomap_dio_rw()
506 dio->i_size = i_size_read(inode); in __iomap_dio_rw()
507 dio->dops = dops; in __iomap_dio_rw()
508 dio->error = 0; in __iomap_dio_rw()
509 dio->flags = 0; in __iomap_dio_rw()
510 dio->done_before = done_before; in __iomap_dio_rw()
512 dio->submit.iter = iter; in __iomap_dio_rw()
513 dio->submit.waiter = current; in __iomap_dio_rw()
514 dio->submit.poll_bio = NULL; in __iomap_dio_rw()
517 if (iomi.pos >= dio->i_size) in __iomap_dio_rw()
530 dio->flags |= IOMAP_DIO_DIRTY; in __iomap_dio_rw()
533 dio->flags |= IOMAP_DIO_WRITE; in __iomap_dio_rw()
545 dio->flags |= IOMAP_DIO_NEED_SYNC; in __iomap_dio_rw()
554 dio->flags |= IOMAP_DIO_WRITE_FUA; in __iomap_dio_rw()
560 if (iomi.pos >= dio->i_size || in __iomap_dio_rw()
561 iomi.pos + iomi.len > dio->i_size) in __iomap_dio_rw()
595 iomi.processed = iomap_dio_iter(&iomi, dio); in __iomap_dio_rw()
610 if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size) in __iomap_dio_rw()
611 iov_iter_revert(iter, iomi.pos - dio->i_size); in __iomap_dio_rw()
613 if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) { in __iomap_dio_rw()
625 iomap_dio_set_error(dio, ret); in __iomap_dio_rw()
631 if (dio->flags & IOMAP_DIO_WRITE_FUA) in __iomap_dio_rw()
632 dio->flags &= ~IOMAP_DIO_NEED_SYNC; in __iomap_dio_rw()
634 WRITE_ONCE(iocb->private, dio->submit.poll_bio); in __iomap_dio_rw()
651 dio->wait_for_completion = wait_for_completion; in __iomap_dio_rw()
652 if (!atomic_dec_and_test(&dio->ref)) { in __iomap_dio_rw()
658 if (!READ_ONCE(dio->submit.waiter)) in __iomap_dio_rw()
666 return dio; in __iomap_dio_rw()
669 kfree(dio); in __iomap_dio_rw()
681 struct iomap_dio *dio; in iomap_dio_rw() local
683 dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, in iomap_dio_rw()
685 if (IS_ERR_OR_NULL(dio)) in iomap_dio_rw()
686 return PTR_ERR_OR_ZERO(dio); in iomap_dio_rw()
687 return iomap_dio_complete(dio); in iomap_dio_rw()