Lines Matching refs:dreq
72 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
75 static inline void get_dreq(struct nfs_direct_req *dreq) in get_dreq() argument
77 atomic_inc(&dreq->io_count); in get_dreq()
80 static inline int put_dreq(struct nfs_direct_req *dreq) in put_dreq() argument
82 return atomic_dec_and_test(&dreq->io_count); in put_dreq()
86 nfs_direct_handle_truncated(struct nfs_direct_req *dreq, in nfs_direct_handle_truncated() argument
93 if (dreq->max_count >= dreq_len) { in nfs_direct_handle_truncated()
94 dreq->max_count = dreq_len; in nfs_direct_handle_truncated()
95 if (dreq->count > dreq_len) in nfs_direct_handle_truncated()
96 dreq->count = dreq_len; in nfs_direct_handle_truncated()
99 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && !dreq->error) in nfs_direct_handle_truncated()
100 dreq->error = hdr->error; in nfs_direct_handle_truncated()
104 nfs_direct_count_bytes(struct nfs_direct_req *dreq, in nfs_direct_count_bytes() argument
110 if (hdr_end > dreq->io_start) in nfs_direct_count_bytes()
111 dreq_len = hdr_end - dreq->io_start; in nfs_direct_count_bytes()
113 nfs_direct_handle_truncated(dreq, hdr, dreq_len); in nfs_direct_count_bytes()
115 if (dreq_len > dreq->max_count) in nfs_direct_count_bytes()
116 dreq_len = dreq->max_count; in nfs_direct_count_bytes()
118 if (dreq->count < dreq_len) in nfs_direct_count_bytes()
119 dreq->count = dreq_len; in nfs_direct_count_bytes()
122 static void nfs_direct_truncate_request(struct nfs_direct_req *dreq, in nfs_direct_truncate_request() argument
126 size_t req_start = (size_t)(offs - dreq->io_start); in nfs_direct_truncate_request()
128 if (req_start < dreq->max_count) in nfs_direct_truncate_request()
129 dreq->max_count = req_start; in nfs_direct_truncate_request()
130 if (req_start < dreq->count) in nfs_direct_truncate_request()
131 dreq->count = req_start; in nfs_direct_truncate_request()
176 struct nfs_direct_req *dreq) in nfs_init_cinfo_from_dreq() argument
178 cinfo->inode = dreq->inode; in nfs_init_cinfo_from_dreq()
179 cinfo->mds = &dreq->mds_cinfo; in nfs_init_cinfo_from_dreq()
180 cinfo->ds = &dreq->ds_cinfo; in nfs_init_cinfo_from_dreq()
181 cinfo->dreq = dreq; in nfs_init_cinfo_from_dreq()
187 struct nfs_direct_req *dreq; in nfs_direct_req_alloc() local
189 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL); in nfs_direct_req_alloc()
190 if (!dreq) in nfs_direct_req_alloc()
193 kref_init(&dreq->kref); in nfs_direct_req_alloc()
194 kref_get(&dreq->kref); in nfs_direct_req_alloc()
195 init_completion(&dreq->completion); in nfs_direct_req_alloc()
196 INIT_LIST_HEAD(&dreq->mds_cinfo.list); in nfs_direct_req_alloc()
197 pnfs_init_ds_commit_info(&dreq->ds_cinfo); in nfs_direct_req_alloc()
198 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); in nfs_direct_req_alloc()
199 spin_lock_init(&dreq->lock); in nfs_direct_req_alloc()
201 return dreq; in nfs_direct_req_alloc()
206 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref); in nfs_direct_req_free() local
208 pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode); in nfs_direct_req_free()
209 if (dreq->l_ctx != NULL) in nfs_direct_req_free()
210 nfs_put_lock_context(dreq->l_ctx); in nfs_direct_req_free()
211 if (dreq->ctx != NULL) in nfs_direct_req_free()
212 put_nfs_open_context(dreq->ctx); in nfs_direct_req_free()
213 kmem_cache_free(nfs_direct_cachep, dreq); in nfs_direct_req_free()
216 static void nfs_direct_req_release(struct nfs_direct_req *dreq) in nfs_direct_req_release() argument
218 kref_put(&dreq->kref, nfs_direct_req_free); in nfs_direct_req_release()
221 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq, loff_t offset) in nfs_dreq_bytes_left() argument
223 loff_t start = offset - dreq->io_start; in nfs_dreq_bytes_left()
224 return dreq->max_count - start; in nfs_dreq_bytes_left()
231 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq) in nfs_direct_wait() argument
236 if (dreq->iocb) in nfs_direct_wait()
239 result = wait_for_completion_killable(&dreq->completion); in nfs_direct_wait()
242 result = dreq->count; in nfs_direct_wait()
243 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_wait()
246 result = dreq->error; in nfs_direct_wait()
256 static void nfs_direct_complete(struct nfs_direct_req *dreq) in nfs_direct_complete() argument
258 struct inode *inode = dreq->inode; in nfs_direct_complete()
262 if (dreq->iocb) { in nfs_direct_complete()
263 long res = (long) dreq->error; in nfs_direct_complete()
264 if (dreq->count != 0) { in nfs_direct_complete()
265 res = (long) dreq->count; in nfs_direct_complete()
266 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_complete()
268 dreq->iocb->ki_complete(dreq->iocb, res); in nfs_direct_complete()
271 complete(&dreq->completion); in nfs_direct_complete()
273 nfs_direct_req_release(dreq); in nfs_direct_complete()
279 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_read_completion() local
281 spin_lock(&dreq->lock); in nfs_direct_read_completion()
283 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
287 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_read_completion()
288 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
290 nfs_update_delegated_atime(dreq->inode); in nfs_direct_read_completion()
297 (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY)) in nfs_direct_read_completion()
304 if (put_dreq(dreq)) in nfs_direct_read_completion()
305 nfs_direct_complete(dreq); in nfs_direct_read_completion()
322 get_dreq(hdr->dreq); in nfs_direct_pgio_init()
340 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_read_schedule_iovec() argument
345 struct inode *inode = dreq->inode; in nfs_direct_read_schedule_iovec()
350 nfs_pageio_init_read(&desc, dreq->inode, false, in nfs_direct_read_schedule_iovec()
352 get_dreq(dreq); in nfs_direct_read_schedule_iovec()
353 desc.pg_dreq = dreq; in nfs_direct_read_schedule_iovec()
373 req = nfs_page_create_from_page(dreq->ctx, pagevec[i], in nfs_direct_read_schedule_iovec()
403 nfs_direct_req_release(dreq); in nfs_direct_read_schedule_iovec()
407 if (put_dreq(dreq)) in nfs_direct_read_schedule_iovec()
408 nfs_direct_complete(dreq); in nfs_direct_read_schedule_iovec()
438 struct nfs_direct_req *dreq; in nfs_file_direct_read() local
454 dreq = nfs_direct_req_alloc(); in nfs_file_direct_read()
455 if (dreq == NULL) in nfs_file_direct_read()
458 dreq->inode = inode; in nfs_file_direct_read()
459 dreq->max_count = count; in nfs_file_direct_read()
460 dreq->io_start = iocb->ki_pos; in nfs_file_direct_read()
461 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_read()
462 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_read()
465 nfs_direct_req_release(dreq); in nfs_file_direct_read()
468 dreq->l_ctx = l_ctx; in nfs_file_direct_read()
470 dreq->iocb = iocb; in nfs_file_direct_read()
473 dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; in nfs_file_direct_read()
481 nfs_direct_req_release(dreq); in nfs_file_direct_read()
487 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos); in nfs_file_direct_read()
493 result = nfs_direct_wait(dreq); in nfs_file_direct_read()
504 nfs_direct_req_release(dreq); in nfs_file_direct_read()
564 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) in nfs_direct_write_reschedule() argument
571 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_reschedule()
572 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_reschedule()
574 nfs_direct_join_group(&reqs, &cinfo, dreq->inode); in nfs_direct_write_reschedule()
576 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo); in nfs_direct_write_reschedule()
577 get_dreq(dreq); in nfs_direct_write_reschedule()
579 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false, in nfs_direct_write_reschedule()
581 desc.pg_dreq = dreq; in nfs_direct_write_reschedule()
588 spin_lock(&dreq->lock); in nfs_direct_write_reschedule()
589 if (dreq->error < 0) { in nfs_direct_write_reschedule()
590 desc.pg_error = dreq->error; in nfs_direct_write_reschedule()
592 dreq->flags = 0; in nfs_direct_write_reschedule()
595 dreq->error = desc.pg_error; in nfs_direct_write_reschedule()
597 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule()
598 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule()
612 spin_lock(&dreq->lock); in nfs_direct_write_reschedule()
613 nfs_direct_truncate_request(dreq, req); in nfs_direct_write_reschedule()
614 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule()
619 if (put_dreq(dreq)) in nfs_direct_write_reschedule()
620 nfs_direct_write_complete(dreq); in nfs_direct_write_reschedule()
626 struct nfs_direct_req *dreq = data->dreq; in nfs_direct_commit_complete() local
631 trace_nfs_direct_commit_complete(dreq); in nfs_direct_commit_complete()
633 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
636 dreq->error = status; in nfs_direct_commit_complete()
637 dreq->flags = NFS_ODIRECT_DONE; in nfs_direct_commit_complete()
639 status = dreq->error; in nfs_direct_commit_complete()
641 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
643 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_complete()
649 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
650 nfs_direct_truncate_request(dreq, req); in nfs_direct_commit_complete()
651 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
654 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
655 if (dreq->flags == 0) in nfs_direct_commit_complete()
656 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_complete()
657 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
670 nfs_direct_write_complete(dreq); in nfs_direct_commit_complete()
676 struct nfs_direct_req *dreq = cinfo->dreq; in nfs_direct_resched_write() local
678 trace_nfs_direct_resched_write(dreq); in nfs_direct_resched_write()
680 spin_lock(&dreq->lock); in nfs_direct_resched_write()
681 if (dreq->flags != NFS_ODIRECT_DONE) in nfs_direct_resched_write()
682 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_resched_write()
683 spin_unlock(&dreq->lock); in nfs_direct_resched_write()
692 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq) in nfs_direct_commit_schedule() argument
698 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_schedule()
700 nfs_scan_commit(dreq->inode, &mds_list, &cinfo); in nfs_direct_commit_schedule()
701 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo); in nfs_direct_commit_schedule()
703 spin_lock(&dreq->lock); in nfs_direct_commit_schedule()
704 if (dreq->flags == 0) in nfs_direct_commit_schedule()
705 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_schedule()
706 spin_unlock(&dreq->lock); in nfs_direct_commit_schedule()
709 nfs_direct_write_complete(dreq); in nfs_direct_commit_schedule()
712 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq) in nfs_direct_write_clear_reqs() argument
718 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_clear_reqs()
719 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_clear_reqs()
724 nfs_direct_truncate_request(dreq, req); in nfs_direct_write_clear_reqs()
732 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work); in nfs_direct_write_schedule_work() local
733 int flags = dreq->flags; in nfs_direct_write_schedule_work()
735 dreq->flags = 0; in nfs_direct_write_schedule_work()
738 nfs_direct_commit_schedule(dreq); in nfs_direct_write_schedule_work()
741 nfs_direct_write_reschedule(dreq); in nfs_direct_write_schedule_work()
744 nfs_direct_write_clear_reqs(dreq); in nfs_direct_write_schedule_work()
745 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping); in nfs_direct_write_schedule_work()
746 nfs_direct_complete(dreq); in nfs_direct_write_schedule_work()
750 static void nfs_direct_write_complete(struct nfs_direct_req *dreq) in nfs_direct_write_complete() argument
752 trace_nfs_direct_write_complete(dreq); in nfs_direct_write_complete()
753 queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */ in nfs_direct_write_complete()
758 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_completion() local
760 struct inode *inode = dreq->inode; in nfs_direct_write_completion()
763 trace_nfs_direct_write_completion(dreq); in nfs_direct_write_completion()
765 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_completion()
767 spin_lock(&dreq->lock); in nfs_direct_write_completion()
769 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
773 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_write_completion()
776 if (!dreq->flags) in nfs_direct_write_completion()
777 dreq->flags = NFS_ODIRECT_DO_COMMIT; in nfs_direct_write_completion()
778 flags = dreq->flags; in nfs_direct_write_completion()
780 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
783 nfs_direct_file_adjust_size_locked(inode, dreq->io_start, dreq->count); in nfs_direct_write_completion()
784 nfs_update_delegated_mtime_locked(dreq->inode); in nfs_direct_write_completion()
806 if (put_dreq(dreq)) in nfs_direct_write_completion()
807 nfs_direct_write_complete(dreq); in nfs_direct_write_completion()
824 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_reschedule_io() local
828 trace_nfs_direct_write_reschedule_io(dreq); in nfs_direct_write_reschedule_io()
830 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_reschedule_io()
831 spin_lock(&dreq->lock); in nfs_direct_write_reschedule_io()
832 if (dreq->error == 0) in nfs_direct_write_reschedule_io()
833 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule_io()
835 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule_io()
863 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_write_schedule_iovec() argument
868 struct inode *inode = dreq->inode; in nfs_direct_write_schedule_iovec()
875 trace_nfs_direct_write_schedule_iovec(dreq); in nfs_direct_write_schedule_iovec()
879 desc.pg_dreq = dreq; in nfs_direct_write_schedule_iovec()
880 get_dreq(dreq); in nfs_direct_write_schedule_iovec()
901 req = nfs_page_create_from_page(dreq->ctx, pagevec[i], in nfs_direct_write_schedule_iovec()
936 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_schedule_iovec()
937 spin_lock(&dreq->lock); in nfs_direct_write_schedule_iovec()
938 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_schedule_iovec()
939 spin_unlock(&dreq->lock); in nfs_direct_write_schedule_iovec()
958 nfs_direct_req_release(dreq); in nfs_direct_write_schedule_iovec()
962 if (put_dreq(dreq)) in nfs_direct_write_schedule_iovec()
963 nfs_direct_write_complete(dreq); in nfs_direct_write_schedule_iovec()
996 struct nfs_direct_req *dreq; in nfs_file_direct_write() local
1019 dreq = nfs_direct_req_alloc(); in nfs_file_direct_write()
1020 if (!dreq) in nfs_file_direct_write()
1023 dreq->inode = inode; in nfs_file_direct_write()
1024 dreq->max_count = count; in nfs_file_direct_write()
1025 dreq->io_start = pos; in nfs_file_direct_write()
1026 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_write()
1027 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_write()
1030 nfs_direct_req_release(dreq); in nfs_file_direct_write()
1033 dreq->l_ctx = l_ctx; in nfs_file_direct_write()
1035 dreq->iocb = iocb; in nfs_file_direct_write()
1036 pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode); in nfs_file_direct_write()
1039 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
1047 nfs_direct_req_release(dreq); in nfs_file_direct_write()
1051 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
1063 result = nfs_direct_wait(dreq); in nfs_file_direct_write()
1076 nfs_direct_req_release(dreq); in nfs_file_direct_write()