Lines Matching refs:dreq
71 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
74 static inline void get_dreq(struct nfs_direct_req *dreq) in get_dreq() argument
76 atomic_inc(&dreq->io_count); in get_dreq()
79 static inline int put_dreq(struct nfs_direct_req *dreq) in put_dreq() argument
81 return atomic_dec_and_test(&dreq->io_count); in put_dreq()
85 nfs_direct_handle_truncated(struct nfs_direct_req *dreq, in nfs_direct_handle_truncated() argument
92 if (dreq->max_count >= dreq_len) { in nfs_direct_handle_truncated()
93 dreq->max_count = dreq_len; in nfs_direct_handle_truncated()
94 if (dreq->count > dreq_len) in nfs_direct_handle_truncated()
95 dreq->count = dreq_len; in nfs_direct_handle_truncated()
98 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && !dreq->error) in nfs_direct_handle_truncated()
99 dreq->error = hdr->error; in nfs_direct_handle_truncated()
103 nfs_direct_count_bytes(struct nfs_direct_req *dreq, in nfs_direct_count_bytes() argument
109 if (hdr_end > dreq->io_start) in nfs_direct_count_bytes()
110 dreq_len = hdr_end - dreq->io_start; in nfs_direct_count_bytes()
112 nfs_direct_handle_truncated(dreq, hdr, dreq_len); in nfs_direct_count_bytes()
114 if (dreq_len > dreq->max_count) in nfs_direct_count_bytes()
115 dreq_len = dreq->max_count; in nfs_direct_count_bytes()
117 if (dreq->count < dreq_len) in nfs_direct_count_bytes()
118 dreq->count = dreq_len; in nfs_direct_count_bytes()
121 static void nfs_direct_truncate_request(struct nfs_direct_req *dreq, in nfs_direct_truncate_request() argument
125 size_t req_start = (size_t)(offs - dreq->io_start); in nfs_direct_truncate_request()
127 if (req_start < dreq->max_count) in nfs_direct_truncate_request()
128 dreq->max_count = req_start; in nfs_direct_truncate_request()
129 if (req_start < dreq->count) in nfs_direct_truncate_request()
130 dreq->count = req_start; in nfs_direct_truncate_request()
161 struct nfs_direct_req *dreq) in nfs_init_cinfo_from_dreq() argument
163 cinfo->inode = dreq->inode; in nfs_init_cinfo_from_dreq()
164 cinfo->mds = &dreq->mds_cinfo; in nfs_init_cinfo_from_dreq()
165 cinfo->ds = &dreq->ds_cinfo; in nfs_init_cinfo_from_dreq()
166 cinfo->dreq = dreq; in nfs_init_cinfo_from_dreq()
172 struct nfs_direct_req *dreq; in nfs_direct_req_alloc() local
174 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL); in nfs_direct_req_alloc()
175 if (!dreq) in nfs_direct_req_alloc()
178 kref_init(&dreq->kref); in nfs_direct_req_alloc()
179 kref_get(&dreq->kref); in nfs_direct_req_alloc()
180 init_completion(&dreq->completion); in nfs_direct_req_alloc()
181 INIT_LIST_HEAD(&dreq->mds_cinfo.list); in nfs_direct_req_alloc()
182 pnfs_init_ds_commit_info(&dreq->ds_cinfo); in nfs_direct_req_alloc()
183 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); in nfs_direct_req_alloc()
184 spin_lock_init(&dreq->lock); in nfs_direct_req_alloc()
186 return dreq; in nfs_direct_req_alloc()
191 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref); in nfs_direct_req_free() local
193 pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode); in nfs_direct_req_free()
194 if (dreq->l_ctx != NULL) in nfs_direct_req_free()
195 nfs_put_lock_context(dreq->l_ctx); in nfs_direct_req_free()
196 if (dreq->ctx != NULL) in nfs_direct_req_free()
197 put_nfs_open_context(dreq->ctx); in nfs_direct_req_free()
198 kmem_cache_free(nfs_direct_cachep, dreq); in nfs_direct_req_free()
201 static void nfs_direct_req_release(struct nfs_direct_req *dreq) in nfs_direct_req_release() argument
203 kref_put(&dreq->kref, nfs_direct_req_free); in nfs_direct_req_release()
206 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq, loff_t offset) in nfs_dreq_bytes_left() argument
208 loff_t start = offset - dreq->io_start; in nfs_dreq_bytes_left()
209 return dreq->max_count - start; in nfs_dreq_bytes_left()
216 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq) in nfs_direct_wait() argument
221 if (dreq->iocb) in nfs_direct_wait()
224 result = wait_for_completion_killable(&dreq->completion); in nfs_direct_wait()
227 result = dreq->count; in nfs_direct_wait()
228 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_wait()
231 result = dreq->error; in nfs_direct_wait()
241 static void nfs_direct_complete(struct nfs_direct_req *dreq) in nfs_direct_complete() argument
243 struct inode *inode = dreq->inode; in nfs_direct_complete()
247 if (dreq->iocb) { in nfs_direct_complete()
248 long res = (long) dreq->error; in nfs_direct_complete()
249 if (dreq->count != 0) { in nfs_direct_complete()
250 res = (long) dreq->count; in nfs_direct_complete()
251 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_complete()
253 dreq->iocb->ki_complete(dreq->iocb, res); in nfs_direct_complete()
256 complete(&dreq->completion); in nfs_direct_complete()
258 nfs_direct_req_release(dreq); in nfs_direct_complete()
264 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_read_completion() local
266 spin_lock(&dreq->lock); in nfs_direct_read_completion()
268 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
272 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_read_completion()
273 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
280 (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY)) in nfs_direct_read_completion()
287 if (put_dreq(dreq)) in nfs_direct_read_completion()
288 nfs_direct_complete(dreq); in nfs_direct_read_completion()
305 get_dreq(hdr->dreq); in nfs_direct_pgio_init()
322 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_read_schedule_iovec() argument
327 struct inode *inode = dreq->inode; in nfs_direct_read_schedule_iovec()
332 nfs_pageio_init_read(&desc, dreq->inode, false, in nfs_direct_read_schedule_iovec()
334 get_dreq(dreq); in nfs_direct_read_schedule_iovec()
335 desc.pg_dreq = dreq; in nfs_direct_read_schedule_iovec()
355 req = nfs_page_create_from_page(dreq->ctx, pagevec[i], in nfs_direct_read_schedule_iovec()
385 nfs_direct_req_release(dreq); in nfs_direct_read_schedule_iovec()
389 if (put_dreq(dreq)) in nfs_direct_read_schedule_iovec()
390 nfs_direct_complete(dreq); in nfs_direct_read_schedule_iovec()
420 struct nfs_direct_req *dreq; in nfs_file_direct_read() local
436 dreq = nfs_direct_req_alloc(); in nfs_file_direct_read()
437 if (dreq == NULL) in nfs_file_direct_read()
440 dreq->inode = inode; in nfs_file_direct_read()
441 dreq->max_count = count; in nfs_file_direct_read()
442 dreq->io_start = iocb->ki_pos; in nfs_file_direct_read()
443 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_read()
444 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_read()
447 nfs_direct_req_release(dreq); in nfs_file_direct_read()
450 dreq->l_ctx = l_ctx; in nfs_file_direct_read()
452 dreq->iocb = iocb; in nfs_file_direct_read()
455 dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; in nfs_file_direct_read()
461 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos); in nfs_file_direct_read()
467 result = nfs_direct_wait(dreq); in nfs_file_direct_read()
478 nfs_direct_req_release(dreq); in nfs_file_direct_read()
538 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) in nfs_direct_write_reschedule() argument
545 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_reschedule()
546 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_reschedule()
548 nfs_direct_join_group(&reqs, &cinfo, dreq->inode); in nfs_direct_write_reschedule()
550 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo); in nfs_direct_write_reschedule()
551 get_dreq(dreq); in nfs_direct_write_reschedule()
553 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false, in nfs_direct_write_reschedule()
555 desc.pg_dreq = dreq; in nfs_direct_write_reschedule()
562 spin_lock(&dreq->lock); in nfs_direct_write_reschedule()
563 if (dreq->error < 0) { in nfs_direct_write_reschedule()
564 desc.pg_error = dreq->error; in nfs_direct_write_reschedule()
566 dreq->flags = 0; in nfs_direct_write_reschedule()
569 dreq->error = desc.pg_error; in nfs_direct_write_reschedule()
571 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule()
572 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule()
586 spin_lock(&dreq->lock); in nfs_direct_write_reschedule()
587 nfs_direct_truncate_request(dreq, req); in nfs_direct_write_reschedule()
588 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule()
593 if (put_dreq(dreq)) in nfs_direct_write_reschedule()
594 nfs_direct_write_complete(dreq); in nfs_direct_write_reschedule()
600 struct nfs_direct_req *dreq = data->dreq; in nfs_direct_commit_complete() local
605 trace_nfs_direct_commit_complete(dreq); in nfs_direct_commit_complete()
607 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
610 dreq->error = status; in nfs_direct_commit_complete()
611 dreq->flags = NFS_ODIRECT_DONE; in nfs_direct_commit_complete()
613 status = dreq->error; in nfs_direct_commit_complete()
615 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
617 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_complete()
623 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
624 nfs_direct_truncate_request(dreq, req); in nfs_direct_commit_complete()
625 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
628 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
629 if (dreq->flags == 0) in nfs_direct_commit_complete()
630 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_complete()
631 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
644 nfs_direct_write_complete(dreq); in nfs_direct_commit_complete()
650 struct nfs_direct_req *dreq = cinfo->dreq; in nfs_direct_resched_write() local
652 trace_nfs_direct_resched_write(dreq); in nfs_direct_resched_write()
654 spin_lock(&dreq->lock); in nfs_direct_resched_write()
655 if (dreq->flags != NFS_ODIRECT_DONE) in nfs_direct_resched_write()
656 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_resched_write()
657 spin_unlock(&dreq->lock); in nfs_direct_resched_write()
666 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq) in nfs_direct_commit_schedule() argument
672 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_schedule()
674 nfs_scan_commit(dreq->inode, &mds_list, &cinfo); in nfs_direct_commit_schedule()
675 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo); in nfs_direct_commit_schedule()
677 spin_lock(&dreq->lock); in nfs_direct_commit_schedule()
678 if (dreq->flags == 0) in nfs_direct_commit_schedule()
679 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_schedule()
680 spin_unlock(&dreq->lock); in nfs_direct_commit_schedule()
683 nfs_direct_write_complete(dreq); in nfs_direct_commit_schedule()
686 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq) in nfs_direct_write_clear_reqs() argument
692 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_clear_reqs()
693 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_clear_reqs()
698 nfs_direct_truncate_request(dreq, req); in nfs_direct_write_clear_reqs()
706 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work); in nfs_direct_write_schedule_work() local
707 int flags = dreq->flags; in nfs_direct_write_schedule_work()
709 dreq->flags = 0; in nfs_direct_write_schedule_work()
712 nfs_direct_commit_schedule(dreq); in nfs_direct_write_schedule_work()
715 nfs_direct_write_reschedule(dreq); in nfs_direct_write_schedule_work()
718 nfs_direct_write_clear_reqs(dreq); in nfs_direct_write_schedule_work()
719 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping); in nfs_direct_write_schedule_work()
720 nfs_direct_complete(dreq); in nfs_direct_write_schedule_work()
724 static void nfs_direct_write_complete(struct nfs_direct_req *dreq) in nfs_direct_write_complete() argument
726 trace_nfs_direct_write_complete(dreq); in nfs_direct_write_complete()
727 queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */ in nfs_direct_write_complete()
732 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_completion() local
737 trace_nfs_direct_write_completion(dreq); in nfs_direct_write_completion()
739 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_completion()
741 spin_lock(&dreq->lock); in nfs_direct_write_completion()
743 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
747 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_write_completion()
750 if (!dreq->flags) in nfs_direct_write_completion()
751 dreq->flags = NFS_ODIRECT_DO_COMMIT; in nfs_direct_write_completion()
752 flags = dreq->flags; in nfs_direct_write_completion()
754 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
774 if (put_dreq(dreq)) in nfs_direct_write_completion()
775 nfs_direct_write_complete(dreq); in nfs_direct_write_completion()
792 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_reschedule_io() local
796 trace_nfs_direct_write_reschedule_io(dreq); in nfs_direct_write_reschedule_io()
798 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_reschedule_io()
799 spin_lock(&dreq->lock); in nfs_direct_write_reschedule_io()
800 if (dreq->error == 0) in nfs_direct_write_reschedule_io()
801 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule_io()
803 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule_io()
831 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_write_schedule_iovec() argument
836 struct inode *inode = dreq->inode; in nfs_direct_write_schedule_iovec()
843 trace_nfs_direct_write_schedule_iovec(dreq); in nfs_direct_write_schedule_iovec()
847 desc.pg_dreq = dreq; in nfs_direct_write_schedule_iovec()
848 get_dreq(dreq); in nfs_direct_write_schedule_iovec()
869 req = nfs_page_create_from_page(dreq->ctx, pagevec[i], in nfs_direct_write_schedule_iovec()
904 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_schedule_iovec()
905 spin_lock(&dreq->lock); in nfs_direct_write_schedule_iovec()
906 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_schedule_iovec()
907 spin_unlock(&dreq->lock); in nfs_direct_write_schedule_iovec()
926 nfs_direct_req_release(dreq); in nfs_direct_write_schedule_iovec()
930 if (put_dreq(dreq)) in nfs_direct_write_schedule_iovec()
931 nfs_direct_write_complete(dreq); in nfs_direct_write_schedule_iovec()
964 struct nfs_direct_req *dreq; in nfs_file_direct_write() local
987 dreq = nfs_direct_req_alloc(); in nfs_file_direct_write()
988 if (!dreq) in nfs_file_direct_write()
991 dreq->inode = inode; in nfs_file_direct_write()
992 dreq->max_count = count; in nfs_file_direct_write()
993 dreq->io_start = pos; in nfs_file_direct_write()
994 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_write()
995 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_write()
998 nfs_direct_req_release(dreq); in nfs_file_direct_write()
1001 dreq->l_ctx = l_ctx; in nfs_file_direct_write()
1003 dreq->iocb = iocb; in nfs_file_direct_write()
1004 pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode); in nfs_file_direct_write()
1007 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
1012 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
1024 result = nfs_direct_wait(dreq); in nfs_file_direct_write()
1037 nfs_direct_req_release(dreq); in nfs_file_direct_write()