Lines Matching refs:dreq
71 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
74 static inline void get_dreq(struct nfs_direct_req *dreq) in get_dreq() argument
76 atomic_inc(&dreq->io_count); in get_dreq()
79 static inline int put_dreq(struct nfs_direct_req *dreq) in put_dreq() argument
81 return atomic_dec_and_test(&dreq->io_count); in put_dreq()
85 nfs_direct_handle_truncated(struct nfs_direct_req *dreq, in nfs_direct_handle_truncated() argument
92 if (dreq->max_count >= dreq_len) { in nfs_direct_handle_truncated()
93 dreq->max_count = dreq_len; in nfs_direct_handle_truncated()
94 if (dreq->count > dreq_len) in nfs_direct_handle_truncated()
95 dreq->count = dreq_len; in nfs_direct_handle_truncated()
98 dreq->error = hdr->error; in nfs_direct_handle_truncated()
100 dreq->error = 0; in nfs_direct_handle_truncated()
105 nfs_direct_count_bytes(struct nfs_direct_req *dreq, in nfs_direct_count_bytes() argument
111 if (hdr_end > dreq->io_start) in nfs_direct_count_bytes()
112 dreq_len = hdr_end - dreq->io_start; in nfs_direct_count_bytes()
114 nfs_direct_handle_truncated(dreq, hdr, dreq_len); in nfs_direct_count_bytes()
116 if (dreq_len > dreq->max_count) in nfs_direct_count_bytes()
117 dreq_len = dreq->max_count; in nfs_direct_count_bytes()
119 if (dreq->count < dreq_len) in nfs_direct_count_bytes()
120 dreq->count = dreq_len; in nfs_direct_count_bytes()
153 struct nfs_direct_req *dreq) in nfs_init_cinfo_from_dreq() argument
155 cinfo->inode = dreq->inode; in nfs_init_cinfo_from_dreq()
156 cinfo->mds = &dreq->mds_cinfo; in nfs_init_cinfo_from_dreq()
157 cinfo->ds = &dreq->ds_cinfo; in nfs_init_cinfo_from_dreq()
158 cinfo->dreq = dreq; in nfs_init_cinfo_from_dreq()
164 struct nfs_direct_req *dreq; in nfs_direct_req_alloc() local
166 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL); in nfs_direct_req_alloc()
167 if (!dreq) in nfs_direct_req_alloc()
170 kref_init(&dreq->kref); in nfs_direct_req_alloc()
171 kref_get(&dreq->kref); in nfs_direct_req_alloc()
172 init_completion(&dreq->completion); in nfs_direct_req_alloc()
173 INIT_LIST_HEAD(&dreq->mds_cinfo.list); in nfs_direct_req_alloc()
174 pnfs_init_ds_commit_info(&dreq->ds_cinfo); in nfs_direct_req_alloc()
175 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); in nfs_direct_req_alloc()
176 spin_lock_init(&dreq->lock); in nfs_direct_req_alloc()
178 return dreq; in nfs_direct_req_alloc()
183 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref); in nfs_direct_req_free() local
185 pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode); in nfs_direct_req_free()
186 if (dreq->l_ctx != NULL) in nfs_direct_req_free()
187 nfs_put_lock_context(dreq->l_ctx); in nfs_direct_req_free()
188 if (dreq->ctx != NULL) in nfs_direct_req_free()
189 put_nfs_open_context(dreq->ctx); in nfs_direct_req_free()
190 kmem_cache_free(nfs_direct_cachep, dreq); in nfs_direct_req_free()
193 static void nfs_direct_req_release(struct nfs_direct_req *dreq) in nfs_direct_req_release() argument
195 kref_put(&dreq->kref, nfs_direct_req_free); in nfs_direct_req_release()
198 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq) in nfs_dreq_bytes_left() argument
200 return dreq->bytes_left; in nfs_dreq_bytes_left()
207 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq) in nfs_direct_wait() argument
212 if (dreq->iocb) in nfs_direct_wait()
215 result = wait_for_completion_killable(&dreq->completion); in nfs_direct_wait()
218 result = dreq->count; in nfs_direct_wait()
219 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_wait()
222 result = dreq->error; in nfs_direct_wait()
232 static void nfs_direct_complete(struct nfs_direct_req *dreq) in nfs_direct_complete() argument
234 struct inode *inode = dreq->inode; in nfs_direct_complete()
238 if (dreq->iocb) { in nfs_direct_complete()
239 long res = (long) dreq->error; in nfs_direct_complete()
240 if (dreq->count != 0) { in nfs_direct_complete()
241 res = (long) dreq->count; in nfs_direct_complete()
242 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_complete()
244 dreq->iocb->ki_complete(dreq->iocb, res); in nfs_direct_complete()
247 complete(&dreq->completion); in nfs_direct_complete()
249 nfs_direct_req_release(dreq); in nfs_direct_complete()
255 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_read_completion() local
257 spin_lock(&dreq->lock); in nfs_direct_read_completion()
259 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
263 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_read_completion()
264 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
271 (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY)) in nfs_direct_read_completion()
278 if (put_dreq(dreq)) in nfs_direct_read_completion()
279 nfs_direct_complete(dreq); in nfs_direct_read_completion()
296 get_dreq(hdr->dreq); in nfs_direct_pgio_init()
313 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_read_schedule_iovec() argument
318 struct inode *inode = dreq->inode; in nfs_direct_read_schedule_iovec()
323 nfs_pageio_init_read(&desc, dreq->inode, false, in nfs_direct_read_schedule_iovec()
325 get_dreq(dreq); in nfs_direct_read_schedule_iovec()
326 desc.pg_dreq = dreq; in nfs_direct_read_schedule_iovec()
346 req = nfs_page_create_from_page(dreq->ctx, pagevec[i], in nfs_direct_read_schedule_iovec()
361 dreq->bytes_left -= req_len; in nfs_direct_read_schedule_iovec()
377 nfs_direct_req_release(dreq); in nfs_direct_read_schedule_iovec()
381 if (put_dreq(dreq)) in nfs_direct_read_schedule_iovec()
382 nfs_direct_complete(dreq); in nfs_direct_read_schedule_iovec()
412 struct nfs_direct_req *dreq; in nfs_file_direct_read() local
428 dreq = nfs_direct_req_alloc(); in nfs_file_direct_read()
429 if (dreq == NULL) in nfs_file_direct_read()
432 dreq->inode = inode; in nfs_file_direct_read()
433 dreq->bytes_left = dreq->max_count = count; in nfs_file_direct_read()
434 dreq->io_start = iocb->ki_pos; in nfs_file_direct_read()
435 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_read()
436 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_read()
439 nfs_direct_req_release(dreq); in nfs_file_direct_read()
442 dreq->l_ctx = l_ctx; in nfs_file_direct_read()
444 dreq->iocb = iocb; in nfs_file_direct_read()
447 dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; in nfs_file_direct_read()
453 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos); in nfs_file_direct_read()
459 result = nfs_direct_wait(dreq); in nfs_file_direct_read()
470 nfs_direct_req_release(dreq); in nfs_file_direct_read()
504 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) in nfs_direct_write_reschedule() argument
512 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_reschedule()
513 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_reschedule()
515 nfs_direct_join_group(&reqs, dreq->inode); in nfs_direct_write_reschedule()
517 dreq->count = 0; in nfs_direct_write_reschedule()
518 dreq->max_count = 0; in nfs_direct_write_reschedule()
520 dreq->max_count += req->wb_bytes; in nfs_direct_write_reschedule()
521 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo); in nfs_direct_write_reschedule()
522 get_dreq(dreq); in nfs_direct_write_reschedule()
524 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false, in nfs_direct_write_reschedule()
526 desc.pg_dreq = dreq; in nfs_direct_write_reschedule()
534 dreq->flags = 0; in nfs_direct_write_reschedule()
536 dreq->error = desc.pg_error; in nfs_direct_write_reschedule()
538 dreq->error = -EIO; in nfs_direct_write_reschedule()
551 if (put_dreq(dreq)) in nfs_direct_write_reschedule()
552 nfs_direct_write_complete(dreq); in nfs_direct_write_reschedule()
558 struct nfs_direct_req *dreq = data->dreq; in nfs_direct_commit_complete() local
563 trace_nfs_direct_commit_complete(dreq); in nfs_direct_commit_complete()
567 dreq->error = status; in nfs_direct_commit_complete()
568 dreq->max_count = 0; in nfs_direct_commit_complete()
569 dreq->count = 0; in nfs_direct_commit_complete()
570 dreq->flags = NFS_ODIRECT_DONE; in nfs_direct_commit_complete()
572 status = dreq->error; in nfs_direct_commit_complete()
575 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_complete()
581 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_complete()
594 nfs_direct_write_complete(dreq); in nfs_direct_commit_complete()
600 struct nfs_direct_req *dreq = cinfo->dreq; in nfs_direct_resched_write() local
602 trace_nfs_direct_resched_write(dreq); in nfs_direct_resched_write()
604 spin_lock(&dreq->lock); in nfs_direct_resched_write()
605 if (dreq->flags != NFS_ODIRECT_DONE) in nfs_direct_resched_write()
606 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_resched_write()
607 spin_unlock(&dreq->lock); in nfs_direct_resched_write()
616 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq) in nfs_direct_commit_schedule() argument
622 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_schedule()
623 nfs_scan_commit(dreq->inode, &mds_list, &cinfo); in nfs_direct_commit_schedule()
624 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo); in nfs_direct_commit_schedule()
626 nfs_direct_write_reschedule(dreq); in nfs_direct_commit_schedule()
629 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq) in nfs_direct_write_clear_reqs() argument
635 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_clear_reqs()
636 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_clear_reqs()
648 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work); in nfs_direct_write_schedule_work() local
649 int flags = dreq->flags; in nfs_direct_write_schedule_work()
651 dreq->flags = 0; in nfs_direct_write_schedule_work()
654 nfs_direct_commit_schedule(dreq); in nfs_direct_write_schedule_work()
657 nfs_direct_write_reschedule(dreq); in nfs_direct_write_schedule_work()
660 nfs_direct_write_clear_reqs(dreq); in nfs_direct_write_schedule_work()
661 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping); in nfs_direct_write_schedule_work()
662 nfs_direct_complete(dreq); in nfs_direct_write_schedule_work()
666 static void nfs_direct_write_complete(struct nfs_direct_req *dreq) in nfs_direct_write_complete() argument
668 trace_nfs_direct_write_complete(dreq); in nfs_direct_write_complete()
669 queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */ in nfs_direct_write_complete()
674 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_completion() local
679 trace_nfs_direct_write_completion(dreq); in nfs_direct_write_completion()
681 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_completion()
683 spin_lock(&dreq->lock); in nfs_direct_write_completion()
685 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
689 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_write_completion()
691 if (!dreq->flags) in nfs_direct_write_completion()
692 dreq->flags = NFS_ODIRECT_DO_COMMIT; in nfs_direct_write_completion()
693 flags = dreq->flags; in nfs_direct_write_completion()
695 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
715 if (put_dreq(dreq)) in nfs_direct_write_completion()
716 nfs_direct_write_complete(dreq); in nfs_direct_write_completion()
733 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_reschedule_io() local
735 trace_nfs_direct_write_reschedule_io(dreq); in nfs_direct_write_reschedule_io()
737 spin_lock(&dreq->lock); in nfs_direct_write_reschedule_io()
738 if (dreq->error == 0) { in nfs_direct_write_reschedule_io()
739 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule_io()
745 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule_io()
767 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_write_schedule_iovec() argument
772 struct inode *inode = dreq->inode; in nfs_direct_write_schedule_iovec()
777 trace_nfs_direct_write_schedule_iovec(dreq); in nfs_direct_write_schedule_iovec()
781 desc.pg_dreq = dreq; in nfs_direct_write_schedule_iovec()
782 get_dreq(dreq); in nfs_direct_write_schedule_iovec()
803 req = nfs_page_create_from_page(dreq->ctx, pagevec[i], in nfs_direct_write_schedule_iovec()
826 dreq->bytes_left -= req_len; in nfs_direct_write_schedule_iovec()
841 nfs_direct_req_release(dreq); in nfs_direct_write_schedule_iovec()
845 if (put_dreq(dreq)) in nfs_direct_write_schedule_iovec()
846 nfs_direct_write_complete(dreq); in nfs_direct_write_schedule_iovec()
879 struct nfs_direct_req *dreq; in nfs_file_direct_write() local
902 dreq = nfs_direct_req_alloc(); in nfs_file_direct_write()
903 if (!dreq) in nfs_file_direct_write()
906 dreq->inode = inode; in nfs_file_direct_write()
907 dreq->bytes_left = dreq->max_count = count; in nfs_file_direct_write()
908 dreq->io_start = pos; in nfs_file_direct_write()
909 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_write()
910 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_write()
913 nfs_direct_req_release(dreq); in nfs_file_direct_write()
916 dreq->l_ctx = l_ctx; in nfs_file_direct_write()
918 dreq->iocb = iocb; in nfs_file_direct_write()
919 pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode); in nfs_file_direct_write()
922 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
927 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
939 result = nfs_direct_wait(dreq); in nfs_file_direct_write()
952 nfs_direct_req_release(dreq); in nfs_file_direct_write()