Lines Matching refs:cs
820 void fuse_copy_init(struct fuse_copy_state *cs, bool write, in fuse_copy_init() argument
823 memset(cs, 0, sizeof(*cs)); in fuse_copy_init()
824 cs->write = write; in fuse_copy_init()
825 cs->iter = iter; in fuse_copy_init()
829 static void fuse_copy_finish(struct fuse_copy_state *cs) in fuse_copy_finish() argument
831 if (cs->currbuf) { in fuse_copy_finish()
832 struct pipe_buffer *buf = cs->currbuf; in fuse_copy_finish()
834 if (cs->write) in fuse_copy_finish()
835 buf->len = PAGE_SIZE - cs->len; in fuse_copy_finish()
836 cs->currbuf = NULL; in fuse_copy_finish()
837 } else if (cs->pg) { in fuse_copy_finish()
838 if (cs->write) { in fuse_copy_finish()
839 flush_dcache_page(cs->pg); in fuse_copy_finish()
840 set_page_dirty_lock(cs->pg); in fuse_copy_finish()
842 put_page(cs->pg); in fuse_copy_finish()
844 cs->pg = NULL; in fuse_copy_finish()
851 static int fuse_copy_fill(struct fuse_copy_state *cs) in fuse_copy_fill() argument
856 err = unlock_request(cs->req); in fuse_copy_fill()
860 fuse_copy_finish(cs); in fuse_copy_fill()
861 if (cs->pipebufs) { in fuse_copy_fill()
862 struct pipe_buffer *buf = cs->pipebufs; in fuse_copy_fill()
864 if (!cs->write) { in fuse_copy_fill()
865 err = pipe_buf_confirm(cs->pipe, buf); in fuse_copy_fill()
869 BUG_ON(!cs->nr_segs); in fuse_copy_fill()
870 cs->currbuf = buf; in fuse_copy_fill()
871 cs->pg = buf->page; in fuse_copy_fill()
872 cs->offset = buf->offset; in fuse_copy_fill()
873 cs->len = buf->len; in fuse_copy_fill()
874 cs->pipebufs++; in fuse_copy_fill()
875 cs->nr_segs--; in fuse_copy_fill()
877 if (cs->nr_segs >= cs->pipe->max_usage) in fuse_copy_fill()
888 cs->currbuf = buf; in fuse_copy_fill()
889 cs->pg = page; in fuse_copy_fill()
890 cs->offset = 0; in fuse_copy_fill()
891 cs->len = PAGE_SIZE; in fuse_copy_fill()
892 cs->pipebufs++; in fuse_copy_fill()
893 cs->nr_segs++; in fuse_copy_fill()
897 err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off); in fuse_copy_fill()
901 cs->len = err; in fuse_copy_fill()
902 cs->offset = off; in fuse_copy_fill()
903 cs->pg = page; in fuse_copy_fill()
906 return lock_request(cs->req); in fuse_copy_fill()
910 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size) in fuse_copy_do() argument
912 unsigned ncpy = min(*size, cs->len); in fuse_copy_do()
914 void *pgaddr = kmap_local_page(cs->pg); in fuse_copy_do()
915 void *buf = pgaddr + cs->offset; in fuse_copy_do()
917 if (cs->write) in fuse_copy_do()
926 cs->len -= ncpy; in fuse_copy_do()
927 cs->offset += ncpy; in fuse_copy_do()
928 if (cs->is_uring) in fuse_copy_do()
929 cs->ring.copied_sz += ncpy; in fuse_copy_do()
959 static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop) in fuse_try_move_folio() argument
964 struct pipe_buffer *buf = cs->pipebufs; in fuse_try_move_folio()
967 err = unlock_request(cs->req); in fuse_try_move_folio()
971 fuse_copy_finish(cs); in fuse_try_move_folio()
973 err = pipe_buf_confirm(cs->pipe, buf); in fuse_try_move_folio()
977 BUG_ON(!cs->nr_segs); in fuse_try_move_folio()
978 cs->currbuf = buf; in fuse_try_move_folio()
979 cs->len = buf->len; in fuse_try_move_folio()
980 cs->pipebufs++; in fuse_try_move_folio()
981 cs->nr_segs--; in fuse_try_move_folio()
983 if (cs->len != folio_size(oldfolio)) in fuse_try_move_folio()
986 if (!pipe_buf_try_steal(cs->pipe, buf)) in fuse_try_move_folio()
1022 pipe_buf_release(cs->pipe, buf); in fuse_try_move_folio()
1025 spin_lock(&cs->req->waitq.lock); in fuse_try_move_folio()
1026 if (test_bit(FR_ABORTED, &cs->req->flags)) in fuse_try_move_folio()
1030 spin_unlock(&cs->req->waitq.lock); in fuse_try_move_folio()
1041 cs->len = 0; in fuse_try_move_folio()
1052 cs->pg = buf->page; in fuse_try_move_folio()
1053 cs->offset = buf->offset; in fuse_try_move_folio()
1055 err = lock_request(cs->req); in fuse_try_move_folio()
1062 static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio, in fuse_ref_folio() argument
1068 if (cs->nr_segs >= cs->pipe->max_usage) in fuse_ref_folio()
1072 err = unlock_request(cs->req); in fuse_ref_folio()
1078 fuse_copy_finish(cs); in fuse_ref_folio()
1080 buf = cs->pipebufs; in fuse_ref_folio()
1085 cs->pipebufs++; in fuse_ref_folio()
1086 cs->nr_segs++; in fuse_ref_folio()
1087 cs->len = 0; in fuse_ref_folio()
1096 static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, in fuse_copy_folio() argument
1110 if (cs->write && cs->pipebufs && folio) { in fuse_copy_folio()
1115 if (cs->req->args->user_pages) { in fuse_copy_folio()
1116 err = fuse_copy_fill(cs); in fuse_copy_folio()
1120 return fuse_ref_folio(cs, folio, offset, count); in fuse_copy_folio()
1122 } else if (!cs->len) { in fuse_copy_folio()
1123 if (cs->move_folios && folio && in fuse_copy_folio()
1125 err = fuse_try_move_folio(cs, foliop); in fuse_copy_folio()
1129 err = fuse_copy_fill(cs); in fuse_copy_folio()
1143 bytes_copied = fuse_copy_do(cs, &buf, ©); in fuse_copy_folio()
1148 offset += fuse_copy_do(cs, NULL, &count); in fuse_copy_folio()
1150 if (folio && !cs->write) in fuse_copy_folio()
1156 static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes, in fuse_copy_folios() argument
1160 struct fuse_req *req = cs->req; in fuse_copy_folios()
1168 err = fuse_copy_folio(cs, &ap->folios[i], offset, count, zeroing); in fuse_copy_folios()
1178 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) in fuse_copy_one() argument
1181 if (!cs->len) { in fuse_copy_one()
1182 int err = fuse_copy_fill(cs); in fuse_copy_one()
1186 fuse_copy_do(cs, &val, &size); in fuse_copy_one()
1192 int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, in fuse_copy_args() argument
1202 err = fuse_copy_folios(cs, arg->size, zeroing); in fuse_copy_args()
1204 err = fuse_copy_one(cs, arg->value, arg->size); in fuse_copy_args()
1229 struct fuse_copy_state *cs, in fuse_read_interrupt() argument
1250 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_interrupt()
1252 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_interrupt()
1253 fuse_copy_finish(cs); in fuse_read_interrupt()
1281 struct fuse_copy_state *cs, in fuse_read_single_forget() argument
1302 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_single_forget()
1304 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_single_forget()
1305 fuse_copy_finish(cs); in fuse_read_single_forget()
1314 struct fuse_copy_state *cs, size_t nbytes) in fuse_read_batch_forget() argument
1339 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_batch_forget()
1341 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_batch_forget()
1347 err = fuse_copy_one(cs, &forget->forget_one, in fuse_read_batch_forget()
1354 fuse_copy_finish(cs); in fuse_read_batch_forget()
1363 struct fuse_copy_state *cs, in fuse_read_forget() argument
1368 return fuse_read_single_forget(fiq, cs, nbytes); in fuse_read_forget()
1370 return fuse_read_batch_forget(fiq, cs, nbytes); in fuse_read_forget()
1383 struct fuse_copy_state *cs, size_t nbytes) in fuse_dev_do_read() argument
1435 return fuse_read_interrupt(fiq, cs, nbytes, req); in fuse_dev_do_read()
1440 return fuse_read_forget(fc, fiq, cs, nbytes); in fuse_dev_do_read()
1475 cs->req = req; in fuse_dev_do_read()
1476 err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h)); in fuse_dev_do_read()
1478 err = fuse_copy_args(cs, args->in_numargs, args->in_pages, in fuse_dev_do_read()
1480 fuse_copy_finish(cs); in fuse_dev_do_read()
1533 struct fuse_copy_state cs; in fuse_dev_read() local
1543 fuse_copy_init(&cs, true, to); in fuse_dev_read()
1545 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to)); in fuse_dev_read()
1555 struct fuse_copy_state cs; in fuse_dev_splice_read() local
1566 fuse_copy_init(&cs, true, NULL); in fuse_dev_splice_read()
1567 cs.pipebufs = bufs; in fuse_dev_splice_read()
1568 cs.pipe = pipe; in fuse_dev_splice_read()
1569 ret = fuse_dev_do_read(fud, in, &cs, len); in fuse_dev_splice_read()
1573 if (pipe_buf_usage(pipe) + cs.nr_segs > pipe->max_usage) { in fuse_dev_splice_read()
1578 for (ret = total = 0; page_nr < cs.nr_segs; total += ret) { in fuse_dev_splice_read()
1592 for (; page_nr < cs.nr_segs; page_nr++) in fuse_dev_splice_read()
1600 struct fuse_copy_state *cs) in fuse_notify_poll() argument
1608 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_poll()
1612 fuse_copy_finish(cs); in fuse_notify_poll()
1616 fuse_copy_finish(cs); in fuse_notify_poll()
1621 struct fuse_copy_state *cs) in fuse_notify_inval_inode() argument
1629 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_inval_inode()
1632 fuse_copy_finish(cs); in fuse_notify_inval_inode()
1641 fuse_copy_finish(cs); in fuse_notify_inval_inode()
1646 struct fuse_copy_state *cs) in fuse_notify_inval_entry() argument
1657 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_inval_entry()
1676 err = fuse_copy_one(cs, buf, outarg.namelen + 1); in fuse_notify_inval_entry()
1679 fuse_copy_finish(cs); in fuse_notify_inval_entry()
1690 fuse_copy_finish(cs); in fuse_notify_inval_entry()
1695 struct fuse_copy_state *cs) in fuse_notify_delete() argument
1706 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_delete()
1725 err = fuse_copy_one(cs, buf, outarg.namelen + 1); in fuse_notify_delete()
1728 fuse_copy_finish(cs); in fuse_notify_delete()
1739 fuse_copy_finish(cs); in fuse_notify_delete()
1744 struct fuse_copy_state *cs) in fuse_notify_store() argument
1761 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_store()
1804 err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0); in fuse_notify_store()
1828 fuse_copy_finish(cs); in fuse_notify_store()
1935 struct fuse_copy_state *cs) in fuse_notify_retrieve() argument
1947 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_retrieve()
1951 fuse_copy_finish(cs); in fuse_notify_retrieve()
1967 fuse_copy_finish(cs); in fuse_notify_retrieve()
2048 unsigned int size, struct fuse_copy_state *cs) in fuse_notify() argument
2051 cs->move_folios = false; in fuse_notify()
2055 return fuse_notify_poll(fc, size, cs); in fuse_notify()
2058 return fuse_notify_inval_inode(fc, size, cs); in fuse_notify()
2061 return fuse_notify_inval_entry(fc, size, cs); in fuse_notify()
2064 return fuse_notify_store(fc, size, cs); in fuse_notify()
2067 return fuse_notify_retrieve(fc, size, cs); in fuse_notify()
2070 return fuse_notify_delete(fc, size, cs); in fuse_notify()
2079 fuse_copy_finish(cs); in fuse_notify()
2097 int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, in fuse_copy_out_args() argument
2106 if (!cs->is_uring) in fuse_copy_out_args()
2121 return fuse_copy_args(cs, args->out_numargs, args->out_pages, in fuse_copy_out_args()
2133 struct fuse_copy_state *cs, size_t nbytes) in fuse_dev_do_write() argument
2145 err = fuse_copy_one(cs, &oh, sizeof(oh)); in fuse_dev_do_write()
2158 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs); in fuse_dev_do_write()
2200 cs->req = req; in fuse_dev_do_write()
2202 cs->move_folios = false; in fuse_dev_do_write()
2207 err = fuse_copy_out_args(cs, req->args, nbytes); in fuse_dev_do_write()
2208 fuse_copy_finish(cs); in fuse_dev_do_write()
2225 fuse_copy_finish(cs); in fuse_dev_do_write()
2231 struct fuse_copy_state cs; in fuse_dev_write() local
2240 fuse_copy_init(&cs, false, from); in fuse_dev_write()
2242 return fuse_dev_do_write(fud, &cs, iov_iter_count(from)); in fuse_dev_write()
2253 struct fuse_copy_state cs; in fuse_dev_splice_write() local
2314 fuse_copy_init(&cs, false, NULL); in fuse_dev_splice_write()
2315 cs.pipebufs = bufs; in fuse_dev_splice_write()
2316 cs.nr_segs = nbuf; in fuse_dev_splice_write()
2317 cs.pipe = pipe; in fuse_dev_splice_write()
2320 cs.move_folios = true; in fuse_dev_splice_write()
2322 ret = fuse_dev_do_write(fud, &cs, len); in fuse_dev_splice_write()