Lines Matching refs:pipe
88 void pipe_lock(struct pipe_inode_info *pipe) in pipe_lock() argument
90 if (pipe->files) in pipe_lock()
91 mutex_lock(&pipe->mutex); in pipe_lock()
95 void pipe_unlock(struct pipe_inode_info *pipe) in pipe_unlock() argument
97 if (pipe->files) in pipe_unlock()
98 mutex_unlock(&pipe->mutex); in pipe_unlock()
114 static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe) in anon_pipe_get_page() argument
116 for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) { in anon_pipe_get_page()
117 if (pipe->tmp_page[i]) { in anon_pipe_get_page()
118 struct page *page = pipe->tmp_page[i]; in anon_pipe_get_page()
119 pipe->tmp_page[i] = NULL; in anon_pipe_get_page()
127 static void anon_pipe_put_page(struct pipe_inode_info *pipe, in anon_pipe_put_page() argument
131 for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) { in anon_pipe_put_page()
132 if (!pipe->tmp_page[i]) { in anon_pipe_put_page()
133 pipe->tmp_page[i] = page; in anon_pipe_put_page()
142 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, in anon_pipe_buf_release() argument
147 anon_pipe_put_page(pipe, page); in anon_pipe_buf_release()
150 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe, in anon_pipe_buf_try_steal() argument
174 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe, in generic_pipe_buf_try_steal() argument
202 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) in generic_pipe_buf_get() argument
216 void generic_pipe_buf_release(struct pipe_inode_info *pipe, in generic_pipe_buf_release() argument
230 static inline bool pipe_readable(const struct pipe_inode_info *pipe) in pipe_readable() argument
232 union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) }; in pipe_readable()
233 unsigned int writers = READ_ONCE(pipe->writers); in pipe_readable()
238 static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe, in pipe_update_tail() argument
242 pipe_buf_release(pipe, buf); in pipe_update_tail()
249 if (pipe_has_watch_queue(pipe)) { in pipe_update_tail()
250 spin_lock_irq(&pipe->rd_wait.lock); in pipe_update_tail()
253 pipe->note_loss = true; in pipe_update_tail()
255 pipe->tail = ++tail; in pipe_update_tail()
256 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_update_tail()
264 pipe->tail = ++tail; in pipe_update_tail()
273 struct pipe_inode_info *pipe = filp->private_data; in anon_pipe_read() local
282 mutex_lock(&pipe->mutex); in anon_pipe_read()
294 unsigned int head = smp_load_acquire(&pipe->head); in anon_pipe_read()
295 unsigned int tail = pipe->tail; in anon_pipe_read()
298 if (pipe->note_loss) { in anon_pipe_read()
317 pipe->note_loss = false; in anon_pipe_read()
322 struct pipe_buffer *buf = pipe_buf(pipe, tail); in anon_pipe_read()
336 error = pipe_buf_confirm(pipe, buf); in anon_pipe_read()
360 wake_writer |= pipe_full(head, tail, pipe->max_usage); in anon_pipe_read()
361 tail = pipe_update_tail(pipe, buf, tail); in anon_pipe_read()
370 if (!pipe->writers) in anon_pipe_read()
379 mutex_unlock(&pipe->mutex); in anon_pipe_read()
388 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0) in anon_pipe_read()
392 mutex_lock(&pipe->mutex); in anon_pipe_read()
394 if (pipe_is_empty(pipe)) in anon_pipe_read()
396 mutex_unlock(&pipe->mutex); in anon_pipe_read()
399 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in anon_pipe_read()
401 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in anon_pipe_read()
402 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in anon_pipe_read()
421 static inline bool pipe_writable(const struct pipe_inode_info *pipe) in pipe_writable() argument
423 union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) }; in pipe_writable()
424 unsigned int max_usage = READ_ONCE(pipe->max_usage); in pipe_writable()
427 !READ_ONCE(pipe->readers); in pipe_writable()
434 struct pipe_inode_info *pipe = filp->private_data; in anon_pipe_write() local
451 if (pipe_has_watch_queue(pipe)) in anon_pipe_write()
458 mutex_lock(&pipe->mutex); in anon_pipe_write()
460 if (!pipe->readers) { in anon_pipe_write()
474 head = pipe->head; in anon_pipe_write()
475 was_empty = pipe_empty(head, pipe->tail); in anon_pipe_write()
478 struct pipe_buffer *buf = pipe_buf(pipe, head - 1); in anon_pipe_write()
483 ret = pipe_buf_confirm(pipe, buf); in anon_pipe_write()
500 if (!pipe->readers) { in anon_pipe_write()
507 head = pipe->head; in anon_pipe_write()
508 if (!pipe_full(head, pipe->tail, pipe->max_usage)) { in anon_pipe_write()
513 page = anon_pipe_get_page(pipe); in anon_pipe_write()
522 anon_pipe_put_page(pipe, page); in anon_pipe_write()
528 pipe->head = head + 1; in anon_pipe_write()
530 buf = pipe_buf(pipe, head); in anon_pipe_write()
567 mutex_unlock(&pipe->mutex); in anon_pipe_write()
569 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in anon_pipe_write()
570 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in anon_pipe_write()
571 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe)); in anon_pipe_write()
572 mutex_lock(&pipe->mutex); in anon_pipe_write()
573 was_empty = pipe_is_empty(pipe); in anon_pipe_write()
577 if (pipe_is_full(pipe)) in anon_pipe_write()
579 mutex_unlock(&pipe->mutex); in anon_pipe_write()
593 if (was_empty || pipe->poll_usage) in anon_pipe_write()
594 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in anon_pipe_write()
595 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in anon_pipe_write()
597 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in anon_pipe_write()
619 struct pipe_inode_info *pipe = filp->private_data; in pipe_ioctl() local
624 mutex_lock(&pipe->mutex); in pipe_ioctl()
626 head = pipe->head; in pipe_ioctl()
627 tail = pipe->tail; in pipe_ioctl()
630 count += pipe_buf(pipe, tail)->len; in pipe_ioctl()
633 mutex_unlock(&pipe->mutex); in pipe_ioctl()
640 mutex_lock(&pipe->mutex); in pipe_ioctl()
641 ret = watch_queue_set_size(pipe, arg); in pipe_ioctl()
642 mutex_unlock(&pipe->mutex); in pipe_ioctl()
648 pipe, (struct watch_notification_filter __user *)arg); in pipe_ioctl()
661 struct pipe_inode_info *pipe = filp->private_data; in pipe_poll() local
665 WRITE_ONCE(pipe->poll_usage, true); in pipe_poll()
674 poll_wait(filp, &pipe->rd_wait, wait); in pipe_poll()
676 poll_wait(filp, &pipe->wr_wait, wait); in pipe_poll()
683 idx.head_tail = READ_ONCE(pipe->head_tail); in pipe_poll()
689 if (!pipe->writers && filp->f_pipe != pipe->w_counter) in pipe_poll()
694 if (!pipe_full(idx.head, idx.tail, pipe->max_usage)) in pipe_poll()
700 if (!pipe->readers) in pipe_poll()
707 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe) in put_pipe_info() argument
712 if (!--pipe->files) { in put_pipe_info()
719 free_pipe_info(pipe); in put_pipe_info()
725 struct pipe_inode_info *pipe = file->private_data; in pipe_release() local
727 mutex_lock(&pipe->mutex); in pipe_release()
729 pipe->readers--; in pipe_release()
731 pipe->writers--; in pipe_release()
734 if (!pipe->readers != !pipe->writers) { in pipe_release()
735 wake_up_interruptible_all(&pipe->rd_wait); in pipe_release()
736 wake_up_interruptible_all(&pipe->wr_wait); in pipe_release()
737 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_release()
738 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_release()
740 mutex_unlock(&pipe->mutex); in pipe_release()
742 put_pipe_info(inode, pipe); in pipe_release()
749 struct pipe_inode_info *pipe = filp->private_data; in pipe_fasync() local
752 mutex_lock(&pipe->mutex); in pipe_fasync()
754 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); in pipe_fasync()
756 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); in pipe_fasync()
759 fasync_helper(-1, filp, 0, &pipe->fasync_readers); in pipe_fasync()
761 mutex_unlock(&pipe->mutex); in pipe_fasync()
792 struct pipe_inode_info *pipe; in alloc_pipe_info() local
798 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT); in alloc_pipe_info()
799 if (pipe == NULL) in alloc_pipe_info()
815 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer), in alloc_pipe_info()
818 if (pipe->bufs) { in alloc_pipe_info()
819 init_waitqueue_head(&pipe->rd_wait); in alloc_pipe_info()
820 init_waitqueue_head(&pipe->wr_wait); in alloc_pipe_info()
821 pipe->r_counter = pipe->w_counter = 1; in alloc_pipe_info()
822 pipe->max_usage = pipe_bufs; in alloc_pipe_info()
823 pipe->ring_size = pipe_bufs; in alloc_pipe_info()
824 pipe->nr_accounted = pipe_bufs; in alloc_pipe_info()
825 pipe->user = user; in alloc_pipe_info()
826 mutex_init(&pipe->mutex); in alloc_pipe_info()
827 lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL); in alloc_pipe_info()
828 return pipe; in alloc_pipe_info()
833 kfree(pipe); in alloc_pipe_info()
839 void free_pipe_info(struct pipe_inode_info *pipe) in free_pipe_info() argument
844 if (pipe->watch_queue) in free_pipe_info()
845 watch_queue_clear(pipe->watch_queue); in free_pipe_info()
848 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0); in free_pipe_info()
849 free_uid(pipe->user); in free_pipe_info()
850 for (i = 0; i < pipe->ring_size; i++) { in free_pipe_info()
851 struct pipe_buffer *buf = pipe->bufs + i; in free_pipe_info()
853 pipe_buf_release(pipe, buf); in free_pipe_info()
856 if (pipe->watch_queue) in free_pipe_info()
857 put_watch_queue(pipe->watch_queue); in free_pipe_info()
859 for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) { in free_pipe_info()
860 if (pipe->tmp_page[i]) in free_pipe_info()
861 __free_page(pipe->tmp_page[i]); in free_pipe_info()
863 kfree(pipe->bufs); in free_pipe_info()
864 kfree(pipe); in free_pipe_info()
887 struct pipe_inode_info *pipe; in get_pipe_inode() local
894 pipe = alloc_pipe_info(); in get_pipe_inode()
895 if (!pipe) in get_pipe_inode()
898 inode->i_pipe = pipe; in get_pipe_inode()
899 pipe->files = 2; in get_pipe_inode()
900 pipe->readers = pipe->writers = 1; in get_pipe_inode()
1057 SYSCALL_DEFINE1(pipe, int __user *, fildes) in SYSCALL_DEFINE1() argument
1071 void pipe_wait_readable(struct pipe_inode_info *pipe) in pipe_wait_readable() argument
1073 pipe_unlock(pipe); in pipe_wait_readable()
1074 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe)); in pipe_wait_readable()
1075 pipe_lock(pipe); in pipe_wait_readable()
1078 void pipe_wait_writable(struct pipe_inode_info *pipe) in pipe_wait_writable() argument
1080 pipe_unlock(pipe); in pipe_wait_writable()
1081 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe)); in pipe_wait_writable()
1082 pipe_lock(pipe); in pipe_wait_writable()
1097 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) in wait_for_partner() argument
1103 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE); in wait_for_partner()
1104 pipe_unlock(pipe); in wait_for_partner()
1106 finish_wait(&pipe->rd_wait, &rdwait); in wait_for_partner()
1107 pipe_lock(pipe); in wait_for_partner()
1114 static void wake_up_partner(struct pipe_inode_info *pipe) in wake_up_partner() argument
1116 wake_up_interruptible_all(&pipe->rd_wait); in wake_up_partner()
1122 struct pipe_inode_info *pipe; in fifo_open() local
1129 pipe = inode->i_pipe; in fifo_open()
1130 pipe->files++; in fifo_open()
1134 pipe = alloc_pipe_info(); in fifo_open()
1135 if (!pipe) in fifo_open()
1137 pipe->files = 1; in fifo_open()
1142 free_pipe_info(pipe); in fifo_open()
1143 pipe = inode->i_pipe; in fifo_open()
1145 inode->i_pipe = pipe; in fifo_open()
1149 filp->private_data = pipe; in fifo_open()
1152 mutex_lock(&pipe->mutex); in fifo_open()
1164 pipe->r_counter++; in fifo_open()
1165 if (pipe->readers++ == 0) in fifo_open()
1166 wake_up_partner(pipe); in fifo_open()
1168 if (!is_pipe && !pipe->writers) { in fifo_open()
1172 filp->f_pipe = pipe->w_counter; in fifo_open()
1174 if (wait_for_partner(pipe, &pipe->w_counter)) in fifo_open()
1187 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) in fifo_open()
1190 pipe->w_counter++; in fifo_open()
1191 if (!pipe->writers++) in fifo_open()
1192 wake_up_partner(pipe); in fifo_open()
1194 if (!is_pipe && !pipe->readers) { in fifo_open()
1195 if (wait_for_partner(pipe, &pipe->r_counter)) in fifo_open()
1208 pipe->readers++; in fifo_open()
1209 pipe->writers++; in fifo_open()
1210 pipe->r_counter++; in fifo_open()
1211 pipe->w_counter++; in fifo_open()
1212 if (pipe->readers == 1 || pipe->writers == 1) in fifo_open()
1213 wake_up_partner(pipe); in fifo_open()
1222 mutex_unlock(&pipe->mutex); in fifo_open()
1226 if (!--pipe->readers) in fifo_open()
1227 wake_up_interruptible(&pipe->wr_wait); in fifo_open()
1232 if (!--pipe->writers) in fifo_open()
1233 wake_up_interruptible_all(&pipe->rd_wait); in fifo_open()
1238 mutex_unlock(&pipe->mutex); in fifo_open()
1240 put_pipe_info(inode, pipe); in fifo_open()
1289 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots) in pipe_resize_ring() argument
1303 spin_lock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1304 mask = pipe->ring_size - 1; in pipe_resize_ring()
1305 head = pipe->head; in pipe_resize_ring()
1306 tail = pipe->tail; in pipe_resize_ring()
1310 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1323 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1326 unsigned int tsize = pipe->ring_size - t; in pipe_resize_ring()
1328 memcpy(bufs + tsize, pipe->bufs, in pipe_resize_ring()
1330 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1338 kfree(pipe->bufs); in pipe_resize_ring()
1339 pipe->bufs = bufs; in pipe_resize_ring()
1340 pipe->ring_size = nr_slots; in pipe_resize_ring()
1341 if (pipe->max_usage > nr_slots) in pipe_resize_ring()
1342 pipe->max_usage = nr_slots; in pipe_resize_ring()
1343 pipe->tail = tail; in pipe_resize_ring()
1344 pipe->head = head; in pipe_resize_ring()
1346 if (!pipe_has_watch_queue(pipe)) { in pipe_resize_ring()
1347 pipe->max_usage = nr_slots; in pipe_resize_ring()
1348 pipe->nr_accounted = nr_slots; in pipe_resize_ring()
1351 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1354 wake_up_interruptible(&pipe->wr_wait); in pipe_resize_ring()
1362 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg) in pipe_set_size() argument
1368 if (pipe_has_watch_queue(pipe)) in pipe_set_size()
1384 if (nr_slots > pipe->max_usage && in pipe_set_size()
1388 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots); in pipe_set_size()
1390 if (nr_slots > pipe->max_usage && in pipe_set_size()
1398 ret = pipe_resize_ring(pipe, nr_slots); in pipe_set_size()
1402 return pipe->max_usage * PAGE_SIZE; in pipe_set_size()
1405 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted); in pipe_set_size()
1415 struct pipe_inode_info *pipe = file->private_data; in get_pipe_info() local
1417 if (!pipe) in get_pipe_info()
1421 if (for_splice && pipe_has_watch_queue(pipe)) in get_pipe_info()
1423 return pipe; in get_pipe_info()
1428 struct pipe_inode_info *pipe; in pipe_fcntl() local
1431 pipe = get_pipe_info(file, false); in pipe_fcntl()
1432 if (!pipe) in pipe_fcntl()
1435 mutex_lock(&pipe->mutex); in pipe_fcntl()
1439 ret = pipe_set_size(pipe, arg); in pipe_fcntl()
1442 ret = pipe->max_usage * PAGE_SIZE; in pipe_fcntl()
1449 mutex_unlock(&pipe->mutex); in pipe_fcntl()