457c899653991 (Thomas Gleixner 2019-05-19 13:08:55 +0100 1) // SPDX-License-Identifier: GPL-2.0-only
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 2) /*
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 3) * "splice": joining two ropes together by interweaving their strands.
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 4) *
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 5) * This is the "extended pipe" functionality, where a pipe is used as
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 6) * an arbitrary in-memory buffer. Think of a pipe as a small kernel
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 7) * buffer that you can use to transfer data from one end to the other.
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 8) *
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 9) * The traditional unix read/write is extended with a "splice()" operation
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 10) * that transfers data buffers to or from a pipe buffer.
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 11) *
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 12) * Named by Larry McVoy, original implementation from Linus, extended by
c2058e0611c2f (Jens Axboe 2006-04-11 13:56:34 +0200 13) * Jens to support splicing to files, network, direct splicing, etc and
c2058e0611c2f (Jens Axboe 2006-04-11 13:56:34 +0200 14) * fixing lots of bugs.
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 15) *
0fe2347957712 (Jens Axboe 2006-09-04 15:41:16 +0200 16) * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
c2058e0611c2f (Jens Axboe 2006-04-11 13:56:34 +0200 17) * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
c2058e0611c2f (Jens Axboe 2006-04-11 13:56:34 +0200 18) * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 19) *
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 20) */
be297968da22c (Christoph Hellwig 2016-11-01 07:40:16 -0600 21) #include <linux/bvec.h>
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 22) #include <linux/fs.h>
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 23) #include <linux/file.h>
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 24) #include <linux/pagemap.h>
d6b29d7cee064 (Jens Axboe 2007-06-04 09:59:47 +0200 25) #include <linux/splice.h>
08e552c69c693 (KAMEZAWA Hiroyuki 2009-01-07 18:08:01 -0800 26) #include <linux/memcontrol.h>
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 27) #include <linux/mm_inline.h>
5abc97aa25b2c (Jens Axboe 2006-03-30 15:16:46 +0200 28) #include <linux/swap.h>
4f6f0bd2ffa4e (Jens Axboe 2006-04-02 23:04:46 +0200 29) #include <linux/writeback.h>
630d9c47274aa (Paul Gortmaker 2011-11-16 23:57:37 -0500 30) #include <linux/export.h>
4f6f0bd2ffa4e (Jens Axboe 2006-04-02 23:04:46 +0200 31) #include <linux/syscalls.h>
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 32) #include <linux/uio.h>
29ce20586be54 (James Morris 2007-07-13 11:44:32 +0200 33) #include <linux/security.h>
5a0e3ad6af866 (Tejun Heo 2010-03-24 17:04:11 +0900 34) #include <linux/gfp.h>
35f9c09fe9c72 (Eric Dumazet 2012-04-05 03:05:35 +0000 35) #include <linux/socket.h>
174cd4b1e5fbd (Ingo Molnar 2017-02-02 19:15:33 +0100 36) #include <linux/sched/signal.h>
174cd4b1e5fbd (Ingo Molnar 2017-02-02 19:15:33 +0100 37)
06ae43f34bcc0 (Al Viro 2013-03-20 13:19:30 -0400 38) #include "internal.h"
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 39)
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 40) /*
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 41) * Attempt to steal a page from a pipe buffer. This should perhaps go into
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 42) * a vm helper function, it's already simplified quite a bit by the
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 43) * addition of remove_mapping(). If success is returned, the caller may
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 44) * attempt to reuse this page for another destination.
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 45) */
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 46) static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 47) struct pipe_buffer *buf)
5abc97aa25b2c (Jens Axboe 2006-03-30 15:16:46 +0200 48) {
5abc97aa25b2c (Jens Axboe 2006-03-30 15:16:46 +0200 49) struct page *page = buf->page;
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 50) struct address_space *mapping;
5abc97aa25b2c (Jens Axboe 2006-03-30 15:16:46 +0200 51)
9e0267c26e237 (Jens Axboe 2006-04-19 15:57:31 +0200 52) lock_page(page);
9e0267c26e237 (Jens Axboe 2006-04-19 15:57:31 +0200 53)
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 54) mapping = page_mapping(page);
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 55) if (mapping) {
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 56) WARN_ON(!PageUptodate(page));
5abc97aa25b2c (Jens Axboe 2006-03-30 15:16:46 +0200 57)
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 58) /*
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 59) * At least for ext2 with nobh option, we need to wait on
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 60) * writeback completing on this page, since we'll remove it
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 61) * from the pagecache. Otherwise truncate wont wait on the
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 62) * page, allowing the disk blocks to be reused by someone else
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 63) * before we actually wrote our data to them. fs corruption
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 64) * ensues.
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 65) */
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 66) wait_on_page_writeback(page);
ad8d6f0a783ff (Jens Axboe 2006-04-02 23:10:32 +0200 67)
266cf658efcf6 (David Howells 2009-04-03 16:42:36 +0100 68) if (page_has_private(page) &&
266cf658efcf6 (David Howells 2009-04-03 16:42:36 +0100 69) !try_to_release_page(page, GFP_KERNEL))
ca39d651d17df (Jens Axboe 2008-05-20 21:27:41 +0200 70) goto out_unlock;
4f6f0bd2ffa4e (Jens Axboe 2006-04-02 23:04:46 +0200 71)
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 72) /*
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 73) * If we succeeded in removing the mapping, set LRU flag
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 74) * and return good.
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 75) */
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 76) if (remove_mapping(mapping, page)) {
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 77) buf->flags |= PIPE_BUF_FLAG_LRU;
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 78) return true;
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 79) }
9e0267c26e237 (Jens Axboe 2006-04-19 15:57:31 +0200 80) }
5abc97aa25b2c (Jens Axboe 2006-03-30 15:16:46 +0200 81)
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 82) /*
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 83) * Raced with truncate or failed to remove page from current
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 84) * address space, unlock and return failure.
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 85) */
ca39d651d17df (Jens Axboe 2008-05-20 21:27:41 +0200 86) out_unlock:
9e94cd4fd1812 (Jens Axboe 2006-06-20 15:01:12 +0200 87) unlock_page(page);
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 88) return false;
5abc97aa25b2c (Jens Axboe 2006-03-30 15:16:46 +0200 89) }
5abc97aa25b2c (Jens Axboe 2006-03-30 15:16:46 +0200 90)
76ad4d11105cc (Jens Axboe 2006-05-03 10:41:33 +0200 91) static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 92) struct pipe_buffer *buf)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 93) {
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 94) put_page(buf->page);
1432873af7ae2 (Jens Axboe 2006-05-03 10:35:26 +0200 95) buf->flags &= ~PIPE_BUF_FLAG_LRU;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 96) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 97)
0845718dafea3 (Jens Axboe 2007-06-12 20:51:32 +0200 98) /*
0845718dafea3 (Jens Axboe 2007-06-12 20:51:32 +0200 99) * Check whether the contents of buf is OK to access. Since the content
0845718dafea3 (Jens Axboe 2007-06-12 20:51:32 +0200 100) * is a page cache page, IO may be in flight.
0845718dafea3 (Jens Axboe 2007-06-12 20:51:32 +0200 101) */
cac36bb06efe4 (Jens Axboe 2007-06-14 13:10:48 +0200 102) static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
cac36bb06efe4 (Jens Axboe 2007-06-14 13:10:48 +0200 103) struct pipe_buffer *buf)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 104) {
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 105) struct page *page = buf->page;
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 106) int err;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 107)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 108) if (!PageUptodate(page)) {
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 109) lock_page(page);
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 110)
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 111) /*
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 112) * Page got truncated/unhashed. This will cause a 0-byte
73d62d83ec362 (Ingo Molnar 2006-04-11 13:57:21 +0200 113) * splice, if this is the first page.
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 114) */
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 115) if (!page->mapping) {
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 116) err = -ENODATA;
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 117) goto error;
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 118) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 119)
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 120) /*
73d62d83ec362 (Ingo Molnar 2006-04-11 13:57:21 +0200 121) * Uh oh, read-error from disk.
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 122) */
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 123) if (!PageUptodate(page)) {
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 124) err = -EIO;
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 125) goto error;
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 126) }
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 127)
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 128) /*
f84d751994441 (Jens Axboe 2006-05-01 19:59:03 +0200 129) * Page is ok afterall, we are done.
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 130) */
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 131) unlock_page(page);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 132) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 133)
f84d751994441 (Jens Axboe 2006-05-01 19:59:03 +0200 134) return 0;
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 135) error:
49d0b21be21ef (Jens Axboe 2006-04-10 09:04:41 +0200 136) unlock_page(page);
f84d751994441 (Jens Axboe 2006-05-01 19:59:03 +0200 137) return err;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 138) }
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 139)
708e3508c2a22 (Hugh Dickins 2011-07-25 17:12:32 -0700 140) const struct pipe_buf_operations page_cache_pipe_buf_ops = {
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 141) .confirm = page_cache_pipe_buf_confirm,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 142) .release = page_cache_pipe_buf_release,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 143) .try_steal = page_cache_pipe_buf_try_steal,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 144) .get = generic_pipe_buf_get,
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 145) };
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 146)
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 147) static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 148) struct pipe_buffer *buf)
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 149) {
7afa6fd037e51 (Jens Axboe 2006-05-01 20:02:33 +0200 150) if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 151) return false;
7afa6fd037e51 (Jens Axboe 2006-05-01 20:02:33 +0200 152)
1432873af7ae2 (Jens Axboe 2006-05-03 10:35:26 +0200 153) buf->flags |= PIPE_BUF_FLAG_LRU;
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 154) return generic_pipe_buf_try_steal(pipe, buf);
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 155) }
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 156)
d4c3cca941b64 (Eric Dumazet 2006-12-13 00:34:04 -0800 157) static const struct pipe_buf_operations user_page_pipe_buf_ops = {
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 158) .release = page_cache_pipe_buf_release,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 159) .try_steal = user_page_pipe_buf_try_steal,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 160) .get = generic_pipe_buf_get,
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 161) };
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 162)
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 163) static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 164) {
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 165) smp_mb();
0ddad21d3e99c (Linus Torvalds 2019-12-09 09:48:27 -0800 166) if (waitqueue_active(&pipe->rd_wait))
0ddad21d3e99c (Linus Torvalds 2019-12-09 09:48:27 -0800 167) wake_up_interruptible(&pipe->rd_wait);
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 168) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 169) }
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 170)
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 171) /**
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 172) * splice_to_pipe - fill passed data into a pipe
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 173) * @pipe: pipe to fill
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 174) * @spd: data to fill
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 175) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 176) * Description:
79685b8deea45 (Randy Dunlap 2007-07-27 08:08:51 +0200 177) * @spd contains a map of pages and len/offset tuples, along with
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 178) * the struct pipe_buf_operations associated with these pages. This
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 179) * function will link that data to the pipe.
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 180) *
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 181) */
d6b29d7cee064 (Jens Axboe 2007-06-04 09:59:47 +0200 182) ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
d6b29d7cee064 (Jens Axboe 2007-06-04 09:59:47 +0200 183) struct splice_pipe_desc *spd)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 184) {
00de00bdad278 (Jens Axboe 2007-06-15 13:14:22 +0200 185) unsigned int spd_pages = spd->nr_pages;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 186) unsigned int tail = pipe->tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 187) unsigned int head = pipe->head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 188) unsigned int mask = pipe->ring_size - 1;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 189) int ret = 0, page_nr = 0;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 190)
d6785d9152147 (Rabin Vincent 2016-03-10 21:19:06 +0100 191) if (!spd_pages)
d6785d9152147 (Rabin Vincent 2016-03-10 21:19:06 +0100 192) return 0;
d6785d9152147 (Rabin Vincent 2016-03-10 21:19:06 +0100 193)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 194) if (unlikely(!pipe->readers)) {
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 195) send_sig(SIGPIPE, current, 0);
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 196) ret = -EPIPE;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 197) goto out;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 198) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 199)
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 200) while (!pipe_full(head, tail, pipe->max_usage)) {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 201) struct pipe_buffer *buf = &pipe->bufs[head & mask];
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 202)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 203) buf->page = spd->pages[page_nr];
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 204) buf->offset = spd->partial[page_nr].offset;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 205) buf->len = spd->partial[page_nr].len;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 206) buf->private = spd->partial[page_nr].private;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 207) buf->ops = spd->ops;
5a81e6a171cdb (Miklos Szeredi 2017-02-16 17:49:02 +0100 208) buf->flags = 0;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 209)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 210) head++;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 211) pipe->head = head;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 212) page_nr++;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 213) ret += buf->len;
29e350944fdc2 (Linus Torvalds 2006-04-02 12:46:35 -0700 214)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 215) if (!--spd->nr_pages)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 216) break;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 217) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 218)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 219) if (!ret)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 220) ret = -EAGAIN;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 221)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 222) out:
00de00bdad278 (Jens Axboe 2007-06-15 13:14:22 +0200 223) while (page_nr < spd_pages)
bbdfc2f70610b (Jens Axboe 2007-11-06 23:29:47 -0800 224) spd->spd_release(spd, page_nr++);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 225)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 226) return ret;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 227) }
2b514574f7e88 (Hannes Frederic Sowa 2015-05-21 17:00:01 +0200 228) EXPORT_SYMBOL_GPL(splice_to_pipe);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 229)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 230) ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 231) {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 232) unsigned int head = pipe->head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 233) unsigned int tail = pipe->tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 234) unsigned int mask = pipe->ring_size - 1;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 235) int ret;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 236)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 237) if (unlikely(!pipe->readers)) {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 238) send_sig(SIGPIPE, current, 0);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 239) ret = -EPIPE;
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 240) } else if (pipe_full(head, tail, pipe->max_usage)) {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 241) ret = -EAGAIN;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 242) } else {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 243) pipe->bufs[head & mask] = *buf;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 244) pipe->head = head + 1;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 245) return buf->len;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 246) }
a779638cf622f (Miklos Szeredi 2016-09-27 10:45:12 +0200 247) pipe_buf_release(pipe, buf);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 248) return ret;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 249) }
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 250) EXPORT_SYMBOL(add_to_pipe);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 251)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 252) /*
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 253) * Check if we need to grow the arrays holding pages and partial page
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 254) * descriptions.
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 255) */
047fe36052358 (Eric Dumazet 2012-06-12 15:24:40 +0200 256) int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 257) {
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 258) unsigned int max_usage = READ_ONCE(pipe->max_usage);
047fe36052358 (Eric Dumazet 2012-06-12 15:24:40 +0200 259)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 260) spd->nr_pages_max = max_usage;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 261) if (max_usage <= PIPE_DEF_BUFFERS)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 262) return 0;
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 263)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 264) spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 265) spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
6da2ec56059c3 (Kees Cook 2018-06-12 13:55:00 -0700 266) GFP_KERNEL);
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 267)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 268) if (spd->pages && spd->partial)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 269) return 0;
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 270)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 271) kfree(spd->pages);
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 272) kfree(spd->partial);
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 273) return -ENOMEM;
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 274) }
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 275)
047fe36052358 (Eric Dumazet 2012-06-12 15:24:40 +0200 276) void splice_shrink_spd(struct splice_pipe_desc *spd)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 277) {
047fe36052358 (Eric Dumazet 2012-06-12 15:24:40 +0200 278) if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 279) return;
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 280)
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 281) kfree(spd->pages);
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 282) kfree(spd->partial);
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 283) }
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 284)
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 285) /**
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 286) * generic_file_splice_read - splice data from file to a pipe
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 287) * @in: file to splice from
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 288) * @ppos: position in @in
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 289) * @pipe: pipe to splice to
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 290) * @len: number of bytes to splice
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 291) * @flags: splice modifier flags
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 292) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 293) * Description:
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 294) * Will read pages from given file and fill them into a pipe. Can be
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 295) * used as long as it has more or less sane ->read_iter().
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 296) *
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 297) */
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 298) ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 299) struct pipe_inode_info *pipe, size_t len,
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 300) unsigned int flags)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 301) {
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 302) struct iov_iter to;
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 303) struct kiocb kiocb;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 304) unsigned int i_head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 305) int ret;
be64f884bed72 (Boaz Harrosh 2015-04-15 16:15:17 -0700 306)
aa563d7bca6e8 (David Howells 2018-10-20 00:57:56 +0100 307) iov_iter_pipe(&to, READ, pipe, len);
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 308) i_head = to.head;
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 309) init_sync_kiocb(&kiocb, in);
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 310) kiocb.ki_pos = *ppos;
bb7462b6fd64e (Miklos Szeredi 2017-02-20 16:51:23 +0100 311) ret = call_read_iter(in, &kiocb, &to);
723590ed52d24 (Miklos Szeredi 2009-08-15 08:43:22 +0200 312) if (ret > 0) {
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 313) *ppos = kiocb.ki_pos;
723590ed52d24 (Miklos Szeredi 2009-08-15 08:43:22 +0200 314) file_accessed(in);
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 315) } else if (ret < 0) {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 316) to.head = i_head;
c3a690240423f (Al Viro 2016-10-10 13:26:27 -0400 317) to.iov_offset = 0;
c3a690240423f (Al Viro 2016-10-10 13:26:27 -0400 318) iov_iter_advance(&to, 0); /* to free what was emitted */
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 319) /*
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 320) * callers of ->splice_read() expect -EAGAIN on
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 321) * "can't put anything in there", rather than -EFAULT.
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 322) */
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 323) if (ret == -EFAULT)
82c156f853840 (Al Viro 2016-09-22 23:35:42 -0400 324) ret = -EAGAIN;
723590ed52d24 (Miklos Szeredi 2009-08-15 08:43:22 +0200 325) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 326)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 327) return ret;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 328) }
059a8f3734a66 (Jens Axboe 2006-04-02 23:06:05 +0200 329) EXPORT_SYMBOL(generic_file_splice_read);
059a8f3734a66 (Jens Axboe 2006-04-02 23:06:05 +0200 330)
241699cd72a84 (Al Viro 2016-09-22 16:33:12 -0400 331) const struct pipe_buf_operations default_pipe_buf_ops = {
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 332) .release = generic_pipe_buf_release,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 333) .try_steal = generic_pipe_buf_try_steal,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 334) .get = generic_pipe_buf_get,
6818173bd6584 (Miklos Szeredi 2009-05-07 15:37:36 +0200 335) };
6818173bd6584 (Miklos Szeredi 2009-05-07 15:37:36 +0200 336)
28a625cbc2a14 (Miklos Szeredi 2014-01-22 19:36:57 +0100 337) /* Pipe buffer operations for a socket and similar. */
28a625cbc2a14 (Miklos Szeredi 2014-01-22 19:36:57 +0100 338) const struct pipe_buf_operations nosteal_pipe_buf_ops = {
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 339) .release = generic_pipe_buf_release,
c928f642c29a5 (Christoph Hellwig 2020-05-20 17:58:16 +0200 340) .get = generic_pipe_buf_get,
28a625cbc2a14 (Miklos Szeredi 2014-01-22 19:36:57 +0100 341) };
28a625cbc2a14 (Miklos Szeredi 2014-01-22 19:36:57 +0100 342) EXPORT_SYMBOL(nosteal_pipe_buf_ops);
28a625cbc2a14 (Miklos Szeredi 2014-01-22 19:36:57 +0100 343)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 344) /*
4f6f0bd2ffa4e (Jens Axboe 2006-04-02 23:04:46 +0200 345) * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
016b661e2f717 (Jens Axboe 2006-04-25 15:42:00 +0200 346) * using sendpage(). Return the number of bytes sent.
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 347) */
76ad4d11105cc (Jens Axboe 2006-05-03 10:41:33 +0200 348) static int pipe_to_sendpage(struct pipe_inode_info *pipe,
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 349) struct pipe_buffer *buf, struct splice_desc *sd)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 350) {
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 351) struct file *file = sd->u.file;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 352) loff_t pos = sd->pos;
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 353) int more;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 354)
72c2d53192004 (Al Viro 2013-09-22 16:27:52 -0400 355) if (!likely(file->f_op->sendpage))
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 356) return -EINVAL;
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 357)
35f9c09fe9c72 (Eric Dumazet 2012-04-05 03:05:35 +0000 358) more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
ae62ca7b03217 (Eric Dumazet 2013-01-06 18:21:49 +0000 359)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 360) if (sd->len < sd->total_len &&
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 361) pipe_occupancy(pipe->head, pipe->tail) > 1)
35f9c09fe9c72 (Eric Dumazet 2012-04-05 03:05:35 +0000 362) more |= MSG_SENDPAGE_NOTLAST;
ae62ca7b03217 (Eric Dumazet 2013-01-06 18:21:49 +0000 363)
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 364) return file->f_op->sendpage(file, buf->page, buf->offset,
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 365) sd->len, &pos, more);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 366) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 367)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 368) static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 369) {
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 370) smp_mb();
0ddad21d3e99c (Linus Torvalds 2019-12-09 09:48:27 -0800 371) if (waitqueue_active(&pipe->wr_wait))
0ddad21d3e99c (Linus Torvalds 2019-12-09 09:48:27 -0800 372) wake_up_interruptible(&pipe->wr_wait);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 373) kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 374) }
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 375)
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 376) /**
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 377) * splice_from_pipe_feed - feed available data from a pipe to a file
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 378) * @pipe: pipe to splice from
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 379) * @sd: information to @actor
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 380) * @actor: handler that splices the data
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 381) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 382) * Description:
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 383) * This function loops over the pipe and calls @actor to do the
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 384) * actual moving of a single struct pipe_buffer to the desired
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 385) * destination. It returns when there's no more buffers left in
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 386) * the pipe or if the requested number of bytes (@sd->total_len)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 387) * have been copied. It returns a positive number (one) if the
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 388) * pipe needs to be filled with more data, zero if the required
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 389) * number of bytes have been copied and -errno on error.
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 390) *
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 391) * This, together with splice_from_pipe_{begin,end,next}, may be
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 392) * used to implement the functionality of __splice_from_pipe() when
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 393) * locking is required around copying the pipe buffers to the
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 394) * destination.
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 395) */
96f9bc8fbc244 (Al Viro 2014-04-05 04:35:49 -0400 396) static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 397) splice_actor *actor)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 398) {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 399) unsigned int head = pipe->head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 400) unsigned int tail = pipe->tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 401) unsigned int mask = pipe->ring_size - 1;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 402) int ret;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 403)
ec057595cb3fb (Linus Torvalds 2019-12-06 12:40:35 -0800 404) while (!pipe_empty(head, tail)) {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 405) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 406)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 407) sd->len = buf->len;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 408) if (sd->len > sd->total_len)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 409) sd->len = sd->total_len;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 410)
fba597db4218a (Miklos Szeredi 2016-09-27 10:45:12 +0200 411) ret = pipe_buf_confirm(pipe, buf);
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 412) if (unlikely(ret)) {
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 413) if (ret == -ENODATA)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 414) ret = 0;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 415) return ret;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 416) }
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 417)
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 418) ret = actor(pipe, buf, sd);
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 419) if (ret <= 0)
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 420) return ret;
a8adbe378b56a (Michał Mirosław 2010-12-17 08:56:44 +0100 421)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 422) buf->offset += ret;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 423) buf->len -= ret;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 424)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 425) sd->num_spliced += ret;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 426) sd->len -= ret;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 427) sd->pos += ret;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 428) sd->total_len -= ret;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 429)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 430) if (!buf->len) {
a779638cf622f (Miklos Szeredi 2016-09-27 10:45:12 +0200 431) pipe_buf_release(pipe, buf);
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 432) tail++;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 433) pipe->tail = tail;
6447a3cf19da8 (Al Viro 2013-03-21 11:01:38 -0400 434) if (pipe->files)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 435) sd->need_wakeup = true;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 436) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 437)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 438) if (!sd->total_len)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 439) return 0;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 440) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 441)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 442) return 1;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 443) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 444)
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 445) /* We know we have a pipe buffer, but maybe it's empty? */
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 446) static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 447) {
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 448) unsigned int tail = pipe->tail;
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 449) unsigned int mask = pipe->ring_size - 1;
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 450) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 451)
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 452) if (unlikely(!buf->len)) {
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 453) pipe_buf_release(pipe, buf);
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 454) pipe->tail = tail+1;
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 455) return true;
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 456) }
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 457)
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 458) return false;
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 459) }
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 460)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 461) /**
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 462) * splice_from_pipe_next - wait for some data to splice from
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 463) * @pipe: pipe to splice from
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 464) * @sd: information about the splice operation
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 465) *
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 466) * Description:
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 467) * This function will wait for some data and return a positive
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 468) * value (one) if pipe buffers are available. It will return zero
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 469) * or -errno if no more data needs to be spliced.
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 470) */
96f9bc8fbc244 (Al Viro 2014-04-05 04:35:49 -0400 471) static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 472) {
c725bfce79680 (Jan Kara 2015-11-23 13:09:50 +0100 473) /*
c725bfce79680 (Jan Kara 2015-11-23 13:09:50 +0100 474) * Check for signal early to make process killable when there are
c725bfce79680 (Jan Kara 2015-11-23 13:09:50 +0100 475) * always buffers available
c725bfce79680 (Jan Kara 2015-11-23 13:09:50 +0100 476) */
c725bfce79680 (Jan Kara 2015-11-23 13:09:50 +0100 477) if (signal_pending(current))
c725bfce79680 (Jan Kara 2015-11-23 13:09:50 +0100 478) return -ERESTARTSYS;
c725bfce79680 (Jan Kara 2015-11-23 13:09:50 +0100 479)
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 480) repeat:
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 481) while (pipe_empty(pipe->head, pipe->tail)) {
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 482) if (!pipe->writers)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 483) return 0;
016b661e2f717 (Jens Axboe 2006-04-25 15:42:00 +0200 484)
a28c8b9db8a10 (Linus Torvalds 2019-12-07 13:21:01 -0800 485) if (sd->num_spliced)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 486) return 0;
73d62d83ec362 (Ingo Molnar 2006-04-11 13:57:21 +0200 487)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 488) if (sd->flags & SPLICE_F_NONBLOCK)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 489) return -EAGAIN;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 490)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 491) if (signal_pending(current))
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 492) return -ERESTARTSYS;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 493)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 494) if (sd->need_wakeup) {
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 495) wakeup_pipe_writers(pipe);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 496) sd->need_wakeup = false;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 497) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 498)
472e5b056f000 (Linus Torvalds 2020-10-01 19:14:36 -0700 499) pipe_wait_readable(pipe);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 500) }
29e350944fdc2 (Linus Torvalds 2006-04-02 12:46:35 -0700 501)
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 502) if (eat_empty_buffer(pipe))
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 503) goto repeat;
d1a819a2ec2d3 (Linus Torvalds 2020-10-05 11:26:27 -0700 504)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 505) return 1;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 506) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 507)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 508) /**
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 509) * splice_from_pipe_begin - start splicing from pipe
b80901bbf5995 (Randy Dunlap 2009-04-16 19:09:55 -0700 510) * @sd: information about the splice operation
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 511) *
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 512) * Description:
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 513) * This function should be called before a loop containing
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 514) * splice_from_pipe_next() and splice_from_pipe_feed() to
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 515) * initialize the necessary fields of @sd.
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 516) */
96f9bc8fbc244 (Al Viro 2014-04-05 04:35:49 -0400 517) static void splice_from_pipe_begin(struct splice_desc *sd)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 518) {
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 519) sd->num_spliced = 0;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 520) sd->need_wakeup = false;
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 521) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 522)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 523) /**
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 524) * splice_from_pipe_end - finish splicing from pipe
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 525) * @pipe: pipe to splice from
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 526) * @sd: information about the splice operation
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 527) *
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 528) * Description:
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 529) * This function will wake up pipe writers if necessary. It should
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 530) * be called after a loop containing splice_from_pipe_next() and
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 531) * splice_from_pipe_feed().
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 532) */
96f9bc8fbc244 (Al Viro 2014-04-05 04:35:49 -0400 533) static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 534) {
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 535) if (sd->need_wakeup)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 536) wakeup_pipe_writers(pipe);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 537) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 538)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 539) /**
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 540) * __splice_from_pipe - splice data from a pipe to given actor
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 541) * @pipe: pipe to splice from
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 542) * @sd: information to @actor
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 543) * @actor: handler that splices the data
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 544) *
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 545) * Description:
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 546) * This function does little more than loop over the pipe and call
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 547) * @actor to do the actual moving of a single struct pipe_buffer to
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 548) * the desired destination. See pipe_to_file, pipe_to_sendpage, or
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 549) * pipe_to_user.
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 550) *
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 551) */
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 552) ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 553) splice_actor *actor)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 554) {
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 555) int ret;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 556)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 557) splice_from_pipe_begin(sd);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 558) do {
c2489e07c0a71 (Jan Kara 2015-11-23 13:09:51 +0100 559) cond_resched();
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 560) ret = splice_from_pipe_next(pipe, sd);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 561) if (ret > 0)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 562) ret = splice_from_pipe_feed(pipe, sd, actor);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 563) } while (ret > 0);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 564) splice_from_pipe_end(pipe, sd);
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 565)
b3c2d2ddd6394 (Miklos Szeredi 2009-04-14 19:48:36 +0200 566) return sd->num_spliced ? sd->num_spliced : ret;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 567) }
40bee44eaef91 (Mark Fasheh 2007-03-21 13:11:02 +0100 568) EXPORT_SYMBOL(__splice_from_pipe);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 569)
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 570) /**
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 571) * splice_from_pipe - splice data from a pipe to a file
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 572) * @pipe: pipe to splice from
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 573) * @out: file to splice to
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 574) * @ppos: position in @out
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 575) * @len: how many bytes to splice
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 576) * @flags: splice modifier flags
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 577) * @actor: handler that splices the data
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 578) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 579) * Description:
2933970b96022 (Miklos Szeredi 2009-04-14 19:48:37 +0200 580) * See __splice_from_pipe. This function locks the pipe inode,
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 581) * otherwise it's identical to __splice_from_pipe().
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 582) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 583) */
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 584) ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 585) loff_t *ppos, size_t len, unsigned int flags,
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 586) splice_actor *actor)
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 587) {
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 588) ssize_t ret;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 589) struct splice_desc sd = {
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 590) .total_len = len,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 591) .flags = flags,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 592) .pos = *ppos,
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 593) .u.file = out,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 594) };
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 595)
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 596) pipe_lock(pipe);
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 597) ret = __splice_from_pipe(pipe, &sd, actor);
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 598) pipe_unlock(pipe);
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 599)
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 600) return ret;
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 601) }
6da61809822c2 (Mark Fasheh 2006-10-17 18:43:07 +0200 602)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 603) /**
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 604) * iter_file_splice_write - splice data from a pipe to a file
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 605) * @pipe: pipe info
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 606) * @out: file to write to
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 607) * @ppos: position in @out
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 608) * @len: number of bytes to splice
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 609) * @flags: splice modifier flags
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 610) *
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 611) * Description:
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 612) * Will either move or copy pages (determined by @flags options) from
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 613) * the given pipe inode to the given file.
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 614) * This one is ->write_iter-based.
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 615) *
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 616) */
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 617) ssize_t
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 618) iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 619) loff_t *ppos, size_t len, unsigned int flags)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 620) {
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 621) struct splice_desc sd = {
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 622) .total_len = len,
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 623) .flags = flags,
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 624) .pos = *ppos,
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 625) .u.file = out,
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 626) };
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 627) int nbufs = pipe->max_usage;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 628) struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 629) GFP_KERNEL);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 630) ssize_t ret;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 631)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 632) if (unlikely(!array))
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 633) return -ENOMEM;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 634)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 635) pipe_lock(pipe);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 636)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 637) splice_from_pipe_begin(&sd);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 638) while (sd.total_len) {
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 639) struct iov_iter from;
ec057595cb3fb (Linus Torvalds 2019-12-06 12:40:35 -0800 640) unsigned int head, tail, mask;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 641) size_t left;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 642) int n;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 643)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 644) ret = splice_from_pipe_next(pipe, &sd);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 645) if (ret <= 0)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 646) break;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 647)
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 648) if (unlikely(nbufs < pipe->max_usage)) {
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 649) kfree(array);
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 650) nbufs = pipe->max_usage;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 651) array = kcalloc(nbufs, sizeof(struct bio_vec),
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 652) GFP_KERNEL);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 653) if (!array) {
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 654) ret = -ENOMEM;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 655) break;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 656) }
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 657) }
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 658)
ec057595cb3fb (Linus Torvalds 2019-12-06 12:40:35 -0800 659) head = pipe->head;
ec057595cb3fb (Linus Torvalds 2019-12-06 12:40:35 -0800 660) tail = pipe->tail;
ec057595cb3fb (Linus Torvalds 2019-12-06 12:40:35 -0800 661) mask = pipe->ring_size - 1;
ec057595cb3fb (Linus Torvalds 2019-12-06 12:40:35 -0800 662)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 663) /* build the vector */
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 664) left = sd.total_len;
0f1d344feb534 (Pavel Begunkov 2021-01-09 16:02:57 +0000 665) for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 666) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 667) size_t this_len = buf->len;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 668)
0f1d344feb534 (Pavel Begunkov 2021-01-09 16:02:57 +0000 669) /* zero-length bvecs are not supported, skip them */
0f1d344feb534 (Pavel Begunkov 2021-01-09 16:02:57 +0000 670) if (!this_len)
0f1d344feb534 (Pavel Begunkov 2021-01-09 16:02:57 +0000 671) continue;
0f1d344feb534 (Pavel Begunkov 2021-01-09 16:02:57 +0000 672) this_len = min(this_len, left);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 673)
fba597db4218a (Miklos Szeredi 2016-09-27 10:45:12 +0200 674) ret = pipe_buf_confirm(pipe, buf);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 675) if (unlikely(ret)) {
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 676) if (ret == -ENODATA)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 677) ret = 0;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 678) goto done;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 679) }
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 680)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 681) array[n].bv_page = buf->page;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 682) array[n].bv_len = this_len;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 683) array[n].bv_offset = buf->offset;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 684) left -= this_len;
0f1d344feb534 (Pavel Begunkov 2021-01-09 16:02:57 +0000 685) n++;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 686) }
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 687)
aa563d7bca6e8 (David Howells 2018-10-20 00:57:56 +0100 688) iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left);
abbb65899aecf (Christoph Hellwig 2017-05-27 11:16:52 +0300 689) ret = vfs_iter_write(out, &from, &sd.pos, 0);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 690) if (ret <= 0)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 691) break;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 692)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 693) sd.num_spliced += ret;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 694) sd.total_len -= ret;
dbe4e192a234c (Christoph Hellwig 2015-01-25 21:11:59 +0100 695) *ppos = sd.pos;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 696)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 697) /* dismiss the fully eaten buffers, adjust the partial one */
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 698) tail = pipe->tail;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 699) while (ret) {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 700) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 701) if (ret >= buf->len) {
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 702) ret -= buf->len;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 703) buf->len = 0;
a779638cf622f (Miklos Szeredi 2016-09-27 10:45:12 +0200 704) pipe_buf_release(pipe, buf);
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 705) tail++;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 706) pipe->tail = tail;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 707) if (pipe->files)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 708) sd.need_wakeup = true;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 709) } else {
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 710) buf->offset += ret;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 711) buf->len -= ret;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 712) ret = 0;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 713) }
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 714) }
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 715) }
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 716) done:
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 717) kfree(array);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 718) splice_from_pipe_end(pipe, &sd);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 719)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 720) pipe_unlock(pipe);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 721)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 722) if (sd.num_spliced)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 723) ret = sd.num_spliced;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 724)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 725) return ret;
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 726) }
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 727)
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 728) EXPORT_SYMBOL(iter_file_splice_write);
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 729)
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 730) /**
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 731) * generic_splice_sendpage - splice data from a pipe to a socket
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 732) * @pipe: pipe to splice from
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 733) * @out: socket to write to
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 734) * @ppos: position in @out
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 735) * @len: number of bytes to splice
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 736) * @flags: splice modifier flags
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 737) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 738) * Description:
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 739) * Will send @len bytes from the pipe to a network socket. No data copying
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 740) * is involved.
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 741) *
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 742) */
3a326a2ce88e7 (Ingo Molnar 2006-04-10 15:18:35 +0200 743) ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 744) loff_t *ppos, size_t len, unsigned int flags)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 745) {
00522fb41a2a9 (Jens Axboe 2006-04-26 14:39:29 +0200 746) return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 747) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 748)
059a8f3734a66 (Jens Axboe 2006-04-02 23:06:05 +0200 749) EXPORT_SYMBOL(generic_splice_sendpage);
a0f067802576d (Jeff Garzik 2006-03-30 23:06:13 -0500 750)
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 751) static int warn_unsupported(struct file *file, const char *op)
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 752) {
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 753) pr_debug_ratelimited(
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 754) "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 755) op, file, current->pid, current->comm);
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 756) return -EINVAL;
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 757) }
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 758)
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 759) /*
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 760) * Attempt to initiate a splice from pipe to file.
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 761) */
3a326a2ce88e7 (Ingo Molnar 2006-04-10 15:18:35 +0200 762) static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 763) loff_t *ppos, size_t len, unsigned int flags)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 764) {
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 765) if (unlikely(!out->f_op->splice_write))
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 766) return warn_unsupported(out, "write");
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 767) return out->f_op->splice_write(pipe, out, ppos, len, flags);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 768) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 769)
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 770) /*
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 771) * Attempt to initiate a splice from a file to a pipe.
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 772) */
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 773) static long do_splice_to(struct file *in, loff_t *ppos,
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 774) struct pipe_inode_info *pipe, size_t len,
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 775) unsigned int flags)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 776) {
313d64a35d36b (Al Viro 2021-01-24 23:49:04 -0500 777) unsigned int p_space;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 778) int ret;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 779)
49570e9b29a3d (Jens Axboe 2006-04-11 13:56:09 +0200 780) if (unlikely(!(in->f_mode & FMODE_READ)))
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 781) return -EBADF;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 782)
313d64a35d36b (Al Viro 2021-01-24 23:49:04 -0500 783) /* Don't try to read more the pipe has space for. */
313d64a35d36b (Al Viro 2021-01-24 23:49:04 -0500 784) p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail);
313d64a35d36b (Al Viro 2021-01-24 23:49:04 -0500 785) len = min_t(size_t, len, p_space << PAGE_SHIFT);
313d64a35d36b (Al Viro 2021-01-24 23:49:04 -0500 786)
cbb7e577e732f (Jens Axboe 2006-04-11 14:57:50 +0200 787) ret = rw_verify_area(READ, in, ppos, len);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 788) if (unlikely(ret < 0))
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 789) return ret;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 790)
03cc0789a690e (Al Viro 2016-04-02 14:56:58 -0400 791) if (unlikely(len > MAX_RW_COUNT))
03cc0789a690e (Al Viro 2016-04-02 14:56:58 -0400 792) len = MAX_RW_COUNT;
03cc0789a690e (Al Viro 2016-04-02 14:56:58 -0400 793)
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 794) if (unlikely(!in->f_op->splice_read))
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 795) return warn_unsupported(in, "read");
36e2c7421f02a (Christoph Hellwig 2020-09-03 16:22:34 +0200 796) return in->f_op->splice_read(in, ppos, pipe, len, flags);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 797) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 798)
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 799) /**
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 800) * splice_direct_to_actor - splices data directly between two non-pipes
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 801) * @in: file to splice from
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 802) * @sd: actor information on where to splice to
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 803) * @actor: handles the data splicing
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 804) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 805) * Description:
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 806) * This is a special case helper to splice directly between two
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 807) * points, without requiring an explicit pipe. Internally an allocated
79685b8deea45 (Randy Dunlap 2007-07-27 08:08:51 +0200 808) * pipe is cached in the process, and reused during the lifetime of
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 809) * that process.
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 810) *
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 811) */
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 812) ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 813) splice_direct_actor *actor)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 814) {
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 815) struct pipe_inode_info *pipe;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 816) long ret, bytes;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 817) umode_t i_mode;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 818) size_t len;
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 819) int i, flags, more;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 820)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 821) /*
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 822) * We require the input being a regular file, as we don't want to
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 823) * randomly drop data for eg socket -> socket splicing. Use the
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 824) * piped splicing for that!
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 825) */
496ad9aa8ef44 (Al Viro 2013-01-23 17:07:38 -0500 826) i_mode = file_inode(in)->i_mode;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 827) if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 828) return -EINVAL;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 829)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 830) /*
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 831) * neither in nor out is a pipe, setup an internal pipe attached to
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 832) * 'out' and transfer the wanted data from 'in' to 'out' through that
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 833) */
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 834) pipe = current->splice_pipe;
49570e9b29a3d (Jens Axboe 2006-04-11 13:56:09 +0200 835) if (unlikely(!pipe)) {
7bee130e222df (Al Viro 2013-03-21 11:04:15 -0400 836) pipe = alloc_pipe_info();
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 837) if (!pipe)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 838) return -ENOMEM;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 839)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 840) /*
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 841) * We don't have an immediate reader, but we'll read the stuff
00522fb41a2a9 (Jens Axboe 2006-04-26 14:39:29 +0200 842) * out of the pipe right after the splice_to_pipe(). So set
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 843) * PIPE_READERS appropriately.
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 844) */
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 845) pipe->readers = 1;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 846)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 847) current->splice_pipe = pipe;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 848) }
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 849)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 850) /*
73d62d83ec362 (Ingo Molnar 2006-04-11 13:57:21 +0200 851) * Do the splice.
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 852) */
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 853) ret = 0;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 854) bytes = 0;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 855) len = sd->total_len;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 856) flags = sd->flags;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 857)
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 858) /*
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 859) * Don't block on output, we have to drain the direct pipe.
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 860) */
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 861) sd->flags &= ~SPLICE_F_NONBLOCK;
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 862) more = sd->flags & SPLICE_F_MORE;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 863)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 864) WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));
17614445576b6 (Darrick J. Wong 2018-11-30 10:37:49 -0800 865)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 866) while (len) {
51a92c0f6ce8f (Jens Axboe 2007-07-13 14:11:43 +0200 867) size_t read_len;
a82c53a0e3f57 (Tom Zanussi 2008-05-09 13:28:36 +0200 868) loff_t pos = sd->pos, prev_pos = pos;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 869)
313d64a35d36b (Al Viro 2021-01-24 23:49:04 -0500 870) ret = do_splice_to(in, &pos, pipe, len, flags);
51a92c0f6ce8f (Jens Axboe 2007-07-13 14:11:43 +0200 871) if (unlikely(ret <= 0))
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 872) goto out_release;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 873)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 874) read_len = ret;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 875) sd->total_len = read_len;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 876)
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 877) /*
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 878) * If more data is pending, set SPLICE_F_MORE
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 879) * If this is the last data and SPLICE_F_MORE was not set
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 880) * initially, clears it.
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 881) */
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 882) if (read_len < len)
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 883) sd->flags |= SPLICE_F_MORE;
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 884) else if (!more)
0ff28d9f4674d (Christophe Leroy 2015-05-06 17:26:47 +0200 885) sd->flags &= ~SPLICE_F_MORE;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 886) /*
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 887) * NOTE: nonblocking mode only applies to the input. We
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 888) * must not do the output in nonblocking mode as then we
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 889) * could get stuck data in the internal pipe:
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 890) */
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 891) ret = actor(pipe, sd);
a82c53a0e3f57 (Tom Zanussi 2008-05-09 13:28:36 +0200 892) if (unlikely(ret <= 0)) {
a82c53a0e3f57 (Tom Zanussi 2008-05-09 13:28:36 +0200 893) sd->pos = prev_pos;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 894) goto out_release;
a82c53a0e3f57 (Tom Zanussi 2008-05-09 13:28:36 +0200 895) }
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 896)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 897) bytes += ret;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 898) len -= ret;
bcd4f3acbaec1 (Jens Axboe 2007-07-16 14:41:49 +0200 899) sd->pos = pos;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 900)
a82c53a0e3f57 (Tom Zanussi 2008-05-09 13:28:36 +0200 901) if (ret < read_len) {
a82c53a0e3f57 (Tom Zanussi 2008-05-09 13:28:36 +0200 902) sd->pos = prev_pos + ret;
51a92c0f6ce8f (Jens Axboe 2007-07-13 14:11:43 +0200 903) goto out_release;
a82c53a0e3f57 (Tom Zanussi 2008-05-09 13:28:36 +0200 904) }
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 905) }
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 906)
9e97198dbf318 (Jens Axboe 2008-01-29 21:05:57 +0100 907) done:
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 908) pipe->tail = pipe->head = 0;
8084870854fe1 (Jens Axboe 2008-01-30 12:24:48 +0100 909) file_accessed(in);
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 910) return bytes;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 911)
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 912) out_release:
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 913) /*
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 914) * If we did an incomplete transfer we must release
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 915) * the pipe buffers in question:
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 916) */
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 917) for (i = 0; i < pipe->ring_size; i++) {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 918) struct pipe_buffer *buf = &pipe->bufs[i];
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 919)
a779638cf622f (Miklos Szeredi 2016-09-27 10:45:12 +0200 920) if (buf->ops)
a779638cf622f (Miklos Szeredi 2016-09-27 10:45:12 +0200 921) pipe_buf_release(pipe, buf);
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 922) }
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 923)
9e97198dbf318 (Jens Axboe 2008-01-29 21:05:57 +0100 924) if (!bytes)
9e97198dbf318 (Jens Axboe 2008-01-29 21:05:57 +0100 925) bytes = ret;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 926)
9e97198dbf318 (Jens Axboe 2008-01-29 21:05:57 +0100 927) goto done;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 928) }
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 929) EXPORT_SYMBOL(splice_direct_to_actor);
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 930)
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 931) static int direct_splice_actor(struct pipe_inode_info *pipe,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 932) struct splice_desc *sd)
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 933) {
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 934) struct file *file = sd->u.file;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 935)
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 936) return do_splice_from(pipe, file, sd->opos, sd->total_len,
2cb4b05e76478 (Changli Gao 2010-06-29 13:09:18 +0200 937) sd->flags);
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 938) }
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 939)
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 940) /**
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 941) * do_splice_direct - splices data directly between two files
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 942) * @in: file to splice from
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 943) * @ppos: input file offset
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 944) * @out: file to splice to
acdb37c361dc8 (Randy Dunlap 2013-06-22 19:44:08 -0700 945) * @opos: output file offset
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 946) * @len: number of bytes to splice
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 947) * @flags: splice modifier flags
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 948) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 949) * Description:
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 950) * For use by do_sendfile(). splice can easily emulate sendfile, but
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 951) * doing it in the application would incur an extra system call
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 952) * (splice in + splice out, as compared to just sendfile()). So this helper
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 953) * can splice directly through a process-private pipe.
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 954) *
932cc6d4f7c35 (Jens Axboe 2007-06-21 13:10:21 +0200 955) */
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 956) long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 957) loff_t *opos, size_t len, unsigned int flags)
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 958) {
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 959) struct splice_desc sd = {
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 960) .len = len,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 961) .total_len = len,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 962) .flags = flags,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 963) .pos = *ppos,
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 964) .u.file = out,
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 965) .opos = opos,
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 966) };
51a92c0f6ce8f (Jens Axboe 2007-07-13 14:11:43 +0200 967) long ret;
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 968)
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 969) if (unlikely(!(out->f_mode & FMODE_WRITE)))
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 970) return -EBADF;
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 971)
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 972) if (unlikely(out->f_flags & O_APPEND))
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 973) return -EINVAL;
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 974)
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 975) ret = rw_verify_area(WRITE, out, opos, len);
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 976) if (unlikely(ret < 0))
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 977) return ret;
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 978)
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 979) ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
51a92c0f6ce8f (Jens Axboe 2007-07-13 14:11:43 +0200 980) if (ret > 0)
a82c53a0e3f57 (Tom Zanussi 2008-05-09 13:28:36 +0200 981) *ppos = sd.pos;
51a92c0f6ce8f (Jens Axboe 2007-07-13 14:11:43 +0200 982)
c66ab6fa705e1 (Jens Axboe 2007-06-12 21:17:17 +0200 983) return ret;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 984) }
1c118596a7682 (Miklos Szeredi 2014-10-24 00:14:35 +0200 985) EXPORT_SYMBOL(do_splice_direct);
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 986)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 987) static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 988) {
52bce91165e5f (Linus Torvalds 2016-12-21 10:59:34 -0800 989) for (;;) {
52bce91165e5f (Linus Torvalds 2016-12-21 10:59:34 -0800 990) if (unlikely(!pipe->readers)) {
52bce91165e5f (Linus Torvalds 2016-12-21 10:59:34 -0800 991) send_sig(SIGPIPE, current, 0);
52bce91165e5f (Linus Torvalds 2016-12-21 10:59:34 -0800 992) return -EPIPE;
52bce91165e5f (Linus Torvalds 2016-12-21 10:59:34 -0800 993) }
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 994) if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
52bce91165e5f (Linus Torvalds 2016-12-21 10:59:34 -0800 995) return 0;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 996) if (flags & SPLICE_F_NONBLOCK)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 997) return -EAGAIN;
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 998) if (signal_pending(current))
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 999) return -ERESTARTSYS;
472e5b056f000 (Linus Torvalds 2020-10-01 19:14:36 -0700 1000) pipe_wait_writable(pipe);
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 1001) }
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 1002) }
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 1003)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1004) static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1005) struct pipe_inode_info *opipe,
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1006) size_t len, unsigned int flags);
ddac0d39cf437 (Jens Axboe 2006-11-04 12:49:32 +0100 1007)
b964bf53e5402 (Al Viro 2021-01-25 22:24:28 -0500 1008) long splice_file_to_pipe(struct file *in,
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1009) struct pipe_inode_info *opipe,
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1010) loff_t *offset,
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1011) size_t len, unsigned int flags)
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1012) {
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1013) long ret;
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1014)
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1015) pipe_lock(opipe);
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1016) ret = wait_for_space(opipe, flags);
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1017) if (!ret)
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1018) ret = do_splice_to(in, offset, opipe, len, flags);
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1019) pipe_unlock(opipe);
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1020) if (ret > 0)
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1021) wakeup_pipe_readers(opipe);
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1022) return ret;
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1023) }
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1024)
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 1025) /*
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 1026) * Determine where to splice to/from.
83f9135bddffd (Jens Axboe 2006-04-02 23:05:09 +0200 1027) */
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1028) long do_splice(struct file *in, loff_t *off_in, struct file *out,
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1029) loff_t *off_out, size_t len, unsigned int flags)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1030) {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1031) struct pipe_inode_info *ipipe;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1032) struct pipe_inode_info *opipe;
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1033) loff_t offset;
a4514ebd8e12c (Jens Axboe 2006-04-19 15:57:05 +0200 1034) long ret;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1035)
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1036) if (unlikely(!(in->f_mode & FMODE_READ) ||
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1037) !(out->f_mode & FMODE_WRITE)))
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1038) return -EBADF;
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1039)
c73be61cede58 (David Howells 2020-01-14 17:07:11 +0000 1040) ipipe = get_pipe_info(in, true);
c73be61cede58 (David Howells 2020-01-14 17:07:11 +0000 1041) opipe = get_pipe_info(out, true);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1042)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1043) if (ipipe && opipe) {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1044) if (off_in || off_out)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1045) return -ESPIPE;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1046)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1047) /* Splicing to self would be fun, but... */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1048) if (ipipe == opipe)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1049) return -EINVAL;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1050)
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1051) if ((in->f_flags | out->f_flags) & O_NONBLOCK)
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1052) flags |= SPLICE_F_NONBLOCK;
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1053)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1054) return splice_pipe_to_pipe(ipipe, opipe, len, flags);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1055) }
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1056)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1057) if (ipipe) {
529565dcb1581 (Ingo Molnar 2006-04-10 15:18:58 +0200 1058) if (off_in)
529565dcb1581 (Ingo Molnar 2006-04-10 15:18:58 +0200 1059) return -ESPIPE;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 1060) if (off_out) {
19c9a49b432f2 (Changli Gao 2010-06-29 13:10:36 +0200 1061) if (!(out->f_mode & FMODE_PWRITE))
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 1062) return -EINVAL;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1063) offset = *off_out;
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1064) } else {
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1065) offset = out->f_pos;
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1066) }
529565dcb1581 (Ingo Molnar 2006-04-10 15:18:58 +0200 1067)
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 1068) if (unlikely(out->f_flags & O_APPEND))
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 1069) return -EINVAL;
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 1070)
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 1071) ret = rw_verify_area(WRITE, out, &offset, len);
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 1072) if (unlikely(ret < 0))
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 1073) return ret;
18c67cb9f0d2a (Al Viro 2013-06-19 15:41:54 +0400 1074)
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1075) if (in->f_flags & O_NONBLOCK)
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1076) flags |= SPLICE_F_NONBLOCK;
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1077)
500368f7fbdd8 (Al Viro 2013-05-23 20:07:11 -0400 1078) file_start_write(out);
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1079) ret = do_splice_from(ipipe, out, &offset, len, flags);
500368f7fbdd8 (Al Viro 2013-05-23 20:07:11 -0400 1080) file_end_write(out);
a4514ebd8e12c (Jens Axboe 2006-04-19 15:57:05 +0200 1081)
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1082) if (!off_out)
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1083) out->f_pos = offset;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1084) else
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1085) *off_out = offset;
a4514ebd8e12c (Jens Axboe 2006-04-19 15:57:05 +0200 1086)
a4514ebd8e12c (Jens Axboe 2006-04-19 15:57:05 +0200 1087) return ret;
529565dcb1581 (Ingo Molnar 2006-04-10 15:18:58 +0200 1088) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1089)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1090) if (opipe) {
529565dcb1581 (Ingo Molnar 2006-04-10 15:18:58 +0200 1091) if (off_out)
529565dcb1581 (Ingo Molnar 2006-04-10 15:18:58 +0200 1092) return -ESPIPE;
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 1093) if (off_in) {
19c9a49b432f2 (Changli Gao 2010-06-29 13:10:36 +0200 1094) if (!(in->f_mode & FMODE_PREAD))
b92ce55893745 (Jens Axboe 2006-04-11 13:52:07 +0200 1095) return -EINVAL;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1096) offset = *off_in;
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1097) } else {
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1098) offset = in->f_pos;
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1099) }
529565dcb1581 (Ingo Molnar 2006-04-10 15:18:58 +0200 1100)
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1101) if (out->f_flags & O_NONBLOCK)
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1102) flags |= SPLICE_F_NONBLOCK;
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1103)
faa97c48c3345 (Al Viro 2021-01-25 22:23:03 -0500 1104) ret = splice_file_to_pipe(in, opipe, &offset, len, flags);
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1105) if (!off_in)
7995bd287134f (Al Viro 2013-06-20 18:58:36 +0400 1106) in->f_pos = offset;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1107) else
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1108) *off_in = offset;
a4514ebd8e12c (Jens Axboe 2006-04-19 15:57:05 +0200 1109)
a4514ebd8e12c (Jens Axboe 2006-04-19 15:57:05 +0200 1110) return ret;
529565dcb1581 (Ingo Molnar 2006-04-10 15:18:58 +0200 1111) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1112)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1113) return -EINVAL;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1114) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1115)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1116) static long __do_splice(struct file *in, loff_t __user *off_in,
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1117) struct file *out, loff_t __user *off_out,
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1118) size_t len, unsigned int flags)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1119) {
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1120) struct pipe_inode_info *ipipe;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1121) struct pipe_inode_info *opipe;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1122) loff_t offset, *__off_in = NULL, *__off_out = NULL;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1123) long ret;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1124)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1125) ipipe = get_pipe_info(in, true);
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1126) opipe = get_pipe_info(out, true);
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1127)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1128) if (ipipe && off_in)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1129) return -ESPIPE;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1130) if (opipe && off_out)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1131) return -ESPIPE;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1132)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1133) if (off_out) {
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1134) if (copy_from_user(&offset, off_out, sizeof(loff_t)))
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1135) return -EFAULT;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1136) __off_out = &offset;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1137) }
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1138) if (off_in) {
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1139) if (copy_from_user(&offset, off_in, sizeof(loff_t)))
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1140) return -EFAULT;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1141) __off_in = &offset;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1142) }
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1143)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1144) ret = do_splice(in, __off_in, out, __off_out, len, flags);
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1145) if (ret < 0)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1146) return ret;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1147)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1148) if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1149) return -EFAULT;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1150) if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1151) return -EFAULT;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1152)
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1153) return ret;
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1154) }
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1155)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1156) static int iter_to_pipe(struct iov_iter *from,
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1157) struct pipe_inode_info *pipe,
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1158) unsigned flags)
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1159) {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1160) struct pipe_buffer buf = {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1161) .ops = &user_page_pipe_buf_ops,
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1162) .flags = flags
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1163) };
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1164) size_t total = 0;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1165) int ret = 0;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1166) bool failed = false;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1167)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1168) while (iov_iter_count(from) && !failed) {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1169) struct page *pages[16];
db85a9eb2e364 (Al Viro 2016-09-17 20:25:06 -0400 1170) ssize_t copied;
db85a9eb2e364 (Al Viro 2016-09-17 20:25:06 -0400 1171) size_t start;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1172) int n;
db85a9eb2e364 (Al Viro 2016-09-17 20:25:06 -0400 1173)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1174) copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1175) if (copied <= 0) {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1176) ret = copied;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1177) break;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1178) }
db85a9eb2e364 (Al Viro 2016-09-17 20:25:06 -0400 1179)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1180) for (n = 0; copied; n++, start = 0) {
db85a9eb2e364 (Al Viro 2016-09-17 20:25:06 -0400 1181) int size = min_t(int, copied, PAGE_SIZE - start);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1182) if (!failed) {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1183) buf.page = pages[n];
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1184) buf.offset = start;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1185) buf.len = size;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1186) ret = add_to_pipe(pipe, &buf);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1187) if (unlikely(ret < 0)) {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1188) failed = true;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1189) } else {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1190) iov_iter_advance(from, ret);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1191) total += ret;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1192) }
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1193) } else {
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1194) put_page(pages[n]);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1195) }
db85a9eb2e364 (Al Viro 2016-09-17 20:25:06 -0400 1196) copied -= size;
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1197) }
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1198) }
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1199) return total ? total : ret;
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1200) }
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1201)
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1202) static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1203) struct splice_desc *sd)
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1204) {
6130f5315ee80 (Al Viro 2014-02-03 18:19:51 -0500 1205) int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
6130f5315ee80 (Al Viro 2014-02-03 18:19:51 -0500 1206) return n == sd->len ? n : -EFAULT;
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1207) }
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1208)
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1209) /*
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1210) * For lack of a better implementation, implement vmsplice() to userspace
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1211) * as a simple copy of the pipes pages to the user iov.
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1212) */
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1213) static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1214) unsigned int flags)
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1215) {
c73be61cede58 (David Howells 2020-01-14 17:07:11 +0000 1216) struct pipe_inode_info *pipe = get_pipe_info(file, true);
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1217) struct splice_desc sd = {
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1218) .total_len = iov_iter_count(iter),
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1219) .flags = flags,
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1220) .u.data = iter
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1221) };
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1222) long ret = 0;
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1223)
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1224) if (!pipe)
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1225) return -EBADF;
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1226)
345995fa48c59 (Al Viro 2015-03-21 19:17:55 -0400 1227) if (sd.total_len) {
345995fa48c59 (Al Viro 2015-03-21 19:17:55 -0400 1228) pipe_lock(pipe);
345995fa48c59 (Al Viro 2015-03-21 19:17:55 -0400 1229) ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
345995fa48c59 (Al Viro 2015-03-21 19:17:55 -0400 1230) pipe_unlock(pipe);
345995fa48c59 (Al Viro 2015-03-21 19:17:55 -0400 1231) }
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1232)
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1233) return ret;
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1234) }
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1235)
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1236) /*
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1237) * vmsplice splices a user address range into a pipe. It can be thought of
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1238) * as splice-from-memory, where the regular splice is splice-from-file (or
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1239) * to file). In both cases the output is a pipe, naturally.
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1240) */
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1241) static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1242) unsigned int flags)
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1243) {
ddac0d39cf437 (Jens Axboe 2006-11-04 12:49:32 +0100 1244) struct pipe_inode_info *pipe;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1245) long ret = 0;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1246) unsigned buf_flag = 0;
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1247)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1248) if (flags & SPLICE_F_GIFT)
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1249) buf_flag = PIPE_BUF_FLAG_GIFT;
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1250)
c73be61cede58 (David Howells 2020-01-14 17:07:11 +0000 1251) pipe = get_pipe_info(file, true);
ddac0d39cf437 (Jens Axboe 2006-11-04 12:49:32 +0100 1252) if (!pipe)
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1253) return -EBADF;
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1254)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 1255) pipe_lock(pipe);
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 1256) ret = wait_for_space(pipe, flags);
79fddc4efd5d4 (Al Viro 2016-09-17 22:38:20 -0400 1257) if (!ret)
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1258) ret = iter_to_pipe(iter, pipe, buf_flag);
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 1259) pipe_unlock(pipe);
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 1260) if (ret > 0)
8924feff66f35 (Al Viro 2016-09-17 20:44:45 -0400 1261) wakeup_pipe_readers(pipe);
35f3d14dbbc58 (Jens Axboe 2010-05-20 10:43:18 +0200 1262) return ret;
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1263) }
912d35f86781e (Jens Axboe 2006-04-26 10:59:21 +0200 1264)
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1265) static int vmsplice_type(struct fd f, int *type)
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1266) {
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1267) if (!f.file)
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1268) return -EBADF;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1269) if (f.file->f_mode & FMODE_WRITE) {
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1270) *type = WRITE;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1271) } else if (f.file->f_mode & FMODE_READ) {
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1272) *type = READ;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1273) } else {
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1274) fdput(f);
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1275) return -EBADF;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1276) }
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1277) return 0;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1278) }
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1279)
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1280) /*
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1281) * Note that vmsplice only really supports true splicing _from_ user memory
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1282) * to a pipe, not the other way around. Splicing from user memory is a simple
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1283) * operation that can be supported without any funky alignment restrictions
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1284) * or nasty vm tricks. We simply map in the user memory and fill them into
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1285) * a pipe. The reverse isn't quite as easy, though. There are two possible
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1286) * solutions for that:
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1287) *
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1288) * - memcpy() the data internally, at which point we might as well just
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1289) * do a regular read() on the buffer anyway.
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1290) * - Lots of nasty vm tricks, that are neither fast nor flexible (it
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1291) * has restriction limitations on both ends of the pipe).
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1292) *
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1293) * Currently we punt and implement it as a normal copy, see pipe_to_user().
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1294) *
6a14b90bb6bc7 (Jens Axboe 2007-06-14 13:08:55 +0200 1295) */
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1296) SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
30cfe4ef8b894 (Dominik Brodowski 2018-03-17 15:00:24 +0100 1297) unsigned long, nr_segs, unsigned int, flags)
30cfe4ef8b894 (Dominik Brodowski 2018-03-17 15:00:24 +0100 1298) {
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1299) struct iovec iovstack[UIO_FASTIOV];
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1300) struct iovec *iov = iovstack;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1301) struct iov_iter iter;
87e5e6dab6c2a (Jens Axboe 2019-05-14 16:02:22 -0600 1302) ssize_t error;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1303) struct fd f;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1304) int type;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1305)
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1306) if (unlikely(flags & ~SPLICE_F_ALL))
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1307) return -EINVAL;
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1308)
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1309) f = fdget(fd);
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1310) error = vmsplice_type(f, &type);
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1311) if (error)
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1312) return error;
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1313)
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1314) error = import_iovec(type, uiov, nr_segs,
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1315) ARRAY_SIZE(iovstack), &iov, &iter);
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1316) if (error < 0)
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1317) goto out_fdput;
30cfe4ef8b894 (Dominik Brodowski 2018-03-17 15:00:24 +0100 1318)
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1319) if (!iov_iter_count(&iter))
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1320) error = 0;
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1321) else if (iov_iter_rw(&iter) == WRITE)
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1322) error = vmsplice_to_pipe(f.file, &iter, flags);
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1323) else
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1324) error = vmsplice_to_user(f.file, &iter, flags);
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1325)
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1326) kfree(iov);
598b3cec831fd (Christoph Hellwig 2020-09-25 06:51:44 +0200 1327) out_fdput:
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1328) fdput(f);
87a3002af9e30 (Al Viro 2018-05-26 21:39:52 -0400 1329) return error;
76b021d053ed0 (Al Viro 2013-03-02 10:19:56 -0500 1330) }
76b021d053ed0 (Al Viro 2013-03-02 10:19:56 -0500 1331)
836f92adf121f (Heiko Carstens 2009-01-14 14:14:33 +0100 1332) SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
836f92adf121f (Heiko Carstens 2009-01-14 14:14:33 +0100 1333) int, fd_out, loff_t __user *, off_out,
836f92adf121f (Heiko Carstens 2009-01-14 14:14:33 +0100 1334) size_t, len, unsigned int, flags)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1335) {
2903ff019b346 (Al Viro 2012-08-28 12:52:22 -0400 1336) struct fd in, out;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1337) long error;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1338)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1339) if (unlikely(!len))
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1340) return 0;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1341)
3d6ea290f337c (Al Viro 2016-12-10 13:17:32 -0500 1342) if (unlikely(flags & ~SPLICE_F_ALL))
3d6ea290f337c (Al Viro 2016-12-10 13:17:32 -0500 1343) return -EINVAL;
3d6ea290f337c (Al Viro 2016-12-10 13:17:32 -0500 1344)
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1345) error = -EBADF;
2903ff019b346 (Al Viro 2012-08-28 12:52:22 -0400 1346) in = fdget(fd_in);
2903ff019b346 (Al Viro 2012-08-28 12:52:22 -0400 1347) if (in.file) {
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1348) out = fdget(fd_out);
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1349) if (out.file) {
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1350) error = __do_splice(in.file, off_in, out.file, off_out,
ee6e00c868221 (Jens Axboe 2020-10-22 14:15:51 -0600 1351) len, flags);
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1352) fdput(out);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1353) }
2903ff019b346 (Al Viro 2012-08-28 12:52:22 -0400 1354) fdput(in);
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1355) }
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1356) return error;
5274f052e7b3d (Jens Axboe 2006-03-30 15:15:30 +0200 1357) }
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1358)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1359) /*
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1360) * Make sure there's data to read. Wait for input if we can, otherwise
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1361) * return an appropriate error.
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1362) */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1363) static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1364) {
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1365) int ret;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1366)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1367) /*
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1368) * Check the pipe occupancy without the inode lock first. This function
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1369) * is speculative anyways, so missing one is ok.
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1370) */
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1371) if (!pipe_empty(pipe->head, pipe->tail))
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1372) return 0;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1373)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1374) ret = 0;
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 1375) pipe_lock(pipe);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1376)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1377) while (pipe_empty(pipe->head, pipe->tail)) {
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1378) if (signal_pending(current)) {
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1379) ret = -ERESTARTSYS;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1380) break;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1381) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1382) if (!pipe->writers)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1383) break;
a28c8b9db8a10 (Linus Torvalds 2019-12-07 13:21:01 -0800 1384) if (flags & SPLICE_F_NONBLOCK) {
a28c8b9db8a10 (Linus Torvalds 2019-12-07 13:21:01 -0800 1385) ret = -EAGAIN;
a28c8b9db8a10 (Linus Torvalds 2019-12-07 13:21:01 -0800 1386) break;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1387) }
472e5b056f000 (Linus Torvalds 2020-10-01 19:14:36 -0700 1388) pipe_wait_readable(pipe);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1389) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1390)
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 1391) pipe_unlock(pipe);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1392) return ret;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1393) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1394)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1395) /*
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1396) * Make sure there's writeable room. Wait for room if we can, otherwise
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1397) * return an appropriate error.
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1398) */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1399) static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1400) {
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1401) int ret;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1402)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1403) /*
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1404) * Check pipe occupancy without the inode lock first. This function
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1405) * is speculative anyways, so missing one is ok.
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1406) */
566d136289dc5 (Tetsuo Handa 2020-05-20 08:51:59 +0900 1407) if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1408) return 0;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1409)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1410) ret = 0;
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 1411) pipe_lock(pipe);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1412)
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 1413) while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1414) if (!pipe->readers) {
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1415) send_sig(SIGPIPE, current, 0);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1416) ret = -EPIPE;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1417) break;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1418) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1419) if (flags & SPLICE_F_NONBLOCK) {
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1420) ret = -EAGAIN;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1421) break;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1422) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1423) if (signal_pending(current)) {
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1424) ret = -ERESTARTSYS;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1425) break;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1426) }
472e5b056f000 (Linus Torvalds 2020-10-01 19:14:36 -0700 1427) pipe_wait_writable(pipe);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1428) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1429)
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 1430) pipe_unlock(pipe);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1431) return ret;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1432) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1433)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1434) /*
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1435) * Splice contents of ipipe to opipe.
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1436) */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1437) static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1438) struct pipe_inode_info *opipe,
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1439) size_t len, unsigned int flags)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1440) {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1441) struct pipe_buffer *ibuf, *obuf;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1442) unsigned int i_head, o_head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1443) unsigned int i_tail, o_tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1444) unsigned int i_mask, o_mask;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1445) int ret = 0;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1446) bool input_wakeup = false;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1447)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1448)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1449) retry:
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1450) ret = ipipe_prep(ipipe, flags);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1451) if (ret)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1452) return ret;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1453)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1454) ret = opipe_prep(opipe, flags);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1455) if (ret)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1456) return ret;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1457)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1458) /*
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1459) * Potential ABBA deadlock, work around it by ordering lock
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1460) * grabbing by pipe info address. Otherwise two different processes
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1461) * could deadlock (one doing tee from A -> B, the other from B -> A).
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1462) */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1463) pipe_double_lock(ipipe, opipe);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1464)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1465) i_tail = ipipe->tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1466) i_mask = ipipe->ring_size - 1;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1467) o_head = opipe->head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1468) o_mask = opipe->ring_size - 1;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1469)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1470) do {
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1471) size_t o_len;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1472)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1473) if (!opipe->readers) {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1474) send_sig(SIGPIPE, current, 0);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1475) if (!ret)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1476) ret = -EPIPE;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1477) break;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1478) }
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1479)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1480) i_head = ipipe->head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1481) o_tail = opipe->tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1482)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1483) if (pipe_empty(i_head, i_tail) && !ipipe->writers)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1484) break;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1485)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1486) /*
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1487) * Cannot make any progress, because either the input
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1488) * pipe is empty or the output pipe is full.
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1489) */
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1490) if (pipe_empty(i_head, i_tail) ||
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 1491) pipe_full(o_head, o_tail, opipe->max_usage)) {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1492) /* Already processed some buffers, break */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1493) if (ret)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1494) break;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1495)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1496) if (flags & SPLICE_F_NONBLOCK) {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1497) ret = -EAGAIN;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1498) break;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1499) }
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1500)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1501) /*
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1502) * We raced with another reader/writer and haven't
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1503) * managed to process any buffers. A zero return
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1504) * value means EOF, so retry instead.
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1505) */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1506) pipe_unlock(ipipe);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1507) pipe_unlock(opipe);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1508) goto retry;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1509) }
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1510)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1511) ibuf = &ipipe->bufs[i_tail & i_mask];
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1512) obuf = &opipe->bufs[o_head & o_mask];
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1513)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1514) if (len >= ibuf->len) {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1515) /*
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1516) * Simply move the whole buffer from ipipe to opipe
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1517) */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1518) *obuf = *ibuf;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1519) ibuf->ops = NULL;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1520) i_tail++;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1521) ipipe->tail = i_tail;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1522) input_wakeup = true;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1523) o_len = obuf->len;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1524) o_head++;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1525) opipe->head = o_head;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1526) } else {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1527) /*
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1528) * Get a reference to this pipe buffer,
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1529) * so we can copy the contents over.
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1530) */
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1531) if (!pipe_buf_get(ipipe, ibuf)) {
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1532) if (ret == 0)
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1533) ret = -EFAULT;
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1534) break;
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1535) }
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1536) *obuf = *ibuf;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1537)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1538) /*
f6dd975583bd8 (Christoph Hellwig 2020-05-20 17:58:12 +0200 1539) * Don't inherit the gift and merge flags, we need to
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1540) * prevent multiple steals of this page.
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1541) */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1542) obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
f6dd975583bd8 (Christoph Hellwig 2020-05-20 17:58:12 +0200 1543) obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
a0ce2f0aa6ad9 (Jann Horn 2019-01-23 15:19:17 +0100 1544)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1545) obuf->len = len;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1546) ibuf->offset += len;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1547) ibuf->len -= len;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1548) o_len = len;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1549) o_head++;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1550) opipe->head = o_head;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1551) }
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1552) ret += o_len;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1553) len -= o_len;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1554) } while (len);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1555)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1556) pipe_unlock(ipipe);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1557) pipe_unlock(opipe);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1558)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1559) /*
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1560) * If we put data in the output pipe, wakeup any potential readers.
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1561) */
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 1562) if (ret > 0)
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 1563) wakeup_pipe_readers(opipe);
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 1564)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1565) if (input_wakeup)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1566) wakeup_pipe_writers(ipipe);
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1567)
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1568) return ret;
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1569) }
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1570)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1571) /*
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1572) * Link contents of ipipe to opipe.
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1573) */
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1574) static int link_pipe(struct pipe_inode_info *ipipe,
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1575) struct pipe_inode_info *opipe,
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1576) size_t len, unsigned int flags)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1577) {
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1578) struct pipe_buffer *ibuf, *obuf;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1579) unsigned int i_head, o_head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1580) unsigned int i_tail, o_tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1581) unsigned int i_mask, o_mask;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1582) int ret = 0;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1583)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1584) /*
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1585) * Potential ABBA deadlock, work around it by ordering lock
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 1586) * grabbing by pipe info address. Otherwise two different processes
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1587) * could deadlock (one doing tee from A -> B, the other from B -> A).
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1588) */
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 1589) pipe_double_lock(ipipe, opipe);
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1590)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1591) i_tail = ipipe->tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1592) i_mask = ipipe->ring_size - 1;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1593) o_head = opipe->head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1594) o_mask = opipe->ring_size - 1;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1595)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1596) do {
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1597) if (!opipe->readers) {
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1598) send_sig(SIGPIPE, current, 0);
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1599) if (!ret)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1600) ret = -EPIPE;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1601) break;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1602) }
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1603)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1604) i_head = ipipe->head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1605) o_tail = opipe->tail;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1606)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1607) /*
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1608) * If we have iterated all input buffers or run out of
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1609) * output room, break.
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1610) */
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1611) if (pipe_empty(i_head, i_tail) ||
6718b6f855a0b (David Howells 2019-10-16 16:47:32 +0100 1612) pipe_full(o_head, o_tail, opipe->max_usage))
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1613) break;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1614)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1615) ibuf = &ipipe->bufs[i_tail & i_mask];
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1616) obuf = &opipe->bufs[o_head & o_mask];
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1617)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1618) /*
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1619) * Get a reference to this pipe buffer,
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1620) * so we can copy the contents over.
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1621) */
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1622) if (!pipe_buf_get(ipipe, ibuf)) {
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1623) if (ret == 0)
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1624) ret = -EFAULT;
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1625) break;
15fab63e1e57b (Matthew Wilcox 2019-04-05 14:02:10 -0700 1626) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1627)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1628) *obuf = *ibuf;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1629)
2a27250e6cf47 (Jens Axboe 2006-04-19 15:56:40 +0200 1630) /*
f6dd975583bd8 (Christoph Hellwig 2020-05-20 17:58:12 +0200 1631) * Don't inherit the gift and merge flag, we need to prevent
f6dd975583bd8 (Christoph Hellwig 2020-05-20 17:58:12 +0200 1632) * multiple steals of this page.
2a27250e6cf47 (Jens Axboe 2006-04-19 15:56:40 +0200 1633) */
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1634) obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
f6dd975583bd8 (Christoph Hellwig 2020-05-20 17:58:12 +0200 1635) obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
a0ce2f0aa6ad9 (Jann Horn 2019-01-23 15:19:17 +0100 1636)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1637) if (obuf->len > len)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1638) obuf->len = len;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1639) ret += obuf->len;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1640) len -= obuf->len;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1641)
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1642) o_head++;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1643) opipe->head = o_head;
8cefc107ca54c (David Howells 2019-11-15 13:30:32 +0000 1644) i_tail++;
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1645) } while (len);
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1646)
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 1647) pipe_unlock(ipipe);
61e0d47c33cc3 (Miklos Szeredi 2009-04-14 19:48:41 +0200 1648) pipe_unlock(opipe);
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1649)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1650) /*
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1651) * If we put data in the output pipe, wakeup any potential readers.
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1652) */
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 1653) if (ret > 0)
825cdcb1a5aaa (Namhyung Kim 2011-05-23 19:58:53 +0200 1654) wakeup_pipe_readers(opipe);
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1655)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1656) return ret;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1657) }
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1658)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1659) /*
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1660) * This is a tee(1) implementation that works on pipes. It doesn't copy
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1661) * any data, it simply references the 'in' pages on the 'out' pipe.
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1662) * The 'flags' used are the SPLICE_F_* variants, currently the only
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1663) * applicable one is SPLICE_F_NONBLOCK.
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1664) */
9dafdfc2f0a3a (Pavel Begunkov 2020-05-17 14:18:05 +0300 1665) long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1666) {
c73be61cede58 (David Howells 2020-01-14 17:07:11 +0000 1667) struct pipe_inode_info *ipipe = get_pipe_info(in, true);
c73be61cede58 (David Howells 2020-01-14 17:07:11 +0000 1668) struct pipe_inode_info *opipe = get_pipe_info(out, true);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1669) int ret = -EINVAL;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1670)
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1671) if (unlikely(!(in->f_mode & FMODE_READ) ||
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1672) !(out->f_mode & FMODE_WRITE)))
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1673) return -EBADF;
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1674)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1675) /*
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1676) * Duplicate the contents of ipipe to opipe without actually
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1677) * copying the data.
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1678) */
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1679) if (ipipe && opipe && ipipe != opipe) {
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1680) if ((in->f_flags | out->f_flags) & O_NONBLOCK)
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1681) flags |= SPLICE_F_NONBLOCK;
ee5e001196d13 (Slavomir Kaslev 2019-02-07 17:45:19 +0200 1682)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1683) /*
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1684) * Keep going, unless we encounter an error. The ipipe/opipe
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1685) * ordering doesn't really matter.
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1686) */
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1687) ret = ipipe_prep(ipipe, flags);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1688) if (!ret) {
7c77f0b3f9208 (Miklos Szeredi 2009-05-07 15:37:35 +0200 1689) ret = opipe_prep(opipe, flags);
02cf01aea5af7 (Jens Axboe 2008-02-20 10:34:51 +0100 1690) if (!ret)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1691) ret = link_pipe(ipipe, opipe, len, flags);
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1692) }
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1693) }
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1694)
aadd06e5c56b9 (Jens Axboe 2006-07-10 11:00:01 +0200 1695) return ret;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1696) }
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1697)
836f92adf121f (Heiko Carstens 2009-01-14 14:14:33 +0100 1698) SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1699) {
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1700) struct fd in, out;
2903ff019b346 (Al Viro 2012-08-28 12:52:22 -0400 1701) int error;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1702)
3d6ea290f337c (Al Viro 2016-12-10 13:17:32 -0500 1703) if (unlikely(flags & ~SPLICE_F_ALL))
3d6ea290f337c (Al Viro 2016-12-10 13:17:32 -0500 1704) return -EINVAL;
3d6ea290f337c (Al Viro 2016-12-10 13:17:32 -0500 1705)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1706) if (unlikely(!len))
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1707) return 0;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1708)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1709) error = -EBADF;
2903ff019b346 (Al Viro 2012-08-28 12:52:22 -0400 1710) in = fdget(fdin);
2903ff019b346 (Al Viro 2012-08-28 12:52:22 -0400 1711) if (in.file) {
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1712) out = fdget(fdout);
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1713) if (out.file) {
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1714) error = do_tee(in.file, out.file, len, flags);
90da2e3f25c8b (Pavel Begunkov 2020-05-04 22:39:35 +0300 1715) fdput(out);
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1716) }
2903ff019b346 (Al Viro 2012-08-28 12:52:22 -0400 1717) fdput(in);
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1718) }
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1719)
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1720) return error;
70524490ee2ea (Jens Axboe 2006-04-11 15:51:17 +0200 1721) }
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags