2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1) // SPDX-License-Identifier: GPL-2.0
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3) * Shared application/kernel submission and completion ring pairs, for
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 4) * supporting fast/efficient IO.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 5) *
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6) * A note on the read/write ordering memory barriers that are matched between
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 7) * the application and kernel side.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 8) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 9) * After the application reads the CQ ring tail, it must use an
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 10) * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 11) * before writing the tail (using smp_load_acquire to read the tail will
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 12) * do). It also needs a smp_mb() before updating CQ head (ordering the
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 13) * entry load(s) with the head store), pairing with an implicit barrier
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 14) * through a control-dependency in io_get_cqring (smp_store_release to
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 15) * store head will do). Failure to do so could lead to reading invalid
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 16) * CQ entries.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 17) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 18) * Likewise, the application must use an appropriate smp_wmb() before
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 19) * writing the SQ tail (ordering SQ entry stores with the tail store),
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 20) * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 21) * to store the tail will do). And it needs a barrier ordering the SQ
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 22) * head load before writing new SQ entries (smp_load_acquire to read
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 23) * head will do).
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 24) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 25) * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 26) * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 27) * updating the SQ tail; a full memory barrier smp_mb() is needed
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 28) * between.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 29) *
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 30) * Also see the examples in the liburing library:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 31) *
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 32) * git://git.kernel.dk/liburing
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 33) *
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 34) * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 35) * from data shared between the kernel and application. This is done both
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 36) * for ordering purposes, but also to ensure that once a value is loaded from
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 37) * data that the application could potentially modify, it remains stable.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 38) *
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 39) * Copyright (C) 2018-2019 Jens Axboe
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 40) * Copyright (c) 2018-2019 Christoph Hellwig
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 41) */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 42) #include <linux/kernel.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 43) #include <linux/init.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 44) #include <linux/errno.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 45) #include <linux/syscalls.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 46) #include <linux/compat.h>
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 47) #include <net/compat.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 48) #include <linux/refcount.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 49) #include <linux/uio.h>
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 50) #include <linux/bits.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 51)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 52) #include <linux/sched/signal.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 53) #include <linux/fs.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 54) #include <linux/file.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 55) #include <linux/fdtable.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 56) #include <linux/mm.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 57) #include <linux/mman.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 58) #include <linux/percpu.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 59) #include <linux/slab.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 60) #include <linux/blkdev.h>
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 61) #include <linux/bvec.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 62) #include <linux/net.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 63) #include <net/sock.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 64) #include <net/af_unix.h>
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 65) #include <net/scm.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 66) #include <linux/anon_inodes.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 67) #include <linux/sched/mm.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 68) #include <linux/uaccess.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 69) #include <linux/nospec.h>
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 70) #include <linux/sizes.h>
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 71) #include <linux/hugetlb.h>
aa4c3967756c6 (Jens Axboe 2019-11-29 10:14:00 -0700 72) #include <linux/highmem.h>
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 73) #include <linux/namei.h>
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 74) #include <linux/fsnotify.h>
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 75) #include <linux/fadvise.h>
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 76) #include <linux/eventpoll.h>
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 77) #include <linux/splice.h>
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 78) #include <linux/task_work.h>
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 79) #include <linux/pagemap.h>
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 80) #include <linux/io_uring.h>
6a4b928211359 (Nadav Amit 2021-08-07 17:13:41 -0700 81) #include <linux/tracehook.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 82)
c826bd7a743f2 (Dmitrii Dolgov 2019-10-15 19:02:01 +0200 83) #define CREATE_TRACE_POINTS
c826bd7a743f2 (Dmitrii Dolgov 2019-10-15 19:02:01 +0200 84) #include <trace/events/io_uring.h>
c826bd7a743f2 (Dmitrii Dolgov 2019-10-15 19:02:01 +0200 85)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 86) #include <uapi/linux/io_uring.h>
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 87)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 88) #include "internal.h"
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 89) #include "io-wq.h"
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 90)
5277deaab9f98 (Daniel Xu 2019-09-14 14:23:45 -0700 91) #define IORING_MAX_ENTRIES 32768
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 92) #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 93)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 94) /*
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 95) * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 96) */
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 97) #define IORING_FILE_TABLE_SHIFT 9
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 98) #define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 99) #define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 100) #define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 101) #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 102) IORING_REGISTER_LAST + IORING_OP_LAST)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 103)
489809e2e22b3 (Pavel Begunkov 2021-05-14 12:06:44 +0100 104) #define IORING_MAX_REG_BUFFERS (1U << 14)
489809e2e22b3 (Pavel Begunkov 2021-05-14 12:06:44 +0100 105)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 106) #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 107) IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 108) IOSQE_BUFFER_SELECT)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 109)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 110) struct io_uring {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 111) u32 head ____cacheline_aligned_in_smp;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 112) u32 tail ____cacheline_aligned_in_smp;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 113) };
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 114)
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 115) /*
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 116) * This data is shared with the application through the mmap at offsets
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 117) * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 118) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 119) * The offsets to the member fields are published through struct
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 120) * io_sqring_offsets when calling io_uring_setup.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 121) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 122) struct io_rings {
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 123) /*
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 124) * Head and tail offsets into the ring; the offsets need to be
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 125) * masked to get valid indices.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 126) *
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 127) * The kernel controls head of the sq ring and the tail of the cq ring,
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 128) * and the application controls tail of the sq ring and the head of the
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 129) * cq ring.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 130) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 131) struct io_uring sq, cq;
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 132) /*
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 133) * Bitmasks to apply to head and tail offsets (constant, equals
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 134) * ring_entries - 1)
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 135) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 136) u32 sq_ring_mask, cq_ring_mask;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 137) /* Ring sizes (constant, power of 2) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 138) u32 sq_ring_entries, cq_ring_entries;
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 139) /*
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 140) * Number of invalid entries dropped by the kernel due to
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 141) * invalid index stored in array
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 142) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 143) * Written by the kernel, shouldn't be modified by the
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 144) * application (i.e. get number of "new events" by comparing to
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 145) * cached value).
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 146) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 147) * After a new SQ head value was read by the application this
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 148) * counter includes all submissions that were dropped reaching
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 149) * the new SQ head (and possibly more).
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 150) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 151) u32 sq_dropped;
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 152) /*
0d9b5b3af134c (Stefano Garzarella 2020-05-15 18:38:04 +0200 153) * Runtime SQ flags
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 154) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 155) * Written by the kernel, shouldn't be modified by the
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 156) * application.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 157) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 158) * The application needs a full memory barrier before checking
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 159) * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 160) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 161) u32 sq_flags;
0d9b5b3af134c (Stefano Garzarella 2020-05-15 18:38:04 +0200 162) /*
0d9b5b3af134c (Stefano Garzarella 2020-05-15 18:38:04 +0200 163) * Runtime CQ flags
0d9b5b3af134c (Stefano Garzarella 2020-05-15 18:38:04 +0200 164) *
0d9b5b3af134c (Stefano Garzarella 2020-05-15 18:38:04 +0200 165) * Written by the application, shouldn't be modified by the
0d9b5b3af134c (Stefano Garzarella 2020-05-15 18:38:04 +0200 166) * kernel.
0d9b5b3af134c (Stefano Garzarella 2020-05-15 18:38:04 +0200 167) */
501449420a42c (Pavel Begunkov 2021-06-24 15:09:57 +0100 168) u32 cq_flags;
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 169) /*
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 170) * Number of completion events lost because the queue was full;
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 171) * this should be avoided by the application by making sure
0b4295b5e2b9b (LimingWu 2019-12-05 20:18:18 +0800 172) * there are not more requests pending than there is space in
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 173) * the completion queue.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 174) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 175) * Written by the kernel, shouldn't be modified by the
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 176) * application (i.e. get number of "new events" by comparing to
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 177) * cached value).
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 178) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 179) * As completion events come in out of order this counter is not
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 180) * ordered with any other data.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 181) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 182) u32 cq_overflow;
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 183) /*
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 184) * Ring buffer of completion events.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 185) *
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 186) * The kernel writes completion events fresh every time they are
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 187) * produced, so the application is allowed to modify pending
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 188) * entries.
1e84b97b7377b (Stefan Bühler 2019-04-24 23:54:16 +0200 189) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 190) struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 191) };
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 192)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 193) enum io_uring_cmd_flags {
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 194) IO_URING_F_NONBLOCK = 1,
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 195) IO_URING_F_COMPLETE_DEFER = 2,
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 196) };
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 197)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 198) struct io_mapped_ubuf {
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 199) u64 ubuf;
4751f53d74a68 (Pavel Begunkov 2021-04-01 15:43:55 +0100 200) u64 ubuf_end;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 201) unsigned int nr_bvecs;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 202) unsigned long acct_pages;
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 203) struct bio_vec bvec[];
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 204) };
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 205)
5023853183699 (Bijan Mottahedeh 2021-01-15 17:37:45 +0000 206) struct io_ring_ctx;
5023853183699 (Bijan Mottahedeh 2021-01-15 17:37:45 +0000 207)
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 208) struct io_overflow_cqe {
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 209) struct io_uring_cqe cqe;
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 210) struct list_head list;
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 211) };
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 212)
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 213) struct io_fixed_file {
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 214) /* file * with additional FFS_* flags */
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 215) unsigned long file_ptr;
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 216) };
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 217)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 218) struct io_rsrc_put {
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 219) struct list_head list;
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 220) u64 tag;
5023853183699 (Bijan Mottahedeh 2021-01-15 17:37:45 +0000 221) union {
5023853183699 (Bijan Mottahedeh 2021-01-15 17:37:45 +0000 222) void *rsrc;
5023853183699 (Bijan Mottahedeh 2021-01-15 17:37:45 +0000 223) struct file *file;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 224) struct io_mapped_ubuf *buf;
5023853183699 (Bijan Mottahedeh 2021-01-15 17:37:45 +0000 225) };
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 226) };
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 227)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 228) struct io_file_table {
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 229) /* two level table */
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 230) struct io_fixed_file **files;
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 231) };
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 232)
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 233) struct io_rsrc_node {
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 234) struct percpu_ref refs;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 235) struct list_head node;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 236) struct list_head rsrc_list;
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 237) struct io_rsrc_data *rsrc_data;
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 238) struct llist_node llist;
e297822b20e7f (Pavel Begunkov 2020-11-18 14:56:26 +0000 239) bool done;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 240) };
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 241)
40ae0ff70fb13 (Pavel Begunkov 2021-04-01 15:43:44 +0100 242) typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
40ae0ff70fb13 (Pavel Begunkov 2021-04-01 15:43:44 +0100 243)
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 244) struct io_rsrc_data {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 245) struct io_ring_ctx *ctx;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 246)
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 247) u64 *tags;
40ae0ff70fb13 (Pavel Begunkov 2021-04-01 15:43:44 +0100 248) rsrc_put_fn *do_put;
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 249) atomic_t refs;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 250) struct completion done;
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 251) bool quiesce;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 252) };
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 253)
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 254) struct io_buffer {
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 255) struct list_head list;
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 256) __u64 addr;
d1f82808877bb (Thadeu Lima de Souza Cascardo 2021-05-05 09:47:06 -0300 257) __u32 len;
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 258) __u16 bid;
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 259) };
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 260)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 261) struct io_restriction {
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 262) DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 263) DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 264) u8 sqe_flags_allowed;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 265) u8 sqe_flags_required;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 266) bool registered;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 267) };
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 268)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 269) enum {
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 270) IO_SQ_THREAD_SHOULD_STOP = 0,
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 271) IO_SQ_THREAD_SHOULD_PARK,
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 272) };
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 273)
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 274) struct io_sq_data {
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 275) refcount_t refs;
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 276) atomic_t park_pending;
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 277) struct mutex lock;
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 278)
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 279) /* ctx's that are using this sqd */
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 280) struct list_head ctx_list;
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 281)
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 282) struct task_struct *thread;
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 283) struct wait_queue_head wait;
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 284)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 285) unsigned sq_thread_idle;
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 286) int sq_cpu;
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 287) pid_t task_pid;
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 288) pid_t task_tgid;
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 289)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 290) unsigned long state;
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 291) struct completion exited;
b7f5a0bfe2061 (Pavel Begunkov 2021-03-15 14:23:08 +0000 292) struct callback_head *park_task_work;
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 293) };
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 294)
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 295) #define IO_IOPOLL_BATCH 8
6dd0be1e2481b (Pavel Begunkov 2021-02-10 00:03:13 +0000 296) #define IO_COMPL_BATCH 32
6ff119a6e4c3f (Pavel Begunkov 2021-02-10 00:03:18 +0000 297) #define IO_REQ_CACHE_SIZE 32
bf019da7fcbe7 (Pavel Begunkov 2021-02-10 00:03:17 +0000 298) #define IO_REQ_ALLOC_BATCH 8
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 299)
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 300) struct io_comp_state {
6dd0be1e2481b (Pavel Begunkov 2021-02-10 00:03:13 +0000 301) struct io_kiocb *reqs[IO_COMPL_BATCH];
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 302) unsigned int nr;
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 303) /* inline/task_work completion list, under ->uring_lock */
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 304) struct list_head free_list;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 305) };
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 306)
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 307) struct io_submit_link {
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 308) struct io_kiocb *head;
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 309) struct io_kiocb *last;
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 310) };
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 311)
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 312) struct io_submit_state {
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 313) struct blk_plug plug;
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 314) struct io_submit_link link;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 315)
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 316) /*
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 317) * io_kiocb alloc cache
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 318) */
bf019da7fcbe7 (Pavel Begunkov 2021-02-10 00:03:17 +0000 319) void *reqs[IO_REQ_CACHE_SIZE];
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 320) unsigned int free_reqs;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 321)
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 322) bool plug_started;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 323)
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 324) /*
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 325) * Batch completion logic
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 326) */
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 327) struct io_comp_state comp;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 328)
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 329) /*
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 330) * File reference cache
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 331) */
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 332) struct file *file;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 333) unsigned int fd;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 334) unsigned int file_refs;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 335) unsigned int ios_left;
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 336) };
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 337)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 338) struct io_ring_ctx {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 339) struct {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 340) struct percpu_ref refs;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 341) } ____cacheline_aligned_in_smp;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 342)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 343) struct {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 344) unsigned int flags;
e1d85334d6238 (Randy Dunlap 2020-02-05 20:57:10 -0800 345) unsigned int compat: 1;
e1d85334d6238 (Randy Dunlap 2020-02-05 20:57:10 -0800 346) unsigned int drain_next: 1;
e1d85334d6238 (Randy Dunlap 2020-02-05 20:57:10 -0800 347) unsigned int eventfd_async: 1;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 348) unsigned int restricted: 1;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 349)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 350) /*
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 351) * Ring buffer of indices into array of io_uring_sqe, which is
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 352) * mmapped by the application using the IORING_OFF_SQES offset.
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 353) *
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 354) * This indirection could e.g. be used to assign fixed
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 355) * io_uring_sqe entries to operations and only submit them to
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 356) * the queue when needed.
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 357) *
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 358) * The kernel modifies neither the indices array nor the entries
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 359) * array.
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 360) */
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 361) u32 *sq_array;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 362) unsigned cached_sq_head;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 363) unsigned sq_entries;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 364) unsigned sq_mask;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 365) unsigned sq_thread_idle;
498ccd9eda491 (Jens Axboe 2019-10-25 10:04:25 -0600 366) unsigned cached_sq_dropped;
2c3bac6dd6c7a (Pavel Begunkov 2020-10-18 10:17:40 +0100 367) unsigned cached_cq_overflow;
ad3eb2c89fb24 (Jens Axboe 2019-12-18 17:12:20 -0700 368) unsigned long sq_check_overflow;
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 369)
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 370) struct list_head defer_list;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 371) struct list_head timeout_list;
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 372) struct list_head cq_overflow_list;
fcb323cc53e29 (Jens Axboe 2019-10-24 12:39:47 -0600 373)
ad3eb2c89fb24 (Jens Axboe 2019-12-18 17:12:20 -0700 374) struct io_uring_sqe *sq_sqes;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 375) } ____cacheline_aligned_in_smp;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 376)
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 377) struct {
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 378) struct mutex uring_lock;
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 379) wait_queue_head_t wait;
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 380) } ____cacheline_aligned_in_smp;
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 381)
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 382) struct io_submit_state submit_state;
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 383) /* IRQ completion list, under ->completion_lock */
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 384) struct list_head locked_free_list;
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 385) unsigned int locked_free_nr;
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 386)
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 387) struct io_rings *rings;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 388)
7c30f36a98ae4 (Stefan Metzmacher 2021-03-07 11:54:28 +0100 389) const struct cred *sq_creds; /* cred used for __io_sq_thread() */
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 390) struct io_sq_data *sq_data; /* if using sq thread polling */
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 391)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 392) struct wait_queue_head sqo_sq_wait;
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 393) struct list_head sqd_list;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 394)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 395) /*
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 396) * If used, fixed file set. Writers must ensure that ->refs is dead,
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 397) * readers must ensure that ->refs is alive as long as the file* is
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 398) * used. Only updated through io_uring_register(2).
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 399) */
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 400) struct io_rsrc_data *file_data;
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 401) struct io_file_table file_table;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 402) unsigned nr_user_files;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 403)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 404) /* if used, fixed mapped user buffers */
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 405) struct io_rsrc_data *buf_data;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 406) unsigned nr_user_bufs;
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 407) struct io_mapped_ubuf **user_bufs;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 408)
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 409) struct xarray io_buffers;
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 410)
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 411) struct xarray personalities;
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 412) u32 pers_next;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 413)
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 414) struct {
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 415) unsigned cached_cq_tail;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 416) unsigned cq_entries;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 417) unsigned cq_mask;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 418) atomic_t cq_timeouts;
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 419) unsigned cq_last_tm_flush;
7b289c38335ec (Hao Xu 2021-04-13 15:20:39 +0800 420) unsigned cq_extra;
ad3eb2c89fb24 (Jens Axboe 2019-12-18 17:12:20 -0700 421) unsigned long cq_check_overflow;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 422) struct wait_queue_head cq_wait;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 423) struct fasync_struct *cq_fasync;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 424) struct eventfd_ctx *cq_ev_fd;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 425) } ____cacheline_aligned_in_smp;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 426)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 427) struct {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 428) spinlock_t completion_lock;
e94f141bd248e (Jens Axboe 2019-12-19 12:06:02 -0700 429)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 430) /*
540e32a0855e7 (Pavel Begunkov 2020-07-13 23:37:09 +0300 431) * ->iopoll_list is protected by the ctx->uring_lock for
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 432) * io_uring instances that don't use IORING_SETUP_SQPOLL.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 433) * For SQPOLL, only the single threaded io_sq_thread() will
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 434) * manipulate the list, hence no extra locking is needed there.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 435) */
540e32a0855e7 (Pavel Begunkov 2020-07-13 23:37:09 +0300 436) struct list_head iopoll_list;
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 437) struct hlist_head *cancel_hash;
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 438) unsigned cancel_hash_bits;
e94f141bd248e (Jens Axboe 2019-12-19 12:06:02 -0700 439) bool poll_multi_file;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 440) } ____cacheline_aligned_in_smp;
85faa7b8346eb (Jens Axboe 2020-04-09 18:14:00 -0600 441)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 442) struct delayed_work rsrc_put_work;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 443) struct llist_head rsrc_put_llist;
d67d2263fb235 (Bijan Mottahedeh 2021-01-15 17:37:46 +0000 444) struct list_head rsrc_ref_list;
d67d2263fb235 (Bijan Mottahedeh 2021-01-15 17:37:46 +0000 445) spinlock_t rsrc_ref_lock;
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 446) struct io_rsrc_node *rsrc_node;
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 447) struct io_rsrc_node *rsrc_backup_node;
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 448) struct io_mapped_ubuf *dummy_ubuf;
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 449)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 450) struct io_restriction restrictions;
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 451)
3c1a2ead915c1 (Jens Axboe 2021-02-11 10:48:03 -0700 452) /* Keep this last, we don't need it for the fast path */
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 453) struct {
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 454) #if defined(CONFIG_UNIX)
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 455) struct socket *ring_sock;
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 456) #endif
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 457) /* hashed buffered write serialization */
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 458) struct io_wq_hash *hash_map;
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 459)
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 460) /* Only used for accounting purposes */
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 461) struct user_struct *user;
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 462) struct mm_struct *mm_account;
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 463)
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 464) /* ctx exit and cancelation */
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 465) struct callback_head *exit_task_work;
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 466) struct work_struct exit_work;
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 467) struct list_head tctx_list;
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 468) struct completion ref_comp;
bea4688b9e592 (Pavel Begunkov 2021-05-16 22:58:06 +0100 469) };
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 470) };
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 471)
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 472) struct io_uring_task {
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 473) /* submission side */
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 474) struct xarray xa;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 475) struct wait_queue_head wait;
ee53fb2b197b7 (Stefan Metzmacher 2021-03-15 12:56:57 +0100 476) const struct io_ring_ctx *last;
ee53fb2b197b7 (Stefan Metzmacher 2021-03-15 12:56:57 +0100 477) struct io_wq *io_wq;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 478) struct percpu_counter inflight;
b303fe2e5a380 (Pavel Begunkov 2021-04-11 01:46:26 +0100 479) atomic_t inflight_tracked;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 480) atomic_t in_idle;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 481)
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 482) spinlock_t task_lock;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 483) struct io_wq_work_list task_list;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 484) unsigned long task_state;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 485) struct callback_head task_work;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 486) };
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100 487)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 488) /*
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 489) * First field must be the file pointer in all the
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 490) * iocb unions! See also 'struct kiocb' in <linux/fs.h>
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 491) */
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 492) struct io_poll_iocb {
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 493) struct file *file;
018043be1f1bc (Pavel Begunkov 2020-10-27 23:17:18 +0000 494) struct wait_queue_head *head;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 495) __poll_t events;
8c838788775a5 (Jens Axboe 2019-03-12 15:48:16 -0600 496) bool done;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 497) bool canceled;
392edb45b2433 (Jens Axboe 2019-12-09 17:52:20 -0700 498) struct wait_queue_entry wait;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 499) };
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 500)
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 501) struct io_poll_update {
018043be1f1bc (Pavel Begunkov 2020-10-27 23:17:18 +0000 502) struct file *file;
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 503) u64 old_user_data;
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 504) u64 new_user_data;
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 505) __poll_t events;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 506) bool update_events;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 507) bool update_user_data;
018043be1f1bc (Pavel Begunkov 2020-10-27 23:17:18 +0000 508) };
018043be1f1bc (Pavel Begunkov 2020-10-27 23:17:18 +0000 509)
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 510) struct io_close {
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 511) struct file *file;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 512) int fd;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 513) };
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 514)
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 515) struct io_timeout_data {
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 516) struct io_kiocb *req;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 517) struct hrtimer timer;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 518) struct timespec64 ts;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 519) enum hrtimer_mode mode;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 520) };
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 521)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 522) struct io_accept {
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 523) struct file *file;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 524) struct sockaddr __user *addr;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 525) int __user *addr_len;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 526) int flags;
09952e3e78261 (Jens Axboe 2020-03-19 20:16:56 -0600 527) unsigned long nofile;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 528) };
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 529)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 530) struct io_sync {
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 531) struct file *file;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 532) loff_t len;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 533) loff_t off;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 534) int flags;
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 535) int mode;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 536) };
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 537)
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 538) struct io_cancel {
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 539) struct file *file;
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 540) u64 addr;
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 541) };
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 542)
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 543) struct io_timeout {
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 544) struct file *file;
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 545) u32 off;
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 546) u32 target_seq;
135fcde8496b0 (Pavel Begunkov 2020-07-13 23:37:12 +0300 547) struct list_head list;
90cd7e424969d (Pavel Begunkov 2020-10-27 23:25:36 +0000 548) /* head of the link, used by linked timeouts only */
90cd7e424969d (Pavel Begunkov 2020-10-27 23:25:36 +0000 549) struct io_kiocb *head;
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 550) };
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 551)
0bdf7a2ddb7d8 (Pavel Begunkov 2020-10-10 18:34:10 +0100 552) struct io_timeout_rem {
0bdf7a2ddb7d8 (Pavel Begunkov 2020-10-10 18:34:10 +0100 553) struct file *file;
0bdf7a2ddb7d8 (Pavel Begunkov 2020-10-10 18:34:10 +0100 554) u64 addr;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 555)
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 556) /* timeout update */
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 557) struct timespec64 ts;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 558) u32 flags;
0bdf7a2ddb7d8 (Pavel Begunkov 2020-10-10 18:34:10 +0100 559) };
0bdf7a2ddb7d8 (Pavel Begunkov 2020-10-10 18:34:10 +0100 560)
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 561) struct io_rw {
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 562) /* NOTE: kiocb has the file as the first member, so don't do it here */
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 563) struct kiocb kiocb;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 564) u64 addr;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 565) u64 len;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 566) };
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 567)
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 568) struct io_connect {
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 569) struct file *file;
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 570) struct sockaddr __user *addr;
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 571) int addr_len;
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 572) };
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 573)
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 574) struct io_sr_msg {
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 575) struct file *file;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 576) union {
4af3417a347d0 (Pavel Begunkov 2021-04-11 01:46:30 +0100 577) struct compat_msghdr __user *umsg_compat;
4af3417a347d0 (Pavel Begunkov 2021-04-11 01:46:30 +0100 578) struct user_msghdr __user *umsg;
4af3417a347d0 (Pavel Begunkov 2021-04-11 01:46:30 +0100 579) void __user *buf;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 580) };
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 581) int msg_flags;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 582) int bgid;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 583) size_t len;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 584) struct io_buffer *kbuf;
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 585) };
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 586)
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 587) struct io_open {
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 588) struct file *file;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 589) int dfd;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 590) struct filename *filename;
c12cedf24e786 (Jens Axboe 2020-01-08 17:41:21 -0700 591) struct open_how how;
4022e7af86be2 (Jens Axboe 2020-03-19 19:23:18 -0600 592) unsigned long nofile;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 593) };
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 594)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 595) struct io_rsrc_update {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 596) struct file *file;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 597) u64 arg;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 598) u32 nr_args;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 599) u32 offset;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 600) };
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 601)
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 602) struct io_fadvise {
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 603) struct file *file;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 604) u64 offset;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 605) u32 len;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 606) u32 advice;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 607) };
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 608)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 609) struct io_madvise {
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 610) struct file *file;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 611) u64 addr;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 612) u32 len;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 613) u32 advice;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 614) };
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 615)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 616) struct io_epoll {
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 617) struct file *file;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 618) int epfd;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 619) int op;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 620) int fd;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 621) struct epoll_event event;
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 622) };
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 623)
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 624) struct io_splice {
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 625) struct file *file_out;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 626) struct file *file_in;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 627) loff_t off_out;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 628) loff_t off_in;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 629) u64 len;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 630) unsigned int flags;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 631) };
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 632)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 633) struct io_provide_buf {
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 634) struct file *file;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 635) __u64 addr;
38134ada0ceea (Pavel Begunkov 2021-04-15 13:07:39 +0100 636) __u32 len;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 637) __u32 bgid;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 638) __u16 nbufs;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 639) __u16 bid;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 640) };
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 641)
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 642) struct io_statx {
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 643) struct file *file;
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 644) int dfd;
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 645) unsigned int mask;
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 646) unsigned int flags;
e62753e4e2926 (Bijan Mottahedeh 2020-05-22 21:31:18 -0700 647) const char __user *filename;
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 648) struct statx __user *buffer;
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 649) };
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 650)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 651) struct io_shutdown {
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 652) struct file *file;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 653) int how;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 654) };
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 655)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 656) struct io_rename {
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 657) struct file *file;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 658) int old_dfd;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 659) int new_dfd;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 660) struct filename *oldpath;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 661) struct filename *newpath;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 662) int flags;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 663) };
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 664)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 665) struct io_unlink {
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 666) struct file *file;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 667) int dfd;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 668) int flags;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 669) struct filename *filename;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 670) };
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 671)
3ca405ebfc1c3 (Pavel Begunkov 2020-07-13 23:37:08 +0300 672) struct io_completion {
3ca405ebfc1c3 (Pavel Begunkov 2020-07-13 23:37:08 +0300 673) struct file *file;
3ca405ebfc1c3 (Pavel Begunkov 2020-07-13 23:37:08 +0300 674) struct list_head list;
8c3f9cd1603d0 (Pavel Begunkov 2021-02-28 22:35:15 +0000 675) u32 cflags;
3ca405ebfc1c3 (Pavel Begunkov 2020-07-13 23:37:08 +0300 676) };
3ca405ebfc1c3 (Pavel Begunkov 2020-07-13 23:37:08 +0300 677)
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 678) struct io_async_connect {
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 679) struct sockaddr_storage address;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 680) };
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 681)
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 682) struct io_async_msghdr {
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 683) struct iovec fast_iov[UIO_FASTIOV];
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 684) /* points to an allocated iov, if NULL we use fast_iov instead */
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 685) struct iovec *free_iov;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 686) struct sockaddr __user *uaddr;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 687) struct msghdr msg;
b537916ca5107 (Jens Axboe 2020-02-09 11:29:15 -0700 688) struct sockaddr_storage addr;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 689) };
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 690)
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 691) struct io_async_rw {
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 692) struct iovec fast_iov[UIO_FASTIOV];
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 693) const struct iovec *free_iovec;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 694) struct iov_iter iter;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 695) size_t bytes_done;
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 696) struct wait_page_queue wpq;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 697) };
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 698)
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 699) enum {
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 700) REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 701) REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 702) REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 703) REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 704) REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 705) REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 706)
dddca22636c90 (Pavel Begunkov 2021-04-27 16:13:52 +0100 707) /* first byte is taken by user flags, shift it to not overlap */
dddca22636c90 (Pavel Begunkov 2021-04-27 16:13:52 +0100 708) REQ_F_FAIL_LINK_BIT = 8,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 709) REQ_F_INFLIGHT_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 710) REQ_F_CUR_POS_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 711) REQ_F_NOWAIT_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 712) REQ_F_LINK_TIMEOUT_BIT,
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 713) REQ_F_NEED_CLEANUP_BIT,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 714) REQ_F_POLLED_BIT,
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 715) REQ_F_BUFFER_SELECTED_BIT,
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 716) REQ_F_LTIMEOUT_ACTIVE_BIT,
e342c807f556d (Pavel Begunkov 2021-01-19 13:32:47 +0000 717) REQ_F_COMPLETE_INLINE_BIT,
230d50d448acb (Jens Axboe 2021-04-01 20:41:15 -0600 718) REQ_F_REISSUE_BIT,
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 719) REQ_F_DONT_REISSUE_BIT,
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 720) /* keep async read/write and isreg together and in order */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 721) REQ_F_ASYNC_READ_BIT,
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 722) REQ_F_ASYNC_WRITE_BIT,
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 723) REQ_F_ISREG_BIT,
84557871f2ff3 (Jens Axboe 2020-03-03 15:28:17 -0700 724)
84557871f2ff3 (Jens Axboe 2020-03-03 15:28:17 -0700 725) /* not a real bit, just to check we're not overflowing the space */
84557871f2ff3 (Jens Axboe 2020-03-03 15:28:17 -0700 726) __REQ_F_LAST_BIT,
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 727) };
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 728)
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 729) enum {
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 730) /* ctx owns file */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 731) REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 732) /* drain existing IO first */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 733) REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 734) /* linked sqes */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 735) REQ_F_LINK = BIT(REQ_F_LINK_BIT),
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 736) /* doesn't sever on completion < 0 */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 737) REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 738) /* IOSQE_ASYNC */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 739) REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 740) /* IOSQE_BUFFER_SELECT */
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 741) REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 742)
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 743) /* fail rest of links */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 744) REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
b05a1bcd40184 (Pavel Begunkov 2021-03-04 13:59:24 +0000 745) /* on inflight list, should be cancelled and waited on exit reliably */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 746) REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 747) /* read/write uses file position */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 748) REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 749) /* must not punt to workers */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 750) REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 751) /* has or had linked timeout */
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 752) REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 753) /* needs cleanup */
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 754) REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 755) /* already went through poll handler */
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 756) REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 757) /* buffer already selected */
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 758) REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 759) /* linked timeout is active, i.e. prepared by link's head */
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 760) REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT),
e342c807f556d (Pavel Begunkov 2021-01-19 13:32:47 +0000 761) /* completion is deferred through io_comp_state */
e342c807f556d (Pavel Begunkov 2021-01-19 13:32:47 +0000 762) REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
230d50d448acb (Jens Axboe 2021-04-01 20:41:15 -0600 763) /* caller should reissue async */
230d50d448acb (Jens Axboe 2021-04-01 20:41:15 -0600 764) REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 765) /* don't attempt request reissue, see io_rw_reissue() */
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 766) REQ_F_DONT_REISSUE = BIT(REQ_F_DONT_REISSUE_BIT),
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 767) /* supports async reads */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 768) REQ_F_ASYNC_READ = BIT(REQ_F_ASYNC_READ_BIT),
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 769) /* supports async writes */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 770) REQ_F_ASYNC_WRITE = BIT(REQ_F_ASYNC_WRITE_BIT),
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 771) /* regular file */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 772) REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 773) };
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 774)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 775) struct async_poll {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 776) struct io_poll_iocb poll;
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 777) struct io_poll_iocb *double_poll;
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 778) };
6b47ee6ecab14 (Pavel Begunkov 2020-01-18 20:22:41 +0300 779)
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 780) struct io_task_work {
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 781) struct io_wq_work_node node;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 782) task_work_func_t func;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 783) };
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 784)
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 785) enum {
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 786) IORING_RSRC_FILE = 0,
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 787) IORING_RSRC_BUFFER = 1,
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 788) };
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 789)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 790) /*
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 791) * NOTE! Each of the iocb union members has the file pointer
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 792) * as the first entry in their struct definition. So you can
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 793) * access the file pointer through any of the sub-structs,
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 794) * or directly as just 'ki_filp' in this struct.
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 795) */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 796) struct io_kiocb {
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 797) union {
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 798) struct file *file;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 799) struct io_rw rw;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 800) struct io_poll_iocb poll;
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 801) struct io_poll_update poll_update;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 802) struct io_accept accept;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 803) struct io_sync sync;
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 804) struct io_cancel cancel;
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 805) struct io_timeout timeout;
0bdf7a2ddb7d8 (Pavel Begunkov 2020-10-10 18:34:10 +0100 806) struct io_timeout_rem timeout_rem;
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 807) struct io_connect connect;
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 808) struct io_sr_msg sr_msg;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 809) struct io_open open;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 810) struct io_close close;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 811) struct io_rsrc_update rsrc_update;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 812) struct io_fadvise fadvise;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 813) struct io_madvise madvise;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 814) struct io_epoll epoll;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 815) struct io_splice splice;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 816) struct io_provide_buf pbuf;
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 817) struct io_statx statx;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 818) struct io_shutdown shutdown;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 819) struct io_rename rename;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 820) struct io_unlink unlink;
3ca405ebfc1c3 (Pavel Begunkov 2020-07-13 23:37:08 +0300 821) /* use only after cleaning per-op data, see io_clean_op() */
3ca405ebfc1c3 (Pavel Begunkov 2020-07-13 23:37:08 +0300 822) struct io_completion compl;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 823) };
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 824)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 825) /* opcode allocated if it needs to store data for async defer */
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 826) void *async_data;
d625c6ee49750 (Jens Axboe 2019-12-17 19:53:05 -0700 827) u8 opcode;
65a6543da3868 (Xiaoguang Wang 2020-06-11 23:39:36 +0800 828) /* polled IO has completed */
65a6543da3868 (Xiaoguang Wang 2020-06-11 23:39:36 +0800 829) u8 iopoll_completed;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 830)
4f4eeba87cc73 (Bijan Mottahedeh 2020-05-19 14:52:49 -0700 831) u16 buf_index;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 832) u32 result;
4f4eeba87cc73 (Bijan Mottahedeh 2020-05-19 14:52:49 -0700 833)
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 834) struct io_ring_ctx *ctx;
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 835) unsigned int flags;
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 836) atomic_t refs;
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 837) struct task_struct *task;
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 838) u64 user_data;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 839)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 840) struct io_kiocb *link;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 841) struct percpu_ref *fixed_rsrc_refs;
fcb323cc53e29 (Jens Axboe 2019-10-24 12:39:47 -0600 842)
b303fe2e5a380 (Pavel Begunkov 2021-04-11 01:46:26 +0100 843) /* used with ctx->iopoll_list with reads/writes */
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 844) struct list_head inflight_entry;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 845) union {
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 846) struct io_task_work io_task_work;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 847) struct callback_head task_work;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 848) };
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 849) /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 850) struct hlist_node hash_node;
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 851) struct async_poll *apoll;
010e8e6be2194 (Pavel Begunkov 2020-07-30 18:43:45 +0300 852) struct io_wq_work work;
501449420a42c (Pavel Begunkov 2021-06-24 15:09:57 +0100 853) const struct cred *creds;
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 854)
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 855) /* store used ubuf, so we can prevent reloading */
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 856) struct io_mapped_ubuf *imu;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 857) };
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 858)
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 859) struct io_tctx_node {
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 860) struct list_head ctx_node;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 861) struct task_struct *task;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 862) struct io_ring_ctx *ctx;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 863) };
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 864)
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 865) struct io_defer_entry {
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 866) struct list_head list;
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 867) struct io_kiocb *req;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 868) u32 seq;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 869) };
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 870)
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 871) struct io_op_def {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 872) /* needs req->file assigned */
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 873) unsigned needs_file : 1;
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 874) /* hash wq insertion if file is a regular file */
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 875) unsigned hash_reg_file : 1;
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 876) /* unbound wq insertion if file is a non-regular file */
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 877) unsigned unbound_nonreg_file : 1;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 878) /* opcode is not supported by this kernel */
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 879) unsigned not_supported : 1;
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 880) /* set if opcode supports polled "wait" */
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 881) unsigned pollin : 1;
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 882) unsigned pollout : 1;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 883) /* op supports buffer selection */
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 884) unsigned buffer_select : 1;
26f0505a9ce57 (Pavel Begunkov 2021-02-28 22:35:18 +0000 885) /* do prep async if is going to be punted */
26f0505a9ce57 (Pavel Begunkov 2021-02-28 22:35:18 +0000 886) unsigned needs_async_setup : 1;
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 887) /* should block plug */
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 888) unsigned plug : 1;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 889) /* size of async data needed, if any */
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 890) unsigned short async_size;
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 891) };
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 892)
0918682be432b (Jens Axboe 2020-10-13 15:01:40 -0600 893) static const struct io_op_def io_op_defs[] = {
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 894) [IORING_OP_NOP] = {},
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 895) [IORING_OP_READV] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 896) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 897) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 898) .pollin = 1,
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 899) .buffer_select = 1,
26f0505a9ce57 (Pavel Begunkov 2021-02-28 22:35:18 +0000 900) .needs_async_setup = 1,
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 901) .plug = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 902) .async_size = sizeof(struct io_async_rw),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 903) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 904) [IORING_OP_WRITEV] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 905) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 906) .hash_reg_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 907) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 908) .pollout = 1,
26f0505a9ce57 (Pavel Begunkov 2021-02-28 22:35:18 +0000 909) .needs_async_setup = 1,
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 910) .plug = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 911) .async_size = sizeof(struct io_async_rw),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 912) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 913) [IORING_OP_FSYNC] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 914) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 915) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 916) [IORING_OP_READ_FIXED] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 917) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 918) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 919) .pollin = 1,
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 920) .plug = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 921) .async_size = sizeof(struct io_async_rw),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 922) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 923) [IORING_OP_WRITE_FIXED] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 924) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 925) .hash_reg_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 926) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 927) .pollout = 1,
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 928) .plug = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 929) .async_size = sizeof(struct io_async_rw),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 930) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 931) [IORING_OP_POLL_ADD] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 932) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 933) .unbound_nonreg_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 934) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 935) [IORING_OP_POLL_REMOVE] = {},
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 936) [IORING_OP_SYNC_FILE_RANGE] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 937) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 938) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 939) [IORING_OP_SENDMSG] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 940) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 941) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 942) .pollout = 1,
26f0505a9ce57 (Pavel Begunkov 2021-02-28 22:35:18 +0000 943) .needs_async_setup = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 944) .async_size = sizeof(struct io_async_msghdr),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 945) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 946) [IORING_OP_RECVMSG] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 947) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 948) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 949) .pollin = 1,
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 950) .buffer_select = 1,
26f0505a9ce57 (Pavel Begunkov 2021-02-28 22:35:18 +0000 951) .needs_async_setup = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 952) .async_size = sizeof(struct io_async_msghdr),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 953) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 954) [IORING_OP_TIMEOUT] = {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 955) .async_size = sizeof(struct io_timeout_data),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 956) },
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 957) [IORING_OP_TIMEOUT_REMOVE] = {
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 958) /* used by timeout updates' prep() */
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 959) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 960) [IORING_OP_ACCEPT] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 961) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 962) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 963) .pollin = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 964) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 965) [IORING_OP_ASYNC_CANCEL] = {},
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 966) [IORING_OP_LINK_TIMEOUT] = {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 967) .async_size = sizeof(struct io_timeout_data),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 968) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 969) [IORING_OP_CONNECT] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 970) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 971) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 972) .pollout = 1,
26f0505a9ce57 (Pavel Begunkov 2021-02-28 22:35:18 +0000 973) .needs_async_setup = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 974) .async_size = sizeof(struct io_async_connect),
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 975) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 976) [IORING_OP_FALLOCATE] = {
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 977) .needs_file = 1,
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 978) },
44526bedc2ff8 (Jens Axboe 2021-02-15 13:32:18 -0700 979) [IORING_OP_OPENAT] = {},
44526bedc2ff8 (Jens Axboe 2021-02-15 13:32:18 -0700 980) [IORING_OP_CLOSE] = {},
44526bedc2ff8 (Jens Axboe 2021-02-15 13:32:18 -0700 981) [IORING_OP_FILES_UPDATE] = {},
44526bedc2ff8 (Jens Axboe 2021-02-15 13:32:18 -0700 982) [IORING_OP_STATX] = {},
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 983) [IORING_OP_READ] = {
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 984) .needs_file = 1,
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 985) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 986) .pollin = 1,
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 987) .buffer_select = 1,
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 988) .plug = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 989) .async_size = sizeof(struct io_async_rw),
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 990) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 991) [IORING_OP_WRITE] = {
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 992) .needs_file = 1,
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 993) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 994) .pollout = 1,
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 995) .plug = 1,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 996) .async_size = sizeof(struct io_async_rw),
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 997) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 998) [IORING_OP_FADVISE] = {
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 999) .needs_file = 1,
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 1000) },
44526bedc2ff8 (Jens Axboe 2021-02-15 13:32:18 -0700 1001) [IORING_OP_MADVISE] = {},
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 1002) [IORING_OP_SEND] = {
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 1003) .needs_file = 1,
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 1004) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 1005) .pollout = 1,
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 1006) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 1007) [IORING_OP_RECV] = {
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 1008) .needs_file = 1,
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 1009) .unbound_nonreg_file = 1,
8a72758c51f8a (Jens Axboe 2020-02-20 09:59:44 -0700 1010) .pollin = 1,
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 1011) .buffer_select = 1,
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 1012) },
0463b6c58e557 (Pavel Begunkov 2020-01-18 21:35:38 +0300 1013) [IORING_OP_OPENAT2] = {
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 1014) },
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 1015) [IORING_OP_EPOLL_CTL] = {
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 1016) .unbound_nonreg_file = 1,
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 1017) },
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 1018) [IORING_OP_SPLICE] = {
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 1019) .needs_file = 1,
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 1020) .hash_reg_file = 1,
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 1021) .unbound_nonreg_file = 1,
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 1022) },
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 1023) [IORING_OP_PROVIDE_BUFFERS] = {},
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 1024) [IORING_OP_REMOVE_BUFFERS] = {},
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 1025) [IORING_OP_TEE] = {
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 1026) .needs_file = 1,
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 1027) .hash_reg_file = 1,
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 1028) .unbound_nonreg_file = 1,
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 1029) },
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 1030) [IORING_OP_SHUTDOWN] = {
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 1031) .needs_file = 1,
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 1032) },
44526bedc2ff8 (Jens Axboe 2021-02-15 13:32:18 -0700 1033) [IORING_OP_RENAMEAT] = {},
44526bedc2ff8 (Jens Axboe 2021-02-15 13:32:18 -0700 1034) [IORING_OP_UNLINKAT] = {},
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 1035) };
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 1036)
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1037) static bool io_disarm_next(struct io_kiocb *req);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 1038) static void io_uring_del_task_file(unsigned long index);
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 1039) static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 1040) struct task_struct *task,
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 1041) bool cancel_all);
734551df6f9be (Pavel Begunkov 2021-04-18 14:52:09 +0100 1042) static void io_uring_cancel_sqpoll(struct io_sq_data *sqd);
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 1043) static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
1ffc54220c444 (Pavel Begunkov 2020-12-30 21:34:15 +0000 1044)
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1045) static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1046) long res, unsigned int cflags);
ec9c02ad4c380 (Jackie Liu 2019-11-08 23:50:36 +0800 1047) static void io_put_req(struct io_kiocb *req);
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 1048) static void io_put_req_deferred(struct io_kiocb *req, int nr);
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1049) static void io_dismantle_req(struct io_kiocb *req);
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1050) static void io_put_task(struct task_struct *task, int nr);
94ae5e77a9150 (Jens Axboe 2019-11-14 19:39:52 -0700 1051) static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
94ae5e77a9150 (Jens Axboe 2019-11-14 19:39:52 -0700 1052) static void io_queue_linked_timeout(struct io_kiocb *req);
fdecb66281e16 (Pavel Begunkov 2021-04-25 14:32:20 +0100 1053) static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 1054) struct io_uring_rsrc_update2 *up,
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 1055) unsigned nr_args);
68fb897966feb (Pavel Begunkov 2021-03-19 17:22:41 +0000 1056) static void io_clean_op(struct io_kiocb *req);
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 1057) static struct file *io_file_get(struct io_submit_state *state,
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 1058) struct io_kiocb *req, int fd, bool fixed);
c5eef2b9449ba (Pavel Begunkov 2021-02-10 00:03:22 +0000 1059) static void __io_queue_sqe(struct io_kiocb *req);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 1060) static void io_rsrc_put_work(struct work_struct *work);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1061)
907d1df30a51c (Pavel Begunkov 2021-01-26 23:35:10 +0000 1062) static void io_req_task_queue(struct io_kiocb *req);
65453d1efbd20 (Jens Axboe 2021-02-10 00:03:21 +0000 1063) static void io_submit_flush_completions(struct io_comp_state *cs,
65453d1efbd20 (Jens Axboe 2021-02-10 00:03:21 +0000 1064) struct io_ring_ctx *ctx);
5082620fb2cab (Jens Axboe 2021-02-23 09:02:26 -0700 1065) static bool io_poll_remove_waitqs(struct io_kiocb *req);
179ae0d15e8b3 (Pavel Begunkov 2021-02-28 22:35:20 +0000 1066) static int io_req_prep_async(struct io_kiocb *req);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1067)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1068) static struct kmem_cache *req_cachep;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1069)
0918682be432b (Jens Axboe 2020-10-13 15:01:40 -0600 1070) static const struct file_operations io_uring_fops;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1071)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1072) struct sock *io_uring_get_socket(struct file *file)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1073) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1074) #if defined(CONFIG_UNIX)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1075) if (file->f_op == &io_uring_fops) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1076) struct io_ring_ctx *ctx = file->private_data;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1077)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1078) return ctx->ring_sock->sk;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1079) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1080) #endif
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1081) return NULL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1082) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1083) EXPORT_SYMBOL(io_uring_get_socket);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1084)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1085) #define io_for_each_link(pos, head) \
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1086) for (pos = (head); pos; pos = pos->link)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1087)
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 1088) static inline void io_req_set_rsrc_node(struct io_kiocb *req)
36f72fe2792c4 (Pavel Begunkov 2020-11-18 19:57:26 +0000 1089) {
36f72fe2792c4 (Pavel Begunkov 2020-11-18 19:57:26 +0000 1090) struct io_ring_ctx *ctx = req->ctx;
36f72fe2792c4 (Pavel Begunkov 2020-11-18 19:57:26 +0000 1091)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 1092) if (!req->fixed_rsrc_refs) {
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 1093) req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 1094) percpu_ref_get(req->fixed_rsrc_refs);
36f72fe2792c4 (Pavel Begunkov 2020-11-18 19:57:26 +0000 1095) }
36f72fe2792c4 (Pavel Begunkov 2020-11-18 19:57:26 +0000 1096) }
36f72fe2792c4 (Pavel Begunkov 2020-11-18 19:57:26 +0000 1097)
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1098) static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1099) {
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1100) bool got = percpu_ref_tryget(ref);
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1101)
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1102) /* already at zero, wait for ->release() */
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1103) if (!got)
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1104) wait_for_completion(compl);
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1105) percpu_ref_resurrect(ref);
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1106) if (got)
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1107) percpu_ref_put(ref);
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1108) }
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 1109)
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 1110) static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 1111) bool cancel_all)
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1112) {
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1113) struct io_kiocb *req;
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1114)
682076801a2f4 (Pavel Begunkov 2021-03-22 01:58:25 +0000 1115) if (task && head->task != task)
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1116) return false;
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 1117) if (cancel_all)
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1118) return true;
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1119)
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1120) io_for_each_link(req, head) {
b05a1bcd40184 (Pavel Begunkov 2021-03-04 13:59:24 +0000 1121) if (req->flags & REQ_F_INFLIGHT)
02a13674fa0e8 (Jens Axboe 2021-01-23 15:49:31 -0700 1122) return true;
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1123) }
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1124) return false;
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1125) }
08d23634643c2 (Pavel Begunkov 2020-11-06 13:00:22 +0000 1126)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 1127) static inline void req_set_fail_links(struct io_kiocb *req)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 1128) {
e4335ed33eb54 (Pavel Begunkov 2021-04-11 01:46:39 +0100 1129) if (req->flags & REQ_F_LINK)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 1130) req->flags |= REQ_F_FAIL_LINK;
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 1131) }
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 1132)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1133) static void io_ring_ctx_ref_free(struct percpu_ref *ref)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1134) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1135) struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1136)
0f158b4cf20e7 (Jens Axboe 2020-05-14 17:18:39 -0600 1137) complete(&ctx->ref_comp);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1138) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1139)
8eb7e2d007633 (Pavel Begunkov 2020-06-29 13:13:02 +0300 1140) static inline bool io_is_timeout_noseq(struct io_kiocb *req)
8eb7e2d007633 (Pavel Begunkov 2020-06-29 13:13:02 +0300 1141) {
8eb7e2d007633 (Pavel Begunkov 2020-06-29 13:13:02 +0300 1142) return !req->timeout.off;
8eb7e2d007633 (Pavel Begunkov 2020-06-29 13:13:02 +0300 1143) }
8eb7e2d007633 (Pavel Begunkov 2020-06-29 13:13:02 +0300 1144)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1145) static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1146) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1147) struct io_ring_ctx *ctx;
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1148) int hash_bits;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1149)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1150) ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1151) if (!ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1152) return NULL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1153)
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1154) /*
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1155) * Use 5 bits less than the max cq entries, that should give us around
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1156) * 32 entries per hash list if totally full and uniformly spread.
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1157) */
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1158) hash_bits = ilog2(p->cq_entries);
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1159) hash_bits -= 5;
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1160) if (hash_bits <= 0)
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1161) hash_bits = 1;
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1162) ctx->cancel_hash_bits = hash_bits;
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1163) ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1164) GFP_KERNEL);
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1165) if (!ctx->cancel_hash)
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1166) goto err;
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1167) __hash_init(ctx->cancel_hash, 1U << hash_bits);
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1168)
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 1169) ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 1170) if (!ctx->dummy_ubuf)
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 1171) goto err;
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 1172) /* set invalid range, so io_import_fixed() fails meeting it */
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 1173) ctx->dummy_ubuf->ubuf = -1UL;
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 1174)
214828962dead (Roman Gushchin 2019-05-07 10:01:48 -0700 1175) if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 1176) PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 1177) goto err;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1178)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1179) ctx->flags = p->flags;
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 1180) init_waitqueue_head(&ctx->sqo_sq_wait);
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 1181) INIT_LIST_HEAD(&ctx->sqd_list);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1182) init_waitqueue_head(&ctx->cq_wait);
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1183) INIT_LIST_HEAD(&ctx->cq_overflow_list);
0f158b4cf20e7 (Jens Axboe 2020-05-14 17:18:39 -0600 1184) init_completion(&ctx->ref_comp);
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 1185) xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 1186) xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1187) mutex_init(&ctx->uring_lock);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1188) init_waitqueue_head(&ctx->wait);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1189) spin_lock_init(&ctx->completion_lock);
540e32a0855e7 (Pavel Begunkov 2020-07-13 23:37:09 +0300 1190) INIT_LIST_HEAD(&ctx->iopoll_list);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1191) INIT_LIST_HEAD(&ctx->defer_list);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 1192) INIT_LIST_HEAD(&ctx->timeout_list);
d67d2263fb235 (Bijan Mottahedeh 2021-01-15 17:37:46 +0000 1193) spin_lock_init(&ctx->rsrc_ref_lock);
d67d2263fb235 (Bijan Mottahedeh 2021-01-15 17:37:46 +0000 1194) INIT_LIST_HEAD(&ctx->rsrc_ref_list);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 1195) INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 1196) init_llist_head(&ctx->rsrc_put_llist);
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 1197) INIT_LIST_HEAD(&ctx->tctx_list);
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 1198) INIT_LIST_HEAD(&ctx->submit_state.comp.free_list);
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 1199) INIT_LIST_HEAD(&ctx->locked_free_list);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1200) return ctx;
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 1201) err:
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 1202) kfree(ctx->dummy_ubuf);
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 1203) kfree(ctx->cancel_hash);
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 1204) kfree(ctx);
206aefde4f886 (Jens Axboe 2019-11-07 18:27:42 -0700 1205) return NULL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1206) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1207)
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 1208) static bool req_need_defer(struct io_kiocb *req, u32 seq)
7adf4eaf60f3d (Jens Axboe 2019-10-10 21:42:58 -0600 1209) {
2bc9930e78fe0 (Jens Axboe 2020-07-09 09:43:27 -0600 1210) if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
2bc9930e78fe0 (Jens Axboe 2020-07-09 09:43:27 -0600 1211) struct io_ring_ctx *ctx = req->ctx;
a197f664a0db8 (Jackie Liu 2019-11-08 08:09:12 -0700 1212)
7b289c38335ec (Hao Xu 2021-04-13 15:20:39 +0800 1213) return seq + ctx->cq_extra != ctx->cached_cq_tail
2c3bac6dd6c7a (Pavel Begunkov 2020-10-18 10:17:40 +0100 1214) + READ_ONCE(ctx->cached_cq_overflow);
2bc9930e78fe0 (Jens Axboe 2020-07-09 09:43:27 -0600 1215) }
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1216)
9d858b2148398 (Bob Liu 2019-11-13 18:06:25 +0800 1217) return false;
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1218) }
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1219)
ce3d5aae331fa (Pavel Begunkov 2021-02-01 18:59:55 +0000 1220) static void io_req_track_inflight(struct io_kiocb *req)
ce3d5aae331fa (Pavel Begunkov 2021-02-01 18:59:55 +0000 1221) {
ce3d5aae331fa (Pavel Begunkov 2021-02-01 18:59:55 +0000 1222) if (!(req->flags & REQ_F_INFLIGHT)) {
ce3d5aae331fa (Pavel Begunkov 2021-02-01 18:59:55 +0000 1223) req->flags |= REQ_F_INFLIGHT;
b303fe2e5a380 (Pavel Begunkov 2021-04-11 01:46:26 +0100 1224) atomic_inc(¤t->io_uring->inflight_tracked);
ce3d5aae331fa (Pavel Begunkov 2021-02-01 18:59:55 +0000 1225) }
ce3d5aae331fa (Pavel Begunkov 2021-02-01 18:59:55 +0000 1226) }
ce3d5aae331fa (Pavel Begunkov 2021-02-01 18:59:55 +0000 1227)
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1228) static void io_prep_async_work(struct io_kiocb *req)
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1229) {
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1230) const struct io_op_def *def = &io_op_defs[req->opcode];
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1231) struct io_ring_ctx *ctx = req->ctx;
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1232)
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 1233) if (!req->creds)
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 1234) req->creds = get_current_cred();
003e8dccdb227 (Jens Axboe 2021-03-06 09:22:27 -0700 1235)
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1236) req->work.list.next = NULL;
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1237) req->work.flags = 0;
feaadc4fc2ebd (Pavel Begunkov 2020-10-22 16:47:16 +0100 1238) if (req->flags & REQ_F_FORCE_ASYNC)
feaadc4fc2ebd (Pavel Begunkov 2020-10-22 16:47:16 +0100 1239) req->work.flags |= IO_WQ_WORK_CONCURRENT;
feaadc4fc2ebd (Pavel Begunkov 2020-10-22 16:47:16 +0100 1240)
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1241) if (req->flags & REQ_F_ISREG) {
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1242) if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1243) io_wq_hash_work(&req->work, file_inode(req->file));
4b982bd0f383d (Jens Axboe 2021-04-01 08:38:34 -0600 1244) } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1245) if (def->unbound_nonreg_file)
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1246) req->work.flags |= IO_WQ_WORK_UNBOUND;
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 1247) }
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1248)
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1249) switch (req->opcode) {
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1250) case IORING_OP_SPLICE:
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1251) case IORING_OP_TEE:
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1252) if (!S_ISREG(file_inode(req->splice.file_in)->i_mode))
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1253) req->work.flags |= IO_WQ_WORK_UNBOUND;
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1254) break;
e1d675df1a36e (Pavel Begunkov 2021-03-22 01:58:29 +0000 1255) }
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 1256) }
cccf0ee834559 (Jens Axboe 2020-01-27 16:34:48 -0700 1257)
cbdcb4357c000 (Pavel Begunkov 2020-06-29 19:18:43 +0300 1258) static void io_prep_async_link(struct io_kiocb *req)
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 1259) {
cbdcb4357c000 (Pavel Begunkov 2020-06-29 19:18:43 +0300 1260) struct io_kiocb *cur;
54a91f3bb9b96 (Jens Axboe 2019-09-10 09:15:04 -0600 1261)
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1262) if (req->flags & REQ_F_LINK_TIMEOUT) {
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1263) struct io_ring_ctx *ctx = req->ctx;
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1264)
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1265) spin_lock_irq(&ctx->completion_lock);
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1266) io_for_each_link(cur, req)
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1267) io_prep_async_work(cur);
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1268) spin_unlock_irq(&ctx->completion_lock);
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1269) } else {
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1270) io_for_each_link(cur, req)
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1271) io_prep_async_work(cur);
5bb49c88472f5 (Pavel Begunkov 2021-07-26 14:14:31 +0100 1272) }
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 1273) }
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 1274)
ebf936670721b (Pavel Begunkov 2021-03-01 18:20:47 +0000 1275) static void io_queue_async_work(struct io_kiocb *req)
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 1276) {
a197f664a0db8 (Jackie Liu 2019-11-08 08:09:12 -0700 1277) struct io_ring_ctx *ctx = req->ctx;
cbdcb4357c000 (Pavel Begunkov 2020-06-29 19:18:43 +0300 1278) struct io_kiocb *link = io_prep_linked_timeout(req);
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 1279) struct io_uring_task *tctx = req->task->io_uring;
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 1280)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 1281) BUG_ON(!tctx);
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 1282) BUG_ON(!tctx->io_wq);
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 1283)
cbdcb4357c000 (Pavel Begunkov 2020-06-29 19:18:43 +0300 1284) /* init ->work of the whole link before punting */
cbdcb4357c000 (Pavel Begunkov 2020-06-29 19:18:43 +0300 1285) io_prep_async_link(req);
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1286)
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1287) /*
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1288) * Not expected to happen, but if we do have a bug where this _can_
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1289) * happen, catch it here and ensure the request is marked as
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1290) * canceled. That will make io-wq go through the usual work cancel
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1291) * procedure rather than attempt to run this request (or create a new
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1292) * worker for it).
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1293) */
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1294) if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1295) req->work.flags |= IO_WQ_WORK_CANCEL;
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 1296)
d07f1e8a42614 (Pavel Begunkov 2021-03-22 01:45:58 +0000 1297) trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
d07f1e8a42614 (Pavel Begunkov 2021-03-22 01:45:58 +0000 1298) &req->work, req->flags);
ebf936670721b (Pavel Begunkov 2021-03-01 18:20:47 +0000 1299) io_wq_enqueue(tctx->io_wq, &req->work);
7271ef3a93a83 (Jens Axboe 2020-08-10 09:55:22 -0600 1300) if (link)
7271ef3a93a83 (Jens Axboe 2020-08-10 09:55:22 -0600 1301) io_queue_linked_timeout(link);
cbdcb4357c000 (Pavel Begunkov 2020-06-29 19:18:43 +0300 1302) }
cbdcb4357c000 (Pavel Begunkov 2020-06-29 19:18:43 +0300 1303)
1ee4160c73b21 (Pavel Begunkov 2021-03-25 18:32:42 +0000 1304) static void io_kill_timeout(struct io_kiocb *req, int status)
8c855885b8b35 (Pavel Begunkov 2021-04-13 02:58:41 +0100 1305) __must_hold(&req->ctx->completion_lock)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 1306) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 1307) struct io_timeout_data *io = req->async_data;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 1308)
fd9c7bc542dae (Pavel Begunkov 2021-04-13 02:58:42 +0100 1309) if (hrtimer_try_to_cancel(&io->timer) != -1) {
01cec8c18f5ad (Pavel Begunkov 2020-07-30 18:43:50 +0300 1310) atomic_set(&req->ctx->cq_timeouts,
01cec8c18f5ad (Pavel Begunkov 2020-07-30 18:43:50 +0300 1311) atomic_read(&req->ctx->cq_timeouts) + 1);
135fcde8496b0 (Pavel Begunkov 2020-07-13 23:37:12 +0300 1312) list_del_init(&req->timeout.list);
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1313) io_cqring_fill_event(req->ctx, req->user_data, status, 0);
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 1314) io_put_req_deferred(req, 1);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 1315) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 1316) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 1317)
0451894522108 (Pavel Begunkov 2020-05-26 20:34:05 +0300 1318) static void __io_queue_deferred(struct io_ring_ctx *ctx)
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1319) {
0451894522108 (Pavel Begunkov 2020-05-26 20:34:05 +0300 1320) do {
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 1321) struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 1322) struct io_defer_entry, list);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1323)
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 1324) if (req_need_defer(de->req, de->seq))
0451894522108 (Pavel Begunkov 2020-05-26 20:34:05 +0300 1325) break;
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 1326) list_del_init(&de->list);
907d1df30a51c (Pavel Begunkov 2021-01-26 23:35:10 +0000 1327) io_req_task_queue(de->req);
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 1328) kfree(de);
0451894522108 (Pavel Begunkov 2020-05-26 20:34:05 +0300 1329) } while (!list_empty(&ctx->defer_list));
0451894522108 (Pavel Begunkov 2020-05-26 20:34:05 +0300 1330) }
0451894522108 (Pavel Begunkov 2020-05-26 20:34:05 +0300 1331)
360428f8c0cd8 (Pavel Begunkov 2020-05-30 14:54:17 +0300 1332) static void io_flush_timeouts(struct io_ring_ctx *ctx)
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1333) {
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1334) u32 seq;
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1335)
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1336) if (list_empty(&ctx->timeout_list))
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1337) return;
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1338)
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1339) seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1340)
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1341) do {
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1342) u32 events_needed, events_got;
360428f8c0cd8 (Pavel Begunkov 2020-05-30 14:54:17 +0300 1343) struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
135fcde8496b0 (Pavel Begunkov 2020-07-13 23:37:12 +0300 1344) struct io_kiocb, timeout.list);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1345)
8eb7e2d007633 (Pavel Begunkov 2020-06-29 13:13:02 +0300 1346) if (io_is_timeout_noseq(req))
360428f8c0cd8 (Pavel Begunkov 2020-05-30 14:54:17 +0300 1347) break;
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1348)
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1349) /*
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1350) * Since seq can easily wrap around over time, subtract
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1351) * the last seq at which timeouts were flushed before comparing.
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1352) * Assuming not more than 2^31-1 events have happened since,
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1353) * these subtractions won't have wrapped, so we can check if
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1354) * target is in [last_seq, current_seq] by comparing the two.
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1355) */
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1356) events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1357) events_got = seq - ctx->cq_last_tm_flush;
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1358) if (events_got < events_needed)
360428f8c0cd8 (Pavel Begunkov 2020-05-30 14:54:17 +0300 1359) break;
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 1360)
135fcde8496b0 (Pavel Begunkov 2020-07-13 23:37:12 +0300 1361) list_del_init(&req->timeout.list);
1ee4160c73b21 (Pavel Begunkov 2021-03-25 18:32:42 +0000 1362) io_kill_timeout(req, 0);
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1363) } while (!list_empty(&ctx->timeout_list));
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1364)
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 1365) ctx->cq_last_tm_flush = seq;
360428f8c0cd8 (Pavel Begunkov 2020-05-30 14:54:17 +0300 1366) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 1367)
360428f8c0cd8 (Pavel Begunkov 2020-05-30 14:54:17 +0300 1368) static void io_commit_cqring(struct io_ring_ctx *ctx)
360428f8c0cd8 (Pavel Begunkov 2020-05-30 14:54:17 +0300 1369) {
360428f8c0cd8 (Pavel Begunkov 2020-05-30 14:54:17 +0300 1370) io_flush_timeouts(ctx);
ec30e04ba4a5c (Pavel Begunkov 2021-01-19 13:32:38 +0000 1371)
ec30e04ba4a5c (Pavel Begunkov 2021-01-19 13:32:38 +0000 1372) /* order cqe stores with ring update */
ec30e04ba4a5c (Pavel Begunkov 2021-01-19 13:32:38 +0000 1373) smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1374)
0451894522108 (Pavel Begunkov 2020-05-26 20:34:05 +0300 1375) if (unlikely(!list_empty(&ctx->defer_list)))
0451894522108 (Pavel Begunkov 2020-05-26 20:34:05 +0300 1376) __io_queue_deferred(ctx);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1377) }
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 1378)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 1379) static inline bool io_sqring_full(struct io_ring_ctx *ctx)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 1380) {
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 1381) struct io_rings *r = ctx->rings;
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 1382)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 1383) return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries;
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 1384) }
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 1385)
888aae2eeddfe (Pavel Begunkov 2021-01-19 13:32:39 +0000 1386) static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
888aae2eeddfe (Pavel Begunkov 2021-01-19 13:32:39 +0000 1387) {
888aae2eeddfe (Pavel Begunkov 2021-01-19 13:32:39 +0000 1388) return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
888aae2eeddfe (Pavel Begunkov 2021-01-19 13:32:39 +0000 1389) }
888aae2eeddfe (Pavel Begunkov 2021-01-19 13:32:39 +0000 1390)
8d13326e56c1a (Pavel Begunkov 2021-04-11 01:46:33 +0100 1391) static inline struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1392) {
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 1393) struct io_rings *rings = ctx->rings;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1394) unsigned tail;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1395)
115e12e58dbc0 (Stefan Bühler 2019-04-24 23:54:18 +0200 1396) /*
115e12e58dbc0 (Stefan Bühler 2019-04-24 23:54:18 +0200 1397) * writes to the cq entry need to come after reading head; the
115e12e58dbc0 (Stefan Bühler 2019-04-24 23:54:18 +0200 1398) * control dependency is enough as we're using WRITE_ONCE to
115e12e58dbc0 (Stefan Bühler 2019-04-24 23:54:18 +0200 1399) * fill the cq entry
115e12e58dbc0 (Stefan Bühler 2019-04-24 23:54:18 +0200 1400) */
888aae2eeddfe (Pavel Begunkov 2021-01-19 13:32:39 +0000 1401) if (__io_cqring_events(ctx) == rings->cq_ring_entries)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1402) return NULL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1403)
888aae2eeddfe (Pavel Begunkov 2021-01-19 13:32:39 +0000 1404) tail = ctx->cached_cq_tail++;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 1405) return &rings->cqes[tail & ctx->cq_mask];
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1406) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1407)
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 1408) static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 1409) {
44c769de6ffc3 (Pavel Begunkov 2021-04-11 01:46:31 +0100 1410) if (likely(!ctx->cq_ev_fd))
f0b493e6b9a89 (Jens Axboe 2020-02-01 21:30:11 -0700 1411) return false;
7e55a19cf6e70 (Stefano Garzarella 2020-05-15 18:38:05 +0200 1412) if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
7e55a19cf6e70 (Stefano Garzarella 2020-05-15 18:38:05 +0200 1413) return false;
44c769de6ffc3 (Pavel Begunkov 2021-04-11 01:46:31 +0100 1414) return !ctx->eventfd_async || io_wq_current_is_worker();
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 1415) }
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 1416)
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 1417) static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1418) {
b1445e59cc9a1 (Pavel Begunkov 2021-01-07 03:15:43 +0000 1419) /* see waitqueue_active() comment */
b1445e59cc9a1 (Pavel Begunkov 2021-01-07 03:15:43 +0000 1420) smp_mb();
b1445e59cc9a1 (Pavel Begunkov 2021-01-07 03:15:43 +0000 1421)
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1422) if (waitqueue_active(&ctx->wait))
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1423) wake_up(&ctx->wait);
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 1424) if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 1425) wake_up(&ctx->sq_data->wait);
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 1426) if (io_should_trigger_evfd(ctx))
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1427) eventfd_signal(ctx->cq_ev_fd, 1);
b1445e59cc9a1 (Pavel Begunkov 2021-01-07 03:15:43 +0000 1428) if (waitqueue_active(&ctx->cq_wait)) {
4aa84f2ffa81f (Pavel Begunkov 2021-01-07 03:15:42 +0000 1429) wake_up_interruptible(&ctx->cq_wait);
4aa84f2ffa81f (Pavel Begunkov 2021-01-07 03:15:42 +0000 1430) kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
4aa84f2ffa81f (Pavel Begunkov 2021-01-07 03:15:42 +0000 1431) }
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1432) }
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1433)
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1434) static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1435) {
b1445e59cc9a1 (Pavel Begunkov 2021-01-07 03:15:43 +0000 1436) /* see waitqueue_active() comment */
b1445e59cc9a1 (Pavel Begunkov 2021-01-07 03:15:43 +0000 1437) smp_mb();
b1445e59cc9a1 (Pavel Begunkov 2021-01-07 03:15:43 +0000 1438)
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1439) if (ctx->flags & IORING_SETUP_SQPOLL) {
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1440) if (waitqueue_active(&ctx->wait))
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1441) wake_up(&ctx->wait);
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1442) }
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1443) if (io_should_trigger_evfd(ctx))
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1444) eventfd_signal(ctx->cq_ev_fd, 1);
b1445e59cc9a1 (Pavel Begunkov 2021-01-07 03:15:43 +0000 1445) if (waitqueue_active(&ctx->cq_wait)) {
4aa84f2ffa81f (Pavel Begunkov 2021-01-07 03:15:42 +0000 1446) wake_up_interruptible(&ctx->cq_wait);
4aa84f2ffa81f (Pavel Begunkov 2021-01-07 03:15:42 +0000 1447) kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
4aa84f2ffa81f (Pavel Begunkov 2021-01-07 03:15:42 +0000 1448) }
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1449) }
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 1450)
c4a2ed72c9a61 (Jens Axboe 2019-11-21 21:01:26 -0700 1451) /* Returns true if there are no backlogged entries after the flush */
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1452) static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1453) {
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1454) struct io_rings *rings = ctx->rings;
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1455) unsigned long flags;
b18032bb0a883 (Jens Axboe 2021-01-24 16:58:56 -0700 1456) bool all_flushed, posted;
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1457)
e23de15fdbd30 (Pavel Begunkov 2020-12-17 00:24:37 +0000 1458) if (!force && __io_cqring_events(ctx) == rings->cq_ring_entries)
e23de15fdbd30 (Pavel Begunkov 2020-12-17 00:24:37 +0000 1459) return false;
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1460)
b18032bb0a883 (Jens Axboe 2021-01-24 16:58:56 -0700 1461) posted = false;
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1462) spin_lock_irqsave(&ctx->completion_lock, flags);
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1463) while (!list_empty(&ctx->cq_overflow_list)) {
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1464) struct io_uring_cqe *cqe = io_get_cqring(ctx);
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1465) struct io_overflow_cqe *ocqe;
e6c8aa9ac33bd (Jens Axboe 2020-09-28 13:10:13 -0600 1466)
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1467) if (!cqe && !force)
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1468) break;
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1469) ocqe = list_first_entry(&ctx->cq_overflow_list,
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1470) struct io_overflow_cqe, list);
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1471) if (cqe)
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1472) memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1473) else
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1474) WRITE_ONCE(ctx->rings->cq_overflow,
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1475) ++ctx->cached_cq_overflow);
b18032bb0a883 (Jens Axboe 2021-01-24 16:58:56 -0700 1476) posted = true;
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1477) list_del(&ocqe->list);
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1478) kfree(ocqe);
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1479) }
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1480)
09e88404f46cc (Pavel Begunkov 2020-12-17 00:24:38 +0000 1481) all_flushed = list_empty(&ctx->cq_overflow_list);
09e88404f46cc (Pavel Begunkov 2020-12-17 00:24:38 +0000 1482) if (all_flushed) {
09e88404f46cc (Pavel Begunkov 2020-12-17 00:24:38 +0000 1483) clear_bit(0, &ctx->sq_check_overflow);
09e88404f46cc (Pavel Begunkov 2020-12-17 00:24:38 +0000 1484) clear_bit(0, &ctx->cq_check_overflow);
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 1485) WRITE_ONCE(ctx->rings->sq_flags,
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 1486) ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW);
09e88404f46cc (Pavel Begunkov 2020-12-17 00:24:38 +0000 1487) }
4693014340808 (Pavel Begunkov 2020-07-30 18:43:49 +0300 1488)
b18032bb0a883 (Jens Axboe 2021-01-24 16:58:56 -0700 1489) if (posted)
b18032bb0a883 (Jens Axboe 2021-01-24 16:58:56 -0700 1490) io_commit_cqring(ctx);
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1491) spin_unlock_irqrestore(&ctx->completion_lock, flags);
b18032bb0a883 (Jens Axboe 2021-01-24 16:58:56 -0700 1492) if (posted)
b18032bb0a883 (Jens Axboe 2021-01-24 16:58:56 -0700 1493) io_cqring_ev_posted(ctx);
09e88404f46cc (Pavel Begunkov 2020-12-17 00:24:38 +0000 1494) return all_flushed;
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1495) }
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1496)
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1497) static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1498) {
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 1499) bool ret = true;
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 1500)
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1501) if (test_bit(0, &ctx->cq_check_overflow)) {
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1502) /* iopoll syncs against uring_lock, not completion_lock */
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1503) if (ctx->flags & IORING_SETUP_IOPOLL)
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1504) mutex_lock(&ctx->uring_lock);
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 1505) ret = __io_cqring_overflow_flush(ctx, force);
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1506) if (ctx->flags & IORING_SETUP_IOPOLL)
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1507) mutex_unlock(&ctx->uring_lock);
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1508) }
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 1509)
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 1510) return ret;
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1511) }
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 1512)
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1513) /*
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1514) * Shamelessly stolen from the mm implementation of page reference checking,
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1515) * see commit f958d7b528b1 for details.
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1516) */
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1517) #define req_ref_zero_or_close_to_overflow(req) \
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1518) ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1519)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1520) static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1521) {
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1522) return atomic_inc_not_zero(&req->refs);
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1523) }
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1524)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1525) static inline bool req_ref_sub_and_test(struct io_kiocb *req, int refs)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1526) {
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1527) WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1528) return atomic_sub_and_test(refs, &req->refs);
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1529) }
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1530)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1531) static inline bool req_ref_put_and_test(struct io_kiocb *req)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1532) {
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1533) WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1534) return atomic_dec_and_test(&req->refs);
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1535) }
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1536)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1537) static inline void req_ref_put(struct io_kiocb *req)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1538) {
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1539) WARN_ON_ONCE(req_ref_put_and_test(req));
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1540) }
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1541)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1542) static inline void req_ref_get(struct io_kiocb *req)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1543) {
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1544) WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 1545) atomic_inc(&req->refs);
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1546) }
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1547)
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1548) static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1549) long res, unsigned int cflags)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1550) {
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1551) struct io_overflow_cqe *ocqe;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1552)
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1553) ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1554) if (!ocqe) {
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1555) /*
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1556) * If we're in ring overflow flush mode, or in task cancel mode,
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1557) * or cannot allocate an overflow entry, then we need to drop it
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1558) * on the floor.
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1559) */
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1560) WRITE_ONCE(ctx->rings->cq_overflow, ++ctx->cached_cq_overflow);
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1561) return false;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1562) }
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1563) if (list_empty(&ctx->cq_overflow_list)) {
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1564) set_bit(0, &ctx->sq_check_overflow);
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1565) set_bit(0, &ctx->cq_check_overflow);
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 1566) WRITE_ONCE(ctx->rings->sq_flags,
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 1567) ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW);
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 1568)
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1569) }
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1570) ocqe->cqe.user_data = user_data;
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1571) ocqe->cqe.res = res;
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1572) ocqe->cqe.flags = cflags;
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1573) list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
cce4b8b0ce1f9 (Pavel Begunkov 2021-04-13 02:58:44 +0100 1574) return true;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1575) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1576)
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1577) static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1578) long res, unsigned int cflags)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1579) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1580) struct io_uring_cqe *cqe;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1581)
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1582) trace_io_uring_complete(ctx, user_data, res, cflags);
51c3ff62cac63 (Jens Axboe 2019-11-03 06:52:50 -0700 1583)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1584) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1585) * If we can't get a cq entry, userspace overflowed the
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1586) * submission (by quite a lot). Increment the overflow count in
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1587) * the ring.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1588) */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1589) cqe = io_get_cqring(ctx);
1d7bb1d50fb4d (Jens Axboe 2019-11-06 11:31:17 -0700 1590) if (likely(cqe)) {
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1591) WRITE_ONCE(cqe->user_data, user_data);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1592) WRITE_ONCE(cqe->res, res);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 1593) WRITE_ONCE(cqe->flags, cflags);
8d13326e56c1a (Pavel Begunkov 2021-04-11 01:46:33 +0100 1594) return true;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1595) }
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1596) return io_cqring_event_overflow(ctx, user_data, res, cflags);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1597) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1598)
8d13326e56c1a (Pavel Begunkov 2021-04-11 01:46:33 +0100 1599) /* not as hot to bloat with inlining */
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1600) static noinline bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1601) long res, unsigned int cflags)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 1602) {
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1603) return __io_cqring_fill_event(ctx, user_data, res, cflags);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 1604) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 1605)
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1606) static void io_req_complete_post(struct io_kiocb *req, long res,
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1607) unsigned int cflags)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1608) {
78e19bbef3836 (Jens Axboe 2019-11-06 15:21:34 -0700 1609) struct io_ring_ctx *ctx = req->ctx;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1610) unsigned long flags;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1611)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1612) spin_lock_irqsave(&ctx->completion_lock, flags);
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1613) __io_cqring_fill_event(ctx, req->user_data, res, cflags);
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1614) /*
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1615) * If we're the last reference to this request, add to our locked
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1616) * free_list cache.
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1617) */
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 1618) if (req_ref_put_and_test(req)) {
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1619) if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1620) if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL_LINK))
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1621) io_disarm_next(req);
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1622) if (req->link) {
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1623) io_req_task_queue(req->link);
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1624) req->link = NULL;
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1625) }
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1626) }
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1627) io_dismantle_req(req);
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1628) io_put_task(req->task, 1);
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 1629) list_add(&req->compl.list, &ctx->locked_free_list);
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 1630) ctx->locked_free_nr++;
180f829fe4026 (Pavel Begunkov 2021-03-14 20:57:09 +0000 1631) } else {
180f829fe4026 (Pavel Begunkov 2021-03-14 20:57:09 +0000 1632) if (!percpu_ref_tryget(&ctx->refs))
180f829fe4026 (Pavel Begunkov 2021-03-14 20:57:09 +0000 1633) req = NULL;
180f829fe4026 (Pavel Begunkov 2021-03-14 20:57:09 +0000 1634) }
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1635) io_commit_cqring(ctx);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1636) spin_unlock_irqrestore(&ctx->completion_lock, flags);
7a612350a9898 (Pavel Begunkov 2021-03-09 00:37:59 +0000 1637)
180f829fe4026 (Pavel Begunkov 2021-03-14 20:57:09 +0000 1638) if (req) {
180f829fe4026 (Pavel Begunkov 2021-03-14 20:57:09 +0000 1639) io_cqring_ev_posted(ctx);
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1640) percpu_ref_put(&ctx->refs);
180f829fe4026 (Pavel Begunkov 2021-03-14 20:57:09 +0000 1641) }
229a7b63507a3 (Jens Axboe 2020-06-22 10:13:11 -0600 1642) }
229a7b63507a3 (Jens Axboe 2020-06-22 10:13:11 -0600 1643)
4e3d9ff905cd3 (Jens Axboe 2021-04-15 17:44:34 -0600 1644) static inline bool io_req_needs_clean(struct io_kiocb *req)
4e3d9ff905cd3 (Jens Axboe 2021-04-15 17:44:34 -0600 1645) {
75652a30ff675 (Jens Axboe 2021-04-15 09:52:40 -0600 1646) return req->flags & (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP |
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 1647) REQ_F_POLLED | REQ_F_INFLIGHT);
4e3d9ff905cd3 (Jens Axboe 2021-04-15 17:44:34 -0600 1648) }
4e3d9ff905cd3 (Jens Axboe 2021-04-15 17:44:34 -0600 1649)
a38d68db6742c (Pavel Begunkov 2021-01-19 13:32:45 +0000 1650) static void io_req_complete_state(struct io_kiocb *req, long res,
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 1651) unsigned int cflags)
229a7b63507a3 (Jens Axboe 2020-06-22 10:13:11 -0600 1652) {
4e3d9ff905cd3 (Jens Axboe 2021-04-15 17:44:34 -0600 1653) if (io_req_needs_clean(req))
68fb897966feb (Pavel Begunkov 2021-03-19 17:22:41 +0000 1654) io_clean_op(req);
a38d68db6742c (Pavel Begunkov 2021-01-19 13:32:45 +0000 1655) req->result = res;
a38d68db6742c (Pavel Begunkov 2021-01-19 13:32:45 +0000 1656) req->compl.cflags = cflags;
e342c807f556d (Pavel Begunkov 2021-01-19 13:32:47 +0000 1657) req->flags |= REQ_F_COMPLETE_INLINE;
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 1658) }
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 1659)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 1660) static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 1661) long res, unsigned cflags)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 1662) {
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 1663) if (issue_flags & IO_URING_F_COMPLETE_DEFER)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 1664) io_req_complete_state(req, res, cflags);
a38d68db6742c (Pavel Begunkov 2021-01-19 13:32:45 +0000 1665) else
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1666) io_req_complete_post(req, res, cflags);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 1667) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 1668)
a38d68db6742c (Pavel Begunkov 2021-01-19 13:32:45 +0000 1669) static inline void io_req_complete(struct io_kiocb *req, long res)
0ddf92e848ab7 (Jens Axboe 2019-11-08 08:52:53 -0700 1670) {
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 1671) __io_req_complete(req, 0, res, 0);
0ddf92e848ab7 (Jens Axboe 2019-11-08 08:52:53 -0700 1672) }
0ddf92e848ab7 (Jens Axboe 2019-11-08 08:52:53 -0700 1673)
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 1674) static void io_req_complete_failed(struct io_kiocb *req, long res)
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 1675) {
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 1676) req_set_fail_links(req);
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 1677) io_put_req(req);
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 1678) io_req_complete_post(req, res, 0);
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 1679) }
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 1680)
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 1681) static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 1682) struct io_comp_state *cs)
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 1683) {
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 1684) spin_lock_irq(&ctx->completion_lock);
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 1685) list_splice_init(&ctx->locked_free_list, &cs->free_list);
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 1686) ctx->locked_free_nr = 0;
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 1687) spin_unlock_irq(&ctx->completion_lock);
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 1688) }
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 1689)
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1690) /* Returns true IFF there are requests in the cache */
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1691) static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
0ddf92e848ab7 (Jens Axboe 2019-11-08 08:52:53 -0700 1692) {
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1693) struct io_submit_state *state = &ctx->submit_state;
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1694) struct io_comp_state *cs = &state->comp;
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1695) int nr;
0ddf92e848ab7 (Jens Axboe 2019-11-08 08:52:53 -0700 1696)
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1697) /*
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1698) * If we have more than a batch's worth of requests in our IRQ side
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1699) * locked cache, grab the lock and move them over to our submission
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1700) * side cache.
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1701) */
16eadf8979564 (Pavel Begunkov 2021-05-16 22:58:12 +0100 1702) if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 1703) io_flush_cached_locked_reqs(ctx, cs);
0ddf92e848ab7 (Jens Axboe 2019-11-08 08:52:53 -0700 1704)
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1705) nr = state->free_reqs;
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1706) while (!list_empty(&cs->free_list)) {
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1707) struct io_kiocb *req = list_first_entry(&cs->free_list,
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1708) struct io_kiocb, compl.list);
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1709)
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 1710) list_del(&req->compl.list);
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1711) state->reqs[nr++] = req;
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1712) if (nr == ARRAY_SIZE(state->reqs))
e5d1bc0a91f16 (Pavel Begunkov 2021-02-10 00:03:23 +0000 1713) break;
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 1714) }
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 1715)
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1716) state->free_reqs = nr;
dd78f49260dd4 (Pavel Begunkov 2021-03-19 17:22:35 +0000 1717) return nr != 0;
0ddf92e848ab7 (Jens Axboe 2019-11-08 08:52:53 -0700 1718) }
0ddf92e848ab7 (Jens Axboe 2019-11-08 08:52:53 -0700 1719)
e5d1bc0a91f16 (Pavel Begunkov 2021-02-10 00:03:23 +0000 1720) static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1721) {
e5d1bc0a91f16 (Pavel Begunkov 2021-02-10 00:03:23 +0000 1722) struct io_submit_state *state = &ctx->submit_state;
e5d1bc0a91f16 (Pavel Begunkov 2021-02-10 00:03:23 +0000 1723)
501449420a42c (Pavel Begunkov 2021-06-24 15:09:57 +0100 1724) BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH);
e5d1bc0a91f16 (Pavel Begunkov 2021-02-10 00:03:23 +0000 1725)
f6b6c7d6a9600 (Pavel Begunkov 2020-06-21 13:09:53 +0300 1726) if (!state->free_reqs) {
291b2821e072e (Pavel Begunkov 2020-09-30 22:57:01 +0300 1727) gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
2579f913d41a0 (Jens Axboe 2019-01-09 09:10:43 -0700 1728) int ret;
2579f913d41a0 (Jens Axboe 2019-01-09 09:10:43 -0700 1729)
c7dae4ba46c9d (Jens Axboe 2021-02-09 19:53:37 -0700 1730) if (io_flush_cached_reqs(ctx))
e5d1bc0a91f16 (Pavel Begunkov 2021-02-10 00:03:23 +0000 1731) goto got_req;
e5d1bc0a91f16 (Pavel Begunkov 2021-02-10 00:03:23 +0000 1732)
bf019da7fcbe7 (Pavel Begunkov 2021-02-10 00:03:17 +0000 1733) ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
bf019da7fcbe7 (Pavel Begunkov 2021-02-10 00:03:17 +0000 1734) state->reqs);
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1735)
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1736) /*
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1737) * Bulk alloc is all-or-nothing. If we fail to get a batch,
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1738) * retry single alloc to be on the safe side.
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1739) */
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1740) if (unlikely(ret <= 0)) {
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1741) state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1742) if (!state->reqs[0])
3893f39f2245e (Pavel Begunkov 2021-02-10 00:03:15 +0000 1743) return NULL;
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1744) ret = 1;
fd6fab2cb78d3 (Jens Axboe 2019-03-14 16:30:06 -0600 1745) }
291b2821e072e (Pavel Begunkov 2020-09-30 22:57:01 +0300 1746) state->free_reqs = ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1747) }
e5d1bc0a91f16 (Pavel Begunkov 2021-02-10 00:03:23 +0000 1748) got_req:
291b2821e072e (Pavel Begunkov 2020-09-30 22:57:01 +0300 1749) state->free_reqs--;
291b2821e072e (Pavel Begunkov 2020-09-30 22:57:01 +0300 1750) return state->reqs[state->free_reqs];
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1751) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1752)
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 1753) static inline void io_put_file(struct file *file)
8da11c19940dd (Pavel Begunkov 2020-02-24 11:32:44 +0300 1754) {
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 1755) if (file)
8da11c19940dd (Pavel Begunkov 2020-02-24 11:32:44 +0300 1756) fput(file);
8da11c19940dd (Pavel Begunkov 2020-02-24 11:32:44 +0300 1757) }
8da11c19940dd (Pavel Begunkov 2020-02-24 11:32:44 +0300 1758)
4edf20f999023 (Pavel Begunkov 2020-10-13 09:43:59 +0100 1759) static void io_dismantle_req(struct io_kiocb *req)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 1760) {
094bae49e5ed9 (Pavel Begunkov 2021-03-19 17:22:42 +0000 1761) unsigned int flags = req->flags;
929a3af90f0f4 (Pavel Begunkov 2020-02-19 00:19:09 +0300 1762)
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 1763) if (io_req_needs_clean(req))
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 1764) io_clean_op(req);
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 1765) if (!(flags & REQ_F_FIXED_FILE))
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 1766) io_put_file(req->file);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 1767) if (req->fixed_rsrc_refs)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 1768) percpu_ref_put(req->fixed_rsrc_refs);
094bae49e5ed9 (Pavel Begunkov 2021-03-19 17:22:42 +0000 1769) if (req->async_data)
094bae49e5ed9 (Pavel Begunkov 2021-03-19 17:22:42 +0000 1770) kfree(req->async_data);
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 1771) if (req->creds) {
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 1772) put_cred(req->creds);
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 1773) req->creds = NULL;
003e8dccdb227 (Jens Axboe 2021-03-06 09:22:27 -0700 1774) }
e65ef56db4945 (Jens Axboe 2019-03-12 10:16:44 -0600 1775) }
e65ef56db4945 (Jens Axboe 2019-03-12 10:16:44 -0600 1776)
b23fcf477f851 (Pavel Begunkov 2021-03-01 18:20:48 +0000 1777) /* must to be called somewhat shortly after putting a request */
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1778) static inline void io_put_task(struct task_struct *task, int nr)
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1779) {
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1780) struct io_uring_task *tctx = task->io_uring;
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1781)
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1782) percpu_counter_sub(&tctx->inflight, nr);
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1783) if (unlikely(atomic_read(&tctx->in_idle)))
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1784) wake_up(&tctx->wait);
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1785) put_task_struct_many(task, nr);
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1786) }
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1787)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 1788) static void __io_free_req(struct io_kiocb *req)
c6ca97b30c47c (Jens Axboe 2019-12-28 12:11:08 -0700 1789) {
51a4cc112c7a4 (Jens Axboe 2020-08-10 10:55:56 -0600 1790) struct io_ring_ctx *ctx = req->ctx;
c6ca97b30c47c (Jens Axboe 2019-12-28 12:11:08 -0700 1791)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 1792) io_dismantle_req(req);
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 1793) io_put_task(req->task, 1);
c6ca97b30c47c (Jens Axboe 2019-12-28 12:11:08 -0700 1794)
3893f39f2245e (Pavel Begunkov 2021-02-10 00:03:15 +0000 1795) kmem_cache_free(req_cachep, req);
ecfc51777487d (Pavel Begunkov 2020-06-29 13:13:03 +0300 1796) percpu_ref_put(&ctx->refs);
e65ef56db4945 (Jens Axboe 2019-03-12 10:16:44 -0600 1797) }
e65ef56db4945 (Jens Axboe 2019-03-12 10:16:44 -0600 1798)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1799) static inline void io_remove_next_linked(struct io_kiocb *req)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1800) {
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1801) struct io_kiocb *nxt = req->link;
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1802)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1803) req->link = nxt->link;
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1804) nxt->link = NULL;
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1805) }
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1806)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1807) static bool io_kill_linked_timeout(struct io_kiocb *req)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1808) __must_hold(&req->ctx->completion_lock)
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 1809) {
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1810) struct io_kiocb *link = req->link;
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1811)
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 1812) /*
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 1813) * Can happen if a linked timeout fired and link had been like
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 1814) * req -> link t-out -> link t-out [-> ...]
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 1815) */
c9abd7ad832b9 (Pavel Begunkov 2020-10-22 16:43:11 +0100 1816) if (link && (link->flags & REQ_F_LTIMEOUT_ACTIVE)) {
c9abd7ad832b9 (Pavel Begunkov 2020-10-22 16:43:11 +0100 1817) struct io_timeout_data *io = link->async_data;
7c86ffeeed303 (Pavel Begunkov 2020-06-29 13:12:59 +0300 1818)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1819) io_remove_next_linked(req);
90cd7e424969d (Pavel Begunkov 2020-10-27 23:25:36 +0000 1820) link->timeout.head = NULL;
fd9c7bc542dae (Pavel Begunkov 2021-04-13 02:58:42 +0100 1821) if (hrtimer_try_to_cancel(&io->timer) != -1) {
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1822) io_cqring_fill_event(link->ctx, link->user_data,
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1823) -ECANCELED, 0);
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1824) io_put_req_deferred(link, 1);
d4729fbde7665 (Pavel Begunkov 2021-03-22 01:58:24 +0000 1825) return true;
c9abd7ad832b9 (Pavel Begunkov 2020-10-22 16:43:11 +0100 1826) }
c9abd7ad832b9 (Pavel Begunkov 2020-10-22 16:43:11 +0100 1827) }
d4729fbde7665 (Pavel Begunkov 2021-03-22 01:58:24 +0000 1828) return false;
7c86ffeeed303 (Pavel Begunkov 2020-06-29 13:12:59 +0300 1829) }
7c86ffeeed303 (Pavel Begunkov 2020-06-29 13:12:59 +0300 1830)
d148ca4b07d05 (Pavel Begunkov 2020-10-18 10:17:39 +0100 1831) static void io_fail_links(struct io_kiocb *req)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1832) __must_hold(&req->ctx->completion_lock)
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1833) {
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1834) struct io_kiocb *nxt, *link = req->link;
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1835)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1836) req->link = NULL;
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1837) while (link) {
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1838) nxt = link->link;
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1839) link->link = NULL;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 1840)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1841) trace_io_uring_fail_link(req, link);
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 1842) io_cqring_fill_event(link->ctx, link->user_data, -ECANCELED, 0);
1575f21a09206 (Jens Axboe 2021-02-27 15:20:49 -0700 1843) io_put_req_deferred(link, 2);
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1844) link = nxt;
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1845) }
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1846) }
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1847)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1848) static bool io_disarm_next(struct io_kiocb *req)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1849) __must_hold(&req->ctx->completion_lock)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1850) {
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1851) bool posted = false;
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1852)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1853) if (likely(req->flags & REQ_F_LINK_TIMEOUT))
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1854) posted = io_kill_linked_timeout(req);
e4335ed33eb54 (Pavel Begunkov 2021-04-11 01:46:39 +0100 1855) if (unlikely((req->flags & REQ_F_FAIL_LINK) &&
e4335ed33eb54 (Pavel Begunkov 2021-04-11 01:46:39 +0100 1856) !(req->flags & REQ_F_HARDLINK))) {
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1857) posted |= (req->link != NULL);
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1858) io_fail_links(req);
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1859) }
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1860) return posted;
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1861) }
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1862)
3fa5e0f331280 (Pavel Begunkov 2020-06-30 15:20:43 +0300 1863) static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
c69f8dbe2426c (Jackie Liu 2019-11-09 11:00:08 +0800 1864) {
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1865) struct io_kiocb *nxt;
944e58bfeda0e (Pavel Begunkov 2019-11-21 23:21:01 +0300 1866)
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1867) /*
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1868) * If LINK is set, we have dependent requests in this chain. If we
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1869) * didn't fail this request, queue the first one up, moving any other
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1870) * dependencies to the next request. In case of failure, fail the rest
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1871) * of the chain.
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1872) */
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1873) if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL_LINK)) {
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1874) struct io_ring_ctx *ctx = req->ctx;
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1875) unsigned long flags;
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1876) bool posted;
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1877)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1878) spin_lock_irqsave(&ctx->completion_lock, flags);
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1879) posted = io_disarm_next(req);
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1880) if (posted)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1881) io_commit_cqring(req->ctx);
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1882) spin_unlock_irqrestore(&ctx->completion_lock, flags);
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1883) if (posted)
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1884) io_cqring_ev_posted(ctx);
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1885) }
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1886) nxt = req->link;
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1887) req->link = NULL;
33cc89a9fc248 (Pavel Begunkov 2021-03-09 00:37:58 +0000 1888) return nxt;
4d7dd46297140 (Jens Axboe 2019-11-20 13:03:52 -0700 1889) }
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 1890)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 1891) static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
3fa5e0f331280 (Pavel Begunkov 2020-06-30 15:20:43 +0300 1892) {
cdbff98223330 (Pavel Begunkov 2021-02-12 18:41:16 +0000 1893) if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
3fa5e0f331280 (Pavel Begunkov 2020-06-30 15:20:43 +0300 1894) return NULL;
3fa5e0f331280 (Pavel Begunkov 2020-06-30 15:20:43 +0300 1895) return __io_req_find_next(req);
3fa5e0f331280 (Pavel Begunkov 2020-06-30 15:20:43 +0300 1896) }
3fa5e0f331280 (Pavel Begunkov 2020-06-30 15:20:43 +0300 1897)
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1898) static void ctx_flush_and_put(struct io_ring_ctx *ctx)
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1899) {
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1900) if (!ctx)
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1901) return;
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1902) if (ctx->submit_state.comp.nr) {
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1903) mutex_lock(&ctx->uring_lock);
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1904) io_submit_flush_completions(&ctx->submit_state.comp, ctx);
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1905) mutex_unlock(&ctx->uring_lock);
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1906) }
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1907) percpu_ref_put(&ctx->refs);
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1908) }
2c32395d81110 (Pavel Begunkov 2021-02-28 22:04:53 +0000 1909)
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1910) static void tctx_task_work(struct callback_head *cb)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 1911) {
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1912) struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1913) task_work);
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 1914)
1d5f360dd1a3c (Jens Axboe 2021-02-26 14:54:16 -0700 1915) clear_bit(0, &tctx->task_state);
1d5f360dd1a3c (Jens Axboe 2021-02-26 14:54:16 -0700 1916)
e95d994d2f8fc (Jens Axboe 2021-07-26 10:42:56 -0600 1917) while (true) {
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1918) struct io_ring_ctx *ctx = NULL;
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1919) struct io_wq_work_list list;
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1920) struct io_wq_work_node *node;
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1921)
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1922) spin_lock_irq(&tctx->task_lock);
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1923) list = tctx->task_list;
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1924) INIT_WQ_LIST(&tctx->task_list);
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1925) spin_unlock_irq(&tctx->task_lock);
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1926)
e95d994d2f8fc (Jens Axboe 2021-07-26 10:42:56 -0600 1927) if (wq_list_empty(&list))
e95d994d2f8fc (Jens Axboe 2021-07-26 10:42:56 -0600 1928) break;
e95d994d2f8fc (Jens Axboe 2021-07-26 10:42:56 -0600 1929)
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1930) node = list.first;
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1931) while (node) {
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1932) struct io_wq_work_node *next = node->next;
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1933) struct io_kiocb *req = container_of(node, struct io_kiocb,
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1934) io_task_work.node);
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1935)
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1936) if (req->ctx != ctx) {
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1937) ctx_flush_and_put(ctx);
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1938) ctx = req->ctx;
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1939) percpu_ref_get(&ctx->refs);
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1940) }
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1941) req->task_work.func(&req->task_work);
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1942) node = next;
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1943) }
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1944)
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1945) ctx_flush_and_put(ctx);
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1946) if (!list.first)
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1947) break;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1948) cond_resched();
dafcb53b550b0 (Pavel Begunkov 2021-06-17 18:14:06 +0100 1949) }
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1950) }
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1951)
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1952) static int io_req_task_work_add(struct io_kiocb *req)
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1953) {
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1954) struct task_struct *tsk = req->task;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1955) struct io_uring_task *tctx = tsk->io_uring;
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1956) enum task_work_notify_mode notify;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1957) struct io_wq_work_node *node, *prev;
0b81e80c813f9 (Jens Axboe 2021-02-16 10:33:53 -0700 1958) unsigned long flags;
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1959) int ret = 0;
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1960)
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1961) if (unlikely(tsk->flags & PF_EXITING))
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1962) return -ESRCH;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1963)
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1964) WARN_ON_ONCE(!tctx);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1965)
0b81e80c813f9 (Jens Axboe 2021-02-16 10:33:53 -0700 1966) spin_lock_irqsave(&tctx->task_lock, flags);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1967) wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
0b81e80c813f9 (Jens Axboe 2021-02-16 10:33:53 -0700 1968) spin_unlock_irqrestore(&tctx->task_lock, flags);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1969)
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1970) /* task_work already pending, we're done */
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1971) if (test_bit(0, &tctx->task_state) ||
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1972) test_and_set_bit(0, &tctx->task_state))
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1973) return 0;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1974)
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1975) /*
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1976) * SQPOLL kernel thread doesn't need notification, just a wakeup. For
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1977) * all other cases, use TWA_SIGNAL unconditionally to ensure we're
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1978) * processing task_work. There's no reliable way to tell if TWA_RESUME
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1979) * will do the job.
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1980) */
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1981) notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1982)
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1983) if (!task_work_add(tsk, &tctx->task_work, notify)) {
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1984) wake_up_process(tsk);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1985) return 0;
c15b79dee51bd (Pavel Begunkov 2021-03-19 17:22:44 +0000 1986) }
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1987)
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1988) /*
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1989) * Slow path - we failed, find and delete work. if the work is not
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1990) * in the list, it got run and we're fine.
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1991) */
0b81e80c813f9 (Jens Axboe 2021-02-16 10:33:53 -0700 1992) spin_lock_irqsave(&tctx->task_lock, flags);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1993) wq_list_for_each(node, prev, &tctx->task_list) {
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1994) if (&req->io_task_work.node == node) {
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1995) wq_list_del(&tctx->task_list, node, prev);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1996) ret = 1;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1997) break;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1998) }
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 1999) }
0b81e80c813f9 (Jens Axboe 2021-02-16 10:33:53 -0700 2000) spin_unlock_irqrestore(&tctx->task_lock, flags);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 2001) clear_bit(0, &tctx->task_state);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 2002) return ret;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 2003) }
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 2004)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2005) static bool io_run_task_work_head(struct callback_head **work_head)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2006) {
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2007) struct callback_head *work, *next;
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2008) bool executed = false;
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2009)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2010) do {
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2011) work = xchg(work_head, NULL);
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2012) if (!work)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2013) break;
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2014)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2015) do {
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2016) next = work->next;
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2017) work->func(work);
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2018) work = next;
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2019) cond_resched();
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2020) } while (work);
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2021) executed = true;
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2022) } while (1);
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2023)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2024) return executed;
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2025) }
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2026)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2027) static void io_task_work_add_head(struct callback_head **work_head,
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2028) struct callback_head *task_work)
eab30c4d20dc7 (Pavel Begunkov 2021-01-19 13:32:42 +0000 2029) {
7c25c0d16ef3c (Jens Axboe 2021-02-16 07:17:00 -0700 2030) struct callback_head *head;
eab30c4d20dc7 (Pavel Begunkov 2021-01-19 13:32:42 +0000 2031)
7c25c0d16ef3c (Jens Axboe 2021-02-16 07:17:00 -0700 2032) do {
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2033) head = READ_ONCE(*work_head);
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2034) task_work->next = head;
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2035) } while (cmpxchg(work_head, head, task_work) != head);
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2036) }
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2037)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2038) static void io_req_task_work_add_fallback(struct io_kiocb *req,
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2039) task_work_func_t cb)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2040) {
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2041) init_task_work(&req->task_work, cb);
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 2042) io_task_work_add_head(&req->ctx->exit_task_work, &req->task_work);
eab30c4d20dc7 (Pavel Begunkov 2021-01-19 13:32:42 +0000 2043) }
eab30c4d20dc7 (Pavel Begunkov 2021-01-19 13:32:42 +0000 2044)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2045) static void io_req_task_cancel(struct callback_head *cb)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2046) {
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2047) struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
87ceb6a6b81ec (Jens Axboe 2020-09-14 08:20:12 -0600 2048) struct io_ring_ctx *ctx = req->ctx;
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2049)
e83acd7d37d83 (Pavel Begunkov 2021-02-28 22:35:09 +0000 2050) /* ctx is guaranteed to stay alive while we hold uring_lock */
792bb6eb86233 (Pavel Begunkov 2021-02-18 22:32:51 +0000 2051) mutex_lock(&ctx->uring_lock);
2593553a01c80 (Pavel Begunkov 2021-03-19 17:22:40 +0000 2052) io_req_complete_failed(req, req->result);
792bb6eb86233 (Pavel Begunkov 2021-02-18 22:32:51 +0000 2053) mutex_unlock(&ctx->uring_lock);
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2054) }
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2055)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2056) static void __io_req_task_submit(struct io_kiocb *req)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2057) {
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2058) struct io_ring_ctx *ctx = req->ctx;
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2059)
04fc6c802dfac (Pavel Begunkov 2021-02-12 03:23:54 +0000 2060) /* ctx stays valid until unlock, even if we drop all ours ctx->refs */
81b6d05ccad4f (Pavel Begunkov 2021-01-04 20:36:35 +0000 2061) mutex_lock(&ctx->uring_lock);
08e1e3b6ddd89 (Pavel Begunkov 2021-07-10 02:45:59 +0100 2062) if (!(req->task->flags & PF_EXITING) && !req->task->in_execve)
c5eef2b9449ba (Pavel Begunkov 2021-02-10 00:03:22 +0000 2063) __io_queue_sqe(req);
81b6d05ccad4f (Pavel Begunkov 2021-01-04 20:36:35 +0000 2064) else
2593553a01c80 (Pavel Begunkov 2021-03-19 17:22:40 +0000 2065) io_req_complete_failed(req, -EFAULT);
81b6d05ccad4f (Pavel Begunkov 2021-01-04 20:36:35 +0000 2066) mutex_unlock(&ctx->uring_lock);
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2067) }
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2068)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2069) static void io_req_task_submit(struct callback_head *cb)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2070) {
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2071) struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2072)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2073) __io_req_task_submit(req);
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2074) }
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2075)
2c4b8eb6435e6 (Pavel Begunkov 2021-02-28 22:35:10 +0000 2076) static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2077) {
2c4b8eb6435e6 (Pavel Begunkov 2021-02-28 22:35:10 +0000 2078) req->result = ret;
2c4b8eb6435e6 (Pavel Begunkov 2021-02-28 22:35:10 +0000 2079) req->task_work.func = io_req_task_cancel;
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2080)
2c4b8eb6435e6 (Pavel Begunkov 2021-02-28 22:35:10 +0000 2081) if (unlikely(io_req_task_work_add(req)))
eab30c4d20dc7 (Pavel Begunkov 2021-01-19 13:32:42 +0000 2082) io_req_task_work_add_fallback(req, io_req_task_cancel);
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2083) }
c40f63790ec95 (Jens Axboe 2020-06-25 15:39:59 -0600 2084)
2c4b8eb6435e6 (Pavel Begunkov 2021-02-28 22:35:10 +0000 2085) static void io_req_task_queue(struct io_kiocb *req)
a3df769899c0b (Pavel Begunkov 2021-02-18 22:32:52 +0000 2086) {
2c4b8eb6435e6 (Pavel Begunkov 2021-02-28 22:35:10 +0000 2087) req->task_work.func = io_req_task_submit;
a3df769899c0b (Pavel Begunkov 2021-02-18 22:32:52 +0000 2088)
a3df769899c0b (Pavel Begunkov 2021-02-18 22:32:52 +0000 2089) if (unlikely(io_req_task_work_add(req)))
2c4b8eb6435e6 (Pavel Begunkov 2021-02-28 22:35:10 +0000 2090) io_req_task_queue_fail(req, -ECANCELED);
a3df769899c0b (Pavel Begunkov 2021-02-18 22:32:52 +0000 2091) }
a3df769899c0b (Pavel Begunkov 2021-02-18 22:32:52 +0000 2092)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 2093) static inline void io_queue_next(struct io_kiocb *req)
c69f8dbe2426c (Jackie Liu 2019-11-09 11:00:08 +0800 2094) {
9b5f7bd932726 (Pavel Begunkov 2020-06-29 13:13:00 +0300 2095) struct io_kiocb *nxt = io_req_find_next(req);
944e58bfeda0e (Pavel Begunkov 2019-11-21 23:21:01 +0300 2096)
944e58bfeda0e (Pavel Begunkov 2019-11-21 23:21:01 +0300 2097) if (nxt)
906a8c3fdbc36 (Pavel Begunkov 2020-06-27 14:04:55 +0300 2098) io_req_task_queue(nxt);
c69f8dbe2426c (Jackie Liu 2019-11-09 11:00:08 +0800 2099) }
c69f8dbe2426c (Jackie Liu 2019-11-09 11:00:08 +0800 2100)
c3524383333e4 (Pavel Begunkov 2020-06-28 12:52:32 +0300 2101) static void io_free_req(struct io_kiocb *req)
7a743e225b2a9 (Pavel Begunkov 2020-03-03 21:33:13 +0300 2102) {
c3524383333e4 (Pavel Begunkov 2020-06-28 12:52:32 +0300 2103) io_queue_next(req);
c3524383333e4 (Pavel Begunkov 2020-06-28 12:52:32 +0300 2104) __io_free_req(req);
c3524383333e4 (Pavel Begunkov 2020-06-28 12:52:32 +0300 2105) }
8766dd516c535 (Pavel Begunkov 2020-03-14 00:31:04 +0300 2106)
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2107) struct req_batch {
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2108) struct task_struct *task;
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2109) int task_refs;
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 2110) int ctx_refs;
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2111) };
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2112)
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2113) static inline void io_init_req_batch(struct req_batch *rb)
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2114) {
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2115) rb->task_refs = 0;
9ae7246321d2b (Pavel Begunkov 2021-02-10 00:03:16 +0000 2116) rb->ctx_refs = 0;
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2117) rb->task = NULL;
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2118) }
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2119)
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2120) static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2121) struct req_batch *rb)
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2122) {
6e833d538b312 (Pavel Begunkov 2021-02-11 18:28:20 +0000 2123) if (rb->task)
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 2124) io_put_task(rb->task, rb->task_refs);
9ae7246321d2b (Pavel Begunkov 2021-02-10 00:03:16 +0000 2125) if (rb->ctx_refs)
9ae7246321d2b (Pavel Begunkov 2021-02-10 00:03:16 +0000 2126) percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2127) }
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2128)
6ff119a6e4c3f (Pavel Begunkov 2021-02-10 00:03:18 +0000 2129) static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
6ff119a6e4c3f (Pavel Begunkov 2021-02-10 00:03:18 +0000 2130) struct io_submit_state *state)
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2131) {
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 2132) io_queue_next(req);
966706579a712 (Pavel Begunkov 2021-03-19 17:22:32 +0000 2133) io_dismantle_req(req);
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2134)
e3bc8e9dad7f2 (Jens Axboe 2020-09-24 08:45:57 -0600 2135) if (req->task != rb->task) {
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 2136) if (rb->task)
7c6607313f032 (Pavel Begunkov 2021-01-25 11:42:21 +0000 2137) io_put_task(rb->task, rb->task_refs);
e3bc8e9dad7f2 (Jens Axboe 2020-09-24 08:45:57 -0600 2138) rb->task = req->task;
e3bc8e9dad7f2 (Jens Axboe 2020-09-24 08:45:57 -0600 2139) rb->task_refs = 0;
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2140) }
e3bc8e9dad7f2 (Jens Axboe 2020-09-24 08:45:57 -0600 2141) rb->task_refs++;
9ae7246321d2b (Pavel Begunkov 2021-02-10 00:03:16 +0000 2142) rb->ctx_refs++;
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2143)
bd75904590de1 (Pavel Begunkov 2021-02-12 03:23:50 +0000 2144) if (state->free_reqs != ARRAY_SIZE(state->reqs))
6ff119a6e4c3f (Pavel Begunkov 2021-02-10 00:03:18 +0000 2145) state->reqs[state->free_reqs++] = req;
bd75904590de1 (Pavel Begunkov 2021-02-12 03:23:50 +0000 2146) else
bd75904590de1 (Pavel Begunkov 2021-02-12 03:23:50 +0000 2147) list_add(&req->compl.list, &state->comp.free_list);
7a743e225b2a9 (Pavel Begunkov 2020-03-03 21:33:13 +0300 2148) }
7a743e225b2a9 (Pavel Begunkov 2020-03-03 21:33:13 +0300 2149)
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2150) static void io_submit_flush_completions(struct io_comp_state *cs,
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2151) struct io_ring_ctx *ctx)
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2152) {
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2153) int i, nr = cs->nr;
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2154) struct io_kiocb *req;
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2155) struct req_batch rb;
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2156)
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2157) io_init_req_batch(&rb);
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2158) spin_lock_irq(&ctx->completion_lock);
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2159) for (i = 0; i < nr; i++) {
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2160) req = cs->reqs[i];
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 2161) __io_cqring_fill_event(ctx, req->user_data, req->result,
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 2162) req->compl.cflags);
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2163) }
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2164) io_commit_cqring(ctx);
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2165) spin_unlock_irq(&ctx->completion_lock);
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2166)
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2167) io_cqring_ev_posted(ctx);
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2168) for (i = 0; i < nr; i++) {
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2169) req = cs->reqs[i];
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2170)
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2171) /* submission and completion refs */
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 2172) if (req_ref_sub_and_test(req, 2))
6ff119a6e4c3f (Pavel Begunkov 2021-02-10 00:03:18 +0000 2173) io_req_free_batch(&rb, req, &ctx->submit_state);
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2174) }
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2175)
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2176) io_req_free_batch_finish(ctx, &rb);
905c172f32c56 (Pavel Begunkov 2021-02-10 00:03:14 +0000 2177) cs->nr = 0;
7a743e225b2a9 (Pavel Begunkov 2020-03-03 21:33:13 +0300 2178) }
7a743e225b2a9 (Pavel Begunkov 2020-03-03 21:33:13 +0300 2179)
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2180) /*
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2181) * Drop reference to request, return next in chain (if there is one) if this
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2182) * was the last reference to this request.
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2183) */
0d85035a7368a (Pavel Begunkov 2021-03-19 17:22:37 +0000 2184) static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
e65ef56db4945 (Jens Axboe 2019-03-12 10:16:44 -0600 2185) {
9b5f7bd932726 (Pavel Begunkov 2020-06-29 13:13:00 +0300 2186) struct io_kiocb *nxt = NULL;
9b5f7bd932726 (Pavel Begunkov 2020-06-29 13:13:00 +0300 2187)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 2188) if (req_ref_put_and_test(req)) {
9b5f7bd932726 (Pavel Begunkov 2020-06-29 13:13:00 +0300 2189) nxt = io_req_find_next(req);
4d7dd46297140 (Jens Axboe 2019-11-20 13:03:52 -0700 2190) __io_free_req(req);
2a44f46781617 (Jens Axboe 2020-02-25 13:25:41 -0700 2191) }
9b5f7bd932726 (Pavel Begunkov 2020-06-29 13:13:00 +0300 2192) return nxt;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2193) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2194)
0d85035a7368a (Pavel Begunkov 2021-03-19 17:22:37 +0000 2195) static inline void io_put_req(struct io_kiocb *req)
e65ef56db4945 (Jens Axboe 2019-03-12 10:16:44 -0600 2196) {
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 2197) if (req_ref_put_and_test(req))
e65ef56db4945 (Jens Axboe 2019-03-12 10:16:44 -0600 2198) io_free_req(req);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2199) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2200)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2201) static void io_put_req_deferred_cb(struct callback_head *cb)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2202) {
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2203) struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2204)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2205) io_free_req(req);
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2206) }
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2207)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2208) static void io_free_req_deferred(struct io_kiocb *req)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2209) {
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 2210) req->task_work.func = io_put_req_deferred_cb;
a05432fb49b64 (Pavel Begunkov 2021-03-19 17:22:38 +0000 2211) if (unlikely(io_req_task_work_add(req)))
eab30c4d20dc7 (Pavel Begunkov 2021-01-19 13:32:42 +0000 2212) io_req_task_work_add_fallback(req, io_put_req_deferred_cb);
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2213) }
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2214)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2215) static inline void io_put_req_deferred(struct io_kiocb *req, int refs)
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2216) {
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 2217) if (req_ref_sub_and_test(req, refs))
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2218) io_free_req_deferred(req);
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2219) }
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 2220)
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 2221) static unsigned io_cqring_events(struct io_ring_ctx *ctx)
a3a0e43fd7701 (Jens Axboe 2019-08-20 11:03:11 -0600 2222) {
a3a0e43fd7701 (Jens Axboe 2019-08-20 11:03:11 -0600 2223) /* See comment at the top of this file */
a3a0e43fd7701 (Jens Axboe 2019-08-20 11:03:11 -0600 2224) smp_rmb();
e23de15fdbd30 (Pavel Begunkov 2020-12-17 00:24:37 +0000 2225) return __io_cqring_events(ctx);
a3a0e43fd7701 (Jens Axboe 2019-08-20 11:03:11 -0600 2226) }
a3a0e43fd7701 (Jens Axboe 2019-08-20 11:03:11 -0600 2227)
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 2228) static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 2229) {
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 2230) struct io_rings *rings = ctx->rings;
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 2231)
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 2232) /* make sure SQ entry isn't read before tail */
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 2233) return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 2234) }
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 2235)
8ff069bf2efd7 (Pavel Begunkov 2020-07-16 23:28:04 +0300 2236) static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
e94f141bd248e (Jens Axboe 2019-12-19 12:06:02 -0700 2237) {
8ff069bf2efd7 (Pavel Begunkov 2020-07-16 23:28:04 +0300 2238) unsigned int cflags;
e94f141bd248e (Jens Axboe 2019-12-19 12:06:02 -0700 2239)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2240) cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2241) cflags |= IORING_CQE_F_BUFFER;
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 2242) req->flags &= ~REQ_F_BUFFER_SELECTED;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2243) kfree(kbuf);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2244) return cflags;
e94f141bd248e (Jens Axboe 2019-12-19 12:06:02 -0700 2245) }
e94f141bd248e (Jens Axboe 2019-12-19 12:06:02 -0700 2246)
8ff069bf2efd7 (Pavel Begunkov 2020-07-16 23:28:04 +0300 2247) static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2248) {
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2249) struct io_buffer *kbuf;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2250)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2251) kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
8ff069bf2efd7 (Pavel Begunkov 2020-07-16 23:28:04 +0300 2252) return io_put_kbuf(req, kbuf);
8ff069bf2efd7 (Pavel Begunkov 2020-07-16 23:28:04 +0300 2253) }
8ff069bf2efd7 (Pavel Begunkov 2020-07-16 23:28:04 +0300 2254)
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 2255) static inline bool io_run_task_work(void)
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 2256) {
6a4b928211359 (Nadav Amit 2021-08-07 17:13:41 -0700 2257) if (test_thread_flag(TIF_NOTIFY_SIGNAL) || current->task_works) {
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 2258) __set_current_state(TASK_RUNNING);
6a4b928211359 (Nadav Amit 2021-08-07 17:13:41 -0700 2259) tracehook_notify_signal();
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 2260) return true;
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 2261) }
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 2262)
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 2263) return false;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2264) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2265)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2266) /*
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2267) * Find and free completed poll iocbs
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2268) */
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2269) static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
f7be9c72d1de9 (Jens Axboe 2021-07-23 11:49:29 -0600 2270) struct list_head *done, bool resubmit)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2271) {
8237e045983d8 (Jens Axboe 2019-12-28 10:48:22 -0700 2272) struct req_batch rb;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2273) struct io_kiocb *req;
bbde017a32b32 (Xiaoguang Wang 2020-06-16 02:06:38 +0800 2274)
bbde017a32b32 (Xiaoguang Wang 2020-06-16 02:06:38 +0800 2275) /* order with ->result store in io_complete_rw_iopoll() */
bbde017a32b32 (Xiaoguang Wang 2020-06-16 02:06:38 +0800 2276) smp_rmb();
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2277)
5af1d13e8f0d8 (Pavel Begunkov 2020-07-18 11:32:52 +0300 2278) io_init_req_batch(&rb);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2279) while (!list_empty(done)) {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2280) int cflags = 0;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2281)
d21ffe7eca82d (Pavel Begunkov 2020-07-13 23:37:10 +0300 2282) req = list_first_entry(done, struct io_kiocb, inflight_entry);
f161340d9e85b (Pavel Begunkov 2021-02-11 18:28:21 +0000 2283) list_del(&req->inflight_entry);
f161340d9e85b (Pavel Begunkov 2021-02-11 18:28:21 +0000 2284)
f7be9c72d1de9 (Jens Axboe 2021-07-23 11:49:29 -0600 2285) if (READ_ONCE(req->result) == -EAGAIN && resubmit &&
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 2286) !(req->flags & REQ_F_DONT_REISSUE)) {
bbde017a32b32 (Xiaoguang Wang 2020-06-16 02:06:38 +0800 2287) req->iopoll_completed = 0;
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 2288) req_ref_get(req);
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 2289) io_queue_async_work(req);
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 2290) continue;
bbde017a32b32 (Xiaoguang Wang 2020-06-16 02:06:38 +0800 2291) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2292)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2293) if (req->flags & REQ_F_BUFFER_SELECTED)
8ff069bf2efd7 (Pavel Begunkov 2020-07-16 23:28:04 +0300 2294) cflags = io_put_rw_kbuf(req);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2295)
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 2296) __io_cqring_fill_event(ctx, req->user_data, req->result, cflags);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2297) (*nr_events)++;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2298)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 2299) if (req_ref_put_and_test(req))
6ff119a6e4c3f (Pavel Begunkov 2021-02-10 00:03:18 +0000 2300) io_req_free_batch(&rb, req, &ctx->submit_state);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2301) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2302)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 2303) io_commit_cqring(ctx);
80c18e4ac20c9 (Pavel Begunkov 2021-01-07 03:15:41 +0000 2304) io_cqring_ev_posted_iopoll(ctx);
2d6500d44c137 (Pavel Begunkov 2020-06-28 12:52:33 +0300 2305) io_req_free_batch_finish(ctx, &rb);
581f981034890 (Bijan Mottahedeh 2020-04-03 13:51:33 -0700 2306) }
581f981034890 (Bijan Mottahedeh 2020-04-03 13:51:33 -0700 2307)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2308) static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
f7be9c72d1de9 (Jens Axboe 2021-07-23 11:49:29 -0600 2309) long min, bool resubmit)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2310) {
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2311) struct io_kiocb *req, *tmp;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2312) LIST_HEAD(done);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2313) bool spin;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2314) int ret;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2315)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2316) /*
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2317) * Only spin for completions if we don't have multiple devices hanging
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2318) * off our complete list, and we're under the requested amount.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2319) */
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2320) spin = !ctx->poll_multi_file && *nr_events < min;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2321)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2322) ret = 0;
d21ffe7eca82d (Pavel Begunkov 2020-07-13 23:37:10 +0300 2323) list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 2324) struct kiocb *kiocb = &req->rw.kiocb;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2325)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2326) /*
581f981034890 (Bijan Mottahedeh 2020-04-03 13:51:33 -0700 2327) * Move completed and retryable entries to our local lists.
581f981034890 (Bijan Mottahedeh 2020-04-03 13:51:33 -0700 2328) * If we find a request that requires polling, break out
581f981034890 (Bijan Mottahedeh 2020-04-03 13:51:33 -0700 2329) * and complete those lists first, if we have entries there.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2330) */
65a6543da3868 (Xiaoguang Wang 2020-06-11 23:39:36 +0800 2331) if (READ_ONCE(req->iopoll_completed)) {
d21ffe7eca82d (Pavel Begunkov 2020-07-13 23:37:10 +0300 2332) list_move_tail(&req->inflight_entry, &done);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2333) continue;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2334) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2335) if (!list_empty(&done))
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2336) break;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2337)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2338) ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2339) if (ret < 0)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2340) break;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2341)
3aadc23e60543 (Pavel Begunkov 2020-07-06 17:59:29 +0300 2342) /* iopoll may have completed current req */
3aadc23e60543 (Pavel Begunkov 2020-07-06 17:59:29 +0300 2343) if (READ_ONCE(req->iopoll_completed))
d21ffe7eca82d (Pavel Begunkov 2020-07-13 23:37:10 +0300 2344) list_move_tail(&req->inflight_entry, &done);
3aadc23e60543 (Pavel Begunkov 2020-07-06 17:59:29 +0300 2345)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2346) if (ret && spin)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2347) spin = false;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2348) ret = 0;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2349) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2350)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2351) if (!list_empty(&done))
f7be9c72d1de9 (Jens Axboe 2021-07-23 11:49:29 -0600 2352) io_iopoll_complete(ctx, nr_events, &done, resubmit);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2353)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2354) return ret;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2355) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2356)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2357) /*
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2358) * We can't just wait for polled events to come to us, we have to actively
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2359) * find and complete them.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2360) */
b2edc0a77fac1 (Pavel Begunkov 2020-07-07 16:36:22 +0300 2361) static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2362) {
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2363) if (!(ctx->flags & IORING_SETUP_IOPOLL))
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2364) return;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2365)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2366) mutex_lock(&ctx->uring_lock);
540e32a0855e7 (Pavel Begunkov 2020-07-13 23:37:09 +0300 2367) while (!list_empty(&ctx->iopoll_list)) {
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2368) unsigned int nr_events = 0;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2369)
f7be9c72d1de9 (Jens Axboe 2021-07-23 11:49:29 -0600 2370) io_do_iopoll(ctx, &nr_events, 0, false);
08f5439f1df25 (Jens Axboe 2019-08-21 22:19:11 -0600 2371)
b2edc0a77fac1 (Pavel Begunkov 2020-07-07 16:36:22 +0300 2372) /* let it sleep and repeat later if can't complete a request */
b2edc0a77fac1 (Pavel Begunkov 2020-07-07 16:36:22 +0300 2373) if (nr_events == 0)
b2edc0a77fac1 (Pavel Begunkov 2020-07-07 16:36:22 +0300 2374) break;
08f5439f1df25 (Jens Axboe 2019-08-21 22:19:11 -0600 2375) /*
08f5439f1df25 (Jens Axboe 2019-08-21 22:19:11 -0600 2376) * Ensure we allow local-to-the-cpu processing to take place,
08f5439f1df25 (Jens Axboe 2019-08-21 22:19:11 -0600 2377) * in this case we need to ensure that we reap all events.
3fcee5a6d5414 (Pavel Begunkov 2020-07-06 17:59:31 +0300 2378) * Also let task_work, etc. to progress by releasing the mutex
08f5439f1df25 (Jens Axboe 2019-08-21 22:19:11 -0600 2379) */
3fcee5a6d5414 (Pavel Begunkov 2020-07-06 17:59:31 +0300 2380) if (need_resched()) {
3fcee5a6d5414 (Pavel Begunkov 2020-07-06 17:59:31 +0300 2381) mutex_unlock(&ctx->uring_lock);
3fcee5a6d5414 (Pavel Begunkov 2020-07-06 17:59:31 +0300 2382) cond_resched();
3fcee5a6d5414 (Pavel Begunkov 2020-07-06 17:59:31 +0300 2383) mutex_lock(&ctx->uring_lock);
3fcee5a6d5414 (Pavel Begunkov 2020-07-06 17:59:31 +0300 2384) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2385) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2386) mutex_unlock(&ctx->uring_lock);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2387) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2388)
7668b92a69b82 (Pavel Begunkov 2020-07-07 16:36:21 +0300 2389) static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2390) {
7668b92a69b82 (Pavel Begunkov 2020-07-07 16:36:21 +0300 2391) unsigned int nr_events = 0;
e9979b36a467d (Pavel Begunkov 2021-04-13 02:58:45 +0100 2392) int ret = 0;
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2393)
c7849be9cc2dd (Xiaoguang Wang 2020-02-22 14:46:05 +0800 2394) /*
c7849be9cc2dd (Xiaoguang Wang 2020-02-22 14:46:05 +0800 2395) * We disallow the app entering submit/complete with polling, but we
c7849be9cc2dd (Xiaoguang Wang 2020-02-22 14:46:05 +0800 2396) * still need to lock the ring to prevent racing with polled issue
c7849be9cc2dd (Xiaoguang Wang 2020-02-22 14:46:05 +0800 2397) * that got punted to a workqueue.
c7849be9cc2dd (Xiaoguang Wang 2020-02-22 14:46:05 +0800 2398) */
c7849be9cc2dd (Xiaoguang Wang 2020-02-22 14:46:05 +0800 2399) mutex_lock(&ctx->uring_lock);
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2400) /*
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2401) * Don't enter poll loop if we already have events pending.
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2402) * If we do, we can potentially be spinning for commands that
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2403) * already triggered a CQE (eg in error).
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2404) */
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2405) if (test_bit(0, &ctx->cq_check_overflow))
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2406) __io_cqring_overflow_flush(ctx, false);
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2407) if (io_cqring_events(ctx))
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2408) goto out;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2409) do {
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2410) /*
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2411) * If a submit got punted to a workqueue, we can have the
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2412) * application entering polling for a command before it gets
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2413) * issued. That app will hold the uring_lock for the duration
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2414) * of the poll right here, so we need to take a breather every
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2415) * now and then to ensure that the issue has a chance to add
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2416) * the poll to the issued list. Otherwise we can spin here
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2417) * forever, while the workqueue is stuck trying to acquire the
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2418) * very same mutex.
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2419) */
e9979b36a467d (Pavel Begunkov 2021-04-13 02:58:45 +0100 2420) if (list_empty(&ctx->iopoll_list)) {
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2421) mutex_unlock(&ctx->uring_lock);
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 2422) io_run_task_work();
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2423) mutex_lock(&ctx->uring_lock);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2424)
e9979b36a467d (Pavel Begunkov 2021-04-13 02:58:45 +0100 2425) if (list_empty(&ctx->iopoll_list))
e9979b36a467d (Pavel Begunkov 2021-04-13 02:58:45 +0100 2426) break;
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2427) }
f7be9c72d1de9 (Jens Axboe 2021-07-23 11:49:29 -0600 2428) ret = io_do_iopoll(ctx, &nr_events, min, true);
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2429) } while (!ret && nr_events < min && !need_resched());
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2430) out:
500f9fbadef86 (Jens Axboe 2019-08-19 12:15:59 -0600 2431) mutex_unlock(&ctx->uring_lock);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2432) return ret;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2433) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2434)
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2435) static void kiocb_end_write(struct io_kiocb *req)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2436) {
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2437) /*
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2438) * Tell lockdep we inherited freeze protection from submission
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2439) * thread.
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2440) */
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2441) if (req->flags & REQ_F_ISREG) {
1c98679db9415 (Pavel Begunkov 2021-03-22 01:58:31 +0000 2442) struct super_block *sb = file_inode(req->file)->i_sb;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2443)
1c98679db9415 (Pavel Begunkov 2021-03-22 01:58:31 +0000 2444) __sb_writers_acquired(sb, SB_FREEZE_WRITE);
1c98679db9415 (Pavel Begunkov 2021-03-22 01:58:31 +0000 2445) sb_end_write(sb);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2446) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2447) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2448)
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2449) #ifdef CONFIG_BLOCK
dc2a6e9aa9c34 (Pavel Begunkov 2021-01-19 13:32:35 +0000 2450) static bool io_resubmit_prep(struct io_kiocb *req)
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2451) {
ab454438aa8dc (Pavel Begunkov 2021-03-22 01:58:33 +0000 2452) struct io_async_rw *rw = req->async_data;
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2453)
ab454438aa8dc (Pavel Begunkov 2021-03-22 01:58:33 +0000 2454) if (!rw)
ab454438aa8dc (Pavel Begunkov 2021-03-22 01:58:33 +0000 2455) return !io_req_prep_async(req);
ab454438aa8dc (Pavel Begunkov 2021-03-22 01:58:33 +0000 2456) /* may have left rw->iter inconsistent on -EIOCBQUEUED */
ab454438aa8dc (Pavel Begunkov 2021-03-22 01:58:33 +0000 2457) iov_iter_revert(&rw->iter, req->result - iov_iter_count(&rw->iter));
ab454438aa8dc (Pavel Begunkov 2021-03-22 01:58:33 +0000 2458) return true;
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2459) }
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2460)
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2461) static bool io_rw_should_reissue(struct io_kiocb *req)
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2462) {
355afaeb578ab (Jens Axboe 2020-09-02 09:30:31 -0600 2463) umode_t mode = file_inode(req->file)->i_mode;
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2464) struct io_ring_ctx *ctx = req->ctx;
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2465)
355afaeb578ab (Jens Axboe 2020-09-02 09:30:31 -0600 2466) if (!S_ISBLK(mode) && !S_ISREG(mode))
355afaeb578ab (Jens Axboe 2020-09-02 09:30:31 -0600 2467) return false;
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2468) if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2469) !(ctx->flags & IORING_SETUP_IOPOLL)))
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2470) return false;
7c977a58dc833 (Jens Axboe 2021-02-23 19:17:35 -0700 2471) /*
7c977a58dc833 (Jens Axboe 2021-02-23 19:17:35 -0700 2472) * If ref is dying, we might be running poll reap from the exit work.
7c977a58dc833 (Jens Axboe 2021-02-23 19:17:35 -0700 2473) * Don't attempt to reissue from that path, just let it fail with
7c977a58dc833 (Jens Axboe 2021-02-23 19:17:35 -0700 2474) * -EAGAIN.
7c977a58dc833 (Jens Axboe 2021-02-23 19:17:35 -0700 2475) */
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2476) if (percpu_ref_is_dying(&ctx->refs))
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2477) return false;
5db0ca0fbebf1 (Jens Axboe 2021-07-27 10:50:31 -0600 2478) /*
5db0ca0fbebf1 (Jens Axboe 2021-07-27 10:50:31 -0600 2479) * Play it safe and assume not safe to re-import and reissue if we're
5db0ca0fbebf1 (Jens Axboe 2021-07-27 10:50:31 -0600 2480) * not in the original thread group (or in task context).
5db0ca0fbebf1 (Jens Axboe 2021-07-27 10:50:31 -0600 2481) */
5db0ca0fbebf1 (Jens Axboe 2021-07-27 10:50:31 -0600 2482) if (!same_thread_group(req->task, current) || !in_task())
5db0ca0fbebf1 (Jens Axboe 2021-07-27 10:50:31 -0600 2483) return false;
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2484) return true;
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2485) }
e82ad48539483 (Jens Axboe 2021-04-02 19:45:34 -0600 2486) #else
a1ff1e3f0e1cb (Jens Axboe 2021-04-12 06:40:02 -0600 2487) static bool io_resubmit_prep(struct io_kiocb *req)
e82ad48539483 (Jens Axboe 2021-04-02 19:45:34 -0600 2488) {
e82ad48539483 (Jens Axboe 2021-04-02 19:45:34 -0600 2489) return false;
e82ad48539483 (Jens Axboe 2021-04-02 19:45:34 -0600 2490) }
e82ad48539483 (Jens Axboe 2021-04-02 19:45:34 -0600 2491) static bool io_rw_should_reissue(struct io_kiocb *req)
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2492) {
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2493) return false;
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2494) }
3e6a0d3c7571c (Jens Axboe 2021-03-01 13:56:00 -0700 2495) #endif
b63534c41e20b (Jens Axboe 2020-06-04 11:28:00 -0600 2496)
a1d7c393c4711 (Jens Axboe 2020-06-22 11:09:46 -0600 2497) static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 2498) unsigned int issue_flags)
a1d7c393c4711 (Jens Axboe 2020-06-22 11:09:46 -0600 2499) {
2f8e45f16c573 (Pavel Begunkov 2021-02-11 18:28:23 +0000 2500) int cflags = 0;
2f8e45f16c573 (Pavel Begunkov 2021-02-11 18:28:23 +0000 2501)
b65c128f963df (Pavel Begunkov 2021-03-22 01:45:59 +0000 2502) if (req->rw.kiocb.ki_flags & IOCB_WRITE)
b65c128f963df (Pavel Begunkov 2021-03-22 01:45:59 +0000 2503) kiocb_end_write(req);
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2504) if (res != req->result) {
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2505) if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2506) io_rw_should_reissue(req)) {
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2507) req->flags |= REQ_F_REISSUE;
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2508) return;
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2509) }
2f8e45f16c573 (Pavel Begunkov 2021-02-11 18:28:23 +0000 2510) req_set_fail_links(req);
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2511) }
2f8e45f16c573 (Pavel Begunkov 2021-02-11 18:28:23 +0000 2512) if (req->flags & REQ_F_BUFFER_SELECTED)
2f8e45f16c573 (Pavel Begunkov 2021-02-11 18:28:23 +0000 2513) cflags = io_put_rw_kbuf(req);
2f8e45f16c573 (Pavel Begunkov 2021-02-11 18:28:23 +0000 2514) __io_req_complete(req, issue_flags, res, cflags);
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2515) }
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2516)
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2517) static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2518) {
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 2519) struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2520)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 2521) __io_complete_rw(req, res, res2, 0);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2522) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2523)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2524) static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2525) {
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 2526) struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2527)
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2528) if (kiocb->ki_flags & IOCB_WRITE)
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2529) kiocb_end_write(req);
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2530) if (unlikely(res != req->result)) {
a1ff1e3f0e1cb (Jens Axboe 2021-04-12 06:40:02 -0600 2531) if (!(res == -EAGAIN && io_rw_should_reissue(req) &&
a1ff1e3f0e1cb (Jens Axboe 2021-04-12 06:40:02 -0600 2532) io_resubmit_prep(req))) {
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2533) req_set_fail_links(req);
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2534) req->flags |= REQ_F_DONT_REISSUE;
9532b99bd9ca3 (Pavel Begunkov 2021-03-22 01:58:34 +0000 2535) }
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 2536) }
bbde017a32b32 (Xiaoguang Wang 2020-06-16 02:06:38 +0800 2537)
bbde017a32b32 (Xiaoguang Wang 2020-06-16 02:06:38 +0800 2538) WRITE_ONCE(req->result, res);
b9b0e0d39c7b4 (Jens Axboe 2021-02-23 08:18:36 -0700 2539) /* order with io_iopoll_complete() checking ->result */
cd664b0e35cb1 (Pavel Begunkov 2020-06-25 12:37:10 +0300 2540) smp_wmb();
cd664b0e35cb1 (Pavel Begunkov 2020-06-25 12:37:10 +0300 2541) WRITE_ONCE(req->iopoll_completed, 1);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2542) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2543)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2544) /*
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2545) * After the iocb has been issued, it's safe to be found on the poll list.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2546) * Adding the kiocb to the list AFTER submission ensures that we don't
f39c8a5b1130f (Pavel Begunkov 2021-04-13 02:58:46 +0100 2547) * find it from a io_do_iopoll() thread before the issuer is done
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2548) * accessing the kiocb cookie.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2549) */
2e9dbe902d102 (Xiaoguang Wang 2020-11-13 00:44:08 +0800 2550) static void io_iopoll_req_issued(struct io_kiocb *req, bool in_async)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2551) {
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2552) struct io_ring_ctx *ctx = req->ctx;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2553)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2554) /*
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2555) * Track whether we have multiple files in our lists. This will impact
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2556) * how we do polling eventually, not spinning if we're on potentially
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2557) * different devices.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2558) */
540e32a0855e7 (Pavel Begunkov 2020-07-13 23:37:09 +0300 2559) if (list_empty(&ctx->iopoll_list)) {
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2560) ctx->poll_multi_file = false;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2561) } else if (!ctx->poll_multi_file) {
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2562) struct io_kiocb *list_req;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2563)
540e32a0855e7 (Pavel Begunkov 2020-07-13 23:37:09 +0300 2564) list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
d21ffe7eca82d (Pavel Begunkov 2020-07-13 23:37:10 +0300 2565) inflight_entry);
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 2566) if (list_req->file != req->file)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2567) ctx->poll_multi_file = true;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2568) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2569)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2570) /*
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2571) * For fast devices, IO may have already completed. If it has, add
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2572) * it to the front so we find it first.
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2573) */
65a6543da3868 (Xiaoguang Wang 2020-06-11 23:39:36 +0800 2574) if (READ_ONCE(req->iopoll_completed))
d21ffe7eca82d (Pavel Begunkov 2020-07-13 23:37:10 +0300 2575) list_add(&req->inflight_entry, &ctx->iopoll_list);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2576) else
d21ffe7eca82d (Pavel Begunkov 2020-07-13 23:37:10 +0300 2577) list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
bdcd3eab2a9ae (Xiaoguang Wang 2020-02-25 22:12:08 +0800 2578)
2e9dbe902d102 (Xiaoguang Wang 2020-11-13 00:44:08 +0800 2579) /*
2e9dbe902d102 (Xiaoguang Wang 2020-11-13 00:44:08 +0800 2580) * If IORING_SETUP_SQPOLL is enabled, sqes are either handled in sq thread
2e9dbe902d102 (Xiaoguang Wang 2020-11-13 00:44:08 +0800 2581) * task context or in io worker task context. If current task context is
2e9dbe902d102 (Xiaoguang Wang 2020-11-13 00:44:08 +0800 2582) * sq thread, we don't need to check whether should wake up sq thread.
2e9dbe902d102 (Xiaoguang Wang 2020-11-13 00:44:08 +0800 2583) */
2e9dbe902d102 (Xiaoguang Wang 2020-11-13 00:44:08 +0800 2584) if (in_async && (ctx->flags & IORING_SETUP_SQPOLL) &&
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 2585) wq_has_sleeper(&ctx->sq_data->wait))
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 2586) wake_up(&ctx->sq_data->wait);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2587) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2588)
9f13c35b33fdd (Pavel Begunkov 2020-05-17 14:13:41 +0300 2589) static inline void io_state_file_put(struct io_submit_state *state)
9f13c35b33fdd (Pavel Begunkov 2020-05-17 14:13:41 +0300 2590) {
02b23a9af5ba4 (Pavel Begunkov 2021-01-19 13:32:41 +0000 2591) if (state->file_refs) {
02b23a9af5ba4 (Pavel Begunkov 2021-01-19 13:32:41 +0000 2592) fput_many(state->file, state->file_refs);
02b23a9af5ba4 (Pavel Begunkov 2021-01-19 13:32:41 +0000 2593) state->file_refs = 0;
02b23a9af5ba4 (Pavel Begunkov 2021-01-19 13:32:41 +0000 2594) }
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2595) }
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2596)
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2597) /*
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2598) * Get as many references to a file as we have IOs left in this submission,
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2599) * assuming most submissions are for one file, or at least that each file
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2600) * has more than one submission.
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2601) */
8da11c19940dd (Pavel Begunkov 2020-02-24 11:32:44 +0300 2602) static struct file *__io_file_get(struct io_submit_state *state, int fd)
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2603) {
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2604) if (!state)
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2605) return fget(fd);
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2606)
6e1271e60c1d5 (Pavel Begunkov 2020-11-20 15:50:50 +0000 2607) if (state->file_refs) {
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2608) if (state->fd == fd) {
6e1271e60c1d5 (Pavel Begunkov 2020-11-20 15:50:50 +0000 2609) state->file_refs--;
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2610) return state->file;
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2611) }
02b23a9af5ba4 (Pavel Begunkov 2021-01-19 13:32:41 +0000 2612) io_state_file_put(state);
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2613) }
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2614) state->file = fget_many(fd, state->ios_left);
6e1271e60c1d5 (Pavel Begunkov 2020-11-20 15:50:50 +0000 2615) if (unlikely(!state->file))
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2616) return NULL;
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2617)
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2618) state->fd = fd;
6e1271e60c1d5 (Pavel Begunkov 2020-11-20 15:50:50 +0000 2619) state->file_refs = state->ios_left - 1;
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2620) return state->file;
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2621) }
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 2622)
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2623) static bool io_bdev_nowait(struct block_device *bdev)
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2624) {
9ba0d0c81284f (Jeffle Xu 2020-10-19 16:59:42 +0800 2625) return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2626) }
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2627)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2628) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2629) * If we tracked the file through the SCM inflight mechanism, we could support
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2630) * any file. For now, just ensure that anything potentially problematic is done
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2631) * inline.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2632) */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2633) static bool __io_file_supports_async(struct file *file, int rw)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2634) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2635) umode_t mode = file_inode(file)->i_mode;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2636)
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2637) if (S_ISBLK(mode)) {
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 2638) if (IS_ENABLED(CONFIG_BLOCK) &&
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 2639) io_bdev_nowait(I_BDEV(file->f_mapping->host)))
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2640) return true;
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2641) return false;
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2642) }
378a830c2608f (Pavel Begunkov 2021-06-09 12:07:25 +0100 2643) if (S_ISSOCK(mode))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2644) return true;
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2645) if (S_ISREG(mode)) {
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 2646) if (IS_ENABLED(CONFIG_BLOCK) &&
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 2647) io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2648) file->f_op != &io_uring_fops)
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2649) return true;
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2650) return false;
4503b7676a2e0 (Jens Axboe 2020-06-01 10:00:27 -0600 2651) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2652)
c5b856255cbc3 (Jens Axboe 2020-06-09 19:23:05 -0600 2653) /* any ->read/write should understand O_NONBLOCK */
c5b856255cbc3 (Jens Axboe 2020-06-09 19:23:05 -0600 2654) if (file->f_flags & O_NONBLOCK)
c5b856255cbc3 (Jens Axboe 2020-06-09 19:23:05 -0600 2655) return true;
c5b856255cbc3 (Jens Axboe 2020-06-09 19:23:05 -0600 2656)
af197f50ac53f (Jens Axboe 2020-04-28 13:15:06 -0600 2657) if (!(file->f_mode & FMODE_NOWAIT))
af197f50ac53f (Jens Axboe 2020-04-28 13:15:06 -0600 2658) return false;
af197f50ac53f (Jens Axboe 2020-04-28 13:15:06 -0600 2659)
af197f50ac53f (Jens Axboe 2020-04-28 13:15:06 -0600 2660) if (rw == READ)
af197f50ac53f (Jens Axboe 2020-04-28 13:15:06 -0600 2661) return file->f_op->read_iter != NULL;
af197f50ac53f (Jens Axboe 2020-04-28 13:15:06 -0600 2662)
af197f50ac53f (Jens Axboe 2020-04-28 13:15:06 -0600 2663) return file->f_op->write_iter != NULL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2664) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2665)
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2666) static bool io_file_supports_async(struct io_kiocb *req, int rw)
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2667) {
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2668) if (rw == READ && (req->flags & REQ_F_ASYNC_READ))
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2669) return true;
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2670) else if (rw == WRITE && (req->flags & REQ_F_ASYNC_WRITE))
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2671) return true;
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2672)
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2673) return __io_file_supports_async(req->file, rw);
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2674) }
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2675)
a88fc400212fc (Pavel Begunkov 2020-09-30 22:57:53 +0300 2676) static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2677) {
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2678) struct io_ring_ctx *ctx = req->ctx;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 2679) struct kiocb *kiocb = &req->rw.kiocb;
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 2680) struct file *file = req->file;
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 2681) unsigned ioprio;
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 2682) int ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2683)
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 2684) if (!(req->flags & REQ_F_ISREG) && S_ISREG(file_inode(file)->i_mode))
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2685) req->flags |= REQ_F_ISREG;
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 2686)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2687) kiocb->ki_pos = READ_ONCE(sqe->off);
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 2688) if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {
ba04291eb66ed (Jens Axboe 2019-12-25 16:33:42 -0700 2689) req->flags |= REQ_F_CUR_POS;
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 2690) kiocb->ki_pos = file->f_pos;
ba04291eb66ed (Jens Axboe 2019-12-25 16:33:42 -0700 2691) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2692) kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
3e577dcd73a1f (Pavel Begunkov 2020-02-01 03:58:42 +0300 2693) kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
3e577dcd73a1f (Pavel Begunkov 2020-02-01 03:58:42 +0300 2694) ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
3e577dcd73a1f (Pavel Begunkov 2020-02-01 03:58:42 +0300 2695) if (unlikely(ret))
3e577dcd73a1f (Pavel Begunkov 2020-02-01 03:58:42 +0300 2696) return ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2697)
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 2698) /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 2699) if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK))
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 2700) req->flags |= REQ_F_NOWAIT;
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 2701)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2702) ioprio = READ_ONCE(sqe->ioprio);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2703) if (ioprio) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2704) ret = ioprio_check_cap(ioprio);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2705) if (ret)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 2706) return ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2707)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2708) kiocb->ki_ioprio = ioprio;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2709) } else
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2710) kiocb->ki_ioprio = get_current_ioprio();
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2711)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2712) if (ctx->flags & IORING_SETUP_IOPOLL) {
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2713) if (!(kiocb->ki_flags & IOCB_DIRECT) ||
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2714) !kiocb->ki_filp->f_op->iopoll)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 2715) return -EOPNOTSUPP;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2716)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2717) kiocb->ki_flags |= IOCB_HIPRI;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2718) kiocb->ki_complete = io_complete_rw_iopoll;
65a6543da3868 (Xiaoguang Wang 2020-06-11 23:39:36 +0800 2719) req->iopoll_completed = 0;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2720) } else {
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 2721) if (kiocb->ki_flags & IOCB_HIPRI)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 2722) return -EINVAL;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2723) kiocb->ki_complete = io_complete_rw;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 2724) }
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 2725)
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2726) if (req->opcode == IORING_OP_READ_FIXED ||
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2727) req->opcode == IORING_OP_WRITE_FIXED) {
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2728) req->imu = NULL;
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2729) io_req_set_rsrc_node(req);
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2730) }
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2731)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 2732) req->rw.addr = READ_ONCE(sqe->addr);
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 2733) req->rw.len = READ_ONCE(sqe->len);
4f4eeba87cc73 (Bijan Mottahedeh 2020-05-19 14:52:49 -0700 2734) req->buf_index = READ_ONCE(sqe->buf_index);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2735) return 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2736) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2737)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2738) static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2739) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2740) switch (ret) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2741) case -EIOCBQUEUED:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2742) break;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2743) case -ERESTARTSYS:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2744) case -ERESTARTNOINTR:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2745) case -ERESTARTNOHAND:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2746) case -ERESTART_RESTARTBLOCK:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2747) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2748) * We can't just restart the syscall, since previously
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2749) * submitted sqes may already be in progress. Just fail this
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2750) * IO with EINTR.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2751) */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2752) ret = -EINTR;
df561f6688fef (Gustavo A. R. Silva 2020-08-23 17:36:59 -0500 2753) fallthrough;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2754) default:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2755) kiocb->ki_complete(kiocb, ret, 0);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2756) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2757) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 2758)
a1d7c393c4711 (Jens Axboe 2020-06-22 11:09:46 -0600 2759) static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 2760) unsigned int issue_flags)
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2761) {
ba04291eb66ed (Jens Axboe 2019-12-25 16:33:42 -0700 2762) struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 2763) struct io_async_rw *io = req->async_data;
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2764) bool check_reissue = kiocb->ki_complete == io_complete_rw;
ba04291eb66ed (Jens Axboe 2019-12-25 16:33:42 -0700 2765)
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 2766) /* add previously done IO, if any */
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 2767) if (io && io->bytes_done > 0) {
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 2768) if (ret < 0)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 2769) ret = io->bytes_done;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 2770) else
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 2771) ret += io->bytes_done;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 2772) }
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 2773)
ba04291eb66ed (Jens Axboe 2019-12-25 16:33:42 -0700 2774) if (req->flags & REQ_F_CUR_POS)
ba04291eb66ed (Jens Axboe 2019-12-25 16:33:42 -0700 2775) req->file->f_pos = kiocb->ki_pos;
bcaec089c5b64 (Pavel Begunkov 2020-02-24 11:30:18 +0300 2776) if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 2777) __io_complete_rw(req, ret, 0, issue_flags);
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2778) else
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2779) io_rw_done(kiocb, ret);
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2780)
501449420a42c (Pavel Begunkov 2021-06-24 15:09:57 +0100 2781) if (check_reissue && (req->flags & REQ_F_REISSUE)) {
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2782) req->flags &= ~REQ_F_REISSUE;
a7be7c23cfdd2 (Jens Axboe 2021-04-15 11:31:14 -0600 2783) if (io_resubmit_prep(req)) {
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 2784) req_ref_get(req);
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 2785) io_queue_async_work(req);
8c130827f417d (Pavel Begunkov 2021-03-22 01:58:32 +0000 2786) } else {
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2787) int cflags = 0;
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2788)
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2789) req_set_fail_links(req);
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2790) if (req->flags & REQ_F_BUFFER_SELECTED)
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2791) cflags = io_put_rw_kbuf(req);
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2792) __io_req_complete(req, issue_flags, ret, cflags);
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2793) }
9728463737db0 (Pavel Begunkov 2021-04-08 19:28:03 +0100 2794) }
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2795) }
ba816ad61fdf3 (Jens Axboe 2019-09-28 11:36:45 -0600 2796)
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2797) static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2798) struct io_mapped_ubuf *imu)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2799) {
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 2800) size_t len = req->rw.len;
75769e3f73571 (Pavel Begunkov 2021-04-01 15:43:54 +0100 2801) u64 buf_end, buf_addr = req->rw.addr;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2802) size_t offset;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2803)
75769e3f73571 (Pavel Begunkov 2021-04-01 15:43:54 +0100 2804) if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2805) return -EFAULT;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2806) /* not inside the mapped region */
4751f53d74a68 (Pavel Begunkov 2021-04-01 15:43:55 +0100 2807) if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2808) return -EFAULT;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2809)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2810) /*
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2811) * May not be a start of buffer, set size appropriately
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2812) * and advance us to the beginning.
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2813) */
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2814) offset = buf_addr - imu->ubuf;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2815) iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2816)
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2817) if (offset) {
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2818) /*
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2819) * Don't use iov_iter_advance() here, as it's really slow for
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2820) * using the latter parts of a big fixed buffer - it iterates
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2821) * over each segment manually. We can cheat a bit here, because
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2822) * we know that:
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2823) *
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2824) * 1) it's a BVEC iter, we set it up
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2825) * 2) all bvecs are PAGE_SIZE in size, except potentially the
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2826) * first and last bvec
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2827) *
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2828) * So just find our index, and adjust the iterator afterwards.
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2829) * If the offset is within the first bvec (or the whole first
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2830) * bvec, just use iov_iter_advance(). This makes it easier
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2831) * since we can just skip the first segment, which may not
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2832) * be PAGE_SIZE aligned.
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2833) */
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2834) const struct bio_vec *bvec = imu->bvec;
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2835)
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2836) if (offset <= bvec->bv_len) {
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2837) iov_iter_advance(iter, offset);
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2838) } else {
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2839) unsigned long seg_skip;
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2840)
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2841) /* skip first vec */
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2842) offset -= bvec->bv_len;
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2843) seg_skip = 1 + (offset >> PAGE_SHIFT);
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2844)
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2845) iter->bvec = bvec + seg_skip;
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2846) iter->nr_segs -= seg_skip;
99c79f6692ccd (Aleix Roca Nonell 2019-08-15 14:03:22 +0200 2847) iter->count -= bvec->bv_len + offset;
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2848) iter->iov_offset = offset & ~PAGE_MASK;
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2849) }
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2850) }
bd11b3a391e3d (Jens Axboe 2019-07-20 08:37:31 -0600 2851)
847595de1732a (Pavel Begunkov 2021-02-04 13:52:06 +0000 2852) return 0;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2853) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 2854)
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2855) static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2856) {
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2857) struct io_ring_ctx *ctx = req->ctx;
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2858) struct io_mapped_ubuf *imu = req->imu;
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2859) u16 index, buf_index = req->buf_index;
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2860)
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2861) if (likely(!imu)) {
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2862) if (unlikely(buf_index >= ctx->nr_user_bufs))
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2863) return -EFAULT;
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2864) index = array_index_nospec(buf_index, ctx->nr_user_bufs);
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2865) imu = READ_ONCE(ctx->user_bufs[index]);
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2866) req->imu = imu;
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2867) }
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2868) return __io_import_fixed(req, rw, iter, imu);
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2869) }
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 2870)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2871) static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2872) {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2873) if (needs_lock)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2874) mutex_unlock(&ctx->uring_lock);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2875) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2876)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2877) static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2878) {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2879) /*
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2880) * "Normal" inline submissions always hold the uring_lock, since we
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2881) * grab it from the system call. Same is true for the SQPOLL offload.
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2882) * The only exception is when we've detached the request and issue it
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2883) * from an async worker thread, grab the lock for that case.
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2884) */
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2885) if (needs_lock)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2886) mutex_lock(&ctx->uring_lock);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2887) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2888)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2889) static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2890) int bgid, struct io_buffer *kbuf,
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2891) bool needs_lock)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2892) {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2893) struct io_buffer *head;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2894)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2895) if (req->flags & REQ_F_BUFFER_SELECTED)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2896) return kbuf;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2897)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2898) io_ring_submit_lock(req->ctx, needs_lock);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2899)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2900) lockdep_assert_held(&req->ctx->uring_lock);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2901)
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 2902) head = xa_load(&req->ctx->io_buffers, bgid);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2903) if (head) {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2904) if (!list_empty(&head->list)) {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2905) kbuf = list_last_entry(&head->list, struct io_buffer,
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2906) list);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2907) list_del(&kbuf->list);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2908) } else {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2909) kbuf = head;
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 2910) xa_erase(&req->ctx->io_buffers, bgid);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2911) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2912) if (*len > kbuf->len)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2913) *len = kbuf->len;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2914) } else {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2915) kbuf = ERR_PTR(-ENOBUFS);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2916) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2917)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2918) io_ring_submit_unlock(req->ctx, needs_lock);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2919)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2920) return kbuf;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2921) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 2922)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2923) static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2924) bool needs_lock)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2925) {
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2926) struct io_buffer *kbuf;
4f4eeba87cc73 (Bijan Mottahedeh 2020-05-19 14:52:49 -0700 2927) u16 bgid;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2928)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2929) kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
4f4eeba87cc73 (Bijan Mottahedeh 2020-05-19 14:52:49 -0700 2930) bgid = req->buf_index;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2931) kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2932) if (IS_ERR(kbuf))
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2933) return kbuf;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2934) req->rw.addr = (u64) (unsigned long) kbuf;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2935) req->flags |= REQ_F_BUFFER_SELECTED;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2936) return u64_to_user_ptr(kbuf->addr);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2937) }
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2938)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2939) #ifdef CONFIG_COMPAT
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2940) static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2941) bool needs_lock)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2942) {
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2943) struct compat_iovec __user *uiov;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2944) compat_ssize_t clen;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2945) void __user *buf;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2946) ssize_t len;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2947)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2948) uiov = u64_to_user_ptr(req->rw.addr);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2949) if (!access_ok(uiov, sizeof(*uiov)))
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2950) return -EFAULT;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2951) if (__get_user(clen, &uiov->iov_len))
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2952) return -EFAULT;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2953) if (clen < 0)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2954) return -EINVAL;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2955)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2956) len = clen;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2957) buf = io_rw_buffer_select(req, &len, needs_lock);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2958) if (IS_ERR(buf))
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2959) return PTR_ERR(buf);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2960) iov[0].iov_base = buf;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2961) iov[0].iov_len = (compat_size_t) len;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2962) return 0;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2963) }
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2964) #endif
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2965)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2966) static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2967) bool needs_lock)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2968) {
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2969) struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2970) void __user *buf;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2971) ssize_t len;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2972)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2973) if (copy_from_user(iov, uiov, sizeof(*uiov)))
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2974) return -EFAULT;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2975)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2976) len = iov[0].iov_len;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2977) if (len < 0)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2978) return -EINVAL;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2979) buf = io_rw_buffer_select(req, &len, needs_lock);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2980) if (IS_ERR(buf))
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2981) return PTR_ERR(buf);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2982) iov[0].iov_base = buf;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2983) iov[0].iov_len = len;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2984) return 0;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2985) }
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2986)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2987) static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2988) bool needs_lock)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2989) {
dddb3e26f6d88 (Jens Axboe 2020-06-04 11:27:01 -0600 2990) if (req->flags & REQ_F_BUFFER_SELECTED) {
dddb3e26f6d88 (Jens Axboe 2020-06-04 11:27:01 -0600 2991) struct io_buffer *kbuf;
dddb3e26f6d88 (Jens Axboe 2020-06-04 11:27:01 -0600 2992)
dddb3e26f6d88 (Jens Axboe 2020-06-04 11:27:01 -0600 2993) kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
dddb3e26f6d88 (Jens Axboe 2020-06-04 11:27:01 -0600 2994) iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
dddb3e26f6d88 (Jens Axboe 2020-06-04 11:27:01 -0600 2995) iov[0].iov_len = kbuf->len;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2996) return 0;
dddb3e26f6d88 (Jens Axboe 2020-06-04 11:27:01 -0600 2997) }
dd20166236953 (Pavel Begunkov 2020-12-19 03:15:43 +0000 2998) if (req->rw.len != 1)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 2999) return -EINVAL;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3000)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3001) #ifdef CONFIG_COMPAT
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3002) if (req->ctx->compat)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3003) return io_compat_import(req, iov, needs_lock);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3004) #endif
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3005)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3006) return __io_iov_buffer_select(req, iov, needs_lock);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3007) }
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3008)
847595de1732a (Pavel Begunkov 2021-02-04 13:52:06 +0000 3009) static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
847595de1732a (Pavel Begunkov 2021-02-04 13:52:06 +0000 3010) struct iov_iter *iter, bool needs_lock)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3011) {
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 3012) void __user *buf = u64_to_user_ptr(req->rw.addr);
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 3013) size_t sqe_len = req->rw.len;
847595de1732a (Pavel Begunkov 2021-02-04 13:52:06 +0000 3014) u8 opcode = req->opcode;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3015) ssize_t ret;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 3016)
7d009165550ad (Pavel Begunkov 2019-11-25 23:14:40 +0300 3017) if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 3018) *iovec = NULL;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 3019) return io_import_fixed(req, rw, iter);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 3020) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3021)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 3022) /* buffer index only valid with fixed read/write, or buffer select */
4f4eeba87cc73 (Bijan Mottahedeh 2020-05-19 14:52:49 -0700 3023) if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 3024) return -EINVAL;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 3025)
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 3026) if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 3027) if (req->flags & REQ_F_BUFFER_SELECT) {
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3028) buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
867a23eab5284 (Pavel Begunkov 2020-08-20 11:34:39 +0300 3029) if (IS_ERR(buf))
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3030) return PTR_ERR(buf);
3f9d64415fdaa (Jens Axboe 2020-03-11 12:27:04 -0600 3031) req->rw.len = sqe_len;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 3032) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 3033)
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 3034) ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 3035) *iovec = NULL;
10fc72e433527 (David Laight 2020-11-07 13:16:25 +0000 3036) return ret;
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 3037) }
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 3038)
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3039) if (req->flags & REQ_F_BUFFER_SELECT) {
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3040) ret = io_iov_buffer_select(req, *iovec, needs_lock);
847595de1732a (Pavel Begunkov 2021-02-04 13:52:06 +0000 3041) if (!ret)
847595de1732a (Pavel Begunkov 2021-02-04 13:52:06 +0000 3042) iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3043) *iovec = NULL;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3044) return ret;
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3045) }
4d954c258a0c3 (Jens Axboe 2020-02-27 07:31:19 -0700 3046)
89cd35c58bc2e (Christoph Hellwig 2020-09-25 06:51:41 +0200 3047) return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
89cd35c58bc2e (Christoph Hellwig 2020-09-25 06:51:41 +0200 3048) req->ctx->compat);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3049) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3050)
0fef948363f62 (Jens Axboe 2020-08-26 10:36:20 -0600 3051) static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
0fef948363f62 (Jens Axboe 2020-08-26 10:36:20 -0600 3052) {
5b09e37e27a87 (Pavel Begunkov 2020-09-30 22:57:15 +0300 3053) return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
0fef948363f62 (Jens Axboe 2020-08-26 10:36:20 -0600 3054) }
0fef948363f62 (Jens Axboe 2020-08-26 10:36:20 -0600 3055)
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 3056) /*
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3057) * For files that don't have ->read_iter() and ->write_iter(), handle them
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3058) * by looping over ->read() or ->write() manually.
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 3059) */
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3060) static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3061) {
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3062) struct kiocb *kiocb = &req->rw.kiocb;
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3063) struct file *file = req->file;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3064) ssize_t ret = 0;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3065)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3066) /*
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3067) * Don't support polled IO through this interface, and we can't
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3068) * support non-blocking either. For the latter, this just causes
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3069) * the kiocb to be handled from an async context.
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3070) */
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3071) if (kiocb->ki_flags & IOCB_HIPRI)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3072) return -EOPNOTSUPP;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3073) if (kiocb->ki_flags & IOCB_NOWAIT)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3074) return -EAGAIN;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3075)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3076) while (iov_iter_count(iter)) {
311ae9e159d81 (Pavel Begunkov 2019-11-24 11:58:24 +0300 3077) struct iovec iovec;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3078) ssize_t nr;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3079)
311ae9e159d81 (Pavel Begunkov 2019-11-24 11:58:24 +0300 3080) if (!iov_iter_is_bvec(iter)) {
311ae9e159d81 (Pavel Begunkov 2019-11-24 11:58:24 +0300 3081) iovec = iov_iter_iovec(iter);
311ae9e159d81 (Pavel Begunkov 2019-11-24 11:58:24 +0300 3082) } else {
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3083) iovec.iov_base = u64_to_user_ptr(req->rw.addr);
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3084) iovec.iov_len = req->rw.len;
311ae9e159d81 (Pavel Begunkov 2019-11-24 11:58:24 +0300 3085) }
311ae9e159d81 (Pavel Begunkov 2019-11-24 11:58:24 +0300 3086)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3087) if (rw == READ) {
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3088) nr = file->f_op->read(file, iovec.iov_base,
0fef948363f62 (Jens Axboe 2020-08-26 10:36:20 -0600 3089) iovec.iov_len, io_kiocb_ppos(kiocb));
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3090) } else {
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3091) nr = file->f_op->write(file, iovec.iov_base,
0fef948363f62 (Jens Axboe 2020-08-26 10:36:20 -0600 3092) iovec.iov_len, io_kiocb_ppos(kiocb));
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3093) }
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3094)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3095) if (nr < 0) {
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3096) if (!ret)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3097) ret = nr;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3098) break;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3099) }
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3100) ret += nr;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3101) if (nr != iovec.iov_len)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3102) break;
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3103) req->rw.len -= nr;
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3104) req->rw.addr += nr;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3105) iov_iter_advance(iter, nr);
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3106) }
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3107)
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3108) return ret;
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3109) }
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3110)
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3111) static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3112) const struct iovec *fast_iov, struct iov_iter *iter)
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3113) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3114) struct io_async_rw *rw = req->async_data;
b64e3444d4e1c (Pavel Begunkov 2020-07-13 22:59:18 +0300 3115)
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3116) memcpy(&rw->iter, iter, sizeof(*iter));
afb87658f89b6 (Pavel Begunkov 2020-09-06 00:45:46 +0300 3117) rw->free_iovec = iovec;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3118) rw->bytes_done = 0;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3119) /* can only be fixed buffers, no need to do anything */
9c3a205c5ffa3 (Pavel Begunkov 2020-11-23 23:20:27 +0000 3120) if (iov_iter_is_bvec(iter))
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3121) return;
b64e3444d4e1c (Pavel Begunkov 2020-07-13 22:59:18 +0300 3122) if (!iovec) {
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3123) unsigned iov_off = 0;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3124)
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3125) rw->iter.iov = rw->fast_iov;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3126) if (iter->iov != fast_iov) {
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3127) iov_off = iter->iov - fast_iov;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3128) rw->iter.iov += iov_off;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3129) }
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3130) if (rw->fast_iov != fast_iov)
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3131) memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
45097daea2f4e (Xiaoguang Wang 2020-04-08 22:29:58 +0800 3132) sizeof(struct iovec) * iter->nr_segs);
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 3133) } else {
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 3134) req->flags |= REQ_F_NEED_CLEANUP;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3135) }
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3136) }
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3137)
6cb78689fa94c (Pavel Begunkov 2021-02-28 22:35:17 +0000 3138) static inline int io_alloc_async_data(struct io_kiocb *req)
3d9932a8b240c (Xiaoguang Wang 2020-03-27 15:36:52 +0800 3139) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3140) WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3141) req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3142) return req->async_data == NULL;
3d9932a8b240c (Xiaoguang Wang 2020-03-27 15:36:52 +0800 3143) }
3d9932a8b240c (Xiaoguang Wang 2020-03-27 15:36:52 +0800 3144)
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3145) static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3146) const struct iovec *fast_iov,
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3147) struct iov_iter *iter, bool force)
b7bb4f7da0a1a (Jens Axboe 2019-12-15 22:13:43 -0700 3148) {
26f0505a9ce57 (Pavel Begunkov 2021-02-28 22:35:18 +0000 3149) if (!force && !io_op_defs[req->opcode].needs_async_setup)
74566df3a71c1 (Jens Axboe 2020-01-13 19:23:24 -0700 3150) return 0;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3151) if (!req->async_data) {
6cb78689fa94c (Pavel Begunkov 2021-02-28 22:35:17 +0000 3152) if (io_alloc_async_data(req)) {
6bf985dc50dd8 (Pavel Begunkov 2021-02-04 13:52:01 +0000 3153) kfree(iovec);
5d204bcfa0933 (Jens Axboe 2020-01-31 12:06:52 -0700 3154) return -ENOMEM;
6bf985dc50dd8 (Pavel Begunkov 2021-02-04 13:52:01 +0000 3155) }
b7bb4f7da0a1a (Jens Axboe 2019-12-15 22:13:43 -0700 3156)
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3157) io_req_map_rw(req, iovec, fast_iov, iter);
5d204bcfa0933 (Jens Axboe 2020-01-31 12:06:52 -0700 3158) }
b7bb4f7da0a1a (Jens Axboe 2019-12-15 22:13:43 -0700 3159) return 0;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3160) }
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3161)
73debe68b300c (Pavel Begunkov 2020-09-30 22:57:54 +0300 3162) static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
c3e330a493740 (Pavel Begunkov 2020-07-13 22:59:19 +0300 3163) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3164) struct io_async_rw *iorw = req->async_data;
f4bff104fffba (Pavel Begunkov 2020-09-06 00:45:45 +0300 3165) struct iovec *iov = iorw->fast_iov;
847595de1732a (Pavel Begunkov 2021-02-04 13:52:06 +0000 3166) int ret;
c3e330a493740 (Pavel Begunkov 2020-07-13 22:59:19 +0300 3167)
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3168) ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
c3e330a493740 (Pavel Begunkov 2020-07-13 22:59:19 +0300 3169) if (unlikely(ret < 0))
c3e330a493740 (Pavel Begunkov 2020-07-13 22:59:19 +0300 3170) return ret;
c3e330a493740 (Pavel Begunkov 2020-07-13 22:59:19 +0300 3171)
ab0b196ce5551 (Pavel Begunkov 2020-09-06 00:45:47 +0300 3172) iorw->bytes_done = 0;
ab0b196ce5551 (Pavel Begunkov 2020-09-06 00:45:47 +0300 3173) iorw->free_iovec = iov;
ab0b196ce5551 (Pavel Begunkov 2020-09-06 00:45:47 +0300 3174) if (iov)
ab0b196ce5551 (Pavel Begunkov 2020-09-06 00:45:47 +0300 3175) req->flags |= REQ_F_NEED_CLEANUP;
c3e330a493740 (Pavel Begunkov 2020-07-13 22:59:19 +0300 3176) return 0;
c3e330a493740 (Pavel Begunkov 2020-07-13 22:59:19 +0300 3177) }
c3e330a493740 (Pavel Begunkov 2020-07-13 22:59:19 +0300 3178)
73debe68b300c (Pavel Begunkov 2020-09-30 22:57:54 +0300 3179) static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3180) {
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 3181) if (unlikely(!(req->file->f_mode & FMODE_READ)))
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 3182) return -EBADF;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 3183) return io_prep_rw(req, sqe);
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3184) }
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3185)
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3186) /*
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3187) * This is our waitqueue callback handler, registered through lock_page_async()
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3188) * when we initially tried to do the IO with the iocb armed our waitqueue.
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3189) * This gets called when the page is unlocked, and we generally expect that to
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3190) * happen when the page IO is completed and the page is now uptodate. This will
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3191) * queue a task_work based retry of the operation, attempting to copy the data
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3192) * again. If the latter fails because the page was NOT uptodate, then we will
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3193) * do a thread based blocking retry of the operation. That's the unexpected
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3194) * slow path.
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3195) */
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3196) static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3197) int sync, void *arg)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3198) {
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3199) struct wait_page_queue *wpq;
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3200) struct io_kiocb *req = wait->private;
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3201) struct wait_page_key *key = arg;
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3202)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3203) wpq = container_of(wait, struct wait_page_queue, wait);
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3204)
cdc8fcb49905c (Linus Torvalds 2020-08-03 13:01:22 -0700 3205) if (!wake_page_match(wpq, key))
cdc8fcb49905c (Linus Torvalds 2020-08-03 13:01:22 -0700 3206) return 0;
cdc8fcb49905c (Linus Torvalds 2020-08-03 13:01:22 -0700 3207)
c8d317aa1887b (Hao Xu 2020-09-29 20:00:45 +0800 3208) req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3209) list_del_init(&wait->entry);
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3210)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3211) /* submit ref gets dropped, acquire a new one */
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 3212) req_ref_get(req);
921b9054e0c4c (Pavel Begunkov 2021-02-12 03:23:53 +0000 3213) io_req_task_queue(req);
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3214) return 1;
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3215) }
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3216)
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3217) /*
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3218) * This controls whether a given IO request should be armed for async page
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3219) * based retry. If we return false here, the request is handed to the async
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3220) * worker threads for retry. If we're doing buffered reads on a regular file,
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3221) * we prepare a private wait_page_queue entry and retry the operation. This
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3222) * will either succeed because the page is now uptodate and unlocked, or it
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3223) * will register a callback when the page is unlocked at IO completion. Through
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3224) * that callback, io_uring uses task_work to setup a retry of the operation.
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3225) * That retry will attempt the buffered read again. The retry will generally
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3226) * succeed, or in rare cases where it fails, we then fall back to using the
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3227) * async worker threads for a blocking retry.
c1dd91d16246b (Jens Axboe 2020-08-03 16:43:59 -0600 3228) */
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3229) static bool io_rw_should_retry(struct io_kiocb *req)
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3230) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3231) struct io_async_rw *rw = req->async_data;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3232) struct wait_page_queue *wait = &rw->wpq;
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3233) struct kiocb *kiocb = &req->rw.kiocb;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3234)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3235) /* never retry for NOWAIT, we just complete with -EAGAIN */
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3236) if (req->flags & REQ_F_NOWAIT)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3237) return false;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3238)
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3239) /* Only for buffered IO */
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3240) if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3241) return false;
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3242)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3243) /*
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3244) * just use poll if we can, and don't attempt if the fs doesn't
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3245) * support callback based unlocks
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3246) */
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3247) if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3248) return false;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3249)
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3250) wait->wait.func = io_async_buf_func;
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3251) wait->wait.private = req;
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3252) wait->wait.flags = 0;
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3253) INIT_LIST_HEAD(&wait->wait.entry);
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3254) kiocb->ki_flags |= IOCB_WAITQ;
c8d317aa1887b (Hao Xu 2020-09-29 20:00:45 +0800 3255) kiocb->ki_flags &= ~IOCB_NOWAIT;
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3256) kiocb->ki_waitq = wait;
3b2a4439e0ae1 (Jens Axboe 2020-08-16 10:58:43 -0700 3257) return true;
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3258) }
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3259)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3260) static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3261) {
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3262) if (req->file->f_op->read_iter)
bcf5a06304d69 (Jens Axboe 2020-05-22 09:24:42 -0600 3263) return call_read_iter(req->file, &req->rw.kiocb, iter);
2dd2111d0d383 (Guoyu Huang 2020-08-05 03:53:50 -0700 3264) else if (req->file->f_op->read)
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3265) return loop_rw_iter(READ, req, iter);
2dd2111d0d383 (Guoyu Huang 2020-08-05 03:53:50 -0700 3266) else
2dd2111d0d383 (Guoyu Huang 2020-08-05 03:53:50 -0700 3267) return -EINVAL;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3268) }
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3269)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 3270) static int io_read(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3271) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3272) struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 3273) struct kiocb *kiocb = &req->rw.kiocb;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3274) struct iov_iter __iter, *iter = &__iter;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3275) struct io_async_rw *rw = req->async_data;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3276) ssize_t io_size, ret, ret2;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3277) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3278)
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3279) if (rw) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3280) iter = &rw->iter;
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3281) iovec = NULL;
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3282) } else {
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3283) ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3284) if (ret < 0)
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3285) return ret;
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3286) }
632546c4b5a4d (Pavel Begunkov 2020-11-07 13:16:26 +0000 3287) io_size = iov_iter_count(iter);
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3288) req->result = io_size;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3289)
fd6c2e4c063d6 (Jens Axboe 2019-12-18 12:19:41 -0700 3290) /* Ensure we clear previously set non-block flag */
fd6c2e4c063d6 (Jens Axboe 2019-12-18 12:19:41 -0700 3291) if (!force_nonblock)
29de5f6a35077 (Jens Axboe 2020-02-20 09:56:08 -0700 3292) kiocb->ki_flags &= ~IOCB_NOWAIT;
a88fc400212fc (Pavel Begunkov 2020-09-30 22:57:53 +0300 3293) else
a88fc400212fc (Pavel Begunkov 2020-09-30 22:57:53 +0300 3294) kiocb->ki_flags |= IOCB_NOWAIT;
a88fc400212fc (Pavel Begunkov 2020-09-30 22:57:53 +0300 3295)
24c74678634b3 (Pavel Begunkov 2020-06-21 13:09:51 +0300 3296) /* If the file doesn't support async, just async punt */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 3297) if (force_nonblock && !io_file_supports_async(req, READ)) {
6713e7a6145a4 (Pavel Begunkov 2021-02-04 13:51:59 +0000 3298) ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
6bf985dc50dd8 (Pavel Begunkov 2021-02-04 13:52:01 +0000 3299) return ret ?: -EAGAIN;
6713e7a6145a4 (Pavel Begunkov 2021-02-04 13:51:59 +0000 3300) }
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 3301)
632546c4b5a4d (Pavel Begunkov 2020-11-07 13:16:26 +0000 3302) ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size);
5ea5dd45844d1 (Pavel Begunkov 2021-02-04 13:52:03 +0000 3303) if (unlikely(ret)) {
5ea5dd45844d1 (Pavel Begunkov 2021-02-04 13:52:03 +0000 3304) kfree(iovec);
5ea5dd45844d1 (Pavel Begunkov 2021-02-04 13:52:03 +0000 3305) return ret;
5ea5dd45844d1 (Pavel Begunkov 2021-02-04 13:52:03 +0000 3306) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3307)
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3308) ret = io_iter_do_read(req, iter);
32960613b7c33 (Jens Axboe 2019-09-23 11:05:34 -0600 3309)
230d50d448acb (Jens Axboe 2021-04-01 20:41:15 -0600 3310) if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
6ad7f2332e84c (Pavel Begunkov 2021-04-08 01:54:39 +0100 3311) req->flags &= ~REQ_F_REISSUE;
eefdf30f3dcb5 (Jens Axboe 2020-08-27 16:40:19 -0600 3312) /* IOPOLL retry should happen for io-wq threads */
eefdf30f3dcb5 (Jens Axboe 2020-08-27 16:40:19 -0600 3313) if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
f91daf565b0e2 (Jens Axboe 2020-08-15 15:58:42 -0700 3314) goto done;
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 3315) /* no retry on NONBLOCK nor RWF_NOWAIT */
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 3316) if (req->flags & REQ_F_NOWAIT)
355afaeb578ab (Jens Axboe 2020-09-02 09:30:31 -0600 3317) goto done;
842163154b87b (Jens Axboe 2020-08-24 11:45:26 -0600 3318) /* some cases will consume bytes even on error returns */
632546c4b5a4d (Pavel Begunkov 2020-11-07 13:16:26 +0000 3319) iov_iter_revert(iter, io_size - iov_iter_count(iter));
f38c7e3abfba9 (Jens Axboe 2020-09-25 15:23:43 -0600 3320) ret = 0;
230d50d448acb (Jens Axboe 2021-04-01 20:41:15 -0600 3321) } else if (ret == -EIOCBQUEUED) {
230d50d448acb (Jens Axboe 2021-04-01 20:41:15 -0600 3322) goto out_free;
7335e3bf9d0a9 (Pavel Begunkov 2021-02-04 13:52:02 +0000 3323) } else if (ret <= 0 || ret == io_size || !force_nonblock ||
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 3324) (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) {
7335e3bf9d0a9 (Pavel Begunkov 2021-02-04 13:52:02 +0000 3325) /* read all, failed, already did sync or don't want to retry */
00d23d516e2e7 (Jens Axboe 2020-08-25 12:59:22 -0600 3326) goto done;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3327) }
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3328)
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3329) ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
6bf985dc50dd8 (Pavel Begunkov 2021-02-04 13:52:01 +0000 3330) if (ret2)
6bf985dc50dd8 (Pavel Begunkov 2021-02-04 13:52:01 +0000 3331) return ret2;
6bf985dc50dd8 (Pavel Begunkov 2021-02-04 13:52:01 +0000 3332)
fe1cdd5586195 (Pavel Begunkov 2021-02-17 21:02:36 +0000 3333) iovec = NULL;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3334) rw = req->async_data;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3335) /* now use our persistent iterator, if we aren't already */
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3336) iter = &rw->iter;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3337)
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3338) do {
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3339) io_size -= ret;
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3340) rw->bytes_done += ret;
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3341) /* if we can retry, do so with the callbacks armed */
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3342) if (!io_rw_should_retry(req)) {
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3343) kiocb->ki_flags &= ~IOCB_WAITQ;
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3344) return -EAGAIN;
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3345) }
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3346)
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3347) /*
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3348) * Now retry read with the IOCB_WAITQ parts set in the iocb. If
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3349) * we get -EIOCBQUEUED, then we'll get a notification when the
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3350) * desired page gets unlocked. We can also get a partial read
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3351) * here, and if we do, then just retry at the new offset.
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3352) */
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3353) ret = io_iter_do_read(req, iter);
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3354) if (ret == -EIOCBQUEUED)
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3355) return 0;
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3356) /* we got some bytes, but not all. retry. */
b5b0ecb736f1c (Jens Axboe 2021-03-04 21:02:58 -0700 3357) kiocb->ki_flags &= ~IOCB_WAITQ;
b23df91bff954 (Pavel Begunkov 2021-02-04 13:52:04 +0000 3358) } while (ret > 0 && ret < io_size);
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3359) done:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 3360) kiocb_done(kiocb, ret, issue_flags);
fe1cdd5586195 (Pavel Begunkov 2021-02-17 21:02:36 +0000 3361) out_free:
fe1cdd5586195 (Pavel Begunkov 2021-02-17 21:02:36 +0000 3362) /* it's faster to check here then delegate to kfree */
fe1cdd5586195 (Pavel Begunkov 2021-02-17 21:02:36 +0000 3363) if (iovec)
fe1cdd5586195 (Pavel Begunkov 2021-02-17 21:02:36 +0000 3364) kfree(iovec);
5ea5dd45844d1 (Pavel Begunkov 2021-02-04 13:52:03 +0000 3365) return 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3366) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3367)
73debe68b300c (Pavel Begunkov 2020-09-30 22:57:54 +0300 3368) static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3369) {
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 3370) if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 3371) return -EBADF;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 3372) return io_prep_rw(req, sqe);
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3373) }
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3374)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 3375) static int io_write(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3376) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3377) struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 3378) struct kiocb *kiocb = &req->rw.kiocb;
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3379) struct iov_iter __iter, *iter = &__iter;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3380) struct io_async_rw *rw = req->async_data;
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3381) ssize_t ret, ret2, io_size;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3382) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3383)
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3384) if (rw) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 3385) iter = &rw->iter;
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3386) iovec = NULL;
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3387) } else {
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3388) ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3389) if (ret < 0)
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3390) return ret;
2846c481c9dd1 (Pavel Begunkov 2020-11-07 13:16:27 +0000 3391) }
632546c4b5a4d (Pavel Begunkov 2020-11-07 13:16:26 +0000 3392) io_size = iov_iter_count(iter);
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3393) req->result = io_size;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3394)
fd6c2e4c063d6 (Jens Axboe 2019-12-18 12:19:41 -0700 3395) /* Ensure we clear previously set non-block flag */
fd6c2e4c063d6 (Jens Axboe 2019-12-18 12:19:41 -0700 3396) if (!force_nonblock)
a88fc400212fc (Pavel Begunkov 2020-09-30 22:57:53 +0300 3397) kiocb->ki_flags &= ~IOCB_NOWAIT;
a88fc400212fc (Pavel Begunkov 2020-09-30 22:57:53 +0300 3398) else
a88fc400212fc (Pavel Begunkov 2020-09-30 22:57:53 +0300 3399) kiocb->ki_flags |= IOCB_NOWAIT;
fd6c2e4c063d6 (Jens Axboe 2019-12-18 12:19:41 -0700 3400)
24c74678634b3 (Pavel Begunkov 2020-06-21 13:09:51 +0300 3401) /* If the file doesn't support async, just async punt */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 3402) if (force_nonblock && !io_file_supports_async(req, WRITE))
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3403) goto copy_iov;
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 3404)
10d59345578a1 (Jens Axboe 2019-12-09 20:16:22 -0700 3405) /* file path doesn't support NOWAIT for non-direct_IO */
10d59345578a1 (Jens Axboe 2019-12-09 20:16:22 -0700 3406) if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
10d59345578a1 (Jens Axboe 2019-12-09 20:16:22 -0700 3407) (req->flags & REQ_F_ISREG))
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3408) goto copy_iov;
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 3409)
632546c4b5a4d (Pavel Begunkov 2020-11-07 13:16:26 +0000 3410) ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size);
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3411) if (unlikely(ret))
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3412) goto out_free;
4ed734b0d0913 (Jens Axboe 2020-03-20 11:23:41 -0600 3413)
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3414) /*
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3415) * Open-code file_start_write here to grab freeze protection,
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3416) * which will be released by another thread in
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3417) * io_complete_rw(). Fool lockdep by telling it the lock got
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3418) * released so that it doesn't complain about the held lock when
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3419) * we return to userspace.
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3420) */
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3421) if (req->flags & REQ_F_ISREG) {
8a3c84b649b03 (Darrick J. Wong 2020-11-10 16:50:21 -0800 3422) sb_start_write(file_inode(req->file)->i_sb);
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3423) __sb_writers_release(file_inode(req->file)->i_sb,
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3424) SB_FREEZE_WRITE);
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3425) }
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3426) kiocb->ki_flags |= IOCB_WRITE;
4ed734b0d0913 (Jens Axboe 2020-03-20 11:23:41 -0600 3427)
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3428) if (req->file->f_op->write_iter)
ff6165b2d7f66 (Jens Axboe 2020-08-13 09:47:43 -0600 3429) ret2 = call_write_iter(req->file, kiocb, iter);
2dd2111d0d383 (Guoyu Huang 2020-08-05 03:53:50 -0700 3430) else if (req->file->f_op->write)
4017eb91a9e79 (Jens Axboe 2020-10-22 14:14:12 -0600 3431) ret2 = loop_rw_iter(WRITE, req, iter);
2dd2111d0d383 (Guoyu Huang 2020-08-05 03:53:50 -0700 3432) else
2dd2111d0d383 (Guoyu Huang 2020-08-05 03:53:50 -0700 3433) ret2 = -EINVAL;
4ed734b0d0913 (Jens Axboe 2020-03-20 11:23:41 -0600 3434)
6ad7f2332e84c (Pavel Begunkov 2021-04-08 01:54:39 +0100 3435) if (req->flags & REQ_F_REISSUE) {
6ad7f2332e84c (Pavel Begunkov 2021-04-08 01:54:39 +0100 3436) req->flags &= ~REQ_F_REISSUE;
230d50d448acb (Jens Axboe 2021-04-01 20:41:15 -0600 3437) ret2 = -EAGAIN;
6ad7f2332e84c (Pavel Begunkov 2021-04-08 01:54:39 +0100 3438) }
230d50d448acb (Jens Axboe 2021-04-01 20:41:15 -0600 3439)
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3440) /*
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3441) * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3442) * retry them without IOCB_NOWAIT.
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3443) */
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3444) if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3445) ret2 = -EAGAIN;
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 3446) /* no retry on NONBLOCK nor RWF_NOWAIT */
75c668cdd6ca0 (Pavel Begunkov 2021-02-04 13:52:05 +0000 3447) if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
355afaeb578ab (Jens Axboe 2020-09-02 09:30:31 -0600 3448) goto done;
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3449) if (!force_nonblock || ret2 != -EAGAIN) {
eefdf30f3dcb5 (Jens Axboe 2020-08-27 16:40:19 -0600 3450) /* IOPOLL retry should happen for io-wq threads */
eefdf30f3dcb5 (Jens Axboe 2020-08-27 16:40:19 -0600 3451) if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
eefdf30f3dcb5 (Jens Axboe 2020-08-27 16:40:19 -0600 3452) goto copy_iov;
355afaeb578ab (Jens Axboe 2020-09-02 09:30:31 -0600 3453) done:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 3454) kiocb_done(kiocb, ret2, issue_flags);
fa15bafb71fd7 (Pavel Begunkov 2020-08-01 13:50:02 +0300 3455) } else {
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 3456) copy_iov:
842163154b87b (Jens Axboe 2020-08-24 11:45:26 -0600 3457) /* some cases will consume bytes even on error returns */
632546c4b5a4d (Pavel Begunkov 2020-11-07 13:16:26 +0000 3458) iov_iter_revert(iter, io_size - iov_iter_count(iter));
227c0c9673d86 (Jens Axboe 2020-08-13 11:51:40 -0600 3459) ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
6bf985dc50dd8 (Pavel Begunkov 2021-02-04 13:52:01 +0000 3460) return ret ?: -EAGAIN;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3461) }
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 3462) out_free:
f261c16861b82 (Pavel Begunkov 2020-08-20 11:34:10 +0300 3463) /* it's reportedly faster than delegating the null check to kfree() */
252917c30f551 (Pavel Begunkov 2020-07-13 22:59:20 +0300 3464) if (iovec)
6f2cc1664db20 (Xiaoguang Wang 2020-06-18 15:01:56 +0800 3465) kfree(iovec);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3466) return ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3467) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3468)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3469) static int io_renameat_prep(struct io_kiocb *req,
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3470) const struct io_uring_sqe *sqe)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3471) {
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3472) struct io_rename *ren = &req->rename;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3473) const char __user *oldf, *newf;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3474)
4f5be8b54bd4f (Jens Axboe 2021-06-23 09:04:13 -0600 3475) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4f5be8b54bd4f (Jens Axboe 2021-06-23 09:04:13 -0600 3476) return -EINVAL;
4f5be8b54bd4f (Jens Axboe 2021-06-23 09:04:13 -0600 3477) if (sqe->ioprio || sqe->buf_index)
4f5be8b54bd4f (Jens Axboe 2021-06-23 09:04:13 -0600 3478) return -EINVAL;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3479) if (unlikely(req->flags & REQ_F_FIXED_FILE))
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3480) return -EBADF;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3481)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3482) ren->old_dfd = READ_ONCE(sqe->fd);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3483) oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3484) newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3485) ren->new_dfd = READ_ONCE(sqe->len);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3486) ren->flags = READ_ONCE(sqe->rename_flags);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3487)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3488) ren->oldpath = getname(oldf);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3489) if (IS_ERR(ren->oldpath))
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3490) return PTR_ERR(ren->oldpath);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3491)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3492) ren->newpath = getname(newf);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3493) if (IS_ERR(ren->newpath)) {
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3494) putname(ren->oldpath);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3495) return PTR_ERR(ren->newpath);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3496) }
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3497)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3498) req->flags |= REQ_F_NEED_CLEANUP;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3499) return 0;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3500) }
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3501)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3502) static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3503) {
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3504) struct io_rename *ren = &req->rename;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3505) int ret;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3506)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3507) if (issue_flags & IO_URING_F_NONBLOCK)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3508) return -EAGAIN;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3509)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3510) ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3511) ren->newpath, ren->flags);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3512)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3513) req->flags &= ~REQ_F_NEED_CLEANUP;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3514) if (ret < 0)
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3515) req_set_fail_links(req);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3516) io_req_complete(req, ret);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3517) return 0;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3518) }
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 3519)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3520) static int io_unlinkat_prep(struct io_kiocb *req,
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3521) const struct io_uring_sqe *sqe)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3522) {
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3523) struct io_unlink *un = &req->unlink;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3524) const char __user *fname;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3525)
03105a24cd2b2 (Jens Axboe 2021-06-23 09:07:45 -0600 3526) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
03105a24cd2b2 (Jens Axboe 2021-06-23 09:07:45 -0600 3527) return -EINVAL;
03105a24cd2b2 (Jens Axboe 2021-06-23 09:07:45 -0600 3528) if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
03105a24cd2b2 (Jens Axboe 2021-06-23 09:07:45 -0600 3529) return -EINVAL;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3530) if (unlikely(req->flags & REQ_F_FIXED_FILE))
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3531) return -EBADF;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3532)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3533) un->dfd = READ_ONCE(sqe->fd);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3534)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3535) un->flags = READ_ONCE(sqe->unlink_flags);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3536) if (un->flags & ~AT_REMOVEDIR)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3537) return -EINVAL;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3538)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3539) fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3540) un->filename = getname(fname);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3541) if (IS_ERR(un->filename))
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3542) return PTR_ERR(un->filename);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3543)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3544) req->flags |= REQ_F_NEED_CLEANUP;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3545) return 0;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3546) }
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3547)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3548) static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3549) {
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3550) struct io_unlink *un = &req->unlink;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3551) int ret;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3552)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3553) if (issue_flags & IO_URING_F_NONBLOCK)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3554) return -EAGAIN;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3555)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3556) if (un->flags & AT_REMOVEDIR)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3557) ret = do_rmdir(un->dfd, un->filename);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3558) else
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3559) ret = do_unlinkat(un->dfd, un->filename);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3560)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3561) req->flags &= ~REQ_F_NEED_CLEANUP;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3562) if (ret < 0)
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3563) req_set_fail_links(req);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3564) io_req_complete(req, ret);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3565) return 0;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3566) }
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 3567)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3568) static int io_shutdown_prep(struct io_kiocb *req,
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3569) const struct io_uring_sqe *sqe)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3570) {
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3571) #if defined(CONFIG_NET)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3572) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3573) return -EINVAL;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3574) if (sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3575) sqe->buf_index)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3576) return -EINVAL;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3577)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3578) req->shutdown.how = READ_ONCE(sqe->len);
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3579) return 0;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3580) #else
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3581) return -EOPNOTSUPP;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3582) #endif
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3583) }
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3584)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3585) static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3586) {
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3587) #if defined(CONFIG_NET)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3588) struct socket *sock;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3589) int ret;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3590)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3591) if (issue_flags & IO_URING_F_NONBLOCK)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3592) return -EAGAIN;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3593)
48aba79bcf6ea (Linus Torvalds 2020-12-16 12:44:05 -0800 3594) sock = sock_from_file(req->file);
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3595) if (unlikely(!sock))
48aba79bcf6ea (Linus Torvalds 2020-12-16 12:44:05 -0800 3596) return -ENOTSOCK;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3597)
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3598) ret = __sys_shutdown_sock(sock, req->shutdown.how);
a146468d76e04 (Jens Axboe 2020-12-14 20:57:27 -0700 3599) if (ret < 0)
a146468d76e04 (Jens Axboe 2020-12-14 20:57:27 -0700 3600) req_set_fail_links(req);
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3601) io_req_complete(req, ret);
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3602) return 0;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3603) #else
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3604) return -EOPNOTSUPP;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3605) #endif
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3606) }
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 3607)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3608) static int __io_splice_prep(struct io_kiocb *req,
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3609) const struct io_uring_sqe *sqe)
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3610) {
501449420a42c (Pavel Begunkov 2021-06-24 15:09:57 +0100 3611) struct io_splice *sp = &req->splice;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3612) unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3613)
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 3614) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 3615) return -EINVAL;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3616)
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3617) sp->file_in = NULL;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3618) sp->len = READ_ONCE(sqe->len);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3619) sp->flags = READ_ONCE(sqe->splice_flags);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3620)
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3621) if (unlikely(sp->flags & ~valid_flags))
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3622) return -EINVAL;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3623)
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 3624) sp->file_in = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in),
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 3625) (sp->flags & SPLICE_F_FD_IN_FIXED));
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 3626) if (!sp->file_in)
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 3627) return -EBADF;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3628) req->flags |= REQ_F_NEED_CLEANUP;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3629) return 0;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3630) }
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3631)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3632) static int io_tee_prep(struct io_kiocb *req,
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3633) const struct io_uring_sqe *sqe)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3634) {
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3635) if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3636) return -EINVAL;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3637) return __io_splice_prep(req, sqe);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3638) }
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3639)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3640) static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3641) {
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3642) struct io_splice *sp = &req->splice;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3643) struct file *in = sp->file_in;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3644) struct file *out = sp->file_out;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3645) unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3646) long ret = 0;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3647)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3648) if (issue_flags & IO_URING_F_NONBLOCK)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3649) return -EAGAIN;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3650) if (sp->len)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3651) ret = do_tee(in, out, sp->len, flags);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3652)
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 3653) if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 3654) io_put_file(in);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3655) req->flags &= ~REQ_F_NEED_CLEANUP;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3656)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3657) if (ret != sp->len)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3658) req_set_fail_links(req);
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 3659) io_req_complete(req, ret);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3660) return 0;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3661) }
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3662)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3663) static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3664) {
501449420a42c (Pavel Begunkov 2021-06-24 15:09:57 +0100 3665) struct io_splice *sp = &req->splice;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3666)
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3667) sp->off_in = READ_ONCE(sqe->splice_off_in);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3668) sp->off_out = READ_ONCE(sqe->off);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3669) return __io_splice_prep(req, sqe);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3670) }
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 3671)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3672) static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3673) {
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3674) struct io_splice *sp = &req->splice;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3675) struct file *in = sp->file_in;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3676) struct file *out = sp->file_out;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3677) unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3678) loff_t *poff_in, *poff_out;
c96874265cd04 (Pavel Begunkov 2020-05-04 23:00:54 +0300 3679) long ret = 0;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3680)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3681) if (issue_flags & IO_URING_F_NONBLOCK)
2fb3e82284fca (Pavel Begunkov 2020-05-01 17:09:38 +0300 3682) return -EAGAIN;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3683)
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3684) poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3685) poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
c96874265cd04 (Pavel Begunkov 2020-05-04 23:00:54 +0300 3686)
948a7749454b1 (Jens Axboe 2020-05-17 14:21:38 -0600 3687) if (sp->len)
c96874265cd04 (Pavel Begunkov 2020-05-04 23:00:54 +0300 3688) ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3689)
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 3690) if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 3691) io_put_file(in);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3692) req->flags &= ~REQ_F_NEED_CLEANUP;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3693)
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3694) if (ret != sp->len)
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3695) req_set_fail_links(req);
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 3696) io_req_complete(req, ret);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3697) return 0;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3698) }
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 3699)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3700) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3701) * IORING_OP_NOP just posts a completion event, nothing else.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3702) */
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 3703) static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3704) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3705) struct io_ring_ctx *ctx = req->ctx;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3706)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 3707) if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 3708) return -EINVAL;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 3709)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 3710) __io_req_complete(req, issue_flags, 0, 0);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3711) return 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3712) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 3713)
1155c76a24836 (Pavel Begunkov 2021-02-18 18:29:38 +0000 3714) static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3715) {
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 3716) struct io_ring_ctx *ctx = req->ctx;
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3717)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 3718) if (!req->file)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 3719) return -EBADF;
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3720)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 3721) if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 3722) return -EINVAL;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 3723) if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3724) return -EINVAL;
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3725)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3726) req->sync.flags = READ_ONCE(sqe->fsync_flags);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3727) if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3728) return -EINVAL;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3729)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3730) req->sync.off = READ_ONCE(sqe->off);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3731) req->sync.len = READ_ONCE(sqe->len);
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3732) return 0;
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3733) }
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3734)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3735) static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3736) {
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3737) loff_t end = req->sync.off + req->sync.len;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3738) int ret;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3739)
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3740) /* fsync always requires a blocking context */
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3741) if (issue_flags & IO_URING_F_NONBLOCK)
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3742) return -EAGAIN;
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3743)
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 3744) ret = vfs_fsync_range(req->file, req->sync.off,
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3745) end > 0 ? end : LLONG_MAX,
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3746) req->sync.flags & IORING_FSYNC_DATASYNC);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3747) if (ret < 0)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 3748) req_set_fail_links(req);
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 3749) io_req_complete(req, ret);
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3750) return 0;
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3751) }
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 3752)
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3753) static int io_fallocate_prep(struct io_kiocb *req,
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3754) const struct io_uring_sqe *sqe)
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3755) {
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3756) if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3757) return -EINVAL;
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 3758) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 3759) return -EINVAL;
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3760)
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3761) req->sync.off = READ_ONCE(sqe->off);
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3762) req->sync.len = READ_ONCE(sqe->addr);
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3763) req->sync.mode = READ_ONCE(sqe->len);
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3764) return 0;
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3765) }
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3766)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3767) static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 3768) {
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3769) int ret;
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3770)
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 3771) /* fallocate always requiring blocking context */
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3772) if (issue_flags & IO_URING_F_NONBLOCK)
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 3773) return -EAGAIN;
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3774) ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3775) req->sync.len);
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3776) if (ret < 0)
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 3777) req_set_fail_links(req);
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 3778) io_req_complete(req, ret);
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 3779) return 0;
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 3780) }
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 3781)
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3782) static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
b7bb4f7da0a1a (Jens Axboe 2019-12-15 22:13:43 -0700 3783) {
f8748881b17dc (Jens Axboe 2020-01-08 17:47:02 -0700 3784) const char __user *fname;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3785) int ret;
b7bb4f7da0a1a (Jens Axboe 2019-12-15 22:13:43 -0700 3786)
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3787) if (unlikely(sqe->ioprio || sqe->buf_index))
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3788) return -EINVAL;
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3789) if (unlikely(req->flags & REQ_F_FIXED_FILE))
cf3040ca55f20 (Jens Axboe 2020-02-06 21:31:40 -0700 3790) return -EBADF;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 3791)
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3792) /* open.how should be already initialised */
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3793) if (!(req->open.how.flags & O_PATH) && force_o_largefile())
08a1d26eb894a (Jens Axboe 2020-04-08 09:20:54 -0600 3794) req->open.how.flags |= O_LARGEFILE;
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 3795)
25e72d1012b30 (Pavel Begunkov 2020-06-03 18:03:23 +0300 3796) req->open.dfd = READ_ONCE(sqe->fd);
25e72d1012b30 (Pavel Begunkov 2020-06-03 18:03:23 +0300 3797) fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
f8748881b17dc (Jens Axboe 2020-01-08 17:47:02 -0700 3798) req->open.filename = getname(fname);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3799) if (IS_ERR(req->open.filename)) {
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3800) ret = PTR_ERR(req->open.filename);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3801) req->open.filename = NULL;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3802) return ret;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3803) }
4022e7af86be2 (Jens Axboe 2020-03-19 19:23:18 -0600 3804) req->open.nofile = rlimit(RLIMIT_NOFILE);
8fef80bf56a49 (Pavel Begunkov 2020-02-07 23:59:53 +0300 3805) req->flags |= REQ_F_NEED_CLEANUP;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3806) return 0;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 3807) }
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 3808)
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3809) static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3810) {
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3811) u64 flags, mode;
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3812)
14587a46646d3 (Jens Axboe 2020-09-05 11:36:08 -0600 3813) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4eb8dded6b82e (Jens Axboe 2020-09-18 19:36:24 -0600 3814) return -EINVAL;
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3815) mode = READ_ONCE(sqe->len);
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3816) flags = READ_ONCE(sqe->open_flags);
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3817) req->open.how = build_open_how(flags, mode);
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3818) return __io_openat_prep(req, sqe);
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3819) }
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3820)
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3821) static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 3822) {
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3823) struct open_how __user *how;
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3824) size_t len;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 3825) int ret;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 3826)
14587a46646d3 (Jens Axboe 2020-09-05 11:36:08 -0600 3827) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4eb8dded6b82e (Jens Axboe 2020-09-18 19:36:24 -0600 3828) return -EINVAL;
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3829) how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3830) len = READ_ONCE(sqe->len);
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3831) if (len < OPEN_HOW_SIZE_VER0)
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3832) return -EINVAL;
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 3833)
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3834) ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3835) len);
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3836) if (ret)
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3837) return ret;
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 3838)
ec65fea5a8d7a (Pavel Begunkov 2020-06-03 18:03:24 +0300 3839) return __io_openat_prep(req, sqe);
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3840) }
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3841)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3842) static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3843) {
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3844) struct open_flags op;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3845) struct file *file;
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3846) bool nonblock_set;
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3847) bool resolve_nonblock;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3848) int ret;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3849)
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3850) ret = build_open_flags(&req->open.how, &op);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3851) if (ret)
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3852) goto err;
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3853) nonblock_set = op.open_flag & O_NONBLOCK;
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3854) resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3855) if (issue_flags & IO_URING_F_NONBLOCK) {
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3856) /*
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3857) * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3858) * it'll always -EAGAIN
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3859) */
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3860) if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3861) return -EAGAIN;
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3862) op.lookup_flags |= LOOKUP_CACHED;
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3863) op.open_flag |= O_NONBLOCK;
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3864) }
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3865)
4022e7af86be2 (Jens Axboe 2020-03-19 19:23:18 -0600 3866) ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3867) if (ret < 0)
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3868) goto err;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3869)
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3870) file = do_filp_open(req->open.dfd, req->open.filename, &op);
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3871) /* only retry if RESOLVE_CACHED wasn't already set by application */
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3872) if ((!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)) &&
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3873) file == ERR_PTR(-EAGAIN)) {
944d1444d53f5 (Jens Axboe 2020-11-13 16:48:44 -0700 3874) /*
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3875) * We could hang on to this 'fd', but seems like marginal
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3876) * gain for something that is now known to be a slower path.
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3877) * So just put it, and we'll get a new one when we retry.
944d1444d53f5 (Jens Axboe 2020-11-13 16:48:44 -0700 3878) */
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3879) put_unused_fd(ret);
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3880) return -EAGAIN;
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3881) }
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3882)
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3883) if (IS_ERR(file)) {
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3884) put_unused_fd(ret);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3885) ret = PTR_ERR(file);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3886) } else {
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3887) if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
3a81fd02045c3 (Jens Axboe 2020-12-10 12:25:36 -0700 3888) file->f_flags &= ~O_NONBLOCK;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3889) fsnotify_open(file);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3890) fd_install(ret, file);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3891) }
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3892) err:
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3893) putname(req->open.filename);
8fef80bf56a49 (Pavel Begunkov 2020-02-07 23:59:53 +0300 3894) req->flags &= ~REQ_F_NEED_CLEANUP;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3895) if (ret < 0)
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3896) req_set_fail_links(req);
0bdf3398b06ef (Pavel Begunkov 2021-04-11 01:46:29 +0100 3897) __io_req_complete(req, issue_flags, ret, 0);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3898) return 0;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3899) }
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 3900)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3901) static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3902) {
e45cff5885888 (Pavel Begunkov 2021-02-28 22:35:14 +0000 3903) return io_openat2(req, issue_flags);
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3904) }
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 3905)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3906) static int io_remove_buffers_prep(struct io_kiocb *req,
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3907) const struct io_uring_sqe *sqe)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3908) {
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3909) struct io_provide_buf *p = &req->pbuf;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3910) u64 tmp;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3911)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3912) if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3913) return -EINVAL;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3914)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3915) tmp = READ_ONCE(sqe->fd);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3916) if (!tmp || tmp > USHRT_MAX)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3917) return -EINVAL;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3918)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3919) memset(p, 0, sizeof(*p));
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3920) p->nbufs = tmp;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3921) p->bgid = READ_ONCE(sqe->buf_group);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3922) return 0;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3923) }
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3924)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3925) static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3926) int bgid, unsigned nbufs)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3927) {
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3928) unsigned i = 0;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3929)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3930) /* shouldn't happen */
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3931) if (!nbufs)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3932) return 0;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3933)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3934) /* the head kbuf is the list itself */
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3935) while (!list_empty(&buf->list)) {
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3936) struct io_buffer *nxt;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3937)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3938) nxt = list_first_entry(&buf->list, struct io_buffer, list);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3939) list_del(&nxt->list);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3940) kfree(nxt);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3941) if (++i == nbufs)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3942) return i;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3943) }
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3944) i++;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3945) kfree(buf);
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 3946) xa_erase(&ctx->io_buffers, bgid);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3947)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3948) return i;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3949) }
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3950)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 3951) static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3952) {
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3953) struct io_provide_buf *p = &req->pbuf;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3954) struct io_ring_ctx *ctx = req->ctx;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3955) struct io_buffer *head;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3956) int ret = 0;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 3957) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3958)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3959) io_ring_submit_lock(ctx, !force_nonblock);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3960)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3961) lockdep_assert_held(&ctx->uring_lock);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3962)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3963) ret = -ENOENT;
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 3964) head = xa_load(&ctx->io_buffers, p->bgid);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3965) if (head)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3966) ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3967) if (ret < 0)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3968) req_set_fail_links(req);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3969)
9fb8cb49c7b63 (Pavel Begunkov 2021-02-28 22:35:13 +0000 3970) /* complete before unlock, IOPOLL may need the lock */
9fb8cb49c7b63 (Pavel Begunkov 2021-02-28 22:35:13 +0000 3971) __io_req_complete(req, issue_flags, ret, 0);
9fb8cb49c7b63 (Pavel Begunkov 2021-02-28 22:35:13 +0000 3972) io_ring_submit_unlock(ctx, !force_nonblock);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3973) return 0;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3974) }
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 3975)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3976) static int io_provide_buffers_prep(struct io_kiocb *req,
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3977) const struct io_uring_sqe *sqe)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3978) {
38134ada0ceea (Pavel Begunkov 2021-04-15 13:07:39 +0100 3979) unsigned long size, tmp_check;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3980) struct io_provide_buf *p = &req->pbuf;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3981) u64 tmp;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3982)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3983) if (sqe->ioprio || sqe->rw_flags)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3984) return -EINVAL;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3985)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3986) tmp = READ_ONCE(sqe->fd);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3987) if (!tmp || tmp > USHRT_MAX)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3988) return -E2BIG;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3989) p->nbufs = tmp;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3990) p->addr = READ_ONCE(sqe->addr);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3991) p->len = READ_ONCE(sqe->len);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 3992)
38134ada0ceea (Pavel Begunkov 2021-04-15 13:07:39 +0100 3993) if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
38134ada0ceea (Pavel Begunkov 2021-04-15 13:07:39 +0100 3994) &size))
38134ada0ceea (Pavel Begunkov 2021-04-15 13:07:39 +0100 3995) return -EOVERFLOW;
38134ada0ceea (Pavel Begunkov 2021-04-15 13:07:39 +0100 3996) if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
38134ada0ceea (Pavel Begunkov 2021-04-15 13:07:39 +0100 3997) return -EOVERFLOW;
38134ada0ceea (Pavel Begunkov 2021-04-15 13:07:39 +0100 3998)
d81269fecb8ce (Pavel Begunkov 2021-03-19 10:21:19 +0000 3999) size = (unsigned long)p->len * p->nbufs;
d81269fecb8ce (Pavel Begunkov 2021-03-19 10:21:19 +0000 4000) if (!access_ok(u64_to_user_ptr(p->addr), size))
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4001) return -EFAULT;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4002)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4003) p->bgid = READ_ONCE(sqe->buf_group);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4004) tmp = READ_ONCE(sqe->off);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4005) if (tmp > USHRT_MAX)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4006) return -E2BIG;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4007) p->bid = tmp;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4008) return 0;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4009) }
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4010)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4011) static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4012) {
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4013) struct io_buffer *buf;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4014) u64 addr = pbuf->addr;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4015) int i, bid = pbuf->bid;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4016)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4017) for (i = 0; i < pbuf->nbufs; i++) {
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4018) buf = kmalloc(sizeof(*buf), GFP_KERNEL);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4019) if (!buf)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4020) break;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4021)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4022) buf->addr = addr;
d1f82808877bb (Thadeu Lima de Souza Cascardo 2021-05-05 09:47:06 -0300 4023) buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4024) buf->bid = bid;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4025) addr += pbuf->len;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4026) bid++;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4027) if (!*head) {
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4028) INIT_LIST_HEAD(&buf->list);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4029) *head = buf;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4030) } else {
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4031) list_add_tail(&buf->list, &(*head)->list);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4032) }
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4033) }
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4034)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4035) return i ? i : -ENOMEM;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4036) }
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4037)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4038) static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4039) {
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4040) struct io_provide_buf *p = &req->pbuf;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4041) struct io_ring_ctx *ctx = req->ctx;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4042) struct io_buffer *head, *list;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4043) int ret = 0;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4044) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4045)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4046) io_ring_submit_lock(ctx, !force_nonblock);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4047)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4048) lockdep_assert_held(&ctx->uring_lock);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4049)
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 4050) list = head = xa_load(&ctx->io_buffers, p->bgid);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4051)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4052) ret = io_add_buffers(p, &head);
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 4053) if (ret >= 0 && !list) {
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 4054) ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL);
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 4055) if (ret < 0)
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 4056) __io_remove_buffers(ctx, head, p->bgid, -1U);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4057) }
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4058) if (ret < 0)
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4059) req_set_fail_links(req);
9fb8cb49c7b63 (Pavel Begunkov 2021-02-28 22:35:13 +0000 4060) /* complete before unlock, IOPOLL may need the lock */
9fb8cb49c7b63 (Pavel Begunkov 2021-02-28 22:35:13 +0000 4061) __io_req_complete(req, issue_flags, ret, 0);
9fb8cb49c7b63 (Pavel Begunkov 2021-02-28 22:35:13 +0000 4062) io_ring_submit_unlock(ctx, !force_nonblock);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 4063) return 0;
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 4064) }
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 4065)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4066) static int io_epoll_ctl_prep(struct io_kiocb *req,
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4067) const struct io_uring_sqe *sqe)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4068) {
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4069) #if defined(CONFIG_EPOLL)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4070) if (sqe->ioprio || sqe->buf_index)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4071) return -EINVAL;
2d74d0421e5af (Pavel Begunkov 2021-05-14 12:05:46 +0100 4072) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 4073) return -EINVAL;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4074)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4075) req->epoll.epfd = READ_ONCE(sqe->fd);
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4076) req->epoll.op = READ_ONCE(sqe->len);
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4077) req->epoll.fd = READ_ONCE(sqe->off);
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4078)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4079) if (ep_op_has_event(req->epoll.op)) {
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4080) struct epoll_event __user *ev;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4081)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4082) ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4083) if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4084) return -EFAULT;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4085) }
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4086)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4087) return 0;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4088) #else
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4089) return -EOPNOTSUPP;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4090) #endif
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4091) }
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4092)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4093) static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4094) {
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4095) #if defined(CONFIG_EPOLL)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4096) struct io_epoll *ie = &req->epoll;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4097) int ret;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4098) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4099)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4100) ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4101) if (force_nonblock && ret == -EAGAIN)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4102) return -EAGAIN;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4103)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4104) if (ret < 0)
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4105) req_set_fail_links(req);
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4106) __io_req_complete(req, issue_flags, ret, 0);
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4107) return 0;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4108) #else
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4109) return -EOPNOTSUPP;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4110) #endif
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4111) }
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 4112)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4113) static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4114) {
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4115) #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4116) if (sqe->ioprio || sqe->buf_index || sqe->off)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4117) return -EINVAL;
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 4118) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 4119) return -EINVAL;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4120)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4121) req->madvise.addr = READ_ONCE(sqe->addr);
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4122) req->madvise.len = READ_ONCE(sqe->len);
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4123) req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4124) return 0;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4125) #else
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4126) return -EOPNOTSUPP;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4127) #endif
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4128) }
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4129)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4130) static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4131) {
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4132) #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4133) struct io_madvise *ma = &req->madvise;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4134) int ret;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4135)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4136) if (issue_flags & IO_URING_F_NONBLOCK)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4137) return -EAGAIN;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4138)
0726b01e70455 (Minchan Kim 2020-10-17 16:14:50 -0700 4139) ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4140) if (ret < 0)
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4141) req_set_fail_links(req);
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 4142) io_req_complete(req, ret);
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4143) return 0;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4144) #else
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4145) return -EOPNOTSUPP;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4146) #endif
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4147) }
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 4148)
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4149) static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4150) {
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4151) if (sqe->ioprio || sqe->buf_index || sqe->addr)
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4152) return -EINVAL;
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 4153) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 4154) return -EINVAL;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4155)
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4156) req->fadvise.offset = READ_ONCE(sqe->off);
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4157) req->fadvise.len = READ_ONCE(sqe->len);
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4158) req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4159) return 0;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4160) }
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4161)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4162) static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4163) {
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4164) struct io_fadvise *fa = &req->fadvise;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4165) int ret;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4166)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4167) if (issue_flags & IO_URING_F_NONBLOCK) {
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4168) switch (fa->advice) {
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4169) case POSIX_FADV_NORMAL:
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4170) case POSIX_FADV_RANDOM:
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4171) case POSIX_FADV_SEQUENTIAL:
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4172) break;
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4173) default:
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4174) return -EAGAIN;
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4175) }
3e69426da2599 (Jens Axboe 2020-02-01 09:22:49 -0700 4176) }
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4177)
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4178) ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4179) if (ret < 0)
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4180) req_set_fail_links(req);
0bdf3398b06ef (Pavel Begunkov 2021-04-11 01:46:29 +0100 4181) __io_req_complete(req, issue_flags, ret, 0);
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4182) return 0;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4183) }
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 4184)
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4185) static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4186) {
2d74d0421e5af (Pavel Begunkov 2021-05-14 12:05:46 +0100 4187) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 4188) return -EINVAL;
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4189) if (sqe->ioprio || sqe->buf_index)
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4190) return -EINVAL;
9c280f9087118 (Pavel Begunkov 2020-04-08 08:58:46 +0300 4191) if (req->flags & REQ_F_FIXED_FILE)
cf3040ca55f20 (Jens Axboe 2020-02-06 21:31:40 -0700 4192) return -EBADF;
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4193)
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 4194) req->statx.dfd = READ_ONCE(sqe->fd);
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 4195) req->statx.mask = READ_ONCE(sqe->len);
e62753e4e2926 (Bijan Mottahedeh 2020-05-22 21:31:18 -0700 4196) req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 4197) req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 4198) req->statx.flags = READ_ONCE(sqe->statx_flags);
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4199)
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4200) return 0;
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4201) }
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4202)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4203) static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4204) {
1d9e1288039a4 (Bijan Mottahedeh 2020-05-22 21:31:16 -0700 4205) struct io_statx *ctx = &req->statx;
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4206) int ret;
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4207)
59d7001345a7b (Pavel Begunkov 2021-03-22 01:58:30 +0000 4208) if (issue_flags & IO_URING_F_NONBLOCK)
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4209) return -EAGAIN;
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4210)
e62753e4e2926 (Bijan Mottahedeh 2020-05-22 21:31:18 -0700 4211) ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
e62753e4e2926 (Bijan Mottahedeh 2020-05-22 21:31:18 -0700 4212) ctx->buffer);
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4213)
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4214) if (ret < 0)
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4215) req_set_fail_links(req);
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 4216) io_req_complete(req, ret);
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4217) return 0;
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4218) }
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 4219)
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4220) static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4221) {
14587a46646d3 (Jens Axboe 2020-09-05 11:36:08 -0600 4222) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02af65f (Pavel Begunkov 2020-06-03 18:03:22 +0300 4223) return -EINVAL;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4224) if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4225) sqe->rw_flags || sqe->buf_index)
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4226) return -EINVAL;
9c280f9087118 (Pavel Begunkov 2020-04-08 08:58:46 +0300 4227) if (req->flags & REQ_F_FIXED_FILE)
cf3040ca55f20 (Jens Axboe 2020-02-06 21:31:40 -0700 4228) return -EBADF;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4229)
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4230) req->close.fd = READ_ONCE(sqe->fd);
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4231) return 0;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4232) }
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4233)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4234) static int io_close(struct io_kiocb *req, unsigned int issue_flags)
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4235) {
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4236) struct files_struct *files = current->files;
3af73b286ccee (Pavel Begunkov 2020-06-08 21:08:17 +0300 4237) struct io_close *close = &req->close;
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4238) struct fdtable *fdt;
a1fde923e3065 (Pavel Begunkov 2021-04-11 01:46:28 +0100 4239) struct file *file = NULL;
a1fde923e3065 (Pavel Begunkov 2021-04-11 01:46:28 +0100 4240) int ret = -EBADF;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4241)
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4242) spin_lock(&files->file_lock);
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4243) fdt = files_fdtable(files);
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4244) if (close->fd >= fdt->max_fds) {
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4245) spin_unlock(&files->file_lock);
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4246) goto err;
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4247) }
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4248) file = fdt->fd[close->fd];
a1fde923e3065 (Pavel Begunkov 2021-04-11 01:46:28 +0100 4249) if (!file || file->f_op == &io_uring_fops) {
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4250) spin_unlock(&files->file_lock);
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4251) file = NULL;
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4252) goto err;
3af73b286ccee (Pavel Begunkov 2020-06-08 21:08:17 +0300 4253) }
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4254)
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4255) /* if the file has a flush method, be safe and punt to async */
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4256) if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4257) spin_unlock(&files->file_lock);
0bf0eefdab52d (Pavel Begunkov 2020-05-26 20:34:06 +0300 4258) return -EAGAIN;
a2100672f3b2a (Pavel Begunkov 2020-03-02 23:45:16 +0300 4259) }
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4260)
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4261) ret = __close_fd_get_file(close->fd, &file);
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4262) spin_unlock(&files->file_lock);
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4263) if (ret < 0) {
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4264) if (ret == -ENOENT)
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4265) ret = -EBADF;
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4266) goto err;
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4267) }
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4268)
3af73b286ccee (Pavel Begunkov 2020-06-08 21:08:17 +0300 4269) /* No ->flush() or already async, safely close from here */
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4270) ret = filp_close(file, current->files);
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4271) err:
3af73b286ccee (Pavel Begunkov 2020-06-08 21:08:17 +0300 4272) if (ret < 0)
3af73b286ccee (Pavel Begunkov 2020-06-08 21:08:17 +0300 4273) req_set_fail_links(req);
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4274) if (file)
9eac1904d3364 (Jens Axboe 2021-01-19 15:50:37 -0700 4275) fput(file);
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4276) __io_req_complete(req, issue_flags, ret, 0);
1a417f4e618e0 (Jens Axboe 2020-01-31 17:16:48 -0700 4277) return 0;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4278) }
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 4279)
1155c76a24836 (Pavel Begunkov 2021-02-18 18:29:38 +0000 4280) static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4281) {
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4282) struct io_ring_ctx *ctx = req->ctx;
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4283)
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4284) if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4285) return -EINVAL;
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4286) if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4287) return -EINVAL;
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4288)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4289) req->sync.off = READ_ONCE(sqe->off);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4290) req->sync.len = READ_ONCE(sqe->len);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4291) req->sync.flags = READ_ONCE(sqe->sync_range_flags);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4292) return 0;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4293) }
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4294)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4295) static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4296) {
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4297) int ret;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4298)
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 4299) /* sync_file_range always requires a blocking context */
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4300) if (issue_flags & IO_URING_F_NONBLOCK)
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 4301) return -EAGAIN;
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 4302)
9adbd45d6d32f (Jens Axboe 2019-12-20 08:45:55 -0700 4303) ret = sync_file_range(req->file, req->sync.off, req->sync.len,
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4304) req->sync.flags);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4305) if (ret < 0)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4306) req_set_fail_links(req);
e1e16097e265d (Jens Axboe 2020-06-22 09:17:17 -0600 4307) io_req_complete(req, ret);
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4308) return 0;
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4309) }
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4310)
469956e853ccd (YueHaibing 2020-03-04 15:53:52 +0800 4311) #if defined(CONFIG_NET)
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4312) static int io_setup_async_msg(struct io_kiocb *req,
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4313) struct io_async_msghdr *kmsg)
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4314) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4315) struct io_async_msghdr *async_msg = req->async_data;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4316)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4317) if (async_msg)
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4318) return -EAGAIN;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4319) if (io_alloc_async_data(req)) {
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4320) kfree(kmsg->free_iov);
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4321) return -ENOMEM;
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4322) }
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4323) async_msg = req->async_data;
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4324) req->flags |= REQ_F_NEED_CLEANUP;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4325) memcpy(async_msg, kmsg, sizeof(*kmsg));
2a7808024b195 (Pavel Begunkov 2021-02-05 00:57:58 +0000 4326) async_msg->msg.msg_name = &async_msg->addr;
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4327) /* if were using fast_iov, set it to the new one */
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4328) if (!async_msg->free_iov)
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4329) async_msg->msg.msg_iter.iov = async_msg->fast_iov;
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4330)
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4331) return -EAGAIN;
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4332) }
02d27d895323c (Pavel Begunkov 2020-02-28 10:36:36 +0300 4333)
2ae523ed07f14 (Pavel Begunkov 2020-07-12 20:41:06 +0300 4334) static int io_sendmsg_copy_hdr(struct io_kiocb *req,
2ae523ed07f14 (Pavel Begunkov 2020-07-12 20:41:06 +0300 4335) struct io_async_msghdr *iomsg)
2ae523ed07f14 (Pavel Begunkov 2020-07-12 20:41:06 +0300 4336) {
2ae523ed07f14 (Pavel Begunkov 2020-07-12 20:41:06 +0300 4337) iomsg->msg.msg_name = &iomsg->addr;
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4338) iomsg->free_iov = iomsg->fast_iov;
2ae523ed07f14 (Pavel Begunkov 2020-07-12 20:41:06 +0300 4339) return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4340) req->sr_msg.msg_flags, &iomsg->free_iov);
2ae523ed07f14 (Pavel Begunkov 2020-07-12 20:41:06 +0300 4341) }
2ae523ed07f14 (Pavel Begunkov 2020-07-12 20:41:06 +0300 4342)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4343) static int io_sendmsg_prep_async(struct io_kiocb *req)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4344) {
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4345) int ret;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4346)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4347) ret = io_sendmsg_copy_hdr(req, req->async_data);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4348) if (!ret)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4349) req->flags |= REQ_F_NEED_CLEANUP;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4350) return ret;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4351) }
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4352)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4353) static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4354) {
e47293fdf9899 (Jens Axboe 2019-12-20 08:58:21 -0700 4355) struct io_sr_msg *sr = &req->sr_msg;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4356)
d2b6f48b691ed (Pavel Begunkov 2020-06-03 18:03:25 +0300 4357) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
d2b6f48b691ed (Pavel Begunkov 2020-06-03 18:03:25 +0300 4358) return -EINVAL;
d2b6f48b691ed (Pavel Begunkov 2020-06-03 18:03:25 +0300 4359)
270a5940700bb (Pavel Begunkov 2020-07-12 20:41:04 +0300 4360) sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4361) sr->len = READ_ONCE(sqe->len);
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4362) sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4363) if (sr->msg_flags & MSG_DONTWAIT)
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4364) req->flags |= REQ_F_NOWAIT;
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4365)
d876836204897 (Jens Axboe 2020-02-27 14:17:49 -0700 4366) #ifdef CONFIG_COMPAT
d876836204897 (Jens Axboe 2020-02-27 14:17:49 -0700 4367) if (req->ctx->compat)
d876836204897 (Jens Axboe 2020-02-27 14:17:49 -0700 4368) sr->msg_flags |= MSG_CMSG_COMPAT;
d876836204897 (Jens Axboe 2020-02-27 14:17:49 -0700 4369) #endif
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4370) return 0;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4371) }
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4372)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4373) static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 4374) {
6b754c8b912a1 (Pavel Begunkov 2020-07-16 23:28:00 +0300 4375) struct io_async_msghdr iomsg, *kmsg;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 4376) struct socket *sock;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4377) unsigned flags;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4378) int min_ret = 0;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 4379) int ret;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 4380)
dba4a9256bb4d (Florent Revest 2020-12-04 12:36:04 +0100 4381) sock = sock_from_file(req->file);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4382) if (unlikely(!sock))
dba4a9256bb4d (Florent Revest 2020-12-04 12:36:04 +0100 4383) return -ENOTSOCK;
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4384)
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4385) kmsg = req->async_data;
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4386) if (!kmsg) {
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4387) ret = io_sendmsg_copy_hdr(req, &iomsg);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4388) if (ret)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4389) return ret;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4390) kmsg = &iomsg;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 4391) }
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 4392)
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4393) flags = req->sr_msg.msg_flags;
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4394) if (issue_flags & IO_URING_F_NONBLOCK)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4395) flags |= MSG_DONTWAIT;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4396) if (flags & MSG_WAITALL)
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4397) min_ret = iov_iter_count(&kmsg->msg.msg_iter);
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4398)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4399) ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4400) if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4401) return io_setup_async_msg(req, kmsg);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4402) if (ret == -ERESTARTSYS)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4403) ret = -EINTR;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 4404)
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4405) /* fast path, check for non-NULL to avoid function call */
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4406) if (kmsg->free_iov)
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4407) kfree(kmsg->free_iov);
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 4408) req->flags &= ~REQ_F_NEED_CLEANUP;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4409) if (ret < min_ret)
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 4410) req_set_fail_links(req);
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4411) __io_req_complete(req, issue_flags, ret, 0);
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4412) return 0;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4413) }
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 4414)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4415) static int io_send(struct io_kiocb *req, unsigned int issue_flags)
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4416) {
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4417) struct io_sr_msg *sr = &req->sr_msg;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4418) struct msghdr msg;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4419) struct iovec iov;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4420) struct socket *sock;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4421) unsigned flags;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4422) int min_ret = 0;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4423) int ret;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4424)
dba4a9256bb4d (Florent Revest 2020-12-04 12:36:04 +0100 4425) sock = sock_from_file(req->file);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4426) if (unlikely(!sock))
dba4a9256bb4d (Florent Revest 2020-12-04 12:36:04 +0100 4427) return -ENOTSOCK;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4428)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4429) ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4430) if (unlikely(ret))
14db84110d489 (Zheng Bin 2020-09-09 20:12:37 +0800 4431) return ret;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4432)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4433) msg.msg_name = NULL;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4434) msg.msg_control = NULL;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4435) msg.msg_controllen = 0;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4436) msg.msg_namelen = 0;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4437)
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4438) flags = req->sr_msg.msg_flags;
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4439) if (issue_flags & IO_URING_F_NONBLOCK)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4440) flags |= MSG_DONTWAIT;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4441) if (flags & MSG_WAITALL)
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4442) min_ret = iov_iter_count(&msg.msg_iter);
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4443)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4444) msg.msg_flags = flags;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4445) ret = sock_sendmsg(sock, &msg);
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4446) if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4447) return -EAGAIN;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4448) if (ret == -ERESTARTSYS)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4449) ret = -EINTR;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4450)
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4451) if (ret < min_ret)
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4452) req_set_fail_links(req);
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4453) __io_req_complete(req, issue_flags, ret, 0);
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4454) return 0;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4455) }
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4456)
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4457) static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4458) struct io_async_msghdr *iomsg)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4459) {
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4460) struct io_sr_msg *sr = &req->sr_msg;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4461) struct iovec __user *uiov;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4462) size_t iov_len;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4463) int ret;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4464)
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4465) ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4466) &iomsg->uaddr, &uiov, &iov_len);
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4467) if (ret)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4468) return ret;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4469)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4470) if (req->flags & REQ_F_BUFFER_SELECT) {
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4471) if (iov_len > 1)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4472) return -EINVAL;
5476dfed29ad9 (Pavel Begunkov 2021-02-05 00:57:59 +0000 4473) if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4474) return -EFAULT;
5476dfed29ad9 (Pavel Begunkov 2021-02-05 00:57:59 +0000 4475) sr->len = iomsg->fast_iov[0].iov_len;
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4476) iomsg->free_iov = NULL;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4477) } else {
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4478) iomsg->free_iov = iomsg->fast_iov;
89cd35c58bc2e (Christoph Hellwig 2020-09-25 06:51:41 +0200 4479) ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4480) &iomsg->free_iov, &iomsg->msg.msg_iter,
89cd35c58bc2e (Christoph Hellwig 2020-09-25 06:51:41 +0200 4481) false);
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4482) if (ret > 0)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4483) ret = 0;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4484) }
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4485)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4486) return ret;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4487) }
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4488)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4489) #ifdef CONFIG_COMPAT
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4490) static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4491) struct io_async_msghdr *iomsg)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4492) {
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4493) struct io_sr_msg *sr = &req->sr_msg;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4494) struct compat_iovec __user *uiov;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4495) compat_uptr_t ptr;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4496) compat_size_t len;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4497) int ret;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4498)
4af3417a347d0 (Pavel Begunkov 2021-04-11 01:46:30 +0100 4499) ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
4af3417a347d0 (Pavel Begunkov 2021-04-11 01:46:30 +0100 4500) &ptr, &len);
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4501) if (ret)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4502) return ret;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4503)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4504) uiov = compat_ptr(ptr);
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4505) if (req->flags & REQ_F_BUFFER_SELECT) {
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4506) compat_ssize_t clen;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4507)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4508) if (len > 1)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4509) return -EINVAL;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4510) if (!access_ok(uiov, sizeof(*uiov)))
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4511) return -EFAULT;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4512) if (__get_user(clen, &uiov->iov_len))
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4513) return -EFAULT;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4514) if (clen < 0)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4515) return -EINVAL;
2d280bc8930ba (Pavel Begunkov 2020-11-29 18:33:32 +0000 4516) sr->len = clen;
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4517) iomsg->free_iov = NULL;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4518) } else {
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4519) iomsg->free_iov = iomsg->fast_iov;
89cd35c58bc2e (Christoph Hellwig 2020-09-25 06:51:41 +0200 4520) ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4521) UIO_FASTIOV, &iomsg->free_iov,
89cd35c58bc2e (Christoph Hellwig 2020-09-25 06:51:41 +0200 4522) &iomsg->msg.msg_iter, true);
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4523) if (ret < 0)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4524) return ret;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4525) }
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4526)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4527) return 0;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4528) }
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4529) #endif
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4530)
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4531) static int io_recvmsg_copy_hdr(struct io_kiocb *req,
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4532) struct io_async_msghdr *iomsg)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4533) {
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4534) iomsg->msg.msg_name = &iomsg->addr;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4535)
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4536) #ifdef CONFIG_COMPAT
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4537) if (req->ctx->compat)
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4538) return __io_compat_recvmsg_copy_hdr(req, iomsg);
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4539) #endif
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4540)
1400e69705baf (Pavel Begunkov 2020-07-12 20:41:05 +0300 4541) return __io_recvmsg_copy_hdr(req, iomsg);
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4542) }
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4543)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4544) static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4545) bool needs_lock)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4546) {
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4547) struct io_sr_msg *sr = &req->sr_msg;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4548) struct io_buffer *kbuf;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4549)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4550) kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4551) if (IS_ERR(kbuf))
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4552) return kbuf;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4553)
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4554) sr->kbuf = kbuf;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4555) req->flags |= REQ_F_BUFFER_SELECTED;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4556) return kbuf;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4557) }
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4558)
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4559) static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4560) {
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4561) return io_put_kbuf(req, req->sr_msg.kbuf);
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4562) }
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4563)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4564) static int io_recvmsg_prep_async(struct io_kiocb *req)
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 4565) {
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 4566) int ret;
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4567)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4568) ret = io_recvmsg_copy_hdr(req, req->async_data);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4569) if (!ret)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4570) req->flags |= REQ_F_NEED_CLEANUP;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4571) return ret;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4572) }
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4573)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4574) static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4575) {
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4576) struct io_sr_msg *sr = &req->sr_msg;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4577)
d2b6f48b691ed (Pavel Begunkov 2020-06-03 18:03:25 +0300 4578) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
d2b6f48b691ed (Pavel Begunkov 2020-06-03 18:03:25 +0300 4579) return -EINVAL;
d2b6f48b691ed (Pavel Begunkov 2020-06-03 18:03:25 +0300 4580)
270a5940700bb (Pavel Begunkov 2020-07-12 20:41:04 +0300 4581) sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
0b7b21e42ba2d (Jens Axboe 2020-01-31 08:34:59 -0700 4582) sr->len = READ_ONCE(sqe->len);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4583) sr->bgid = READ_ONCE(sqe->buf_group);
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4584) sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4585) if (sr->msg_flags & MSG_DONTWAIT)
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4586) req->flags |= REQ_F_NOWAIT;
06b76d44ba25e (Jens Axboe 2019-12-19 14:44:26 -0700 4587)
d876836204897 (Jens Axboe 2020-02-27 14:17:49 -0700 4588) #ifdef CONFIG_COMPAT
d876836204897 (Jens Axboe 2020-02-27 14:17:49 -0700 4589) if (req->ctx->compat)
d876836204897 (Jens Axboe 2020-02-27 14:17:49 -0700 4590) sr->msg_flags |= MSG_CMSG_COMPAT;
d876836204897 (Jens Axboe 2020-02-27 14:17:49 -0700 4591) #endif
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4592) return 0;
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 4593) }
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 4594)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4595) static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 4596) {
6b754c8b912a1 (Pavel Begunkov 2020-07-16 23:28:00 +0300 4597) struct io_async_msghdr iomsg, *kmsg;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4598) struct socket *sock;
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4599) struct io_buffer *kbuf;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4600) unsigned flags;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4601) int min_ret = 0;
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4602) int ret, cflags = 0;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4603) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4604)
dba4a9256bb4d (Florent Revest 2020-12-04 12:36:04 +0100 4605) sock = sock_from_file(req->file);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4606) if (unlikely(!sock))
dba4a9256bb4d (Florent Revest 2020-12-04 12:36:04 +0100 4607) return -ENOTSOCK;
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4608)
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4609) kmsg = req->async_data;
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4610) if (!kmsg) {
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4611) ret = io_recvmsg_copy_hdr(req, &iomsg);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4612) if (ret)
681fda8d27a66 (Pavel Begunkov 2020-07-15 22:20:45 +0300 4613) return ret;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4614) kmsg = &iomsg;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4615) }
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4616)
bc02ef3325e3e (Pavel Begunkov 2020-07-16 23:28:03 +0300 4617) if (req->flags & REQ_F_BUFFER_SELECT) {
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4618) kbuf = io_recv_buffer_select(req, !force_nonblock);
bc02ef3325e3e (Pavel Begunkov 2020-07-16 23:28:03 +0300 4619) if (IS_ERR(kbuf))
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4620) return PTR_ERR(kbuf);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4621) kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
5476dfed29ad9 (Pavel Begunkov 2021-02-05 00:57:59 +0000 4622) kmsg->fast_iov[0].iov_len = req->sr_msg.len;
5476dfed29ad9 (Pavel Begunkov 2021-02-05 00:57:59 +0000 4623) iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4624) 1, req->sr_msg.len);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4625) }
52de1fe122408 (Jens Axboe 2020-02-27 10:15:42 -0700 4626)
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4627) flags = req->sr_msg.msg_flags;
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4628) if (force_nonblock)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4629) flags |= MSG_DONTWAIT;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4630) if (flags & MSG_WAITALL)
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4631) min_ret = iov_iter_count(&kmsg->msg.msg_iter);
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4632)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4633) ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4634) kmsg->uaddr, flags);
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 4635) if (force_nonblock && ret == -EAGAIN)
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 4636) return io_setup_async_msg(req, kmsg);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4637) if (ret == -ERESTARTSYS)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4638) ret = -EINTR;
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4639)
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4640) if (req->flags & REQ_F_BUFFER_SELECTED)
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4641) cflags = io_put_recv_kbuf(req);
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4642) /* fast path, check for non-NULL to avoid function call */
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4643) if (kmsg->free_iov)
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 4644) kfree(kmsg->free_iov);
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 4645) req->flags &= ~REQ_F_NEED_CLEANUP;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4646) if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 4647) req_set_fail_links(req);
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4648) __io_req_complete(req, issue_flags, ret, cflags);
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 4649) return 0;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 4650) }
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 4651)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4652) static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4653) {
6b754c8b912a1 (Pavel Begunkov 2020-07-16 23:28:00 +0300 4654) struct io_buffer *kbuf;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4655) struct io_sr_msg *sr = &req->sr_msg;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4656) struct msghdr msg;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4657) void __user *buf = sr->buf;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4658) struct socket *sock;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4659) struct iovec iov;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4660) unsigned flags;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4661) int min_ret = 0;
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4662) int ret, cflags = 0;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4663) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4664)
dba4a9256bb4d (Florent Revest 2020-12-04 12:36:04 +0100 4665) sock = sock_from_file(req->file);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4666) if (unlikely(!sock))
dba4a9256bb4d (Florent Revest 2020-12-04 12:36:04 +0100 4667) return -ENOTSOCK;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4668)
bc02ef3325e3e (Pavel Begunkov 2020-07-16 23:28:03 +0300 4669) if (req->flags & REQ_F_BUFFER_SELECT) {
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4670) kbuf = io_recv_buffer_select(req, !force_nonblock);
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4671) if (IS_ERR(kbuf))
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4672) return PTR_ERR(kbuf);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4673) buf = u64_to_user_ptr(kbuf->addr);
bc02ef3325e3e (Pavel Begunkov 2020-07-16 23:28:03 +0300 4674) }
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 4675)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4676) ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
14c32eee92866 (Pavel Begunkov 2020-07-16 23:28:01 +0300 4677) if (unlikely(ret))
14c32eee92866 (Pavel Begunkov 2020-07-16 23:28:01 +0300 4678) goto out_free;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4679)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4680) msg.msg_name = NULL;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4681) msg.msg_control = NULL;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4682) msg.msg_controllen = 0;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4683) msg.msg_namelen = 0;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4684) msg.msg_iocb = NULL;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4685) msg.msg_flags = 0;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4686)
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4687) flags = req->sr_msg.msg_flags;
044118069a23f (Pavel Begunkov 2021-04-01 15:44:00 +0100 4688) if (force_nonblock)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4689) flags |= MSG_DONTWAIT;
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4690) if (flags & MSG_WAITALL)
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4691) min_ret = iov_iter_count(&msg.msg_iter);
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4692)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4693) ret = sock_recvmsg(sock, &msg, flags);
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4694) if (force_nonblock && ret == -EAGAIN)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4695) return -EAGAIN;
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4696) if (ret == -ERESTARTSYS)
7a7cacba8b456 (Pavel Begunkov 2020-07-16 23:27:59 +0300 4697) ret = -EINTR;
14c32eee92866 (Pavel Begunkov 2020-07-16 23:28:01 +0300 4698) out_free:
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4699) if (req->flags & REQ_F_BUFFER_SELECTED)
7fbb1b541f428 (Pavel Begunkov 2020-07-16 23:28:05 +0300 4700) cflags = io_put_recv_kbuf(req);
0031275d119ef (Stefan Metzmacher 2021-03-20 20:33:36 +0100 4701) if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4702) req_set_fail_links(req);
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4703) __io_req_complete(req, issue_flags, ret, cflags);
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4704) return 0;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4705) }
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 4706)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4707) static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 4708) {
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4709) struct io_accept *accept = &req->accept;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4710)
14587a46646d3 (Jens Axboe 2020-09-05 11:36:08 -0600 4711) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 4712) return -EINVAL;
8042d6ce8c40d (Hrvoje Zeba 2019-11-25 14:40:22 -0500 4713) if (sqe->ioprio || sqe->len || sqe->buf_index)
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 4714) return -EINVAL;
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 4715)
d55e5f5b70dd6 (Jens Axboe 2019-12-11 16:12:15 -0700 4716) accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
d55e5f5b70dd6 (Jens Axboe 2019-12-11 16:12:15 -0700 4717) accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4718) accept->flags = READ_ONCE(sqe->accept_flags);
09952e3e78261 (Jens Axboe 2020-03-19 20:16:56 -0600 4719) accept->nofile = rlimit(RLIMIT_NOFILE);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4720) return 0;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4721) }
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 4722)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4723) static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4724) {
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4725) struct io_accept *accept = &req->accept;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4726) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 4727) unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4728) int ret;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4729)
e697deed834de (Jiufei Xue 2020-06-10 13:41:59 +0800 4730) if (req->file->f_flags & O_NONBLOCK)
e697deed834de (Jiufei Xue 2020-06-10 13:41:59 +0800 4731) req->flags |= REQ_F_NOWAIT;
e697deed834de (Jiufei Xue 2020-06-10 13:41:59 +0800 4732)
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4733) ret = __sys_accept4_file(req->file, file_flags, accept->addr,
09952e3e78261 (Jens Axboe 2020-03-19 20:16:56 -0600 4734) accept->addr_len, accept->flags,
09952e3e78261 (Jens Axboe 2020-03-19 20:16:56 -0600 4735) accept->nofile);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4736) if (ret == -EAGAIN && force_nonblock)
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 4737) return -EAGAIN;
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 4738) if (ret < 0) {
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 4739) if (ret == -ERESTARTSYS)
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 4740) ret = -EINTR;
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 4741) req_set_fail_links(req);
ac45abc0e2a8e (Pavel Begunkov 2020-06-08 21:08:18 +0300 4742) }
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4743) __io_req_complete(req, issue_flags, ret, 0);
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 4744) return 0;
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4745) }
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 4746)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4747) static int io_connect_prep_async(struct io_kiocb *req)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4748) {
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4749) struct io_async_connect *io = req->async_data;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4750) struct io_connect *conn = &req->connect;
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4751)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4752) return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4753) }
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4754)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4755) static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4756) {
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4757) struct io_connect *conn = &req->connect;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4758)
14587a46646d3 (Jens Axboe 2020-09-05 11:36:08 -0600 4759) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 4760) return -EINVAL;
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 4761) if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 4762) return -EINVAL;
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 4763)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4764) conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4765) conn->addr_len = READ_ONCE(sqe->addr2);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 4766) return 0;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4767) }
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4768)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4769) static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 4770) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4771) struct io_async_connect __io, *io;
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 4772) unsigned file_flags;
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 4773) int ret;
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 4774) bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 4775)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4776) if (req->async_data) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4777) io = req->async_data;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4778) } else {
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4779) ret = move_addr_to_kernel(req->connect.addr,
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 4780) req->connect.addr_len,
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4781) &__io.address);
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4782) if (ret)
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4783) goto out;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4784) io = &__io;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4785) }
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4786)
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 4787) file_flags = force_nonblock ? O_NONBLOCK : 0;
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 4788)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4789) ret = __sys_connect_file(req->file, &io->address,
3fbb51c18f5c1 (Jens Axboe 2019-12-20 08:51:52 -0700 4790) req->connect.addr_len, file_flags);
87f80d623c6c9 (Jens Axboe 2019-12-03 11:23:54 -0700 4791) if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4792) if (req->async_data)
b7bb4f7da0a1a (Jens Axboe 2019-12-15 22:13:43 -0700 4793) return -EAGAIN;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4794) if (io_alloc_async_data(req)) {
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4795) ret = -ENOMEM;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4796) goto out;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4797) }
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4798) memcpy(req->async_data, &__io, sizeof(__io));
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 4799) return -EAGAIN;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4800) }
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 4801) if (ret == -ERESTARTSYS)
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 4802) ret = -EINTR;
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 4803) out:
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 4804) if (ret < 0)
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 4805) req_set_fail_links(req);
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 4806) __io_req_complete(req, issue_flags, ret, 0);
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 4807) return 0;
469956e853ccd (YueHaibing 2020-03-04 15:53:52 +0800 4808) }
469956e853ccd (YueHaibing 2020-03-04 15:53:52 +0800 4809) #else /* !CONFIG_NET */
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4810) #define IO_NETOP_FN(op) \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4811) static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4812) { \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4813) return -EOPNOTSUPP; \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4814) }
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4815)
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4816) #define IO_NETOP_PREP(op) \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4817) IO_NETOP_FN(op) \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4818) static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4819) { \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4820) return -EOPNOTSUPP; \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4821) } \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4822)
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4823) #define IO_NETOP_PREP_ASYNC(op) \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4824) IO_NETOP_PREP(op) \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4825) static int io_##op##_prep_async(struct io_kiocb *req) \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4826) { \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4827) return -EOPNOTSUPP; \
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4828) }
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4829)
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4830) IO_NETOP_PREP_ASYNC(sendmsg);
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4831) IO_NETOP_PREP_ASYNC(recvmsg);
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4832) IO_NETOP_PREP_ASYNC(connect);
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4833) IO_NETOP_PREP(accept);
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4834) IO_NETOP_FN(send);
99a1008164716 (Jens Axboe 2021-02-19 09:35:19 -0700 4835) IO_NETOP_FN(recv);
469956e853ccd (YueHaibing 2020-03-04 15:53:52 +0800 4836) #endif /* CONFIG_NET */
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 4837)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4838) struct io_poll_table {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4839) struct poll_table_struct pt;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4840) struct io_kiocb *req;
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 4841) int nr_entries;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4842) int error;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4843) };
ce593a6c480a2 (Jens Axboe 2020-06-30 12:39:05 -0600 4844)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4845) static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4846) __poll_t mask, task_work_func_t func)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4847) {
aa96bf8a9ee33 (Jens Axboe 2020-04-03 11:26:26 -0600 4848) int ret;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4849)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4850) /* for instances that support it check for an event match first: */
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4851) if (mask && !(mask & poll->events))
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4852) return 0;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4853)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4854) trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4855)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4856) list_del_init(&poll->wait.entry);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4857)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4858) req->result = mask;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 4859) req->task_work.func = func;
6d816e088c359 (Jens Axboe 2020-08-11 08:04:14 -0600 4860)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4861) /*
e3aabf9554fd0 (Jens Axboe 2020-05-18 11:04:17 -0600 4862) * If this fails, then the task is exiting. When a task exits, the
e3aabf9554fd0 (Jens Axboe 2020-05-18 11:04:17 -0600 4863) * work gets canceled, so just cancel this request as well instead
e3aabf9554fd0 (Jens Axboe 2020-05-18 11:04:17 -0600 4864) * of executing it. We can't safely execute it anyway, as we may not
e3aabf9554fd0 (Jens Axboe 2020-05-18 11:04:17 -0600 4865) * have the needed state needed for it anyway.
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4866) */
355fb9e2b78e7 (Jens Axboe 2020-10-22 20:19:35 -0600 4867) ret = io_req_task_work_add(req);
aa96bf8a9ee33 (Jens Axboe 2020-04-03 11:26:26 -0600 4868) if (unlikely(ret)) {
e3aabf9554fd0 (Jens Axboe 2020-05-18 11:04:17 -0600 4869) WRITE_ONCE(poll->canceled, true);
eab30c4d20dc7 (Pavel Begunkov 2021-01-19 13:32:42 +0000 4870) io_req_task_work_add_fallback(req, func);
aa96bf8a9ee33 (Jens Axboe 2020-04-03 11:26:26 -0600 4871) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4872) return 1;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4873) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 4874)
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4875) static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4876) __acquires(&req->ctx->completion_lock)
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4877) {
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4878) struct io_ring_ctx *ctx = req->ctx;
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4879)
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4880) if (!req->result && !READ_ONCE(poll->canceled)) {
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4881) struct poll_table_struct pt = { ._key = poll->events };
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4882)
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4883) req->result = vfs_poll(req->file, &pt) & poll->events;
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4884) }
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4885)
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4886) spin_lock_irq(&ctx->completion_lock);
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4887) if (!req->result && !READ_ONCE(poll->canceled)) {
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4888) add_wait_queue(poll->head, &poll->wait);
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4889) return true;
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4890) }
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4891)
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4892) return false;
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4893) }
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 4894)
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4895) static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4896) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4897) /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4898) if (req->opcode == IORING_OP_POLL_ADD)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 4899) return req->async_data;
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4900) return req->apoll->double_poll;
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4901) }
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4902)
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4903) static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4904) {
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4905) if (req->opcode == IORING_OP_POLL_ADD)
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4906) return &req->poll;
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4907) return &req->apoll->poll;
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4908) }
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4909)
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4910) static void io_poll_remove_double(struct io_kiocb *req)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 4911) __must_hold(&req->ctx->completion_lock)
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4912) {
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4913) struct io_poll_iocb *poll = io_poll_get_double(req);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4914)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4915) lockdep_assert_held(&req->ctx->completion_lock);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4916)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4917) if (poll && poll->head) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4918) struct wait_queue_head *head = poll->head;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4919)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4920) spin_lock(&head->lock);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4921) list_del_init(&poll->wait.entry);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4922) if (poll->wait.private)
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 4923) req_ref_put(req);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4924) poll->head = NULL;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4925) spin_unlock(&head->lock);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4926) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4927) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4928)
e27414bef7b4f (Pavel Begunkov 2021-04-09 09:13:20 +0100 4929) static bool io_poll_complete(struct io_kiocb *req, __poll_t mask)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 4930) __must_hold(&req->ctx->completion_lock)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4931) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4932) struct io_ring_ctx *ctx = req->ctx;
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4933) unsigned flags = IORING_CQE_F_MORE;
e27414bef7b4f (Pavel Begunkov 2021-04-09 09:13:20 +0100 4934) int error;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4935)
e27414bef7b4f (Pavel Begunkov 2021-04-09 09:13:20 +0100 4936) if (READ_ONCE(req->poll.canceled)) {
45ab03b19e8bf (Jens Axboe 2021-02-23 08:19:33 -0700 4937) error = -ECANCELED;
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4938) req->poll.events |= EPOLLONESHOT;
e27414bef7b4f (Pavel Begunkov 2021-04-09 09:13:20 +0100 4939) } else {
5082620fb2cab (Jens Axboe 2021-02-23 09:02:26 -0700 4940) error = mangle_poll(mask);
e27414bef7b4f (Pavel Begunkov 2021-04-09 09:13:20 +0100 4941) }
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 4942) if (req->poll.events & EPOLLONESHOT)
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 4943) flags = 0;
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 4944) if (!io_cqring_fill_event(ctx, req->user_data, error, flags)) {
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4945) req->poll.done = true;
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4946) flags = 0;
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4947) }
7b289c38335ec (Hao Xu 2021-04-13 15:20:39 +0800 4948) if (flags & IORING_CQE_F_MORE)
7b289c38335ec (Hao Xu 2021-04-13 15:20:39 +0800 4949) ctx->cq_extra++;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4950)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4951) io_commit_cqring(ctx);
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4952) return !(flags & IORING_CQE_F_MORE);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4953) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4954)
dd221f46f68ad (Pavel Begunkov 2020-10-18 10:17:42 +0100 4955) static void io_poll_task_func(struct callback_head *cb)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4956) {
dd221f46f68ad (Pavel Begunkov 2020-10-18 10:17:42 +0100 4957) struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4958) struct io_ring_ctx *ctx = req->ctx;
dd221f46f68ad (Pavel Begunkov 2020-10-18 10:17:42 +0100 4959) struct io_kiocb *nxt;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4960)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4961) if (io_poll_rewait(req, &req->poll)) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4962) spin_unlock_irq(&ctx->completion_lock);
dd221f46f68ad (Pavel Begunkov 2020-10-18 10:17:42 +0100 4963) } else {
f40b964a66ace (Pavel Begunkov 2021-04-09 09:13:19 +0100 4964) bool done;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4965)
e27414bef7b4f (Pavel Begunkov 2021-04-09 09:13:20 +0100 4966) done = io_poll_complete(req, req->result);
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4967) if (done) {
a8d4169f924ad (Hao Xu 2021-07-28 11:03:22 +0800 4968) io_poll_remove_double(req);
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4969) hash_del(&req->hash_node);
f40b964a66ace (Pavel Begunkov 2021-04-09 09:13:19 +0100 4970) } else {
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4971) req->result = 0;
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4972) add_wait_queue(req->poll.head, &req->poll.wait);
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4973) }
dd221f46f68ad (Pavel Begunkov 2020-10-18 10:17:42 +0100 4974) spin_unlock_irq(&ctx->completion_lock);
dd221f46f68ad (Pavel Begunkov 2020-10-18 10:17:42 +0100 4975) io_cqring_ev_posted(ctx);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4976)
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4977) if (done) {
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4978) nxt = io_put_req_find_next(req);
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4979) if (nxt)
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4980) __io_req_task_submit(nxt);
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4981) }
dd221f46f68ad (Pavel Begunkov 2020-10-18 10:17:42 +0100 4982) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4983) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4984)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4985) static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4986) int sync, void *key)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4987) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4988) struct io_kiocb *req = wait->private;
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 4989) struct io_poll_iocb *poll = io_poll_get_single(req);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4990) __poll_t mask = key_to_poll(key);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4991)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4992) /* for instances that support it check for an event match first: */
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4993) if (mask && !(mask & poll->events))
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4994) return 0;
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4995) if (!(poll->events & EPOLLONESHOT))
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 4996) return poll->wait.func(&poll->wait, mode, sync, key);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 4997)
8706e04ed7d6c (Jens Axboe 2020-09-28 08:38:54 -0600 4998) list_del_init(&wait->entry);
8706e04ed7d6c (Jens Axboe 2020-09-28 08:38:54 -0600 4999)
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5000) if (poll && poll->head) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5001) bool done;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5002)
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5003) spin_lock(&poll->head->lock);
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5004) done = list_empty(&poll->wait.entry);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5005) if (!done)
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5006) list_del_init(&poll->wait.entry);
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 5007) /* make sure double remove sees this as being gone */
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 5008) wait->private = NULL;
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5009) spin_unlock(&poll->head->lock);
c8b5e2600a2cf (Jens Axboe 2020-10-25 13:53:26 -0600 5010) if (!done) {
c8b5e2600a2cf (Jens Axboe 2020-10-25 13:53:26 -0600 5011) /* use wait func handler, so it matches the rq type */
c8b5e2600a2cf (Jens Axboe 2020-10-25 13:53:26 -0600 5012) poll->wait.func(&poll->wait, mode, sync, key);
c8b5e2600a2cf (Jens Axboe 2020-10-25 13:53:26 -0600 5013) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5014) }
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 5015) req_ref_put(req);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5016) return 1;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5017) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5018)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5019) static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5020) wait_queue_func_t wake_func)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5021) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5022) poll->head = NULL;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5023) poll->done = false;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5024) poll->canceled = false;
464dca612bc6b (Jens Axboe 2021-03-19 14:06:24 -0600 5025) #define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
464dca612bc6b (Jens Axboe 2021-03-19 14:06:24 -0600 5026) /* mask in events that we always want/need */
464dca612bc6b (Jens Axboe 2021-03-19 14:06:24 -0600 5027) poll->events = events | IO_POLL_UNMASK;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5028) INIT_LIST_HEAD(&poll->wait.entry);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5029) init_waitqueue_func_entry(&poll->wait, wake_func);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5030) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5031)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5032) static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5033) struct wait_queue_head *head,
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5034) struct io_poll_iocb **poll_ptr)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5035) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5036) struct io_kiocb *req = pt->req;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5037)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5038) /*
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5039) * The file being polled uses multiple waitqueues for poll handling
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5040) * (e.g. one for read, one for write). Setup a separate io_poll_iocb
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5041) * if this happens.
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5042) */
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5043) if (unlikely(pt->nr_entries)) {
58852d4d67376 (Pavel Begunkov 2020-10-16 20:55:56 +0100 5044) struct io_poll_iocb *poll_one = poll;
58852d4d67376 (Pavel Begunkov 2020-10-16 20:55:56 +0100 5045)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5046) /* already have a 2nd entry, fail a third attempt */
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5047) if (*poll_ptr) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5048) pt->error = -EINVAL;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5049) return;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5050) }
ea6a693d862d4 (Jens Axboe 2021-04-15 09:47:13 -0600 5051) /*
ea6a693d862d4 (Jens Axboe 2021-04-15 09:47:13 -0600 5052) * Can't handle multishot for double wait for now, turn it
ea6a693d862d4 (Jens Axboe 2021-04-15 09:47:13 -0600 5053) * into one-shot mode.
ea6a693d862d4 (Jens Axboe 2021-04-15 09:47:13 -0600 5054) */
7a274727702cc (Pavel Begunkov 2021-05-17 12:43:34 +0100 5055) if (!(poll_one->events & EPOLLONESHOT))
7a274727702cc (Pavel Begunkov 2021-05-17 12:43:34 +0100 5056) poll_one->events |= EPOLLONESHOT;
1c3b3e6527e57 (Jens Axboe 2021-02-28 16:07:30 -0700 5057) /* double add on the same waitqueue head, ignore */
7a274727702cc (Pavel Begunkov 2021-05-17 12:43:34 +0100 5058) if (poll_one->head == head)
1c3b3e6527e57 (Jens Axboe 2021-02-28 16:07:30 -0700 5059) return;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5060) poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5061) if (!poll) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5062) pt->error = -ENOMEM;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5063) return;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5064) }
58852d4d67376 (Pavel Begunkov 2020-10-16 20:55:56 +0100 5065) io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 5066) req_ref_get(req);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5067) poll->wait.private = req;
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5068) *poll_ptr = poll;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5069) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5070)
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5071) pt->nr_entries++;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5072) poll->head = head;
a31eb4a2f1650 (Jiufei Xue 2020-06-17 17:53:56 +0800 5073)
a31eb4a2f1650 (Jiufei Xue 2020-06-17 17:53:56 +0800 5074) if (poll->events & EPOLLEXCLUSIVE)
a31eb4a2f1650 (Jiufei Xue 2020-06-17 17:53:56 +0800 5075) add_wait_queue_exclusive(head, &poll->wait);
a31eb4a2f1650 (Jiufei Xue 2020-06-17 17:53:56 +0800 5076) else
a31eb4a2f1650 (Jiufei Xue 2020-06-17 17:53:56 +0800 5077) add_wait_queue(head, &poll->wait);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5078) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5079)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5080) static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5081) struct poll_table_struct *p)
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5082) {
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5083) struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5084) struct async_poll *apoll = pt->req->apoll;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5085)
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5086) __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5087) }
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5088)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5089) static void io_async_task_func(struct callback_head *cb)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5090) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5091) struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5092) struct async_poll *apoll = req->apoll;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5093) struct io_ring_ctx *ctx = req->ctx;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5094)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5095) trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5096)
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 5097) if (io_poll_rewait(req, &apoll->poll)) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5098) spin_unlock_irq(&ctx->completion_lock);
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 5099) return;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5100) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5101)
0ea13b448ee75 (Pavel Begunkov 2021-04-09 09:13:21 +0100 5102) hash_del(&req->hash_node);
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 5103) io_poll_remove_double(req);
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 5104) spin_unlock_irq(&ctx->completion_lock);
74ce6ce43d4fc (Jens Axboe 2020-04-13 11:09:12 -0600 5105)
0be0b0e33b0bf (Pavel Begunkov 2020-06-30 15:20:42 +0300 5106) if (!READ_ONCE(apoll->poll.canceled))
0be0b0e33b0bf (Pavel Begunkov 2020-06-30 15:20:42 +0300 5107) __io_req_task_submit(req);
0be0b0e33b0bf (Pavel Begunkov 2020-06-30 15:20:42 +0300 5108) else
2593553a01c80 (Pavel Begunkov 2021-03-19 17:22:40 +0000 5109) io_req_complete_failed(req, -ECANCELED);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5110) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5111)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5112) static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5113) void *key)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5114) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5115) struct io_kiocb *req = wait->private;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5116) struct io_poll_iocb *poll = &req->apoll->poll;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5117)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5118) trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5119) key_to_poll(key));
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5120)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5121) return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5122) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5123)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5124) static void io_poll_req_insert(struct io_kiocb *req)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5125) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5126) struct io_ring_ctx *ctx = req->ctx;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5127) struct hlist_head *list;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5128)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5129) list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5130) hlist_add_head(&req->hash_node, list);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5131) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5132)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5133) static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5134) struct io_poll_iocb *poll,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5135) struct io_poll_table *ipt, __poll_t mask,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5136) wait_queue_func_t wake_func)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5137) __acquires(&ctx->completion_lock)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5138) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5139) struct io_ring_ctx *ctx = req->ctx;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5140) bool cancel = false;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5141)
4d52f338992bf (Pavel Begunkov 2020-10-18 10:17:43 +0100 5142) INIT_HLIST_NODE(&req->hash_node);
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5143) io_init_poll_iocb(poll, mask, wake_func);
b90cd197f9315 (Pavel Begunkov 2020-06-21 13:09:52 +0300 5144) poll->file = req->file;
18bceab101add (Jens Axboe 2020-05-15 11:56:54 -0600 5145) poll->wait.private = req;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5146)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5147) ipt->pt._key = mask;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5148) ipt->req = req;
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5149) ipt->error = 0;
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5150) ipt->nr_entries = 0;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5151)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5152) mask = vfs_poll(req->file, &ipt->pt) & poll->events;
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5153) if (unlikely(!ipt->nr_entries) && !ipt->error)
0d80ae099a495 (Pavel Begunkov 2021-07-20 10:50:43 +0100 5154) ipt->error = -EINVAL;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5155)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5156) spin_lock_irq(&ctx->completion_lock);
a8d4169f924ad (Hao Xu 2021-07-28 11:03:22 +0800 5157) if (ipt->error || (mask && (poll->events & EPOLLONESHOT)))
81cebadedc37c (Pavel Begunkov 2021-07-20 10:50:44 +0100 5158) io_poll_remove_double(req);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5159) if (likely(poll->head)) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5160) spin_lock(&poll->head->lock);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5161) if (unlikely(list_empty(&poll->wait.entry))) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5162) if (ipt->error)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5163) cancel = true;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5164) ipt->error = 0;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5165) mask = 0;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5166) }
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 5167) if ((mask && (poll->events & EPOLLONESHOT)) || ipt->error)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5168) list_del_init(&poll->wait.entry);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5169) else if (cancel)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5170) WRITE_ONCE(poll->canceled, true);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5171) else if (!poll->done) /* actually waiting for an event */
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5172) io_poll_req_insert(req);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5173) spin_unlock(&poll->head->lock);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5174) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5175)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5176) return mask;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5177) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5178)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5179) static bool io_arm_poll_handler(struct io_kiocb *req)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5180) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5181) const struct io_op_def *def = &io_op_defs[req->opcode];
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5182) struct io_ring_ctx *ctx = req->ctx;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5183) struct async_poll *apoll;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5184) struct io_poll_table ipt;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5185) __poll_t mask, ret;
9dab14b81807a (Jens Axboe 2020-08-25 12:27:50 -0600 5186) int rw;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5187)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5188) if (!req->file || !file_can_poll(req->file))
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5189) return false;
24c74678634b3 (Pavel Begunkov 2020-06-21 13:09:51 +0300 5190) if (req->flags & REQ_F_POLLED)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5191) return false;
9dab14b81807a (Jens Axboe 2020-08-25 12:27:50 -0600 5192) if (def->pollin)
9dab14b81807a (Jens Axboe 2020-08-25 12:27:50 -0600 5193) rw = READ;
9dab14b81807a (Jens Axboe 2020-08-25 12:27:50 -0600 5194) else if (def->pollout)
9dab14b81807a (Jens Axboe 2020-08-25 12:27:50 -0600 5195) rw = WRITE;
9dab14b81807a (Jens Axboe 2020-08-25 12:27:50 -0600 5196) else
9dab14b81807a (Jens Axboe 2020-08-25 12:27:50 -0600 5197) return false;
9dab14b81807a (Jens Axboe 2020-08-25 12:27:50 -0600 5198) /* if we can't nonblock try, then no point in arming a poll handler */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 5199) if (!io_file_supports_async(req, rw))
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5200) return false;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5201)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5202) apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5203) if (unlikely(!apoll))
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5204) return false;
807abcb088343 (Jens Axboe 2020-07-17 17:09:27 -0600 5205) apoll->double_poll = NULL;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5206)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5207) req->flags |= REQ_F_POLLED;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5208) req->apoll = apoll;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5209)
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 5210) mask = EPOLLONESHOT;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5211) if (def->pollin)
8755d97a09fed (Nathan Chancellor 2020-03-02 16:01:19 -0700 5212) mask |= POLLIN | POLLRDNORM;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5213) if (def->pollout)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5214) mask |= POLLOUT | POLLWRNORM;
901341bb97188 (Luke Hsiao 2020-08-21 21:41:05 -0700 5215)
901341bb97188 (Luke Hsiao 2020-08-21 21:41:05 -0700 5216) /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
901341bb97188 (Luke Hsiao 2020-08-21 21:41:05 -0700 5217) if ((req->opcode == IORING_OP_RECVMSG) &&
901341bb97188 (Luke Hsiao 2020-08-21 21:41:05 -0700 5218) (req->sr_msg.msg_flags & MSG_ERRQUEUE))
901341bb97188 (Luke Hsiao 2020-08-21 21:41:05 -0700 5219) mask &= ~POLLIN;
901341bb97188 (Luke Hsiao 2020-08-21 21:41:05 -0700 5220)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5221) mask |= POLLERR | POLLPRI;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5222)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5223) ipt.pt._qproc = io_async_queue_proc;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5224)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5225) ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5226) io_async_wake);
a36da65c46565 (Jens Axboe 2020-08-11 09:50:19 -0600 5227) if (ret || ipt.error) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5228) spin_unlock_irq(&ctx->completion_lock);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5229) return false;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5230) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5231) spin_unlock_irq(&ctx->completion_lock);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5232) trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5233) apoll->poll.events);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5234) return true;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5235) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5236)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5237) static bool __io_poll_remove_one(struct io_kiocb *req,
b2e720ace221f (Jens Axboe 2021-03-31 09:03:03 -0600 5238) struct io_poll_iocb *poll, bool do_cancel)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 5239) __must_hold(&req->ctx->completion_lock)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5240) {
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5241) bool do_complete = false;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5242)
5082620fb2cab (Jens Axboe 2021-02-23 09:02:26 -0700 5243) if (!poll->head)
5082620fb2cab (Jens Axboe 2021-02-23 09:02:26 -0700 5244) return false;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5245) spin_lock(&poll->head->lock);
b2e720ace221f (Jens Axboe 2021-03-31 09:03:03 -0600 5246) if (do_cancel)
b2e720ace221f (Jens Axboe 2021-03-31 09:03:03 -0600 5247) WRITE_ONCE(poll->canceled, true);
392edb45b2433 (Jens Axboe 2019-12-09 17:52:20 -0700 5248) if (!list_empty(&poll->wait.entry)) {
392edb45b2433 (Jens Axboe 2019-12-09 17:52:20 -0700 5249) list_del_init(&poll->wait.entry);
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5250) do_complete = true;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5251) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5252) spin_unlock(&poll->head->lock);
3bfa5bcb26f0b (Jens Axboe 2020-05-17 13:54:12 -0600 5253) hash_del(&req->hash_node);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5254) return do_complete;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5255) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5256)
b2c3f7e171560 (Jens Axboe 2021-02-23 08:58:04 -0700 5257) static bool io_poll_remove_waitqs(struct io_kiocb *req)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 5258) __must_hold(&req->ctx->completion_lock)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5259) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5260) bool do_complete;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5261)
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 5262) io_poll_remove_double(req);
e31001a3abb81 (Pavel Begunkov 2021-04-13 02:58:43 +0100 5263) do_complete = __io_poll_remove_one(req, io_poll_get_single(req), true);
d4e7cd36a90e3 (Jens Axboe 2020-08-15 11:44:50 -0700 5264)
e31001a3abb81 (Pavel Begunkov 2021-04-13 02:58:43 +0100 5265) if (req->opcode != IORING_OP_POLL_ADD && do_complete) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5266) /* non-poll requests have submit ref still */
e31001a3abb81 (Pavel Begunkov 2021-04-13 02:58:43 +0100 5267) req_ref_put(req);
b1f573bd15fda (Xiaoguang Wang 2020-04-12 14:50:54 +0800 5268) }
b2c3f7e171560 (Jens Axboe 2021-02-23 08:58:04 -0700 5269) return do_complete;
b2c3f7e171560 (Jens Axboe 2021-02-23 08:58:04 -0700 5270) }
b2c3f7e171560 (Jens Axboe 2021-02-23 08:58:04 -0700 5271)
b2c3f7e171560 (Jens Axboe 2021-02-23 08:58:04 -0700 5272) static bool io_poll_remove_one(struct io_kiocb *req)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 5273) __must_hold(&req->ctx->completion_lock)
b2c3f7e171560 (Jens Axboe 2021-02-23 08:58:04 -0700 5274) {
b2c3f7e171560 (Jens Axboe 2021-02-23 08:58:04 -0700 5275) bool do_complete;
b1f573bd15fda (Xiaoguang Wang 2020-04-12 14:50:54 +0800 5276)
b2c3f7e171560 (Jens Axboe 2021-02-23 08:58:04 -0700 5277) do_complete = io_poll_remove_waitqs(req);
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5278) if (do_complete) {
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 5279) io_cqring_fill_event(req->ctx, req->user_data, -ECANCELED, 0);
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5280) io_commit_cqring(req->ctx);
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 5281) req_set_fail_links(req);
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 5282) io_put_req_deferred(req, 1);
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5283) }
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5284)
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5285) return do_complete;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5286) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5287)
76e1b6427fd82 (Jens Axboe 2020-09-26 15:05:03 -0600 5288) /*
76e1b6427fd82 (Jens Axboe 2020-09-26 15:05:03 -0600 5289) * Returns true if we found and killed one or more poll requests
76e1b6427fd82 (Jens Axboe 2020-09-26 15:05:03 -0600 5290) */
6b81928d4ca86 (Pavel Begunkov 2020-11-06 13:00:25 +0000 5291) static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 5292) bool cancel_all)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5293) {
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 5294) struct hlist_node *tmp;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5295) struct io_kiocb *req;
8e2e1faf28b3e (Jens Axboe 2020-04-13 17:05:14 -0600 5296) int posted = 0, i;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5297)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5298) spin_lock_irq(&ctx->completion_lock);
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 5299) for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 5300) struct hlist_head *list;
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 5301)
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 5302) list = &ctx->cancel_hash[i];
f3606e3a92ddd (Jens Axboe 2020-09-22 08:18:24 -0600 5303) hlist_for_each_entry_safe(req, tmp, list, hash_node) {
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 5304) if (io_match_task(req, tsk, cancel_all))
f3606e3a92ddd (Jens Axboe 2020-09-22 08:18:24 -0600 5305) posted += io_poll_remove_one(req);
f3606e3a92ddd (Jens Axboe 2020-09-22 08:18:24 -0600 5306) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5307) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5308) spin_unlock_irq(&ctx->completion_lock);
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5309)
8e2e1faf28b3e (Jens Axboe 2020-04-13 17:05:14 -0600 5310) if (posted)
8e2e1faf28b3e (Jens Axboe 2020-04-13 17:05:14 -0600 5311) io_cqring_ev_posted(ctx);
76e1b6427fd82 (Jens Axboe 2020-09-26 15:05:03 -0600 5312)
76e1b6427fd82 (Jens Axboe 2020-09-26 15:05:03 -0600 5313) return posted != 0;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5314) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5315)
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5316) static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5317) bool poll_only)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 5318) __must_hold(&ctx->completion_lock)
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5319) {
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 5320) struct hlist_head *list;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5321) struct io_kiocb *req;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5322)
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 5323) list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 5324) hlist_for_each_entry(req, list, hash_node) {
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5325) if (sqe_addr != req->user_data)
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 5326) continue;
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5327) if (poll_only && req->opcode != IORING_OP_POLL_ADD)
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5328) continue;
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5329) return req;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5330) }
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5331) return NULL;
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5332) }
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5333)
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5334) static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5335) bool poll_only)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 5336) __must_hold(&ctx->completion_lock)
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5337) {
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5338) struct io_kiocb *req;
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5339)
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5340) req = io_poll_find(ctx, sqe_addr, poll_only);
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5341) if (!req)
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5342) return -ENOENT;
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5343) if (io_poll_remove_one(req))
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5344) return 0;
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5345)
b2cb805f6dd40 (Jens Axboe 2021-03-17 08:17:19 -0600 5346) return -EALREADY;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5347) }
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5348)
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5349) static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5350) unsigned int flags)
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5351) {
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5352) u32 events;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5353)
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5354) events = READ_ONCE(sqe->poll32_events);
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5355) #ifdef __BIG_ENDIAN
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5356) events = swahw32(events);
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5357) #endif
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5358) if (!(flags & IORING_POLL_ADD_MULTI))
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5359) events |= EPOLLONESHOT;
9096af3e9c873 (Pavel Begunkov 2021-04-14 13:38:36 +0100 5360) return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT));
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5361) }
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5362)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5363) static int io_poll_update_prep(struct io_kiocb *req,
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 5364) const struct io_uring_sqe *sqe)
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5365) {
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5366) struct io_poll_update *upd = &req->poll_update;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5367) u32 flags;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5368)
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5369) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5370) return -EINVAL;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5371) if (sqe->ioprio || sqe->buf_index)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5372) return -EINVAL;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5373) flags = READ_ONCE(sqe->len);
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5374) if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5375) IORING_POLL_ADD_MULTI))
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5376) return -EINVAL;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5377) /* meaningless without update */
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5378) if (flags == IORING_POLL_ADD_MULTI)
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5379) return -EINVAL;
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5380)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5381) upd->old_user_data = READ_ONCE(sqe->addr);
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5382) upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5383) upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5384)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5385) upd->new_user_data = READ_ONCE(sqe->off);
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5386) if (!upd->update_user_data && upd->new_user_data)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5387) return -EINVAL;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5388) if (upd->update_events)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5389) upd->events = io_poll_parse_events(sqe, flags);
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5390) else if (sqe->poll32_events)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5391) return -EINVAL;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5392)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5393) return 0;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5394) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5395)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5396) static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5397) void *key)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5398) {
c2f2eb7d2c1cd (Jens Axboe 2020-02-10 09:07:05 -0700 5399) struct io_kiocb *req = wait->private;
c2f2eb7d2c1cd (Jens Axboe 2020-02-10 09:07:05 -0700 5400) struct io_poll_iocb *poll = &req->poll;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5401)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5402) return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5403) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5404)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5405) static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5406) struct poll_table_struct *p)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5407) {
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5408) struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5409)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 5410) __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data);
eac406c61cd0e (Jens Axboe 2019-11-14 12:09:58 -0700 5411) }
eac406c61cd0e (Jens Axboe 2019-11-14 12:09:58 -0700 5412)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 5413) static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5414) {
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5415) struct io_poll_iocb *poll = &req->poll;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5416) u32 flags;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5417)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5418) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5419) return -EINVAL;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5420) if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 5421) return -EINVAL;
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 5422) flags = READ_ONCE(sqe->len);
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5423) if (flags & ~IORING_POLL_ADD_MULTI)
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5424) return -EINVAL;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5425)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5426) poll->events = io_poll_parse_events(sqe, flags);
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5427) return 0;
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5428) }
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5429)
61e9820304798 (Pavel Begunkov 2021-02-10 00:03:08 +0000 5430) static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5431) {
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5432) struct io_poll_iocb *poll = &req->poll;
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5433) struct io_ring_ctx *ctx = req->ctx;
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5434) struct io_poll_table ipt;
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5435) __poll_t mask;
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5436)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5437) ipt.pt._qproc = io_poll_queue_proc;
36703247d5f52 (Jens Axboe 2019-07-25 10:20:18 -0600 5438)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5439) mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 5440) io_poll_wake);
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5441)
8c838788775a5 (Jens Axboe 2019-03-12 15:48:16 -0600 5442) if (mask) { /* no async, we'd stolen it */
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5443) ipt.error = 0;
e27414bef7b4f (Pavel Begunkov 2021-04-09 09:13:20 +0100 5444) io_poll_complete(req, mask);
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5445) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5446) spin_unlock_irq(&ctx->completion_lock);
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5447)
8c838788775a5 (Jens Axboe 2019-03-12 15:48:16 -0600 5448) if (mask) {
8c838788775a5 (Jens Axboe 2019-03-12 15:48:16 -0600 5449) io_cqring_ev_posted(ctx);
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 5450) if (poll->events & EPOLLONESHOT)
88e41cf928a6e (Jens Axboe 2021-02-22 22:08:01 -0700 5451) io_put_req(req);
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5452) }
8c838788775a5 (Jens Axboe 2019-03-12 15:48:16 -0600 5453) return ipt.error;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5454) }
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 5455)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5456) static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5457) {
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5458) struct io_ring_ctx *ctx = req->ctx;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5459) struct io_kiocb *preq;
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5460) bool completing;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5461) int ret;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5462)
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5463) spin_lock_irq(&ctx->completion_lock);
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5464) preq = io_poll_find(ctx, req->poll_update.old_user_data, true);
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5465) if (!preq) {
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5466) ret = -ENOENT;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5467) goto err;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5468) }
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5469)
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5470) if (!req->poll_update.update_events && !req->poll_update.update_user_data) {
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5471) completing = true;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5472) ret = io_poll_remove_one(preq) ? 0 : -EALREADY;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5473) goto err;
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5474) }
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5475)
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5476) /*
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5477) * Don't allow racy completion with singleshot, as we cannot safely
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5478) * update those. For multishot, if we're racing with completion, just
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5479) * let completion re-add it.
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5480) */
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5481) completing = !__io_poll_remove_one(preq, &preq->poll, false);
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5482) if (completing && (preq->poll.events & EPOLLONESHOT)) {
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5483) ret = -EALREADY;
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5484) goto err;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5485) }
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5486) /* we now have a detached poll request. reissue. */
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5487) ret = 0;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5488) err:
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5489) if (ret < 0) {
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5490) spin_unlock_irq(&ctx->completion_lock);
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5491) req_set_fail_links(req);
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5492) io_req_complete(req, ret);
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5493) return 0;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5494) }
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5495) /* only mask one event flags, keep behavior flags */
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 5496) if (req->poll_update.update_events) {
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5497) preq->poll.events &= ~0xffff;
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 5498) preq->poll.events |= req->poll_update.events & 0xffff;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5499) preq->poll.events |= IO_POLL_UNMASK;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5500) }
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 5501) if (req->poll_update.update_user_data)
9d8058926be70 (Pavel Begunkov 2021-04-13 02:58:40 +0100 5502) preq->user_data = req->poll_update.new_user_data;
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5503) spin_unlock_irq(&ctx->completion_lock);
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5504)
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5505) /* complete update request, we're done with it */
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5506) io_req_complete(req, ret);
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5507)
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5508) if (!completing) {
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5509) ret = io_poll_add(preq, issue_flags);
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5510) if (ret < 0) {
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5511) req_set_fail_links(preq);
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5512) io_req_complete(preq, ret);
cb3b200e4f665 (Jens Axboe 2021-04-06 09:49:31 -0600 5513) }
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5514) }
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5515) return 0;
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5516) }
b69de288e9130 (Jens Axboe 2021-03-17 08:37:41 -0600 5517)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5518) static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5519) {
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5520) struct io_timeout_data *data = container_of(timer,
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5521) struct io_timeout_data, timer);
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5522) struct io_kiocb *req = data->req;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5523) struct io_ring_ctx *ctx = req->ctx;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5524) unsigned long flags;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5525)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5526) spin_lock_irqsave(&ctx->completion_lock, flags);
a71976f3fa474 (Pavel Begunkov 2020-10-10 18:34:11 +0100 5527) list_del_init(&req->timeout.list);
01cec8c18f5ad (Pavel Begunkov 2020-07-30 18:43:50 +0300 5528) atomic_set(&req->ctx->cq_timeouts,
01cec8c18f5ad (Pavel Begunkov 2020-07-30 18:43:50 +0300 5529) atomic_read(&req->ctx->cq_timeouts) + 1);
01cec8c18f5ad (Pavel Begunkov 2020-07-30 18:43:50 +0300 5530)
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 5531) io_cqring_fill_event(ctx, req->user_data, -ETIME, 0);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5532) io_commit_cqring(ctx);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5533) spin_unlock_irqrestore(&ctx->completion_lock, flags);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5534)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5535) io_cqring_ev_posted(ctx);
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 5536) req_set_fail_links(req);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5537) io_put_req(req);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5538) return HRTIMER_NORESTART;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5539) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5540)
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5541) static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5542) __u64 user_data)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 5543) __must_hold(&ctx->completion_lock)
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 5544) {
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5545) struct io_timeout_data *io;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5546) struct io_kiocb *req;
fd9c7bc542dae (Pavel Begunkov 2021-04-13 02:58:42 +0100 5547) bool found = false;
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 5548)
135fcde8496b0 (Pavel Begunkov 2020-07-13 23:37:12 +0300 5549) list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
fd9c7bc542dae (Pavel Begunkov 2021-04-13 02:58:42 +0100 5550) found = user_data == req->user_data;
fd9c7bc542dae (Pavel Begunkov 2021-04-13 02:58:42 +0100 5551) if (found)
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5552) break;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5553) }
fd9c7bc542dae (Pavel Begunkov 2021-04-13 02:58:42 +0100 5554) if (!found)
fd9c7bc542dae (Pavel Begunkov 2021-04-13 02:58:42 +0100 5555) return ERR_PTR(-ENOENT);
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5556)
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5557) io = req->async_data;
fd9c7bc542dae (Pavel Begunkov 2021-04-13 02:58:42 +0100 5558) if (hrtimer_try_to_cancel(&io->timer) == -1)
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5559) return ERR_PTR(-EALREADY);
a71976f3fa474 (Pavel Begunkov 2020-10-10 18:34:11 +0100 5560) list_del_init(&req->timeout.list);
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5561) return req;
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5562) }
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5563)
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5564) static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 5565) __must_hold(&ctx->completion_lock)
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5566) {
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5567) struct io_kiocb *req = io_timeout_extract(ctx, user_data);
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5568)
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5569) if (IS_ERR(req))
fbd15848f3c13 (Pavel Begunkov 2020-11-30 19:11:15 +0000 5570) return PTR_ERR(req);
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 5571)
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 5572) req_set_fail_links(req);
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 5573) io_cqring_fill_event(ctx, req->user_data, -ECANCELED, 0);
216578e55ac93 (Pavel Begunkov 2020-10-13 09:44:00 +0100 5574) io_put_req_deferred(req, 1);
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 5575) return 0;
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 5576) }
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 5577)
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5578) static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5579) struct timespec64 *ts, enum hrtimer_mode mode)
e07785b002916 (Pavel Begunkov 2021-04-01 15:43:57 +0100 5580) __must_hold(&ctx->completion_lock)
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5581) {
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5582) struct io_kiocb *req = io_timeout_extract(ctx, user_data);
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5583) struct io_timeout_data *data;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5584)
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5585) if (IS_ERR(req))
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5586) return PTR_ERR(req);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5587)
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5588) req->timeout.off = 0; /* noseq */
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5589) data = req->async_data;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5590) list_add_tail(&req->timeout.list, &ctx->timeout_list);
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5591) hrtimer_init(&data->timer, CLOCK_MONOTONIC, mode);
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5592) data->timer.function = io_timeout_fn;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5593) hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5594) return 0;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5595) }
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5596)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 5597) static int io_timeout_remove_prep(struct io_kiocb *req,
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 5598) const struct io_uring_sqe *sqe)
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5599) {
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5600) struct io_timeout_rem *tr = &req->timeout_rem;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5601)
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5602) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5603) return -EINVAL;
61710e437f280 (Daniele Albano 2020-07-18 14:15:16 -0600 5604) if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
61710e437f280 (Daniele Albano 2020-07-18 14:15:16 -0600 5605) return -EINVAL;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5606) if (sqe->ioprio || sqe->buf_index || sqe->len)
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5607) return -EINVAL;
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5608)
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5609) tr->addr = READ_ONCE(sqe->addr);
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5610) tr->flags = READ_ONCE(sqe->timeout_flags);
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5611) if (tr->flags & IORING_TIMEOUT_UPDATE) {
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5612) if (tr->flags & ~(IORING_TIMEOUT_UPDATE|IORING_TIMEOUT_ABS))
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5613) return -EINVAL;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5614) if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5615) return -EFAULT;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5616) } else if (tr->flags) {
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5617) /* timeout removal doesn't support flags */
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5618) return -EINVAL;
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5619) }
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5620)
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5621) return 0;
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5622) }
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5623)
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5624) static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5625) {
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5626) return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5627) : HRTIMER_MODE_REL;
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5628) }
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5629)
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5630) /*
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5631) * Remove or update an existing timeout command
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5632) */
61e9820304798 (Pavel Begunkov 2021-02-10 00:03:08 +0000 5633) static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5634) {
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5635) struct io_timeout_rem *tr = &req->timeout_rem;
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5636) struct io_ring_ctx *ctx = req->ctx;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5637) int ret;
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5638)
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5639) spin_lock_irq(&ctx->completion_lock);
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5640) if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE))
9c8e11b36c9b6 (Pavel Begunkov 2020-11-30 19:11:16 +0000 5641) ret = io_timeout_cancel(ctx, tr->addr);
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5642) else
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5643) ret = io_timeout_update(ctx, tr->addr, &tr->ts,
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5644) io_translate_timeout_mode(tr->flags));
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5645)
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 5646) io_cqring_fill_event(ctx, req->user_data, ret, 0);
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5647) io_commit_cqring(ctx);
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5648) spin_unlock_irq(&ctx->completion_lock);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5649) io_cqring_ev_posted(ctx);
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 5650) if (ret < 0)
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 5651) req_set_fail_links(req);
ec9c02ad4c380 (Jackie Liu 2019-11-08 23:50:36 +0800 5652) io_put_req(req);
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 5653) return 0;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5654) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5655)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 5656) static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2d28390aff879 (Jens Axboe 2019-12-04 11:08:05 -0700 5657) bool is_timeout_link)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5658) {
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5659) struct io_timeout_data *data;
a41525ab2e759 (Jens Axboe 2019-10-15 16:48:15 -0600 5660) unsigned flags;
56080b02ed6e7 (Pavel Begunkov 2020-05-26 20:34:04 +0300 5661) u32 off = READ_ONCE(sqe->off);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5662)
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5663) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5664) return -EINVAL;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5665) if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
a41525ab2e759 (Jens Axboe 2019-10-15 16:48:15 -0600 5666) return -EINVAL;
56080b02ed6e7 (Pavel Begunkov 2020-05-26 20:34:04 +0300 5667) if (off && is_timeout_link)
2d28390aff879 (Jens Axboe 2019-12-04 11:08:05 -0700 5668) return -EINVAL;
a41525ab2e759 (Jens Axboe 2019-10-15 16:48:15 -0600 5669) flags = READ_ONCE(sqe->timeout_flags);
a41525ab2e759 (Jens Axboe 2019-10-15 16:48:15 -0600 5670) if (flags & ~IORING_TIMEOUT_ABS)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5671) return -EINVAL;
bdf200731145f (Arnd Bergmann 2019-10-01 09:53:29 -0600 5672)
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 5673) req->timeout.off = off;
26a61679f10c6 (Jens Axboe 2019-12-20 09:02:01 -0700 5674)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 5675) if (!req->async_data && io_alloc_async_data(req))
26a61679f10c6 (Jens Axboe 2019-12-20 09:02:01 -0700 5676) return -ENOMEM;
26a61679f10c6 (Jens Axboe 2019-12-20 09:02:01 -0700 5677)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 5678) data = req->async_data;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5679) data->req = req;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5680)
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5681) if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5682) return -EFAULT;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5683)
8662daec09edc (Pavel Begunkov 2021-01-19 13:32:44 +0000 5684) data->mode = io_translate_timeout_mode(flags);
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5685) hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
2482b58ffbdc8 (Pavel Begunkov 2021-03-25 18:32:44 +0000 5686) if (is_timeout_link)
2482b58ffbdc8 (Pavel Begunkov 2021-03-25 18:32:44 +0000 5687) io_req_track_inflight(req);
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5688) return 0;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5689) }
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5690)
61e9820304798 (Pavel Begunkov 2021-02-10 00:03:08 +0000 5691) static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5692) {
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5693) struct io_ring_ctx *ctx = req->ctx;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 5694) struct io_timeout_data *data = req->async_data;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5695) struct list_head *entry;
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 5696) u32 tail, off = req->timeout.off;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5697)
733f5c95e6fda (Pavel Begunkov 2020-05-26 20:34:03 +0300 5698) spin_lock_irq(&ctx->completion_lock);
93bd25bb69f46 (Jens Axboe 2019-11-11 23:34:31 -0700 5699)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5700) /*
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5701) * sqe->off holds how many events that need to occur for this
93bd25bb69f46 (Jens Axboe 2019-11-11 23:34:31 -0700 5702) * timeout event to be satisfied. If it isn't set, then this is
93bd25bb69f46 (Jens Axboe 2019-11-11 23:34:31 -0700 5703) * a pure timeout request, sequence isn't used.
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5704) */
8eb7e2d007633 (Pavel Begunkov 2020-06-29 13:13:02 +0300 5705) if (io_is_timeout_noseq(req)) {
93bd25bb69f46 (Jens Axboe 2019-11-11 23:34:31 -0700 5706) entry = ctx->timeout_list.prev;
93bd25bb69f46 (Jens Axboe 2019-11-11 23:34:31 -0700 5707) goto add;
93bd25bb69f46 (Jens Axboe 2019-11-11 23:34:31 -0700 5708) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5709)
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 5710) tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 5711) req->timeout.target_seq = tail + off;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5712)
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 5713) /* Update the last seq here in case io_flush_timeouts() hasn't.
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 5714) * This is safe because ->completion_lock is held, and submissions
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 5715) * and completions are never mixed in the same ->completion_lock section.
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 5716) */
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 5717) ctx->cq_last_tm_flush = tail;
f010505b78a4f (Marcelo Diop-Gonzalez 2021-01-15 11:54:40 -0500 5718)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5719) /*
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5720) * Insertion sort, ensuring the first entry in the list is always
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5721) * the one we need first.
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5722) */
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5723) list_for_each_prev(entry, &ctx->timeout_list) {
135fcde8496b0 (Pavel Begunkov 2020-07-13 23:37:12 +0300 5724) struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
135fcde8496b0 (Pavel Begunkov 2020-07-13 23:37:12 +0300 5725) timeout.list);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5726)
8eb7e2d007633 (Pavel Begunkov 2020-06-29 13:13:02 +0300 5727) if (io_is_timeout_noseq(nxt))
93bd25bb69f46 (Jens Axboe 2019-11-11 23:34:31 -0700 5728) continue;
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 5729) /* nxt.seq is behind @tail, otherwise would've been completed */
bfe68a221905d (Pavel Begunkov 2020-05-30 14:54:18 +0300 5730) if (off >= nxt->timeout.target_seq - tail)
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5731) break;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5732) }
93bd25bb69f46 (Jens Axboe 2019-11-11 23:34:31 -0700 5733) add:
135fcde8496b0 (Pavel Begunkov 2020-07-13 23:37:12 +0300 5734) list_add(&req->timeout.list, entry);
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5735) data->timer.function = io_timeout_fn;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 5736) hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5737) spin_unlock_irq(&ctx->completion_lock);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5738) return 0;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5739) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5740)
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5741) struct io_cancel_data {
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5742) struct io_ring_ctx *ctx;
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5743) u64 user_data;
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5744) };
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5745)
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5746) static bool io_cancel_cb(struct io_wq_work *work, void *data)
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5747) {
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5748) struct io_kiocb *req = container_of(work, struct io_kiocb, work);
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5749) struct io_cancel_data *cd = data;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5750)
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5751) return req->ctx == cd->ctx && req->user_data == cd->user_data;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5752) }
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5753)
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5754) static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5755) struct io_ring_ctx *ctx)
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5756) {
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5757) struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5758) enum io_wq_cancel cancel_ret;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5759) int ret = 0;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5760)
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5761) if (!tctx || !tctx->io_wq)
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 5762) return -ENOENT;
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 5763)
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5764) cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5765) switch (cancel_ret) {
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5766) case IO_WQ_CANCEL_OK:
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5767) ret = 0;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5768) break;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5769) case IO_WQ_CANCEL_RUNNING:
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5770) ret = -EALREADY;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5771) break;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5772) case IO_WQ_CANCEL_NOTFOUND:
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5773) ret = -ENOENT;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5774) break;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5775) }
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 5776)
e977d6d34f0c0 (Jens Axboe 2019-11-05 12:39:45 -0700 5777) return ret;
e977d6d34f0c0 (Jens Axboe 2019-11-05 12:39:45 -0700 5778) }
e977d6d34f0c0 (Jens Axboe 2019-11-05 12:39:45 -0700 5779)
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5780) static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5781) struct io_kiocb *req, __u64 sqe_addr,
014db0073cc6a (Pavel Begunkov 2020-03-03 21:33:12 +0300 5782) int success_ret)
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5783) {
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5784) unsigned long flags;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5785) int ret;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5786)
f458dd8441e56 (Pavel Begunkov 2021-03-08 12:14:14 +0000 5787) ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5788) spin_lock_irqsave(&ctx->completion_lock, flags);
df9727affa058 (Pavel Begunkov 2021-04-01 15:43:59 +0100 5789) if (ret != -ENOENT)
df9727affa058 (Pavel Begunkov 2021-04-01 15:43:59 +0100 5790) goto done;
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5791) ret = io_timeout_cancel(ctx, sqe_addr);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5792) if (ret != -ENOENT)
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5793) goto done;
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5794) ret = io_poll_cancel(ctx, sqe_addr, false);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5795) done:
b0dd8a412699a (Jens Axboe 2019-11-18 12:14:54 -0700 5796) if (!ret)
b0dd8a412699a (Jens Axboe 2019-11-18 12:14:54 -0700 5797) ret = success_ret;
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 5798) io_cqring_fill_event(ctx, req->user_data, ret, 0);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5799) io_commit_cqring(ctx);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5800) spin_unlock_irqrestore(&ctx->completion_lock, flags);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5801) io_cqring_ev_posted(ctx);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5802)
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 5803) if (ret < 0)
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 5804) req_set_fail_links(req);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5805) }
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 5806)
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 5807) static int io_async_cancel_prep(struct io_kiocb *req,
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 5808) const struct io_uring_sqe *sqe)
e977d6d34f0c0 (Jens Axboe 2019-11-05 12:39:45 -0700 5809) {
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5810) if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
e977d6d34f0c0 (Jens Axboe 2019-11-05 12:39:45 -0700 5811) return -EINVAL;
61710e437f280 (Daniele Albano 2020-07-18 14:15:16 -0600 5812) if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
61710e437f280 (Daniele Albano 2020-07-18 14:15:16 -0600 5813) return -EINVAL;
61710e437f280 (Daniele Albano 2020-07-18 14:15:16 -0600 5814) if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
e977d6d34f0c0 (Jens Axboe 2019-11-05 12:39:45 -0700 5815) return -EINVAL;
e977d6d34f0c0 (Jens Axboe 2019-11-05 12:39:45 -0700 5816)
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5817) req->cancel.addr = READ_ONCE(sqe->addr);
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5818) return 0;
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5819) }
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5820)
61e9820304798 (Pavel Begunkov 2021-02-10 00:03:08 +0000 5821) static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5822) {
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5823) struct io_ring_ctx *ctx = req->ctx;
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5824) u64 sqe_addr = req->cancel.addr;
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5825) struct io_tctx_node *node;
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5826) int ret;
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5827)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5828) /* tasks should wait for their io-wq threads, so safe w/o sync */
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5829) ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5830) spin_lock_irq(&ctx->completion_lock);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5831) if (ret != -ENOENT)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5832) goto done;
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5833) ret = io_timeout_cancel(ctx, sqe_addr);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5834) if (ret != -ENOENT)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5835) goto done;
9ba5fac8cf3b6 (Pavel Begunkov 2021-04-14 13:38:35 +0100 5836) ret = io_poll_cancel(ctx, sqe_addr, false);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5837) if (ret != -ENOENT)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5838) goto done;
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5839) spin_unlock_irq(&ctx->completion_lock);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5840)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5841) /* slow path, try all io-wq's */
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5842) io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5843) ret = -ENOENT;
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5844) list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5845) struct io_uring_task *tctx = node->task->io_uring;
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5846)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5847) ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5848) if (ret != -ENOENT)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5849) break;
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5850) }
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5851) io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5852)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5853) spin_lock_irq(&ctx->completion_lock);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5854) done:
d4d19c19d6ae9 (Pavel Begunkov 2021-04-25 14:32:17 +0100 5855) io_cqring_fill_event(ctx, req->user_data, ret, 0);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5856) io_commit_cqring(ctx);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5857) spin_unlock_irq(&ctx->completion_lock);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5858) io_cqring_ev_posted(ctx);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5859)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5860) if (ret < 0)
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5861) req_set_fail_links(req);
58f9937383415 (Pavel Begunkov 2021-03-12 16:25:55 +0000 5862) io_put_req(req);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5863) return 0;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5864) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5865)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 5866) static int io_rsrc_update_prep(struct io_kiocb *req,
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5867) const struct io_uring_sqe *sqe)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5868) {
61710e437f280 (Daniele Albano 2020-07-18 14:15:16 -0600 5869) if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
61710e437f280 (Daniele Albano 2020-07-18 14:15:16 -0600 5870) return -EINVAL;
61710e437f280 (Daniele Albano 2020-07-18 14:15:16 -0600 5871) if (sqe->ioprio || sqe->rw_flags)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5872) return -EINVAL;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5873)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 5874) req->rsrc_update.offset = READ_ONCE(sqe->off);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 5875) req->rsrc_update.nr_args = READ_ONCE(sqe->len);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 5876) if (!req->rsrc_update.nr_args)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5877) return -EINVAL;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 5878) req->rsrc_update.arg = READ_ONCE(sqe->addr);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5879) return 0;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5880) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5881)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 5882) static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5883) {
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5884) struct io_ring_ctx *ctx = req->ctx;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 5885) struct io_uring_rsrc_update2 up;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5886) int ret;
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5887)
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 5888) if (issue_flags & IO_URING_F_NONBLOCK)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5889) return -EAGAIN;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5890)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 5891) up.offset = req->rsrc_update.offset;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 5892) up.data = req->rsrc_update.arg;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 5893) up.nr = 0;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 5894) up.tags = 0;
615cee49b3ca5 (Colin Ian King 2021-04-26 10:47:35 +0100 5895) up.resv = 0;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5896)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5897) mutex_lock(&ctx->uring_lock);
fdecb66281e16 (Pavel Begunkov 2021-04-25 14:32:20 +0100 5898) ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 5899) &up, req->rsrc_update.nr_args);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5900) mutex_unlock(&ctx->uring_lock);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5901)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5902) if (ret < 0)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5903) req_set_fail_links(req);
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 5904) __io_req_complete(req, issue_flags, ret, 0);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5905) return 0;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5906) }
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 5907)
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5908) static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 5909) {
d625c6ee49750 (Jens Axboe 2019-12-17 19:53:05 -0700 5910) switch (req->opcode) {
e781573e2fb1b (Jens Axboe 2019-12-17 19:45:06 -0700 5911) case IORING_OP_NOP:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5912) return 0;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 5913) case IORING_OP_READV:
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 5914) case IORING_OP_READ_FIXED:
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 5915) case IORING_OP_READ:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5916) return io_read_prep(req, sqe);
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 5917) case IORING_OP_WRITEV:
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 5918) case IORING_OP_WRITE_FIXED:
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 5919) case IORING_OP_WRITE:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5920) return io_write_prep(req, sqe);
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5921) case IORING_OP_POLL_ADD:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5922) return io_poll_add_prep(req, sqe);
0969e783e3a89 (Jens Axboe 2019-12-17 18:40:57 -0700 5923) case IORING_OP_POLL_REMOVE:
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 5924) return io_poll_update_prep(req, sqe);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 5925) case IORING_OP_FSYNC:
1155c76a24836 (Pavel Begunkov 2021-02-18 18:29:38 +0000 5926) return io_fsync_prep(req, sqe);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 5927) case IORING_OP_SYNC_FILE_RANGE:
1155c76a24836 (Pavel Begunkov 2021-02-18 18:29:38 +0000 5928) return io_sfr_prep(req, sqe);
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 5929) case IORING_OP_SENDMSG:
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 5930) case IORING_OP_SEND:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5931) return io_sendmsg_prep(req, sqe);
03b1230ca12a1 (Jens Axboe 2019-12-02 18:50:25 -0700 5932) case IORING_OP_RECVMSG:
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 5933) case IORING_OP_RECV:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5934) return io_recvmsg_prep(req, sqe);
f499a021ea8c9 (Jens Axboe 2019-12-02 16:28:46 -0700 5935) case IORING_OP_CONNECT:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5936) return io_connect_prep(req, sqe);
2d28390aff879 (Jens Axboe 2019-12-04 11:08:05 -0700 5937) case IORING_OP_TIMEOUT:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5938) return io_timeout_prep(req, sqe, false);
b29472ee7b537 (Jens Axboe 2019-12-17 18:50:29 -0700 5939) case IORING_OP_TIMEOUT_REMOVE:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5940) return io_timeout_remove_prep(req, sqe);
fbf23849b1724 (Jens Axboe 2019-12-17 18:45:56 -0700 5941) case IORING_OP_ASYNC_CANCEL:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5942) return io_async_cancel_prep(req, sqe);
2d28390aff879 (Jens Axboe 2019-12-04 11:08:05 -0700 5943) case IORING_OP_LINK_TIMEOUT:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5944) return io_timeout_prep(req, sqe, true);
8ed8d3c3bc32b (Jens Axboe 2019-12-16 11:55:28 -0700 5945) case IORING_OP_ACCEPT:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5946) return io_accept_prep(req, sqe);
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 5947) case IORING_OP_FALLOCATE:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5948) return io_fallocate_prep(req, sqe);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 5949) case IORING_OP_OPENAT:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5950) return io_openat_prep(req, sqe);
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 5951) case IORING_OP_CLOSE:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5952) return io_close_prep(req, sqe);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 5953) case IORING_OP_FILES_UPDATE:
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 5954) return io_rsrc_update_prep(req, sqe);
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 5955) case IORING_OP_STATX:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5956) return io_statx_prep(req, sqe);
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 5957) case IORING_OP_FADVISE:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5958) return io_fadvise_prep(req, sqe);
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 5959) case IORING_OP_MADVISE:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5960) return io_madvise_prep(req, sqe);
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 5961) case IORING_OP_OPENAT2:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5962) return io_openat2_prep(req, sqe);
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 5963) case IORING_OP_EPOLL_CTL:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5964) return io_epoll_ctl_prep(req, sqe);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 5965) case IORING_OP_SPLICE:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5966) return io_splice_prep(req, sqe);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 5967) case IORING_OP_PROVIDE_BUFFERS:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5968) return io_provide_buffers_prep(req, sqe);
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 5969) case IORING_OP_REMOVE_BUFFERS:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5970) return io_remove_buffers_prep(req, sqe);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 5971) case IORING_OP_TEE:
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5972) return io_tee_prep(req, sqe);
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 5973) case IORING_OP_SHUTDOWN:
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 5974) return io_shutdown_prep(req, sqe);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 5975) case IORING_OP_RENAMEAT:
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 5976) return io_renameat_prep(req, sqe);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 5977) case IORING_OP_UNLINKAT:
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 5978) return io_unlinkat_prep(req, sqe);
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 5979) }
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 5980)
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5981) printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5982) req->opcode);
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 5983) return -EINVAL;
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5984) }
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5985)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 5986) static int io_req_prep_async(struct io_kiocb *req)
bfe76559833d5 (Pavel Begunkov 2020-09-30 22:57:55 +0300 5987) {
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 5988) if (!io_op_defs[req->opcode].needs_async_setup)
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 5989) return 0;
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 5990) if (WARN_ON_ONCE(req->async_data))
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 5991) return -EFAULT;
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 5992) if (io_alloc_async_data(req))
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 5993) return -EAGAIN;
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 5994)
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 5995) switch (req->opcode) {
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 5996) case IORING_OP_READV:
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 5997) return io_rw_prep_async(req, READ);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 5998) case IORING_OP_WRITEV:
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 5999) return io_rw_prep_async(req, WRITE);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 6000) case IORING_OP_SENDMSG:
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 6001) return io_sendmsg_prep_async(req);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 6002) case IORING_OP_RECVMSG:
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 6003) return io_recvmsg_prep_async(req);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 6004) case IORING_OP_CONNECT:
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 6005) return io_connect_prep_async(req);
93642ef884344 (Pavel Begunkov 2021-02-18 18:29:44 +0000 6006) }
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 6007) printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 6008) req->opcode);
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 6009) return -EFAULT;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 6010) }
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 6011)
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6012) static u32 io_get_sequence(struct io_kiocb *req)
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6013) {
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6014) struct io_kiocb *pos;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6015) struct io_ring_ctx *ctx = req->ctx;
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6016) u32 total_submitted, nr_reqs = 0;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6017)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6018) io_for_each_link(pos, req)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6019) nr_reqs++;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6020)
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6021) total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6022) return total_submitted - nr_reqs;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6023) }
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6024)
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6025) static int io_req_defer(struct io_kiocb *req)
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6026) {
a197f664a0db8 (Jackie Liu 2019-11-08 08:09:12 -0700 6027) struct io_ring_ctx *ctx = req->ctx;
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 6028) struct io_defer_entry *de;
f67676d160c6e (Jens Axboe 2019-12-02 11:03:47 -0700 6029) int ret;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6030) u32 seq;
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6031)
9d858b2148398 (Bob Liu 2019-11-13 18:06:25 +0800 6032) /* Still need defer if there is pending req in defer list. */
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6033) if (likely(list_empty_careful(&ctx->defer_list) &&
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6034) !(req->flags & REQ_F_IO_DRAIN)))
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6035) return 0;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6036)
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6037) seq = io_get_sequence(req);
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6038) /* Still a chance to pass the sequence check */
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6039) if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6040) return 0;
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6041)
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 6042) ret = io_req_prep_async(req);
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6043) if (ret)
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6044) return ret;
cbdcb4357c000 (Pavel Begunkov 2020-06-29 19:18:43 +0300 6045) io_prep_async_link(req);
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 6046) de = kmalloc(sizeof(*de), GFP_KERNEL);
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 6047) if (!de)
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 6048) return -ENOMEM;
2d28390aff879 (Jens Axboe 2019-12-04 11:08:05 -0700 6049)
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6050) spin_lock_irq(&ctx->completion_lock);
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6051) if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6052) spin_unlock_irq(&ctx->completion_lock);
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 6053) kfree(de);
ae34817bd93e3 (Pavel Begunkov 2020-07-23 20:25:20 +0300 6054) io_queue_async_work(req);
ae34817bd93e3 (Pavel Begunkov 2020-07-23 20:25:20 +0300 6055) return -EIOCBQUEUED;
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6056) }
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6057)
915967f69c591 (Jens Axboe 2019-11-21 09:01:20 -0700 6058) trace_io_uring_defer(ctx, req, req->user_data);
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 6059) de->req = req;
9cf7c104deaef (Pavel Begunkov 2020-07-13 23:37:15 +0300 6060) de->seq = seq;
27dc8338e5fb0 (Pavel Begunkov 2020-07-13 23:37:14 +0300 6061) list_add_tail(&de->list, &ctx->defer_list);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6062) spin_unlock_irq(&ctx->completion_lock);
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6063) return -EIOCBQUEUED;
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6064) }
de0617e467171 (Jens Axboe 2019-04-06 21:51:27 -0600 6065)
68fb897966feb (Pavel Begunkov 2021-03-19 17:22:41 +0000 6066) static void io_clean_op(struct io_kiocb *req)
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 6067) {
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6068) if (req->flags & REQ_F_BUFFER_SELECTED) {
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6069) switch (req->opcode) {
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6070) case IORING_OP_READV:
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6071) case IORING_OP_READ_FIXED:
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6072) case IORING_OP_READ:
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 6073) kfree((void *)(unsigned long)req->rw.addr);
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6074) break;
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6075) case IORING_OP_RECVMSG:
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6076) case IORING_OP_RECV:
bcda7baaa3f15 (Jens Axboe 2020-02-23 16:42:51 -0700 6077) kfree(req->sr_msg.kbuf);
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6078) break;
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6079) }
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6080) req->flags &= ~REQ_F_BUFFER_SELECTED;
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 6081) }
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 6082)
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6083) if (req->flags & REQ_F_NEED_CLEANUP) {
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6084) switch (req->opcode) {
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6085) case IORING_OP_READV:
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6086) case IORING_OP_READ_FIXED:
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6087) case IORING_OP_READ:
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6088) case IORING_OP_WRITEV:
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6089) case IORING_OP_WRITE_FIXED:
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6090) case IORING_OP_WRITE: {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6091) struct io_async_rw *io = req->async_data;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6092) if (io->free_iovec)
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6093) kfree(io->free_iovec);
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6094) break;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6095) }
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6096) case IORING_OP_RECVMSG:
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6097) case IORING_OP_SENDMSG: {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6098) struct io_async_msghdr *io = req->async_data;
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 6099)
257e84a5377fb (Pavel Begunkov 2021-02-05 00:58:00 +0000 6100) kfree(io->free_iov);
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6101) break;
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6102) }
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6103) case IORING_OP_SPLICE:
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6104) case IORING_OP_TEE:
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 6105) if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
e1d767f078b88 (Pavel Begunkov 2021-03-19 17:22:43 +0000 6106) io_put_file(req->splice.file_in);
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6107) break;
f3cd4850504ff (Jens Axboe 2020-09-24 14:55:54 -0600 6108) case IORING_OP_OPENAT:
f3cd4850504ff (Jens Axboe 2020-09-24 14:55:54 -0600 6109) case IORING_OP_OPENAT2:
f3cd4850504ff (Jens Axboe 2020-09-24 14:55:54 -0600 6110) if (req->open.filename)
f3cd4850504ff (Jens Axboe 2020-09-24 14:55:54 -0600 6111) putname(req->open.filename);
f3cd4850504ff (Jens Axboe 2020-09-24 14:55:54 -0600 6112) break;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 6113) case IORING_OP_RENAMEAT:
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 6114) putname(req->rename.oldpath);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 6115) putname(req->rename.newpath);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 6116) break;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 6117) case IORING_OP_UNLINKAT:
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 6118) putname(req->unlink.filename);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 6119) break;
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6120) }
0e1b6fe3d1e5f (Pavel Begunkov 2020-07-16 23:28:02 +0300 6121) req->flags &= ~REQ_F_NEED_CLEANUP;
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 6122) }
75652a30ff675 (Jens Axboe 2021-04-15 09:52:40 -0600 6123) if ((req->flags & REQ_F_POLLED) && req->apoll) {
75652a30ff675 (Jens Axboe 2021-04-15 09:52:40 -0600 6124) kfree(req->apoll->double_poll);
75652a30ff675 (Jens Axboe 2021-04-15 09:52:40 -0600 6125) kfree(req->apoll);
75652a30ff675 (Jens Axboe 2021-04-15 09:52:40 -0600 6126) req->apoll = NULL;
75652a30ff675 (Jens Axboe 2021-04-15 09:52:40 -0600 6127) }
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 6128) if (req->flags & REQ_F_INFLIGHT) {
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 6129) struct io_uring_task *tctx = req->task->io_uring;
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 6130)
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 6131) atomic_dec(&tctx->inflight_tracked);
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 6132) req->flags &= ~REQ_F_INFLIGHT;
3a0a690235923 (Pavel Begunkov 2021-04-20 12:03:31 +0100 6133) }
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 6134) }
99bc4c38537d7 (Pavel Begunkov 2020-02-07 22:04:45 +0300 6135)
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6136) static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6137) {
a197f664a0db8 (Jackie Liu 2019-11-08 08:09:12 -0700 6138) struct io_ring_ctx *ctx = req->ctx;
5730b27e84fdb (Jens Axboe 2021-02-27 15:57:30 -0700 6139) const struct cred *creds = NULL;
d625c6ee49750 (Jens Axboe 2019-12-17 19:53:05 -0700 6140) int ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6141)
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 6142) if (req->creds && req->creds != current_cred())
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 6143) creds = override_creds(req->creds);
5730b27e84fdb (Jens Axboe 2021-02-27 15:57:30 -0700 6144)
d625c6ee49750 (Jens Axboe 2019-12-17 19:53:05 -0700 6145) switch (req->opcode) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6146) case IORING_OP_NOP:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6147) ret = io_nop(req, issue_flags);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6148) break;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6149) case IORING_OP_READV:
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 6150) case IORING_OP_READ_FIXED:
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 6151) case IORING_OP_READ:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6152) ret = io_read(req, issue_flags);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 6153) break;
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 6154) case IORING_OP_WRITEV:
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 6155) case IORING_OP_WRITE_FIXED:
3a6820f2bb8a0 (Jens Axboe 2019-12-22 15:19:35 -0700 6156) case IORING_OP_WRITE:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6157) ret = io_write(req, issue_flags);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6158) break;
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 6159) case IORING_OP_FSYNC:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6160) ret = io_fsync(req, issue_flags);
c992fe2925d77 (Christoph Hellwig 2019-01-11 09:43:02 -0700 6161) break;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 6162) case IORING_OP_POLL_ADD:
61e9820304798 (Pavel Begunkov 2021-02-10 00:03:08 +0000 6163) ret = io_poll_add(req, issue_flags);
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 6164) break;
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 6165) case IORING_OP_POLL_REMOVE:
c5de00366e3e6 (Pavel Begunkov 2021-04-14 13:38:37 +0100 6166) ret = io_poll_update(req, issue_flags);
221c5eb233823 (Jens Axboe 2019-01-17 09:41:58 -0700 6167) break;
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 6168) case IORING_OP_SYNC_FILE_RANGE:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6169) ret = io_sync_file_range(req, issue_flags);
5d17b4a4b7fa1 (Jens Axboe 2019-04-09 14:56:44 -0600 6170) break;
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 6171) case IORING_OP_SENDMSG:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6172) ret = io_sendmsg(req, issue_flags);
062d04d73168d (Pavel Begunkov 2020-10-10 18:34:12 +0100 6173) break;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 6174) case IORING_OP_SEND:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6175) ret = io_send(req, issue_flags);
0fa03c624d8fc (Jens Axboe 2019-04-19 13:34:07 -0600 6176) break;
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 6177) case IORING_OP_RECVMSG:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6178) ret = io_recvmsg(req, issue_flags);
062d04d73168d (Pavel Begunkov 2020-10-10 18:34:12 +0100 6179) break;
fddafacee287b (Jens Axboe 2020-01-04 20:19:44 -0700 6180) case IORING_OP_RECV:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6181) ret = io_recv(req, issue_flags);
aa1fa28fc73ea (Jens Axboe 2019-04-19 13:38:09 -0600 6182) break;
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 6183) case IORING_OP_TIMEOUT:
61e9820304798 (Pavel Begunkov 2021-02-10 00:03:08 +0000 6184) ret = io_timeout(req, issue_flags);
5262f567987d3 (Jens Axboe 2019-09-17 12:26:57 -0600 6185) break;
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 6186) case IORING_OP_TIMEOUT_REMOVE:
61e9820304798 (Pavel Begunkov 2021-02-10 00:03:08 +0000 6187) ret = io_timeout_remove(req, issue_flags);
11365043e5271 (Jens Axboe 2019-10-16 09:08:32 -0600 6188) break;
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 6189) case IORING_OP_ACCEPT:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6190) ret = io_accept(req, issue_flags);
17f2fe35d080d (Jens Axboe 2019-10-17 14:42:58 -0600 6191) break;
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 6192) case IORING_OP_CONNECT:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6193) ret = io_connect(req, issue_flags);
f8e85cf255ad5 (Jens Axboe 2019-11-23 14:24:24 -0700 6194) break;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 6195) case IORING_OP_ASYNC_CANCEL:
61e9820304798 (Pavel Begunkov 2021-02-10 00:03:08 +0000 6196) ret = io_async_cancel(req, issue_flags);
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 6197) break;
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 6198) case IORING_OP_FALLOCATE:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6199) ret = io_fallocate(req, issue_flags);
d63d1b5edb7b8 (Jens Axboe 2019-12-10 10:38:56 -0700 6200) break;
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 6201) case IORING_OP_OPENAT:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6202) ret = io_openat(req, issue_flags);
15b71abe7b52d (Jens Axboe 2019-12-11 11:20:36 -0700 6203) break;
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 6204) case IORING_OP_CLOSE:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6205) ret = io_close(req, issue_flags);
b5dba59e0cf7e (Jens Axboe 2019-12-11 14:02:38 -0700 6206) break;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 6207) case IORING_OP_FILES_UPDATE:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6208) ret = io_files_update(req, issue_flags);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 6209) break;
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 6210) case IORING_OP_STATX:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6211) ret = io_statx(req, issue_flags);
eddc7ef52a6b3 (Jens Axboe 2019-12-13 21:18:10 -0700 6212) break;
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 6213) case IORING_OP_FADVISE:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6214) ret = io_fadvise(req, issue_flags);
4840e418c2fc5 (Jens Axboe 2019-12-25 22:03:45 -0700 6215) break;
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 6216) case IORING_OP_MADVISE:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6217) ret = io_madvise(req, issue_flags);
c1ca757bd6f46 (Jens Axboe 2019-12-25 22:18:28 -0700 6218) break;
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 6219) case IORING_OP_OPENAT2:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6220) ret = io_openat2(req, issue_flags);
cebdb98617ae3 (Jens Axboe 2020-01-08 17:59:24 -0700 6221) break;
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 6222) case IORING_OP_EPOLL_CTL:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6223) ret = io_epoll_ctl(req, issue_flags);
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 6224) break;
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 6225) case IORING_OP_SPLICE:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6226) ret = io_splice(req, issue_flags);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 6227) break;
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 6228) case IORING_OP_PROVIDE_BUFFERS:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6229) ret = io_provide_buffers(req, issue_flags);
ddf0322db79c5 (Jens Axboe 2020-02-23 16:41:33 -0700 6230) break;
067524e914cb2 (Jens Axboe 2020-03-02 16:32:28 -0700 6231) case IORING_OP_REMOVE_BUFFERS:
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6232) ret = io_remove_buffers(req, issue_flags);
3e4827b05d2ac (Jens Axboe 2020-01-08 15:18:09 -0700 6233) break;
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 6234) case IORING_OP_TEE:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6235) ret = io_tee(req, issue_flags);
f2a8d5c7a218b (Pavel Begunkov 2020-05-17 14:18:06 +0300 6236) break;
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 6237) case IORING_OP_SHUTDOWN:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6238) ret = io_shutdown(req, issue_flags);
36f4fa6886a81 (Jens Axboe 2020-09-05 11:14:22 -0600 6239) break;
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 6240) case IORING_OP_RENAMEAT:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6241) ret = io_renameat(req, issue_flags);
80a261fd00327 (Jens Axboe 2020-09-28 14:23:58 -0600 6242) break;
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 6243) case IORING_OP_UNLINKAT:
45d189c606292 (Pavel Begunkov 2021-02-10 00:03:07 +0000 6244) ret = io_unlinkat(req, issue_flags);
14a1143b68ee2 (Jens Axboe 2020-09-28 14:27:37 -0600 6245) break;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6246) default:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6247) ret = -EINVAL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6248) break;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6249) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6250)
5730b27e84fdb (Jens Axboe 2021-02-27 15:57:30 -0700 6251) if (creds)
5730b27e84fdb (Jens Axboe 2021-02-27 15:57:30 -0700 6252) revert_creds(creds);
5730b27e84fdb (Jens Axboe 2021-02-27 15:57:30 -0700 6253)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 6254) if (ret)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 6255) return ret;
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 6256)
b532576ed39ef (Jens Axboe 2020-05-19 21:20:27 -0600 6257) /* If the op doesn't have a file, we're not polling for it */
b532576ed39ef (Jens Axboe 2020-05-19 21:20:27 -0600 6258) if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6259) const bool in_async = io_wq_current_is_worker();
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6260)
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6261) /* workqueue context doesn't hold uring_lock, grab it now */
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6262) if (in_async)
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6263) mutex_lock(&ctx->uring_lock);
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6264)
2e9dbe902d102 (Xiaoguang Wang 2020-11-13 00:44:08 +0800 6265) io_iopoll_req_issued(req, in_async);
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6266)
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6267) if (in_async)
11ba820bf163e (Jens Axboe 2020-01-15 21:51:17 -0700 6268) mutex_unlock(&ctx->uring_lock);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 6269) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 6270)
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 6271) return 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6272) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6273)
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 6274) static void io_wq_submit_work(struct io_wq_work *work)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6275) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6276) struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6df1db6b54243 (Pavel Begunkov 2020-07-03 22:15:06 +0300 6277) struct io_kiocb *timeout;
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6278) int ret = 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6279)
6df1db6b54243 (Pavel Begunkov 2020-07-03 22:15:06 +0300 6280) timeout = io_prep_linked_timeout(req);
6df1db6b54243 (Pavel Begunkov 2020-07-03 22:15:06 +0300 6281) if (timeout)
6df1db6b54243 (Pavel Begunkov 2020-07-03 22:15:06 +0300 6282) io_queue_linked_timeout(timeout);
d4c81f38522f3 (Pavel Begunkov 2020-06-08 21:08:19 +0300 6283)
4014d943cb62d (Jens Axboe 2021-01-19 15:53:54 -0700 6284) if (work->flags & IO_WQ_WORK_CANCEL)
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6285) ret = -ECANCELED;
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 6286)
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6287) if (!ret) {
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6288) do {
889fca73287b0 (Pavel Begunkov 2021-02-10 00:03:09 +0000 6289) ret = io_issue_sqe(req, 0);
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6290) /*
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6291) * We can get EAGAIN for polled IO even though we're
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6292) * forcing a sync submission from here, since we can't
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6293) * wait for request slots on the block side.
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6294) */
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6295) if (ret != -EAGAIN)
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6296) break;
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6297) cond_resched();
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6298) } while (1);
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6299) }
31b515106428b (Jens Axboe 2019-01-18 22:56:34 -0700 6300)
a3df769899c0b (Pavel Begunkov 2021-02-18 22:32:52 +0000 6301) /* avoid locking problems by failing it from a clean context */
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 6302) if (ret) {
a3df769899c0b (Pavel Begunkov 2021-02-18 22:32:52 +0000 6303) /* io-wq is going to take one down */
de9b4ccad750f (Jens Axboe 2021-02-24 13:28:27 -0700 6304) req_ref_get(req);
a3df769899c0b (Pavel Begunkov 2021-02-18 22:32:52 +0000 6305) io_req_task_queue_fail(req, ret);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 6306) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6307) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6308)
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6309) #define FFS_ASYNC_READ 0x1UL
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6310) #define FFS_ASYNC_WRITE 0x2UL
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6311) #ifdef CONFIG_64BIT
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6312) #define FFS_ISREG 0x4UL
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6313) #else
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6314) #define FFS_ISREG 0x0UL
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6315) #endif
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6316) #define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6317)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 6318) static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 6319) unsigned i)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6320) {
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 6321) struct io_fixed_file *table_l2;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6322)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 6323) table_l2 = table->files[i >> IORING_FILE_TABLE_SHIFT];
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 6324) return &table_l2[i & IORING_FILE_TABLE_MASK];
dafecf19e25f9 (Pavel Begunkov 2021-02-28 22:35:11 +0000 6325) }
dafecf19e25f9 (Pavel Begunkov 2021-02-28 22:35:11 +0000 6326)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6327) static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6328) int index)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6329) {
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 6330) struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index);
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6331)
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 6332) return (struct file *) (slot->file_ptr & FFS_MASK);
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6333) }
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6334)
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 6335) static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file)
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6336) {
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6337) unsigned long file_ptr = (unsigned long) file;
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6338)
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6339) if (__io_file_supports_async(file, READ))
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6340) file_ptr |= FFS_ASYNC_READ;
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6341) if (__io_file_supports_async(file, WRITE))
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6342) file_ptr |= FFS_ASYNC_WRITE;
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6343) if (S_ISREG(file_inode(file)->i_mode))
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 6344) file_ptr |= FFS_ISREG;
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 6345) file_slot->file_ptr = file_ptr;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6346) }
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 6347)
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 6348) static struct file *io_file_get(struct io_submit_state *state,
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 6349) struct io_kiocb *req, int fd, bool fixed)
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 6350) {
a197f664a0db8 (Jackie Liu 2019-11-08 08:09:12 -0700 6351) struct io_ring_ctx *ctx = req->ctx;
8da11c19940dd (Pavel Begunkov 2020-02-24 11:32:44 +0300 6352) struct file *file;
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 6353)
8da11c19940dd (Pavel Begunkov 2020-02-24 11:32:44 +0300 6354) if (fixed) {
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6355) unsigned long file_ptr;
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6356)
479f517be5718 (Pavel Begunkov 2020-10-10 18:34:07 +0100 6357) if (unlikely((unsigned int)fd >= ctx->nr_user_files))
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 6358) return NULL;
b7620121dc04e (Jens Axboe 2019-10-26 07:22:55 -0600 6359) fd = array_index_nospec(fd, ctx->nr_user_files);
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 6360) file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6361) file = (struct file *) (file_ptr & FFS_MASK);
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6362) file_ptr &= ~FFS_MASK;
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6363) /* mask in overlapping REQ_F and FFS bits */
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 6364) req->flags |= (file_ptr << REQ_F_ASYNC_READ_BIT);
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 6365) io_req_set_rsrc_node(req);
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 6366) } else {
c826bd7a743f2 (Dmitrii Dolgov 2019-10-15 19:02:01 +0200 6367) trace_io_uring_file_get(ctx, fd);
8da11c19940dd (Pavel Begunkov 2020-02-24 11:32:44 +0300 6368) file = __io_file_get(state, fd);
d44f554e105b0 (Jens Axboe 2021-03-12 08:27:05 -0700 6369)
d44f554e105b0 (Jens Axboe 2021-03-12 08:27:05 -0700 6370) /* we don't allow fixed io_uring files */
d44f554e105b0 (Jens Axboe 2021-03-12 08:27:05 -0700 6371) if (file && unlikely(file->f_op == &io_uring_fops))
d44f554e105b0 (Jens Axboe 2021-03-12 08:27:05 -0700 6372) io_req_track_inflight(req);
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 6373) }
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 6374)
8371adf53c3c5 (Pavel Begunkov 2020-10-10 18:34:08 +0100 6375) return file;
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 6376) }
09bb839434bd8 (Jens Axboe 2019-03-13 12:39:28 -0600 6377)
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6378) static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6379) {
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 6380) struct io_timeout_data *data = container_of(timer,
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 6381) struct io_timeout_data, timer);
90cd7e424969d (Pavel Begunkov 2020-10-27 23:25:36 +0000 6382) struct io_kiocb *prev, *req = data->req;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6383) struct io_ring_ctx *ctx = req->ctx;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6384) unsigned long flags;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6385)
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6386) spin_lock_irqsave(&ctx->completion_lock, flags);
90cd7e424969d (Pavel Begunkov 2020-10-27 23:25:36 +0000 6387) prev = req->timeout.head;
90cd7e424969d (Pavel Begunkov 2020-10-27 23:25:36 +0000 6388) req->timeout.head = NULL;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6389)
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6390) /*
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6391) * We don't expect the list to be empty, that will only happen if we
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6392) * race with the completion of the linked work.
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6393) */
447c19f3b5074 (Pavel Begunkov 2021-05-14 12:02:50 +0100 6394) if (prev) {
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6395) io_remove_next_linked(prev);
447c19f3b5074 (Pavel Begunkov 2021-05-14 12:02:50 +0100 6396) if (!req_ref_inc_not_zero(prev))
447c19f3b5074 (Pavel Begunkov 2021-05-14 12:02:50 +0100 6397) prev = NULL;
447c19f3b5074 (Pavel Begunkov 2021-05-14 12:02:50 +0100 6398) }
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6399) spin_unlock_irqrestore(&ctx->completion_lock, flags);
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6400)
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6401) if (prev) {
014db0073cc6a (Pavel Begunkov 2020-03-03 21:33:12 +0300 6402) io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
9ae1f8dd372e0 (Pavel Begunkov 2021-02-01 18:59:51 +0000 6403) io_put_req_deferred(prev, 1);
a298232ee6b9a (Pavel Begunkov 2021-05-07 21:06:38 +0100 6404) io_put_req_deferred(req, 1);
47f467686ec02 (Jens Axboe 2019-11-09 17:43:02 -0700 6405) } else {
9ae1f8dd372e0 (Pavel Begunkov 2021-02-01 18:59:51 +0000 6406) io_req_complete_post(req, -ETIME, 0);
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6407) }
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6408) return HRTIMER_NORESTART;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6409) }
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6410)
de968c182b4f4 (Pavel Begunkov 2021-03-19 17:22:33 +0000 6411) static void io_queue_linked_timeout(struct io_kiocb *req)
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6412) {
de968c182b4f4 (Pavel Begunkov 2021-03-19 17:22:33 +0000 6413) struct io_ring_ctx *ctx = req->ctx;
de968c182b4f4 (Pavel Begunkov 2021-03-19 17:22:33 +0000 6414)
de968c182b4f4 (Pavel Begunkov 2021-03-19 17:22:33 +0000 6415) spin_lock_irq(&ctx->completion_lock);
76a46e066e2d9 (Jens Axboe 2019-11-10 23:34:16 -0700 6416) /*
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6417) * If the back reference is NULL, then our linked request finished
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6418) * before we got a chance to setup the timer
76a46e066e2d9 (Jens Axboe 2019-11-10 23:34:16 -0700 6419) */
90cd7e424969d (Pavel Begunkov 2020-10-27 23:25:36 +0000 6420) if (req->timeout.head) {
e8c2bc1fb6c94 (Jens Axboe 2020-08-15 18:44:09 -0700 6421) struct io_timeout_data *data = req->async_data;
94ae5e77a9150 (Jens Axboe 2019-11-14 19:39:52 -0700 6422)
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 6423) data->timer.function = io_link_timeout_fn;
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 6424) hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 6425) data->mode);
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6426) }
76a46e066e2d9 (Jens Axboe 2019-11-10 23:34:16 -0700 6427) spin_unlock_irq(&ctx->completion_lock);
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6428) /* drop submission reference */
76a46e066e2d9 (Jens Axboe 2019-11-10 23:34:16 -0700 6429) io_put_req(req);
76a46e066e2d9 (Jens Axboe 2019-11-10 23:34:16 -0700 6430) }
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6431)
ad8a48acc23cb (Jens Axboe 2019-11-15 08:49:11 -0700 6432) static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6433) {
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6434) struct io_kiocb *nxt = req->link;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6435)
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6436) if (!nxt || (req->flags & REQ_F_LINK_TIMEOUT) ||
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6437) nxt->opcode != IORING_OP_LINK_TIMEOUT)
76a46e066e2d9 (Jens Axboe 2019-11-10 23:34:16 -0700 6438) return NULL;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6439)
90cd7e424969d (Pavel Begunkov 2020-10-27 23:25:36 +0000 6440) nxt->timeout.head = req;
900fad45dc75c (Pavel Begunkov 2020-10-19 16:39:16 +0100 6441) nxt->flags |= REQ_F_LTIMEOUT_ACTIVE;
76a46e066e2d9 (Jens Axboe 2019-11-10 23:34:16 -0700 6442) req->flags |= REQ_F_LINK_TIMEOUT;
76a46e066e2d9 (Jens Axboe 2019-11-10 23:34:16 -0700 6443) return nxt;
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6444) }
2665abfd757fb (Jens Axboe 2019-11-05 12:40:47 -0700 6445)
c5eef2b9449ba (Pavel Begunkov 2021-02-10 00:03:22 +0000 6446) static void __io_queue_sqe(struct io_kiocb *req)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6447) {
d3d7298d05cb0 (Pavel Begunkov 2021-02-12 03:23:51 +0000 6448) struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
e0c5c576d5074 (Jens Axboe 2019-03-12 10:18:47 -0600 6449) int ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6450)
c5eef2b9449ba (Pavel Begunkov 2021-02-10 00:03:22 +0000 6451) ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
193155c8c9429 (Jens Axboe 2020-02-22 23:22:19 -0700 6452)
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 6453) /*
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 6454) * We async punt it if the file wasn't marked NOWAIT, or if the file
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 6455) * doesn't support non-blocking read/write attempts
491381ce07ca5 (Jens Axboe 2019-10-17 09:20:46 -0600 6456) */
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6457) if (likely(!ret)) {
0d63c148d6d9a (Pavel Begunkov 2020-10-22 16:47:18 +0100 6458) /* drop submission reference */
e342c807f556d (Pavel Begunkov 2021-01-19 13:32:47 +0000 6459) if (req->flags & REQ_F_COMPLETE_INLINE) {
c5eef2b9449ba (Pavel Begunkov 2021-02-10 00:03:22 +0000 6460) struct io_ring_ctx *ctx = req->ctx;
c5eef2b9449ba (Pavel Begunkov 2021-02-10 00:03:22 +0000 6461) struct io_comp_state *cs = &ctx->submit_state.comp;
e65ef56db4945 (Jens Axboe 2019-03-12 10:16:44 -0600 6462)
6dd0be1e2481b (Pavel Begunkov 2021-02-10 00:03:13 +0000 6463) cs->reqs[cs->nr++] = req;
d3d7298d05cb0 (Pavel Begunkov 2021-02-12 03:23:51 +0000 6464) if (cs->nr == ARRAY_SIZE(cs->reqs))
c5eef2b9449ba (Pavel Begunkov 2021-02-10 00:03:22 +0000 6465) io_submit_flush_completions(cs, ctx);
9affd664f0e05 (Pavel Begunkov 2021-01-19 13:32:46 +0000 6466) } else {
d3d7298d05cb0 (Pavel Begunkov 2021-02-12 03:23:51 +0000 6467) io_put_req(req);
0d63c148d6d9a (Pavel Begunkov 2020-10-22 16:47:18 +0100 6468) }
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6469) } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6470) if (!io_arm_poll_handler(req)) {
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6471) /*
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6472) * Queued up for async execution, worker will release
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6473) * submit reference when the iocb is actually submitted.
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6474) */
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6475) io_queue_async_work(req);
1840038e11957 (Pavel Begunkov 2021-03-19 17:22:34 +0000 6476) }
0d63c148d6d9a (Pavel Begunkov 2020-10-22 16:47:18 +0100 6477) } else {
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 6478) io_req_complete_failed(req, ret);
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6479) }
d3d7298d05cb0 (Pavel Begunkov 2021-02-12 03:23:51 +0000 6480) if (linked_timeout)
d3d7298d05cb0 (Pavel Begunkov 2021-02-12 03:23:51 +0000 6481) io_queue_linked_timeout(linked_timeout);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6482) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6483)
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6484) static void io_queue_sqe(struct io_kiocb *req)
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6485) {
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6486) int ret;
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6487)
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6488) ret = io_req_defer(req);
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6489) if (ret) {
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6490) if (ret != -EIOCBQUEUED) {
1118591ab883f (Pavel Begunkov 2020-01-22 23:09:35 +0300 6491) fail_req:
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 6492) io_req_complete_failed(req, ret);
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6493) }
2550878f8421f (Pavel Begunkov 2019-12-30 21:24:47 +0300 6494) } else if (req->flags & REQ_F_FORCE_ASYNC) {
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 6495) ret = io_req_prep_async(req);
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6496) if (unlikely(ret))
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6497) goto fail_req;
ce35a47a3a020 (Jens Axboe 2019-12-17 08:04:44 -0700 6498) io_queue_async_work(req);
ce35a47a3a020 (Jens Axboe 2019-12-17 08:04:44 -0700 6499) } else {
c5eef2b9449ba (Pavel Begunkov 2021-02-10 00:03:22 +0000 6500) __io_queue_sqe(req);
ce35a47a3a020 (Jens Axboe 2019-12-17 08:04:44 -0700 6501) }
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6502) }
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6503)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6504) /*
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6505) * Check SQE restrictions (opcode and flags).
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6506) *
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6507) * Returns 'true' if SQE is allowed, 'false' otherwise.
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6508) */
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6509) static inline bool io_check_restriction(struct io_ring_ctx *ctx,
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6510) struct io_kiocb *req,
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6511) unsigned int sqe_flags)
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6512) {
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6513) if (!ctx->restricted)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6514) return true;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6515)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6516) if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6517) return false;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6518)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6519) if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6520) ctx->restrictions.sqe_flags_required)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6521) return false;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6522)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6523) if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6524) ctx->restrictions.sqe_flags_required))
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6525) return false;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6526)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6527) return true;
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6528) }
4fe2c963154c3 (Jackie Liu 2019-09-09 20:50:40 +0800 6529)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6530) static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6531) const struct io_uring_sqe *sqe)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6532) {
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6533) struct io_submit_state *state;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6534) unsigned int sqe_flags;
003e8dccdb227 (Jens Axboe 2021-03-06 09:22:27 -0700 6535) int personality, ret = 0;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6536)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6537) req->opcode = READ_ONCE(sqe->opcode);
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6538) /* same numerical values with corresponding REQ_F_*, safe to copy */
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6539) req->flags = sqe_flags = READ_ONCE(sqe->flags);
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6540) req->user_data = READ_ONCE(sqe->user_data);
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6541) req->async_data = NULL;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6542) req->file = NULL;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6543) req->ctx = ctx;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6544) req->link = NULL;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6545) req->fixed_rsrc_refs = NULL;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6546) /* one is dropped after submission, the other at completion */
abc54d634334f (Jens Axboe 2021-02-24 13:32:30 -0700 6547) atomic_set(&req->refs, 2);
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6548) req->task = current;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6549) req->result = 0;
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 6550) req->creds = NULL;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6551)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6552) /* enforce forwards compatibility on users */
dddca22636c90 (Pavel Begunkov 2021-04-27 16:13:52 +0100 6553) if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6554) return -EINVAL;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6555) if (unlikely(req->opcode >= IORING_OP_LAST))
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6556) return -EINVAL;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6557) if (unlikely(!io_check_restriction(ctx, req, sqe_flags)))
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6558) return -EACCES;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6559)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6560) if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6561) !io_op_defs[req->opcode].buffer_select)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6562) return -EOPNOTSUPP;
863e05604a6fb (Pavel Begunkov 2020-10-27 23:25:35 +0000 6563)
003e8dccdb227 (Jens Axboe 2021-03-06 09:22:27 -0700 6564) personality = READ_ONCE(sqe->personality);
003e8dccdb227 (Jens Axboe 2021-03-06 09:22:27 -0700 6565) if (personality) {
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 6566) req->creds = xa_load(&ctx->personalities, personality);
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 6567) if (!req->creds)
003e8dccdb227 (Jens Axboe 2021-03-06 09:22:27 -0700 6568) return -EINVAL;
7511a4f524f3d (Pavel Begunkov 2021-06-17 18:14:01 +0100 6569) get_cred(req->creds);
003e8dccdb227 (Jens Axboe 2021-03-06 09:22:27 -0700 6570) }
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6571) state = &ctx->submit_state;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6572)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6573) /*
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6574) * Plug now if we have more than 1 IO left after this, and the target
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6575) * is potentially a read/write to block based storage.
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6576) */
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6577) if (!state->plug_started && state->ios_left > 1 &&
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6578) io_op_defs[req->opcode].plug) {
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6579) blk_start_plug(&state->plug);
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6580) state->plug_started = true;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6581) }
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6582)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6583) if (io_op_defs[req->opcode].needs_file) {
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6584) bool fixed = req->flags & REQ_F_FIXED_FILE;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6585)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6586) req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed);
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6587) if (unlikely(!req->file))
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6588) ret = -EBADF;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6589) }
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6590)
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6591) state->ios_left--;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6592) return ret;
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6593) }
b16fed66bc7dc (Pavel Begunkov 2021-02-18 18:29:40 +0000 6594)
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6595) static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 6596) const struct io_uring_sqe *sqe)
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6597) {
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 6598) struct io_submit_link *link = &ctx->submit_state.link;
ef4ff581102a9 (Pavel Begunkov 2020-04-12 02:05:05 +0300 6599) int ret;
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6600)
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6601) ret = io_init_req(ctx, req, sqe);
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6602) if (unlikely(ret)) {
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6603) fail_req:
de59bc104c24f (Pavel Begunkov 2021-02-18 18:29:47 +0000 6604) if (link->head) {
de59bc104c24f (Pavel Begunkov 2021-02-18 18:29:47 +0000 6605) /* fail even hard links since we don't submit */
cf10960426515 (Pavel Begunkov 2021-02-18 18:29:43 +0000 6606) link->head->flags |= REQ_F_FAIL_LINK;
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 6607) io_req_complete_failed(link->head, -ECANCELED);
de59bc104c24f (Pavel Begunkov 2021-02-18 18:29:47 +0000 6608) link->head = NULL;
de59bc104c24f (Pavel Begunkov 2021-02-18 18:29:47 +0000 6609) }
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 6610) io_req_complete_failed(req, ret);
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6611) return ret;
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6612) }
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6613) ret = io_req_prep(req, sqe);
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6614) if (unlikely(ret))
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6615) goto fail_req;
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6616)
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6617) /* don't need @sqe from now on */
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6618) trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6619) true, ctx->flags & IORING_SETUP_SQPOLL);
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6620)
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6621) /*
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6622) * If we already have a head request, queue this one for async
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6623) * submittal once the head completes. If we don't have a head but
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6624) * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6625) * submitted sync once the chain is complete. If none of those
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6626) * conditions are true (normal request), then just queue it.
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6627) */
863e05604a6fb (Pavel Begunkov 2020-10-27 23:25:35 +0000 6628) if (link->head) {
863e05604a6fb (Pavel Begunkov 2020-10-27 23:25:35 +0000 6629) struct io_kiocb *head = link->head;
4e88d6e7793f2 (Jens Axboe 2019-12-07 20:59:47 -0700 6630)
8cdf2193a3335 (Pavel Begunkov 2020-01-25 00:40:24 +0300 6631) /*
8cdf2193a3335 (Pavel Begunkov 2020-01-25 00:40:24 +0300 6632) * Taking sequential execution of a link, draining both sides
8cdf2193a3335 (Pavel Begunkov 2020-01-25 00:40:24 +0300 6633) * of the link also fullfils IOSQE_IO_DRAIN semantics for all
8cdf2193a3335 (Pavel Begunkov 2020-01-25 00:40:24 +0300 6634) * requests in the link. So, it drains the head and the
8cdf2193a3335 (Pavel Begunkov 2020-01-25 00:40:24 +0300 6635) * next after the link request. The last one is done via
8cdf2193a3335 (Pavel Begunkov 2020-01-25 00:40:24 +0300 6636) * drain_next flag to persist the effect across calls.
8cdf2193a3335 (Pavel Begunkov 2020-01-25 00:40:24 +0300 6637) */
ef4ff581102a9 (Pavel Begunkov 2020-04-12 02:05:05 +0300 6638) if (req->flags & REQ_F_IO_DRAIN) {
711be0312df4d (Pavel Begunkov 2020-01-17 03:57:59 +0300 6639) head->flags |= REQ_F_IO_DRAIN;
711be0312df4d (Pavel Begunkov 2020-01-17 03:57:59 +0300 6640) ctx->drain_next = 1;
711be0312df4d (Pavel Begunkov 2020-01-17 03:57:59 +0300 6641) }
b7e298d265f20 (Pavel Begunkov 2021-02-28 22:35:19 +0000 6642) ret = io_req_prep_async(req);
cf10960426515 (Pavel Begunkov 2021-02-18 18:29:43 +0000 6643) if (unlikely(ret))
a6b8cadcea86d (Pavel Begunkov 2021-02-18 18:29:41 +0000 6644) goto fail_req;
9d76377f7e13c (Pavel Begunkov 2019-12-17 02:22:07 +0300 6645) trace_io_uring_link(ctx, req, head);
f2f87370bb666 (Pavel Begunkov 2020-10-27 23:25:37 +0000 6646) link->last->link = req;
863e05604a6fb (Pavel Begunkov 2020-10-27 23:25:35 +0000 6647) link->last = req;
32fe525b6d10f (Pavel Begunkov 2019-12-17 22:26:58 +0300 6648)
32fe525b6d10f (Pavel Begunkov 2019-12-17 22:26:58 +0300 6649) /* last request of a link, enqueue the link */
ef4ff581102a9 (Pavel Begunkov 2020-04-12 02:05:05 +0300 6650) if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
de59bc104c24f (Pavel Begunkov 2021-02-18 18:29:47 +0000 6651) io_queue_sqe(head);
863e05604a6fb (Pavel Begunkov 2020-10-27 23:25:35 +0000 6652) link->head = NULL;
32fe525b6d10f (Pavel Begunkov 2019-12-17 22:26:58 +0300 6653) }
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6654) } else {
711be0312df4d (Pavel Begunkov 2020-01-17 03:57:59 +0300 6655) if (unlikely(ctx->drain_next)) {
711be0312df4d (Pavel Begunkov 2020-01-17 03:57:59 +0300 6656) req->flags |= REQ_F_IO_DRAIN;
ef4ff581102a9 (Pavel Begunkov 2020-04-12 02:05:05 +0300 6657) ctx->drain_next = 0;
711be0312df4d (Pavel Begunkov 2020-01-17 03:57:59 +0300 6658) }
ef4ff581102a9 (Pavel Begunkov 2020-04-12 02:05:05 +0300 6659) if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
863e05604a6fb (Pavel Begunkov 2020-10-27 23:25:35 +0000 6660) link->head = req;
863e05604a6fb (Pavel Begunkov 2020-10-27 23:25:35 +0000 6661) link->last = req;
711be0312df4d (Pavel Begunkov 2020-01-17 03:57:59 +0300 6662) } else {
be7053b7d028d (Pavel Begunkov 2021-02-18 18:29:45 +0000 6663) io_queue_sqe(req);
711be0312df4d (Pavel Begunkov 2020-01-17 03:57:59 +0300 6664) }
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6665) }
2e6e1fde32d7d (Pavel Begunkov 2019-12-05 16:15:45 +0300 6666)
1d4240cc9e7bb (Pavel Begunkov 2020-04-12 02:05:03 +0300 6667) return 0;
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6668) }
9e645e1105ca6 (Jens Axboe 2019-05-10 16:07:28 -0600 6669)
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6670) /*
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6671) * Batched submission is done, ensure local IO is flushed out.
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6672) */
ba88ff112bdfd (Pavel Begunkov 2021-02-10 00:03:11 +0000 6673) static void io_submit_state_end(struct io_submit_state *state,
ba88ff112bdfd (Pavel Begunkov 2021-02-10 00:03:11 +0000 6674) struct io_ring_ctx *ctx)
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6675) {
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 6676) if (state->link.head)
de59bc104c24f (Pavel Begunkov 2021-02-18 18:29:47 +0000 6677) io_queue_sqe(state->link.head);
6dd0be1e2481b (Pavel Begunkov 2021-02-10 00:03:13 +0000 6678) if (state->comp.nr)
ba88ff112bdfd (Pavel Begunkov 2021-02-10 00:03:11 +0000 6679) io_submit_flush_completions(&state->comp, ctx);
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 6680) if (state->plug_started)
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 6681) blk_finish_plug(&state->plug);
9f13c35b33fdd (Pavel Begunkov 2020-05-17 14:13:41 +0300 6682) io_state_file_put(state);
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6683) }
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6684)
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6685) /*
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6686) * Start submission side cache.
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6687) */
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6688) static void io_submit_state_start(struct io_submit_state *state,
ba88ff112bdfd (Pavel Begunkov 2021-02-10 00:03:11 +0000 6689) unsigned int max_ios)
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6690) {
27926b683db03 (Jens Axboe 2020-10-28 09:33:23 -0600 6691) state->plug_started = false;
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6692) state->ios_left = max_ios;
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 6693) /* set only head, no need to init link_last in advance */
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 6694) state->link.head = NULL;
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6695) }
9a56a2323dbbd (Jens Axboe 2019-01-09 09:06:50 -0700 6696)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6697) static void io_commit_sqring(struct io_ring_ctx *ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6698) {
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 6699) struct io_rings *rings = ctx->rings;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6700)
caf582c652fec (Pavel Begunkov 2019-12-30 21:24:46 +0300 6701) /*
caf582c652fec (Pavel Begunkov 2019-12-30 21:24:46 +0300 6702) * Ensure any loads from the SQEs are done at this point,
caf582c652fec (Pavel Begunkov 2019-12-30 21:24:46 +0300 6703) * since once we write the new head, the application could
caf582c652fec (Pavel Begunkov 2019-12-30 21:24:46 +0300 6704) * write new data to them.
caf582c652fec (Pavel Begunkov 2019-12-30 21:24:46 +0300 6705) */
caf582c652fec (Pavel Begunkov 2019-12-30 21:24:46 +0300 6706) smp_store_release(&rings->sq.head, ctx->cached_sq_head);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6707) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6708)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6709) /*
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 6710) * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6711) * that is mapped by userspace. This means that care needs to be taken to
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6712) * ensure that reads are stable, as we cannot rely on userspace always
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6713) * being a good citizen. If members of the sqe are validated and then later
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6714) * used, it's important that those reads are done through READ_ONCE() to
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6715) * prevent a re-load down the line.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6716) */
709b302faddfa (Pavel Begunkov 2020-04-08 08:58:43 +0300 6717) static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6718) {
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 6719) u32 *sq_array = ctx->sq_array;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6720) unsigned head;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6721)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6722) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6723) * The cached sq head (or cq tail) serves two purposes:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6724) *
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6725) * 1) allows us to batch the cost of updating the user visible
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6726) * head updates.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6727) * 2) allows the kernel side to track the head on its own, even
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6728) * though the application is the one updating it.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6729) */
4fccfcbb73379 (Pavel Begunkov 2021-02-12 11:55:17 +0000 6730) head = READ_ONCE(sq_array[ctx->cached_sq_head++ & ctx->sq_mask]);
709b302faddfa (Pavel Begunkov 2020-04-08 08:58:43 +0300 6731) if (likely(head < ctx->sq_entries))
709b302faddfa (Pavel Begunkov 2020-04-08 08:58:43 +0300 6732) return &ctx->sq_sqes[head];
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6733)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 6734) /* drop invalid entries */
498ccd9eda491 (Jens Axboe 2019-10-25 10:04:25 -0600 6735) ctx->cached_sq_dropped++;
ee7d46d9db19d (Pavel Begunkov 2019-12-30 21:24:45 +0300 6736) WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
709b302faddfa (Pavel Begunkov 2020-04-08 08:58:43 +0300 6737) return NULL;
709b302faddfa (Pavel Begunkov 2020-04-08 08:58:43 +0300 6738) }
709b302faddfa (Pavel Begunkov 2020-04-08 08:58:43 +0300 6739)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 6740) static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6741) {
46c4e16a8625f (Pavel Begunkov 2021-02-18 18:29:37 +0000 6742) int submitted = 0;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6743)
ee7d46d9db19d (Pavel Begunkov 2019-12-30 21:24:45 +0300 6744) /* make sure SQ entry isn't read before tail */
ee7d46d9db19d (Pavel Begunkov 2019-12-30 21:24:45 +0300 6745) nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
9ef4f124894b7 (Pavel Begunkov 2019-12-30 21:24:44 +0300 6746)
2b85edfc0c90e (Pavel Begunkov 2019-12-28 14:13:03 +0300 6747) if (!percpu_ref_tryget_many(&ctx->refs, nr))
2b85edfc0c90e (Pavel Begunkov 2019-12-28 14:13:03 +0300 6748) return -EAGAIN;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6749)
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 6750) percpu_counter_add(¤t->io_uring->inflight, nr);
faf7b51c06973 (Jens Axboe 2020-10-07 12:48:53 -0600 6751) refcount_add(nr, ¤t->usage);
ba88ff112bdfd (Pavel Begunkov 2021-02-10 00:03:11 +0000 6752) io_submit_state_start(&ctx->submit_state, nr);
b14cca0c84c76 (Pavel Begunkov 2020-01-17 04:45:59 +0300 6753)
46c4e16a8625f (Pavel Begunkov 2021-02-18 18:29:37 +0000 6754) while (submitted < nr) {
3529d8c2b353e (Jens Axboe 2019-12-19 18:24:38 -0700 6755) const struct io_uring_sqe *sqe;
196be95cd5572 (Pavel Begunkov 2019-11-07 01:41:06 +0300 6756) struct io_kiocb *req;
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 6757)
258b29a93bfe7 (Pavel Begunkov 2021-02-10 00:03:10 +0000 6758) req = io_alloc_req(ctx);
196be95cd5572 (Pavel Begunkov 2019-11-07 01:41:06 +0300 6759) if (unlikely(!req)) {
196be95cd5572 (Pavel Begunkov 2019-11-07 01:41:06 +0300 6760) if (!submitted)
196be95cd5572 (Pavel Begunkov 2019-11-07 01:41:06 +0300 6761) submitted = -EAGAIN;
fb5ccc98782f6 (Pavel Begunkov 2019-10-25 12:31:30 +0300 6762) break;
196be95cd5572 (Pavel Begunkov 2019-11-07 01:41:06 +0300 6763) }
4fccfcbb73379 (Pavel Begunkov 2021-02-12 11:55:17 +0000 6764) sqe = io_get_sqe(ctx);
4fccfcbb73379 (Pavel Begunkov 2021-02-12 11:55:17 +0000 6765) if (unlikely(!sqe)) {
4fccfcbb73379 (Pavel Begunkov 2021-02-12 11:55:17 +0000 6766) kmem_cache_free(req_cachep, req);
4fccfcbb73379 (Pavel Begunkov 2021-02-12 11:55:17 +0000 6767) break;
4fccfcbb73379 (Pavel Begunkov 2021-02-12 11:55:17 +0000 6768) }
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 6769) /* will complete beyond this point, count as submitted */
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 6770) submitted++;
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 6771) if (io_submit_sqe(ctx, req, sqe))
196be95cd5572 (Pavel Begunkov 2019-11-07 01:41:06 +0300 6772) break;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6773) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6774)
9466f43741bc0 (Pavel Begunkov 2020-01-25 22:34:01 +0300 6775) if (unlikely(submitted != nr)) {
9466f43741bc0 (Pavel Begunkov 2020-01-25 22:34:01 +0300 6776) int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 6777) struct io_uring_task *tctx = current->io_uring;
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 6778) int unused = nr - ref_used;
9466f43741bc0 (Pavel Begunkov 2020-01-25 22:34:01 +0300 6779)
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 6780) percpu_ref_put_many(&ctx->refs, unused);
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 6781) percpu_counter_sub(&tctx->inflight, unused);
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 6782) put_task_struct_many(current, unused);
9466f43741bc0 (Pavel Begunkov 2020-01-25 22:34:01 +0300 6783) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6784)
a1ab7b35db8f2 (Pavel Begunkov 2021-02-18 18:29:42 +0000 6785) io_submit_state_end(&ctx->submit_state, ctx);
ae9428ca61271 (Pavel Begunkov 2019-11-06 00:22:14 +0300 6786) /* Commit SQ ring head once we've consumed and submitted all SQEs */
ae9428ca61271 (Pavel Begunkov 2019-11-06 00:22:14 +0300 6787) io_commit_sqring(ctx);
ae9428ca61271 (Pavel Begunkov 2019-11-06 00:22:14 +0300 6788)
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6789) return submitted;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6790) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6791)
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6792) static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6793) {
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6794) /* Tell userspace we may need a wakeup call */
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6795) spin_lock_irq(&ctx->completion_lock);
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 6796) WRITE_ONCE(ctx->rings->sq_flags,
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 6797) ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP);
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6798) spin_unlock_irq(&ctx->completion_lock);
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6799) }
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6800)
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6801) static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6802) {
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6803) spin_lock_irq(&ctx->completion_lock);
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 6804) WRITE_ONCE(ctx->rings->sq_flags,
261d195d5fe6c (Nadav Amit 2021-08-07 17:13:42 -0700 6805) ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP);
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6806) spin_unlock_irq(&ctx->completion_lock);
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6807) }
23b3628e45924 (Xiaoguang Wang 2020-07-23 20:57:24 +0800 6808)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6809) static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6810) {
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6811) unsigned int to_submit;
bdcd3eab2a9ae (Xiaoguang Wang 2020-02-25 22:12:08 +0800 6812) int ret = 0;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6813)
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6814) to_submit = io_sqring_entries(ctx);
e95eee2dee786 (Jens Axboe 2020-09-08 09:11:32 -0600 6815) /* if we're handling multiple rings, cap submit size for fairness */
e95eee2dee786 (Jens Axboe 2020-09-08 09:11:32 -0600 6816) if (cap_entries && to_submit > 8)
e95eee2dee786 (Jens Axboe 2020-09-08 09:11:32 -0600 6817) to_submit = 8;
e95eee2dee786 (Jens Axboe 2020-09-08 09:11:32 -0600 6818)
906a3c6f9ca07 (Xiaoguang Wang 2020-11-12 14:56:00 +0800 6819) if (!list_empty(&ctx->iopoll_list) || to_submit) {
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6820) unsigned nr_events = 0;
a4c0b3decb33f (Jackie Liu 2019-07-08 13:41:12 +0800 6821)
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6822) mutex_lock(&ctx->uring_lock);
906a3c6f9ca07 (Xiaoguang Wang 2020-11-12 14:56:00 +0800 6823) if (!list_empty(&ctx->iopoll_list))
f7be9c72d1de9 (Jens Axboe 2021-07-23 11:49:29 -0600 6824) io_do_iopoll(ctx, &nr_events, 0, true);
906a3c6f9ca07 (Xiaoguang Wang 2020-11-12 14:56:00 +0800 6825)
3b763ba1c77da (Pavel Begunkov 2021-04-18 14:52:08 +0100 6826) /*
3b763ba1c77da (Pavel Begunkov 2021-04-18 14:52:08 +0100 6827) * Don't submit if refs are dying, good for io_uring_register(),
3b763ba1c77da (Pavel Begunkov 2021-04-18 14:52:08 +0100 6828) * but also it is relied upon by io_ring_exit_work()
3b763ba1c77da (Pavel Begunkov 2021-04-18 14:52:08 +0100 6829) */
0298ef969a110 (Pavel Begunkov 2021-03-08 13:20:57 +0000 6830) if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
0298ef969a110 (Pavel Begunkov 2021-03-08 13:20:57 +0000 6831) !(ctx->flags & IORING_SETUP_R_DISABLED))
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6832) ret = io_submit_sqes(ctx, to_submit);
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6833) mutex_unlock(&ctx->uring_lock);
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6834) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6835)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 6836) if (!io_sqring_full(ctx) && wq_has_sleeper(&ctx->sqo_sq_wait))
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 6837) wake_up(&ctx->sqo_sq_wait);
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6838)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6839) return ret;
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6840) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6841)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6842) static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6843) {
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6844) struct io_ring_ctx *ctx;
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6845) unsigned sq_thread_idle = 0;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6846)
c9dca27dc7f9c (Pavel Begunkov 2021-03-10 13:13:55 +0000 6847) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
c9dca27dc7f9c (Pavel Begunkov 2021-03-10 13:13:55 +0000 6848) sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6849) sqd->sq_thread_idle = sq_thread_idle;
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6850) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6851)
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6852) static int io_sq_thread(void *data)
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6853) {
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 6854) struct io_sq_data *sqd = data;
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 6855) struct io_ring_ctx *ctx;
a0d9205f7d36b (Xiaoguang Wang 2020-11-12 14:55:59 +0800 6856) unsigned long timeout = 0;
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6857) char buf[TASK_COMM_LEN];
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6858) DEFINE_WAIT(wait);
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6859)
696ee88a7c50f (Pavel Begunkov 2021-04-01 09:55:04 +0100 6860) snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6861) set_task_comm(current, buf);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6862)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6863) if (sqd->sq_cpu != -1)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6864) set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6865) else
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6866) set_cpus_allowed_ptr(current, cpu_online_mask);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6867) current->flags |= PF_NO_SETAFFINITY;
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6868)
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 6869) mutex_lock(&sqd->lock);
c7d95613c7d6e (Pavel Begunkov 2021-04-13 11:43:00 +0100 6870) /* a user may had exited before the thread started */
c7d95613c7d6e (Pavel Begunkov 2021-04-13 11:43:00 +0100 6871) io_run_task_work_head(&sqd->park_task_work);
c7d95613c7d6e (Pavel Begunkov 2021-04-13 11:43:00 +0100 6872)
05962f95f9ac7 (Jens Axboe 2021-03-06 13:58:48 -0700 6873) while (!test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) {
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6874) int ret;
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6875) bool cap_entries, sqt_spin, needs_sched;
c1edbf5f081be (Jens Axboe 2019-11-10 16:56:04 -0700 6876)
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6877) if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6878) signal_pending(current)) {
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6879) bool did_sig = false;
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6880)
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 6881) mutex_unlock(&sqd->lock);
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6882) if (signal_pending(current)) {
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6883) struct ksignal ksig;
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6884)
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6885) did_sig = get_signal(&ksig);
82734c5b1b24c (Jens Axboe 2021-03-29 06:52:44 -0600 6886) }
05962f95f9ac7 (Jens Axboe 2021-03-06 13:58:48 -0700 6887) cond_resched();
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 6888) mutex_lock(&sqd->lock);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 6889) io_run_task_work();
b7f5a0bfe2061 (Pavel Begunkov 2021-03-15 14:23:08 +0000 6890) io_run_task_work_head(&sqd->park_task_work);
c7d95613c7d6e (Pavel Begunkov 2021-04-13 11:43:00 +0100 6891) if (did_sig)
c7d95613c7d6e (Pavel Begunkov 2021-04-13 11:43:00 +0100 6892) break;
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6893) timeout = jiffies + sqd->sq_thread_idle;
7d41e8543d809 (Pavel Begunkov 2021-03-10 13:13:54 +0000 6894) continue;
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6895) }
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6896) sqt_spin = false;
e95eee2dee786 (Jens Axboe 2020-09-08 09:11:32 -0600 6897) cap_entries = !list_is_singular(&sqd->ctx_list);
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 6898) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
7c30f36a98ae4 (Stefan Metzmacher 2021-03-07 11:54:28 +0100 6899) const struct cred *creds = NULL;
7c30f36a98ae4 (Stefan Metzmacher 2021-03-07 11:54:28 +0100 6900)
7c30f36a98ae4 (Stefan Metzmacher 2021-03-07 11:54:28 +0100 6901) if (ctx->sq_creds != current_cred())
7c30f36a98ae4 (Stefan Metzmacher 2021-03-07 11:54:28 +0100 6902) creds = override_creds(ctx->sq_creds);
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6903) ret = __io_sq_thread(ctx, cap_entries);
7c30f36a98ae4 (Stefan Metzmacher 2021-03-07 11:54:28 +0100 6904) if (creds)
7c30f36a98ae4 (Stefan Metzmacher 2021-03-07 11:54:28 +0100 6905) revert_creds(creds);
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6906) if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6907) sqt_spin = true;
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 6908) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6909)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6910) if (sqt_spin || !time_after(jiffies, timeout)) {
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6911) io_run_task_work();
c8d1ba583fe67 (Jens Axboe 2020-09-14 11:07:26 -0600 6912) cond_resched();
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6913) if (sqt_spin)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6914) timeout = jiffies + sqd->sq_thread_idle;
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6915) continue;
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6916) }
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6917)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6918) prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
025b6262dc963 (Olivier Langlois 2021-06-23 11:50:11 -0700 6919) if (!test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) &&
025b6262dc963 (Olivier Langlois 2021-06-23 11:50:11 -0700 6920) !io_run_task_work()) {
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 6921) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 6922) io_ring_set_wakeup_flag(ctx);
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6923)
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6924) needs_sched = true;
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6925) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6926) if ((ctx->flags & IORING_SETUP_IOPOLL) &&
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6927) !list_empty_careful(&ctx->iopoll_list)) {
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6928) needs_sched = false;
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6929) break;
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6930) }
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6931) if (io_sqring_entries(ctx)) {
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6932) needs_sched = false;
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6933) break;
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6934) }
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6935) }
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6936)
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6937) if (needs_sched) {
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6938) mutex_unlock(&sqd->lock);
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6939) schedule();
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6940) mutex_lock(&sqd->lock);
724cb4f9ec905 (Hao Xu 2021-04-21 23:19:11 +0800 6941) }
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 6942) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 6943) io_ring_clear_wakeup_flag(ctx);
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6944) }
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6945)
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6946) finish_wait(&sqd->wait, &wait);
b7f5a0bfe2061 (Pavel Begunkov 2021-03-15 14:23:08 +0000 6947) io_run_task_work_head(&sqd->park_task_work);
0836924634407 (Xiaoguang Wang 2020-11-03 14:15:59 +0800 6948) timeout = jiffies + sqd->sq_thread_idle;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6949) }
28cea78af4491 (Jens Axboe 2020-09-14 10:51:17 -0600 6950)
734551df6f9be (Pavel Begunkov 2021-04-18 14:52:09 +0100 6951) io_uring_cancel_sqpoll(sqd);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6952) sqd->thread = NULL;
05962f95f9ac7 (Jens Axboe 2021-03-06 13:58:48 -0700 6953) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
5f3f26f98ae48 (Jens Axboe 2021-02-25 10:17:46 -0700 6954) io_ring_set_wakeup_flag(ctx);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 6955) io_run_task_work();
b7f5a0bfe2061 (Pavel Begunkov 2021-03-15 14:23:08 +0000 6956) io_run_task_work_head(&sqd->park_task_work);
734551df6f9be (Pavel Begunkov 2021-04-18 14:52:09 +0100 6957) mutex_unlock(&sqd->lock);
734551df6f9be (Pavel Begunkov 2021-04-18 14:52:09 +0100 6958)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6959) complete(&sqd->exited);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 6960) do_exit(0);
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6961) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 6962)
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6963) struct io_wait_queue {
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6964) struct wait_queue_entry wq;
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6965) struct io_ring_ctx *ctx;
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6966) unsigned to_wait;
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6967) unsigned nr_timeouts;
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6968) };
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6969)
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6970) static inline bool io_should_wake(struct io_wait_queue *iowq)
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6971) {
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6972) struct io_ring_ctx *ctx = iowq->ctx;
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6973)
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6974) /*
d195a66e367b3 (Brian Gianforcaro 2019-12-13 03:09:50 -0800 6975) * Wake up if we have enough events, or if a timeout occurred since we
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6976) * started waiting. For timeouts, we always want to return to userspace,
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6977) * regardless of event count.
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6978) */
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6979) return io_cqring_events(ctx) >= iowq->to_wait ||
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6980) atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6981) }
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6982)
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6983) static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6984) int wake_flags, void *key)
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6985) {
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6986) struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6987) wq);
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6988)
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6989) /*
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6990) * Cannot safely flush overflowed CQEs from here, ensure we wake up
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6991) * the task, and the next invocation will do it.
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6992) */
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6993) if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->cq_check_overflow))
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6994) return autoremove_wake_function(curr, mode, wake_flags, key);
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 6995) return -1;
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6996) }
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 6997)
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 6998) static int io_run_task_work_sig(void)
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 6999) {
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 7000) if (io_run_task_work())
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 7001) return 1;
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 7002) if (!signal_pending(current))
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 7003) return 0;
0b8cfa974dfc9 (Jens Axboe 2021-03-21 14:16:08 -0600 7004) if (test_thread_flag(TIF_NOTIFY_SIGNAL))
792ee0f6db5b9 (Jens Axboe 2020-10-22 20:17:18 -0600 7005) return -ERESTARTSYS;
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 7006) return -EINTR;
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 7007) }
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 7008)
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7009) /* when returns >0, the caller should retry */
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7010) static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7011) struct io_wait_queue *iowq,
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7012) signed long *timeout)
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7013) {
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7014) int ret;
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7015)
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7016) /* make sure we run task_work before checking for signals */
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7017) ret = io_run_task_work_sig();
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7018) if (ret || io_should_wake(iowq))
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7019) return ret;
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7020) /* let the caller flush overflows, retry */
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7021) if (test_bit(0, &ctx->cq_check_overflow))
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7022) return 1;
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7023)
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7024) *timeout = schedule_timeout(*timeout);
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7025) return !*timeout ? -ETIME : 1;
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7026) }
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7027)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7028) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7029) * Wait until events become available, if we don't already have some. The
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7030) * application must reap them itself, as they reside on the shared cq ring.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7031) */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7032) static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 7033) const sigset_t __user *sig, size_t sigsz,
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 7034) struct __kernel_timespec __user *uts)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7035) {
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7036) struct io_wait_queue iowq = {
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7037) .wq = {
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7038) .private = current,
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7039) .func = io_wake_function,
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7040) .entry = LIST_HEAD_INIT(iowq.wq.entry),
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7041) },
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7042) .ctx = ctx,
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7043) .to_wait = min_events,
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7044) };
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 7045) struct io_rings *rings = ctx->rings;
c1d5a224683b3 (Pavel Begunkov 2021-02-04 13:51:57 +0000 7046) signed long timeout = MAX_SCHEDULE_TIMEOUT;
c1d5a224683b3 (Pavel Begunkov 2021-02-04 13:51:57 +0000 7047) int ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7048)
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 7049) do {
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 7050) io_cqring_overflow_flush(ctx, false);
6c503150ae33e (Pavel Begunkov 2021-01-04 20:36:36 +0000 7051) if (io_cqring_events(ctx) >= min_events)
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 7052) return 0;
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 7053) if (!io_run_task_work())
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 7054) break;
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 7055) } while (1);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7056)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7057) if (sig) {
9e75ad5d8f399 (Arnd Bergmann 2019-03-25 15:34:53 +0100 7058) #ifdef CONFIG_COMPAT
9e75ad5d8f399 (Arnd Bergmann 2019-03-25 15:34:53 +0100 7059) if (in_compat_syscall())
9e75ad5d8f399 (Arnd Bergmann 2019-03-25 15:34:53 +0100 7060) ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
b772434be0891 (Oleg Nesterov 2019-07-16 16:29:53 -0700 7061) sigsz);
9e75ad5d8f399 (Arnd Bergmann 2019-03-25 15:34:53 +0100 7062) else
9e75ad5d8f399 (Arnd Bergmann 2019-03-25 15:34:53 +0100 7063) #endif
b772434be0891 (Oleg Nesterov 2019-07-16 16:29:53 -0700 7064) ret = set_user_sigmask(sig, sigsz);
9e75ad5d8f399 (Arnd Bergmann 2019-03-25 15:34:53 +0100 7065)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7066) if (ret)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7067) return ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7068) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7069)
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 7070) if (uts) {
c1d5a224683b3 (Pavel Begunkov 2021-02-04 13:51:57 +0000 7071) struct timespec64 ts;
c1d5a224683b3 (Pavel Begunkov 2021-02-04 13:51:57 +0000 7072)
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 7073) if (get_timespec64(&ts, uts))
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 7074) return -EFAULT;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 7075) timeout = timespec64_to_jiffies(&ts);
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 7076) }
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 7077)
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7078) iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
c826bd7a743f2 (Dmitrii Dolgov 2019-10-15 19:02:01 +0200 7079) trace_io_uring_cqring_wait(ctx, min_events);
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7080) do {
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 7081) /* if we can't even flush overflow, don't wait for more */
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 7082) if (!io_cqring_overflow_flush(ctx, false)) {
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 7083) ret = -EBUSY;
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 7084) break;
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 7085) }
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7086) prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7087) TASK_INTERRUPTIBLE);
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7088) ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7089) finish_wait(&ctx->wait, &iowq.wq);
ca0a26511c679 (Jens Axboe 2021-03-04 17:15:48 -0700 7090) cond_resched();
eeb60b9ab4000 (Pavel Begunkov 2021-02-04 13:51:58 +0000 7091) } while (ret > 0);
bda521624e75c (Jens Axboe 2019-09-24 13:47:15 -0600 7092)
b7db41c9e03b5 (Jens Axboe 2020-07-04 08:55:50 -0600 7093) restore_saved_sigmask_unless(ret == -EINTR);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7094)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 7095) return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7096) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7097)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7098) static void io_free_file_tables(struct io_file_table *table, unsigned nr_files)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7099) {
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7100) unsigned i, nr_tables = DIV_ROUND_UP(nr_files, IORING_MAX_FILES_TABLE);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7101)
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7102) for (i = 0; i < nr_tables; i++)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7103) kfree(table->files[i]);
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7104) kfree(table->files);
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7105) table->files = NULL;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7106) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7107)
28a9fe2521348 (Pavel Begunkov 2021-04-01 15:43:47 +0100 7108) static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
1642b4450d20e (Pavel Begunkov 2020-12-30 21:34:14 +0000 7109) {
28a9fe2521348 (Pavel Begunkov 2021-04-01 15:43:47 +0100 7110) percpu_ref_exit(&ref_node->refs);
28a9fe2521348 (Pavel Begunkov 2021-04-01 15:43:47 +0100 7111) kfree(ref_node);
1642b4450d20e (Pavel Begunkov 2020-12-30 21:34:14 +0000 7112) }
1642b4450d20e (Pavel Begunkov 2020-12-30 21:34:14 +0000 7113)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7114) static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7115) struct io_rsrc_data *data_to_kill)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7116) {
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7117) WARN_ON_ONCE(!ctx->rsrc_backup_node);
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7118) WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7119)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7120) if (data_to_kill) {
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7121) struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
82fbcfa996e0b (Pavel Begunkov 2021-04-01 15:43:43 +0100 7122)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7123) rsrc_node->rsrc_data = data_to_kill;
de14d2fc6a624 (Jens Axboe 2021-08-09 07:49:41 -0600 7124) spin_lock_irq(&ctx->rsrc_ref_lock);
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7125) list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
de14d2fc6a624 (Jens Axboe 2021-08-09 07:49:41 -0600 7126) spin_unlock_irq(&ctx->rsrc_ref_lock);
82fbcfa996e0b (Pavel Begunkov 2021-04-01 15:43:43 +0100 7127)
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7128) atomic_inc(&data_to_kill->refs);
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7129) percpu_ref_kill(&rsrc_node->refs);
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7130) ctx->rsrc_node = NULL;
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7131) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7132)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7133) if (!ctx->rsrc_node) {
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7134) ctx->rsrc_node = ctx->rsrc_backup_node;
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7135) ctx->rsrc_backup_node = NULL;
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7136) }
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7137) }
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7138)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7139) static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
8dd03afe611d3 (Pavel Begunkov 2021-03-19 17:22:36 +0000 7140) {
8dd03afe611d3 (Pavel Begunkov 2021-03-19 17:22:36 +0000 7141) if (ctx->rsrc_backup_node)
8dd03afe611d3 (Pavel Begunkov 2021-03-19 17:22:36 +0000 7142) return 0;
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7143) ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
8dd03afe611d3 (Pavel Begunkov 2021-03-19 17:22:36 +0000 7144) return ctx->rsrc_backup_node ? 0 : -ENOMEM;
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7145) }
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7146)
40ae0ff70fb13 (Pavel Begunkov 2021-04-01 15:43:44 +0100 7147) static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7148) {
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7149) int ret;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7150)
215c39026023d (Pavel Begunkov 2021-04-01 15:43:48 +0100 7151) /* As we may drop ->uring_lock, other task may have started quiesce */
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7152) if (data->quiesce)
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7153) return -ENXIO;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7154)
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7155) data->quiesce = true;
1ffc54220c444 (Pavel Begunkov 2020-12-30 21:34:15 +0000 7156) do {
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7157) ret = io_rsrc_node_switch_start(ctx);
8dd03afe611d3 (Pavel Begunkov 2021-03-19 17:22:36 +0000 7158) if (ret)
f2303b1f8244d (Pavel Begunkov 2021-02-20 18:03:49 +0000 7159) break;
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7160) io_rsrc_node_switch(ctx, data);
f2303b1f8244d (Pavel Begunkov 2021-02-20 18:03:49 +0000 7161)
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7162) /* kill initial ref, already quiesced if zero */
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7163) if (atomic_dec_and_test(&data->refs))
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7164) break;
e1c5046e341db (Jens Axboe 2021-08-09 08:15:50 -0600 7165) mutex_unlock(&ctx->uring_lock);
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7166) flush_delayed_work(&ctx->rsrc_put_work);
1ffc54220c444 (Pavel Begunkov 2020-12-30 21:34:15 +0000 7167) ret = wait_for_completion_interruptible(&data->done);
e1c5046e341db (Jens Axboe 2021-08-09 08:15:50 -0600 7168) if (!ret) {
e1c5046e341db (Jens Axboe 2021-08-09 08:15:50 -0600 7169) mutex_lock(&ctx->uring_lock);
1ffc54220c444 (Pavel Begunkov 2020-12-30 21:34:15 +0000 7170) break;
e1c5046e341db (Jens Axboe 2021-08-09 08:15:50 -0600 7171) }
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7172)
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7173) atomic_inc(&data->refs);
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7174) /* wait for all works potentially completing data->done */
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7175) flush_delayed_work(&ctx->rsrc_put_work);
cb5e1b81304e0 (Jens Axboe 2021-02-25 07:37:35 -0700 7176) reinit_completion(&data->done);
8dd03afe611d3 (Pavel Begunkov 2021-03-19 17:22:36 +0000 7177)
1ffc54220c444 (Pavel Begunkov 2020-12-30 21:34:15 +0000 7178) ret = io_run_task_work_sig();
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7179) mutex_lock(&ctx->uring_lock);
f2303b1f8244d (Pavel Begunkov 2021-02-20 18:03:49 +0000 7180) } while (ret >= 0);
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7181) data->quiesce = false;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7182)
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 7183) return ret;
d7954b2ba9463 (Bijan Mottahedeh 2021-01-15 17:37:50 +0000 7184) }
d7954b2ba9463 (Bijan Mottahedeh 2021-01-15 17:37:50 +0000 7185)
44b31f2fa2c4b (Pavel Begunkov 2021-04-25 14:32:16 +0100 7186) static void io_rsrc_data_free(struct io_rsrc_data *data)
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7187) {
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7188) kvfree(data->tags);
44b31f2fa2c4b (Pavel Begunkov 2021-04-25 14:32:16 +0100 7189) kfree(data);
44b31f2fa2c4b (Pavel Begunkov 2021-04-25 14:32:16 +0100 7190) }
44b31f2fa2c4b (Pavel Begunkov 2021-04-25 14:32:16 +0100 7191)
40ae0ff70fb13 (Pavel Begunkov 2021-04-01 15:43:44 +0100 7192) static struct io_rsrc_data *io_rsrc_data_alloc(struct io_ring_ctx *ctx,
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7193) rsrc_put_fn *do_put,
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7194) unsigned nr)
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7195) {
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7196) struct io_rsrc_data *data;
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7197)
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7198) data = kzalloc(sizeof(*data), GFP_KERNEL);
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7199) if (!data)
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7200) return NULL;
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7201)
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7202) data->tags = kvcalloc(nr, sizeof(*data->tags), GFP_KERNEL);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7203) if (!data->tags) {
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7204) kfree(data);
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7205) return NULL;
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7206) }
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7207)
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7208) atomic_set(&data->refs, 1);
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7209) data->ctx = ctx;
40ae0ff70fb13 (Pavel Begunkov 2021-04-01 15:43:44 +0100 7210) data->do_put = do_put;
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7211) init_completion(&data->done);
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7212) return data;
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7213) }
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7214)
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7215) static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7216) {
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7217) #if defined(CONFIG_UNIX)
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7218) if (ctx->ring_sock) {
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7219) struct sock *sock = ctx->ring_sock->sk;
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7220) struct sk_buff *skb;
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7221)
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7222) while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7223) kfree_skb(skb);
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7224) }
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7225) #else
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7226) int i;
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7227)
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7228) for (i = 0; i < ctx->nr_user_files; i++) {
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7229) struct file *file;
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7230)
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7231) file = io_file_from_index(ctx, i);
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7232) if (file)
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7233) fput(file);
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7234) }
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7235) #endif
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7236) io_free_file_tables(&ctx->file_table, ctx->nr_user_files);
44b31f2fa2c4b (Pavel Begunkov 2021-04-25 14:32:16 +0100 7237) io_rsrc_data_free(ctx->file_data);
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7238) ctx->file_data = NULL;
fff4db76be297 (Pavel Begunkov 2021-04-25 14:32:15 +0100 7239) ctx->nr_user_files = 0;
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7240) }
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7241)
d7954b2ba9463 (Bijan Mottahedeh 2021-01-15 17:37:50 +0000 7242) static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
d7954b2ba9463 (Bijan Mottahedeh 2021-01-15 17:37:50 +0000 7243) {
d7954b2ba9463 (Bijan Mottahedeh 2021-01-15 17:37:50 +0000 7244) int ret;
d7954b2ba9463 (Bijan Mottahedeh 2021-01-15 17:37:50 +0000 7245)
084804002e512 (Pavel Begunkov 2021-04-13 02:58:38 +0100 7246) if (!ctx->file_data)
d7954b2ba9463 (Bijan Mottahedeh 2021-01-15 17:37:50 +0000 7247) return -ENXIO;
084804002e512 (Pavel Begunkov 2021-04-13 02:58:38 +0100 7248) ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
084804002e512 (Pavel Begunkov 2021-04-13 02:58:38 +0100 7249) if (!ret)
084804002e512 (Pavel Begunkov 2021-04-13 02:58:38 +0100 7250) __io_sqe_files_unregister(ctx);
084804002e512 (Pavel Begunkov 2021-04-13 02:58:38 +0100 7251) return ret;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7252) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7253)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7254) static void io_sq_thread_unpark(struct io_sq_data *sqd)
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 7255) __releases(&sqd->lock)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7256) {
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 7257) WARN_ON_ONCE(sqd->thread == current);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 7258)
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7259) /*
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7260) * Do the dance but not conditional clear_bit() because it'd race with
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7261) * other threads incrementing park_pending and setting the bit.
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7262) */
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7263) clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7264) if (atomic_dec_return(&sqd->park_pending))
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7265) set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 7266) mutex_unlock(&sqd->lock);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7267) }
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7268)
86e0d6766cf90 (Jens Axboe 2021-03-05 08:44:39 -0700 7269) static void io_sq_thread_park(struct io_sq_data *sqd)
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 7270) __acquires(&sqd->lock)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7271) {
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 7272) WARN_ON_ONCE(sqd->thread == current);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 7273)
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7274) atomic_inc(&sqd->park_pending);
86e0d6766cf90 (Jens Axboe 2021-03-05 08:44:39 -0700 7275) set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 7276) mutex_lock(&sqd->lock);
05962f95f9ac7 (Jens Axboe 2021-03-06 13:58:48 -0700 7277) if (sqd->thread)
86e0d6766cf90 (Jens Axboe 2021-03-05 08:44:39 -0700 7278) wake_up_process(sqd->thread);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7279) }
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7280)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7281) static void io_sq_thread_stop(struct io_sq_data *sqd)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7282) {
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 7283) WARN_ON_ONCE(sqd->thread == current);
88885f66e8c66 (Pavel Begunkov 2021-04-11 01:46:38 +0100 7284) WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state));
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 7285)
05962f95f9ac7 (Jens Axboe 2021-03-06 13:58:48 -0700 7286) set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
88885f66e8c66 (Pavel Begunkov 2021-04-11 01:46:38 +0100 7287) mutex_lock(&sqd->lock);
e8f98f24549d6 (Jens Axboe 2021-03-09 16:32:13 -0700 7288) if (sqd->thread)
e8f98f24549d6 (Jens Axboe 2021-03-09 16:32:13 -0700 7289) wake_up_process(sqd->thread);
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 7290) mutex_unlock(&sqd->lock);
05962f95f9ac7 (Jens Axboe 2021-03-06 13:58:48 -0700 7291) wait_for_completion(&sqd->exited);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7292) }
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7293)
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7294) static void io_put_sq_data(struct io_sq_data *sqd)
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 7295) {
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7296) if (refcount_dec_and_test(&sqd->refs)) {
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7297) WARN_ON_ONCE(atomic_read(&sqd->park_pending));
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7298)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7299) io_sq_thread_stop(sqd);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7300) kfree(sqd);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7301) }
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7302) }
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7303)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7304) static void io_sq_thread_finish(struct io_ring_ctx *ctx)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7305) {
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7306) struct io_sq_data *sqd = ctx->sq_data;
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7307)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7308) if (sqd) {
05962f95f9ac7 (Jens Axboe 2021-03-06 13:58:48 -0700 7309) io_sq_thread_park(sqd);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 7310) list_del_init(&ctx->sqd_list);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7311) io_sqd_update_thread_idle(sqd);
05962f95f9ac7 (Jens Axboe 2021-03-06 13:58:48 -0700 7312) io_sq_thread_unpark(sqd);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7313)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7314) io_put_sq_data(sqd);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7315) ctx->sq_data = NULL;
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7316) }
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7317) }
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7318)
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7319) static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7320) {
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7321) struct io_ring_ctx *ctx_attach;
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7322) struct io_sq_data *sqd;
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7323) struct fd f;
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7324)
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7325) f = fdget(p->wq_fd);
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7326) if (!f.file)
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7327) return ERR_PTR(-ENXIO);
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7328) if (f.file->f_op != &io_uring_fops) {
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7329) fdput(f);
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7330) return ERR_PTR(-EINVAL);
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7331) }
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7332)
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7333) ctx_attach = f.file->private_data;
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7334) sqd = ctx_attach->sq_data;
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7335) if (!sqd) {
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7336) fdput(f);
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7337) return ERR_PTR(-EINVAL);
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7338) }
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7339) if (sqd->task_tgid != current->tgid) {
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7340) fdput(f);
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7341) return ERR_PTR(-EPERM);
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7342) }
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7343)
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7344) refcount_inc(&sqd->refs);
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7345) fdput(f);
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7346) return sqd;
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7347) }
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7348)
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 7349) static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 7350) bool *attached)
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7351) {
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7352) struct io_sq_data *sqd;
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7353)
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 7354) *attached = false;
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7355) if (p->flags & IORING_SETUP_ATTACH_WQ) {
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7356) sqd = io_attach_sq_data(p);
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 7357) if (!IS_ERR(sqd)) {
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 7358) *attached = true;
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7359) return sqd;
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 7360) }
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7361) /* fall through for EPERM case, setup new sqd/task */
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7362) if (PTR_ERR(sqd) != -EPERM)
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7363) return sqd;
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 7364) }
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 7365)
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7366) sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7367) if (!sqd)
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7368) return ERR_PTR(-ENOMEM);
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7369)
9e138a4834542 (Pavel Begunkov 2021-03-14 20:57:12 +0000 7370) atomic_set(&sqd->park_pending, 0);
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7371) refcount_set(&sqd->refs, 1);
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 7372) INIT_LIST_HEAD(&sqd->ctx_list);
09a6f4efaa653 (Pavel Begunkov 2021-03-14 20:57:10 +0000 7373) mutex_init(&sqd->lock);
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7374) init_waitqueue_head(&sqd->wait);
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 7375) init_completion(&sqd->exited);
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7376) return sqd;
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7377) }
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7378)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7379) #if defined(CONFIG_UNIX)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7380) /*
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7381) * Ensure the UNIX gc is aware of our file set, so we are certain that
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7382) * the io_uring can be safely unregistered on process exit, even if we have
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7383) * loops in the file referencing.
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7384) */
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7385) static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7386) {
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7387) struct sock *sk = ctx->ring_sock->sk;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7388) struct scm_fp_list *fpl;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7389) struct sk_buff *skb;
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7390) int i, nr_files;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7391)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7392) fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7393) if (!fpl)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7394) return -ENOMEM;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7395)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7396) skb = alloc_skb(0, GFP_KERNEL);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7397) if (!skb) {
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7398) kfree(fpl);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7399) return -ENOMEM;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7400) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7401)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7402) skb->sk = sk;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7403)
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7404) nr_files = 0;
62e398be275a6 (Jens Axboe 2021-02-21 16:19:37 -0700 7405) fpl->user = get_uid(current_user());
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7406) for (i = 0; i < nr; i++) {
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7407) struct file *file = io_file_from_index(ctx, i + offset);
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7408)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7409) if (!file)
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7410) continue;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7411) fpl->fp[nr_files] = get_file(file);
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7412) unix_inflight(fpl->user, fpl->fp[nr_files]);
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7413) nr_files++;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7414) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7415)
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7416) if (nr_files) {
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7417) fpl->max = SCM_MAX_FD;
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7418) fpl->count = nr_files;
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7419) UNIXCB(skb).fp = fpl;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7420) skb->destructor = unix_destruct_scm;
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7421) refcount_add(skb->truesize, &sk->sk_wmem_alloc);
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7422) skb_queue_head(&sk->sk_receive_queue, skb);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7423)
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7424) for (i = 0; i < nr_files; i++)
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7425) fput(fpl->fp[i]);
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7426) } else {
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7427) kfree_skb(skb);
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7428) kfree(fpl);
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7429) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7430)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7431) return 0;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7432) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7433)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7434) /*
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7435) * If UNIX sockets are enabled, fd passing can cause a reference cycle which
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7436) * causes regular reference counting to break down. We rely on the UNIX
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7437) * garbage collection to take care of this problem for us.
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7438) */
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7439) static int io_sqe_files_scm(struct io_ring_ctx *ctx)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7440) {
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7441) unsigned left, total;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7442) int ret = 0;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7443)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7444) total = 0;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7445) left = ctx->nr_user_files;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7446) while (left) {
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7447) unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7448)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7449) ret = __io_sqe_files_scm(ctx, this_files, total);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7450) if (ret)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7451) break;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7452) left -= this_files;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7453) total += this_files;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7454) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7455)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7456) if (!ret)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7457) return 0;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7458)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7459) while (total < ctx->nr_user_files) {
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7460) struct file *file = io_file_from_index(ctx, total);
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7461)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7462) if (file)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7463) fput(file);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7464) total++;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7465) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7466)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7467) return ret;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7468) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7469) #else
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7470) static int io_sqe_files_scm(struct io_ring_ctx *ctx)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7471) {
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7472) return 0;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7473) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7474) #endif
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7475)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7476) static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7477) {
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7478) unsigned i, nr_tables = DIV_ROUND_UP(nr_files, IORING_MAX_FILES_TABLE);
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7479)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7480) table->files = kcalloc(nr_tables, sizeof(*table->files), GFP_KERNEL);
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7481) if (!table->files)
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7482) return false;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7483)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7484) for (i = 0; i < nr_tables; i++) {
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7485) unsigned int this_files = min(nr_files, IORING_MAX_FILES_TABLE);
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7486)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7487) table->files[i] = kcalloc(this_files, sizeof(*table->files[i]),
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7488) GFP_KERNEL);
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7489) if (!table->files[i])
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7490) break;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7491) nr_files -= this_files;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7492) }
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7493)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7494) if (i == nr_tables)
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7495) return true;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7496)
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7497) io_free_file_tables(table, nr_tables * IORING_MAX_FILES_TABLE);
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7498) return false;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7499) }
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7500)
47e90392c8ad9 (Pavel Begunkov 2021-04-01 15:43:56 +0100 7501) static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7502) {
5023853183699 (Bijan Mottahedeh 2021-01-15 17:37:45 +0000 7503) struct file *file = prsrc->file;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7504) #if defined(CONFIG_UNIX)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7505) struct sock *sock = ctx->ring_sock->sk;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7506) struct sk_buff_head list, *head = &sock->sk_receive_queue;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7507) struct sk_buff *skb;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7508) int i;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7509)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7510) __skb_queue_head_init(&list);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7511)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7512) /*
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7513) * Find the skb that holds this file in its SCM_RIGHTS. When found,
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7514) * remove this entry and rearrange the file array.
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7515) */
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7516) skb = skb_dequeue(head);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7517) while (skb) {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7518) struct scm_fp_list *fp;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7519)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7520) fp = UNIXCB(skb).fp;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7521) for (i = 0; i < fp->count; i++) {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7522) int left;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7523)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7524) if (fp->fp[i] != file)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7525) continue;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7526)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7527) unix_notinflight(fp->user, fp->fp[i]);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7528) left = fp->count - 1 - i;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7529) if (left) {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7530) memmove(&fp->fp[i], &fp->fp[i + 1],
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7531) left * sizeof(struct file *));
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7532) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7533) fp->count--;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7534) if (!fp->count) {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7535) kfree_skb(skb);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7536) skb = NULL;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7537) } else {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7538) __skb_queue_tail(&list, skb);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7539) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7540) fput(file);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7541) file = NULL;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7542) break;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7543) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7544)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7545) if (!file)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7546) break;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7547)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7548) __skb_queue_tail(&list, skb);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7549)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7550) skb = skb_dequeue(head);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7551) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7552)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7553) if (skb_peek(&list)) {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7554) spin_lock_irq(&head->lock);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7555) while ((skb = __skb_dequeue(&list)) != NULL)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7556) __skb_queue_tail(head, skb);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7557) spin_unlock_irq(&head->lock);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7558) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7559) #else
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7560) fput(file);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7561) #endif
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7562) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7563)
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7564) static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7565) {
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7566) struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7567) struct io_ring_ctx *ctx = rsrc_data->ctx;
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7568) struct io_rsrc_put *prsrc, *tmp;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7569)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7570) list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7571) list_del(&prsrc->list);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7572)
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7573) if (prsrc->tag) {
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7574) bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7575) unsigned long flags;
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7576)
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7577) io_ring_submit_lock(ctx, lock_ring);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7578) spin_lock_irqsave(&ctx->completion_lock, flags);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7579) io_cqring_fill_event(ctx, prsrc->tag, 0, 0);
2840f710f23a3 (Pavel Begunkov 2021-04-27 16:13:51 +0100 7580) ctx->cq_extra++;
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7581) io_commit_cqring(ctx);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7582) spin_unlock_irqrestore(&ctx->completion_lock, flags);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7583) io_cqring_ev_posted(ctx);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7584) io_ring_submit_unlock(ctx, lock_ring);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7585) }
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7586)
40ae0ff70fb13 (Pavel Begunkov 2021-04-01 15:43:44 +0100 7587) rsrc_data->do_put(ctx, prsrc);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7588) kfree(prsrc);
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7589) }
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7590)
28a9fe2521348 (Pavel Begunkov 2021-04-01 15:43:47 +0100 7591) io_rsrc_node_destroy(ref_node);
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7592) if (atomic_dec_and_test(&rsrc_data->refs))
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7593) complete(&rsrc_data->done);
2faf852d1be8a (Jens Axboe 2020-02-04 19:54:55 -0700 7594) }
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7595)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7596) static void io_rsrc_put_work(struct work_struct *work)
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7597) {
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7598) struct io_ring_ctx *ctx;
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7599) struct llist_node *node;
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7600)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7601) ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7602) node = llist_del_all(&ctx->rsrc_put_llist);
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7603)
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7604) while (node) {
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7605) struct io_rsrc_node *ref_node;
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7606) struct llist_node *next = node->next;
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7607)
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7608) ref_node = llist_entry(node, struct io_rsrc_node, llist);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7609) __io_rsrc_put_work(ref_node);
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7610) node = next;
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7611) }
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7612) }
4a38aed2a0a72 (Jens Axboe 2020-05-14 17:21:15 -0600 7613)
00835dce1406e (Bijan Mottahedeh 2021-01-15 17:37:52 +0000 7614) static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
2faf852d1be8a (Jens Axboe 2020-02-04 19:54:55 -0700 7615) {
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7616) struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7617) struct io_ring_ctx *ctx = node->rsrc_data->ctx;
de14d2fc6a624 (Jens Axboe 2021-08-09 07:49:41 -0600 7618) unsigned long flags;
e297822b20e7f (Pavel Begunkov 2020-11-18 14:56:26 +0000 7619) bool first_add = false;
e297822b20e7f (Pavel Begunkov 2020-11-18 14:56:26 +0000 7620)
de14d2fc6a624 (Jens Axboe 2021-08-09 07:49:41 -0600 7621) spin_lock_irqsave(&ctx->rsrc_ref_lock, flags);
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7622) node->done = true;
e297822b20e7f (Pavel Begunkov 2020-11-18 14:56:26 +0000 7623)
d67d2263fb235 (Bijan Mottahedeh 2021-01-15 17:37:46 +0000 7624) while (!list_empty(&ctx->rsrc_ref_list)) {
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7625) node = list_first_entry(&ctx->rsrc_ref_list,
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7626) struct io_rsrc_node, node);
e297822b20e7f (Pavel Begunkov 2020-11-18 14:56:26 +0000 7627) /* recycle ref nodes in order */
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7628) if (!node->done)
e297822b20e7f (Pavel Begunkov 2020-11-18 14:56:26 +0000 7629) break;
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7630) list_del(&node->node);
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7631) first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
e297822b20e7f (Pavel Begunkov 2020-11-18 14:56:26 +0000 7632) }
de14d2fc6a624 (Jens Axboe 2021-08-09 07:49:41 -0600 7633) spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags);
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7634)
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7635) if (first_add)
3e9424989b59f (Pavel Begunkov 2021-04-11 01:46:34 +0100 7636) mod_delayed_work(system_wq, &ctx->rsrc_put_work, HZ);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7637) }
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7638)
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7639) static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7640) {
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7641) struct io_rsrc_node *ref_node;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7642)
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7643) ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7644) if (!ref_node)
3e2224c5867fe (Matthew Wilcox (Oracle) 2021-01-06 16:09:26 +0000 7645) return NULL;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7646)
00835dce1406e (Bijan Mottahedeh 2021-01-15 17:37:52 +0000 7647) if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7648) 0, GFP_KERNEL)) {
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7649) kfree(ref_node);
3e2224c5867fe (Matthew Wilcox (Oracle) 2021-01-06 16:09:26 +0000 7650) return NULL;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7651) }
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7652) INIT_LIST_HEAD(&ref_node->node);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7653) INIT_LIST_HEAD(&ref_node->rsrc_list);
e297822b20e7f (Pavel Begunkov 2020-11-18 14:56:26 +0000 7654) ref_node->done = false;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7655) return ref_node;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7656) }
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7657)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7658) static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7659) unsigned nr_args, u64 __user *tags)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7660) {
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7661) __s32 __user *fds = (__s32 __user *) arg;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7662) struct file *file;
f3baed39929ed (Pavel Begunkov 2021-04-01 15:43:42 +0100 7663) int fd, ret;
846a4ef22bf6d (Pavel Begunkov 2021-04-01 15:44:03 +0100 7664) unsigned i;
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7665) struct io_rsrc_data *file_data;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7666)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7667) if (ctx->file_data)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7668) return -EBUSY;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7669) if (!nr_args)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7670) return -EINVAL;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7671) if (nr_args > IORING_MAX_FIXED_FILES)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7672) return -EMFILE;
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7673) ret = io_rsrc_node_switch_start(ctx);
f3baed39929ed (Pavel Begunkov 2021-04-01 15:43:42 +0100 7674) if (ret)
f3baed39929ed (Pavel Begunkov 2021-04-01 15:43:42 +0100 7675) return ret;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7676)
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7677) file_data = io_rsrc_data_alloc(ctx, io_rsrc_file_put, nr_args);
5398ae6985255 (Pavel Begunkov 2020-10-10 18:34:14 +0100 7678) if (!file_data)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7679) return -ENOMEM;
13770a71ed355 (Dan Carpenter 2021-02-01 15:23:42 +0300 7680) ctx->file_data = file_data;
f3baed39929ed (Pavel Begunkov 2021-04-01 15:43:42 +0100 7681) ret = -ENOMEM;
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7682) if (!io_alloc_file_tables(&ctx->file_table, nr_args))
1ad555c6ae6e2 (Bijan Mottahedeh 2021-01-15 17:37:51 +0000 7683) goto out_free;
65e19f54d29cd (Jens Axboe 2019-10-26 07:20:21 -0600 7684)
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7685) for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7686) u64 tag = 0;
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7687)
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7688) if ((tags && copy_from_user(&tag, &tags[i], sizeof(tag))) ||
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7689) copy_from_user(&fd, &fds[i], sizeof(fd))) {
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7690) ret = -EFAULT;
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7691) goto out_fput;
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7692) }
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7693) /* allow sparse sets */
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7694) if (fd == -1) {
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7695) ret = -EINVAL;
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7696) if (unlikely(tag))
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7697) goto out_fput;
08a451739a9b5 (Jens Axboe 2019-10-03 08:11:03 -0600 7698) continue;
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7699) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7700)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7701) file = fget(fd);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7702) ret = -EBADF;
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7703) if (unlikely(!file))
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7704) goto out_fput;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7705)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7706) /*
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7707) * Don't allow io_uring instances to be registered. If UNIX
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7708) * isn't enabled, then this causes a reference cycle and this
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7709) * instance can never get freed. If UNIX is enabled we'll
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7710) * handle it just fine, but there's still no point in allowing
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7711) * a ring fd as it doesn't support regular read/write anyway.
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7712) */
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7713) if (file->f_op == &io_uring_fops) {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7714) fput(file);
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7715) goto out_fput;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7716) }
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 7717) ctx->file_data->tags[i] = tag;
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7718) io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7719) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7720)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7721) ret = io_sqe_files_scm(ctx);
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7722) if (ret) {
084804002e512 (Pavel Begunkov 2021-04-13 02:58:38 +0100 7723) __io_sqe_files_unregister(ctx);
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7724) return ret;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7725) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7726)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7727) io_rsrc_node_switch(ctx, NULL);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7728) return ret;
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7729) out_fput:
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7730) for (i = 0; i < ctx->nr_user_files; i++) {
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7731) file = io_file_from_index(ctx, i);
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7732) if (file)
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7733) fput(file);
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7734) }
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7735) io_free_file_tables(&ctx->file_table, nr_args);
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7736) ctx->nr_user_files = 0;
600cf3f8b3f68 (Pavel Begunkov 2020-10-10 18:34:15 +0100 7737) out_free:
44b31f2fa2c4b (Pavel Begunkov 2021-04-25 14:32:16 +0100 7738) io_rsrc_data_free(ctx->file_data);
55cbc2564ab2f (Jens Axboe 2020-10-14 07:35:57 -0600 7739) ctx->file_data = NULL;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7740) return ret;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7741) }
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 7742)
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7743) static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7744) int index)
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7745) {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7746) #if defined(CONFIG_UNIX)
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7747) struct sock *sock = ctx->ring_sock->sk;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7748) struct sk_buff_head *head = &sock->sk_receive_queue;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7749) struct sk_buff *skb;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7750)
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7751) /*
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7752) * See if we can merge this file into an existing skb SCM_RIGHTS
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7753) * file set. If there's no room, fall back to allocating a new skb
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7754) * and filling it in.
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7755) */
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7756) spin_lock_irq(&head->lock);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7757) skb = skb_peek(head);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7758) if (skb) {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7759) struct scm_fp_list *fpl = UNIXCB(skb).fp;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7760)
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7761) if (fpl->count < SCM_MAX_FD) {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7762) __skb_unlink(skb, head);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7763) spin_unlock_irq(&head->lock);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7764) fpl->fp[fpl->count] = get_file(file);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7765) unix_inflight(fpl->user, fpl->fp[fpl->count]);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7766) fpl->count++;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7767) spin_lock_irq(&head->lock);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7768) __skb_queue_head(head, skb);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7769) } else {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7770) skb = NULL;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7771) }
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7772) }
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7773) spin_unlock_irq(&head->lock);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7774)
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7775) if (skb) {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7776) fput(file);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7777) return 0;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7778) }
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7779)
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7780) return __io_sqe_files_scm(ctx, 1, index);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7781) #else
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7782) return 0;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7783) #endif
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7784) }
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7785)
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7786) static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
e7c78371bbf74 (Pavel Begunkov 2021-04-01 15:43:45 +0100 7787) struct io_rsrc_node *node, void *rsrc)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7788) {
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7789) struct io_rsrc_put *prsrc;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7790)
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7791) prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
269bbe5fd4d2f (Bijan Mottahedeh 2021-01-15 17:37:44 +0000 7792) if (!prsrc)
a5318d3cdffbe (Hillf Danton 2020-03-23 17:47:15 +0800 7793) return -ENOMEM;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7794)
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7795) prsrc->tag = data->tags[idx];
5023853183699 (Bijan Mottahedeh 2021-01-15 17:37:45 +0000 7796) prsrc->rsrc = rsrc;
e7c78371bbf74 (Pavel Begunkov 2021-04-01 15:43:45 +0100 7797) list_add(&prsrc->list, &node->rsrc_list);
a5318d3cdffbe (Hillf Danton 2020-03-23 17:47:15 +0800 7798) return 0;
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7799) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7800)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7801) static int __io_sqe_files_update(struct io_ring_ctx *ctx,
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7802) struct io_uring_rsrc_update2 *up,
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7803) unsigned nr_args)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7804) {
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7805) u64 __user *tags = u64_to_user_ptr(up->tags);
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 7806) __s32 __user *fds = u64_to_user_ptr(up->data);
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 7807) struct io_rsrc_data *data = ctx->file_data;
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 7808) struct io_fixed_file *file_slot;
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 7809) struct file *file;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 7810) int fd, i, err = 0;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 7811) unsigned int done;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7812) bool needs_switch = false;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7813)
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 7814) if (!ctx->file_data)
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 7815) return -ENXIO;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 7816) if (up->offset + nr_args > ctx->nr_user_files)
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7817) return -EINVAL;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7818)
67973b933e347 (Pavel Begunkov 2021-01-26 13:51:09 +0000 7819) for (done = 0; done < nr_args; done++) {
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7820) u64 tag = 0;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7821)
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7822) if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7823) copy_from_user(&fd, &fds[done], sizeof(fd))) {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7824) err = -EFAULT;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7825) break;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7826) }
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7827) if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7828) err = -EINVAL;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7829) break;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7830) }
4e0377a1c5c63 (noah 2021-01-26 15:23:28 -0500 7831) if (fd == IORING_REGISTER_FILES_SKIP)
4e0377a1c5c63 (noah 2021-01-26 15:23:28 -0500 7832) continue;
4e0377a1c5c63 (noah 2021-01-26 15:23:28 -0500 7833)
67973b933e347 (Pavel Begunkov 2021-01-26 13:51:09 +0000 7834) i = array_index_nospec(up->offset + done, ctx->nr_user_files);
aeca241b0bdd8 (Pavel Begunkov 2021-04-11 01:46:37 +0100 7835) file_slot = io_fixed_file_slot(&ctx->file_table, i);
ea64ec02b31d5 (Pavel Begunkov 2021-02-04 13:52:07 +0000 7836)
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 7837) if (file_slot->file_ptr) {
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 7838) file = (struct file *)(file_slot->file_ptr & FFS_MASK);
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7839) err = io_queue_rsrc_removal(data, up->offset + done,
b60c8dce33895 (Pavel Begunkov 2021-04-25 14:32:18 +0100 7840) ctx->rsrc_node, file);
a5318d3cdffbe (Hillf Danton 2020-03-23 17:47:15 +0800 7841) if (err)
a5318d3cdffbe (Hillf Danton 2020-03-23 17:47:15 +0800 7842) break;
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 7843) file_slot->file_ptr = 0;
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7844) needs_switch = true;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7845) }
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7846) if (fd != -1) {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7847) file = fget(fd);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7848) if (!file) {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7849) err = -EBADF;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7850) break;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7851) }
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7852) /*
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7853) * Don't allow io_uring instances to be registered. If
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7854) * UNIX isn't enabled, then this causes a reference
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7855) * cycle and this instance can never get freed. If UNIX
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7856) * is enabled we'll handle it just fine, but there's
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7857) * still no point in allowing a ring fd as it doesn't
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7858) * support regular read/write anyway.
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7859) */
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7860) if (file->f_op == &io_uring_fops) {
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7861) fput(file);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7862) err = -EBADF;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7863) break;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7864) }
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 7865) data->tags[up->offset + done] = tag;
9a321c98490c7 (Pavel Begunkov 2021-04-01 15:44:01 +0100 7866) io_fixed_file_set(file_slot, file);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7867) err = io_sqe_file_register(ctx, file, i);
f3bd9dae3708a (Yang Yingliang 2020-07-09 10:11:41 +0000 7868) if (err) {
a04b0ac0cb64f (Pavel Begunkov 2021-04-01 15:44:04 +0100 7869) file_slot->file_ptr = 0;
f3bd9dae3708a (Yang Yingliang 2020-07-09 10:11:41 +0000 7870) fput(file);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7871) break;
f3bd9dae3708a (Yang Yingliang 2020-07-09 10:11:41 +0000 7872) }
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7873) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7874) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 7875)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7876) if (needs_switch)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 7877) io_rsrc_node_switch(ctx, data);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7878) return done ? done : err;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 7879) }
0558955373023 (Xiaoguang Wang 2020-03-31 14:05:18 +0800 7880)
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 7881) static struct io_wq_work *io_free_work(struct io_wq_work *work)
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 7882) {
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 7883) struct io_kiocb *req = container_of(work, struct io_kiocb, work);
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 7884)
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 7885) req = io_put_req_find_next(req);
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 7886) return req ? &req->work : NULL;
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 7887) }
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 7888)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 7889) static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 7890) struct task_struct *task)
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7891) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7892) struct io_wq_hash *hash;
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7893) struct io_wq_data data;
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7894) unsigned int concurrency;
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7895)
502731a03f27c (Yang Yingliang 2021-07-20 16:38:05 +0800 7896) mutex_lock(&ctx->uring_lock);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7897) hash = ctx->hash_map;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7898) if (!hash) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7899) hash = kzalloc(sizeof(*hash), GFP_KERNEL);
502731a03f27c (Yang Yingliang 2021-07-20 16:38:05 +0800 7900) if (!hash) {
502731a03f27c (Yang Yingliang 2021-07-20 16:38:05 +0800 7901) mutex_unlock(&ctx->uring_lock);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7902) return ERR_PTR(-ENOMEM);
502731a03f27c (Yang Yingliang 2021-07-20 16:38:05 +0800 7903) }
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7904) refcount_set(&hash->refs, 1);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7905) init_waitqueue_head(&hash->wait);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7906) ctx->hash_map = hash;
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7907) }
502731a03f27c (Yang Yingliang 2021-07-20 16:38:05 +0800 7908) mutex_unlock(&ctx->uring_lock);
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7909)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 7910) data.hash = hash;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 7911) data.task = task;
e9fd939654f17 (Pavel Begunkov 2020-03-04 16:14:12 +0300 7912) data.free_work = io_free_work;
f5fa38c59cb0b (Pavel Begunkov 2020-06-08 21:08:20 +0300 7913) data.do_work = io_wq_submit_work;
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7914)
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7915) /* Do QD, or 4 * CPUS, whatever is smallest */
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7916) concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7917)
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7918) return io_wq_create(concurrency, &data);
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7919) }
24369c2e3bb06 (Pavel Begunkov 2020-01-28 03:15:48 +0300 7920)
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7921) static int io_uring_alloc_task_context(struct task_struct *task,
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7922) struct io_ring_ctx *ctx)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7923) {
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7924) struct io_uring_task *tctx;
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 7925) int ret;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7926)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7927) tctx = kmalloc(sizeof(*tctx), GFP_KERNEL);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7928) if (unlikely(!tctx))
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7929) return -ENOMEM;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7930)
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 7931) ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 7932) if (unlikely(ret)) {
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 7933) kfree(tctx);
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 7934) return ret;
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 7935) }
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 7936)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 7937) tctx->io_wq = io_init_wq_offload(ctx, task);
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7938) if (IS_ERR(tctx->io_wq)) {
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7939) ret = PTR_ERR(tctx->io_wq);
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7940) percpu_counter_destroy(&tctx->inflight);
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7941) kfree(tctx);
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7942) return ret;
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7943) }
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 7944)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7945) xa_init(&tctx->xa);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7946) init_waitqueue_head(&tctx->wait);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7947) tctx->last = NULL;
fdaf083cdfb55 (Jens Axboe 2020-10-30 09:37:30 -0600 7948) atomic_set(&tctx->in_idle, 0);
b303fe2e5a380 (Pavel Begunkov 2021-04-11 01:46:26 +0100 7949) atomic_set(&tctx->inflight_tracked, 0);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7950) task->io_uring = tctx;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 7951) spin_lock_init(&tctx->task_lock);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 7952) INIT_WQ_LIST(&tctx->task_list);
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 7953) tctx->task_state = 0;
7cbf1722d5fc5 (Jens Axboe 2021-02-10 00:03:20 +0000 7954) init_task_work(&tctx->task_work, tctx_task_work);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7955) return 0;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7956) }
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7957)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7958) void __io_uring_free(struct task_struct *tsk)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7959) {
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7960) struct io_uring_task *tctx = tsk->io_uring;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7961)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7962) WARN_ON_ONCE(!xa_empty(&tctx->xa));
ef8eaa4e65fac (Pavel Begunkov 2021-02-27 11:16:45 +0000 7963) WARN_ON_ONCE(tctx->io_wq);
ef8eaa4e65fac (Pavel Begunkov 2021-02-27 11:16:45 +0000 7964)
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 7965) percpu_counter_destroy(&tctx->inflight);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7966) kfree(tctx);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7967) tsk->io_uring = NULL;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7968) }
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 7969)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 7970) static int io_sq_offload_create(struct io_ring_ctx *ctx,
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 7971) struct io_uring_params *p)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7972) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7973) int ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 7974)
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7975) /* Retain compatibility with failing for an invalid attach attempt */
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7976) if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7977) IORING_SETUP_ATTACH_WQ) {
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7978) struct fd f;
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7979)
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7980) f = fdget(p->wq_fd);
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7981) if (!f.file)
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7982) return -ENXIO;
a6ead78130ad6 (Jens Axboe 2021-07-22 17:08:07 -0600 7983) if (f.file->f_op != &io_uring_fops) {
a6ead78130ad6 (Jens Axboe 2021-07-22 17:08:07 -0600 7984) fdput(f);
f2a48dd09b8e9 (Pavel Begunkov 2021-04-20 12:03:33 +0100 7985) return -EINVAL;
a6ead78130ad6 (Jens Axboe 2021-07-22 17:08:07 -0600 7986) }
a6ead78130ad6 (Jens Axboe 2021-07-22 17:08:07 -0600 7987) fdput(f);
d25e3a3de0d6f (Jens Axboe 2021-02-16 11:41:41 -0700 7988) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 7989) if (ctx->flags & IORING_SETUP_SQPOLL) {
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 7990) struct task_struct *tsk;
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7991) struct io_sq_data *sqd;
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 7992) bool attached;
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7993)
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 7994) sqd = io_get_sq_data(p, &attached);
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7995) if (IS_ERR(sqd)) {
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7996) ret = PTR_ERR(sqd);
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7997) goto err;
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 7998) }
69fb21310fd36 (Jens Axboe 2020-09-14 11:16:23 -0600 7999)
7c30f36a98ae4 (Stefan Metzmacher 2021-03-07 11:54:28 +0100 8000) ctx->sq_creds = get_current_cred();
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 8001) ctx->sq_data = sqd;
917257daa0fea (Jens Axboe 2019-04-13 09:28:55 -0600 8002) ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
917257daa0fea (Jens Axboe 2019-04-13 09:28:55 -0600 8003) if (!ctx->sq_thread_idle)
917257daa0fea (Jens Axboe 2019-04-13 09:28:55 -0600 8004) ctx->sq_thread_idle = HZ;
917257daa0fea (Jens Axboe 2019-04-13 09:28:55 -0600 8005)
78d7f6ba82edb (Pavel Begunkov 2021-03-10 13:13:53 +0000 8006) io_sq_thread_park(sqd);
de75a3d3f5a14 (Pavel Begunkov 2021-03-18 11:54:35 +0000 8007) list_add(&ctx->sqd_list, &sqd->ctx_list);
de75a3d3f5a14 (Pavel Begunkov 2021-03-18 11:54:35 +0000 8008) io_sqd_update_thread_idle(sqd);
26984fbf3ad9d (Pavel Begunkov 2021-03-11 23:29:37 +0000 8009) /* don't attach to a dying SQPOLL thread, would be racy */
f2a48dd09b8e9 (Pavel Begunkov 2021-04-20 12:03:33 +0100 8010) ret = (attached && !sqd->thread) ? -ENXIO : 0;
78d7f6ba82edb (Pavel Begunkov 2021-03-10 13:13:53 +0000 8011) io_sq_thread_unpark(sqd);
78d7f6ba82edb (Pavel Begunkov 2021-03-10 13:13:53 +0000 8012)
de75a3d3f5a14 (Pavel Begunkov 2021-03-18 11:54:35 +0000 8013) if (ret < 0)
de75a3d3f5a14 (Pavel Begunkov 2021-03-18 11:54:35 +0000 8014) goto err;
de75a3d3f5a14 (Pavel Begunkov 2021-03-18 11:54:35 +0000 8015) if (attached)
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 8016) return 0;
aa06165de863a (Jens Axboe 2020-09-02 14:50:27 -0600 8017)
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8018) if (p->flags & IORING_SETUP_SQ_AFF) {
44a9bd18a0f06 (Jens Axboe 2019-05-14 20:00:30 -0600 8019) int cpu = p->sq_thread_cpu;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8020)
917257daa0fea (Jens Axboe 2019-04-13 09:28:55 -0600 8021) ret = -EINVAL;
f2a48dd09b8e9 (Pavel Begunkov 2021-04-20 12:03:33 +0100 8022) if (cpu >= nr_cpu_ids || !cpu_online(cpu))
e8f98f24549d6 (Jens Axboe 2021-03-09 16:32:13 -0700 8023) goto err_sqpoll;
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 8024) sqd->sq_cpu = cpu;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8025) } else {
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 8026) sqd->sq_cpu = -1;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8027) }
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 8028)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 8029) sqd->task_pid = current->pid;
5c2469e0a22e0 (Jens Axboe 2021-03-11 10:17:56 -0700 8030) sqd->task_tgid = current->tgid;
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 8031) tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 8032) if (IS_ERR(tsk)) {
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 8033) ret = PTR_ERR(tsk);
e8f98f24549d6 (Jens Axboe 2021-03-09 16:32:13 -0700 8034) goto err_sqpoll;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8035) }
97a73a0f9fbfb (Pavel Begunkov 2021-03-08 17:30:54 +0000 8036)
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 8037) sqd->thread = tsk;
97a73a0f9fbfb (Pavel Begunkov 2021-03-08 17:30:54 +0000 8038) ret = io_uring_alloc_task_context(tsk, ctx);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 8039) wake_up_new_task(tsk);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 8040) if (ret)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 8041) goto err;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8042) } else if (p->flags & IORING_SETUP_SQ_AFF) {
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8043) /* Can't have SQ_AFF without SQPOLL */
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8044) ret = -EINVAL;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8045) goto err;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8046) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 8047)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8048) return 0;
f2a48dd09b8e9 (Pavel Begunkov 2021-04-20 12:03:33 +0100 8049) err_sqpoll:
f2a48dd09b8e9 (Pavel Begunkov 2021-04-20 12:03:33 +0100 8050) complete(&ctx->sq_data->exited);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8051) err:
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 8052) io_sq_thread_finish(ctx);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8053) return ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8054) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8055)
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8056) static inline void __io_unaccount_mem(struct user_struct *user,
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8057) unsigned long nr_pages)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8058) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8059) atomic_long_sub(nr_pages, &user->locked_vm);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8060) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8061)
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8062) static inline int __io_account_mem(struct user_struct *user,
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8063) unsigned long nr_pages)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8064) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8065) unsigned long page_limit, cur_pages, new_pages;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8066)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8067) /* Don't allow more pages than we can safely lock */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8068) page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8069)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8070) do {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8071) cur_pages = atomic_long_read(&user->locked_vm);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8072) new_pages = cur_pages + nr_pages;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8073) if (new_pages > page_limit)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8074) return -ENOMEM;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8075) } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8076) new_pages) != cur_pages);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8077)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8078) return 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8079) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8080)
26bfa89e25f42 (Jens Axboe 2021-02-09 20:14:12 -0700 8081) static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8082) {
62e398be275a6 (Jens Axboe 2021-02-21 16:19:37 -0700 8083) if (ctx->user)
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8084) __io_unaccount_mem(ctx->user, nr_pages);
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8085)
26bfa89e25f42 (Jens Axboe 2021-02-09 20:14:12 -0700 8086) if (ctx->mm_account)
26bfa89e25f42 (Jens Axboe 2021-02-09 20:14:12 -0700 8087) atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8088) }
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8089)
26bfa89e25f42 (Jens Axboe 2021-02-09 20:14:12 -0700 8090) static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8091) {
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8092) int ret;
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8093)
62e398be275a6 (Jens Axboe 2021-02-21 16:19:37 -0700 8094) if (ctx->user) {
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8095) ret = __io_account_mem(ctx->user, nr_pages);
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8096) if (ret)
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8097) return ret;
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8098) }
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8099)
26bfa89e25f42 (Jens Axboe 2021-02-09 20:14:12 -0700 8100) if (ctx->mm_account)
26bfa89e25f42 (Jens Axboe 2021-02-09 20:14:12 -0700 8101) atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8102)
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8103) return 0;
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8104) }
a087e2b519929 (Bijan Mottahedeh 2020-06-16 16:36:07 -0700 8105)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8106) static void io_mem_free(void *ptr)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8107) {
52e04ef4c9d45 (Mark Rutland 2019-04-30 17:30:21 +0100 8108) struct page *page;
52e04ef4c9d45 (Mark Rutland 2019-04-30 17:30:21 +0100 8109)
52e04ef4c9d45 (Mark Rutland 2019-04-30 17:30:21 +0100 8110) if (!ptr)
52e04ef4c9d45 (Mark Rutland 2019-04-30 17:30:21 +0100 8111) return;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8112)
52e04ef4c9d45 (Mark Rutland 2019-04-30 17:30:21 +0100 8113) page = virt_to_head_page(ptr);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8114) if (put_page_testzero(page))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8115) free_compound_page(page);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8116) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8117)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8118) static void *io_mem_alloc(size_t size)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8119) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8120) gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
26bfa89e25f42 (Jens Axboe 2021-02-09 20:14:12 -0700 8121) __GFP_NORETRY | __GFP_ACCOUNT;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8122)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8123) return (void *) __get_free_pages(gfp_flags, get_order(size));
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8124) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8125)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8126) static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8127) size_t *sq_offset)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8128) {
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8129) struct io_rings *rings;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8130) size_t off, sq_array_size;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8131)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8132) off = struct_size(rings, cqes, cq_entries);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8133) if (off == SIZE_MAX)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8134) return SIZE_MAX;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8135)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8136) #ifdef CONFIG_SMP
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8137) off = ALIGN(off, SMP_CACHE_BYTES);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8138) if (off == 0)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8139) return SIZE_MAX;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8140) #endif
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8141)
b36200f543ff0 (Dmitry Vyukov 2020-07-11 11:31:11 +0200 8142) if (sq_offset)
b36200f543ff0 (Dmitry Vyukov 2020-07-11 11:31:11 +0200 8143) *sq_offset = off;
b36200f543ff0 (Dmitry Vyukov 2020-07-11 11:31:11 +0200 8144)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8145) sq_array_size = array_size(sizeof(u32), sq_entries);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8146) if (sq_array_size == SIZE_MAX)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8147) return SIZE_MAX;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8148)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8149) if (check_add_overflow(off, sq_array_size, &off))
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8150) return SIZE_MAX;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8151)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8152) return off;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8153) }
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8154)
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8155) static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
7f61a1e9ef511 (Pavel Begunkov 2021-04-11 01:46:35 +0100 8156) {
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8157) struct io_mapped_ubuf *imu = *slot;
7f61a1e9ef511 (Pavel Begunkov 2021-04-11 01:46:35 +0100 8158) unsigned int i;
7f61a1e9ef511 (Pavel Begunkov 2021-04-11 01:46:35 +0100 8159)
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8160) if (imu != ctx->dummy_ubuf) {
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8161) for (i = 0; i < imu->nr_bvecs; i++)
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8162) unpin_user_page(imu->bvec[i].bv_page);
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8163) if (imu->acct_pages)
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8164) io_unaccount_mem(ctx, imu->acct_pages);
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8165) kvfree(imu);
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8166) }
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8167) *slot = NULL;
7f61a1e9ef511 (Pavel Begunkov 2021-04-11 01:46:35 +0100 8168) }
7f61a1e9ef511 (Pavel Begunkov 2021-04-11 01:46:35 +0100 8169)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8170) static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8171) {
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8172) io_buffer_unmap(ctx, &prsrc->buf);
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8173) prsrc->buf = NULL;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8174) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8175)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8176) static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8177) {
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8178) unsigned int i;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8179)
7f61a1e9ef511 (Pavel Begunkov 2021-04-11 01:46:35 +0100 8180) for (i = 0; i < ctx->nr_user_bufs; i++)
7f61a1e9ef511 (Pavel Begunkov 2021-04-11 01:46:35 +0100 8181) io_buffer_unmap(ctx, &ctx->user_bufs[i]);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8182) kfree(ctx->user_bufs);
bb6659cc0ad3c (Zqiang 2021-04-30 16:25:15 +0800 8183) io_rsrc_data_free(ctx->buf_data);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8184) ctx->user_bufs = NULL;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8185) ctx->buf_data = NULL;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8186) ctx->nr_user_bufs = 0;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8187) }
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8188)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8189) static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8190) {
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8191) int ret;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8192)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8193) if (!ctx->buf_data)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8194) return -ENXIO;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8195)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8196) ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8197) if (!ret)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8198) __io_sqe_buffers_unregister(ctx);
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8199) return ret;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8200) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8201)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8202) static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8203) void __user *arg, unsigned index)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8204) {
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8205) struct iovec __user *src;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8206)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8207) #ifdef CONFIG_COMPAT
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8208) if (ctx->compat) {
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8209) struct compat_iovec __user *ciovs;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8210) struct compat_iovec ciov;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8211)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8212) ciovs = (struct compat_iovec __user *) arg;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8213) if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8214) return -EFAULT;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8215)
d55e5f5b70dd6 (Jens Axboe 2019-12-11 16:12:15 -0700 8216) dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8217) dst->iov_len = ciov.iov_len;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8218) return 0;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8219) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8220) #endif
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8221) src = (struct iovec __user *) arg;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8222) if (copy_from_user(dst, &src[index], sizeof(*dst)))
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8223) return -EFAULT;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8224) return 0;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8225) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8226)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8227) /*
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8228) * Not super efficient, but this is just a registration time. And we do cache
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8229) * the last compound head, so generally we'll only do a full search if we don't
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8230) * match that one.
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8231) *
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8232) * We check if the given compound head page has already been accounted, to
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8233) * avoid double accounting it. This allows us to account the full size of the
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8234) * page, not just the constituent pages of a huge page.
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8235) */
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8236) static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8237) int nr_pages, struct page *hpage)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8238) {
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8239) int i, j;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8240)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8241) /* check current page array */
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8242) for (i = 0; i < nr_pages; i++) {
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8243) if (!PageCompound(pages[i]))
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8244) continue;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8245) if (compound_head(pages[i]) == hpage)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8246) return true;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8247) }
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8248)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8249) /* check previously registered pages */
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8250) for (i = 0; i < ctx->nr_user_bufs; i++) {
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8251) struct io_mapped_ubuf *imu = ctx->user_bufs[i];
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8252)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8253) for (j = 0; j < imu->nr_bvecs; j++) {
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8254) if (!PageCompound(imu->bvec[j].bv_page))
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8255) continue;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8256) if (compound_head(imu->bvec[j].bv_page) == hpage)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8257) return true;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8258) }
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8259) }
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8260)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8261) return false;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8262) }
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8263)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8264) static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8265) int nr_pages, struct io_mapped_ubuf *imu,
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8266) struct page **last_hpage)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8267) {
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8268) int i, ret;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8269)
216e5835966a7 (Pavel Begunkov 2021-05-29 12:01:02 +0100 8270) imu->acct_pages = 0;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8271) for (i = 0; i < nr_pages; i++) {
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8272) if (!PageCompound(pages[i])) {
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8273) imu->acct_pages++;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8274) } else {
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8275) struct page *hpage;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8276)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8277) hpage = compound_head(pages[i]);
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8278) if (hpage == *last_hpage)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8279) continue;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8280) *last_hpage = hpage;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8281) if (headpage_already_acct(ctx, pages, i, hpage))
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8282) continue;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8283) imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8284) }
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8285) }
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8286)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8287) if (!imu->acct_pages)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8288) return 0;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8289)
26bfa89e25f42 (Jens Axboe 2021-02-09 20:14:12 -0700 8290) ret = io_account_mem(ctx, imu->acct_pages);
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8291) if (ret)
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8292) imu->acct_pages = 0;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8293) return ret;
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8294) }
de2939388be56 (Jens Axboe 2020-09-17 16:19:16 -0600 8295)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8296) static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8297) struct io_mapped_ubuf **pimu,
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8298) struct page **last_hpage)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8299) {
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8300) struct io_mapped_ubuf *imu = NULL;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8301) struct vm_area_struct **vmas = NULL;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8302) struct page **pages = NULL;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8303) unsigned long off, start, end, ubuf;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8304) size_t size;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8305) int ret, pret, nr_pages, i;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8306)
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8307) if (!iov->iov_base) {
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8308) *pimu = ctx->dummy_ubuf;
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8309) return 0;
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8310) }
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8311)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8312) ubuf = (unsigned long) iov->iov_base;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8313) end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8314) start = ubuf >> PAGE_SHIFT;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8315) nr_pages = end - start;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8316)
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8317) *pimu = NULL;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8318) ret = -ENOMEM;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8319)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8320) pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8321) if (!pages)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8322) goto done;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8323)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8324) vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8325) GFP_KERNEL);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8326) if (!vmas)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8327) goto done;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8328)
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8329) imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
a2b4198cab7e3 (Pavel Begunkov 2021-04-26 00:16:31 +0100 8330) if (!imu)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8331) goto done;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8332)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8333) ret = 0;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8334) mmap_read_lock(current->mm);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8335) pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8336) pages, vmas);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8337) if (pret == nr_pages) {
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8338) /* don't support file backed memory */
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8339) for (i = 0; i < nr_pages; i++) {
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8340) struct vm_area_struct *vma = vmas[i];
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8341)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8342) if (vma->vm_file &&
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8343) !is_file_hugepages(vma->vm_file)) {
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8344) ret = -EOPNOTSUPP;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8345) break;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8346) }
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8347) }
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8348) } else {
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8349) ret = pret < 0 ? pret : -EFAULT;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8350) }
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8351) mmap_read_unlock(current->mm);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8352) if (ret) {
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8353) /*
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8354) * if we did partial map, or found file backed vmas,
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8355) * release any pages we did get
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8356) */
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8357) if (pret > 0)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8358) unpin_user_pages(pages, pret);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8359) goto done;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8360) }
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8361)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8362) ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8363) if (ret) {
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8364) unpin_user_pages(pages, pret);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8365) goto done;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8366) }
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8367)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8368) off = ubuf & ~PAGE_MASK;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8369) size = iov->iov_len;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8370) for (i = 0; i < nr_pages; i++) {
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8371) size_t vec_len;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8372)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8373) vec_len = min_t(size_t, size, PAGE_SIZE - off);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8374) imu->bvec[i].bv_page = pages[i];
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8375) imu->bvec[i].bv_len = vec_len;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8376) imu->bvec[i].bv_offset = off;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8377) off = 0;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8378) size -= vec_len;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8379) }
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8380) /* store original address for later verification */
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8381) imu->ubuf = ubuf;
4751f53d74a68 (Pavel Begunkov 2021-04-01 15:43:55 +0100 8382) imu->ubuf_end = ubuf + iov->iov_len;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8383) imu->nr_bvecs = nr_pages;
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8384) *pimu = imu;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8385) ret = 0;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8386) done:
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8387) if (ret)
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8388) kvfree(imu);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8389) kvfree(pages);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8390) kvfree(vmas);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8391) return ret;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8392) }
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8393)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8394) static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8395) {
87094465d01a2 (Pavel Begunkov 2021-04-11 01:46:36 +0100 8396) ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
87094465d01a2 (Pavel Begunkov 2021-04-11 01:46:36 +0100 8397) return ctx->user_bufs ? 0 : -ENOMEM;
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8398) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8399)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8400) static int io_buffer_validate(struct iovec *iov)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8401) {
50e96989d736b (Pavel Begunkov 2021-03-24 22:59:01 +0000 8402) unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
50e96989d736b (Pavel Begunkov 2021-03-24 22:59:01 +0000 8403)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8404) /*
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8405) * Don't impose further limits on the size and buffer
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8406) * constraints here, we'll -EINVAL later when IO is
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8407) * submitted if they are wrong.
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8408) */
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8409) if (!iov->iov_base)
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8410) return iov->iov_len ? -EFAULT : 0;
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8411) if (!iov->iov_len)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8412) return -EFAULT;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8413)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8414) /* arbitrary limit, but we need something */
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8415) if (iov->iov_len > SZ_1G)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8416) return -EFAULT;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8417)
50e96989d736b (Pavel Begunkov 2021-03-24 22:59:01 +0000 8418) if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
50e96989d736b (Pavel Begunkov 2021-03-24 22:59:01 +0000 8419) return -EOVERFLOW;
50e96989d736b (Pavel Begunkov 2021-03-24 22:59:01 +0000 8420)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8421) return 0;
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8422) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8423)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8424) static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8425) unsigned int nr_args, u64 __user *tags)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8426) {
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8427) struct page *last_hpage = NULL;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8428) struct io_rsrc_data *data;
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8429) int i, ret;
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8430) struct iovec iov;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8431)
87094465d01a2 (Pavel Begunkov 2021-04-11 01:46:36 +0100 8432) if (ctx->user_bufs)
87094465d01a2 (Pavel Begunkov 2021-04-11 01:46:36 +0100 8433) return -EBUSY;
489809e2e22b3 (Pavel Begunkov 2021-05-14 12:06:44 +0100 8434) if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
87094465d01a2 (Pavel Begunkov 2021-04-11 01:46:36 +0100 8435) return -EINVAL;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8436) ret = io_rsrc_node_switch_start(ctx);
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8437) if (ret)
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8438) return ret;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8439) data = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, nr_args);
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8440) if (!data)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8441) return -ENOMEM;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8442) ret = io_buffers_map_alloc(ctx, nr_args);
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8443) if (ret) {
bb6659cc0ad3c (Zqiang 2021-04-30 16:25:15 +0800 8444) io_rsrc_data_free(data);
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8445) return ret;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8446) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8447)
87094465d01a2 (Pavel Begunkov 2021-04-11 01:46:36 +0100 8448) for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8449) u64 tag = 0;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8450)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8451) if (tags && copy_from_user(&tag, &tags[i], sizeof(tag))) {
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8452) ret = -EFAULT;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8453) break;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8454) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8455) ret = io_copy_iov(ctx, &iov, arg, i);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8456) if (ret)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8457) break;
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8458) ret = io_buffer_validate(&iov);
2b358604aa6e8 (Bijan Mottahedeh 2021-01-06 12:39:11 -0800 8459) if (ret)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8460) break;
cf3770e78421f (Colin Ian King 2021-04-29 11:46:02 +0100 8461) if (!iov.iov_base && tag) {
cf3770e78421f (Colin Ian King 2021-04-29 11:46:02 +0100 8462) ret = -EINVAL;
cf3770e78421f (Colin Ian King 2021-04-29 11:46:02 +0100 8463) break;
cf3770e78421f (Colin Ian King 2021-04-29 11:46:02 +0100 8464) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8465)
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8466) ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 8467) &last_hpage);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8468) if (ret)
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8469) break;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8470) data->tags[i] = tag;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8471) }
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8472)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8473) WARN_ON_ONCE(ctx->buf_data);
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 8474)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8475) ctx->buf_data = data;
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8476) if (ret)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8477) __io_sqe_buffers_unregister(ctx);
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8478) else
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8479) io_rsrc_node_switch(ctx, NULL);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8480) return ret;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8481) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 8482)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8483) static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8484) struct io_uring_rsrc_update2 *up,
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8485) unsigned int nr_args)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8486) {
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8487) u64 __user *tags = u64_to_user_ptr(up->tags);
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8488) struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8489) struct page *last_hpage = NULL;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8490) bool needs_switch = false;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8491) __u32 done;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8492) int i, err;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8493)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8494) if (!ctx->buf_data)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8495) return -ENXIO;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8496) if (up->offset + nr_args > ctx->nr_user_bufs)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8497) return -EINVAL;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8498)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8499) for (done = 0; done < nr_args; done++) {
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8500) struct io_mapped_ubuf *imu;
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8501) int offset = up->offset + done;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8502) u64 tag = 0;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8503)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8504) err = io_copy_iov(ctx, &iov, iovs, done);
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8505) if (err)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8506) break;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8507) if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8508) err = -EFAULT;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8509) break;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8510) }
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8511) err = io_buffer_validate(&iov);
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8512) if (err)
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8513) break;
cf3770e78421f (Colin Ian King 2021-04-29 11:46:02 +0100 8514) if (!iov.iov_base && tag) {
cf3770e78421f (Colin Ian King 2021-04-29 11:46:02 +0100 8515) err = -EINVAL;
cf3770e78421f (Colin Ian King 2021-04-29 11:46:02 +0100 8516) break;
cf3770e78421f (Colin Ian King 2021-04-29 11:46:02 +0100 8517) }
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8518) err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8519) if (err)
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8520) break;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8521)
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8522) i = array_index_nospec(offset, ctx->nr_user_bufs);
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8523) if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8524) err = io_queue_rsrc_removal(ctx->buf_data, offset,
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8525) ctx->rsrc_node, ctx->user_bufs[i]);
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8526) if (unlikely(err)) {
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8527) io_buffer_unmap(ctx, &imu);
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8528) break;
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8529) }
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8530) ctx->user_bufs[i] = NULL;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8531) needs_switch = true;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8532) }
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8533)
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8534) ctx->user_bufs[i] = imu;
0b8c0e7c9692c (Pavel Begunkov 2021-04-26 15:17:38 +0100 8535) ctx->buf_data->tags[offset] = tag;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8536) }
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8537)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8538) if (needs_switch)
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8539) io_rsrc_node_switch(ctx, ctx->buf_data);
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8540) return done ? done : err;
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8541) }
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 8542)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8543) static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8544) {
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8545) __s32 __user *fds = arg;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8546) int fd;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8547)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8548) if (ctx->cq_ev_fd)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8549) return -EBUSY;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8550)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8551) if (copy_from_user(&fd, fds, sizeof(*fds)))
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8552) return -EFAULT;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8553)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8554) ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8555) if (IS_ERR(ctx->cq_ev_fd)) {
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8556) int ret = PTR_ERR(ctx->cq_ev_fd);
501449420a42c (Pavel Begunkov 2021-06-24 15:09:57 +0100 8557)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8558) ctx->cq_ev_fd = NULL;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8559) return ret;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8560) }
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8561)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8562) return 0;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8563) }
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8564)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8565) static int io_eventfd_unregister(struct io_ring_ctx *ctx)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8566) {
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8567) if (ctx->cq_ev_fd) {
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8568) eventfd_ctx_put(ctx->cq_ev_fd);
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8569) ctx->cq_ev_fd = NULL;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8570) return 0;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8571) }
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8572)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8573) return -ENXIO;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8574) }
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8575)
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 8576) static void io_destroy_buffers(struct io_ring_ctx *ctx)
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 8577) {
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 8578) struct io_buffer *buf;
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 8579) unsigned long index;
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 8580)
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 8581) xa_for_each(&ctx->io_buffers, index, buf)
9e15c3a0ced5a (Jens Axboe 2021-03-13 12:29:43 -0700 8582) __io_remove_buffers(ctx, buf, index, -1U);
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 8583) }
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 8584)
68e68ee6e3593 (Jens Axboe 2021-02-13 09:00:02 -0700 8585) static void io_req_cache_free(struct list_head *list, struct task_struct *tsk)
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 8586) {
68e68ee6e3593 (Jens Axboe 2021-02-13 09:00:02 -0700 8587) struct io_kiocb *req, *nxt;
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 8588)
68e68ee6e3593 (Jens Axboe 2021-02-13 09:00:02 -0700 8589) list_for_each_entry_safe(req, nxt, list, compl.list) {
68e68ee6e3593 (Jens Axboe 2021-02-13 09:00:02 -0700 8590) if (tsk && req->task != tsk)
68e68ee6e3593 (Jens Axboe 2021-02-13 09:00:02 -0700 8591) continue;
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 8592) list_del(&req->compl.list);
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 8593) kmem_cache_free(req_cachep, req);
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 8594) }
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 8595) }
1b4c351f6eb74 (Jens Axboe 2021-02-10 00:03:19 +0000 8596)
4010fec41fd9f (Jens Axboe 2021-02-27 15:04:18 -0700 8597) static void io_req_caches_free(struct io_ring_ctx *ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8598) {
bf019da7fcbe7 (Pavel Begunkov 2021-02-10 00:03:17 +0000 8599) struct io_submit_state *submit_state = &ctx->submit_state;
e5547d2c5eb36 (Pavel Begunkov 2021-02-23 22:17:20 +0000 8600) struct io_comp_state *cs = &ctx->submit_state.comp;
bf019da7fcbe7 (Pavel Begunkov 2021-02-10 00:03:17 +0000 8601)
9a4fdbd8ee0d8 (Jens Axboe 2021-02-13 09:09:44 -0700 8602) mutex_lock(&ctx->uring_lock);
9a4fdbd8ee0d8 (Jens Axboe 2021-02-13 09:09:44 -0700 8603)
8e5c66c485a8a (Pavel Begunkov 2021-02-22 11:45:55 +0000 8604) if (submit_state->free_reqs) {
9a4fdbd8ee0d8 (Jens Axboe 2021-02-13 09:09:44 -0700 8605) kmem_cache_free_bulk(req_cachep, submit_state->free_reqs,
9a4fdbd8ee0d8 (Jens Axboe 2021-02-13 09:09:44 -0700 8606) submit_state->reqs);
8e5c66c485a8a (Pavel Begunkov 2021-02-22 11:45:55 +0000 8607) submit_state->free_reqs = 0;
8e5c66c485a8a (Pavel Begunkov 2021-02-22 11:45:55 +0000 8608) }
9a4fdbd8ee0d8 (Jens Axboe 2021-02-13 09:09:44 -0700 8609)
dac7a09864938 (Pavel Begunkov 2021-03-19 17:22:39 +0000 8610) io_flush_cached_locked_reqs(ctx, cs);
e5547d2c5eb36 (Pavel Begunkov 2021-02-23 22:17:20 +0000 8611) io_req_cache_free(&cs->free_list, NULL);
9a4fdbd8ee0d8 (Jens Axboe 2021-02-13 09:09:44 -0700 8612) mutex_unlock(&ctx->uring_lock);
9a4fdbd8ee0d8 (Jens Axboe 2021-02-13 09:09:44 -0700 8613) }
9a4fdbd8ee0d8 (Jens Axboe 2021-02-13 09:09:44 -0700 8614)
3e35c7dbf7064 (Pavel Begunkov 2021-08-10 02:44:23 +0100 8615) static void io_wait_rsrc_data(struct io_rsrc_data *data)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8616) {
3e35c7dbf7064 (Pavel Begunkov 2021-08-10 02:44:23 +0100 8617) if (data && !atomic_dec_and_test(&data->refs))
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8618) wait_for_completion(&data->done);
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8619) }
04fc6c802dfac (Pavel Begunkov 2021-02-12 03:23:54 +0000 8620)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8621) static void io_ring_ctx_free(struct io_ring_ctx *ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8622) {
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 8623) io_sq_thread_finish(ctx);
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 8624)
37d1e2e3642e2 (Jens Axboe 2021-02-17 21:03:43 -0700 8625) if (ctx->mm_account) {
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 8626) mmdrop(ctx->mm_account);
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 8627) ctx->mm_account = NULL;
309758254ea62 (Bijan Mottahedeh 2020-06-16 16:36:09 -0700 8628) }
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 8629)
3e35c7dbf7064 (Pavel Begunkov 2021-08-10 02:44:23 +0100 8630) /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
3e35c7dbf7064 (Pavel Begunkov 2021-08-10 02:44:23 +0100 8631) io_wait_rsrc_data(ctx->buf_data);
3e35c7dbf7064 (Pavel Begunkov 2021-08-10 02:44:23 +0100 8632) io_wait_rsrc_data(ctx->file_data);
3e35c7dbf7064 (Pavel Begunkov 2021-08-10 02:44:23 +0100 8633)
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 8634) mutex_lock(&ctx->uring_lock);
3e35c7dbf7064 (Pavel Begunkov 2021-08-10 02:44:23 +0100 8635) if (ctx->buf_data)
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 8636) __io_sqe_buffers_unregister(ctx);
3e35c7dbf7064 (Pavel Begunkov 2021-08-10 02:44:23 +0100 8637) if (ctx->file_data)
084804002e512 (Pavel Begunkov 2021-04-13 02:58:38 +0100 8638) __io_sqe_files_unregister(ctx);
c4ea060e85eab (Pavel Begunkov 2021-04-01 15:43:58 +0100 8639) if (ctx->rings)
c4ea060e85eab (Pavel Begunkov 2021-04-01 15:43:58 +0100 8640) __io_cqring_overflow_flush(ctx, true);
8bad28d8a305b (Hao Xu 2021-02-19 17:19:36 +0800 8641) mutex_unlock(&ctx->uring_lock);
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 8642) io_eventfd_unregister(ctx);
5a2e745d4d430 (Jens Axboe 2020-02-23 16:23:11 -0700 8643) io_destroy_buffers(ctx);
07db298a1c96b (Pavel Begunkov 2021-04-20 12:03:32 +0100 8644) if (ctx->sq_creds)
07db298a1c96b (Pavel Begunkov 2021-04-20 12:03:32 +0100 8645) put_cred(ctx->sq_creds);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 8646)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 8647) /* there are no registered resources left, nobody uses it */
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 8648) if (ctx->rsrc_node)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 8649) io_rsrc_node_destroy(ctx->rsrc_node);
8dd03afe611d3 (Pavel Begunkov 2021-03-19 17:22:36 +0000 8650) if (ctx->rsrc_backup_node)
b895c9a632e70 (Pavel Begunkov 2021-04-01 15:43:40 +0100 8651) io_rsrc_node_destroy(ctx->rsrc_backup_node);
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 8652) flush_delayed_work(&ctx->rsrc_put_work);
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 8653)
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 8654) WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
a7f0ed5acdc9c (Pavel Begunkov 2021-04-01 15:43:46 +0100 8655) WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist));
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 8656)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8657) #if defined(CONFIG_UNIX)
355e8d26f719c (Eric Biggers 2019-06-12 14:58:43 -0700 8658) if (ctx->ring_sock) {
355e8d26f719c (Eric Biggers 2019-06-12 14:58:43 -0700 8659) ctx->ring_sock->file = NULL; /* so that iput() is called */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8660) sock_release(ctx->ring_sock);
355e8d26f719c (Eric Biggers 2019-06-12 14:58:43 -0700 8661) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8662) #endif
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8663)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 8664) io_mem_free(ctx->rings);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8665) io_mem_free(ctx->sq_sqes);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8666)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8667) percpu_ref_exit(&ctx->refs);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8668) free_uid(ctx->user);
4010fec41fd9f (Jens Axboe 2021-02-27 15:04:18 -0700 8669) io_req_caches_free(ctx);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 8670) if (ctx->hash_map)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 8671) io_wq_put_hash(ctx->hash_map);
78076bb64aa8b (Jens Axboe 2019-12-04 19:56:40 -0700 8672) kfree(ctx->cancel_hash);
6224843d56e0c (Pavel Begunkov 2021-04-28 13:11:29 +0100 8673) kfree(ctx->dummy_ubuf);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8674) kfree(ctx);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8675) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8676)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8677) static __poll_t io_uring_poll(struct file *file, poll_table *wait)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8678) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8679) struct io_ring_ctx *ctx = file->private_data;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8680) __poll_t mask = 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8681)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8682) poll_wait(file, &ctx->cq_wait, wait);
4f7067c3fb7f2 (Stefan Bühler 2019-04-24 23:54:17 +0200 8683) /*
4f7067c3fb7f2 (Stefan Bühler 2019-04-24 23:54:17 +0200 8684) * synchronizes with barrier from wq_has_sleeper call in
4f7067c3fb7f2 (Stefan Bühler 2019-04-24 23:54:17 +0200 8685) * io_commit_cqring
4f7067c3fb7f2 (Stefan Bühler 2019-04-24 23:54:17 +0200 8686) */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8687) smp_rmb();
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 8688) if (!io_sqring_full(ctx))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8689) mask |= EPOLLOUT | EPOLLWRNORM;
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8690)
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8691) /*
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8692) * Don't flush cqring overflow list here, just do a simple check.
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8693) * Otherwise there could possible be ABBA deadlock:
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8694) * CPU0 CPU1
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8695) * ---- ----
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8696) * lock(&ctx->uring_lock);
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8697) * lock(&ep->mtx);
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8698) * lock(&ctx->uring_lock);
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8699) * lock(&ep->mtx);
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8700) *
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8701) * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8702) * pushs them to do the flush.
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8703) */
ed670c3f90a67 (Hao Xu 2021-02-05 16:34:21 +0800 8704) if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8705) mask |= EPOLLIN | EPOLLRDNORM;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8706)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8707) return mask;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8708) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8709)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8710) static int io_uring_fasync(int fd, struct file *file, int on)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8711) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8712) struct io_ring_ctx *ctx = file->private_data;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8713)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8714) return fasync_helper(fd, file, on, &ctx->cq_fasync);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8715) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8716)
0bead8cd39b9c (Yejune Deng 2020-12-24 11:02:20 +0800 8717) static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 8718) {
4379bf8bd70b5 (Jens Axboe 2021-02-15 13:40:22 -0700 8719) const struct cred *creds;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 8720)
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 8721) creds = xa_erase(&ctx->personalities, id);
4379bf8bd70b5 (Jens Axboe 2021-02-15 13:40:22 -0700 8722) if (creds) {
4379bf8bd70b5 (Jens Axboe 2021-02-15 13:40:22 -0700 8723) put_cred(creds);
0bead8cd39b9c (Yejune Deng 2020-12-24 11:02:20 +0800 8724) return 0;
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 8725) }
0bead8cd39b9c (Yejune Deng 2020-12-24 11:02:20 +0800 8726)
0bead8cd39b9c (Yejune Deng 2020-12-24 11:02:20 +0800 8727) return -EINVAL;
0bead8cd39b9c (Yejune Deng 2020-12-24 11:02:20 +0800 8728) }
0bead8cd39b9c (Yejune Deng 2020-12-24 11:02:20 +0800 8729)
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 8730) static inline bool io_run_ctx_fallback(struct io_ring_ctx *ctx)
7c25c0d16ef3c (Jens Axboe 2021-02-16 07:17:00 -0700 8731) {
9b46571142e47 (Pavel Begunkov 2021-03-15 14:23:07 +0000 8732) return io_run_task_work_head(&ctx->exit_task_work);
7c25c0d16ef3c (Jens Axboe 2021-02-16 07:17:00 -0700 8733) }
7c25c0d16ef3c (Jens Axboe 2021-02-16 07:17:00 -0700 8734)
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8735) struct io_tctx_exit {
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8736) struct callback_head task_work;
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8737) struct completion completion;
baf186c4d345f (Pavel Begunkov 2021-03-06 11:02:15 +0000 8738) struct io_ring_ctx *ctx;
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8739) };
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8740)
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8741) static void io_tctx_exit_cb(struct callback_head *cb)
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8742) {
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8743) struct io_uring_task *tctx = current->io_uring;
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8744) struct io_tctx_exit *work;
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8745)
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8746) work = container_of(cb, struct io_tctx_exit, task_work);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8747) /*
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8748) * When @in_idle, we're in cancellation and it's racy to remove the
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8749) * node. It'll be removed by the end of cancellation, just ignore it.
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8750) */
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8751) if (!atomic_read(&tctx->in_idle))
baf186c4d345f (Pavel Begunkov 2021-03-06 11:02:15 +0000 8752) io_uring_del_task_file((unsigned long)work->ctx);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8753) complete(&work->completion);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8754) }
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8755)
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8756) static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8757) {
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8758) struct io_kiocb *req = container_of(work, struct io_kiocb, work);
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8759)
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8760) return req->ctx == data;
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8761) }
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8762)
85faa7b8346eb (Jens Axboe 2020-04-09 18:14:00 -0600 8763) static void io_ring_exit_work(struct work_struct *work)
85faa7b8346eb (Jens Axboe 2020-04-09 18:14:00 -0600 8764) {
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8765) struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
b5bb3a24f69da (Pavel Begunkov 2021-03-06 11:02:16 +0000 8766) unsigned long timeout = jiffies + HZ * 60 * 5;
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8767) struct io_tctx_exit exit;
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8768) struct io_tctx_node *node;
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8769) int ret;
85faa7b8346eb (Jens Axboe 2020-04-09 18:14:00 -0600 8770)
56952e91acc93 (Jens Axboe 2020-06-17 15:00:04 -0600 8771) /*
56952e91acc93 (Jens Axboe 2020-06-17 15:00:04 -0600 8772) * If we're doing polled IO and end up having requests being
56952e91acc93 (Jens Axboe 2020-06-17 15:00:04 -0600 8773) * submitted async (out-of-line), then completions can come in while
56952e91acc93 (Jens Axboe 2020-06-17 15:00:04 -0600 8774) * we're waiting for refs to drop. We need to reap these manually,
56952e91acc93 (Jens Axboe 2020-06-17 15:00:04 -0600 8775) * as nobody else will be looking for them.
56952e91acc93 (Jens Axboe 2020-06-17 15:00:04 -0600 8776) */
b2edc0a77fac1 (Pavel Begunkov 2020-07-07 16:36:22 +0300 8777) do {
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8778) io_uring_try_cancel_requests(ctx, NULL, true);
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8779) if (ctx->sq_data) {
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8780) struct io_sq_data *sqd = ctx->sq_data;
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8781) struct task_struct *tsk;
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8782)
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8783) io_sq_thread_park(sqd);
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8784) tsk = sqd->thread;
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8785) if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8786) io_wq_cancel_cb(tsk->io_uring->io_wq,
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8787) io_cancel_ctx_cb, ctx, true);
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8788) io_sq_thread_unpark(sqd);
28090c133869b (Pavel Begunkov 2021-04-25 23:34:45 +0100 8789) }
b5bb3a24f69da (Pavel Begunkov 2021-03-06 11:02:16 +0000 8790)
b5bb3a24f69da (Pavel Begunkov 2021-03-06 11:02:16 +0000 8791) WARN_ON_ONCE(time_after(jiffies, timeout));
b2edc0a77fac1 (Pavel Begunkov 2020-07-07 16:36:22 +0300 8792) } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8793)
7f00651aebc9a (Pavel Begunkov 2021-04-14 13:38:34 +0100 8794) init_completion(&exit.completion);
7f00651aebc9a (Pavel Begunkov 2021-04-14 13:38:34 +0100 8795) init_task_work(&exit.task_work, io_tctx_exit_cb);
7f00651aebc9a (Pavel Begunkov 2021-04-14 13:38:34 +0100 8796) exit.ctx = ctx;
89b5066ea1d96 (Pavel Begunkov 2021-04-01 15:43:50 +0100 8797) /*
89b5066ea1d96 (Pavel Begunkov 2021-04-01 15:43:50 +0100 8798) * Some may use context even when all refs and requests have been put,
89b5066ea1d96 (Pavel Begunkov 2021-04-01 15:43:50 +0100 8799) * and they are free to do so while still holding uring_lock or
89b5066ea1d96 (Pavel Begunkov 2021-04-01 15:43:50 +0100 8800) * completion_lock, see __io_req_task_submit(). Apart from other work,
89b5066ea1d96 (Pavel Begunkov 2021-04-01 15:43:50 +0100 8801) * this lock/unlock section also waits them to finish.
89b5066ea1d96 (Pavel Begunkov 2021-04-01 15:43:50 +0100 8802) */
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8803) mutex_lock(&ctx->uring_lock);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8804) while (!list_empty(&ctx->tctx_list)) {
b5bb3a24f69da (Pavel Begunkov 2021-03-06 11:02:16 +0000 8805) WARN_ON_ONCE(time_after(jiffies, timeout));
b5bb3a24f69da (Pavel Begunkov 2021-03-06 11:02:16 +0000 8806)
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8807) node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8808) ctx_node);
7f00651aebc9a (Pavel Begunkov 2021-04-14 13:38:34 +0100 8809) /* don't spin on a single task if cancellation failed */
7f00651aebc9a (Pavel Begunkov 2021-04-14 13:38:34 +0100 8810) list_rotate_left(&ctx->tctx_list);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8811) ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8812) if (WARN_ON_ONCE(ret))
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8813) continue;
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8814) wake_up_process(node->task);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8815)
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8816) mutex_unlock(&ctx->uring_lock);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8817) wait_for_completion(&exit.completion);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8818) mutex_lock(&ctx->uring_lock);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8819) }
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8820) mutex_unlock(&ctx->uring_lock);
89b5066ea1d96 (Pavel Begunkov 2021-04-01 15:43:50 +0100 8821) spin_lock_irq(&ctx->completion_lock);
89b5066ea1d96 (Pavel Begunkov 2021-04-01 15:43:50 +0100 8822) spin_unlock_irq(&ctx->completion_lock);
d56d938b4bef3 (Pavel Begunkov 2021-03-06 11:02:13 +0000 8823)
85faa7b8346eb (Jens Axboe 2020-04-09 18:14:00 -0600 8824) io_ring_ctx_free(ctx);
85faa7b8346eb (Jens Axboe 2020-04-09 18:14:00 -0600 8825) }
85faa7b8346eb (Jens Axboe 2020-04-09 18:14:00 -0600 8826)
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8827) /* Returns true if we found and killed one or more timeouts */
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8828) static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8829) bool cancel_all)
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8830) {
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8831) struct io_kiocb *req, *tmp;
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8832) int canceled = 0;
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8833)
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8834) spin_lock_irq(&ctx->completion_lock);
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8835) list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8836) if (io_match_task(req, tsk, cancel_all)) {
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8837) io_kill_timeout(req, -ECANCELED);
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8838) canceled++;
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8839) }
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8840) }
51520426f4bc3 (Pavel Begunkov 2021-03-29 11:39:29 +0100 8841) if (canceled != 0)
51520426f4bc3 (Pavel Begunkov 2021-03-29 11:39:29 +0100 8842) io_commit_cqring(ctx);
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8843) spin_unlock_irq(&ctx->completion_lock);
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8844) if (canceled != 0)
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8845) io_cqring_ev_posted(ctx);
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8846) return canceled != 0;
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8847) }
80c4cbdb5ee60 (Pavel Begunkov 2021-03-25 18:32:43 +0000 8848)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8849) static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8850) {
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 8851) unsigned long index;
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 8852) struct creds *creds;
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 8853)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8854) mutex_lock(&ctx->uring_lock);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8855) percpu_ref_kill(&ctx->refs);
634578f800652 (Pavel Begunkov 2020-12-06 22:22:44 +0000 8856) if (ctx->rings)
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 8857) __io_cqring_overflow_flush(ctx, true);
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 8858) xa_for_each(&ctx->personalities, index, creds)
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 8859) io_unregister_personality(ctx, index);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8860) mutex_unlock(&ctx->uring_lock);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8861)
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8862) io_kill_timeouts(ctx, NULL, true);
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8863) io_poll_remove_all(ctx, NULL, true);
561fb04a6a225 (Jens Axboe 2019-10-24 07:25:42 -0600 8864)
15dff286d0e00 (Jens Axboe 2019-11-13 09:09:23 -0700 8865) /* if we failed setting up the ctx, we might not have any rings */
b2edc0a77fac1 (Pavel Begunkov 2020-07-07 16:36:22 +0300 8866) io_iopoll_try_reap_events(ctx);
309fc03a3284a (Jens Axboe 2020-07-10 09:13:34 -0600 8867)
85faa7b8346eb (Jens Axboe 2020-04-09 18:14:00 -0600 8868) INIT_WORK(&ctx->exit_work, io_ring_exit_work);
fc666777da9da (Jens Axboe 2020-08-19 11:10:51 -0600 8869) /*
fc666777da9da (Jens Axboe 2020-08-19 11:10:51 -0600 8870) * Use system_unbound_wq to avoid spawning tons of event kworkers
fc666777da9da (Jens Axboe 2020-08-19 11:10:51 -0600 8871) * if we're exiting a ton of rings at the same time. It just adds
fc666777da9da (Jens Axboe 2020-08-19 11:10:51 -0600 8872) * noise and overhead, there's no discernable change in runtime
fc666777da9da (Jens Axboe 2020-08-19 11:10:51 -0600 8873) * over using system_wq.
fc666777da9da (Jens Axboe 2020-08-19 11:10:51 -0600 8874) */
fc666777da9da (Jens Axboe 2020-08-19 11:10:51 -0600 8875) queue_work(system_unbound_wq, &ctx->exit_work);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8876) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8877)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8878) static int io_uring_release(struct inode *inode, struct file *file)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8879) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8880) struct io_ring_ctx *ctx = file->private_data;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8881)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8882) file->private_data = NULL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8883) io_ring_ctx_wait_and_kill(ctx);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8884) return 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8885) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 8886)
f6edbabb83597 (Pavel Begunkov 2020-11-06 13:00:26 +0000 8887) struct io_task_cancel {
f6edbabb83597 (Pavel Begunkov 2020-11-06 13:00:26 +0000 8888) struct task_struct *task;
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8889) bool all;
f6edbabb83597 (Pavel Begunkov 2020-11-06 13:00:26 +0000 8890) };
f254ac04c8744 (Jens Axboe 2020-08-12 17:33:30 -0600 8891)
f6edbabb83597 (Pavel Begunkov 2020-11-06 13:00:26 +0000 8892) static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
b711d4eaf0c40 (Jens Axboe 2020-08-16 08:23:05 -0700 8893) {
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8894) struct io_kiocb *req = container_of(work, struct io_kiocb, work);
f6edbabb83597 (Pavel Begunkov 2020-11-06 13:00:26 +0000 8895) struct io_task_cancel *cancel = data;
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8896) bool ret;
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8897)
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8898) if (!cancel->all && (req->flags & REQ_F_LINK_TIMEOUT)) {
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8899) unsigned long flags;
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8900) struct io_ring_ctx *ctx = req->ctx;
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8901)
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8902) /* protect against races with linked timeouts */
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8903) spin_lock_irqsave(&ctx->completion_lock, flags);
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8904) ret = io_match_task(req, cancel->task, cancel->all);
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8905) spin_unlock_irqrestore(&ctx->completion_lock, flags);
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8906) } else {
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8907) ret = io_match_task(req, cancel->task, cancel->all);
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8908) }
9a472ef7a3690 (Pavel Begunkov 2020-11-05 22:31:37 +0000 8909) return ret;
b711d4eaf0c40 (Jens Axboe 2020-08-16 08:23:05 -0700 8910) }
b711d4eaf0c40 (Jens Axboe 2020-08-16 08:23:05 -0700 8911)
e1915f76a8981 (Pavel Begunkov 2021-03-11 23:29:35 +0000 8912) static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8913) struct task_struct *task, bool cancel_all)
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8914) {
e1915f76a8981 (Pavel Begunkov 2021-03-11 23:29:35 +0000 8915) struct io_defer_entry *de;
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8916) LIST_HEAD(list);
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8917)
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8918) spin_lock_irq(&ctx->completion_lock);
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8919) list_for_each_entry_reverse(de, &ctx->defer_list, list) {
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8920) if (io_match_task(de->req, task, cancel_all)) {
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8921) list_cut_position(&list, &ctx->defer_list, &de->list);
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8922) break;
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8923) }
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8924) }
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8925) spin_unlock_irq(&ctx->completion_lock);
e1915f76a8981 (Pavel Begunkov 2021-03-11 23:29:35 +0000 8926) if (list_empty(&list))
e1915f76a8981 (Pavel Begunkov 2021-03-11 23:29:35 +0000 8927) return false;
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8928)
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8929) while (!list_empty(&list)) {
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8930) de = list_first_entry(&list, struct io_defer_entry, list);
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8931) list_del_init(&de->list);
f41db2732d483 (Pavel Begunkov 2021-02-28 22:35:12 +0000 8932) io_req_complete_failed(de->req, -ECANCELED);
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8933) kfree(de);
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8934) }
e1915f76a8981 (Pavel Begunkov 2021-03-11 23:29:35 +0000 8935) return true;
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8936) }
b7ddce3cbf010 (Pavel Begunkov 2020-09-06 00:45:14 +0300 8937)
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8938) static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8939) {
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8940) struct io_tctx_node *node;
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8941) enum io_wq_cancel cret;
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8942) bool ret = false;
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8943)
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8944) mutex_lock(&ctx->uring_lock);
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8945) list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8946) struct io_uring_task *tctx = node->task->io_uring;
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8947)
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8948) /*
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8949) * io_wq will stay alive while we hold uring_lock, because it's
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8950) * killed after ctx nodes, which requires to take the lock.
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8951) */
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8952) if (!tctx || !tctx->io_wq)
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8953) continue;
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8954) cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8955) ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8956) }
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8957) mutex_unlock(&ctx->uring_lock);
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8958)
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8959) return ret;
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8960) }
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8961)
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8962) static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8963) struct task_struct *task,
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8964) bool cancel_all)
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8965) {
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8966) struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8967) struct io_uring_task *tctx = task ? task->io_uring : NULL;
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8968)
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8969) while (1) {
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8970) enum io_wq_cancel cret;
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8971) bool ret = false;
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8972)
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8973) if (!task) {
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8974) ret |= io_uring_try_cancel_iowq(ctx);
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8975) } else if (tctx && tctx->io_wq) {
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8976) /*
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8977) * Cancels requests of all rings, not only @ctx, but
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8978) * it's fine as the task is in exit/exec.
1b00764f09b69 (Pavel Begunkov 2021-03-06 11:02:17 +0000 8979) */
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 8980) cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8981) &cancel, true);
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8982) ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8983) }
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8984)
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8985) /* SQPOLL thread does its own polling */
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8986) if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
d052d1d685f51 (Jens Axboe 2021-03-11 10:49:20 -0700 8987) (ctx->sq_data && ctx->sq_data->thread == current)) {
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8988) while (!list_empty_careful(&ctx->iopoll_list)) {
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8989) io_iopoll_try_reap_events(ctx);
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8990) ret = true;
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8991) }
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8992) }
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 8993)
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8994) ret |= io_cancel_defer_files(ctx, task, cancel_all);
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8995) ret |= io_poll_remove_all(ctx, task, cancel_all);
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 8996) ret |= io_kill_timeouts(ctx, task, cancel_all);
c31fab84b23c7 (Pavel Begunkov 2021-06-26 21:40:46 +0100 8997) if (task)
c31fab84b23c7 (Pavel Begunkov 2021-06-26 21:40:46 +0100 8998) ret |= io_run_task_work();
ba50a036f23c4 (Pavel Begunkov 2021-02-26 15:47:56 +0000 8999) ret |= io_run_ctx_fallback(ctx);
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 9000) if (!ret)
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 9001) break;
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 9002) cond_resched();
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 9003) }
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 9004) }
9936c7c2bc76a (Pavel Begunkov 2021-02-04 13:51:56 +0000 9005)
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9006) static int __io_uring_add_task_file(struct io_ring_ctx *ctx)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9007) {
236434c3438c4 (Matthew Wilcox (Oracle) 2020-10-09 13:49:52 +0100 9008) struct io_uring_task *tctx = current->io_uring;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9009) struct io_tctx_node *node;
a528b04ea4069 (Pavel Begunkov 2020-12-21 18:34:04 +0000 9010) int ret;
236434c3438c4 (Matthew Wilcox (Oracle) 2020-10-09 13:49:52 +0100 9011)
236434c3438c4 (Matthew Wilcox (Oracle) 2020-10-09 13:49:52 +0100 9012) if (unlikely(!tctx)) {
5aa75ed5b93f0 (Jens Axboe 2021-02-16 12:56:50 -0700 9013) ret = io_uring_alloc_task_context(current, ctx);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9014) if (unlikely(ret))
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9015) return ret;
236434c3438c4 (Matthew Wilcox (Oracle) 2020-10-09 13:49:52 +0100 9016) tctx = current->io_uring;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9017) }
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9018) if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9019) node = kmalloc(sizeof(*node), GFP_KERNEL);
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9020) if (!node)
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9021) return -ENOMEM;
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9022) node->ctx = ctx;
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9023) node->task = current;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9024)
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9025) ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9026) node, GFP_KERNEL));
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9027) if (ret) {
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9028) kfree(node);
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9029) return ret;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9030) }
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9031)
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9032) mutex_lock(&ctx->uring_lock);
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9033) list_add(&node->ctx_node, &ctx->tctx_list);
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9034) mutex_unlock(&ctx->uring_lock);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9035) }
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9036) tctx->last = ctx;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9037) return 0;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9038) }
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9039)
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9040) /*
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9041) * Note that this task has used io_uring. We use it for cancelation purposes.
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9042) */
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9043) static inline int io_uring_add_task_file(struct io_ring_ctx *ctx)
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9044) {
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9045) struct io_uring_task *tctx = current->io_uring;
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9046)
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9047) if (likely(tctx && tctx->last == ctx))
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9048) return 0;
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9049) return __io_uring_add_task_file(ctx);
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9050) }
cf27f3b149618 (Pavel Begunkov 2021-03-19 17:22:31 +0000 9051)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9052) /*
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9053) * Remove this io_uring_file -> task mapping.
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9054) */
2941267bd3dad (Pavel Begunkov 2021-03-06 11:02:11 +0000 9055) static void io_uring_del_task_file(unsigned long index)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9056) {
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9057) struct io_uring_task *tctx = current->io_uring;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9058) struct io_tctx_node *node;
2941267bd3dad (Pavel Begunkov 2021-03-06 11:02:11 +0000 9059)
eebd2e37e6626 (Pavel Begunkov 2021-03-06 11:02:14 +0000 9060) if (!tctx)
eebd2e37e6626 (Pavel Begunkov 2021-03-06 11:02:14 +0000 9061) return;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9062) node = xa_erase(&tctx->xa, index);
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9063) if (!node)
2941267bd3dad (Pavel Begunkov 2021-03-06 11:02:11 +0000 9064) return;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9065)
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9066) WARN_ON_ONCE(current != node->task);
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9067) WARN_ON_ONCE(list_empty(&node->ctx_node));
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9068)
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9069) mutex_lock(&node->ctx->uring_lock);
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9070) list_del(&node->ctx_node);
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9071) mutex_unlock(&node->ctx->uring_lock);
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9072)
baf186c4d345f (Pavel Begunkov 2021-03-06 11:02:15 +0000 9073) if (tctx->last == node->ctx)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9074) tctx->last = NULL;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9075) kfree(node);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9076) }
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9077)
8452d4a674b0e (Pavel Begunkov 2021-02-27 11:16:46 +0000 9078) static void io_uring_clean_tctx(struct io_uring_task *tctx)
de7f1d9e99d8b (Pavel Begunkov 2021-01-04 20:43:29 +0000 9079) {
ba5ef6dc8a827 (Pavel Begunkov 2021-05-20 13:21:20 +0100 9080) struct io_wq *wq = tctx->io_wq;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9081) struct io_tctx_node *node;
de7f1d9e99d8b (Pavel Begunkov 2021-01-04 20:43:29 +0000 9082) unsigned long index;
de7f1d9e99d8b (Pavel Begunkov 2021-01-04 20:43:29 +0000 9083)
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9084) xa_for_each(&tctx->xa, index, node)
2941267bd3dad (Pavel Begunkov 2021-03-06 11:02:11 +0000 9085) io_uring_del_task_file(index);
b16ef427adf31 (Marco Elver 2021-05-27 11:25:48 +0200 9086) if (wq) {
b16ef427adf31 (Marco Elver 2021-05-27 11:25:48 +0200 9087) /*
b16ef427adf31 (Marco Elver 2021-05-27 11:25:48 +0200 9088) * Must be after io_uring_del_task_file() (removes nodes under
b16ef427adf31 (Marco Elver 2021-05-27 11:25:48 +0200 9089) * uring_lock) to avoid race with io_uring_try_cancel_iowq().
b16ef427adf31 (Marco Elver 2021-05-27 11:25:48 +0200 9090) */
b16ef427adf31 (Marco Elver 2021-05-27 11:25:48 +0200 9091) tctx->io_wq = NULL;
ba5ef6dc8a827 (Pavel Begunkov 2021-05-20 13:21:20 +0100 9092) io_wq_put_and_exit(wq);
b16ef427adf31 (Marco Elver 2021-05-27 11:25:48 +0200 9093) }
de7f1d9e99d8b (Pavel Begunkov 2021-01-04 20:43:29 +0000 9094) }
de7f1d9e99d8b (Pavel Begunkov 2021-01-04 20:43:29 +0000 9095)
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9096) static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9097) {
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9098) if (tracked)
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9099) return atomic_read(&tctx->inflight_tracked);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9100) return percpu_counter_sum(&tctx->inflight);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9101) }
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9102)
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 9103) static void io_uring_try_cancel(bool cancel_all)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9104) {
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9105) struct io_uring_task *tctx = current->io_uring;
13bf43f5f4739 (Pavel Begunkov 2021-03-06 11:02:12 +0000 9106) struct io_tctx_node *node;
ce765372bc443 (Matthew Wilcox (Oracle) 2020-10-09 13:49:51 +0100 9107) unsigned long index;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9108)
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9109) xa_for_each(&tctx->xa, index, node) {
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9110) struct io_ring_ctx *ctx = node->ctx;
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9111)
9f59a9d88d3bb (Pavel Begunkov 2021-04-25 23:34:46 +0100 9112) /* sqpoll task will cancel all its requests */
9f59a9d88d3bb (Pavel Begunkov 2021-04-25 23:34:46 +0100 9113) if (!ctx->sq_data)
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 9114) io_uring_try_cancel_requests(ctx, current, cancel_all);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9115) }
fdaf083cdfb55 (Jens Axboe 2020-10-30 09:37:30 -0600 9116) }
fdaf083cdfb55 (Jens Axboe 2020-10-30 09:37:30 -0600 9117)
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9118) /* should only be called by SQPOLL task */
734551df6f9be (Pavel Begunkov 2021-04-18 14:52:09 +0100 9119) static void io_uring_cancel_sqpoll(struct io_sq_data *sqd)
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9120) {
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9121) struct io_uring_task *tctx = current->io_uring;
734551df6f9be (Pavel Begunkov 2021-04-18 14:52:09 +0100 9122) struct io_ring_ctx *ctx;
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9123) s64 inflight;
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9124) DEFINE_WAIT(wait);
fdaf083cdfb55 (Jens Axboe 2020-10-30 09:37:30 -0600 9125)
6d042ffb598ed (Palash Oswal 2021-04-27 18:21:49 +0530 9126) if (!current->io_uring)
6d042ffb598ed (Palash Oswal 2021-04-27 18:21:49 +0530 9127) return;
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 9128) if (tctx->io_wq)
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 9129) io_wq_exit_start(tctx->io_wq);
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 9130)
734551df6f9be (Pavel Begunkov 2021-04-18 14:52:09 +0100 9131) WARN_ON_ONCE(!sqd || sqd->thread != current);
521d6a737a31c (Pavel Begunkov 2021-03-11 23:29:38 +0000 9132)
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9133) atomic_inc(&tctx->in_idle);
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9134) do {
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9135) /* read completions before cancelations */
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9136) inflight = tctx_inflight(tctx, false);
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9137) if (!inflight)
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9138) break;
734551df6f9be (Pavel Begunkov 2021-04-18 14:52:09 +0100 9139) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 9140) io_uring_try_cancel_requests(ctx, current, true);
fdaf083cdfb55 (Jens Axboe 2020-10-30 09:37:30 -0600 9141)
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9142) prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9143) /*
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9144) * If we've seen completions, retry without waiting. This
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9145) * avoids a race where a completion comes in before we did
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9146) * prepare_to_wait().
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9147) */
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9148) if (inflight == tctx_inflight(tctx, false))
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9149) schedule();
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9150) finish_wait(&tctx->wait, &wait);
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9151) } while (1);
0e9ddb39b7d96 (Pavel Begunkov 2021-02-07 22:34:26 +0000 9152) atomic_dec(&tctx->in_idle);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9153) }
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9154)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9155) /*
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9156) * Find any io_uring fd that this task has registered or done IO on, and cancel
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9157) * requests.
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9158) */
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9159) void __io_uring_cancel(struct files_struct *files)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9160) {
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9161) struct io_uring_task *tctx = current->io_uring;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9162) DEFINE_WAIT(wait);
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 9163) s64 inflight;
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 9164) bool cancel_all = !files;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9165)
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 9166) if (tctx->io_wq)
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 9167) io_wq_exit_start(tctx->io_wq);
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 9168)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9169) /* make sure overflow events are dropped */
fdaf083cdfb55 (Jens Axboe 2020-10-30 09:37:30 -0600 9170) atomic_inc(&tctx->in_idle);
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 9171) do {
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9172) /* read completions before cancelations */
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 9173) inflight = tctx_inflight(tctx, !cancel_all);
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 9174) if (!inflight)
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 9175) break;
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 9176) io_uring_try_cancel(cancel_all);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9177) prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9178)
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9179) /*
a1bb3cd589133 (Pavel Begunkov 2021-01-26 15:28:26 +0000 9180) * If we've seen completions, retry without waiting. This
a1bb3cd589133 (Pavel Begunkov 2021-01-26 15:28:26 +0000 9181) * avoids a race where a completion comes in before we did
a1bb3cd589133 (Pavel Begunkov 2021-01-26 15:28:26 +0000 9182) * prepare_to_wait().
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9183) */
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 9184) if (inflight == tctx_inflight(tctx, !cancel_all))
a1bb3cd589133 (Pavel Begunkov 2021-01-26 15:28:26 +0000 9185) schedule();
f57555eda979c (Pavel Begunkov 2020-12-20 13:21:44 +0000 9186) finish_wait(&tctx->wait, &wait);
d8a6df10aac9f (Jens Axboe 2020-10-15 16:24:45 -0600 9187) } while (1);
fdaf083cdfb55 (Jens Axboe 2020-10-30 09:37:30 -0600 9188) atomic_dec(&tctx->in_idle);
de7f1d9e99d8b (Pavel Begunkov 2021-01-04 20:43:29 +0000 9189)
8452d4a674b0e (Pavel Begunkov 2021-02-27 11:16:46 +0000 9190) io_uring_clean_tctx(tctx);
f564954855fcc (Pavel Begunkov 2021-05-16 22:58:04 +0100 9191) if (cancel_all) {
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9192) /* for exec all current's requests should be gone, kill tctx */
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9193) __io_uring_free(current);
3f48cf18f886c (Pavel Begunkov 2021-04-11 01:46:27 +0100 9194) }
44e728b8aae0b (Pavel Begunkov 2020-06-15 10:24:04 +0300 9195) }
44e728b8aae0b (Pavel Begunkov 2020-06-15 10:24:04 +0300 9196)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9197) static void *io_uring_validate_mmap_request(struct file *file,
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9198) loff_t pgoff, size_t sz)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9199) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9200) struct io_ring_ctx *ctx = file->private_data;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9201) loff_t offset = pgoff << PAGE_SHIFT;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9202) struct page *page;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9203) void *ptr;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9204)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9205) switch (offset) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9206) case IORING_OFF_SQ_RING:
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9207) case IORING_OFF_CQ_RING:
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9208) ptr = ctx->rings;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9209) break;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9210) case IORING_OFF_SQES:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9211) ptr = ctx->sq_sqes;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9212) break;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9213) default:
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9214) return ERR_PTR(-EINVAL);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9215) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9216)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9217) page = virt_to_head_page(ptr);
a50b854e073cd (Matthew Wilcox (Oracle) 2019-09-23 15:34:25 -0700 9218) if (sz > page_size(page))
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9219) return ERR_PTR(-EINVAL);
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9220)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9221) return ptr;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9222) }
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9223)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9224) #ifdef CONFIG_MMU
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9225)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9226) static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9227) {
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9228) size_t sz = vma->vm_end - vma->vm_start;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9229) unsigned long pfn;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9230) void *ptr;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9231)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9232) ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9233) if (IS_ERR(ptr))
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9234) return PTR_ERR(ptr);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9235)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9236) pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9237) return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9238) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9239)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9240) #else /* !CONFIG_MMU */
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9241)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9242) static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9243) {
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9244) return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9245) }
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9246)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9247) static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9248) {
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9249) return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9250) }
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9251)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9252) static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9253) unsigned long addr, unsigned long len,
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9254) unsigned long pgoff, unsigned long flags)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9255) {
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9256) void *ptr;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9257)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9258) ptr = io_uring_validate_mmap_request(file, pgoff, len);
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9259) if (IS_ERR(ptr))
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9260) return PTR_ERR(ptr);
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9261)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9262) return (unsigned long) ptr;
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9263) }
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9264)
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9265) #endif /* !CONFIG_MMU */
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9266)
d9d05217cb699 (Pavel Begunkov 2021-01-08 20:57:25 +0000 9267) static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9268) {
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9269) DEFINE_WAIT(wait);
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9270)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9271) do {
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9272) if (!io_sqring_full(ctx))
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9273) break;
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9274) prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9275)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9276) if (!io_sqring_full(ctx))
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9277) break;
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9278) schedule();
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9279) } while (!signal_pending(current));
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9280)
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9281) finish_wait(&ctx->sqo_sq_wait, &wait);
5199328a0d415 (Yang Li 2021-03-09 14:30:41 +0800 9282) return 0;
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9283) }
90554200724d5 (Jens Axboe 2020-09-03 12:12:41 -0600 9284)
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9285) static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9286) struct __kernel_timespec __user **ts,
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9287) const sigset_t __user **sig)
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9288) {
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9289) struct io_uring_getevents_arg arg;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9290)
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9291) /*
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9292) * If EXT_ARG isn't set, then we have no timespec and the argp pointer
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9293) * is just a pointer to the sigset_t.
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9294) */
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9295) if (!(flags & IORING_ENTER_EXT_ARG)) {
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9296) *sig = (const sigset_t __user *) argp;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9297) *ts = NULL;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9298) return 0;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9299) }
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9300)
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9301) /*
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9302) * EXT_ARG is set - ensure we agree on the size of it and copy in our
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9303) * timespec and sigset_t pointers if good.
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9304) */
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9305) if (*argsz != sizeof(arg))
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9306) return -EINVAL;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9307) if (copy_from_user(&arg, argp, sizeof(arg)))
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9308) return -EFAULT;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9309) *sig = u64_to_user_ptr(arg.sigmask);
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9310) *argsz = arg.sigmask_sz;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9311) *ts = u64_to_user_ptr(arg.ts);
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9312) return 0;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9313) }
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9314)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9315) SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9316) u32, min_complete, u32, flags, const void __user *, argp,
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9317) size_t, argsz)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9318) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9319) struct io_ring_ctx *ctx;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9320) int submitted = 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9321) struct fd f;
33f993da98297 (Pavel Begunkov 2021-03-19 17:22:30 +0000 9322) long ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9323)
4c6e277c4cc4a (Jens Axboe 2020-07-01 11:29:10 -0600 9324) io_run_task_work();
b41e98524e424 (Jens Axboe 2020-02-17 09:52:41 -0700 9325)
33f993da98297 (Pavel Begunkov 2021-03-19 17:22:30 +0000 9326) if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
33f993da98297 (Pavel Begunkov 2021-03-19 17:22:30 +0000 9327) IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9328) return -EINVAL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9329)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9330) f = fdget(fd);
33f993da98297 (Pavel Begunkov 2021-03-19 17:22:30 +0000 9331) if (unlikely(!f.file))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9332) return -EBADF;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9333)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9334) ret = -EOPNOTSUPP;
33f993da98297 (Pavel Begunkov 2021-03-19 17:22:30 +0000 9335) if (unlikely(f.file->f_op != &io_uring_fops))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9336) goto out_fput;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9337)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9338) ret = -ENXIO;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9339) ctx = f.file->private_data;
33f993da98297 (Pavel Begunkov 2021-03-19 17:22:30 +0000 9340) if (unlikely(!percpu_ref_tryget(&ctx->refs)))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9341) goto out_fput;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9342)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9343) ret = -EBADFD;
33f993da98297 (Pavel Begunkov 2021-03-19 17:22:30 +0000 9344) if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9345) goto out;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9346)
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9347) /*
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9348) * For SQ polling, the thread will do all submissions and completions.
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9349) * Just return the requested submit count, and wake the thread if
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9350) * we were asked to.
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9351) */
b2a9eadab8573 (Jens Axboe 2019-09-12 14:19:16 -0600 9352) ret = 0;
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9353) if (ctx->flags & IORING_SETUP_SQPOLL) {
6c2450ae55656 (Pavel Begunkov 2021-02-23 12:40:22 +0000 9354) io_cqring_overflow_flush(ctx, false);
89448c47b8452 (Pavel Begunkov 2020-12-17 00:24:39 +0000 9355)
da8495661fe9e (Jens Axboe 2021-08-14 09:04:40 -0600 9356) if (unlikely(ctx->sq_data->thread == NULL)) {
da8495661fe9e (Jens Axboe 2021-08-14 09:04:40 -0600 9357) ret = -EOWNERDEAD;
041474885e970 (Stefan Metzmacher 2021-03-07 11:54:29 +0100 9358) goto out;
da8495661fe9e (Jens Axboe 2021-08-14 09:04:40 -0600 9359) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9360) if (flags & IORING_ENTER_SQ_WAKEUP)
534ca6d684f1f (Jens Axboe 2020-09-02 13:52:19 -0600 9361) wake_up(&ctx->sq_data->wait);
d9d05217cb699 (Pavel Begunkov 2021-01-08 20:57:25 +0000 9362) if (flags & IORING_ENTER_SQ_WAIT) {
d9d05217cb699 (Pavel Begunkov 2021-01-08 20:57:25 +0000 9363) ret = io_sqpoll_wait_sq(ctx);
d9d05217cb699 (Pavel Begunkov 2021-01-08 20:57:25 +0000 9364) if (ret)
d9d05217cb699 (Pavel Begunkov 2021-01-08 20:57:25 +0000 9365) goto out;
d9d05217cb699 (Pavel Begunkov 2021-01-08 20:57:25 +0000 9366) }
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9367) submitted = to_submit;
b2a9eadab8573 (Jens Axboe 2019-09-12 14:19:16 -0600 9368) } else if (to_submit) {
baf186c4d345f (Pavel Begunkov 2021-03-06 11:02:15 +0000 9369) ret = io_uring_add_task_file(ctx);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9370) if (unlikely(ret))
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9371) goto out;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9372) mutex_lock(&ctx->uring_lock);
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9373) submitted = io_submit_sqes(ctx, to_submit);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9374) mutex_unlock(&ctx->uring_lock);
7c504e65206a4 (Pavel Begunkov 2019-12-18 19:53:45 +0300 9375)
7c504e65206a4 (Pavel Begunkov 2019-12-18 19:53:45 +0300 9376) if (submitted != to_submit)
7c504e65206a4 (Pavel Begunkov 2019-12-18 19:53:45 +0300 9377) goto out;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9378) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9379) if (flags & IORING_ENTER_GETEVENTS) {
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9380) const sigset_t __user *sig;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9381) struct __kernel_timespec __user *ts;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9382)
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9383) ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9384) if (unlikely(ret))
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9385) goto out;
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9386)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9387) min_complete = min(min_complete, ctx->cq_entries);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9388)
32b2244a840a9 (Xiaoguang Wang 2020-03-11 09:26:09 +0800 9389) /*
32b2244a840a9 (Xiaoguang Wang 2020-03-11 09:26:09 +0800 9390) * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
32b2244a840a9 (Xiaoguang Wang 2020-03-11 09:26:09 +0800 9391) * space applications don't need to do io completion events
32b2244a840a9 (Xiaoguang Wang 2020-03-11 09:26:09 +0800 9392) * polling again, they can rely on io_sq_thread to do polling
32b2244a840a9 (Xiaoguang Wang 2020-03-11 09:26:09 +0800 9393) * work, which can reduce cpu usage and uring_lock contention.
32b2244a840a9 (Xiaoguang Wang 2020-03-11 09:26:09 +0800 9394) */
32b2244a840a9 (Xiaoguang Wang 2020-03-11 09:26:09 +0800 9395) if (ctx->flags & IORING_SETUP_IOPOLL &&
32b2244a840a9 (Xiaoguang Wang 2020-03-11 09:26:09 +0800 9396) !(ctx->flags & IORING_SETUP_SQPOLL)) {
7668b92a69b82 (Pavel Begunkov 2020-07-07 16:36:21 +0300 9397) ret = io_iopoll_check(ctx, min_complete);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 9398) } else {
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9399) ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
def596e9557c9 (Jens Axboe 2019-01-09 08:59:42 -0700 9400) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9401) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9402)
7c504e65206a4 (Pavel Begunkov 2019-12-18 19:53:45 +0300 9403) out:
6805b32ec2b08 (Pavel Begunkov 2019-10-08 02:18:42 +0300 9404) percpu_ref_put(&ctx->refs);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9405) out_fput:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9406) fdput(f);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9407) return submitted ? submitted : ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9408) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9409)
bebdb65e07726 (Tobias Klauser 2020-02-26 18:38:32 +0100 9410) #ifdef CONFIG_PROC_FS
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9411) static int io_uring_show_cred(struct seq_file *m, unsigned int id,
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9412) const struct cred *cred)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9413) {
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9414) struct user_namespace *uns = seq_user_ns(m);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9415) struct group_info *gi;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9416) kernel_cap_t cap;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9417) unsigned __capi;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9418) int g;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9419)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9420) seq_printf(m, "%5d\n", id);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9421) seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9422) seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9423) seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9424) seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9425) seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9426) seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9427) seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9428) seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9429) seq_puts(m, "\n\tGroups:\t");
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9430) gi = cred->group_info;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9431) for (g = 0; g < gi->ngroups; g++) {
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9432) seq_put_decimal_ull(m, g ? " " : "",
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9433) from_kgid_munged(uns, gi->gid[g]));
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9434) }
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9435) seq_puts(m, "\n\tCapEff:\t");
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9436) cap = cred->cap_effective;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9437) CAP_FOR_EACH_U32(__capi)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9438) seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9439) seq_putc(m, '\n');
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9440) return 0;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9441) }
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9442)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9443) static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9444) {
dbbe9c642411c (Joseph Qi 2020-09-29 09:01:22 -0600 9445) struct io_sq_data *sq = NULL;
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9446) bool has_lock;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9447) int i;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9448)
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9449) /*
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9450) * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9451) * since fdinfo case grabs it in the opposite direction of normal use
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9452) * cases. If we fail to get the lock, we just don't iterate any
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9453) * structures that could be going away outside the io_uring mutex.
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9454) */
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9455) has_lock = mutex_trylock(&ctx->uring_lock);
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9456)
5f3f26f98ae48 (Jens Axboe 2021-02-25 10:17:46 -0700 9457) if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) {
dbbe9c642411c (Joseph Qi 2020-09-29 09:01:22 -0600 9458) sq = ctx->sq_data;
5f3f26f98ae48 (Jens Axboe 2021-02-25 10:17:46 -0700 9459) if (!sq->thread)
5f3f26f98ae48 (Jens Axboe 2021-02-25 10:17:46 -0700 9460) sq = NULL;
5f3f26f98ae48 (Jens Axboe 2021-02-25 10:17:46 -0700 9461) }
dbbe9c642411c (Joseph Qi 2020-09-29 09:01:22 -0600 9462)
dbbe9c642411c (Joseph Qi 2020-09-29 09:01:22 -0600 9463) seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
dbbe9c642411c (Joseph Qi 2020-09-29 09:01:22 -0600 9464) seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9465) seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9466) for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
7b29f92da377c (Jens Axboe 2021-03-12 08:30:14 -0700 9467) struct file *f = io_file_from_index(ctx, i);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9468)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9469) if (f)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9470) seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9471) else
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9472) seq_printf(m, "%5u: <none>\n", i);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9473) }
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9474) seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9475) for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
41edf1a5ec967 (Pavel Begunkov 2021-04-25 14:32:23 +0100 9476) struct io_mapped_ubuf *buf = ctx->user_bufs[i];
4751f53d74a68 (Pavel Begunkov 2021-04-01 15:43:55 +0100 9477) unsigned int len = buf->ubuf_end - buf->ubuf;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9478)
4751f53d74a68 (Pavel Begunkov 2021-04-01 15:43:55 +0100 9479) seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9480) }
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9481) if (has_lock && !xa_empty(&ctx->personalities)) {
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9482) unsigned long index;
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9483) const struct cred *cred;
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9484)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9485) seq_printf(m, "Personalities:\n");
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9486) xa_for_each(&ctx->personalities, index, cred)
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9487) io_uring_show_cred(m, index, cred);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9488) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9489) seq_printf(m, "PollList:\n");
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9490) spin_lock_irq(&ctx->completion_lock);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9491) for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9492) struct hlist_head *list = &ctx->cancel_hash[i];
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9493) struct io_kiocb *req;
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9494)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9495) hlist_for_each_entry(req, list, hash_node)
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9496) seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9497) req->task->task_works != NULL);
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9498) }
d7718a9d25a61 (Jens Axboe 2020-02-14 22:23:12 -0700 9499) spin_unlock_irq(&ctx->completion_lock);
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9500) if (has_lock)
fad8e0de4426a (Jens Axboe 2020-09-28 08:57:48 -0600 9501) mutex_unlock(&ctx->uring_lock);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9502) }
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9503)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9504) static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9505) {
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9506) struct io_ring_ctx *ctx = f->private_data;
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9507)
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9508) if (percpu_ref_tryget(&ctx->refs)) {
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9509) __io_uring_show_fdinfo(ctx, m);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9510) percpu_ref_put(&ctx->refs);
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9511) }
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9512) }
bebdb65e07726 (Tobias Klauser 2020-02-26 18:38:32 +0100 9513) #endif
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9514)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9515) static const struct file_operations io_uring_fops = {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9516) .release = io_uring_release,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9517) .mmap = io_uring_mmap,
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9518) #ifndef CONFIG_MMU
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9519) .get_unmapped_area = io_uring_nommu_get_unmapped_area,
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9520) .mmap_capabilities = io_uring_nommu_mmap_capabilities,
6c5c240e41268 (Roman Penyaev 2019-11-28 12:53:22 +0100 9521) #endif
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9522) .poll = io_uring_poll,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9523) .fasync = io_uring_fasync,
bebdb65e07726 (Tobias Klauser 2020-02-26 18:38:32 +0100 9524) #ifdef CONFIG_PROC_FS
87ce955b24c99 (Jens Axboe 2020-01-30 08:25:34 -0700 9525) .show_fdinfo = io_uring_show_fdinfo,
bebdb65e07726 (Tobias Klauser 2020-02-26 18:38:32 +0100 9526) #endif
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9527) };
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9528)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9529) static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9530) struct io_uring_params *p)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9531) {
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9532) struct io_rings *rings;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9533) size_t size, sq_array_offset;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9534)
bd74048108c17 (Jens Axboe 2020-08-05 12:58:23 -0600 9535) /* make sure these are sane, as we already accounted them */
bd74048108c17 (Jens Axboe 2020-08-05 12:58:23 -0600 9536) ctx->sq_entries = p->sq_entries;
bd74048108c17 (Jens Axboe 2020-08-05 12:58:23 -0600 9537) ctx->cq_entries = p->cq_entries;
bd74048108c17 (Jens Axboe 2020-08-05 12:58:23 -0600 9538)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9539) size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9540) if (size == SIZE_MAX)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9541) return -EOVERFLOW;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9542)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9543) rings = io_mem_alloc(size);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9544) if (!rings)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9545) return -ENOMEM;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9546)
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9547) ctx->rings = rings;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9548) ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9549) rings->sq_ring_mask = p->sq_entries - 1;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9550) rings->cq_ring_mask = p->cq_entries - 1;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9551) rings->sq_ring_entries = p->sq_entries;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9552) rings->cq_ring_entries = p->cq_entries;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9553) ctx->sq_mask = rings->sq_ring_mask;
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9554) ctx->cq_mask = rings->cq_ring_mask;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9555)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9556) size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
eb065d301e8c8 (Jens Axboe 2019-11-20 09:26:29 -0700 9557) if (size == SIZE_MAX) {
eb065d301e8c8 (Jens Axboe 2019-11-20 09:26:29 -0700 9558) io_mem_free(ctx->rings);
eb065d301e8c8 (Jens Axboe 2019-11-20 09:26:29 -0700 9559) ctx->rings = NULL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9560) return -EOVERFLOW;
eb065d301e8c8 (Jens Axboe 2019-11-20 09:26:29 -0700 9561) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9562)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9563) ctx->sq_sqes = io_mem_alloc(size);
eb065d301e8c8 (Jens Axboe 2019-11-20 09:26:29 -0700 9564) if (!ctx->sq_sqes) {
eb065d301e8c8 (Jens Axboe 2019-11-20 09:26:29 -0700 9565) io_mem_free(ctx->rings);
eb065d301e8c8 (Jens Axboe 2019-11-20 09:26:29 -0700 9566) ctx->rings = NULL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9567) return -ENOMEM;
eb065d301e8c8 (Jens Axboe 2019-11-20 09:26:29 -0700 9568) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9569)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9570) return 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9571) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9572)
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9573) static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9574) {
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9575) int ret, fd;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9576)
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9577) fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9578) if (fd < 0)
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9579) return fd;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9580)
baf186c4d345f (Pavel Begunkov 2021-03-06 11:02:15 +0000 9581) ret = io_uring_add_task_file(ctx);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9582) if (ret) {
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9583) put_unused_fd(fd);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9584) return ret;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9585) }
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9586) fd_install(fd, file);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9587) return fd;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9588) }
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9589)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9590) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9591) * Allocate an anonymous fd, this is what constitutes the application
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9592) * visible backing of an io_uring instance. The application mmaps this
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9593) * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9594) * we have to tie this fd to a socket for file garbage collection purposes.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9595) */
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9596) static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9597) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9598) struct file *file;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9599) #if defined(CONFIG_UNIX)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9600) int ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9601)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9602) ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9603) &ctx->ring_sock);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9604) if (ret)
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9605) return ERR_PTR(ret);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9606) #endif
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9607)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9608) file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9609) O_RDWR | O_CLOEXEC);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9610) #if defined(CONFIG_UNIX)
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9611) if (IS_ERR(file)) {
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9612) sock_release(ctx->ring_sock);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9613) ctx->ring_sock = NULL;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9614) } else {
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9615) ctx->ring_sock->file = file;
0f2122045b946 (Jens Axboe 2020-09-13 13:09:39 -0600 9616) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9617) #endif
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9618) return file;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9619) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9620)
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9621) static int io_uring_create(unsigned entries, struct io_uring_params *p,
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9622) struct io_uring_params __user *params)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9623) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9624) struct io_ring_ctx *ctx;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9625) struct file *file;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9626) int ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9627)
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9628) if (!entries)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9629) return -EINVAL;
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9630) if (entries > IORING_MAX_ENTRIES) {
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9631) if (!(p->flags & IORING_SETUP_CLAMP))
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9632) return -EINVAL;
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9633) entries = IORING_MAX_ENTRIES;
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9634) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9635)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9636) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9637) * Use twice as many entries for the CQ ring. It's possible for the
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9638) * application to drive a higher depth than the size of the SQ ring,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9639) * since the sqes are only used at submission time. This allows for
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9640) * some flexibility in overcommitting a bit. If the application has
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9641) * set IORING_SETUP_CQSIZE, it will have passed in the desired number
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9642) * of CQ ring entries manually.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9643) */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9644) p->sq_entries = roundup_pow_of_two(entries);
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9645) if (p->flags & IORING_SETUP_CQSIZE) {
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9646) /*
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9647) * If IORING_SETUP_CQSIZE is set, we do the same roundup
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9648) * to a power-of-two, if it isn't already. We do NOT impose
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9649) * any cq vs sq ring sizing.
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9650) */
eb2667b343361 (Joseph Qi 2020-11-24 15:03:03 +0800 9651) if (!p->cq_entries)
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9652) return -EINVAL;
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9653) if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9654) if (!(p->flags & IORING_SETUP_CLAMP))
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9655) return -EINVAL;
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9656) p->cq_entries = IORING_MAX_CQ_ENTRIES;
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9657) }
eb2667b343361 (Joseph Qi 2020-11-24 15:03:03 +0800 9658) p->cq_entries = roundup_pow_of_two(p->cq_entries);
eb2667b343361 (Joseph Qi 2020-11-24 15:03:03 +0800 9659) if (p->cq_entries < p->sq_entries)
eb2667b343361 (Joseph Qi 2020-11-24 15:03:03 +0800 9660) return -EINVAL;
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9661) } else {
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9662) p->cq_entries = 2 * p->sq_entries;
33a107f0a1b8d (Jens Axboe 2019-10-04 12:10:03 -0600 9663) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9664)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9665) ctx = io_ring_ctx_alloc(p);
62e398be275a6 (Jens Axboe 2021-02-21 16:19:37 -0700 9666) if (!ctx)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9667) return -ENOMEM;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9668) ctx->compat = in_compat_syscall();
62e398be275a6 (Jens Axboe 2021-02-21 16:19:37 -0700 9669) if (!capable(CAP_IPC_LOCK))
62e398be275a6 (Jens Axboe 2021-02-21 16:19:37 -0700 9670) ctx->user = get_uid(current_user());
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 9671)
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 9672) /*
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 9673) * This is just grabbed for accounting purposes. When a process exits,
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 9674) * the mm is exited and dropped before the files, hence we need to hang
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 9675) * on to this mm purely for the purposes of being able to unaccount
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 9676) * memory (locked/pinned vm). It's not used for anything else.
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 9677) */
6b7898eb180df (Jens Axboe 2020-08-25 07:58:00 -0600 9678) mmgrab(current->mm);
2aede0e417db8 (Jens Axboe 2020-09-14 10:45:53 -0600 9679) ctx->mm_account = current->mm;
6b7898eb180df (Jens Axboe 2020-08-25 07:58:00 -0600 9680)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9681) ret = io_allocate_scq_urings(ctx, p);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9682) if (ret)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9683) goto err;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9684)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9685) ret = io_sq_offload_create(ctx, p);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9686) if (ret)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9687) goto err;
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 9688) /* always set a rsrc node */
47b228ce6f668 (Pavel Begunkov 2021-04-29 11:46:48 +0100 9689) ret = io_rsrc_node_switch_start(ctx);
47b228ce6f668 (Pavel Begunkov 2021-04-29 11:46:48 +0100 9690) if (ret)
47b228ce6f668 (Pavel Begunkov 2021-04-29 11:46:48 +0100 9691) goto err;
eae071c9b4cef (Pavel Begunkov 2021-04-25 14:32:24 +0100 9692) io_rsrc_node_switch(ctx, NULL);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9693)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9694) memset(&p->sq_off, 0, sizeof(p->sq_off));
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9695) p->sq_off.head = offsetof(struct io_rings, sq.head);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9696) p->sq_off.tail = offsetof(struct io_rings, sq.tail);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9697) p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9698) p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9699) p->sq_off.flags = offsetof(struct io_rings, sq_flags);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9700) p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9701) p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9702)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9703) memset(&p->cq_off, 0, sizeof(p->cq_off));
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9704) p->cq_off.head = offsetof(struct io_rings, cq.head);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9705) p->cq_off.tail = offsetof(struct io_rings, cq.tail);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9706) p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9707) p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9708) p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
75b28affdd6ae (Hristo Venev 2019-08-26 17:23:46 +0000 9709) p->cq_off.cqes = offsetof(struct io_rings, cqes);
0d9b5b3af134c (Stefano Garzarella 2020-05-15 18:38:04 +0200 9710) p->cq_off.flags = offsetof(struct io_rings, cq_flags);
ac90f249e15cd (Jens Axboe 2019-09-06 10:26:21 -0600 9711)
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9712) p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9713) IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
5769a351b89cd (Jiufei Xue 2020-06-17 17:53:55 +0800 9714) IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
c73ebb685fb6d (Hao Xu 2020-11-03 10:54:37 +0800 9715) IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
9690557e22d63 (Pavel Begunkov 2021-06-10 16:37:38 +0100 9716) IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
9690557e22d63 (Pavel Begunkov 2021-06-10 16:37:38 +0100 9717) IORING_FEAT_RSRC_TAGS;
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9718)
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9719) if (copy_to_user(params, p, sizeof(*p))) {
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9720) ret = -EFAULT;
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9721) goto err;
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9722) }
d1719f70d0a5b (Jens Axboe 2020-07-30 13:43:53 -0600 9723)
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9724) file = io_uring_get_file(ctx);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9725) if (IS_ERR(file)) {
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9726) ret = PTR_ERR(file);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9727) goto err;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9728) }
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9729)
044c1ab399afb (Jens Axboe 2019-10-28 09:15:33 -0600 9730) /*
044c1ab399afb (Jens Axboe 2019-10-28 09:15:33 -0600 9731) * Install ring fd as the very last thing, so we don't risk someone
044c1ab399afb (Jens Axboe 2019-10-28 09:15:33 -0600 9732) * having closed it before we finish setup
044c1ab399afb (Jens Axboe 2019-10-28 09:15:33 -0600 9733) */
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9734) ret = io_uring_install_fd(ctx, file);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9735) if (ret < 0) {
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9736) /* fput will clean it up */
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9737) fput(file);
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9738) return ret;
9faadcc8abe4b (Pavel Begunkov 2020-12-21 18:34:05 +0000 9739) }
044c1ab399afb (Jens Axboe 2019-10-28 09:15:33 -0600 9740)
c826bd7a743f2 (Dmitrii Dolgov 2019-10-15 19:02:01 +0200 9741) trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9742) return ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9743) err:
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9744) io_ring_ctx_wait_and_kill(ctx);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9745) return ret;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9746) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9747)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9748) /*
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9749) * Sets up an aio uring context, and returns the fd. Applications asks for a
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9750) * ring size, we return the actual sq/cq ring sizes (among other things) in the
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9751) * params structure passed in.
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9752) */
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9753) static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9754) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9755) struct io_uring_params p;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9756) int i;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9757)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9758) if (copy_from_user(&p, params, sizeof(p)))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9759) return -EFAULT;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9760) for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9761) if (p.resv[i])
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9762) return -EINVAL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9763) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9764)
6c271ce2f1d57 (Jens Axboe 2019-01-10 11:22:30 -0700 9765) if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
8110c1a6212e4 (Jens Axboe 2019-12-28 15:39:54 -0700 9766) IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9767) IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9768) IORING_SETUP_R_DISABLED))
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9769) return -EINVAL;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9770)
7f13657d14134 (Xiaoguang Wang 2020-05-05 16:28:53 +0800 9771) return io_uring_create(entries, &p, params);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9772) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9773)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9774) SYSCALL_DEFINE2(io_uring_setup, u32, entries,
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9775) struct io_uring_params __user *, params)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9776) {
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9777) return io_uring_setup(entries, params);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9778) }
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 9779)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9780) static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9781) {
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9782) struct io_uring_probe *p;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9783) size_t size;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9784) int i, ret;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9785)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9786) size = struct_size(p, ops, nr_args);
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9787) if (size == SIZE_MAX)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9788) return -EOVERFLOW;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9789) p = kzalloc(size, GFP_KERNEL);
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9790) if (!p)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9791) return -ENOMEM;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9792)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9793) ret = -EFAULT;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9794) if (copy_from_user(p, arg, size))
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9795) goto out;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9796) ret = -EINVAL;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9797) if (memchr_inv(p, 0, size))
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9798) goto out;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9799)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9800) p->last_op = IORING_OP_LAST - 1;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9801) if (nr_args > IORING_OP_LAST)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9802) nr_args = IORING_OP_LAST;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9803)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9804) for (i = 0; i < nr_args; i++) {
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9805) p->ops[i].op = i;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9806) if (!io_op_defs[i].not_supported)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9807) p->ops[i].flags = IO_URING_OP_SUPPORTED;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9808) }
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9809) p->ops_len = i;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9810)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9811) ret = 0;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9812) if (copy_to_user(arg, p, size))
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9813) ret = -EFAULT;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9814) out:
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9815) kfree(p);
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9816) return ret;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9817) }
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 9818)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 9819) static int io_register_personality(struct io_ring_ctx *ctx)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 9820) {
4379bf8bd70b5 (Jens Axboe 2021-02-15 13:40:22 -0700 9821) const struct cred *creds;
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9822) u32 id;
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 9823) int ret;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 9824)
4379bf8bd70b5 (Jens Axboe 2021-02-15 13:40:22 -0700 9825) creds = get_current_cred();
1e6fa5216a0e5 (Jens Axboe 2020-10-15 08:46:24 -0600 9826)
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9827) ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
61cf93700fe63 (Matthew Wilcox (Oracle) 2021-03-08 14:16:16 +0000 9828) XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
a57b2a703e444 (Jens Axboe 2021-08-20 14:53:59 -0600 9829) if (ret < 0) {
a57b2a703e444 (Jens Axboe 2021-08-20 14:53:59 -0600 9830) put_cred(creds);
a57b2a703e444 (Jens Axboe 2021-08-20 14:53:59 -0600 9831) return ret;
a57b2a703e444 (Jens Axboe 2021-08-20 14:53:59 -0600 9832) }
a57b2a703e444 (Jens Axboe 2021-08-20 14:53:59 -0600 9833) return id;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 9834) }
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 9835)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9836) static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9837) unsigned int nr_args)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9838) {
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9839) struct io_uring_restriction *res;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9840) size_t size;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9841) int i, ret;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9842)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9843) /* Restrictions allowed only if rings started disabled */
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9844) if (!(ctx->flags & IORING_SETUP_R_DISABLED))
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9845) return -EBADFD;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9846)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9847) /* We allow only a single restrictions registration */
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9848) if (ctx->restrictions.registered)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9849) return -EBUSY;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9850)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9851) if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9852) return -EINVAL;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9853)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9854) size = array_size(nr_args, sizeof(*res));
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9855) if (size == SIZE_MAX)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9856) return -EOVERFLOW;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9857)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9858) res = memdup_user(arg, size);
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9859) if (IS_ERR(res))
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9860) return PTR_ERR(res);
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9861)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9862) ret = 0;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9863)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9864) for (i = 0; i < nr_args; i++) {
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9865) switch (res[i].opcode) {
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9866) case IORING_RESTRICTION_REGISTER_OP:
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9867) if (res[i].register_op >= IORING_REGISTER_LAST) {
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9868) ret = -EINVAL;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9869) goto out;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9870) }
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9871)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9872) __set_bit(res[i].register_op,
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9873) ctx->restrictions.register_op);
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9874) break;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9875) case IORING_RESTRICTION_SQE_OP:
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9876) if (res[i].sqe_op >= IORING_OP_LAST) {
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9877) ret = -EINVAL;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9878) goto out;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9879) }
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9880)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9881) __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9882) break;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9883) case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9884) ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9885) break;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9886) case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9887) ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9888) break;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9889) default:
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9890) ret = -EINVAL;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9891) goto out;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9892) }
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9893) }
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9894)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9895) out:
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9896) /* Reset all restrictions if an error happened */
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9897) if (ret != 0)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9898) memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9899) else
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9900) ctx->restrictions.registered = true;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9901)
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9902) kfree(res);
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9903) return ret;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9904) }
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 9905)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9906) static int io_register_enable_rings(struct io_ring_ctx *ctx)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9907) {
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9908) if (!(ctx->flags & IORING_SETUP_R_DISABLED))
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9909) return -EBADFD;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9910)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9911) if (ctx->restrictions.registered)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9912) ctx->restricted = 1;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9913)
0298ef969a110 (Pavel Begunkov 2021-03-08 13:20:57 +0000 9914) ctx->flags &= ~IORING_SETUP_R_DISABLED;
0298ef969a110 (Pavel Begunkov 2021-03-08 13:20:57 +0000 9915) if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
0298ef969a110 (Pavel Begunkov 2021-03-08 13:20:57 +0000 9916) wake_up(&ctx->sq_data->wait);
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9917) return 0;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9918) }
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 9919)
fdecb66281e16 (Pavel Begunkov 2021-04-25 14:32:20 +0100 9920) static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9921) struct io_uring_rsrc_update2 *up,
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9922) unsigned nr_args)
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9923) {
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9924) __u32 tmp;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9925) int err;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9926)
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9927) if (up->resv)
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9928) return -EINVAL;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9929) if (check_add_overflow(up->offset, nr_args, &tmp))
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9930) return -EOVERFLOW;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9931) err = io_rsrc_node_switch_start(ctx);
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9932) if (err)
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9933) return err;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9934)
fdecb66281e16 (Pavel Begunkov 2021-04-25 14:32:20 +0100 9935) switch (type) {
fdecb66281e16 (Pavel Begunkov 2021-04-25 14:32:20 +0100 9936) case IORING_RSRC_FILE:
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9937) return __io_sqe_files_update(ctx, up, nr_args);
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 9938) case IORING_RSRC_BUFFER:
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 9939) return __io_sqe_buffers_update(ctx, up, nr_args);
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9940) }
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9941) return -EINVAL;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9942) }
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9943)
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9944) static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9945) unsigned nr_args)
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9946) {
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9947) struct io_uring_rsrc_update2 up;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9948)
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9949) if (!nr_args)
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9950) return -EINVAL;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9951) memset(&up, 0, sizeof(up));
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9952) if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9953) return -EFAULT;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9954) return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9955) }
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9956)
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9957) static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 9958) unsigned size, unsigned type)
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9959) {
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9960) struct io_uring_rsrc_update2 up;
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9961)
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9962) if (size != sizeof(up))
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 9963) return -EINVAL;
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9964) if (copy_from_user(&up, arg, sizeof(up)))
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9965) return -EFAULT;
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 9966) if (!up.nr || up.resv)
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9967) return -EINVAL;
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 9968) return __io_register_rsrc_update(ctx, type, &up, up.nr);
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9969) }
98f0b3b4f1d51 (Pavel Begunkov 2021-04-25 14:32:19 +0100 9970)
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9971) static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 9972) unsigned int size, unsigned int type)
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9973) {
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9974) struct io_uring_rsrc_register rr;
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9975)
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9976) /* keep it extendible */
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9977) if (size != sizeof(rr))
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9978) return -EINVAL;
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9979)
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9980) memset(&rr, 0, sizeof(rr));
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9981) if (copy_from_user(&rr, arg, size))
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9982) return -EFAULT;
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 9983) if (!rr.nr || rr.resv || rr.resv2)
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9984) return -EINVAL;
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9985)
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 9986) switch (type) {
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9987) case IORING_RSRC_FILE:
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9988) return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9989) rr.nr, u64_to_user_ptr(rr.tags));
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 9990) case IORING_RSRC_BUFFER:
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 9991) return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 9992) rr.nr, u64_to_user_ptr(rr.tags));
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9993) }
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9994) return -EINVAL;
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9995) }
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 9996)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 9997) static bool io_register_op_must_quiesce(int op)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 9998) {
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 9999) switch (op) {
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 10000) case IORING_REGISTER_BUFFERS:
bd54b6fe3316e (Bijan Mottahedeh 2021-04-25 14:32:25 +0100 10001) case IORING_UNREGISTER_BUFFERS:
f4f7d21ce4647 (Pavel Begunkov 2021-04-01 15:44:02 +0100 10002) case IORING_REGISTER_FILES:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10003) case IORING_UNREGISTER_FILES:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10004) case IORING_REGISTER_FILES_UPDATE:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10005) case IORING_REGISTER_PROBE:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10006) case IORING_REGISTER_PERSONALITY:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10007) case IORING_UNREGISTER_PERSONALITY:
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10008) case IORING_REGISTER_FILES2:
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10009) case IORING_REGISTER_FILES_UPDATE2:
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10010) case IORING_REGISTER_BUFFERS2:
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10011) case IORING_REGISTER_BUFFERS_UPDATE:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10012) return false;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10013) default:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10014) return true;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10015) }
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10016) }
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10017)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10018) static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10019) void __user *arg, unsigned nr_args)
b19062a567266 (Jens Axboe 2019-04-15 10:49:38 -0600 10020) __releases(ctx->uring_lock)
b19062a567266 (Jens Axboe 2019-04-15 10:49:38 -0600 10021) __acquires(ctx->uring_lock)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10022) {
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10023) int ret;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10024)
35fa71a030caa (Jens Axboe 2019-04-22 10:23:23 -0600 10025) /*
35fa71a030caa (Jens Axboe 2019-04-22 10:23:23 -0600 10026) * We're inside the ring mutex, if the ref is already dying, then
35fa71a030caa (Jens Axboe 2019-04-22 10:23:23 -0600 10027) * someone else killed the ctx or is already going through
35fa71a030caa (Jens Axboe 2019-04-22 10:23:23 -0600 10028) * io_uring_register().
35fa71a030caa (Jens Axboe 2019-04-22 10:23:23 -0600 10029) */
35fa71a030caa (Jens Axboe 2019-04-22 10:23:23 -0600 10030) if (percpu_ref_is_dying(&ctx->refs))
35fa71a030caa (Jens Axboe 2019-04-22 10:23:23 -0600 10031) return -ENXIO;
35fa71a030caa (Jens Axboe 2019-04-22 10:23:23 -0600 10032)
75c4021aacbd9 (Pavel Begunkov 2021-04-15 13:07:40 +0100 10033) if (ctx->restricted) {
75c4021aacbd9 (Pavel Begunkov 2021-04-15 13:07:40 +0100 10034) if (opcode >= IORING_REGISTER_LAST)
75c4021aacbd9 (Pavel Begunkov 2021-04-15 13:07:40 +0100 10035) return -EINVAL;
75c4021aacbd9 (Pavel Begunkov 2021-04-15 13:07:40 +0100 10036) opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
75c4021aacbd9 (Pavel Begunkov 2021-04-15 13:07:40 +0100 10037) if (!test_bit(opcode, ctx->restrictions.register_op))
75c4021aacbd9 (Pavel Begunkov 2021-04-15 13:07:40 +0100 10038) return -EACCES;
75c4021aacbd9 (Pavel Begunkov 2021-04-15 13:07:40 +0100 10039) }
75c4021aacbd9 (Pavel Begunkov 2021-04-15 13:07:40 +0100 10040)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10041) if (io_register_op_must_quiesce(opcode)) {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10042) percpu_ref_kill(&ctx->refs);
b19062a567266 (Jens Axboe 2019-04-15 10:49:38 -0600 10043)
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10044) /*
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10045) * Drop uring mutex before waiting for references to exit. If
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10046) * another thread is currently inside io_uring_enter() it might
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10047) * need to grab the uring_lock to make progress. If we hold it
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10048) * here across the drain wait, then we can deadlock. It's safe
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10049) * to drop the mutex here, since no new references will come in
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10050) * after we've killed the percpu ref.
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10051) */
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10052) mutex_unlock(&ctx->uring_lock);
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 10053) do {
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 10054) ret = wait_for_completion_interruptible(&ctx->ref_comp);
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 10055) if (!ret)
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 10056) break;
ed6930c9201cd (Jens Axboe 2020-10-08 19:09:46 -0600 10057) ret = io_run_task_work_sig();
ed6930c9201cd (Jens Axboe 2020-10-08 19:09:46 -0600 10058) if (ret < 0)
ed6930c9201cd (Jens Axboe 2020-10-08 19:09:46 -0600 10059) break;
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 10060) } while (1);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10061) mutex_lock(&ctx->uring_lock);
af9c1a44f8dee (Jens Axboe 2020-09-24 13:32:18 -0600 10062)
c150368b49683 (Jens Axboe 2020-01-08 08:26:07 -0700 10063) if (ret) {
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 10064) io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
f70865db5ff35 (Pavel Begunkov 2021-04-11 01:46:40 +0100 10065) return ret;
c150368b49683 (Jens Axboe 2020-01-08 08:26:07 -0700 10066) }
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10067) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10068)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10069) switch (opcode) {
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10070) case IORING_REGISTER_BUFFERS:
634d00df5e1cf (Pavel Begunkov 2021-04-25 14:32:26 +0100 10071) ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10072) break;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10073) case IORING_UNREGISTER_BUFFERS:
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10074) ret = -EINVAL;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10075) if (arg || nr_args)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10076) break;
0a96bbe49994a (Bijan Mottahedeh 2021-01-06 12:39:10 -0800 10077) ret = io_sqe_buffers_unregister(ctx);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10078) break;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 10079) case IORING_REGISTER_FILES:
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 10080) ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 10081) break;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 10082) case IORING_UNREGISTER_FILES:
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 10083) ret = -EINVAL;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 10084) if (arg || nr_args)
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 10085) break;
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 10086) ret = io_sqe_files_unregister(ctx);
6b06314c47e14 (Jens Axboe 2019-01-10 22:13:58 -0700 10087) break;
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 10088) case IORING_REGISTER_FILES_UPDATE:
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 10089) ret = io_register_files_update(ctx, arg, nr_args);
c3a31e605620c (Jens Axboe 2019-10-03 13:59:56 -0600 10090) break;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10091) case IORING_REGISTER_EVENTFD:
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 10092) case IORING_REGISTER_EVENTFD_ASYNC:
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10093) ret = -EINVAL;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10094) if (nr_args != 1)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10095) break;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10096) ret = io_eventfd_register(ctx, arg);
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 10097) if (ret)
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 10098) break;
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 10099) if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 10100) ctx->eventfd_async = 1;
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 10101) else
f2842ab5b72d7 (Jens Axboe 2020-01-08 11:04:00 -0700 10102) ctx->eventfd_async = 0;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10103) break;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10104) case IORING_UNREGISTER_EVENTFD:
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10105) ret = -EINVAL;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10106) if (arg || nr_args)
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10107) break;
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10108) ret = io_eventfd_unregister(ctx);
9b402849e80c8 (Jens Axboe 2019-04-11 11:45:41 -0600 10109) break;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 10110) case IORING_REGISTER_PROBE:
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 10111) ret = -EINVAL;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 10112) if (!arg || nr_args > 256)
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 10113) break;
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 10114) ret = io_probe(ctx, arg, nr_args);
66f4af93da576 (Jens Axboe 2020-01-16 15:36:52 -0700 10115) break;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10116) case IORING_REGISTER_PERSONALITY:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10117) ret = -EINVAL;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10118) if (arg || nr_args)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10119) break;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10120) ret = io_register_personality(ctx);
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10121) break;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10122) case IORING_UNREGISTER_PERSONALITY:
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10123) ret = -EINVAL;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10124) if (arg)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10125) break;
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10126) ret = io_unregister_personality(ctx, nr_args);
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10127) break;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 10128) case IORING_REGISTER_ENABLE_RINGS:
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 10129) ret = -EINVAL;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 10130) if (arg || nr_args)
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 10131) break;
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 10132) ret = io_register_enable_rings(ctx);
7e84e1c7566a1 (Stefano Garzarella 2020-08-27 16:58:31 +0200 10133) break;
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 10134) case IORING_REGISTER_RESTRICTIONS:
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 10135) ret = io_register_restrictions(ctx, arg, nr_args);
21b55dbc06530 (Stefano Garzarella 2020-08-27 16:58:30 +0200 10136) break;
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10137) case IORING_REGISTER_FILES2:
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10138) ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10139) break;
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10140) case IORING_REGISTER_FILES_UPDATE2:
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10141) ret = io_register_rsrc_update(ctx, arg, nr_args,
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10142) IORING_RSRC_FILE);
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10143) break;
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10144) case IORING_REGISTER_BUFFERS2:
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10145) ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
792e35824be9a (Pavel Begunkov 2021-04-25 14:32:21 +0100 10146) break;
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10147) case IORING_REGISTER_BUFFERS_UPDATE:
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10148) ret = io_register_rsrc_update(ctx, arg, nr_args,
992da01aa932b (Pavel Begunkov 2021-06-10 16:37:37 +0100 10149) IORING_RSRC_BUFFER);
c3bdad0271834 (Pavel Begunkov 2021-04-25 14:32:22 +0100 10150) break;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10151) default:
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10152) ret = -EINVAL;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10153) break;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10154) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10155)
071698e13ac6b (Jens Axboe 2020-01-28 10:04:42 -0700 10156) if (io_register_op_must_quiesce(opcode)) {
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10157) /* bring the ctx back to life */
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10158) percpu_ref_reinit(&ctx->refs);
0f158b4cf20e7 (Jens Axboe 2020-05-14 17:18:39 -0600 10159) reinit_completion(&ctx->ref_comp);
05f3fb3c53975 (Jens Axboe 2019-12-09 11:22:50 -0700 10160) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10161) return ret;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10162) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10163)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10164) SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10165) void __user *, arg, unsigned int, nr_args)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10166) {
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10167) struct io_ring_ctx *ctx;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10168) long ret = -EBADF;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10169) struct fd f;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10170)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10171) f = fdget(fd);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10172) if (!f.file)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10173) return -EBADF;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10174)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10175) ret = -EOPNOTSUPP;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10176) if (f.file->f_op != &io_uring_fops)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10177) goto out_fput;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10178)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10179) ctx = f.file->private_data;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10180)
b6c23dd5a4831 (Pavel Begunkov 2021-02-20 15:17:18 +0000 10181) io_run_task_work();
b6c23dd5a4831 (Pavel Begunkov 2021-02-20 15:17:18 +0000 10182)
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10183) mutex_lock(&ctx->uring_lock);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10184) ret = __io_uring_register(ctx, opcode, arg, nr_args);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10185) mutex_unlock(&ctx->uring_lock);
c826bd7a743f2 (Dmitrii Dolgov 2019-10-15 19:02:01 +0200 10186) trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
c826bd7a743f2 (Dmitrii Dolgov 2019-10-15 19:02:01 +0200 10187) ctx->cq_ev_fd != NULL, ret);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10188) out_fput:
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10189) fdput(f);
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10190) return ret;
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10191) }
edafccee56ff3 (Jens Axboe 2019-01-09 09:16:05 -0700 10192)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 10193) static int __init io_uring_init(void)
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 10194) {
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10195) #define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10196) BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10197) BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10198) } while (0)
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10199)
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10200) #define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10201) __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10202) BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10203) BUILD_BUG_SQE_ELEM(0, __u8, opcode);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10204) BUILD_BUG_SQE_ELEM(1, __u8, flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10205) BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10206) BUILD_BUG_SQE_ELEM(4, __s32, fd);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10207) BUILD_BUG_SQE_ELEM(8, __u64, off);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10208) BUILD_BUG_SQE_ELEM(8, __u64, addr2);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10209) BUILD_BUG_SQE_ELEM(16, __u64, addr);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 10210) BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10211) BUILD_BUG_SQE_ELEM(24, __u32, len);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10212) BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10213) BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10214) BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10215) BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
5769a351b89cd (Jiufei Xue 2020-06-17 17:53:55 +0800 10216) BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
5769a351b89cd (Jiufei Xue 2020-06-17 17:53:55 +0800 10217) BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10218) BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10219) BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10220) BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10221) BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10222) BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10223) BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10224) BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10225) BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 10226) BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10227) BUILD_BUG_SQE_ELEM(32, __u64, user_data);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10228) BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10229) BUILD_BUG_SQE_ELEM(42, __u16, personality);
7d67af2c01340 (Pavel Begunkov 2020-02-24 11:32:45 +0300 10230) BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
d7f62e825fd19 (Stefan Metzmacher 2020-01-29 14:39:41 +0100 10231)
b0d658ec88a69 (Pavel Begunkov 2021-04-27 16:13:53 +0100 10232) BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
b0d658ec88a69 (Pavel Begunkov 2021-04-27 16:13:53 +0100 10233) sizeof(struct io_uring_rsrc_update));
b0d658ec88a69 (Pavel Begunkov 2021-04-27 16:13:53 +0100 10234) BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
b0d658ec88a69 (Pavel Begunkov 2021-04-27 16:13:53 +0100 10235) sizeof(struct io_uring_rsrc_update2));
b0d658ec88a69 (Pavel Begunkov 2021-04-27 16:13:53 +0100 10236) /* should fit into one byte */
b0d658ec88a69 (Pavel Begunkov 2021-04-27 16:13:53 +0100 10237) BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
b0d658ec88a69 (Pavel Begunkov 2021-04-27 16:13:53 +0100 10238)
d3656344fea03 (Jens Axboe 2019-12-18 09:50:26 -0700 10239) BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
84557871f2ff3 (Jens Axboe 2020-03-03 15:28:17 -0700 10240) BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
91f245d5d5de0 (Jens Axboe 2021-02-09 13:48:50 -0700 10241) req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
91f245d5d5de0 (Jens Axboe 2021-02-09 13:48:50 -0700 10242) SLAB_ACCOUNT);
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 10243) return 0;
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 10244) };
2b188cc1bb857 (Jens Axboe 2019-01-07 10:46:33 -0700 10245) __initcall(io_uring_init);
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags