771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 1) // SPDX-License-Identifier: GPL-2.0
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 2) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 3) * Basic worker thread pool for io_uring
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 4) *
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 5) * Copyright (C) 2019 Jens Axboe
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 6) *
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 7) */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 8) #include <linux/kernel.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 9) #include <linux/init.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 10) #include <linux/errno.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 11) #include <linux/sched/signal.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 12) #include <linux/mm.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 13) #include <linux/sched/mm.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 14) #include <linux/percpu.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 15) #include <linux/slab.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 16) #include <linux/rculist_nulls.h>
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 17) #include <linux/cpu.h>
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 18) #include <linux/tracehook.h>
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 19)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 20) #include "io-wq.h"
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 21)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 22) #define WORKER_IDLE_TIMEOUT (5 * HZ)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 23)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 24) enum {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 25) IO_WORKER_F_UP = 1, /* up and active */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 26) IO_WORKER_F_RUNNING = 2, /* account as running */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 27) IO_WORKER_F_FREE = 4, /* worker on free list */
145cc8c665f40 (Jens Axboe 2020-09-26 12:37:46 -0600 28) IO_WORKER_F_FIXED = 8, /* static idle worker */
145cc8c665f40 (Jens Axboe 2020-09-26 12:37:46 -0600 29) IO_WORKER_F_BOUND = 16, /* is doing bounded work */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 30) };
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 31)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 32) enum {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 33) IO_WQ_BIT_EXIT = 0, /* wq exiting */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 34) };
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 35)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 36) enum {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 37) IO_WQE_FLAG_STALLED = 1, /* stalled on hash */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 38) };
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 39)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 40) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 41) * One for each thread in a wqe pool
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 42) */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 43) struct io_worker {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 44) refcount_t ref;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 45) unsigned flags;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 46) struct hlist_nulls_node nulls_node;
e61df66c69b11 (Jens Axboe 2019-11-13 13:54:49 -0700 47) struct list_head all_list;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 48) struct task_struct *task;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 49) struct io_wqe *wqe;
36c2f9223e84c (Jens Axboe 2019-11-13 09:43:34 -0700 50)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 51) struct io_wq_work *cur_work;
36c2f9223e84c (Jens Axboe 2019-11-13 09:43:34 -0700 52) spinlock_t lock;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 53)
eb2de9418d56b (Jens Axboe 2021-02-23 19:59:06 -0700 54) struct completion ref_done;
eb2de9418d56b (Jens Axboe 2021-02-23 19:59:06 -0700 55)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 56) struct rcu_head rcu;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 57) };
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 58)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 59) #if BITS_PER_LONG == 64
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 60) #define IO_WQ_HASH_ORDER 6
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 61) #else
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 62) #define IO_WQ_HASH_ORDER 5
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 63) #endif
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 64)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 65) #define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 66)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 67) struct io_wqe_acct {
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 68) unsigned nr_workers;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 69) unsigned max_workers;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 70) int index;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 71) atomic_t nr_running;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 72) };
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 73)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 74) enum {
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 75) IO_WQ_ACCT_BOUND,
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 76) IO_WQ_ACCT_UNBOUND,
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 77) };
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 78)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 79) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 80) * Per-node worker thread pool
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 81) */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 82) struct io_wqe {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 83) struct {
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 84) raw_spinlock_t lock;
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 85) struct io_wq_work_list work_list;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 86) unsigned flags;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 87) } ____cacheline_aligned_in_smp;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 88)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 89) int node;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 90) struct io_wqe_acct acct[2];
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 91)
021d1cdda3875 (Jens Axboe 2019-11-14 08:00:41 -0700 92) struct hlist_nulls_head free_list;
e61df66c69b11 (Jens Axboe 2019-11-13 13:54:49 -0700 93) struct list_head all_list;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 94)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 95) struct wait_queue_entry wait;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 96)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 97) struct io_wq *wq;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 98) struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS];
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 99) };
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 100)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 101) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 102) * Per io_wq state
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 103) */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 104) struct io_wq {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 105) struct io_wqe **wqes;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 106) unsigned long state;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 107)
e9fd939654f17 (Pavel Begunkov 2020-03-04 16:14:12 +0300 108) free_work_fn *free_work;
f5fa38c59cb0b (Pavel Begunkov 2020-06-08 21:08:20 +0300 109) io_wq_work_fn *do_work;
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 110)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 111) struct io_wq_hash *hash;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 112)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 113) refcount_t refs;
848f7e1887c46 (Jens Axboe 2020-01-23 15:33:32 -0700 114)
fb3a1f6c745cc (Jens Axboe 2021-02-26 09:47:20 -0700 115) atomic_t worker_refs;
fb3a1f6c745cc (Jens Axboe 2021-02-26 09:47:20 -0700 116) struct completion worker_done;
fb3a1f6c745cc (Jens Axboe 2021-02-26 09:47:20 -0700 117)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 118) struct hlist_node cpuhp_node;
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 119)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 120) struct task_struct *task;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 121) };
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 122)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 123) static enum cpuhp_state io_wq_online;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 124)
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 125) struct io_cb_cancel_data {
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 126) work_cancel_fn *fn;
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 127) void *data;
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 128) int nr_running;
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 129) int nr_pending;
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 130) bool cancel_all;
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 131) };
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 132)
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 133) static void create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index, bool first);
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 134) static void io_wqe_dec_running(struct io_worker *worker);
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 135)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 136) static bool io_worker_get(struct io_worker *worker)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 137) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 138) return refcount_inc_not_zero(&worker->ref);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 139) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 140)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 141) static void io_worker_release(struct io_worker *worker)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 142) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 143) if (refcount_dec_and_test(&worker->ref))
eb2de9418d56b (Jens Axboe 2021-02-23 19:59:06 -0700 144) complete(&worker->ref_done);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 145) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 146)
8418f22a53795 (Pavel Begunkov 2021-03-22 01:58:28 +0000 147) static inline struct io_wqe_acct *io_get_acct(struct io_wqe *wqe, bool bound)
8418f22a53795 (Pavel Begunkov 2021-03-22 01:58:28 +0000 148) {
8418f22a53795 (Pavel Begunkov 2021-03-22 01:58:28 +0000 149) return &wqe->acct[bound ? IO_WQ_ACCT_BOUND : IO_WQ_ACCT_UNBOUND];
8418f22a53795 (Pavel Begunkov 2021-03-22 01:58:28 +0000 150) }
8418f22a53795 (Pavel Begunkov 2021-03-22 01:58:28 +0000 151)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 152) static inline struct io_wqe_acct *io_work_get_acct(struct io_wqe *wqe,
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 153) struct io_wq_work *work)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 154) {
8418f22a53795 (Pavel Begunkov 2021-03-22 01:58:28 +0000 155) return io_get_acct(wqe, !(work->flags & IO_WQ_WORK_UNBOUND));
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 156) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 157)
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 158) static inline struct io_wqe_acct *io_wqe_get_acct(struct io_worker *worker)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 159) {
8418f22a53795 (Pavel Begunkov 2021-03-22 01:58:28 +0000 160) return io_get_acct(worker->wqe, worker->flags & IO_WORKER_F_BOUND);
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 161) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 162)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 163) static void io_worker_ref_put(struct io_wq *wq)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 164) {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 165) if (atomic_dec_and_test(&wq->worker_refs))
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 166) complete(&wq->worker_done);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 167) }
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 168)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 169) static void io_worker_exit(struct io_worker *worker)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 170) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 171) struct io_wqe *wqe = worker->wqe;
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 172) struct io_wqe_acct *acct = io_wqe_get_acct(worker);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 173)
eb2de9418d56b (Jens Axboe 2021-02-23 19:59:06 -0700 174) if (refcount_dec_and_test(&worker->ref))
eb2de9418d56b (Jens Axboe 2021-02-23 19:59:06 -0700 175) complete(&worker->ref_done);
eb2de9418d56b (Jens Axboe 2021-02-23 19:59:06 -0700 176) wait_for_completion(&worker->ref_done);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 177)
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 178) raw_spin_lock_irq(&wqe->lock);
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 179) if (worker->flags & IO_WORKER_F_FREE)
bf1daa4bfc77a (Jens Axboe 2021-02-16 18:00:55 -0700 180) hlist_nulls_del_rcu(&worker->nulls_node);
e61df66c69b11 (Jens Axboe 2019-11-13 13:54:49 -0700 181) list_del_rcu(&worker->all_list);
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 182) acct->nr_workers--;
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 183) preempt_disable();
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 184) io_wqe_dec_running(worker);
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 185) worker->flags = 0;
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 186) current->flags &= ~PF_IO_WORKER;
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 187) preempt_enable();
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 188) raw_spin_unlock_irq(&wqe->lock);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 189)
364b05fd06e87 (YueHaibing 2019-11-02 15:55:01 +0800 190) kfree_rcu(worker, rcu);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 191) io_worker_ref_put(wqe->wq);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 192) do_exit(0);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 193) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 194)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 195) static inline bool io_wqe_run_queue(struct io_wqe *wqe)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 196) __must_hold(wqe->lock)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 197) {
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 198) if (!wq_list_empty(&wqe->work_list) &&
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 199) !(wqe->flags & IO_WQE_FLAG_STALLED))
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 200) return true;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 201) return false;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 202) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 203)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 204) /*
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 205) * Check head of free list for an available worker. If one isn't available,
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 206) * caller must create one.
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 207) */
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 208) static bool io_wqe_activate_free_worker(struct io_wqe *wqe)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 209) __must_hold(RCU)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 210) {
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 211) struct hlist_nulls_node *n;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 212) struct io_worker *worker;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 213)
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 214) /*
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 215) * Iterate free_list and see if we can find an idle worker to
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 216) * activate. If a given worker is on the free_list but in the process
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 217) * of exiting, keep trying.
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 218) */
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 219) hlist_nulls_for_each_entry_rcu(worker, n, &wqe->free_list, nulls_node) {
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 220) if (!io_worker_get(worker))
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 221) continue;
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 222) if (wake_up_process(worker->task)) {
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 223) io_worker_release(worker);
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 224) return true;
6ed983ea4a12a (Jens Axboe 2021-08-03 09:14:35 -0600 225) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 226) io_worker_release(worker);
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 227) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 228)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 229) return false;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 230) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 231)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 232) /*
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 233) * We need a worker. If we find a free one, we're good. If not, and we're
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 234) * below the max number of workers, create one.
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 235) */
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 236) static void io_wqe_wake_worker(struct io_wqe *wqe, struct io_wqe_acct *acct)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 237) {
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 238) bool ret;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 239)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 240) /*
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 241) * Most likely an attempt to queue unbounded work on an io_wq that
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 242) * wasn't setup with any unbounded workers.
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 243) */
a8eca6968b1e6 (Pavel Begunkov 2021-06-17 18:13:59 +0100 244) if (unlikely(!acct->max_workers))
a8eca6968b1e6 (Pavel Begunkov 2021-06-17 18:13:59 +0100 245) pr_warn_once("io-wq is not configured for unbound workers");
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 246)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 247) rcu_read_lock();
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 248) ret = io_wqe_activate_free_worker(wqe);
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 249) rcu_read_unlock();
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 250)
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 251) if (!ret) {
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 252) bool do_create = false, first = false;
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 253)
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 254) raw_spin_lock_irq(&wqe->lock);
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 255) if (acct->nr_workers < acct->max_workers) {
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 256) atomic_inc(&acct->nr_running);
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 257) atomic_inc(&wqe->wq->worker_refs);
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 258) if (!acct->nr_workers)
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 259) first = true;
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 260) acct->nr_workers++;
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 261) do_create = true;
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 262) }
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 263) raw_spin_unlock_irq(&wqe->lock);
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 264) if (do_create)
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 265) create_io_worker(wqe->wq, wqe, acct->index, first);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 266) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 267) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 268)
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 269) static void io_wqe_inc_running(struct io_worker *worker)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 270) {
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 271) struct io_wqe_acct *acct = io_wqe_get_acct(worker);
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 272)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 273) atomic_inc(&acct->nr_running);
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 274) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 275)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 276) struct create_worker_data {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 277) struct callback_head work;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 278) struct io_wqe *wqe;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 279) int index;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 280) };
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 281)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 282) static void create_worker_cb(struct callback_head *cb)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 283) {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 284) struct create_worker_data *cwd;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 285) struct io_wq *wq;
d92eaad3ed951 (Hao Xu 2021-08-05 18:05:38 +0800 286) struct io_wqe *wqe;
d92eaad3ed951 (Hao Xu 2021-08-05 18:05:38 +0800 287) struct io_wqe_acct *acct;
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 288) bool do_create = false, first = false;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 289)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 290) cwd = container_of(cb, struct create_worker_data, work);
d92eaad3ed951 (Hao Xu 2021-08-05 18:05:38 +0800 291) wqe = cwd->wqe;
d92eaad3ed951 (Hao Xu 2021-08-05 18:05:38 +0800 292) wq = wqe->wq;
d92eaad3ed951 (Hao Xu 2021-08-05 18:05:38 +0800 293) acct = &wqe->acct[cwd->index];
d92eaad3ed951 (Hao Xu 2021-08-05 18:05:38 +0800 294) raw_spin_lock_irq(&wqe->lock);
815a0fe3f415e (Hao Xu 2021-08-08 21:54:33 +0800 295) if (acct->nr_workers < acct->max_workers) {
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 296) if (!acct->nr_workers)
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 297) first = true;
d92eaad3ed951 (Hao Xu 2021-08-05 18:05:38 +0800 298) acct->nr_workers++;
815a0fe3f415e (Hao Xu 2021-08-08 21:54:33 +0800 299) do_create = true;
815a0fe3f415e (Hao Xu 2021-08-08 21:54:33 +0800 300) }
d92eaad3ed951 (Hao Xu 2021-08-05 18:05:38 +0800 301) raw_spin_unlock_irq(&wqe->lock);
815a0fe3f415e (Hao Xu 2021-08-08 21:54:33 +0800 302) if (do_create) {
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 303) create_io_worker(wq, wqe, cwd->index, first);
815a0fe3f415e (Hao Xu 2021-08-08 21:54:33 +0800 304) } else {
815a0fe3f415e (Hao Xu 2021-08-08 21:54:33 +0800 305) atomic_dec(&acct->nr_running);
815a0fe3f415e (Hao Xu 2021-08-08 21:54:33 +0800 306) io_worker_ref_put(wq);
815a0fe3f415e (Hao Xu 2021-08-08 21:54:33 +0800 307) }
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 308) kfree(cwd);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 309) }
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 310)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 311) static void io_queue_worker_create(struct io_wqe *wqe, struct io_wqe_acct *acct)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 312) {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 313) struct create_worker_data *cwd;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 314) struct io_wq *wq = wqe->wq;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 315)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 316) /* raced with exit, just ignore create call */
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 317) if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 318) goto fail;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 319)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 320) cwd = kmalloc(sizeof(*cwd), GFP_ATOMIC);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 321) if (cwd) {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 322) init_task_work(&cwd->work, create_worker_cb);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 323) cwd->wqe = wqe;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 324) cwd->index = acct->index;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 325) if (!task_work_add(wq->task, &cwd->work, TWA_SIGNAL))
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 326) return;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 327)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 328) kfree(cwd);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 329) }
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 330) fail:
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 331) atomic_dec(&acct->nr_running);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 332) io_worker_ref_put(wq);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 333) }
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 334)
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 335) static void io_wqe_dec_running(struct io_worker *worker)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 336) __must_hold(wqe->lock)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 337) {
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 338) struct io_wqe_acct *acct = io_wqe_get_acct(worker);
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 339) struct io_wqe *wqe = worker->wqe;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 340)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 341) if (!(worker->flags & IO_WORKER_F_UP))
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 342) return;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 343)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 344) if (atomic_dec_and_test(&acct->nr_running) && io_wqe_run_queue(wqe)) {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 345) atomic_inc(&acct->nr_running);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 346) atomic_inc(&wqe->wq->worker_refs);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 347) io_queue_worker_create(wqe, acct);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 348) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 349) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 350)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 351) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 352) * Worker will start processing some work. Move it to the busy list, if
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 353) * it's currently on the freelist
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 354) */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 355) static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker,
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 356) struct io_wq_work *work)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 357) __must_hold(wqe->lock)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 358) {
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 359) bool worker_bound, work_bound;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 360)
417b5052be9ec (Hao Xu 2021-04-06 11:08:45 +0800 361) BUILD_BUG_ON((IO_WQ_ACCT_UNBOUND ^ IO_WQ_ACCT_BOUND) != 1);
417b5052be9ec (Hao Xu 2021-04-06 11:08:45 +0800 362)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 363) if (worker->flags & IO_WORKER_F_FREE) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 364) worker->flags &= ~IO_WORKER_F_FREE;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 365) hlist_nulls_del_init_rcu(&worker->nulls_node);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 366) }
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 367)
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 368) /*
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 369) * If worker is moving from bound to unbound (or vice versa), then
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 370) * ensure we update the running accounting.
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 371) */
b2e9c7d64b7ec (Dan Carpenter 2019-11-19 09:22:16 +0300 372) worker_bound = (worker->flags & IO_WORKER_F_BOUND) != 0;
b2e9c7d64b7ec (Dan Carpenter 2019-11-19 09:22:16 +0300 373) work_bound = (work->flags & IO_WQ_WORK_UNBOUND) == 0;
b2e9c7d64b7ec (Dan Carpenter 2019-11-19 09:22:16 +0300 374) if (worker_bound != work_bound) {
417b5052be9ec (Hao Xu 2021-04-06 11:08:45 +0800 375) int index = work_bound ? IO_WQ_ACCT_UNBOUND : IO_WQ_ACCT_BOUND;
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 376) io_wqe_dec_running(worker);
417b5052be9ec (Hao Xu 2021-04-06 11:08:45 +0800 377) worker->flags ^= IO_WORKER_F_BOUND;
417b5052be9ec (Hao Xu 2021-04-06 11:08:45 +0800 378) wqe->acct[index].nr_workers--;
417b5052be9ec (Hao Xu 2021-04-06 11:08:45 +0800 379) wqe->acct[index ^ 1].nr_workers++;
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 380) io_wqe_inc_running(worker);
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 381) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 382) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 383)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 384) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 385) * No work, worker going to sleep. Move to freelist, and unuse mm if we
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 386) * have one attached. Dropping the mm may potentially sleep, so we drop
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 387) * the lock in that case and return success. Since the caller has to
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 388) * retry the loop in that case (we changed task state), we don't regrab
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 389) * the lock if we return success.
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 390) */
c6d77d92b7e53 (Jens Axboe 2021-02-15 13:26:34 -0700 391) static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 392) __must_hold(wqe->lock)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 393) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 394) if (!(worker->flags & IO_WORKER_F_FREE)) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 395) worker->flags |= IO_WORKER_F_FREE;
021d1cdda3875 (Jens Axboe 2019-11-14 08:00:41 -0700 396) hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 397) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 398) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 399)
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 400) static inline unsigned int io_get_work_hash(struct io_wq_work *work)
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 401) {
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 402) return work->flags >> IO_WQ_HASH_SHIFT;
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 403) }
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 404)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 405) static void io_wait_on_hash(struct io_wqe *wqe, unsigned int hash)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 406) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 407) struct io_wq *wq = wqe->wq;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 408)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 409) spin_lock(&wq->hash->wait.lock);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 410) if (list_empty(&wqe->wait.entry)) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 411) __add_wait_queue(&wq->hash->wait, &wqe->wait);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 412) if (!test_bit(hash, &wq->hash->map)) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 413) __set_current_state(TASK_RUNNING);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 414) list_del_init(&wqe->wait.entry);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 415) }
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 416) }
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 417) spin_unlock(&wq->hash->wait.lock);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 418) }
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 419)
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 420) static struct io_wq_work *io_get_next_work(struct io_wqe *wqe)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 421) __must_hold(wqe->lock)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 422) {
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 423) struct io_wq_work_node *node, *prev;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 424) struct io_wq_work *work, *tail;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 425) unsigned int stall_hash = -1U;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 426)
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 427) wq_list_for_each(node, prev, &wqe->work_list) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 428) unsigned int hash;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 429)
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 430) work = container_of(node, struct io_wq_work, list);
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 431)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 432) /* not hashed, can run anytime */
8766dd516c535 (Pavel Begunkov 2020-03-14 00:31:04 +0300 433) if (!io_wq_is_hashed(work)) {
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 434) wq_list_del(&wqe->work_list, node, prev);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 435) return work;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 436) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 437)
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 438) hash = io_get_work_hash(work);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 439) /* all items with this hash lie in [work, tail] */
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 440) tail = wqe->hash_tail[hash];
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 441)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 442) /* hashed, can run if not already running */
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 443) if (!test_and_set_bit(hash, &wqe->wq->hash->map)) {
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 444) wqe->hash_tail[hash] = NULL;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 445) wq_list_cut(&wqe->work_list, &tail->list, prev);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 446) return work;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 447) }
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 448) if (stall_hash == -1U)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 449) stall_hash = hash;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 450) /* fast forward to a next hash, for-each will fix up @prev */
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 451) node = &tail->list;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 452) }
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 453)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 454) if (stall_hash != -1U) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 455) raw_spin_unlock(&wqe->lock);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 456) io_wait_on_hash(wqe, stall_hash);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 457) raw_spin_lock(&wqe->lock);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 458) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 459)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 460) return NULL;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 461) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 462)
00ddff431a458 (Jens Axboe 2021-03-21 07:06:56 -0600 463) static bool io_flush_signals(void)
cccf0ee834559 (Jens Axboe 2020-01-27 16:34:48 -0700 464) {
0b8cfa974dfc9 (Jens Axboe 2021-03-21 14:16:08 -0600 465) if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL))) {
00ddff431a458 (Jens Axboe 2021-03-21 07:06:56 -0600 466) __set_current_state(TASK_RUNNING);
0b8cfa974dfc9 (Jens Axboe 2021-03-21 14:16:08 -0600 467) tracehook_notify_signal();
00ddff431a458 (Jens Axboe 2021-03-21 07:06:56 -0600 468) return true;
cccf0ee834559 (Jens Axboe 2020-01-27 16:34:48 -0700 469) }
00ddff431a458 (Jens Axboe 2021-03-21 07:06:56 -0600 470) return false;
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 471) }
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 472)
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 473) static void io_assign_current_work(struct io_worker *worker,
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 474) struct io_wq_work *work)
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 475) {
d78298e73a344 (Pavel Begunkov 2020-03-14 00:31:03 +0300 476) if (work) {
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 477) io_flush_signals();
d78298e73a344 (Pavel Begunkov 2020-03-14 00:31:03 +0300 478) cond_resched();
d78298e73a344 (Pavel Begunkov 2020-03-14 00:31:03 +0300 479) }
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 480)
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 481) spin_lock_irq(&worker->lock);
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 482) worker->cur_work = work;
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 483) spin_unlock_irq(&worker->lock);
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 484) }
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 485)
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 486) static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 487)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 488) static void io_worker_handle_work(struct io_worker *worker)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 489) __releases(wqe->lock)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 490) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 491) struct io_wqe *wqe = worker->wqe;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 492) struct io_wq *wq = wqe->wq;
c60eb049f4a19 (Pavel Begunkov 2021-04-08 01:54:42 +0100 493) bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 494)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 495) do {
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 496) struct io_wq_work *work;
f462fd36fc436 (Pavel Begunkov 2020-03-04 16:14:11 +0300 497) get_next:
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 498) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 499) * If we got some work, mark us as busy. If we didn't, but
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 500) * the list isn't empty, it means we stalled on hashed work.
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 501) * Mark us stalled so we don't keep looking for work when we
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 502) * can't make progress, any work completion or insertion will
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 503) * clear the stalled flag.
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 504) */
60cf46ae60544 (Pavel Begunkov 2020-03-14 00:31:05 +0300 505) work = io_get_next_work(wqe);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 506) if (work)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 507) __io_worker_busy(wqe, worker, work);
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 508) else if (!wq_list_empty(&wqe->work_list))
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 509) wqe->flags |= IO_WQE_FLAG_STALLED;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 510)
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 511) raw_spin_unlock_irq(&wqe->lock);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 512) if (!work)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 513) break;
58e3931987377 (Pavel Begunkov 2020-03-04 16:14:10 +0300 514) io_assign_current_work(worker, work);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 515) __set_current_state(TASK_RUNNING);
36c2f9223e84c (Jens Axboe 2019-11-13 09:43:34 -0700 516)
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 517) /* handle a whole dependent link */
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 518) do {
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 519) struct io_wq_work *next_hashed, *linked;
b089ed390b5c9 (Pavel Begunkov 2020-07-25 14:42:00 +0300 520) unsigned int hash = io_get_work_hash(work);
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 521)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 522) next_hashed = wq_next_work(work);
c60eb049f4a19 (Pavel Begunkov 2021-04-08 01:54:42 +0100 523)
c60eb049f4a19 (Pavel Begunkov 2021-04-08 01:54:42 +0100 524) if (unlikely(do_kill) && (work->flags & IO_WQ_WORK_UNBOUND))
c60eb049f4a19 (Pavel Begunkov 2021-04-08 01:54:42 +0100 525) work->flags |= IO_WQ_WORK_CANCEL;
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 526) wq->do_work(work);
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 527) io_assign_current_work(worker, NULL);
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 528)
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 529) linked = wq->free_work(work);
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 530) work = next_hashed;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 531) if (!work && linked && !io_wq_is_hashed(linked)) {
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 532) work = linked;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 533) linked = NULL;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 534) }
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 535) io_assign_current_work(worker, work);
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 536) if (linked)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 537) io_wqe_enqueue(wqe, linked);
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 538)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 539) if (hash != -1U && !next_hashed) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 540) clear_bit(hash, &wq->hash->map);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 541) if (wq_has_sleeper(&wq->hash->wait))
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 542) wake_up(&wq->hash->wait);
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 543) raw_spin_lock_irq(&wqe->lock);
dc026a73c7221 (Pavel Begunkov 2020-03-04 16:14:09 +0300 544) wqe->flags &= ~IO_WQE_FLAG_STALLED;
f462fd36fc436 (Pavel Begunkov 2020-03-04 16:14:11 +0300 545) /* skip unnecessary unlock-lock wqe->lock */
f462fd36fc436 (Pavel Begunkov 2020-03-04 16:14:11 +0300 546) if (!work)
f462fd36fc436 (Pavel Begunkov 2020-03-04 16:14:11 +0300 547) goto get_next;
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 548) raw_spin_unlock_irq(&wqe->lock);
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 549) }
58e3931987377 (Pavel Begunkov 2020-03-04 16:14:10 +0300 550) } while (work);
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 551)
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 552) raw_spin_lock_irq(&wqe->lock);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 553) } while (1);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 554) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 555)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 556) static int io_wqe_worker(void *data)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 557) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 558) struct io_worker *worker = data;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 559) struct io_wqe *wqe = worker->wqe;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 560) struct io_wq *wq = wqe->wq;
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 561) char buf[TASK_COMM_LEN];
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 562)
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 563) worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 564)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 565) snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task->pid);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 566) set_task_comm(current, buf);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 567)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 568) while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
16efa4fce3b7a (Jens Axboe 2021-03-12 20:26:13 -0700 569) long ret;
16efa4fce3b7a (Jens Axboe 2021-03-12 20:26:13 -0700 570)
506d95ff5d6aa (Jens Axboe 2019-12-07 21:03:59 -0700 571) set_current_state(TASK_INTERRUPTIBLE);
e995d5123ed43 (Jens Axboe 2019-12-07 21:06:46 -0700 572) loop:
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 573) raw_spin_lock_irq(&wqe->lock);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 574) if (io_wqe_run_queue(wqe)) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 575) io_worker_handle_work(worker);
e995d5123ed43 (Jens Axboe 2019-12-07 21:06:46 -0700 576) goto loop;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 577) }
c6d77d92b7e53 (Jens Axboe 2021-02-15 13:26:34 -0700 578) __io_worker_idle(wqe, worker);
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 579) raw_spin_unlock_irq(&wqe->lock);
00ddff431a458 (Jens Axboe 2021-03-21 07:06:56 -0600 580) if (io_flush_signals())
00ddff431a458 (Jens Axboe 2021-03-21 07:06:56 -0600 581) continue;
16efa4fce3b7a (Jens Axboe 2021-03-12 20:26:13 -0700 582) ret = schedule_timeout(WORKER_IDLE_TIMEOUT);
dbe1bdbb39db7 (Jens Axboe 2021-03-25 18:16:06 -0600 583) if (signal_pending(current)) {
dbe1bdbb39db7 (Jens Axboe 2021-03-25 18:16:06 -0600 584) struct ksignal ksig;
dbe1bdbb39db7 (Jens Axboe 2021-03-25 18:16:06 -0600 585)
dbe1bdbb39db7 (Jens Axboe 2021-03-25 18:16:06 -0600 586) if (!get_signal(&ksig))
dbe1bdbb39db7 (Jens Axboe 2021-03-25 18:16:06 -0600 587) continue;
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 588) break;
dbe1bdbb39db7 (Jens Axboe 2021-03-25 18:16:06 -0600 589) }
dbe1bdbb39db7 (Jens Axboe 2021-03-25 18:16:06 -0600 590) if (ret)
dbe1bdbb39db7 (Jens Axboe 2021-03-25 18:16:06 -0600 591) continue;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 592) /* timed out, exit unless we're the fixed worker */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 593) if (test_bit(IO_WQ_BIT_EXIT, &wq->state) ||
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 594) !(worker->flags & IO_WORKER_F_FIXED))
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 595) break;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 596) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 597)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 598) if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 599) raw_spin_lock_irq(&wqe->lock);
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 600) if (!wq_list_empty(&wqe->work_list))
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 601) io_worker_handle_work(worker);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 602) else
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 603) raw_spin_unlock_irq(&wqe->lock);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 604) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 605)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 606) io_worker_exit(worker);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 607) return 0;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 608) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 609)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 610) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 611) * Called when a worker is scheduled in. Mark us as currently running.
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 612) */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 613) void io_wq_worker_running(struct task_struct *tsk)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 614) {
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 615) struct io_worker *worker = tsk->pf_io_worker;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 616)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 617) if (!worker)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 618) return;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 619) if (!(worker->flags & IO_WORKER_F_UP))
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 620) return;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 621) if (worker->flags & IO_WORKER_F_RUNNING)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 622) return;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 623) worker->flags |= IO_WORKER_F_RUNNING;
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 624) io_wqe_inc_running(worker);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 625) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 626)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 627) /*
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 628) * Called when worker is going to sleep. If there are no workers currently
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 629) * running and we have work pending, wake up a free one or create a new one.
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 630) */
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 631) void io_wq_worker_sleeping(struct task_struct *tsk)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 632) {
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 633) struct io_worker *worker = tsk->pf_io_worker;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 634)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 635) if (!worker)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 636) return;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 637) if (!(worker->flags & IO_WORKER_F_UP))
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 638) return;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 639) if (!(worker->flags & IO_WORKER_F_RUNNING))
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 640) return;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 641)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 642) worker->flags &= ~IO_WORKER_F_RUNNING;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 643)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 644) raw_spin_lock_irq(&worker->wqe->lock);
958234d5ec932 (Jens Axboe 2021-02-17 09:00:57 -0700 645) io_wqe_dec_running(worker);
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 646) raw_spin_unlock_irq(&worker->wqe->lock);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 647) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 648)
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 649) static void create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index, bool first)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 650) {
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 651) struct io_wqe_acct *acct = &wqe->acct[index];
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 652) struct io_worker *worker;
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 653) struct task_struct *tsk;
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 654)
8b3e78b5955ab (Jens Axboe 2021-02-23 15:34:06 -0700 655) __set_current_state(TASK_RUNNING);
8b3e78b5955ab (Jens Axboe 2021-02-23 15:34:06 -0700 656)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 657) worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node);
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 658) if (!worker)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 659) goto fail;
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 660)
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 661) refcount_set(&worker->ref, 1);
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 662) worker->nulls_node.pprev = NULL;
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 663) worker->wqe = wqe;
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 664) spin_lock_init(&worker->lock);
eb2de9418d56b (Jens Axboe 2021-02-23 19:59:06 -0700 665) init_completion(&worker->ref_done);
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 666)
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 667) tsk = create_io_thread(io_wqe_worker, worker, wqe->node);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 668) if (IS_ERR(tsk)) {
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 669) kfree(worker);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 670) fail:
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 671) atomic_dec(&acct->nr_running);
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 672) raw_spin_lock_irq(&wqe->lock);
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 673) acct->nr_workers--;
ebad5646c0598 (Hao Xu 2021-08-05 18:05:37 +0800 674) raw_spin_unlock_irq(&wqe->lock);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 675) io_worker_ref_put(wq);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 676) return;
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 677) }
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 678)
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 679) tsk->pf_io_worker = worker;
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 680) worker->task = tsk;
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 681) set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node));
e22bc9b481a90 (Jens Axboe 2021-03-09 19:49:02 -0700 682) tsk->flags |= PF_NO_SETAFFINITY;
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 683)
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 684) raw_spin_lock_irq(&wqe->lock);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 685) hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 686) list_add_tail_rcu(&worker->all_list, &wqe->all_list);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 687) worker->flags |= IO_WORKER_F_FREE;
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 688) if (index == IO_WQ_ACCT_BOUND)
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 689) worker->flags |= IO_WORKER_F_BOUND;
f49d457950b99 (Hao Xu 2021-08-08 21:54:34 +0800 690) if (first && (worker->flags & IO_WORKER_F_BOUND))
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 691) worker->flags |= IO_WORKER_F_FIXED;
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 692) raw_spin_unlock_irq(&wqe->lock);
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 693) wake_up_new_task(tsk);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 694) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 695)
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 696) /*
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 697) * Iterate the passed in list and call the specific function for each
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 698) * worker that isn't exiting
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 699) */
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 700) static bool io_wq_for_each_worker(struct io_wqe *wqe,
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 701) bool (*func)(struct io_worker *, void *),
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 702) void *data)
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 703) {
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 704) struct io_worker *worker;
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 705) bool ret = false;
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 706)
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 707) list_for_each_entry_rcu(worker, &wqe->all_list, all_list) {
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 708) if (io_worker_get(worker)) {
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 709) /* no task if node is/was offline */
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 710) if (worker->task)
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 711) ret = func(worker, data);
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 712) io_worker_release(worker);
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 713) if (ret)
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 714) break;
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 715) }
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 716) }
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 717)
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 718) return ret;
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 719) }
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 720)
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 721) static bool io_wq_worker_wake(struct io_worker *worker, void *data)
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 722) {
46fe18b16c465 (Jens Axboe 2021-03-04 12:39:36 -0700 723) set_notify_signal(worker->task);
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 724) wake_up_process(worker->task);
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 725) return false;
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 726) }
c4068bf898dda (Hillf Danton 2020-09-26 21:26:55 +0800 727)
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 728) static bool io_wq_work_match_all(struct io_wq_work *work, void *data)
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 729) {
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 730) return true;
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 731) }
f01272541d2cd (Jens Axboe 2021-03-03 15:47:04 -0700 732)
e9fd939654f17 (Pavel Begunkov 2020-03-04 16:14:12 +0300 733) static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)
fc04c39bae01a (Pavel Begunkov 2020-03-01 19:18:19 +0300 734) {
e9fd939654f17 (Pavel Begunkov 2020-03-04 16:14:12 +0300 735) struct io_wq *wq = wqe->wq;
e9fd939654f17 (Pavel Begunkov 2020-03-04 16:14:12 +0300 736)
fc04c39bae01a (Pavel Begunkov 2020-03-01 19:18:19 +0300 737) do {
fc04c39bae01a (Pavel Begunkov 2020-03-01 19:18:19 +0300 738) work->flags |= IO_WQ_WORK_CANCEL;
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 739) wq->do_work(work);
5280f7e530f71 (Pavel Begunkov 2021-02-04 13:52:08 +0000 740) work = wq->free_work(work);
fc04c39bae01a (Pavel Begunkov 2020-03-01 19:18:19 +0300 741) } while (work);
fc04c39bae01a (Pavel Begunkov 2020-03-01 19:18:19 +0300 742) }
fc04c39bae01a (Pavel Begunkov 2020-03-01 19:18:19 +0300 743)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 744) static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 745) {
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 746) unsigned int hash;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 747) struct io_wq_work *tail;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 748)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 749) if (!io_wq_is_hashed(work)) {
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 750) append:
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 751) wq_list_add_tail(&work->list, &wqe->work_list);
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 752) return;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 753) }
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 754)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 755) hash = io_get_work_hash(work);
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 756) tail = wqe->hash_tail[hash];
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 757) wqe->hash_tail[hash] = work;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 758) if (!tail)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 759) goto append;
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 760)
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 761) wq_list_add_after(&work->list, &tail->list, &wqe->work_list);
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 762) }
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 763)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 764) static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 765) {
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 766) struct io_wqe_acct *acct = io_work_get_acct(wqe, work);
895e2ca0f693c (Jens Axboe 2019-12-17 08:46:33 -0700 767) int work_flags;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 768) unsigned long flags;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 769)
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 770) /*
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 771) * If io-wq is exiting for this task, or if the request has explicitly
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 772) * been marked as one that should not get executed, cancel it here.
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 773) */
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 774) if (test_bit(IO_WQ_BIT_EXIT, &wqe->wq->state) ||
989b27104a976 (Jens Axboe 2021-07-23 11:53:54 -0600 775) (work->flags & IO_WQ_WORK_CANCEL)) {
70e35125093b0 (yangerkun 2021-03-09 11:04:10 +0800 776) io_run_cancel(work, wqe);
4fb6ac326204b (Jens Axboe 2021-02-25 10:17:09 -0700 777) return;
4fb6ac326204b (Jens Axboe 2021-02-25 10:17:09 -0700 778) }
4fb6ac326204b (Jens Axboe 2021-02-25 10:17:09 -0700 779)
895e2ca0f693c (Jens Axboe 2019-12-17 08:46:33 -0700 780) work_flags = work->flags;
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 781) raw_spin_lock_irqsave(&wqe->lock, flags);
86f3cd1b589a1 (Pavel Begunkov 2020-03-23 22:57:22 +0300 782) io_wqe_insert_work(wqe, work);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 783) wqe->flags &= ~IO_WQE_FLAG_STALLED;
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 784) raw_spin_unlock_irqrestore(&wqe->lock, flags);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 785)
895e2ca0f693c (Jens Axboe 2019-12-17 08:46:33 -0700 786) if ((work_flags & IO_WQ_WORK_CONCURRENT) ||
895e2ca0f693c (Jens Axboe 2019-12-17 08:46:33 -0700 787) !atomic_read(&acct->nr_running))
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 788) io_wqe_wake_worker(wqe, acct);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 789) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 790)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 791) void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 792) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 793) struct io_wqe *wqe = wq->wqes[numa_node_id()];
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 794)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 795) io_wqe_enqueue(wqe, work);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 796) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 797)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 798) /*
8766dd516c535 (Pavel Begunkov 2020-03-14 00:31:04 +0300 799) * Work items that hash to the same value will not be done in parallel.
8766dd516c535 (Pavel Begunkov 2020-03-14 00:31:04 +0300 800) * Used to limit concurrent writes, generally hashed by inode.
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 801) */
8766dd516c535 (Pavel Begunkov 2020-03-14 00:31:04 +0300 802) void io_wq_hash_work(struct io_wq_work *work, void *val)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 803) {
8766dd516c535 (Pavel Begunkov 2020-03-14 00:31:04 +0300 804) unsigned int bit;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 805)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 806) bit = hash_ptr(val, IO_WQ_HASH_ORDER);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 807) work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT));
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 808) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 809)
2293b4195800f (Pavel Begunkov 2020-03-07 01:15:39 +0300 810) static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 811) {
2293b4195800f (Pavel Begunkov 2020-03-07 01:15:39 +0300 812) struct io_cb_cancel_data *match = data;
6f72653e76a51 (Jens Axboe 2019-11-05 13:51:51 -0700 813) unsigned long flags;
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 814)
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 815) /*
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 816) * Hold the lock to avoid ->cur_work going out of scope, caller
36c2f9223e84c (Jens Axboe 2019-11-13 09:43:34 -0700 817) * may dereference the passed in work.
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 818) */
36c2f9223e84c (Jens Axboe 2019-11-13 09:43:34 -0700 819) spin_lock_irqsave(&worker->lock, flags);
62755e35dfb2b (Jens Axboe 2019-10-28 21:49:21 -0600 820) if (worker->cur_work &&
2293b4195800f (Pavel Begunkov 2020-03-07 01:15:39 +0300 821) match->fn(worker->cur_work, match->data)) {
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 822) set_notify_signal(worker->task);
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 823) match->nr_running++;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 824) }
36c2f9223e84c (Jens Axboe 2019-11-13 09:43:34 -0700 825) spin_unlock_irqrestore(&worker->lock, flags);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 826)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 827) return match->nr_running && !match->cancel_all;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 828) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 829)
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 830) static inline void io_wqe_remove_pending(struct io_wqe *wqe,
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 831) struct io_wq_work *work,
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 832) struct io_wq_work_node *prev)
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 833) {
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 834) unsigned int hash = io_get_work_hash(work);
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 835) struct io_wq_work *prev_work = NULL;
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 836)
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 837) if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) {
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 838) if (prev)
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 839) prev_work = container_of(prev, struct io_wq_work, list);
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 840) if (prev_work && io_get_work_hash(prev_work) == hash)
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 841) wqe->hash_tail[hash] = prev_work;
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 842) else
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 843) wqe->hash_tail[hash] = NULL;
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 844) }
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 845) wq_list_del(&wqe->work_list, &work->list, prev);
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 846) }
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 847)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 848) static void io_wqe_cancel_pending_work(struct io_wqe *wqe,
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 849) struct io_cb_cancel_data *match)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 850) {
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 851) struct io_wq_work_node *node, *prev;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 852) struct io_wq_work *work;
6f72653e76a51 (Jens Axboe 2019-11-05 13:51:51 -0700 853) unsigned long flags;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 854)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 855) retry:
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 856) raw_spin_lock_irqsave(&wqe->lock, flags);
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 857) wq_list_for_each(node, prev, &wqe->work_list) {
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 858) work = container_of(node, struct io_wq_work, list);
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 859) if (!match->fn(work, match->data))
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 860) continue;
204361a77f401 (Pavel Begunkov 2020-08-23 20:33:10 +0300 861) io_wqe_remove_pending(wqe, work, prev);
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 862) raw_spin_unlock_irqrestore(&wqe->lock, flags);
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 863) io_run_cancel(work, wqe);
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 864) match->nr_pending++;
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 865) if (!match->cancel_all)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 866) return;
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 867)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 868) /* not safe to continue after unlock */
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 869) goto retry;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 870) }
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 871) raw_spin_unlock_irqrestore(&wqe->lock, flags);
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 872) }
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 873)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 874) static void io_wqe_cancel_running_work(struct io_wqe *wqe,
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 875) struct io_cb_cancel_data *match)
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 876) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 877) rcu_read_lock();
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 878) io_wq_for_each_worker(wqe, io_wq_worker_cancel, match);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 879) rcu_read_unlock();
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 880) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 881)
2293b4195800f (Pavel Begunkov 2020-03-07 01:15:39 +0300 882) enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 883) void *data, bool cancel_all)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 884) {
2293b4195800f (Pavel Begunkov 2020-03-07 01:15:39 +0300 885) struct io_cb_cancel_data match = {
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 886) .fn = cancel,
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 887) .data = data,
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 888) .cancel_all = cancel_all,
00bcda13dcbf6 (Jens Axboe 2020-02-08 19:13:32 -0700 889) };
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 890) int node;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 891)
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 892) /*
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 893) * First check pending list, if we're lucky we can just remove it
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 894) * from there. CANCEL_OK means that the work is returned as-new,
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 895) * no completion will be posted for it.
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 896) */
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 897) for_each_node(node) {
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 898) struct io_wqe *wqe = wq->wqes[node];
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 899)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 900) io_wqe_cancel_pending_work(wqe, &match);
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 901) if (match.nr_pending && !match.cancel_all)
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 902) return IO_WQ_CANCEL_OK;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 903) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 904)
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 905) /*
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 906) * Now check if a free (going busy) or busy worker has the work
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 907) * currently running. If we find it there, we'll return CANCEL_RUNNING
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 908) * as an indication that we attempt to signal cancellation. The
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 909) * completion will run normally in this case.
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 910) */
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 911) for_each_node(node) {
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 912) struct io_wqe *wqe = wq->wqes[node];
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 913)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 914) io_wqe_cancel_running_work(wqe, &match);
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 915) if (match.nr_running && !match.cancel_all)
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 916) return IO_WQ_CANCEL_RUNNING;
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 917) }
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 918)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 919) if (match.nr_running)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 920) return IO_WQ_CANCEL_RUNNING;
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 921) if (match.nr_pending)
4f26bda1522c3 (Pavel Begunkov 2020-06-15 10:24:03 +0300 922) return IO_WQ_CANCEL_OK;
f4c2665e33f48 (Pavel Begunkov 2020-06-15 10:24:02 +0300 923) return IO_WQ_CANCEL_NOTFOUND;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 924) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 925)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 926) static int io_wqe_hash_wake(struct wait_queue_entry *wait, unsigned mode,
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 927) int sync, void *key)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 928) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 929) struct io_wqe *wqe = container_of(wait, struct io_wqe, wait);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 930)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 931) list_del_init(&wait->entry);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 932)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 933) rcu_read_lock();
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 934) io_wqe_activate_free_worker(wqe);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 935) rcu_read_unlock();
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 936) return 1;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 937) }
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 938)
576a347b7af8a (Jens Axboe 2019-11-25 08:49:20 -0700 939) struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 940) {
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 941) int ret = -ENOMEM, node;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 942) struct io_wq *wq;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 943)
f5fa38c59cb0b (Pavel Begunkov 2020-06-08 21:08:20 +0300 944) if (WARN_ON_ONCE(!data->free_work || !data->do_work))
e9fd939654f17 (Pavel Begunkov 2020-03-04 16:14:12 +0300 945) return ERR_PTR(-EINVAL);
a8eca6968b1e6 (Pavel Begunkov 2021-06-17 18:13:59 +0100 946) if (WARN_ON_ONCE(!bounded))
a8eca6968b1e6 (Pavel Begunkov 2021-06-17 18:13:59 +0100 947) return ERR_PTR(-EINVAL);
e9fd939654f17 (Pavel Begunkov 2020-03-04 16:14:12 +0300 948)
ad6e005ca68de (Jann Horn 2019-11-26 17:39:45 +0100 949) wq = kzalloc(sizeof(*wq), GFP_KERNEL);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 950) if (!wq)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 951) return ERR_PTR(-ENOMEM);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 952)
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 953) wq->wqes = kcalloc(nr_node_ids, sizeof(struct io_wqe *), GFP_KERNEL);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 954) if (!wq->wqes)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 955) goto err_wq;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 956)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 957) ret = cpuhp_state_add_instance_nocalls(io_wq_online, &wq->cpuhp_node);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 958) if (ret)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 959) goto err_wqes;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 960)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 961) refcount_inc(&data->hash->refs);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 962) wq->hash = data->hash;
e9fd939654f17 (Pavel Begunkov 2020-03-04 16:14:12 +0300 963) wq->free_work = data->free_work;
f5fa38c59cb0b (Pavel Begunkov 2020-06-08 21:08:20 +0300 964) wq->do_work = data->do_work;
7d7230652e7c7 (Jens Axboe 2019-11-12 22:31:31 -0700 965)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 966) ret = -ENOMEM;
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 967) for_each_node(node) {
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 968) struct io_wqe *wqe;
7563439adfae1 (Jens Axboe 2020-02-11 06:30:06 -0700 969) int alloc_node = node;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 970)
7563439adfae1 (Jens Axboe 2020-02-11 06:30:06 -0700 971) if (!node_online(alloc_node))
7563439adfae1 (Jens Axboe 2020-02-11 06:30:06 -0700 972) alloc_node = NUMA_NO_NODE;
7563439adfae1 (Jens Axboe 2020-02-11 06:30:06 -0700 973) wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 974) if (!wqe)
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 975) goto err;
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 976) wq->wqes[node] = wqe;
7563439adfae1 (Jens Axboe 2020-02-11 06:30:06 -0700 977) wqe->node = alloc_node;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 978) wqe->acct[IO_WQ_ACCT_BOUND].index = IO_WQ_ACCT_BOUND;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 979) wqe->acct[IO_WQ_ACCT_UNBOUND].index = IO_WQ_ACCT_UNBOUND;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 980) wqe->acct[IO_WQ_ACCT_BOUND].max_workers = bounded;
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 981) atomic_set(&wqe->acct[IO_WQ_ACCT_BOUND].nr_running, 0);
728f13e730093 (Jens Axboe 2021-02-21 16:02:53 -0700 982) wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers =
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 983) task_rlimit(current, RLIMIT_NPROC);
c5def4ab84949 (Jens Axboe 2019-11-07 11:41:16 -0700 984) atomic_set(&wqe->acct[IO_WQ_ACCT_UNBOUND].nr_running, 0);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 985) wqe->wait.func = io_wqe_hash_wake;
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 986) INIT_LIST_HEAD(&wqe->wait.entry);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 987) wqe->wq = wq;
95da84659226d (Sebastian Andrzej Siewior 2020-09-01 10:41:46 +0200 988) raw_spin_lock_init(&wqe->lock);
6206f0e180d4e (Jens Axboe 2019-11-26 11:59:32 -0700 989) INIT_WQ_LIST(&wqe->work_list);
021d1cdda3875 (Jens Axboe 2019-11-14 08:00:41 -0700 990) INIT_HLIST_NULLS_HEAD(&wqe->free_list, 0);
e61df66c69b11 (Jens Axboe 2019-11-13 13:54:49 -0700 991) INIT_LIST_HEAD(&wqe->all_list);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 992) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 993)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 994) wq->task = get_task_struct(data->task);
3bfe6106693b6 (Jens Axboe 2021-02-16 14:15:30 -0700 995) refcount_set(&wq->refs, 1);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 996) atomic_set(&wq->worker_refs, 1);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 997) init_completion(&wq->worker_done);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 998) return wq;
b60fda6000a99 (Jens Axboe 2019-11-19 08:37:07 -0700 999) err:
dc7bbc9ef361b (Jens Axboe 2021-03-01 09:09:56 -0700 1000) io_wq_put_hash(data->hash);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1001) cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 1002) for_each_node(node)
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 1003) kfree(wq->wqes[node]);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1004) err_wqes:
b60fda6000a99 (Jens Axboe 2019-11-19 08:37:07 -0700 1005) kfree(wq->wqes);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1006) err_wq:
b60fda6000a99 (Jens Axboe 2019-11-19 08:37:07 -0700 1007) kfree(wq);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 1008) return ERR_PTR(ret);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 1009) }
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 1010)
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1011) static bool io_task_work_match(struct callback_head *cb, void *data)
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1012) {
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1013) struct create_worker_data *cwd;
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1014)
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1015) if (cb->func != create_worker_cb)
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1016) return false;
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1017) cwd = container_of(cb, struct create_worker_data, work);
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1018) return cwd->wqe->wq == data;
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1019) }
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1020)
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1021) void io_wq_exit_start(struct io_wq *wq)
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1022) {
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1023) set_bit(IO_WQ_BIT_EXIT, &wq->state);
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1024) }
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1025)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1026) static void io_wq_exit_workers(struct io_wq *wq)
afcc4015d1bf5 (Jens Axboe 2021-02-26 13:48:19 -0700 1027) {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1028) struct callback_head *cb;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1029) int node;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1030)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1031) if (!wq->task)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1032) return;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1033)
c80ca4707d1aa (Jens Axboe 2021-04-01 19:57:07 -0600 1034) while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1035) struct create_worker_data *cwd;
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1036)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1037) cwd = container_of(cb, struct create_worker_data, work);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1038) atomic_dec(&cwd->wqe->acct[cwd->index].nr_running);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1039) io_worker_ref_put(wq);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1040) kfree(cwd);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1041) }
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1042)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1043) rcu_read_lock();
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1044) for_each_node(node) {
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1045) struct io_wqe *wqe = wq->wqes[node];
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1046)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1047) io_wq_for_each_worker(wqe, io_wq_worker_wake, NULL);
afcc4015d1bf5 (Jens Axboe 2021-02-26 13:48:19 -0700 1048) }
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1049) rcu_read_unlock();
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1050) io_worker_ref_put(wq);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1051) wait_for_completion(&wq->worker_done);
3743c1723bfc6 (Zqiang 2021-05-26 13:08:26 +0800 1052)
3743c1723bfc6 (Zqiang 2021-05-26 13:08:26 +0800 1053) for_each_node(node) {
3743c1723bfc6 (Zqiang 2021-05-26 13:08:26 +0800 1054) spin_lock_irq(&wq->hash->wait.lock);
3743c1723bfc6 (Zqiang 2021-05-26 13:08:26 +0800 1055) list_del_init(&wq->wqes[node]->wait.entry);
3743c1723bfc6 (Zqiang 2021-05-26 13:08:26 +0800 1056) spin_unlock_irq(&wq->hash->wait.lock);
3743c1723bfc6 (Zqiang 2021-05-26 13:08:26 +0800 1057) }
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1058) put_task_struct(wq->task);
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1059) wq->task = NULL;
afcc4015d1bf5 (Jens Axboe 2021-02-26 13:48:19 -0700 1060) }
afcc4015d1bf5 (Jens Axboe 2021-02-26 13:48:19 -0700 1061)
4fb6ac326204b (Jens Axboe 2021-02-25 10:17:09 -0700 1062) static void io_wq_destroy(struct io_wq *wq)
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 1063) {
3fc50ab559f5a (Jann Horn 2019-11-26 19:10:20 +0100 1064) int node;
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 1065)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1066) cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1067)
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 1068) for_each_node(node) {
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 1069) struct io_wqe *wqe = wq->wqes[node];
f5d2d23bf0d94 (Jens Axboe 2021-03-25 10:16:12 -0600 1070) struct io_cb_cancel_data match = {
f5d2d23bf0d94 (Jens Axboe 2021-03-25 10:16:12 -0600 1071) .fn = io_wq_work_match_all,
f5d2d23bf0d94 (Jens Axboe 2021-03-25 10:16:12 -0600 1072) .cancel_all = true,
f5d2d23bf0d94 (Jens Axboe 2021-03-25 10:16:12 -0600 1073) };
f5d2d23bf0d94 (Jens Axboe 2021-03-25 10:16:12 -0600 1074) io_wqe_cancel_pending_work(wqe, &match);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 1075) kfree(wqe);
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 1076) }
e941894eae31b (Jens Axboe 2021-02-19 12:33:30 -0700 1077) io_wq_put_hash(wq->hash);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 1078) kfree(wq->wqes);
771b53d033e86 (Jens Axboe 2019-10-22 10:25:58 -0600 1079) kfree(wq);
4fb6ac326204b (Jens Axboe 2021-02-25 10:17:09 -0700 1080) }
4fb6ac326204b (Jens Axboe 2021-02-25 10:17:09 -0700 1081)
afcc4015d1bf5 (Jens Axboe 2021-02-26 13:48:19 -0700 1082) void io_wq_put_and_exit(struct io_wq *wq)
afcc4015d1bf5 (Jens Axboe 2021-02-26 13:48:19 -0700 1083) {
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1084) WARN_ON_ONCE(!test_bit(IO_WQ_BIT_EXIT, &wq->state));
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1085)
685fe7feedb96 (Jens Axboe 2021-03-08 09:37:51 -0700 1086) io_wq_exit_workers(wq);
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1087) if (refcount_dec_and_test(&wq->refs))
17a91051fe63b (Pavel Begunkov 2021-05-23 15:48:39 +0100 1088) io_wq_destroy(wq);
afcc4015d1bf5 (Jens Axboe 2021-02-26 13:48:19 -0700 1089) }
afcc4015d1bf5 (Jens Axboe 2021-02-26 13:48:19 -0700 1090)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1091) static bool io_wq_worker_affinity(struct io_worker *worker, void *data)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1092) {
e0051d7d18e0b (Peter Zijlstra 2021-04-08 11:44:50 +0200 1093) set_cpus_allowed_ptr(worker->task, cpumask_of_node(worker->wqe->node));
e0051d7d18e0b (Peter Zijlstra 2021-04-08 11:44:50 +0200 1094)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1095) return false;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1096) }
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1097)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1098) static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1099) {
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1100) struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1101) int i;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1102)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1103) rcu_read_lock();
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1104) for_each_node(i)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1105) io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, NULL);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1106) rcu_read_unlock();
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1107) return 0;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1108) }
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1109)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1110) static __init int io_wq_init(void)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1111) {
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1112) int ret;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1113)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1114) ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online",
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1115) io_wq_cpu_online, NULL);
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1116) if (ret < 0)
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1117) return ret;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1118) io_wq_online = ret;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1119) return 0;
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1120) }
43c01fbefdf11 (Jens Axboe 2020-10-22 09:02:50 -0600 1121) subsys_initcall(io_wq_init);
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags