VisionFive2 Linux kernel

771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600   1) #ifndef INTERNAL_IO_WQ_H
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600   2) #define INTERNAL_IO_WQ_H
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600   3) 
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700   4) #include <linux/refcount.h>
98447d65b4a7a (Jens Axboe        2020-10-14 10:48:51 -0600   5) 
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600   6) struct io_wq;
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600   7) 
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600   8) enum {
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600   9) 	IO_WQ_WORK_CANCEL	= 1,
e883a79d8ced8 (Pavel Begunkov    2020-06-25 18:20:53 +0300  10) 	IO_WQ_WORK_HASHED	= 2,
e883a79d8ced8 (Pavel Begunkov    2020-06-25 18:20:53 +0300  11) 	IO_WQ_WORK_UNBOUND	= 4,
e883a79d8ced8 (Pavel Begunkov    2020-06-25 18:20:53 +0300  12) 	IO_WQ_WORK_CONCURRENT	= 16,
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  13) 
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  14) 	IO_WQ_HASH_SHIFT	= 24,	/* upper 8 bits are used for hash key */
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  15) };
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  16) 
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  17) enum io_wq_cancel {
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  18) 	IO_WQ_CANCEL_OK,	/* cancelled before started */
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  19) 	IO_WQ_CANCEL_RUNNING,	/* found, running, and attempted cancelled */
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  20) 	IO_WQ_CANCEL_NOTFOUND,	/* work not found */
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  21) };
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  22) 
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  23) struct io_wq_work_node {
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  24) 	struct io_wq_work_node *next;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  25) };
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  26) 
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  27) struct io_wq_work_list {
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  28) 	struct io_wq_work_node *first;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  29) 	struct io_wq_work_node *last;
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  30) };
53e043b2b432e (Stefan Metzmacher 2021-03-15 12:56:56 +0100  31) 
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  32) static inline void wq_list_add_after(struct io_wq_work_node *node,
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  33) 				     struct io_wq_work_node *pos,
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  34) 				     struct io_wq_work_list *list)
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  35) {
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  36) 	struct io_wq_work_node *next = pos->next;
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  37) 
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  38) 	pos->next = node;
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  39) 	node->next = next;
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  40) 	if (!next)
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  41) 		list->last = node;
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  42) }
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  43) 
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  44) static inline void wq_list_add_tail(struct io_wq_work_node *node,
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  45) 				    struct io_wq_work_list *list)
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  46) {
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  47) 	if (!list->first) {
e995d5123ed43 (Jens Axboe        2019-12-07 21:06:46 -0700  48) 		list->last = node;
e995d5123ed43 (Jens Axboe        2019-12-07 21:06:46 -0700  49) 		WRITE_ONCE(list->first, node);
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  50) 	} else {
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  51) 		list->last->next = node;
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  52) 		list->last = node;
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  53) 	}
0020ef04e4857 (Xiaoguang Wang    2020-12-18 15:26:48 +0800  54) 	node->next = NULL;
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  55) }
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  56) 
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  57) static inline void wq_list_cut(struct io_wq_work_list *list,
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  58) 			       struct io_wq_work_node *last,
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  59) 			       struct io_wq_work_node *prev)
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  60) {
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  61) 	/* first in the list, if prev==NULL */
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  62) 	if (!prev)
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  63) 		WRITE_ONCE(list->first, last->next);
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  64) 	else
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  65) 		prev->next = last->next;
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  66) 
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  67) 	if (last == list->last)
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  68) 		list->last = prev;
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  69) 	last->next = NULL;
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  70) }
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  71) 
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  72) static inline void wq_list_del(struct io_wq_work_list *list,
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  73) 			       struct io_wq_work_node *node,
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  74) 			       struct io_wq_work_node *prev)
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  75) {
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  76) 	wq_list_cut(list, node, prev);
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  77) }
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  78) 
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  79) #define wq_list_for_each(pos, prv, head)			\
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  80) 	for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  81) 
e995d5123ed43 (Jens Axboe        2019-12-07 21:06:46 -0700  82) #define wq_list_empty(list)	(READ_ONCE((list)->first) == NULL)
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  83) #define INIT_WQ_LIST(list)	do {				\
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  84) 	(list)->first = NULL;					\
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  85) 	(list)->last = NULL;					\
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  86) } while (0)
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  87) 
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  88) struct io_wq_work {
18a542ff19ad1 (Pavel Begunkov    2020-03-23 00:23:29 +0300  89) 	struct io_wq_work_node list;
6206f0e180d4e (Jens Axboe        2019-11-26 11:59:32 -0700  90) 	unsigned flags;
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  91) };
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600  92) 
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  93) static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  94) {
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  95) 	if (!work->list.next)
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  96) 		return NULL;
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  97) 
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  98) 	return container_of(work->list.next, struct io_wq_work, list);
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300  99) }
86f3cd1b589a1 (Pavel Begunkov    2020-03-23 22:57:22 +0300 100) 
5280f7e530f71 (Pavel Begunkov    2021-02-04 13:52:08 +0000 101) typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *);
5280f7e530f71 (Pavel Begunkov    2021-02-04 13:52:08 +0000 102) typedef void (io_wq_work_fn)(struct io_wq_work *);
7d7230652e7c7 (Jens Axboe        2019-11-12 22:31:31 -0700 103) 
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 104) struct io_wq_hash {
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 105) 	refcount_t refs;
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 106) 	unsigned long map;
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 107) 	struct wait_queue_head wait;
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 108) };
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 109) 
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 110) static inline void io_wq_put_hash(struct io_wq_hash *hash)
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 111) {
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 112) 	if (refcount_dec_and_test(&hash->refs))
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 113) 		kfree(hash);
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 114) }
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 115) 
576a347b7af8a (Jens Axboe        2019-11-25 08:49:20 -0700 116) struct io_wq_data {
e941894eae31b (Jens Axboe        2021-02-19 12:33:30 -0700 117) 	struct io_wq_hash *hash;
685fe7feedb96 (Jens Axboe        2021-03-08 09:37:51 -0700 118) 	struct task_struct *task;
f5fa38c59cb0b (Pavel Begunkov    2020-06-08 21:08:20 +0300 119) 	io_wq_work_fn *do_work;
e9fd939654f17 (Pavel Begunkov    2020-03-04 16:14:12 +0300 120) 	free_work_fn *free_work;
576a347b7af8a (Jens Axboe        2019-11-25 08:49:20 -0700 121) };
576a347b7af8a (Jens Axboe        2019-11-25 08:49:20 -0700 122) 
576a347b7af8a (Jens Axboe        2019-11-25 08:49:20 -0700 123) struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
17a91051fe63b (Pavel Begunkov    2021-05-23 15:48:39 +0100 124) void io_wq_exit_start(struct io_wq *wq);
afcc4015d1bf5 (Jens Axboe        2021-02-26 13:48:19 -0700 125) void io_wq_put_and_exit(struct io_wq *wq);
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 126) 
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 127) void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
8766dd516c535 (Pavel Begunkov    2020-03-14 00:31:04 +0300 128) void io_wq_hash_work(struct io_wq_work *work, void *val);
8766dd516c535 (Pavel Begunkov    2020-03-14 00:31:04 +0300 129) 
8766dd516c535 (Pavel Begunkov    2020-03-14 00:31:04 +0300 130) static inline bool io_wq_is_hashed(struct io_wq_work *work)
8766dd516c535 (Pavel Begunkov    2020-03-14 00:31:04 +0300 131) {
8766dd516c535 (Pavel Begunkov    2020-03-14 00:31:04 +0300 132) 	return work->flags & IO_WQ_WORK_HASHED;
8766dd516c535 (Pavel Begunkov    2020-03-14 00:31:04 +0300 133) }
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 134) 
62755e35dfb2b (Jens Axboe        2019-10-28 21:49:21 -0600 135) typedef bool (work_cancel_fn)(struct io_wq_work *, void *);
62755e35dfb2b (Jens Axboe        2019-10-28 21:49:21 -0600 136) 
62755e35dfb2b (Jens Axboe        2019-10-28 21:49:21 -0600 137) enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
4f26bda1522c3 (Pavel Begunkov    2020-06-15 10:24:03 +0300 138) 					void *data, bool cancel_all);
62755e35dfb2b (Jens Axboe        2019-10-28 21:49:21 -0600 139) 
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 140) #if defined(CONFIG_IO_WQ)
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 141) extern void io_wq_worker_sleeping(struct task_struct *);
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 142) extern void io_wq_worker_running(struct task_struct *);
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 143) #else
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 144) static inline void io_wq_worker_sleeping(struct task_struct *tsk)
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 145) {
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 146) }
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 147) static inline void io_wq_worker_running(struct task_struct *tsk)
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 148) {
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 149) }
525b305d61ede (Jens Axboe        2019-12-17 14:13:37 -0700 150) #endif
771b53d033e86 (Jens Axboe        2019-10-22 10:25:58 -0600 151) 
525b305d61ede (Jens Axboe        2019-12-17 14:13:37 -0700 152) static inline bool io_wq_current_is_worker(void)
525b305d61ede (Jens Axboe        2019-12-17 14:13:37 -0700 153) {
3bfe6106693b6 (Jens Axboe        2021-02-16 14:15:30 -0700 154) 	return in_task() && (current->flags & PF_IO_WORKER) &&
3bfe6106693b6 (Jens Axboe        2021-02-16 14:15:30 -0700 155) 		current->pf_io_worker;
525b305d61ede (Jens Axboe        2019-12-17 14:13:37 -0700 156) }
525b305d61ede (Jens Axboe        2019-12-17 14:13:37 -0700 157) #endif