457c899653991 (Thomas Gleixner 2019-05-19 13:08:55 +0100 1) // SPDX-License-Identifier: GPL-2.0-only
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 2) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 3) * fs/direct-io.c
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 4) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 5) * Copyright (C) 2002, Linus Torvalds.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 6) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 7) * O_DIRECT
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 8) *
e1f8e87449147 (Francois Cami 2008-10-15 22:01:59 -0700 9) * 04Jul2002 Andrew Morton
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 10) * Initial version
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 11) * 11Sep2002 janetinc@us.ibm.com
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 12) * added readv/writev support.
e1f8e87449147 (Francois Cami 2008-10-15 22:01:59 -0700 13) * 29Oct2002 Andrew Morton
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 14) * rewrote bio_add_page() support.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 15) * 30Oct2002 pbadari@us.ibm.com
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 16) * added support for non-aligned IO.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 17) * 06Nov2002 pbadari@us.ibm.com
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 18) * added asynchronous IO support.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 19) * 21Jul2003 nathans@sgi.com
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 20) * added IO completion notifier.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 21) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 22)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 23) #include <linux/kernel.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 24) #include <linux/module.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 25) #include <linux/types.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 26) #include <linux/fs.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 27) #include <linux/mm.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 28) #include <linux/slab.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 29) #include <linux/highmem.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 30) #include <linux/pagemap.h>
98c4d57decf97 (Andrew Morton 2006-12-10 02:19:47 -0800 31) #include <linux/task_io_accounting_ops.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 32) #include <linux/bio.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 33) #include <linux/wait.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 34) #include <linux/err.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 35) #include <linux/blkdev.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 36) #include <linux/buffer_head.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 37) #include <linux/rwsem.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 38) #include <linux/uio.h>
60063497a95e7 (Arun Sharma 2011-07-26 16:09:06 -0700 39) #include <linux/atomic.h>
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 40) #include <linux/prefetch.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 41)
b16155a0b01ae (Eric Biggers 2020-01-04 12:59:49 -0800 42) #include "internal.h"
b16155a0b01ae (Eric Biggers 2020-01-04 12:59:49 -0800 43)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 44) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 45) * How many user pages to map in one call to get_user_pages(). This determines
cde1ecb3247f6 (Andi Kleen 2011-08-01 21:38:04 -0700 46) * the size of a structure in the slab cache
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 47) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 48) #define DIO_PAGES 64
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 49)
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 50) /*
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 51) * Flags for dio_complete()
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 52) */
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 53) #define DIO_COMPLETE_ASYNC 0x01 /* This is async IO */
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 54) #define DIO_COMPLETE_INVALIDATE 0x02 /* Can invalidate pages */
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 55)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 56) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 57) * This code generally works in units of "dio_blocks". A dio_block is
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 58) * somewhere between the hard sector size and the filesystem block size. it
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 59) * is determined on a per-invocation basis. When talking to the filesystem
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 60) * we need to convert dio_blocks to fs_blocks by scaling the dio_block quantity
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 61) * down by dio->blkfactor. Similarly, fs-blocksize quantities are converted
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 62) * to bio_block quantities by shifting left by blkfactor.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 63) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 64) * If blkfactor is zero then the user's request was aligned to the filesystem's
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 65) * blocksize.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 66) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 67)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 68) /* dio_state only used in the submission path */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 69)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 70) struct dio_submit {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 71) struct bio *bio; /* bio under assembly */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 72) unsigned blkbits; /* doesn't change */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 73) unsigned blkfactor; /* When we're using an alignment which
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 74) is finer than the filesystem's soft
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 75) blocksize, this specifies how much
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 76) finer. blkfactor=2 means 1/4-block
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 77) alignment. Does not change */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 78) unsigned start_zero_done; /* flag: sub-blocksize zeroing has
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 79) been performed at the start of a
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 80) write */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 81) int pages_in_io; /* approximate total IO pages */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 82) sector_t block_in_file; /* Current offset into the underlying
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 83) file in dio_block units. */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 84) unsigned blocks_available; /* At block_in_file. changes */
0dc2bc49be545 (Andi Kleen 2011-08-01 21:38:05 -0700 85) int reap_counter; /* rate limit reaping */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 86) sector_t final_block_in_request;/* doesn't change */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 87) int boundary; /* prev block is at a boundary */
1d8fa7a2b9a39 (Badari Pulavarty 2006-03-26 01:38:02 -0800 88) get_block_t *get_block; /* block mapping function */
facd07b07d2a7 (Josef Bacik 2010-05-23 11:00:55 -0400 89) dio_submit_t *submit_io; /* IO submition function */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 90)
facd07b07d2a7 (Josef Bacik 2010-05-23 11:00:55 -0400 91) loff_t logical_offset_in_bio; /* current first logical block in bio */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 92) sector_t final_block_in_bio; /* current final block in bio + 1 */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 93) sector_t next_block_for_io; /* next block to be put under IO,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 94) in dio_blocks units */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 95)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 96) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 97) * Deferred addition of a page to the dio. These variables are
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 98) * private to dio_send_cur_page(), submit_page_section() and
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 99) * dio_bio_add_page().
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 100) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 101) struct page *cur_page; /* The page */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 102) unsigned cur_page_offset; /* Offset into it, in bytes */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 103) unsigned cur_page_len; /* Nr of bytes at cur_page_offset */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 104) sector_t cur_page_block; /* Where it starts */
facd07b07d2a7 (Josef Bacik 2010-05-23 11:00:55 -0400 105) loff_t cur_page_fs_offset; /* Offset in file */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 106)
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 107) struct iov_iter *iter;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 108) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 109) * Page queue. These variables belong to dio_refill_pages() and
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 110) * dio_get_page().
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 111) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 112) unsigned head; /* next page to process */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 113) unsigned tail; /* last valid page + 1 */
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 114) size_t from, to;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 115) };
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 116)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 117) /* dio_state communicated between submission path and end_io */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 118) struct dio {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 119) int flags; /* doesn't change */
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 120) int op;
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 121) int op_flags;
15c4f638f3d41 (Jens Axboe 2015-10-27 14:09:51 +0900 122) blk_qc_t bio_cookie;
74d46992e0d9d (Christoph Hellwig 2017-08-23 19:10:32 +0200 123) struct gendisk *bio_disk;
0dc2bc49be545 (Andi Kleen 2011-08-01 21:38:05 -0700 124) struct inode *inode;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 125) loff_t i_size; /* i_size when submitted */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 126) dio_iodone_t *end_io; /* IO completion function */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 127)
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 128) void *private; /* copy from map_bh.b_private */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 129)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 130) /* BIO completion state */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 131) spinlock_t bio_lock; /* protects BIO fields below */
0dc2bc49be545 (Andi Kleen 2011-08-01 21:38:05 -0700 132) int page_errors; /* errno from get_user_pages() */
0dc2bc49be545 (Andi Kleen 2011-08-01 21:38:05 -0700 133) int is_async; /* is IO async ? */
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 134) bool defer_completion; /* defer AIO completion to workqueue? */
53cbf3b157a04 (Ming Lei 2015-08-17 10:31:46 +0800 135) bool should_dirty; /* if pages should be dirtied */
0dc2bc49be545 (Andi Kleen 2011-08-01 21:38:05 -0700 136) int io_error; /* IO error in completion path */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 137) unsigned long refcount; /* direct_io_worker() and bios */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 138) struct bio *bio_list; /* singly linked via bi_private */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 139) struct task_struct *waiter; /* waiting task (NULL if none) */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 140)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 141) /* AIO related stuff */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 142) struct kiocb *iocb; /* kiocb */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 143) ssize_t result; /* IO result */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 144)
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 145) /*
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 146) * pages[] (and any fields placed after it) are not zeroed out at
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 147) * allocation time. Don't add new fields after pages[] unless you
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 148) * wish that they not be zeroed.
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 149) */
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 150) union {
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 151) struct page *pages[DIO_PAGES]; /* page buffer */
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 152) struct work_struct complete_work;/* deferred AIO completion */
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 153) };
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 154) } ____cacheline_aligned_in_smp;
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 155)
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 156) static struct kmem_cache *dio_cache __read_mostly;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 157)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 158) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 159) * How many pages are in the queue?
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 160) */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 161) static inline unsigned dio_pages_present(struct dio_submit *sdio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 162) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 163) return sdio->tail - sdio->head;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 164) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 165)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 166) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 167) * Go grab and pin some userspace pages. Typically we'll get 64 at a time.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 168) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 169) static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 170) {
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 171) ssize_t ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 172)
2c80929c4c4d5 (Miklos Szeredi 2014-09-24 17:09:11 +0200 173) ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES,
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 174) &sdio->from);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 175)
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 176) if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) {
557ed1fa2620d (Nicholas Piggin 2007-10-16 01:24:40 -0700 177) struct page *page = ZERO_PAGE(0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 178) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 179) * A memory fault, but the filesystem has some outstanding
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 180) * mapped blocks. We need to use those blocks up to avoid
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 181) * leaking stale data in the file.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 182) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 183) if (dio->page_errors == 0)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 184) dio->page_errors = ret;
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 185) get_page(page);
b5810039a54e5 (Nicholas Piggin 2005-10-29 18:16:12 -0700 186) dio->pages[0] = page;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 187) sdio->head = 0;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 188) sdio->tail = 1;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 189) sdio->from = 0;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 190) sdio->to = PAGE_SIZE;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 191) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 192) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 193)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 194) if (ret >= 0) {
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 195) iov_iter_advance(sdio->iter, ret);
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 196) ret += sdio->from;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 197) sdio->head = 0;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 198) sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 199) sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 200) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 201) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 202) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 203) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 204)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 205) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 206) * Get another userspace page. Returns an ERR_PTR on error. Pages are
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 207) * buffered inside the dio so that we can call get_user_pages() against a
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 208) * decent number of pages, less frequently. To provide nicer use of the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 209) * L1 cache.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 210) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 211) static inline struct page *dio_get_page(struct dio *dio,
6fcc5420bfb91 (Boaz Harrosh 2014-07-20 12:09:04 +0300 212) struct dio_submit *sdio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 213) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 214) if (dio_pages_present(sdio) == 0) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 215) int ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 216)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 217) ret = dio_refill_pages(dio, sdio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 218) if (ret)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 219) return ERR_PTR(ret);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 220) BUG_ON(dio_pages_present(sdio) == 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 221) }
6fcc5420bfb91 (Boaz Harrosh 2014-07-20 12:09:04 +0300 222) return dio->pages[sdio->head];
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 223) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 224)
c70d868f272be (Randy Dunlap 2019-10-14 14:12:11 -0700 225) /*
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 226) * dio_complete() - called when all DIO BIO I/O has been completed
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 227) *
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 228) * This drops i_dio_count, lets interested parties know that a DIO operation
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 229) * has completed, and calculates the resulting return code for the operation.
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 230) *
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 231) * It lets the filesystem know if it registered an interest earlier via
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 232) * get_block. Pass the private field of the map buffer_head so that
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 233) * filesystems can use it to hold additional state between get_block calls and
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 234) * dio_complete.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 235) */
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 236) static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 237) {
716b9bc0cb50b (Christoph Hellwig 2016-04-07 08:51:59 -0700 238) loff_t offset = dio->iocb->ki_pos;
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 239) ssize_t transferred = 0;
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 240) int err;
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 241)
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 242) /*
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 243) * AIO submission can race with bio completion to get here while
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 244) * expecting to have the last io completed by bio completion.
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 245) * In that case -EIOCBQUEUED is in fact not an error we want
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 246) * to preserve through this call.
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 247) */
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 248) if (ret == -EIOCBQUEUED)
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 249) ret = 0;
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 250)
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 251) if (dio->result) {
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 252) transferred = dio->result;
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 253)
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 254) /* Check for short read case */
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 255) if ((dio->op == REQ_OP_READ) &&
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 256) ((offset + transferred) > dio->i_size))
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 257) transferred = dio->i_size - offset;
4038acdb187e2 (Al Viro 2016-10-03 20:38:55 -0400 258) /* ignore EFAULT if some IO has been done */
4038acdb187e2 (Al Viro 2016-10-03 20:38:55 -0400 259) if (unlikely(ret == -EFAULT) && transferred)
4038acdb187e2 (Al Viro 2016-10-03 20:38:55 -0400 260) ret = 0;
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 261) }
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 262)
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 263) if (ret == 0)
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 264) ret = dio->page_errors;
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 265) if (ret == 0)
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 266) ret = dio->io_error;
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 267) if (ret == 0)
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 268) ret = transferred;
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 269)
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 270) if (dio->end_io) {
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 271) // XXX: ki_pos??
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 272) err = dio->end_io(dio->iocb, offset, ret, dio->private);
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 273) if (err)
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 274) ret = err;
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 275) }
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 276)
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 277) /*
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 278) * Try again to invalidate clean pages which might have been cached by
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 279) * non-direct readahead, or faulted in by get_user_pages() if the source
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 280) * of the write was an mmap'ed region of the file we're writing. Either
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 281) * one is a pretty crazy thing to do, so we don't support it 100%. If
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 282) * this invalidation fails, tough, the write still worked...
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 283) *
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 284) * And this page cache invalidation has to be after dio->end_io(), as
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 285) * some filesystems convert unwritten extents to real allocations in
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 286) * end_io() when necessary, otherwise a racing buffer read would cache
5e25c269e17de (Eryu Guan 2017-10-13 09:47:46 -0700 287) * zeros from unwritten extents.
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 288) */
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 289) if (flags & DIO_COMPLETE_INVALIDATE &&
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 290) ret > 0 && dio->op == REQ_OP_WRITE &&
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 291) dio->inode->i_mapping->nrpages) {
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 292) err = invalidate_inode_pages2_range(dio->inode->i_mapping,
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 293) offset >> PAGE_SHIFT,
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 294) (offset + ret - 1) >> PAGE_SHIFT);
5a9d929d6e132 (Darrick J. Wong 2018-01-08 10:41:39 -0800 295) if (err)
5a9d929d6e132 (Darrick J. Wong 2018-01-08 10:41:39 -0800 296) dio_warn_stale_pagecache(dio->iocb->ki_filp);
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 297) }
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 298)
ce3077ee80d6a (Nikolay Borisov 2018-02-23 13:45:29 +0200 299) inode_dio_end(dio->inode);
fe0f07d08ee35 (Jens Axboe 2015-04-15 17:05:48 -0600 300)
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 301) if (flags & DIO_COMPLETE_ASYNC) {
e259221763a40 (Christoph Hellwig 2016-04-07 08:52:01 -0700 302) /*
e259221763a40 (Christoph Hellwig 2016-04-07 08:52:01 -0700 303) * generic_write_sync expects ki_pos to have been updated
e259221763a40 (Christoph Hellwig 2016-04-07 08:52:01 -0700 304) * already, but the submission path only does this for
e259221763a40 (Christoph Hellwig 2016-04-07 08:52:01 -0700 305) * synchronous I/O.
e259221763a40 (Christoph Hellwig 2016-04-07 08:52:01 -0700 306) */
e259221763a40 (Christoph Hellwig 2016-04-07 08:52:01 -0700 307) dio->iocb->ki_pos += transferred;
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 308)
41e817bca3acd (Maximilian Heyne 2018-11-30 08:35:14 -0700 309) if (ret > 0 && dio->op == REQ_OP_WRITE)
41e817bca3acd (Maximilian Heyne 2018-11-30 08:35:14 -0700 310) ret = generic_write_sync(dio->iocb, ret);
04b2fa9f8f36e (Christoph Hellwig 2015-02-02 14:49:06 +0100 311) dio->iocb->ki_complete(dio->iocb, ret, 0);
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 312) }
40e2e97316af6 (Christoph Hellwig 2010-07-18 21:17:09 +0000 313)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 314) kmem_cache_free(dio_cache, dio);
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 315) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 316) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 317)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 318) static void dio_aio_complete_work(struct work_struct *work)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 319) {
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 320) struct dio *dio = container_of(work, struct dio, complete_work);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 321)
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 322) dio_complete(dio, 0, DIO_COMPLETE_ASYNC | DIO_COMPLETE_INVALIDATE);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 323) }
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 324)
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 325) static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 326)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 327) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 328) * Asynchronous IO callback.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 329) */
4246a0b63bd8f (Christoph Hellwig 2015-07-20 15:29:37 +0200 330) static void dio_bio_end_aio(struct bio *bio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 331) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 332) struct dio *dio = bio->bi_private;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 333) unsigned long remaining;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 334) unsigned long flags;
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 335) bool defer_completion = false;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 336)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 337) /* cleanup the bio */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 338) dio_bio_complete(dio, bio);
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 339)
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 340) spin_lock_irqsave(&dio->bio_lock, flags);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 341) remaining = --dio->refcount;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 342) if (remaining == 1 && dio->waiter)
20258b2b39703 (Zach Brown 2006-12-10 02:21:01 -0800 343) wake_up_process(dio->waiter);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 344) spin_unlock_irqrestore(&dio->bio_lock, flags);
20258b2b39703 (Zach Brown 2006-12-10 02:21:01 -0800 345)
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 346) if (remaining == 0) {
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 347) /*
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 348) * Defer completion when defer_completion is set or
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 349) * when the inode has pages mapped and this is AIO write.
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 350) * We need to invalidate those pages because there is a
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 351) * chance they contain stale data in the case buffered IO
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 352) * went in between AIO submission and completion into the
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 353) * same region.
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 354) */
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 355) if (dio->result)
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 356) defer_completion = dio->defer_completion ||
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 357) (dio->op == REQ_OP_WRITE &&
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 358) dio->inode->i_mapping->nrpages);
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 359) if (defer_completion) {
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 360) INIT_WORK(&dio->complete_work, dio_aio_complete_work);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 361) queue_work(dio->inode->i_sb->s_dio_done_wq,
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 362) &dio->complete_work);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 363) } else {
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 364) dio_complete(dio, 0, DIO_COMPLETE_ASYNC);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 365) }
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 366) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 367) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 368)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 369) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 370) * The BIO completion handler simply queues the BIO up for the process-context
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 371) * handler.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 372) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 373) * During I/O bi_private points at the dio. After I/O, bi_private is used to
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 374) * implement a singly-linked list of completed BIOs, at dio->bio_list.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 375) */
4246a0b63bd8f (Christoph Hellwig 2015-07-20 15:29:37 +0200 376) static void dio_bio_end_io(struct bio *bio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 377) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 378) struct dio *dio = bio->bi_private;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 379) unsigned long flags;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 380)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 381) spin_lock_irqsave(&dio->bio_lock, flags);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 382) bio->bi_private = dio->bio_list;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 383) dio->bio_list = bio;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 384) if (--dio->refcount == 1 && dio->waiter)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 385) wake_up_process(dio->waiter);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 386) spin_unlock_irqrestore(&dio->bio_lock, flags);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 387) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 388)
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 389) static inline void
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 390) dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 391) struct block_device *bdev,
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 392) sector_t first_sector, int nr_vecs)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 393) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 394) struct bio *bio;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 395)
20d9600cb407b (David Dillow 2011-01-20 14:44:22 -0800 396) /*
0eb0b63c1d1a8 (Christoph Hellwig 2018-05-09 09:54:08 +0200 397) * bio_alloc() is guaranteed to return a bio when allowed to sleep and
0eb0b63c1d1a8 (Christoph Hellwig 2018-05-09 09:54:08 +0200 398) * we request a valid number of vectors.
20d9600cb407b (David Dillow 2011-01-20 14:44:22 -0800 399) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 400) bio = bio_alloc(GFP_KERNEL, nr_vecs);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 401)
74d46992e0d9d (Christoph Hellwig 2017-08-23 19:10:32 +0200 402) bio_set_dev(bio, bdev);
4f024f3797c43 (Kent Overstreet 2013-10-11 15:44:27 -0700 403) bio->bi_iter.bi_sector = first_sector;
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 404) bio_set_op_attrs(bio, dio->op, dio->op_flags);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 405) if (dio->is_async)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 406) bio->bi_end_io = dio_bio_end_aio;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 407) else
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 408) bio->bi_end_io = dio_bio_end_io;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 409)
45d06cf701a38 (Jens Axboe 2017-06-27 11:01:22 -0600 410) bio->bi_write_hint = dio->iocb->ki_hint;
45d06cf701a38 (Jens Axboe 2017-06-27 11:01:22 -0600 411)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 412) sdio->bio = bio;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 413) sdio->logical_offset_in_bio = sdio->cur_page_fs_offset;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 414) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 415)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 416) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 417) * In the AIO read case we speculatively dirty the pages before starting IO.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 418) * During IO completion, any of these pages which happen to have been written
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 419) * back will be redirtied by bio_check_pages_dirty().
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 420) *
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 421) * bios hold a dio reference between submit_bio and ->end_io.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 422) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 423) static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 424) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 425) struct bio *bio = sdio->bio;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 426) unsigned long flags;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 427)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 428) bio->bi_private = dio;
0cf41e5e9bafc (Pavel Begunkov 2021-01-09 16:02:59 +0000 429) /* don't account direct I/O as memory stall */
0cf41e5e9bafc (Pavel Begunkov 2021-01-09 16:02:59 +0000 430) bio_clear_flag(bio, BIO_WORKINGSET);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 431)
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 432) spin_lock_irqsave(&dio->bio_lock, flags);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 433) dio->refcount++;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 434) spin_unlock_irqrestore(&dio->bio_lock, flags);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 435)
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 436) if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 437) bio_set_pages_dirty(bio);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 438)
309dca309fc39 (Christoph Hellwig 2021-01-24 11:02:34 +0100 439) dio->bio_disk = bio->bi_bdev->bd_disk;
c1c534609fe8a (Jens Axboe 2015-11-10 10:14:38 -0700 440)
15c4f638f3d41 (Jens Axboe 2015-10-27 14:09:51 +0900 441) if (sdio->submit_io) {
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 442) sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio);
15c4f638f3d41 (Jens Axboe 2015-10-27 14:09:51 +0900 443) dio->bio_cookie = BLK_QC_T_NONE;
c1c534609fe8a (Jens Axboe 2015-11-10 10:14:38 -0700 444) } else
4e49ea4a3d276 (Mike Christie 2016-06-05 14:31:41 -0500 445) dio->bio_cookie = submit_bio(bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 446)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 447) sdio->bio = NULL;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 448) sdio->boundary = 0;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 449) sdio->logical_offset_in_bio = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 450) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 451)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 452) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 453) * Release any resources in case of a failure
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 454) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 455) static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 456) {
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 457) while (sdio->head < sdio->tail)
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 458) put_page(dio->pages[sdio->head++]);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 459) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 460)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 461) /*
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 462) * Wait for the next BIO to complete. Remove it and return it. NULL is
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 463) * returned once all BIOs have been completed. This must only be called once
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 464) * all bios have been issued so that dio->refcount can only decrease. This
3d742d4b6ebb3 (Randy Dunlap 2021-02-24 12:00:48 -0800 465) * requires that the caller hold a reference on the dio.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 466) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 467) static struct bio *dio_await_one(struct dio *dio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 468) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 469) unsigned long flags;
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 470) struct bio *bio = NULL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 471)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 472) spin_lock_irqsave(&dio->bio_lock, flags);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 473)
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 474) /*
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 475) * Wait as long as the list is empty and there are bios in flight. bio
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 476) * completion drops the count, maybe adds to the list, and wakes while
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 477) * holding the bio_lock so we don't need set_current_state()'s barrier
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 478) * and can call it after testing our condition.
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 479) */
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 480) while (dio->refcount > 1 && dio->bio_list == NULL) {
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 481) __set_current_state(TASK_UNINTERRUPTIBLE);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 482) dio->waiter = current;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 483) spin_unlock_irqrestore(&dio->bio_lock, flags);
c43c83a294e8d (Christoph Hellwig 2016-03-03 16:04:02 +0100 484) if (!(dio->iocb->ki_flags & IOCB_HIPRI) ||
0a1b8b87d064a (Jens Axboe 2018-11-26 08:24:43 -0700 485) !blk_poll(dio->bio_disk->queue, dio->bio_cookie, true))
e6249cdd46e43 (Ming Lei 2020-05-03 09:54:22 +0800 486) blk_io_schedule();
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 487) /* wake up sets us TASK_RUNNING */
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 488) spin_lock_irqsave(&dio->bio_lock, flags);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 489) dio->waiter = NULL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 490) }
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 491) if (dio->bio_list) {
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 492) bio = dio->bio_list;
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 493) dio->bio_list = bio->bi_private;
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 494) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 495) spin_unlock_irqrestore(&dio->bio_lock, flags);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 496) return bio;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 497) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 498)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 499) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 500) * Process one completed BIO. No locks are held.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 501) */
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 502) static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 503) {
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 504) blk_status_t err = bio->bi_status;
d7c8aa85ed204 (Christoph Hellwig 2019-06-26 15:49:27 +0200 505) bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 506)
03a07c92a9ed9 (Goldwyn Rodrigues 2017-06-20 07:05:46 -0500 507) if (err) {
03a07c92a9ed9 (Goldwyn Rodrigues 2017-06-20 07:05:46 -0500 508) if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT))
03a07c92a9ed9 (Goldwyn Rodrigues 2017-06-20 07:05:46 -0500 509) dio->io_error = -EAGAIN;
03a07c92a9ed9 (Goldwyn Rodrigues 2017-06-20 07:05:46 -0500 510) else
03a07c92a9ed9 (Goldwyn Rodrigues 2017-06-20 07:05:46 -0500 511) dio->io_error = -EIO;
03a07c92a9ed9 (Goldwyn Rodrigues 2017-06-20 07:05:46 -0500 512) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 513)
d7c8aa85ed204 (Christoph Hellwig 2019-06-26 15:49:27 +0200 514) if (dio->is_async && should_dirty) {
7ddc971f86aa0 (Mike Krinkin 2016-01-30 19:09:59 +0300 515) bio_check_pages_dirty(bio); /* transfers ownership */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 516) } else {
d7c8aa85ed204 (Christoph Hellwig 2019-06-26 15:49:27 +0200 517) bio_release_pages(bio, should_dirty);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 518) bio_put(bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 519) }
9b81c842355ac (Sasha Levin 2015-08-10 19:05:18 -0400 520) return err;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 521) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 522)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 523) /*
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 524) * Wait on and process all in-flight BIOs. This must only be called once
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 525) * all bios have been issued so that the refcount can only decrease.
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 526) * This just waits for all bios to make it through dio_bio_complete. IO
beb7dd86a1012 (Robert P. J. Day 2007-05-09 07:14:03 +0200 527) * errors are propagated through dio->io_error and should be propagated via
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 528) * dio_complete().
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 529) */
6d544bb4d9019 (Zach Brown 2006-12-10 02:20:54 -0800 530) static void dio_await_completion(struct dio *dio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 531) {
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 532) struct bio *bio;
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 533) do {
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 534) bio = dio_await_one(dio);
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 535) if (bio)
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 536) dio_bio_complete(dio, bio);
0273201e693fd (Zach Brown 2006-12-10 02:20:59 -0800 537) } while (bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 538) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 539)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 540) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 541) * A really large O_DIRECT read or write can generate a lot of BIOs. So
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 542) * to keep the memory consumption sane we periodically reap any completed BIOs
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 543) * during the BIO generation phase.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 544) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 545) * This also helps to limit the peak amount of pinned userspace memory.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 546) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 547) static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 548) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 549) int ret = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 550)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 551) if (sdio->reap_counter++ >= 64) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 552) while (dio->bio_list) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 553) unsigned long flags;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 554) struct bio *bio;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 555) int ret2;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 556)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 557) spin_lock_irqsave(&dio->bio_lock, flags);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 558) bio = dio->bio_list;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 559) dio->bio_list = bio->bi_private;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 560) spin_unlock_irqrestore(&dio->bio_lock, flags);
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 561) ret2 = blk_status_to_errno(dio_bio_complete(dio, bio));
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 562) if (ret == 0)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 563) ret = ret2;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 564) }
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 565) sdio->reap_counter = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 566) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 567) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 568) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 569)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 570) /*
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 571) * Create workqueue for deferred direct IO completions. We allocate the
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 572) * workqueue when it's first needed. This avoids creating workqueue for
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 573) * filesystems that don't need it and also allows us to create the workqueue
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 574) * late enough so the we can include s_id in the name of the workqueue.
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 575) */
ec1b826097f30 (Christoph Hellwig 2016-11-30 14:33:53 +1100 576) int sb_init_dio_done_wq(struct super_block *sb)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 577) {
45150c43b1b0c (Olof Johansson 2013-09-09 10:34:23 -0700 578) struct workqueue_struct *old;
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 579) struct workqueue_struct *wq = alloc_workqueue("dio/%s",
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 580) WQ_MEM_RECLAIM, 0,
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 581) sb->s_id);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 582) if (!wq)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 583) return -ENOMEM;
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 584) /*
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 585) * This has to be atomic as more DIOs can race to create the workqueue
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 586) */
45150c43b1b0c (Olof Johansson 2013-09-09 10:34:23 -0700 587) old = cmpxchg(&sb->s_dio_done_wq, NULL, wq);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 588) /* Someone created workqueue before us? Free ours... */
45150c43b1b0c (Olof Johansson 2013-09-09 10:34:23 -0700 589) if (old)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 590) destroy_workqueue(wq);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 591) return 0;
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 592) }
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 593)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 594) static int dio_set_defer_completion(struct dio *dio)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 595) {
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 596) struct super_block *sb = dio->inode->i_sb;
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 597)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 598) if (dio->defer_completion)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 599) return 0;
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 600) dio->defer_completion = true;
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 601) if (!sb->s_dio_done_wq)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 602) return sb_init_dio_done_wq(sb);
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 603) return 0;
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 604) }
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 605)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 606) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 607) * Call into the fs to map some more disk blocks. We record the current number
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 608) * of available blocks at sdio->blocks_available. These are in units of the
93407472a21b8 (Fabian Frederick 2017-02-27 14:28:32 -0800 609) * fs blocksize, i_blocksize(inode).
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 610) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 611) * The fs is allowed to map lots of blocks at once. If it wants to do that,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 612) * it uses the passed inode-relative block number as the file offset, as usual.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 613) *
1d8fa7a2b9a39 (Badari Pulavarty 2006-03-26 01:38:02 -0800 614) * get_block() is passed the number of i_blkbits-sized blocks which direct_io
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 615) * has remaining to do. The fs should not map more than this number of blocks.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 616) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 617) * If the fs has mapped a lot of blocks, it should populate bh->b_size to
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 618) * indicate how much contiguous disk space has been made available at
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 619) * bh->b_blocknr.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 620) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 621) * If *any* of the mapped blocks are new, then the fs must set buffer_new().
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 622) * This isn't very efficient...
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 623) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 624) * In the case of filesystem holes: the fs may return an arbitrarily-large
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 625) * hole by returning an appropriate value in b_size and by clearing
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 626) * buffer_mapped(). However the direct-io code will only process holes one
1d8fa7a2b9a39 (Badari Pulavarty 2006-03-26 01:38:02 -0800 627) * block at a time - it will repeatedly call get_block() as it walks the hole.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 628) */
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 629) static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 630) struct buffer_head *map_bh)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 631) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 632) int ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 633) sector_t fs_startblk; /* Into file, in filesystem-sized blocks */
ae55e1aaa7e2e (Tao Ma 2012-01-12 17:20:33 -0800 634) sector_t fs_endblk; /* Into file, in filesystem-sized blocks */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 635) unsigned long fs_count; /* Number of filesystem-sized blocks */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 636) int create;
ab73857e354ab (Linus Torvalds 2012-11-29 12:27:00 -0800 637) unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor;
8b9433eb4de3c (Ernesto A. Fernández 2018-10-08 20:58:23 -0300 638) loff_t i_size;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 639)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 640) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 641) * If there was a memory error and we've overwritten all the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 642) * mapped blocks then we can now return that memory error
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 643) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 644) ret = dio->page_errors;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 645) if (ret == 0) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 646) BUG_ON(sdio->block_in_file >= sdio->final_block_in_request);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 647) fs_startblk = sdio->block_in_file >> sdio->blkfactor;
ae55e1aaa7e2e (Tao Ma 2012-01-12 17:20:33 -0800 648) fs_endblk = (sdio->final_block_in_request - 1) >>
ae55e1aaa7e2e (Tao Ma 2012-01-12 17:20:33 -0800 649) sdio->blkfactor;
ae55e1aaa7e2e (Tao Ma 2012-01-12 17:20:33 -0800 650) fs_count = fs_endblk - fs_startblk + 1;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 651)
3c674e74238cb (Nathan Scott 2006-03-29 09:26:15 +1000 652) map_bh->b_state = 0;
ab73857e354ab (Linus Torvalds 2012-11-29 12:27:00 -0800 653) map_bh->b_size = fs_count << i_blkbits;
3c674e74238cb (Nathan Scott 2006-03-29 09:26:15 +1000 654)
5fe878ae7f82f (Christoph Hellwig 2009-12-15 16:47:50 -0800 655) /*
9ecd10b7a0270 (Eryu Guan 2016-05-27 14:27:18 -0700 656) * For writes that could fill holes inside i_size on a
9ecd10b7a0270 (Eryu Guan 2016-05-27 14:27:18 -0700 657) * DIO_SKIP_HOLES filesystem we forbid block creations: only
9ecd10b7a0270 (Eryu Guan 2016-05-27 14:27:18 -0700 658) * overwrites are permitted. We will return early to the caller
9ecd10b7a0270 (Eryu Guan 2016-05-27 14:27:18 -0700 659) * once we see an unmapped buffer head returned, and the caller
9ecd10b7a0270 (Eryu Guan 2016-05-27 14:27:18 -0700 660) * will fall back to buffered I/O.
5fe878ae7f82f (Christoph Hellwig 2009-12-15 16:47:50 -0800 661) *
5fe878ae7f82f (Christoph Hellwig 2009-12-15 16:47:50 -0800 662) * Otherwise the decision is left to the get_blocks method,
5fe878ae7f82f (Christoph Hellwig 2009-12-15 16:47:50 -0800 663) * which may decide to handle it or also return an unmapped
5fe878ae7f82f (Christoph Hellwig 2009-12-15 16:47:50 -0800 664) * buffer head.
5fe878ae7f82f (Christoph Hellwig 2009-12-15 16:47:50 -0800 665) */
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 666) create = dio->op == REQ_OP_WRITE;
5fe878ae7f82f (Christoph Hellwig 2009-12-15 16:47:50 -0800 667) if (dio->flags & DIO_SKIP_HOLES) {
8b9433eb4de3c (Ernesto A. Fernández 2018-10-08 20:58:23 -0300 668) i_size = i_size_read(dio->inode);
8b9433eb4de3c (Ernesto A. Fernández 2018-10-08 20:58:23 -0300 669) if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 670) create = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 671) }
3c674e74238cb (Nathan Scott 2006-03-29 09:26:15 +1000 672)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 673) ret = (*sdio->get_block)(dio->inode, fs_startblk,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 674) map_bh, create);
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 675)
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 676) /* Store for completion */
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 677) dio->private = map_bh->b_private;
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 678)
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 679) if (ret == 0 && buffer_defer_completion(map_bh))
7b7a8665edd8d (Christoph Hellwig 2013-09-04 15:04:39 +0200 680) ret = dio_set_defer_completion(dio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 681) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 682) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 683) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 684)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 685) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 686) * There is no bio. Make one now.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 687) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 688) static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 689) sector_t start_sector, struct buffer_head *map_bh)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 690) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 691) sector_t sector;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 692) int ret, nr_pages;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 693)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 694) ret = dio_bio_reap(dio, sdio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 695) if (ret)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 696) goto out;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 697) sector = start_sector << (sdio->blkbits - 9);
5f7136db82996 (Matthew Wilcox (Oracle) 2021-01-29 04:38:57 +0000 698) nr_pages = bio_max_segs(sdio->pages_in_io);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 699) BUG_ON(nr_pages <= 0);
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 700) dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 701) sdio->boundary = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 702) out:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 703) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 704) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 705)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 706) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 707) * Attempt to put the current chunk of 'cur_page' into the current BIO. If
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 708) * that was successful then update final_block_in_bio and take a ref against
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 709) * the just-added page.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 710) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 711) * Return zero on success. Non-zero means the caller needs to start a new BIO.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 712) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 713) static inline int dio_bio_add_page(struct dio_submit *sdio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 714) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 715) int ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 716)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 717) ret = bio_add_page(sdio->bio, sdio->cur_page,
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 718) sdio->cur_page_len, sdio->cur_page_offset);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 719) if (ret == sdio->cur_page_len) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 720) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 721) * Decrement count only, if we are done with this page
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 722) */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 723) if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 724) sdio->pages_in_io--;
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 725) get_page(sdio->cur_page);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 726) sdio->final_block_in_bio = sdio->cur_page_block +
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 727) (sdio->cur_page_len >> sdio->blkbits);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 728) ret = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 729) } else {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 730) ret = 1;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 731) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 732) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 733) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 734)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 735) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 736) * Put cur_page under IO. The section of cur_page which is described by
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 737) * cur_page_offset,cur_page_len is put into a BIO. The section of cur_page
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 738) * starts on-disk at cur_page_block.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 739) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 740) * We take a ref against the page here (on behalf of its presence in the bio).
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 741) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 742) * The caller of this function is responsible for removing cur_page from the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 743) * dio, and for dropping the refcount which came from that presence.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 744) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 745) static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 746) struct buffer_head *map_bh)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 747) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 748) int ret = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 749)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 750) if (sdio->bio) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 751) loff_t cur_offset = sdio->cur_page_fs_offset;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 752) loff_t bio_next_offset = sdio->logical_offset_in_bio +
4f024f3797c43 (Kent Overstreet 2013-10-11 15:44:27 -0700 753) sdio->bio->bi_iter.bi_size;
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 754)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 755) /*
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 756) * See whether this new request is contiguous with the old.
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 757) *
f0940cee22279 (Namhyung Kim 2011-01-11 21:15:03 +0900 758) * Btrfs cannot handle having logically non-contiguous requests
f0940cee22279 (Namhyung Kim 2011-01-11 21:15:03 +0900 759) * submitted. For example if you have
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 760) *
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 761) * Logical: [0-4095][HOLE][8192-12287]
f0940cee22279 (Namhyung Kim 2011-01-11 21:15:03 +0900 762) * Physical: [0-4095] [4096-8191]
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 763) *
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 764) * We cannot submit those pages together as one BIO. So if our
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 765) * current logical offset in the file does not equal what would
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 766) * be the next logical offset in the bio, submit the bio we
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 767) * have.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 768) */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 769) if (sdio->final_block_in_bio != sdio->cur_page_block ||
c2c6ca417e2db (Josef Bacik 2010-05-23 11:00:55 -0400 770) cur_offset != bio_next_offset)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 771) dio_bio_submit(dio, sdio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 772) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 773)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 774) if (sdio->bio == NULL) {
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 775) ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 776) if (ret)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 777) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 778) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 779)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 780) if (dio_bio_add_page(sdio) != 0) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 781) dio_bio_submit(dio, sdio);
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 782) ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 783) if (ret == 0) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 784) ret = dio_bio_add_page(sdio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 785) BUG_ON(ret != 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 786) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 787) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 788) out:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 789) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 790) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 791)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 792) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 793) * An autonomous function to put a chunk of a page under deferred IO.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 794) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 795) * The caller doesn't actually know (or care) whether this piece of page is in
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 796) * a BIO, or is under IO or whatever. We just take care of all possible
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 797) * situations here. The separation between the logic of do_direct_IO() and
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 798) * that of submit_page_section() is important for clarity. Please don't break.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 799) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 800) * The chunk of page starts on-disk at blocknr.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 801) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 802) * We perform deferred IO, by recording the last-submitted page inside our
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 803) * private part of the dio structure. If possible, we just expand the IO
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 804) * across that page here.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 805) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 806) * If that doesn't work out then we put the old page into the bio and add this
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 807) * page to the dio instead.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 808) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 809) static inline int
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 810) submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 811) unsigned offset, unsigned len, sector_t blocknr,
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 812) struct buffer_head *map_bh)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 813) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 814) int ret = 0;
df41872b68601 (Jack Qiu 2021-04-09 13:27:35 -0700 815) int boundary = sdio->boundary; /* dio_send_cur_page may clear it */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 816)
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 817) if (dio->op == REQ_OP_WRITE) {
98c4d57decf97 (Andrew Morton 2006-12-10 02:19:47 -0800 818) /*
98c4d57decf97 (Andrew Morton 2006-12-10 02:19:47 -0800 819) * Read accounting is performed in submit_bio()
98c4d57decf97 (Andrew Morton 2006-12-10 02:19:47 -0800 820) */
98c4d57decf97 (Andrew Morton 2006-12-10 02:19:47 -0800 821) task_io_account_write(len);
98c4d57decf97 (Andrew Morton 2006-12-10 02:19:47 -0800 822) }
98c4d57decf97 (Andrew Morton 2006-12-10 02:19:47 -0800 823)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 824) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 825) * Can we just grow the current page's presence in the dio?
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 826) */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 827) if (sdio->cur_page == page &&
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 828) sdio->cur_page_offset + sdio->cur_page_len == offset &&
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 829) sdio->cur_page_block +
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 830) (sdio->cur_page_len >> sdio->blkbits) == blocknr) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 831) sdio->cur_page_len += len;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 832) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 833) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 834)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 835) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 836) * If there's a deferred page already there then send it.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 837) */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 838) if (sdio->cur_page) {
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 839) ret = dio_send_cur_page(dio, sdio, map_bh);
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 840) put_page(sdio->cur_page);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 841) sdio->cur_page = NULL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 842) if (ret)
b1058b981272f (Jan Kara 2013-04-29 15:06:18 -0700 843) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 844) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 845)
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 846) get_page(page); /* It is in dio */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 847) sdio->cur_page = page;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 848) sdio->cur_page_offset = offset;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 849) sdio->cur_page_len = len;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 850) sdio->cur_page_block = blocknr;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 851) sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 852) out:
b1058b981272f (Jan Kara 2013-04-29 15:06:18 -0700 853) /*
df41872b68601 (Jack Qiu 2021-04-09 13:27:35 -0700 854) * If boundary then we want to schedule the IO now to
b1058b981272f (Jan Kara 2013-04-29 15:06:18 -0700 855) * avoid metadata seeks.
b1058b981272f (Jan Kara 2013-04-29 15:06:18 -0700 856) */
df41872b68601 (Jack Qiu 2021-04-09 13:27:35 -0700 857) if (boundary) {
b1058b981272f (Jan Kara 2013-04-29 15:06:18 -0700 858) ret = dio_send_cur_page(dio, sdio, map_bh);
899f0429c7d3e (Andreas Gruenbacher 2017-10-09 11:13:18 +0200 859) if (sdio->bio)
899f0429c7d3e (Andreas Gruenbacher 2017-10-09 11:13:18 +0200 860) dio_bio_submit(dio, sdio);
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 861) put_page(sdio->cur_page);
b1058b981272f (Jan Kara 2013-04-29 15:06:18 -0700 862) sdio->cur_page = NULL;
b1058b981272f (Jan Kara 2013-04-29 15:06:18 -0700 863) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 864) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 865) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 866)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 867) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 868) * If we are not writing the entire block and get_block() allocated
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 869) * the block for us, we need to fill-in the unused portion of the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 870) * block with zeros. This happens only if user-buffer, fileoffset or
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 871) * io length is not filesystem block-size multiple.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 872) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 873) * `end' is zero if we're doing the start of the IO, 1 at the end of the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 874) * IO.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 875) */
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 876) static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 877) int end, struct buffer_head *map_bh)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 878) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 879) unsigned dio_blocks_per_fs_block;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 880) unsigned this_chunk_blocks; /* In dio_blocks */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 881) unsigned this_chunk_bytes;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 882) struct page *page;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 883)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 884) sdio->start_zero_done = 1;
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 885) if (!sdio->blkfactor || !buffer_new(map_bh))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 886) return;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 887)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 888) dio_blocks_per_fs_block = 1 << sdio->blkfactor;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 889) this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 890)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 891) if (!this_chunk_blocks)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 892) return;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 893)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 894) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 895) * We need to zero out part of an fs block. It is either at the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 896) * beginning or the end of the fs block.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 897) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 898) if (end)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 899) this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 900)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 901) this_chunk_bytes = this_chunk_blocks << sdio->blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 902)
557ed1fa2620d (Nicholas Piggin 2007-10-16 01:24:40 -0700 903) page = ZERO_PAGE(0);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 904) if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 905) sdio->next_block_for_io, map_bh))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 906) return;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 907)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 908) sdio->next_block_for_io += this_chunk_blocks;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 909) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 910)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 911) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 912) * Walk the user pages, and the file, mapping blocks to disk and generating
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 913) * a sequence of (page,offset,len,block) mappings. These mappings are injected
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 914) * into submit_page_section(), which takes care of the next stage of submission
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 915) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 916) * Direct IO against a blockdev is different from a file. Because we can
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 917) * happily perform page-sized but 512-byte aligned IOs. It is important that
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 918) * blockdev IO be able to have fine alignment and large sizes.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 919) *
1d8fa7a2b9a39 (Badari Pulavarty 2006-03-26 01:38:02 -0800 920) * So what we do is to permit the ->get_block function to populate bh.b_size
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 921) * with the size of IO which is permitted at this offset and this i_blkbits.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 922) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 923) * For best results, the blockdev should be set up with 512-byte i_blkbits and
1d8fa7a2b9a39 (Badari Pulavarty 2006-03-26 01:38:02 -0800 924) * it should set b_size to PAGE_SIZE or more inside get_block(). This gives
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 925) * fine alignment but still allows this function to work in PAGE_SIZE units.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 926) */
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 927) static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 928) struct buffer_head *map_bh)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 929) {
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 930) const unsigned blkbits = sdio->blkbits;
dd545b52a3e1e (Chandan Rajendra 2017-01-10 13:29:54 -0700 931) const unsigned i_blkbits = blkbits + sdio->blkfactor;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 932) int ret = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 933)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 934) while (sdio->block_in_file < sdio->final_block_in_request) {
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 935) struct page *page;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 936) size_t from, to;
6fcc5420bfb91 (Boaz Harrosh 2014-07-20 12:09:04 +0300 937)
6fcc5420bfb91 (Boaz Harrosh 2014-07-20 12:09:04 +0300 938) page = dio_get_page(dio, sdio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 939) if (IS_ERR(page)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 940) ret = PTR_ERR(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 941) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 942) }
6fcc5420bfb91 (Boaz Harrosh 2014-07-20 12:09:04 +0300 943) from = sdio->head ? 0 : sdio->from;
6fcc5420bfb91 (Boaz Harrosh 2014-07-20 12:09:04 +0300 944) to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE;
6fcc5420bfb91 (Boaz Harrosh 2014-07-20 12:09:04 +0300 945) sdio->head++;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 946)
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 947) while (from < to) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 948) unsigned this_chunk_bytes; /* # of bytes mapped */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 949) unsigned this_chunk_blocks; /* # of blocks */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 950) unsigned u;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 951)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 952) if (sdio->blocks_available == 0) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 953) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 954) * Need to go and map some more disk
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 955) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 956) unsigned long blkmask;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 957) unsigned long dio_remainder;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 958)
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 959) ret = get_more_blocks(dio, sdio, map_bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 960) if (ret) {
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 961) put_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 962) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 963) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 964) if (!buffer_mapped(map_bh))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 965) goto do_holes;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 966)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 967) sdio->blocks_available =
f734c89cc96e9 (Jan Kara 2016-11-04 18:08:12 +0100 968) map_bh->b_size >> blkbits;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 969) sdio->next_block_for_io =
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 970) map_bh->b_blocknr << sdio->blkfactor;
f734c89cc96e9 (Jan Kara 2016-11-04 18:08:12 +0100 971) if (buffer_new(map_bh)) {
f734c89cc96e9 (Jan Kara 2016-11-04 18:08:12 +0100 972) clean_bdev_aliases(
f734c89cc96e9 (Jan Kara 2016-11-04 18:08:12 +0100 973) map_bh->b_bdev,
f734c89cc96e9 (Jan Kara 2016-11-04 18:08:12 +0100 974) map_bh->b_blocknr,
dd545b52a3e1e (Chandan Rajendra 2017-01-10 13:29:54 -0700 975) map_bh->b_size >> i_blkbits);
f734c89cc96e9 (Jan Kara 2016-11-04 18:08:12 +0100 976) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 977)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 978) if (!sdio->blkfactor)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 979) goto do_holes;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 980)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 981) blkmask = (1 << sdio->blkfactor) - 1;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 982) dio_remainder = (sdio->block_in_file & blkmask);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 983)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 984) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 985) * If we are at the start of IO and that IO
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 986) * starts partway into a fs-block,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 987) * dio_remainder will be non-zero. If the IO
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 988) * is a read then we can simply advance the IO
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 989) * cursor to the first block which is to be
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 990) * read. But if the IO is a write and the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 991) * block was newly allocated we cannot do that;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 992) * the start of the fs block must be zeroed out
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 993) * on-disk
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 994) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 995) if (!buffer_new(map_bh))
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 996) sdio->next_block_for_io += dio_remainder;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 997) sdio->blocks_available -= dio_remainder;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 998) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 999) do_holes:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1000) /* Handle holes */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1001) if (!buffer_mapped(map_bh)) {
35dc8161d0a6f (Jeff Moyer 2006-02-03 03:04:27 -0800 1002) loff_t i_size_aligned;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1003)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1004) /* AKPM: eargh, -ENOTBLK is a hack */
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 1005) if (dio->op == REQ_OP_WRITE) {
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 1006) put_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1007) return -ENOTBLK;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1008) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1009)
35dc8161d0a6f (Jeff Moyer 2006-02-03 03:04:27 -0800 1010) /*
35dc8161d0a6f (Jeff Moyer 2006-02-03 03:04:27 -0800 1011) * Be sure to account for a partial block as the
35dc8161d0a6f (Jeff Moyer 2006-02-03 03:04:27 -0800 1012) * last block in the file
35dc8161d0a6f (Jeff Moyer 2006-02-03 03:04:27 -0800 1013) */
35dc8161d0a6f (Jeff Moyer 2006-02-03 03:04:27 -0800 1014) i_size_aligned = ALIGN(i_size_read(dio->inode),
35dc8161d0a6f (Jeff Moyer 2006-02-03 03:04:27 -0800 1015) 1 << blkbits);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1016) if (sdio->block_in_file >=
35dc8161d0a6f (Jeff Moyer 2006-02-03 03:04:27 -0800 1017) i_size_aligned >> blkbits) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1018) /* We hit eof */
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 1019) put_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1020) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1021) }
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1022) zero_user(page, from, 1 << blkbits);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1023) sdio->block_in_file++;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1024) from += 1 << blkbits;
3320c60b3a26d (Al Viro 2014-03-10 02:30:55 -0400 1025) dio->result += 1 << blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1026) goto next_block;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1027) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1028)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1029) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1030) * If we're performing IO which has an alignment which
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1031) * is finer than the underlying fs, go check to see if
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1032) * we must zero out the start of this block.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1033) */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1034) if (unlikely(sdio->blkfactor && !sdio->start_zero_done))
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 1035) dio_zero_block(dio, sdio, 0, map_bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1036)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1037) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1038) * Work out, in this_chunk_blocks, how much disk we
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1039) * can add to this page
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1040) */
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1041) this_chunk_blocks = sdio->blocks_available;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1042) u = (to - from) >> blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1043) if (this_chunk_blocks > u)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1044) this_chunk_blocks = u;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1045) u = sdio->final_block_in_request - sdio->block_in_file;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1046) if (this_chunk_blocks > u)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1047) this_chunk_blocks = u;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1048) this_chunk_bytes = this_chunk_blocks << blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1049) BUG_ON(this_chunk_bytes == 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1050)
092c8d46e348b (Jan Kara 2013-04-29 15:06:17 -0700 1051) if (this_chunk_blocks == sdio->blocks_available)
092c8d46e348b (Jan Kara 2013-04-29 15:06:17 -0700 1052) sdio->boundary = buffer_boundary(map_bh);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1053) ret = submit_page_section(dio, sdio, page,
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1054) from,
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1055) this_chunk_bytes,
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 1056) sdio->next_block_for_io,
18772641dbe2c (Andi Kleen 2011-08-01 21:38:07 -0700 1057) map_bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1058) if (ret) {
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 1059) put_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1060) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1061) }
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1062) sdio->next_block_for_io += this_chunk_blocks;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1063)
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1064) sdio->block_in_file += this_chunk_blocks;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1065) from += this_chunk_bytes;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1066) dio->result += this_chunk_bytes;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1067) sdio->blocks_available -= this_chunk_blocks;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1068) next_block:
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1069) BUG_ON(sdio->block_in_file > sdio->final_block_in_request);
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1070) if (sdio->block_in_file == sdio->final_block_in_request)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1071) break;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1072) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1073)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1074) /* Drop the ref which was taken in get_user_pages() */
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 1075) put_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1076) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1077) out:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1078) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1079) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1080)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1081) static inline int drop_refcount(struct dio *dio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1082) {
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1083) int ret2;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1084) unsigned long flags;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1085)
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 1086) /*
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 1087) * Sync will always be dropping the final ref and completing the
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1088) * operation. AIO can if it was a broken operation described above or
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1089) * in fact if all the bios race to complete before we get here. In
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1090) * that case dio_complete() translates the EIOCBQUEUED into the proper
04b2fa9f8f36e (Christoph Hellwig 2015-02-02 14:49:06 +0100 1091) * return code that the caller will hand to ->complete().
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1092) *
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1093) * This is managed by the bio_lock instead of being an atomic_t so that
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1094) * completion paths can drop their ref and use the remaining count to
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1095) * decide to wake the submission path atomically.
8459d86aff04f (Zach Brown 2006-12-10 02:21:05 -0800 1096) */
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1097) spin_lock_irqsave(&dio->bio_lock, flags);
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1098) ret2 = --dio->refcount;
5eb6c7a2ab413 (Zach Brown 2006-12-10 02:21:07 -0800 1099) spin_unlock_irqrestore(&dio->bio_lock, flags);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1100) return ret2;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1101) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1102)
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1103) /*
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1104) * This is a library function for use by filesystem drivers.
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1105) *
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1106) * The locking rules are governed by the flags parameter:
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1107) * - if the flags value contains DIO_LOCKING we use a fancy locking
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1108) * scheme for dumb filesystems.
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1109) * For writes this function is called under i_mutex and returns with
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1110) * i_mutex held, for reads, i_mutex is not held on entry, but it is
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1111) * taken and dropped again before returning.
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1112) * - if the flags value does NOT contain DIO_LOCKING we don't use any
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1113) * internal locking but rather rely on the filesystem to synchronize
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1114) * direct I/O reads/writes versus each other and truncate.
df2d6f26586f1 (Christoph Hellwig 2011-06-24 14:29:46 -0400 1115) *
df2d6f26586f1 (Christoph Hellwig 2011-06-24 14:29:46 -0400 1116) * To help with locking against truncate we incremented the i_dio_count
df2d6f26586f1 (Christoph Hellwig 2011-06-24 14:29:46 -0400 1117) * counter before starting direct I/O, and decrement it once we are done.
df2d6f26586f1 (Christoph Hellwig 2011-06-24 14:29:46 -0400 1118) * Truncate can wait for it to reach zero to provide exclusion. It is
df2d6f26586f1 (Christoph Hellwig 2011-06-24 14:29:46 -0400 1119) * expected that filesystem provide exclusion between new direct I/O
df2d6f26586f1 (Christoph Hellwig 2011-06-24 14:29:46 -0400 1120) * and truncates. For DIO_LOCKING filesystems this is done by i_mutex,
df2d6f26586f1 (Christoph Hellwig 2011-06-24 14:29:46 -0400 1121) * but other filesystems need to take care of this on their own.
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 1122) *
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 1123) * NOTE: if you pass "sdio" to anything by pointer make sure that function
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 1124) * is always inlined. Otherwise gcc is unable to split the structure into
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 1125) * individual fields and will generate much worse code. This is important
ba253fbf6d350 (Andi Kleen 2011-08-01 21:38:08 -0700 1126) * for the whole file.
eafdc7d190a94 (Christoph Hellwig 2010-06-04 11:29:53 +0200 1127) */
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1128) static inline ssize_t
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1129) do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1130) struct block_device *bdev, struct iov_iter *iter,
c8b8e32d700fe (Christoph Hellwig 2016-04-07 08:51:58 -0700 1131) get_block_t get_block, dio_iodone_t end_io,
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1132) dio_submit_t submit_io, int flags)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1133) {
6aa7de059173a (Mark Rutland 2017-10-23 14:07:29 -0700 1134) unsigned i_blkbits = READ_ONCE(inode->i_blkbits);
ab73857e354ab (Linus Torvalds 2012-11-29 12:27:00 -0800 1135) unsigned blkbits = i_blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1136) unsigned blocksize_mask = (1 << blkbits) - 1;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1137) ssize_t retval = -EINVAL;
1c0ff0f1bdeb1 (Nikolay Borisov 2018-04-05 16:24:36 -0700 1138) const size_t count = iov_iter_count(iter);
c8b8e32d700fe (Christoph Hellwig 2016-04-07 08:51:58 -0700 1139) loff_t offset = iocb->ki_pos;
1c0ff0f1bdeb1 (Nikolay Borisov 2018-04-05 16:24:36 -0700 1140) const loff_t end = offset + count;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1141) struct dio *dio;
eb28be2b4c0a0 (Andi Kleen 2011-08-01 21:38:03 -0700 1142) struct dio_submit sdio = { 0, };
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1143) struct buffer_head map_bh = { 0, };
647d1e4c52357 (Fengguang Wu 2012-08-09 15:23:09 +0200 1144) struct blk_plug plug;
886a39115005c (Al Viro 2014-03-05 13:50:45 -0500 1145) unsigned long align = offset | iov_iter_alignment(iter);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1146)
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1147) /*
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1148) * Avoid references to bdev if not absolutely needed to give
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1149) * the early prefetch in the caller enough time.
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1150) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1151)
f9b5570d7fded (Christoph Hellwig 2011-06-24 14:29:42 -0400 1152) /* watch out for a 0 len io from a tricksy fs */
1c0ff0f1bdeb1 (Nikolay Borisov 2018-04-05 16:24:36 -0700 1153) if (iov_iter_rw(iter) == READ && !count)
f9b5570d7fded (Christoph Hellwig 2011-06-24 14:29:42 -0400 1154) return 0;
f9b5570d7fded (Christoph Hellwig 2011-06-24 14:29:42 -0400 1155)
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 1156) dio = kmem_cache_alloc(dio_cache, GFP_KERNEL);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1157) if (!dio)
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1158) return -ENOMEM;
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 1159) /*
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 1160) * Believe it or not, zeroing out the page array caused a .5%
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 1161) * performance regression in a database benchmark. So, we take
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 1162) * care to only zero out what's needed.
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 1163) */
23aee091d804e (Jeff Moyer 2009-12-15 16:47:49 -0800 1164) memset(dio, 0, offsetof(struct dio, pages));
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1165)
5fe878ae7f82f (Christoph Hellwig 2009-12-15 16:47:50 -0800 1166) dio->flags = flags;
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1167) if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) {
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1168) /* will be released by direct_io_worker */
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1169) inode_lock(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1170) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1171)
74cedf9b6c603 (Jan Kara 2015-11-30 10:15:42 -0700 1172) /* Once we sampled i_size check for reads beyond EOF */
74cedf9b6c603 (Jan Kara 2015-11-30 10:15:42 -0700 1173) dio->i_size = i_size_read(inode);
74cedf9b6c603 (Jan Kara 2015-11-30 10:15:42 -0700 1174) if (iov_iter_rw(iter) == READ && offset >= dio->i_size) {
2d4594acbf6d8 (Al Viro 2015-12-08 12:22:47 -0500 1175) retval = 0;
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1176) goto fail_dio;
74cedf9b6c603 (Jan Kara 2015-11-30 10:15:42 -0700 1177) }
74cedf9b6c603 (Jan Kara 2015-11-30 10:15:42 -0700 1178)
41b21af388f94 (Gabriel Krisman Bertazi 2020-10-08 02:26:20 -0400 1179) if (align & blocksize_mask) {
41b21af388f94 (Gabriel Krisman Bertazi 2020-10-08 02:26:20 -0400 1180) if (bdev)
41b21af388f94 (Gabriel Krisman Bertazi 2020-10-08 02:26:20 -0400 1181) blkbits = blksize_bits(bdev_logical_block_size(bdev));
41b21af388f94 (Gabriel Krisman Bertazi 2020-10-08 02:26:20 -0400 1182) blocksize_mask = (1 << blkbits) - 1;
41b21af388f94 (Gabriel Krisman Bertazi 2020-10-08 02:26:20 -0400 1183) if (align & blocksize_mask)
41b21af388f94 (Gabriel Krisman Bertazi 2020-10-08 02:26:20 -0400 1184) goto fail_dio;
41b21af388f94 (Gabriel Krisman Bertazi 2020-10-08 02:26:20 -0400 1185) }
41b21af388f94 (Gabriel Krisman Bertazi 2020-10-08 02:26:20 -0400 1186)
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1187) if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) {
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1188) struct address_space *mapping = iocb->ki_filp->f_mapping;
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1189)
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1190) retval = filemap_write_and_wait_range(mapping, offset, end - 1);
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1191) if (retval)
0a9164cb7ff32 (Gabriel Krisman Bertazi 2020-10-08 02:26:19 -0400 1192) goto fail_dio;
74cedf9b6c603 (Jan Kara 2015-11-30 10:15:42 -0700 1193) }
74cedf9b6c603 (Jan Kara 2015-11-30 10:15:42 -0700 1194)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1195) /*
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1196) * For file extending writes updating i_size before data writeouts
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1197) * complete can expose uninitialized blocks in dumb filesystems.
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1198) * In that case we need to wait for I/O completion even if asked
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1199) * for an asynchronous write.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1200) */
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1201) if (is_sync_kiocb(iocb))
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1202) dio->is_async = false;
c8f4c36f81623 (Nikolay Borisov 2018-02-23 13:45:28 +0200 1203) else if (iov_iter_rw(iter) == WRITE && end > i_size_read(inode))
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1204) dio->is_async = false;
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1205) else
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1206) dio->is_async = true;
6039257378e4c (Christoph Hellwig 2014-02-10 10:27:11 +1100 1207)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1208) dio->inode = inode;
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 1209) if (iov_iter_rw(iter) == WRITE) {
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 1210) dio->op = REQ_OP_WRITE;
70fd76140a6cb (Christoph Hellwig 2016-11-01 07:40:10 -0600 1211) dio->op_flags = REQ_SYNC | REQ_IDLE;
03a07c92a9ed9 (Goldwyn Rodrigues 2017-06-20 07:05:46 -0500 1212) if (iocb->ki_flags & IOCB_NOWAIT)
03a07c92a9ed9 (Goldwyn Rodrigues 2017-06-20 07:05:46 -0500 1213) dio->op_flags |= REQ_NOWAIT;
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 1214) } else {
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 1215) dio->op = REQ_OP_READ;
8a4c1e42e0eb7 (Mike Christie 2016-06-05 14:31:50 -0500 1216) }
d1e36282b0bbd (Jens Axboe 2018-08-29 10:36:56 -0600 1217) if (iocb->ki_flags & IOCB_HIPRI)
d1e36282b0bbd (Jens Axboe 2018-08-29 10:36:56 -0600 1218) dio->op_flags |= REQ_HIPRI;
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1219)
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1220) /*
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1221) * For AIO O_(D)SYNC writes we need to defer completions to a workqueue
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1222) * so that we can call ->fsync.
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1223) */
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1224) if (dio->is_async && iov_iter_rw(iter) == WRITE) {
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1225) retval = 0;
d9c10e5b8863c (Jan Kara 2018-02-26 12:51:43 +0100 1226) if (iocb->ki_flags & IOCB_DSYNC)
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1227) retval = dio_set_defer_completion(dio);
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1228) else if (!dio->inode->i_sb->s_dio_done_wq) {
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1229) /*
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1230) * In case of AIO write racing with buffered read we
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1231) * need to defer completion. We can't decide this now,
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1232) * however the workqueue needs to be initialized here.
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1233) */
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1234) retval = sb_init_dio_done_wq(dio->inode->i_sb);
332391a9935da (Lukas Czerner 2017-09-21 08:16:29 -0600 1235) }
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1236) if (retval)
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1237) goto fail_dio;
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1238) }
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1239)
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1240) /*
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1241) * Will be decremented at I/O completion time.
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1242) */
ce3077ee80d6a (Nikolay Borisov 2018-02-23 13:45:29 +0200 1243) inode_dio_begin(inode);
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1244)
02afc27faec94 (Christoph Hellwig 2013-09-04 15:04:40 +0200 1245) retval = 0;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1246) sdio.blkbits = blkbits;
ab73857e354ab (Linus Torvalds 2012-11-29 12:27:00 -0800 1247) sdio.blkfactor = i_blkbits - blkbits;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1248) sdio.block_in_file = offset >> blkbits;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1249)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1250) sdio.get_block = get_block;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1251) dio->end_io = end_io;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1252) sdio.submit_io = submit_io;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1253) sdio.final_block_in_bio = -1;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1254) sdio.next_block_for_io = -1;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1255)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1256) dio->iocb = iocb;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1257)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1258) spin_lock_init(&dio->bio_lock);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1259) dio->refcount = 1;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1260)
00e23707442a7 (David Howells 2018-10-22 13:07:28 +0100 1261) dio->should_dirty = iter_is_iovec(iter) && iov_iter_rw(iter) == READ;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1262) sdio.iter = iter;
1c0ff0f1bdeb1 (Nikolay Borisov 2018-04-05 16:24:36 -0700 1263) sdio.final_block_in_request = end >> blkbits;
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1264)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1265) /*
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1266) * In case of non-aligned buffers, we may need 2 more
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1267) * pages since we need to zero out first and last block.
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1268) */
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1269) if (unlikely(sdio.blkfactor))
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1270) sdio.pages_in_io = 2;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1271)
f67da30c1d5fc (Al Viro 2014-03-19 01:16:16 -0400 1272) sdio.pages_in_io += iov_iter_npages(iter, INT_MAX);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1273)
647d1e4c52357 (Fengguang Wu 2012-08-09 15:23:09 +0200 1274) blk_start_plug(&plug);
647d1e4c52357 (Fengguang Wu 2012-08-09 15:23:09 +0200 1275)
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1276) retval = do_direct_IO(dio, &sdio, &map_bh);
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1277) if (retval)
7b2c99d15559e (Al Viro 2014-03-15 04:05:57 -0400 1278) dio_cleanup(dio, &sdio);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1279)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1280) if (retval == -ENOTBLK) {
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1281) /*
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1282) * The remaining part of the request will be
3d742d4b6ebb3 (Randy Dunlap 2021-02-24 12:00:48 -0800 1283) * handled by buffered I/O when we return
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1284) */
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1285) retval = 0;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1286) }
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1287) /*
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1288) * There may be some unwritten disk at the end of a part-written
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1289) * fs-block-sized block. Go zero that now.
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1290) */
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1291) dio_zero_block(dio, &sdio, 1, &map_bh);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1292)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1293) if (sdio.cur_page) {
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1294) ssize_t ret2;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1295)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1296) ret2 = dio_send_cur_page(dio, &sdio, &map_bh);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1297) if (retval == 0)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1298) retval = ret2;
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 1299) put_page(sdio.cur_page);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1300) sdio.cur_page = NULL;
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1301) }
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1302) if (sdio.bio)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1303) dio_bio_submit(dio, &sdio);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1304)
647d1e4c52357 (Fengguang Wu 2012-08-09 15:23:09 +0200 1305) blk_finish_plug(&plug);
647d1e4c52357 (Fengguang Wu 2012-08-09 15:23:09 +0200 1306)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1307) /*
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1308) * It is possible that, we return short IO due to end of file.
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1309) * In that case, we need to release all the pages we got hold on.
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1310) */
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1311) dio_cleanup(dio, &sdio);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1312)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1313) /*
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1314) * All block lookups have been performed. For READ requests
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1315) * we can let i_mutex go now that its achieved its purpose
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1316) * of protecting us from looking up uninitialized blocks.
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1317) */
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1318) if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING))
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 1319) inode_unlock(dio->inode);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1320)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1321) /*
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1322) * The only time we want to leave bios in flight is when a successful
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1323) * partial aio read or full aio write have been setup. In that case
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1324) * bio completion will call aio_complete. The only time it's safe to
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1325) * call aio_complete is when we return -EIOCBQUEUED, so we key on that.
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1326) * This had *better* be the only place that raises -EIOCBQUEUED.
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1327) */
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1328) BUG_ON(retval == -EIOCBQUEUED);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1329) if (dio->is_async && retval == 0 && dio->result &&
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1330) (iov_iter_rw(iter) == READ || dio->result == count))
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1331) retval = -EIOCBQUEUED;
af436472772d4 (Christoph Hellwig 2014-07-30 07:18:48 -0400 1332) else
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1333) dio_await_completion(dio);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1334)
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1335) if (drop_refcount(dio) == 0) {
ffe51f0142a29 (Lukas Czerner 2017-10-17 08:43:09 -0600 1336) retval = dio_complete(dio, retval, DIO_COMPLETE_INVALIDATE);
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1337) } else
847cc6371ba82 (Andi Kleen 2011-08-01 21:38:09 -0700 1338) BUG_ON(retval != -EIOCBQUEUED);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1339)
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1340) return retval;
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1341)
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1342) fail_dio:
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1343) if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ)
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1344) inode_unlock(inode);
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1345)
46d716025a228 (Gabriel Krisman Bertazi 2020-10-08 02:26:18 -0400 1346) kmem_cache_free(dio_cache, dio);
7bb46a6734a7e (Nicholas Piggin 2010-05-27 01:05:33 +1000 1347) return retval;
7bb46a6734a7e (Nicholas Piggin 2010-05-27 01:05:33 +1000 1348) }
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1349)
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1350) ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1351) struct block_device *bdev, struct iov_iter *iter,
c8b8e32d700fe (Christoph Hellwig 2016-04-07 08:51:58 -0700 1352) get_block_t get_block,
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1353) dio_iodone_t end_io, dio_submit_t submit_io,
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1354) int flags)
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1355) {
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1356) /*
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1357) * The block device state is needed in the end to finally
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1358) * submit everything. Since it's likely to be cache cold
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1359) * prefetch it here as first thing to hide some of the
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1360) * latency.
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1361) *
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1362) * Attempt to prefetch the pieces we likely need later.
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1363) */
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1364) prefetch(&bdev->bd_disk->part_tbl);
e556f6ba10f0f (Christoph Hellwig 2020-06-26 10:01:56 +0200 1365) prefetch(bdev->bd_disk->queue);
e556f6ba10f0f (Christoph Hellwig 2020-06-26 10:01:56 +0200 1366) prefetch((char *)bdev->bd_disk->queue + SMP_CACHE_BYTES);
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1367)
c8b8e32d700fe (Christoph Hellwig 2016-04-07 08:51:58 -0700 1368) return do_blockdev_direct_IO(iocb, inode, bdev, iter, get_block,
17f8c842d24ac (Omar Sandoval 2015-03-16 04:33:50 -0700 1369) end_io, submit_io, flags);
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1370) }
65dd2aa90aa17 (Andi Kleen 2012-01-12 17:20:35 -0800 1371)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1372) EXPORT_SYMBOL(__blockdev_direct_IO);
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 1373)
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 1374) static __init int dio_init(void)
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 1375) {
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 1376) dio_cache = KMEM_CACHE(dio, SLAB_PANIC);
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 1377) return 0;
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 1378) }
6e8267f532a17 (Andi Kleen 2011-08-01 21:38:06 -0700 1379) module_init(dio_init)
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags