db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 1) // SPDX-License-Identifier: GPL-2.0
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 2) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 3) * Copyright (C) 2010 Red Hat, Inc.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 4) * Copyright (c) 2016-2018 Christoph Hellwig.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 5) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 6) #include <linux/module.h>
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 7) #include <linux/compiler.h>
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 8) #include <linux/fs.h>
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 9) #include <linux/iomap.h>
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 10) #include <linux/backing-dev.h>
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 11) #include <linux/uio.h>
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 12) #include <linux/task_io_accounting_ops.h>
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 13) #include "trace.h"
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 14)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 15) #include "../internal.h"
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 16)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 17) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 18) * Private flags for iomap_dio, must not overlap with the public ones in
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 19) * iomap.h:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 20) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 21) #define IOMAP_DIO_WRITE_FUA (1 << 28)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 22) #define IOMAP_DIO_NEED_SYNC (1 << 29)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 23) #define IOMAP_DIO_WRITE (1 << 30)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 24) #define IOMAP_DIO_DIRTY (1 << 31)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 25)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 26) struct iomap_dio {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 27) struct kiocb *iocb;
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 28) const struct iomap_dio_ops *dops;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 29) loff_t i_size;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 30) loff_t size;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 31) atomic_t ref;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 32) unsigned flags;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 33) int error;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 34) bool wait_for_completion;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 35)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 36) union {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 37) /* used during submission and for synchronous completion: */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 38) struct {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 39) struct iov_iter *iter;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 40) struct task_struct *waiter;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 41) struct request_queue *last_queue;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 42) blk_qc_t cookie;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 43) } submit;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 44)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 45) /* used for aio completion: */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 46) struct {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 47) struct work_struct work;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 48) } aio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 49) };
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 50) };
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 51)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 52) int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 53) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 54) struct request_queue *q = READ_ONCE(kiocb->private);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 55)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 56) if (!q)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 57) return 0;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 58) return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 59) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 60) EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 61)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 62) static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap,
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 63) struct bio *bio, loff_t pos)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 64) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 65) atomic_inc(&dio->ref);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 66)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 67) if (dio->iocb->ki_flags & IOCB_HIPRI)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 68) bio_set_polled(bio, dio->iocb);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 69)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 70) dio->submit.last_queue = bdev_get_queue(iomap->bdev);
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 71) if (dio->dops && dio->dops->submit_io)
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 72) dio->submit.cookie = dio->dops->submit_io(
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 73) file_inode(dio->iocb->ki_filp),
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 74) iomap, bio, pos);
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 75) else
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 76) dio->submit.cookie = submit_bio(bio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 77) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 78)
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 79) ssize_t iomap_dio_complete(struct iomap_dio *dio)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 80) {
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 81) const struct iomap_dio_ops *dops = dio->dops;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 82) struct kiocb *iocb = dio->iocb;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 83) struct inode *inode = file_inode(iocb->ki_filp);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 84) loff_t offset = iocb->ki_pos;
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 85) ssize_t ret = dio->error;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 86)
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 87) if (dops && dops->end_io)
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 88) ret = dops->end_io(iocb, dio->size, ret, dio->flags);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 89)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 90) if (likely(!ret)) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 91) ret = dio->size;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 92) /* check for short read */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 93) if (offset + ret > dio->i_size &&
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 94) !(dio->flags & IOMAP_DIO_WRITE))
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 95) ret = dio->i_size - offset;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 96) iocb->ki_pos += ret;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 97) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 98)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 99) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 100) * Try again to invalidate clean pages which might have been cached by
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 101) * non-direct readahead, or faulted in by get_user_pages() if the source
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 102) * of the write was an mmap'ed region of the file we're writing. Either
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 103) * one is a pretty crazy thing to do, so we don't support it 100%. If
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 104) * this invalidation fails, tough, the write still worked...
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 105) *
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 106) * And this page cache invalidation has to be after ->end_io(), as some
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 107) * filesystems convert unwritten extents to real allocations in
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 108) * ->end_io() when necessary, otherwise a racing buffer read would cache
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 109) * zeros from unwritten extents.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 110) */
c114bbc6c423a (Andreas Gruenbacher 2020-09-10 08:26:16 -0700 111) if (!dio->error && dio->size &&
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 112) (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 113) int err;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 114) err = invalidate_inode_pages2_range(inode->i_mapping,
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 115) offset >> PAGE_SHIFT,
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 116) (offset + dio->size - 1) >> PAGE_SHIFT);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 117) if (err)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 118) dio_warn_stale_pagecache(iocb->ki_filp);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 119) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 120)
1a31182edd008 (Goldwyn Rodrigues 2020-09-28 08:51:08 -0700 121) inode_dio_end(file_inode(iocb->ki_filp));
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 122) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 123) * If this is a DSYNC write, make sure we push it to stable storage now
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 124) * that we've written data.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 125) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 126) if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC))
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 127) ret = generic_write_sync(iocb, ret);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 128)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 129) kfree(dio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 130)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 131) return ret;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 132) }
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 133) EXPORT_SYMBOL_GPL(iomap_dio_complete);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 134)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 135) static void iomap_dio_complete_work(struct work_struct *work)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 136) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 137) struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 138) struct kiocb *iocb = dio->iocb;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 139)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 140) iocb->ki_complete(iocb, iomap_dio_complete(dio), 0);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 141) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 142)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 143) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 144) * Set an error in the dio if none is set yet. We have to use cmpxchg
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 145) * as the submission context and the completion context(s) can race to
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 146) * update the error.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 147) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 148) static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 149) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 150) cmpxchg(&dio->error, 0, ret);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 151) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 152)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 153) static void iomap_dio_bio_end_io(struct bio *bio)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 154) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 155) struct iomap_dio *dio = bio->bi_private;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 156) bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 157)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 158) if (bio->bi_status)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 159) iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 160)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 161) if (atomic_dec_and_test(&dio->ref)) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 162) if (dio->wait_for_completion) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 163) struct task_struct *waiter = dio->submit.waiter;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 164) WRITE_ONCE(dio->submit.waiter, NULL);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 165) blk_wake_io_task(waiter);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 166) } else if (dio->flags & IOMAP_DIO_WRITE) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 167) struct inode *inode = file_inode(dio->iocb->ki_filp);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 168)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 169) INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 170) queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 171) } else {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 172) iomap_dio_complete_work(&dio->aio.work);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 173) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 174) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 175)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 176) if (should_dirty) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 177) bio_check_pages_dirty(bio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 178) } else {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 179) bio_release_pages(bio, false);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 180) bio_put(bio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 181) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 182) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 183)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 184) static void
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 185) iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos,
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 186) unsigned len)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 187) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 188) struct page *page = ZERO_PAGE(0);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 189) int flags = REQ_SYNC | REQ_IDLE;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 190) struct bio *bio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 191)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 192) bio = bio_alloc(GFP_KERNEL, 1);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 193) bio_set_dev(bio, iomap->bdev);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 194) bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 195) bio->bi_private = dio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 196) bio->bi_end_io = iomap_dio_bio_end_io;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 197)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 198) get_page(page);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 199) __bio_add_page(bio, page, len, 0);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 200) bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 201) iomap_dio_submit_bio(dio, iomap, bio, pos);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 202) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 203)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 204) /*
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 205) * Figure out the bio's operation flags from the dio request, the
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 206) * mapping, and whether or not we want FUA. Note that we can end up
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 207) * clearing the WRITE_FUA flag in the dio request.
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 208) */
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 209) static inline unsigned int
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 210) iomap_dio_bio_opflags(struct iomap_dio *dio, struct iomap *iomap, bool use_fua)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 211) {
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 212) unsigned int opflags = REQ_SYNC | REQ_IDLE;
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 213)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 214) if (!(dio->flags & IOMAP_DIO_WRITE)) {
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 215) WARN_ON_ONCE(iomap->flags & IOMAP_F_ZONE_APPEND);
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 216) return REQ_OP_READ;
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 217) }
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 218)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 219) if (iomap->flags & IOMAP_F_ZONE_APPEND)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 220) opflags |= REQ_OP_ZONE_APPEND;
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 221) else
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 222) opflags |= REQ_OP_WRITE;
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 223)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 224) if (use_fua)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 225) opflags |= REQ_FUA;
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 226) else
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 227) dio->flags &= ~IOMAP_DIO_WRITE_FUA;
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 228)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 229) return opflags;
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 230) }
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 231)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 232) static loff_t
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 233) iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 234) struct iomap_dio *dio, struct iomap *iomap)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 235) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 236) unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 237) unsigned int fs_block_size = i_blocksize(inode), pad;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 238) unsigned int align = iov_iter_alignment(dio->submit.iter);
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 239) unsigned int bio_opf;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 240) struct bio *bio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 241) bool need_zeroout = false;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 242) bool use_fua = false;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 243) int nr_pages, ret = 0;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 244) size_t copied = 0;
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 245) size_t orig_count;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 246)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 247) if ((pos | length | align) & ((1 << blkbits) - 1))
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 248) return -EINVAL;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 249)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 250) if (iomap->type == IOMAP_UNWRITTEN) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 251) dio->flags |= IOMAP_DIO_UNWRITTEN;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 252) need_zeroout = true;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 253) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 254)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 255) if (iomap->flags & IOMAP_F_SHARED)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 256) dio->flags |= IOMAP_DIO_COW;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 257)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 258) if (iomap->flags & IOMAP_F_NEW) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 259) need_zeroout = true;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 260) } else if (iomap->type == IOMAP_MAPPED) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 261) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 262) * Use a FUA write if we need datasync semantics, this is a pure
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 263) * data IO that doesn't require any metadata updates (including
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 264) * after IO completion such as unwritten extent conversion) and
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 265) * the underlying device supports FUA. This allows us to avoid
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 266) * cache flushes on IO completion.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 267) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 268) if (!(iomap->flags & (IOMAP_F_SHARED|IOMAP_F_DIRTY)) &&
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 269) (dio->flags & IOMAP_DIO_WRITE_FUA) &&
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 270) blk_queue_fua(bdev_get_queue(iomap->bdev)))
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 271) use_fua = true;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 272) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 273)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 274) /*
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 275) * Save the original count and trim the iter to just the extent we
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 276) * are operating on right now. The iter will be re-expanded once
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 277) * we are done.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 278) */
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 279) orig_count = iov_iter_count(dio->submit.iter);
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 280) iov_iter_truncate(dio->submit.iter, length);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 281)
3e1a88ec96259 (Pavel Begunkov 2021-01-09 16:03:02 +0000 282) if (!iov_iter_count(dio->submit.iter))
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 283) goto out;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 284)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 285) if (need_zeroout) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 286) /* zero out from the start of the block to the write offset */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 287) pad = pos & (fs_block_size - 1);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 288) if (pad)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 289) iomap_dio_zero(dio, iomap, pos - pad, pad);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 290) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 291)
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 292) /*
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 293) * Set the operation flags early so that bio_iov_iter_get_pages
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 294) * can set up the page vector appropriately for a ZONE_APPEND
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 295) * operation.
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 296) */
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 297) bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua);
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 298)
a8affc03a9b37 (Christoph Hellwig 2021-03-11 12:01:37 +0100 299) nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 300) do {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 301) size_t n;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 302) if (dio->error) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 303) iov_iter_revert(dio->submit.iter, copied);
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 304) copied = ret = 0;
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 305) goto out;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 306) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 307)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 308) bio = bio_alloc(GFP_KERNEL, nr_pages);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 309) bio_set_dev(bio, iomap->bdev);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 310) bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 311) bio->bi_write_hint = dio->iocb->ki_hint;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 312) bio->bi_ioprio = dio->iocb->ki_ioprio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 313) bio->bi_private = dio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 314) bio->bi_end_io = iomap_dio_bio_end_io;
c3b0e880bbfaf (Naohiro Aota 2021-02-04 19:21:41 +0900 315) bio->bi_opf = bio_opf;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 316)
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 317) ret = bio_iov_iter_get_pages(bio, dio->submit.iter);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 318) if (unlikely(ret)) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 319) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 320) * We have to stop part way through an IO. We must fall
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 321) * through to the sub-block tail zeroing here, otherwise
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 322) * this short IO may expose stale data in the tail of
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 323) * the block we haven't written data to.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 324) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 325) bio_put(bio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 326) goto zero_tail;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 327) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 328)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 329) n = bio->bi_iter.bi_size;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 330) if (dio->flags & IOMAP_DIO_WRITE) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 331) task_io_account_write(n);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 332) } else {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 333) if (dio->flags & IOMAP_DIO_DIRTY)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 334) bio_set_pages_dirty(bio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 335) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 336)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 337) dio->size += n;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 338) copied += n;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 339)
3e1a88ec96259 (Pavel Begunkov 2021-01-09 16:03:02 +0000 340) nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter,
a8affc03a9b37 (Christoph Hellwig 2021-03-11 12:01:37 +0100 341) BIO_MAX_VECS);
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 342) iomap_dio_submit_bio(dio, iomap, bio, pos);
8cecd0ba85479 (Goldwyn Rodrigues 2019-05-14 18:54:27 -0500 343) pos += n;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 344) } while (nr_pages);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 345)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 346) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 347) * We need to zeroout the tail of a sub-block write if the extent type
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 348) * requires zeroing or the write extends beyond EOF. If we don't zero
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 349) * the block tail in the latter case, we can expose stale data via mmap
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 350) * reads of the EOF block.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 351) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 352) zero_tail:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 353) if (need_zeroout ||
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 354) ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 355) /* zero out from the end of the write to the end of the block */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 356) pad = pos & (fs_block_size - 1);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 357) if (pad)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 358) iomap_dio_zero(dio, iomap, pos, fs_block_size - pad);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 359) }
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 360) out:
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 361) /* Undo iter limitation to current extent */
f550ee9b85fd4 (Jan Kara 2019-11-26 09:28:47 -0800 362) iov_iter_reexpand(dio->submit.iter, orig_count - copied);
e9f930ac88a89 (Jan Stancek 2019-11-11 12:58:24 -0800 363) if (copied)
e9f930ac88a89 (Jan Stancek 2019-11-11 12:58:24 -0800 364) return copied;
e9f930ac88a89 (Jan Stancek 2019-11-11 12:58:24 -0800 365) return ret;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 366) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 367)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 368) static loff_t
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 369) iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 370) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 371) length = iov_iter_zero(length, dio->submit.iter);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 372) dio->size += length;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 373) return length;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 374) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 375)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 376) static loff_t
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 377) iomap_dio_inline_actor(struct inode *inode, loff_t pos, loff_t length,
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 378) struct iomap_dio *dio, struct iomap *iomap)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 379) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 380) struct iov_iter *iter = dio->submit.iter;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 381) size_t copied;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 382)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 383) BUG_ON(pos + length > PAGE_SIZE - offset_in_page(iomap->inline_data));
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 384)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 385) if (dio->flags & IOMAP_DIO_WRITE) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 386) loff_t size = inode->i_size;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 387)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 388) if (pos > size)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 389) memset(iomap->inline_data + size, 0, pos - size);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 390) copied = copy_from_iter(iomap->inline_data + pos, length, iter);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 391) if (copied) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 392) if (pos + copied > size)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 393) i_size_write(inode, pos + copied);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 394) mark_inode_dirty(inode);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 395) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 396) } else {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 397) copied = copy_to_iter(iomap->inline_data + pos, length, iter);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 398) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 399) dio->size += copied;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 400) return copied;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 401) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 402)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 403) static loff_t
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 404) iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
c039b99792726 (Goldwyn Rodrigues 2019-10-18 16:44:10 -0700 405) void *data, struct iomap *iomap, struct iomap *srcmap)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 406) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 407) struct iomap_dio *dio = data;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 408)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 409) switch (iomap->type) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 410) case IOMAP_HOLE:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 411) if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 412) return -EIO;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 413) return iomap_dio_hole_actor(length, dio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 414) case IOMAP_UNWRITTEN:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 415) if (!(dio->flags & IOMAP_DIO_WRITE))
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 416) return iomap_dio_hole_actor(length, dio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 417) return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 418) case IOMAP_MAPPED:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 419) return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 420) case IOMAP_INLINE:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 421) return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 422) case IOMAP_DELALLOC:
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 423) /*
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 424) * DIO is not serialised against mmap() access at all, and so
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 425) * if the page_mkwrite occurs between the writeback and the
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 426) * iomap_apply() call in the DIO path, then it will see the
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 427) * DELALLOC block that the page-mkwrite allocated.
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 428) */
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 429) pr_warn_ratelimited("Direct I/O collision with buffered writes! File: %pD4 Comm: %.20s\n",
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 430) dio->iocb->ki_filp, current->comm);
a805c111650cd (Qian Cai 2020-09-10 08:26:15 -0700 431) return -EIO;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 432) default:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 433) WARN_ON_ONCE(1);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 434) return -EIO;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 435) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 436) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 437)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 438) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 439) * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 440) * is being issued as AIO or not. This allows us to optimise pure data writes
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 441) * to use REQ_FUA rather than requiring generic_write_sync() to issue a
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 442) * REQ_FLUSH post write. This is slightly tricky because a single request here
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 443) * can be mapped into multiple disjoint IOs and only a subset of the IOs issued
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 444) * may be pure data writes. In that case, we still need to do a full data sync
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 445) * completion.
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 446) *
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 447) * Returns -ENOTBLK In case of a page invalidation invalidation failure for
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 448) * writes. The callers needs to fall back to buffered I/O in this case.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 449) */
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 450) struct iomap_dio *
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 451) __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
13ef954445df4 (Jan Kara 2019-10-15 08:43:42 -0700 452) const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
2f63296578cad (Christoph Hellwig 2021-01-23 10:06:09 -0800 453) unsigned int dio_flags)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 454) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 455) struct address_space *mapping = iocb->ki_filp->f_mapping;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 456) struct inode *inode = file_inode(iocb->ki_filp);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 457) size_t count = iov_iter_count(iter);
88cfd30e188fc (Johannes Thumshirn 2019-11-26 09:28:47 -0800 458) loff_t pos = iocb->ki_pos;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 459) loff_t end = iocb->ki_pos + count - 1, ret = 0;
2f63296578cad (Christoph Hellwig 2021-01-23 10:06:09 -0800 460) bool wait_for_completion =
2f63296578cad (Christoph Hellwig 2021-01-23 10:06:09 -0800 461) is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT);
5724be5de88f5 (Christoph Hellwig 2021-01-23 10:06:09 -0800 462) unsigned int iomap_flags = IOMAP_DIRECT;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 463) struct blk_plug plug;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 464) struct iomap_dio *dio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 465)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 466) if (!count)
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 467) return NULL;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 468)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 469) dio = kmalloc(sizeof(*dio), GFP_KERNEL);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 470) if (!dio)
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 471) return ERR_PTR(-ENOMEM);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 472)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 473) dio->iocb = iocb;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 474) atomic_set(&dio->ref, 1);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 475) dio->size = 0;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 476) dio->i_size = i_size_read(inode);
838c4f3d7515e (Christoph Hellwig 2019-09-19 15:32:45 -0700 477) dio->dops = dops;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 478) dio->error = 0;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 479) dio->flags = 0;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 480)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 481) dio->submit.iter = iter;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 482) dio->submit.waiter = current;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 483) dio->submit.cookie = BLK_QC_T_NONE;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 484) dio->submit.last_queue = NULL;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 485)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 486) if (iov_iter_rw(iter) == READ) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 487) if (pos >= dio->i_size)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 488) goto out_free_dio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 489)
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 490) if (iocb->ki_flags & IOCB_NOWAIT) {
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 491) if (filemap_range_needs_writeback(mapping, pos, end)) {
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 492) ret = -EAGAIN;
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 493) goto out_free_dio;
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 494) }
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 495) iomap_flags |= IOMAP_NOWAIT;
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 496) }
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 497)
a901004214994 (Joseph Qi 2019-10-29 09:51:24 -0700 498) if (iter_is_iovec(iter))
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 499) dio->flags |= IOMAP_DIO_DIRTY;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 500) } else {
5724be5de88f5 (Christoph Hellwig 2021-01-23 10:06:09 -0800 501) iomap_flags |= IOMAP_WRITE;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 502) dio->flags |= IOMAP_DIO_WRITE;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 503)
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 504) if (iocb->ki_flags & IOCB_NOWAIT) {
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 505) if (filemap_range_has_page(mapping, pos, end)) {
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 506) ret = -EAGAIN;
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 507) goto out_free_dio;
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 508) }
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 509) iomap_flags |= IOMAP_NOWAIT;
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 510) }
985b71db17506 (Jens Axboe 2021-04-29 22:55:24 -0700 511)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 512) /* for data sync or sync, we need sync completion processing */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 513) if (iocb->ki_flags & IOCB_DSYNC)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 514) dio->flags |= IOMAP_DIO_NEED_SYNC;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 515)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 516) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 517) * For datasync only writes, we optimistically try using FUA for
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 518) * this IO. Any non-FUA write that occurs will clear this flag,
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 519) * hence we know before completion whether a cache flush is
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 520) * necessary.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 521) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 522) if ((iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC)) == IOCB_DSYNC)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 523) dio->flags |= IOMAP_DIO_WRITE_FUA;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 524) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 525)
213f627104daf (Christoph Hellwig 2021-01-23 10:06:10 -0800 526) if (dio_flags & IOMAP_DIO_OVERWRITE_ONLY) {
213f627104daf (Christoph Hellwig 2021-01-23 10:06:10 -0800 527) ret = -EAGAIN;
213f627104daf (Christoph Hellwig 2021-01-23 10:06:10 -0800 528) if (pos >= dio->i_size || pos + count > dio->i_size)
213f627104daf (Christoph Hellwig 2021-01-23 10:06:10 -0800 529) goto out_free_dio;
213f627104daf (Christoph Hellwig 2021-01-23 10:06:10 -0800 530) iomap_flags |= IOMAP_OVERWRITE_ONLY;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 531) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 532)
88cfd30e188fc (Johannes Thumshirn 2019-11-26 09:28:47 -0800 533) ret = filemap_write_and_wait_range(mapping, pos, end);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 534) if (ret)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 535) goto out_free_dio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 536)
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 537) if (iov_iter_rw(iter) == WRITE) {
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 538) /*
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 539) * Try to invalidate cache pages for the range we are writing.
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 540) * If this invalidation fails, let the caller fall back to
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 541) * buffered I/O.
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 542) */
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 543) if (invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT,
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 544) end >> PAGE_SHIFT)) {
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 545) trace_iomap_dio_invalidate_fail(inode, pos, count);
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 546) ret = -ENOTBLK;
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 547) goto out_free_dio;
60263d5889e6d (Christoph Hellwig 2020-07-23 22:45:59 -0700 548) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 549)
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 550) if (!wait_for_completion && !inode->i_sb->s_dio_done_wq) {
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 551) ret = sb_init_dio_done_wq(inode->i_sb);
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 552) if (ret < 0)
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 553) goto out_free_dio;
54752de928c40 (Dave Chinner 2020-07-23 22:45:58 -0700 554) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 555) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 556)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 557) inode_dio_begin(inode);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 558)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 559) blk_start_plug(&plug);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 560) do {
5724be5de88f5 (Christoph Hellwig 2021-01-23 10:06:09 -0800 561) ret = iomap_apply(inode, pos, count, iomap_flags, ops, dio,
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 562) iomap_dio_actor);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 563) if (ret <= 0) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 564) /* magic error code to fall back to buffered I/O */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 565) if (ret == -ENOTBLK) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 566) wait_for_completion = true;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 567) ret = 0;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 568) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 569) break;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 570) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 571) pos += ret;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 572)
419e9c38aa075 (Jan Kara 2019-11-21 16:14:38 -0800 573) if (iov_iter_rw(iter) == READ && pos >= dio->i_size) {
419e9c38aa075 (Jan Kara 2019-11-21 16:14:38 -0800 574) /*
419e9c38aa075 (Jan Kara 2019-11-21 16:14:38 -0800 575) * We only report that we've read data up to i_size.
419e9c38aa075 (Jan Kara 2019-11-21 16:14:38 -0800 576) * Revert iter to a state corresponding to that as
419e9c38aa075 (Jan Kara 2019-11-21 16:14:38 -0800 577) * some callers (such as splice code) rely on it.
419e9c38aa075 (Jan Kara 2019-11-21 16:14:38 -0800 578) */
419e9c38aa075 (Jan Kara 2019-11-21 16:14:38 -0800 579) iov_iter_revert(iter, pos - dio->i_size);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 580) break;
419e9c38aa075 (Jan Kara 2019-11-21 16:14:38 -0800 581) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 582) } while ((count = iov_iter_count(iter)) > 0);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 583) blk_finish_plug(&plug);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 584)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 585) if (ret < 0)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 586) iomap_dio_set_error(dio, ret);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 587)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 588) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 589) * If all the writes we issued were FUA, we don't need to flush the
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 590) * cache on IO completion. Clear the sync flag for this case.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 591) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 592) if (dio->flags & IOMAP_DIO_WRITE_FUA)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 593) dio->flags &= ~IOMAP_DIO_NEED_SYNC;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 594)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 595) WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 596) WRITE_ONCE(iocb->private, dio->submit.last_queue);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 597)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 598) /*
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 599) * We are about to drop our additional submission reference, which
d9973ce2fe5bc (yangerkun 2020-03-18 08:04:36 -0700 600) * might be the last reference to the dio. There are three different
d9973ce2fe5bc (yangerkun 2020-03-18 08:04:36 -0700 601) * ways we can progress here:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 602) *
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 603) * (a) If this is the last reference we will always complete and free
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 604) * the dio ourselves.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 605) * (b) If this is not the last reference, and we serve an asynchronous
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 606) * iocb, we must never touch the dio after the decrement, the
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 607) * I/O completion handler will complete and free it.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 608) * (c) If this is not the last reference, but we serve a synchronous
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 609) * iocb, the I/O completion handler will wake us up on the drop
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 610) * of the final reference, and we will complete and free it here
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 611) * after we got woken by the I/O completion handler.
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 612) */
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 613) dio->wait_for_completion = wait_for_completion;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 614) if (!atomic_dec_and_test(&dio->ref)) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 615) if (!wait_for_completion)
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 616) return ERR_PTR(-EIOCBQUEUED);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 617)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 618) for (;;) {
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 619) set_current_state(TASK_UNINTERRUPTIBLE);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 620) if (!READ_ONCE(dio->submit.waiter))
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 621) break;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 622)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 623) if (!(iocb->ki_flags & IOCB_HIPRI) ||
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 624) !dio->submit.last_queue ||
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 625) !blk_poll(dio->submit.last_queue,
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 626) dio->submit.cookie, true))
e6249cdd46e43 (Ming Lei 2020-05-03 09:54:22 +0800 627) blk_io_schedule();
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 628) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 629) __set_current_state(TASK_RUNNING);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 630) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 631)
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 632) return dio;
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 633)
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 634) out_free_dio:
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 635) kfree(dio);
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 636) if (ret)
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 637) return ERR_PTR(ret);
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 638) return NULL;
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 639) }
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 640) EXPORT_SYMBOL_GPL(__iomap_dio_rw);
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 641)
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 642) ssize_t
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 643) iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 644) const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
2f63296578cad (Christoph Hellwig 2021-01-23 10:06:09 -0800 645) unsigned int dio_flags)
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 646) {
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 647) struct iomap_dio *dio;
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 648)
2f63296578cad (Christoph Hellwig 2021-01-23 10:06:09 -0800 649) dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags);
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 650) if (IS_ERR_OR_NULL(dio))
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 651) return PTR_ERR_OR_ZERO(dio);
c3d4ed1abecfc (Christoph Hellwig 2020-09-28 08:51:08 -0700 652) return iomap_dio_complete(dio);
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 653) }
db074436f4219 (Darrick J. Wong 2019-07-15 08:50:59 -0700 654) EXPORT_SYMBOL_GPL(iomap_dio_rw);
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags