2874c5fd28426 (Thomas Gleixner 2019-05-27 08:55:01 +0200 1) // SPDX-License-Identifier: GPL-2.0-or-later
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 2) /* Cache page management and data I/O routines
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 3) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 4) * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 5) * Written by David Howells (dhowells@redhat.com)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 6) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 7)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 8) #define FSCACHE_DEBUG_LEVEL PAGE
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 9) #include <linux/module.h>
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 10) #include <linux/fscache-cache.h>
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 11) #include <linux/buffer_head.h>
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 12) #include <linux/pagevec.h>
5a0e3ad6af866 (Tejun Heo 2010-03-24 17:04:11 +0900 13) #include <linux/slab.h>
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 14) #include "internal.h"
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 15)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 16) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 17) * check to see if a page is being written to the cache
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 18) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 19) bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 20) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 21) void *val;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 22)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 23) rcu_read_lock();
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 24) val = radix_tree_lookup(&cookie->stores, page->index);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 25) rcu_read_unlock();
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 26) trace_fscache_check_page(cookie, page, val, 0);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 27)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 28) return val != NULL;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 29) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 30) EXPORT_SYMBOL(__fscache_check_page_write);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 31)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 32) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 33) * wait for a page to finish being written to the cache
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 34) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 35) void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 36) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 37) wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 38)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 39) trace_fscache_page(cookie, page, fscache_page_write_wait);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 40)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 41) wait_event(*wq, !__fscache_check_page_write(cookie, page));
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 42) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 43) EXPORT_SYMBOL(__fscache_wait_on_page_write);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 44)
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 45) /*
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 46) * wait for a page to finish being written to the cache. Put a timeout here
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 47) * since we might be called recursively via parent fs.
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 48) */
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 49) static
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 50) bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page)
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 51) {
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 52) wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 53)
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 54) return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page),
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 55) HZ);
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 56) }
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 57)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 58) /*
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 59) * decide whether a page can be released, possibly by cancelling a store to it
d0164adc89f6b (Mel Gorman 2015-11-06 16:28:21 -0800 60) * - we're allowed to sleep if __GFP_DIRECT_RECLAIM is flagged
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 61) */
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 62) bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 63) struct page *page,
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 64) gfp_t gfp)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 65) {
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 66) struct page *xpage;
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 67) void *val;
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 68)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 69) _enter("%p,%p,%x", cookie, page, gfp);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 70)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 71) trace_fscache_page(cookie, page, fscache_page_maybe_release);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 72)
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 73) try_again:
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 74) rcu_read_lock();
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 75) val = radix_tree_lookup(&cookie->stores, page->index);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 76) if (!val) {
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 77) rcu_read_unlock();
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 78) fscache_stat(&fscache_n_store_vmscan_not_storing);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 79) __fscache_uncache_page(cookie, page);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 80) return true;
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 81) }
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 82)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 83) /* see if the page is actually undergoing storage - if so we can't get
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 84) * rid of it till the cache has finished with it */
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 85) if (radix_tree_tag_get(&cookie->stores, page->index,
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 86) FSCACHE_COOKIE_STORING_TAG)) {
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 87) rcu_read_unlock();
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 88) goto page_busy;
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 89) }
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 90)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 91) /* the page is pending storage, so we attempt to cancel the store and
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 92) * discard the store request so that the page can be reclaimed */
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 93) spin_lock(&cookie->stores_lock);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 94) rcu_read_unlock();
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 95)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 96) if (radix_tree_tag_get(&cookie->stores, page->index,
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 97) FSCACHE_COOKIE_STORING_TAG)) {
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 98) /* the page started to undergo storage whilst we were looking,
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 99) * so now we can only wait or return */
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 100) spin_unlock(&cookie->stores_lock);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 101) goto page_busy;
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 102) }
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 103)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 104) xpage = radix_tree_delete(&cookie->stores, page->index);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 105) trace_fscache_page(cookie, page, fscache_page_radix_delete);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 106) spin_unlock(&cookie->stores_lock);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 107)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 108) if (xpage) {
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 109) fscache_stat(&fscache_n_store_vmscan_cancelled);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 110) fscache_stat(&fscache_n_store_radix_deletes);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 111) ASSERTCMP(xpage, ==, page);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 112) } else {
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 113) fscache_stat(&fscache_n_store_vmscan_gone);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 114) }
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 115)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 116) wake_up_bit(&cookie->flags, 0);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 117) trace_fscache_wake_cookie(cookie);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 118) if (xpage)
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 119) put_page(xpage);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 120) __fscache_uncache_page(cookie, page);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 121) return true;
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 122)
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 123) page_busy:
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 124) /* We will wait here if we're allowed to, but that could deadlock the
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 125) * allocator as the work threads writing to the cache may all end up
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 126) * sleeping on memory allocation, so we may need to impose a timeout
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 127) * too. */
d0164adc89f6b (Mel Gorman 2015-11-06 16:28:21 -0800 128) if (!(gfp & __GFP_DIRECT_RECLAIM) || !(gfp & __GFP_FS)) {
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 129) fscache_stat(&fscache_n_store_vmscan_busy);
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 130) return false;
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 131) }
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 132)
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 133) fscache_stat(&fscache_n_store_vmscan_wait);
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 134) if (!release_page_wait_timeout(cookie, page))
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 135) _debug("fscache writeout timeout page: %p{%lx}",
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 136) page, page->index);
9776de96e5156 (Milosz Tanski 2014-08-13 12:58:16 -0400 137)
d0164adc89f6b (Mel Gorman 2015-11-06 16:28:21 -0800 138) gfp &= ~__GFP_DIRECT_RECLAIM;
8c209ce721444 (David Howells 2012-12-05 13:34:49 +0000 139) goto try_again;
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 140) }
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 141) EXPORT_SYMBOL(__fscache_maybe_release_page);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 142)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 143) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 144) * note that a page has finished being written to the cache
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 145) */
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 146) static void fscache_end_page_write(struct fscache_object *object,
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 147) struct page *page)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 148) {
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 149) struct fscache_cookie *cookie;
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 150) struct page *xpage = NULL, *val;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 151)
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 152) spin_lock(&object->lock);
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 153) cookie = object->cookie;
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 154) if (cookie) {
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 155) /* delete the page from the tree if it is now no longer
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 156) * pending */
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 157) spin_lock(&cookie->stores_lock);
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 158) radix_tree_tag_clear(&cookie->stores, page->index,
201a15428bd54 (David Howells 2009-11-19 18:11:35 +0000 159) FSCACHE_COOKIE_STORING_TAG);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 160) trace_fscache_page(cookie, page, fscache_page_radix_clear_store);
285e728b0ac55 (David Howells 2009-11-19 18:11:29 +0000 161) if (!radix_tree_tag_get(&cookie->stores, page->index,
285e728b0ac55 (David Howells 2009-11-19 18:11:29 +0000 162) FSCACHE_COOKIE_PENDING_TAG)) {
285e728b0ac55 (David Howells 2009-11-19 18:11:29 +0000 163) fscache_stat(&fscache_n_store_radix_deletes);
285e728b0ac55 (David Howells 2009-11-19 18:11:29 +0000 164) xpage = radix_tree_delete(&cookie->stores, page->index);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 165) trace_fscache_page(cookie, page, fscache_page_radix_delete);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 166) trace_fscache_page(cookie, page, fscache_page_write_end);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 167)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 168) val = radix_tree_lookup(&cookie->stores, page->index);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 169) trace_fscache_check_page(cookie, page, val, 1);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 170) } else {
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 171) trace_fscache_page(cookie, page, fscache_page_write_end_pend);
285e728b0ac55 (David Howells 2009-11-19 18:11:29 +0000 172) }
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 173) spin_unlock(&cookie->stores_lock);
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 174) wake_up_bit(&cookie->flags, 0);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 175) trace_fscache_wake_cookie(cookie);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 176) } else {
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 177) trace_fscache_page(cookie, page, fscache_page_write_end_noc);
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 178) }
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 179) spin_unlock(&object->lock);
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 180) if (xpage)
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 181) put_page(xpage);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 182) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 183)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 184) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 185) * actually apply the changed attributes to a cache object
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 186) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 187) static void fscache_attr_changed_op(struct fscache_operation *op)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 188) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 189) struct fscache_object *object = op->object;
440f0affe247e (David Howells 2009-11-19 18:11:01 +0000 190) int ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 191)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 192) _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 193)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 194) fscache_stat(&fscache_n_attr_changed_calls);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 195)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 196) if (fscache_object_is_active(object)) {
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 197) fscache_stat(&fscache_n_cop_attr_changed);
440f0affe247e (David Howells 2009-11-19 18:11:01 +0000 198) ret = object->cache->ops->attr_changed(object);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 199) fscache_stat_d(&fscache_n_cop_attr_changed);
440f0affe247e (David Howells 2009-11-19 18:11:01 +0000 200) if (ret < 0)
440f0affe247e (David Howells 2009-11-19 18:11:01 +0000 201) fscache_abort_object(object);
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 202) fscache_op_complete(op, ret < 0);
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 203) } else {
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 204) fscache_op_complete(op, true);
440f0affe247e (David Howells 2009-11-19 18:11:01 +0000 205) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 206)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 207) _leave("");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 208) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 209)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 210) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 211) * notification that the attributes on an object have changed
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 212) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 213) int __fscache_attr_changed(struct fscache_cookie *cookie)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 214) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 215) struct fscache_operation *op;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 216) struct fscache_object *object;
3e1199dcad004 (Milosz Tanski 2014-08-13 12:58:26 -0400 217) bool wake_cookie = false;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 218)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 219) _enter("%p", cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 220)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 221) ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 222)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 223) fscache_stat(&fscache_n_attr_changed);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 224)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 225) op = kzalloc(sizeof(*op), GFP_KERNEL);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 226) if (!op) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 227) fscache_stat(&fscache_n_attr_changed_nomem);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 228) _leave(" = -ENOMEM");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 229) return -ENOMEM;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 230) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 231)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 232) fscache_operation_init(cookie, op, fscache_attr_changed_op, NULL, NULL);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 233) trace_fscache_page_op(cookie, NULL, op, fscache_page_op_attr_changed);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 234) op->flags = FSCACHE_OP_ASYNC |
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 235) (1 << FSCACHE_OP_EXCLUSIVE) |
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 236) (1 << FSCACHE_OP_UNUSE_COOKIE);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 237)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 238) spin_lock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 239)
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 240) if (!fscache_cookie_enabled(cookie) ||
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 241) hlist_empty(&cookie->backing_objects))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 242) goto nobufs;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 243) object = hlist_entry(cookie->backing_objects.first,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 244) struct fscache_object, cookie_link);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 245)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 246) __fscache_use_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 247) if (fscache_submit_exclusive_op(object, op) < 0)
3e1199dcad004 (Milosz Tanski 2014-08-13 12:58:26 -0400 248) goto nobufs_dec;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 249) spin_unlock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 250) fscache_stat(&fscache_n_attr_changed_ok);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 251) fscache_put_operation(op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 252) _leave(" = 0");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 253) return 0;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 254)
3e1199dcad004 (Milosz Tanski 2014-08-13 12:58:26 -0400 255) nobufs_dec:
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 256) wake_cookie = __fscache_unuse_cookie(cookie);
3e1199dcad004 (Milosz Tanski 2014-08-13 12:58:26 -0400 257) nobufs:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 258) spin_unlock(&cookie->lock);
a39caadf06879 (David Howells 2015-02-25 14:22:40 +0000 259) fscache_put_operation(op);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 260) if (wake_cookie)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 261) __fscache_wake_unused_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 262) fscache_stat(&fscache_n_attr_changed_nobufs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 263) _leave(" = %d", -ENOBUFS);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 264) return -ENOBUFS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 265) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 266) EXPORT_SYMBOL(__fscache_attr_changed);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 267)
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 268) /*
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 269) * Handle cancellation of a pending retrieval op
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 270) */
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 271) static void fscache_do_cancel_retrieval(struct fscache_operation *_op)
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 272) {
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 273) struct fscache_retrieval *op =
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 274) container_of(_op, struct fscache_retrieval, op);
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 275)
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 276) atomic_set(&op->n_pages, 0);
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 277) }
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 278)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 279) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 280) * release a retrieval op reference
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 281) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 282) static void fscache_release_retrieval_op(struct fscache_operation *_op)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 283) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 284) struct fscache_retrieval *op =
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 285) container_of(_op, struct fscache_retrieval, op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 286)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 287) _enter("{OP%x}", op->op.debug_id);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 288)
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 289) ASSERTIFCMP(op->op.state != FSCACHE_OP_ST_INITIALISED,
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 290) atomic_read(&op->n_pages), ==, 0);
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 291)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 292) fscache_hist(fscache_retrieval_histogram, op->start_time);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 293) if (op->context)
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 294) fscache_put_context(op->cookie, op->context);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 295)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 296) _leave("");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 297) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 298)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 299) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 300) * allocate a retrieval op
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 301) */
26aaeffcafe6c (David Howells 2021-02-22 11:39:47 +0000 302) struct fscache_retrieval *fscache_alloc_retrieval(
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 303) struct fscache_cookie *cookie,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 304) struct address_space *mapping,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 305) fscache_rw_complete_t end_io_func,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 306) void *context)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 307) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 308) struct fscache_retrieval *op;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 309)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 310) /* allocate a retrieval operation and attempt to submit it */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 311) op = kzalloc(sizeof(*op), GFP_NOIO);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 312) if (!op) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 313) fscache_stat(&fscache_n_retrievals_nomem);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 314) return NULL;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 315) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 316)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 317) fscache_operation_init(cookie, &op->op, NULL,
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 318) fscache_do_cancel_retrieval,
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 319) fscache_release_retrieval_op);
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 320) op->op.flags = FSCACHE_OP_MYTHREAD |
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 321) (1UL << FSCACHE_OP_WAITING) |
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 322) (1UL << FSCACHE_OP_UNUSE_COOKIE);
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 323) op->cookie = cookie;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 324) op->mapping = mapping;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 325) op->end_io_func = end_io_func;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 326) op->context = context;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 327) op->start_time = jiffies;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 328) INIT_LIST_HEAD(&op->to_do);
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 329)
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 330) /* Pin the netfs read context in case we need to do the actual netfs
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 331) * read because we've encountered a cache read failure.
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 332) */
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 333) if (context)
4a47132ff472a (David Howells 2015-02-24 10:05:29 +0000 334) fscache_get_context(op->cookie, context);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 335) return op;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 336) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 337)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 338) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 339) * wait for a deferred lookup to complete
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 340) */
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 341) int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 342) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 343) unsigned long jif;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 344)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 345) _enter("");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 346)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 347) if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 348) _leave(" = 0 [imm]");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 349) return 0;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 350) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 351)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 352) fscache_stat(&fscache_n_retrievals_wait);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 353)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 354) jif = jiffies;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 355) if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 356) TASK_INTERRUPTIBLE) != 0) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 357) fscache_stat(&fscache_n_retrievals_intr);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 358) _leave(" = -ERESTARTSYS");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 359) return -ERESTARTSYS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 360) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 361)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 362) ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 363)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 364) smp_rmb();
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 365) fscache_hist(fscache_retrieval_delay_histogram, jif);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 366) _leave(" = 0 [dly]");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 367) return 0;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 368) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 369)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 370) /*
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 371) * wait for an object to become active (or dead)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 372) */
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 373) int fscache_wait_for_operation_activation(struct fscache_object *object,
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 374) struct fscache_operation *op,
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 375) atomic_t *stat_op_waits,
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 376) atomic_t *stat_object_dead)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 377) {
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 378) int ret;
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 379)
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 380) if (!test_bit(FSCACHE_OP_WAITING, &op->flags))
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 381) goto check_if_dead;
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 382)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 383) _debug(">>> WT");
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 384) if (stat_op_waits)
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 385) fscache_stat(stat_op_waits);
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 386) if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
9c04caa81b876 (David Howells 2012-12-07 18:08:02 +0000 387) TASK_INTERRUPTIBLE) != 0) {
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 388) trace_fscache_op(object->cookie, op, fscache_op_signal);
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 389) ret = fscache_cancel_op(op, false);
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 390) if (ret == 0)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 391) return -ERESTARTSYS;
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 392)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 393) /* it's been removed from the pending queue by another party,
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 394) * so we should get to run shortly */
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 395) wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
743162013d40c (NeilBrown 2014-07-07 15:16:04 +1000 396) TASK_UNINTERRUPTIBLE);
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 397) }
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 398) _debug("<<< GO");
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 399)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 400) check_if_dead:
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 401) if (op->state == FSCACHE_OP_ST_CANCELLED) {
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 402) if (stat_object_dead)
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 403) fscache_stat(stat_object_dead);
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 404) _leave(" = -ENOBUFS [cancelled]");
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 405) return -ENOBUFS;
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 406) }
87021526300f1 (David Howells 2015-02-24 10:52:51 +0000 407) if (unlikely(fscache_object_is_dying(object) ||
87021526300f1 (David Howells 2015-02-24 10:52:51 +0000 408) fscache_cache_is_broken(object))) {
87021526300f1 (David Howells 2015-02-24 10:52:51 +0000 409) enum fscache_operation_state state = op->state;
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 410) trace_fscache_op(object->cookie, op, fscache_op_signal);
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 411) fscache_cancel_op(op, true);
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 412) if (stat_object_dead)
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 413) fscache_stat(stat_object_dead);
87021526300f1 (David Howells 2015-02-24 10:52:51 +0000 414) _leave(" = -ENOBUFS [obj dead %d]", state);
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 415) return -ENOBUFS;
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 416) }
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 417) return 0;
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 418) }
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 419)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 420) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 421) * read a page from the cache or allocate a block in which to store it
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 422) * - we return:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 423) * -ENOMEM - out of memory, nothing done
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 424) * -ERESTARTSYS - interrupted
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 425) * -ENOBUFS - no backing object available in which to cache the block
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 426) * -ENODATA - no data available in the backing object for this block
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 427) * 0 - dispatched a read - it'll call end_io_func() when finished
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 428) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 429) int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 430) struct page *page,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 431) fscache_rw_complete_t end_io_func,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 432) void *context,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 433) gfp_t gfp)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 434) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 435) struct fscache_retrieval *op;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 436) struct fscache_object *object;
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 437) bool wake_cookie = false;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 438) int ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 439)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 440) _enter("%p,%p,,,", cookie, page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 441)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 442) fscache_stat(&fscache_n_retrievals);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 443)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 444) if (hlist_empty(&cookie->backing_objects))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 445) goto nobufs;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 446)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 447) if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 448) _leave(" = -ENOBUFS [invalidating]");
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 449) return -ENOBUFS;
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 450) }
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 451)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 452) ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 453) ASSERTCMP(page, !=, NULL);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 454)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 455) if (fscache_wait_for_deferred_lookup(cookie) < 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 456) return -ERESTARTSYS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 457)
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 458) op = fscache_alloc_retrieval(cookie, page->mapping,
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 459) end_io_func, context);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 460) if (!op) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 461) _leave(" = -ENOMEM");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 462) return -ENOMEM;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 463) }
1bb4b7f98f361 (David Howells 2013-05-21 13:44:15 +0100 464) atomic_set(&op->n_pages, 1);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 465) trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_retr_one);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 466)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 467) spin_lock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 468)
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 469) if (!fscache_cookie_enabled(cookie) ||
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 470) hlist_empty(&cookie->backing_objects))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 471) goto nobufs_unlock;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 472) object = hlist_entry(cookie->backing_objects.first,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 473) struct fscache_object, cookie_link);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 474)
caaef6900befb (David Howells 2013-05-10 19:50:26 +0100 475) ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags));
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 476)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 477) __fscache_use_cookie(cookie);
4fbf4291aa159 (David Howells 2009-11-19 18:11:04 +0000 478) atomic_inc(&object->n_reads);
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 479) __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
4fbf4291aa159 (David Howells 2009-11-19 18:11:04 +0000 480)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 481) if (fscache_submit_op(object, &op->op) < 0)
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 482) goto nobufs_unlock_dec;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 483) spin_unlock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 484)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 485) fscache_stat(&fscache_n_retrieval_ops);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 486)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 487) /* we wait for the operation to become active, and then process it
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 488) * *here*, in this thread, and not in the thread pool */
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 489) ret = fscache_wait_for_operation_activation(
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 490) object, &op->op,
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 491) __fscache_stat(&fscache_n_retrieval_op_waits),
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 492) __fscache_stat(&fscache_n_retrievals_object_dead));
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 493) if (ret < 0)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 494) goto error;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 495)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 496) /* ask the cache to honour the operation */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 497) if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 498) fscache_stat(&fscache_n_cop_allocate_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 499) ret = object->cache->ops->allocate_page(op, page, gfp);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 500) fscache_stat_d(&fscache_n_cop_allocate_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 501) if (ret == 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 502) ret = -ENODATA;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 503) } else {
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 504) fscache_stat(&fscache_n_cop_read_or_alloc_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 505) ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 506) fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 507) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 508)
5753c44188925 (David Howells 2009-11-19 18:11:19 +0000 509) error:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 510) if (ret == -ENOMEM)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 511) fscache_stat(&fscache_n_retrievals_nomem);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 512) else if (ret == -ERESTARTSYS)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 513) fscache_stat(&fscache_n_retrievals_intr);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 514) else if (ret == -ENODATA)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 515) fscache_stat(&fscache_n_retrievals_nodata);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 516) else if (ret < 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 517) fscache_stat(&fscache_n_retrievals_nobufs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 518) else
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 519) fscache_stat(&fscache_n_retrievals_ok);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 520)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 521) fscache_put_retrieval(op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 522) _leave(" = %d", ret);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 523) return ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 524)
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 525) nobufs_unlock_dec:
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 526) atomic_dec(&object->n_reads);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 527) wake_cookie = __fscache_unuse_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 528) nobufs_unlock:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 529) spin_unlock(&cookie->lock);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 530) if (wake_cookie)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 531) __fscache_wake_unused_cookie(cookie);
a39caadf06879 (David Howells 2015-02-25 14:22:40 +0000 532) fscache_put_retrieval(op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 533) nobufs:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 534) fscache_stat(&fscache_n_retrievals_nobufs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 535) _leave(" = -ENOBUFS");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 536) return -ENOBUFS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 537) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 538) EXPORT_SYMBOL(__fscache_read_or_alloc_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 539)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 540) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 541) * read a list of page from the cache or allocate a block in which to store
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 542) * them
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 543) * - we return:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 544) * -ENOMEM - out of memory, some pages may be being read
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 545) * -ERESTARTSYS - interrupted, some pages may be being read
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 546) * -ENOBUFS - no backing object or space available in which to cache any
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 547) * pages not being read
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 548) * -ENODATA - no data available in the backing object for some or all of
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 549) * the pages
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 550) * 0 - dispatched a read on all pages
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 551) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 552) * end_io_func() will be called for each page read from the cache as it is
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 553) * finishes being read
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 554) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 555) * any pages for which a read is dispatched will be removed from pages and
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 556) * nr_pages
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 557) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 558) int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 559) struct address_space *mapping,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 560) struct list_head *pages,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 561) unsigned *nr_pages,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 562) fscache_rw_complete_t end_io_func,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 563) void *context,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 564) gfp_t gfp)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 565) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 566) struct fscache_retrieval *op;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 567) struct fscache_object *object;
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 568) bool wake_cookie = false;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 569) int ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 570)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 571) _enter("%p,,%d,,,", cookie, *nr_pages);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 572)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 573) fscache_stat(&fscache_n_retrievals);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 574)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 575) if (hlist_empty(&cookie->backing_objects))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 576) goto nobufs;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 577)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 578) if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 579) _leave(" = -ENOBUFS [invalidating]");
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 580) return -ENOBUFS;
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 581) }
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 582)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 583) ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 584) ASSERTCMP(*nr_pages, >, 0);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 585) ASSERT(!list_empty(pages));
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 586)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 587) if (fscache_wait_for_deferred_lookup(cookie) < 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 588) return -ERESTARTSYS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 589)
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 590) op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 591) if (!op)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 592) return -ENOMEM;
1bb4b7f98f361 (David Howells 2013-05-21 13:44:15 +0100 593) atomic_set(&op->n_pages, *nr_pages);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 594) trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 595)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 596) spin_lock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 597)
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 598) if (!fscache_cookie_enabled(cookie) ||
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 599) hlist_empty(&cookie->backing_objects))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 600) goto nobufs_unlock;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 601) object = hlist_entry(cookie->backing_objects.first,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 602) struct fscache_object, cookie_link);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 603)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 604) __fscache_use_cookie(cookie);
4fbf4291aa159 (David Howells 2009-11-19 18:11:04 +0000 605) atomic_inc(&object->n_reads);
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 606) __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
4fbf4291aa159 (David Howells 2009-11-19 18:11:04 +0000 607)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 608) if (fscache_submit_op(object, &op->op) < 0)
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 609) goto nobufs_unlock_dec;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 610) spin_unlock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 611)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 612) fscache_stat(&fscache_n_retrieval_ops);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 613)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 614) /* we wait for the operation to become active, and then process it
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 615) * *here*, in this thread, and not in the thread pool */
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 616) ret = fscache_wait_for_operation_activation(
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 617) object, &op->op,
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 618) __fscache_stat(&fscache_n_retrieval_op_waits),
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 619) __fscache_stat(&fscache_n_retrievals_object_dead));
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 620) if (ret < 0)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 621) goto error;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 622)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 623) /* ask the cache to honour the operation */
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 624) if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 625) fscache_stat(&fscache_n_cop_allocate_pages);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 626) ret = object->cache->ops->allocate_pages(
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 627) op, pages, nr_pages, gfp);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 628) fscache_stat_d(&fscache_n_cop_allocate_pages);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 629) } else {
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 630) fscache_stat(&fscache_n_cop_read_or_alloc_pages);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 631) ret = object->cache->ops->read_or_alloc_pages(
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 632) op, pages, nr_pages, gfp);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 633) fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 634) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 635)
5753c44188925 (David Howells 2009-11-19 18:11:19 +0000 636) error:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 637) if (ret == -ENOMEM)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 638) fscache_stat(&fscache_n_retrievals_nomem);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 639) else if (ret == -ERESTARTSYS)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 640) fscache_stat(&fscache_n_retrievals_intr);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 641) else if (ret == -ENODATA)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 642) fscache_stat(&fscache_n_retrievals_nodata);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 643) else if (ret < 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 644) fscache_stat(&fscache_n_retrievals_nobufs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 645) else
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 646) fscache_stat(&fscache_n_retrievals_ok);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 647)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 648) fscache_put_retrieval(op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 649) _leave(" = %d", ret);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 650) return ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 651)
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 652) nobufs_unlock_dec:
9f10523f89192 (David Howells 2012-12-20 21:52:35 +0000 653) atomic_dec(&object->n_reads);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 654) wake_cookie = __fscache_unuse_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 655) nobufs_unlock:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 656) spin_unlock(&cookie->lock);
a39caadf06879 (David Howells 2015-02-25 14:22:40 +0000 657) fscache_put_retrieval(op);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 658) if (wake_cookie)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 659) __fscache_wake_unused_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 660) nobufs:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 661) fscache_stat(&fscache_n_retrievals_nobufs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 662) _leave(" = -ENOBUFS");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 663) return -ENOBUFS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 664) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 665) EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 666)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 667) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 668) * allocate a block in the cache on which to store a page
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 669) * - we return:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 670) * -ENOMEM - out of memory, nothing done
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 671) * -ERESTARTSYS - interrupted
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 672) * -ENOBUFS - no backing object available in which to cache the block
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 673) * 0 - block allocated
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 674) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 675) int __fscache_alloc_page(struct fscache_cookie *cookie,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 676) struct page *page,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 677) gfp_t gfp)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 678) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 679) struct fscache_retrieval *op;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 680) struct fscache_object *object;
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 681) bool wake_cookie = false;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 682) int ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 683)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 684) _enter("%p,%p,,,", cookie, page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 685)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 686) fscache_stat(&fscache_n_allocs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 687)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 688) if (hlist_empty(&cookie->backing_objects))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 689) goto nobufs;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 690)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 691) ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 692) ASSERTCMP(page, !=, NULL);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 693)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 694) if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 695) _leave(" = -ENOBUFS [invalidating]");
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 696) return -ENOBUFS;
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 697) }
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 698)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 699) if (fscache_wait_for_deferred_lookup(cookie) < 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 700) return -ERESTARTSYS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 701)
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 702) op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 703) if (!op)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 704) return -ENOMEM;
1bb4b7f98f361 (David Howells 2013-05-21 13:44:15 +0100 705) atomic_set(&op->n_pages, 1);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 706) trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_alloc_one);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 707)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 708) spin_lock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 709)
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 710) if (!fscache_cookie_enabled(cookie) ||
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 711) hlist_empty(&cookie->backing_objects))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 712) goto nobufs_unlock;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 713) object = hlist_entry(cookie->backing_objects.first,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 714) struct fscache_object, cookie_link);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 715)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 716) __fscache_use_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 717) if (fscache_submit_op(object, &op->op) < 0)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 718) goto nobufs_unlock_dec;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 719) spin_unlock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 720)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 721) fscache_stat(&fscache_n_alloc_ops);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 722)
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 723) ret = fscache_wait_for_operation_activation(
da9803bc8812f (David Howells 2013-08-21 17:29:38 -0400 724) object, &op->op,
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 725) __fscache_stat(&fscache_n_alloc_op_waits),
d3b97ca4a99e4 (David Howells 2015-02-24 10:05:29 +0000 726) __fscache_stat(&fscache_n_allocs_object_dead));
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 727) if (ret < 0)
60d543ca724be (David Howells 2009-11-19 18:11:45 +0000 728) goto error;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 729)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 730) /* ask the cache to honour the operation */
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 731) fscache_stat(&fscache_n_cop_allocate_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 732) ret = object->cache->ops->allocate_page(op, page, gfp);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 733) fscache_stat_d(&fscache_n_cop_allocate_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 734)
5753c44188925 (David Howells 2009-11-19 18:11:19 +0000 735) error:
5753c44188925 (David Howells 2009-11-19 18:11:19 +0000 736) if (ret == -ERESTARTSYS)
5753c44188925 (David Howells 2009-11-19 18:11:19 +0000 737) fscache_stat(&fscache_n_allocs_intr);
5753c44188925 (David Howells 2009-11-19 18:11:19 +0000 738) else if (ret < 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 739) fscache_stat(&fscache_n_allocs_nobufs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 740) else
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 741) fscache_stat(&fscache_n_allocs_ok);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 742)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 743) fscache_put_retrieval(op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 744) _leave(" = %d", ret);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 745) return ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 746)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 747) nobufs_unlock_dec:
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 748) wake_cookie = __fscache_unuse_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 749) nobufs_unlock:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 750) spin_unlock(&cookie->lock);
a39caadf06879 (David Howells 2015-02-25 14:22:40 +0000 751) fscache_put_retrieval(op);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 752) if (wake_cookie)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 753) __fscache_wake_unused_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 754) nobufs:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 755) fscache_stat(&fscache_n_allocs_nobufs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 756) _leave(" = -ENOBUFS");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 757) return -ENOBUFS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 758) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 759) EXPORT_SYMBOL(__fscache_alloc_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 760)
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 761) /*
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 762) * Unmark pages allocate in the readahead code path (via:
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 763) * fscache_readpages_or_alloc) after delegating to the base filesystem
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 764) */
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 765) void __fscache_readpages_cancel(struct fscache_cookie *cookie,
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 766) struct list_head *pages)
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 767) {
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 768) struct page *page;
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 769)
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 770) list_for_each_entry(page, pages, lru) {
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 771) if (PageFsCache(page))
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 772) __fscache_uncache_page(cookie, page);
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 773) }
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 774) }
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 775) EXPORT_SYMBOL(__fscache_readpages_cancel);
5a6f282a2052b (Milosz Tanski 2013-08-21 17:30:11 -0400 776)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 777) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 778) * release a write op reference
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 779) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 780) static void fscache_release_write_op(struct fscache_operation *_op)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 781) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 782) _enter("{OP%x}", _op->debug_id);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 783) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 784)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 785) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 786) * perform the background storage of a page into the cache
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 787) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 788) static void fscache_write_op(struct fscache_operation *_op)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 789) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 790) struct fscache_storage *op =
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 791) container_of(_op, struct fscache_storage, op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 792) struct fscache_object *object = op->op.object;
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 793) struct fscache_cookie *cookie;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 794) struct page *page;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 795) unsigned n;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 796) void *results[1];
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 797) int ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 798)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 799) _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 800)
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 801) again:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 802) spin_lock(&object->lock);
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 803) cookie = object->cookie;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 804)
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 805) if (!fscache_object_is_active(object)) {
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 806) /* If we get here, then the on-disk cache object likely no
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 807) * longer exists, so we should just cancel this write
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 808) * operation.
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 809) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 810) spin_unlock(&object->lock);
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 811) fscache_op_complete(&op->op, true);
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 812) _leave(" [inactive]");
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 813) return;
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 814) }
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 815)
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 816) if (!cookie) {
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 817) /* If we get here, then the cookie belonging to the object was
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 818) * detached, probably by the cookie being withdrawn due to
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 819) * memory pressure, which means that the pages we might write
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 820) * to the cache from no longer exist - therefore, we can just
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 821) * cancel this write operation.
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 822) */
7ef001e937e8b (David Howells 2012-12-07 10:41:26 +0000 823) spin_unlock(&object->lock);
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 824) fscache_op_complete(&op->op, true);
caaef6900befb (David Howells 2013-05-10 19:50:26 +0100 825) _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}",
caaef6900befb (David Howells 2013-05-10 19:50:26 +0100 826) _op->flags, _op->state, object->state->short_name,
caaef6900befb (David Howells 2013-05-10 19:50:26 +0100 827) object->flags);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 828) return;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 829) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 830)
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 831) spin_lock(&cookie->stores_lock);
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 832)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 833) fscache_stat(&fscache_n_store_calls);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 834)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 835) /* find a page to store */
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 836) results[0] = NULL;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 837) page = NULL;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 838) n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 839) FSCACHE_COOKIE_PENDING_TAG);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 840) trace_fscache_gang_lookup(cookie, &op->op, results, n, op->store_limit);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 841) if (n != 1)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 842) goto superseded;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 843) page = results[0];
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 844) _debug("gang %d [%lx]", n, page->index);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 845)
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 846) radix_tree_tag_set(&cookie->stores, page->index,
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 847) FSCACHE_COOKIE_STORING_TAG);
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 848) radix_tree_tag_clear(&cookie->stores, page->index,
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 849) FSCACHE_COOKIE_PENDING_TAG);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 850) trace_fscache_page(cookie, page, fscache_page_radix_pend2store);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 851)
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 852) spin_unlock(&cookie->stores_lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 853) spin_unlock(&object->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 854)
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 855) if (page->index >= op->store_limit)
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 856) goto discard_page;
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 857)
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 858) fscache_stat(&fscache_n_store_pages);
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 859) fscache_stat(&fscache_n_cop_write_page);
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 860) ret = object->cache->ops->write_page(op, page);
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 861) fscache_stat_d(&fscache_n_cop_write_page);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 862) trace_fscache_wrote_page(cookie, page, &op->op, ret);
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 863) fscache_end_page_write(object, page);
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 864) if (ret < 0) {
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 865) fscache_abort_object(object);
1f372dff1da37 (David Howells 2012-12-13 20:03:13 +0000 866) fscache_op_complete(&op->op, true);
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 867) } else {
08a66859e6926 (Dan Carpenter 2010-06-01 20:58:22 +0100 868) fscache_enqueue_operation(&op->op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 869) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 870)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 871) _leave("");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 872) return;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 873)
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 874) discard_page:
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 875) fscache_stat(&fscache_n_store_pages_over_limit);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 876) trace_fscache_wrote_page(cookie, page, &op->op, -ENOBUFS);
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 877) fscache_end_page_write(object, page);
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 878) goto again;
2c98425720233 (David Howells 2018-04-04 13:41:26 +0100 879)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 880) superseded:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 881) /* this writer is going away and there aren't any more things to
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 882) * write */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 883) _debug("cease");
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 884) spin_unlock(&cookie->stores_lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 885) clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 886) spin_unlock(&object->lock);
b27ddd4624531 (David Howells 2018-04-04 13:41:26 +0100 887) fscache_op_complete(&op->op, false);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 888) _leave("");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 889) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 890)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 891) /*
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 892) * Clear the pages pending writing for invalidation
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 893) */
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 894) void fscache_invalidate_writes(struct fscache_cookie *cookie)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 895) {
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 896) struct page *page;
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 897) void *results[16];
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 898) int n, i;
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 899)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 900) _enter("");
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 901)
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 902) for (;;) {
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 903) spin_lock(&cookie->stores_lock);
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 904) n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 905) ARRAY_SIZE(results),
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 906) FSCACHE_COOKIE_PENDING_TAG);
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 907) if (n == 0) {
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 908) spin_unlock(&cookie->stores_lock);
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 909) break;
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 910) }
ee8be57bc331f (Sebastian Andrzej Siewior 2013-05-10 19:50:24 +0100 911)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 912) for (i = n - 1; i >= 0; i--) {
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 913) page = results[i];
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 914) radix_tree_delete(&cookie->stores, page->index);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 915) trace_fscache_page(cookie, page, fscache_page_radix_delete);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 916) trace_fscache_page(cookie, page, fscache_page_inval);
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 917) }
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 918)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 919) spin_unlock(&cookie->stores_lock);
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 920)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 921) for (i = n - 1; i >= 0; i--)
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 922) put_page(results[i]);
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 923) }
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 924)
d2138455286f9 (Yan, Zheng 2016-05-17 11:52:48 +0800 925) wake_up_bit(&cookie->flags, 0);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 926) trace_fscache_wake_cookie(cookie);
d2138455286f9 (Yan, Zheng 2016-05-17 11:52:48 +0800 927)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 928) _leave("");
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 929) }
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 930)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 931) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 932) * request a page be stored in the cache
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 933) * - returns:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 934) * -ENOMEM - out of memory, nothing done
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 935) * -ENOBUFS - no backing object available in which to cache the page
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 936) * 0 - dispatched a write - it'll call end_io_func() when finished
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 937) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 938) * if the cookie still has a backing object at this point, that object can be
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 939) * in one of a few states with respect to storage processing:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 940) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 941) * (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 942) * set)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 943) *
caaef6900befb (David Howells 2013-05-10 19:50:26 +0100 944) * (a) no writes yet
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 945) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 946) * (b) writes deferred till post-creation (mark page for writing and
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 947) * return immediately)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 948) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 949) * (2) negative lookup, object created, initial fill being made from netfs
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 950) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 951) * (a) fill point not yet reached this page (mark page for writing and
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 952) * return)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 953) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 954) * (b) fill point passed this page (queue op to store this page)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 955) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 956) * (3) object extant (queue op to store this page)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 957) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 958) * any other state is invalid
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 959) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 960) int __fscache_write_page(struct fscache_cookie *cookie,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 961) struct page *page,
ee1235a9a0681 (David Howells 2018-04-04 13:41:28 +0100 962) loff_t object_size,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 963) gfp_t gfp)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 964) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 965) struct fscache_storage *op;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 966) struct fscache_object *object;
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 967) bool wake_cookie = false;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 968) int ret;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 969)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 970) _enter("%p,%x,", cookie, (u32) page->flags);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 971)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 972) ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 973) ASSERT(PageFsCache(page));
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 974)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 975) fscache_stat(&fscache_n_stores);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 976)
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 977) if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 978) _leave(" = -ENOBUFS [invalidating]");
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 979) return -ENOBUFS;
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 980) }
ef778e7ae67cd (David Howells 2012-12-20 21:52:36 +0000 981)
5f4f9f4af185d (David Howells 2012-12-20 21:52:33 +0000 982) op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 983) if (!op)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 984) goto nomem;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 985)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 986) fscache_operation_init(cookie, &op->op, fscache_write_op, NULL,
8af7c12436803 (Tejun Heo 2010-07-20 22:09:01 +0200 987) fscache_release_write_op);
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 988) op->op.flags = FSCACHE_OP_ASYNC |
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 989) (1 << FSCACHE_OP_WAITING) |
1362729b169b7 (David Howells 2013-05-10 19:50:26 +0100 990) (1 << FSCACHE_OP_UNUSE_COOKIE);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 991)
5e4c0d974139a (Jan Kara 2013-09-11 14:26:05 -0700 992) ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 993) if (ret < 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 994) goto nomem_free;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 995)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 996) trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_write_one);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 997)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 998) ret = -ENOBUFS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 999) spin_lock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1000)
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 1001) if (!fscache_cookie_enabled(cookie) ||
94d30ae90a00c (David Howells 2013-09-21 00:09:31 +0100 1002) hlist_empty(&cookie->backing_objects))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1003) goto nobufs;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1004) object = hlist_entry(cookie->backing_objects.first,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1005) struct fscache_object, cookie_link);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1006) if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1007) goto nobufs;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1008)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1009) trace_fscache_page(cookie, page, fscache_page_write);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1010)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1011) /* add the page to the pending-storage radix tree on the backing
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1012) * object */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1013) spin_lock(&object->lock);
ee1235a9a0681 (David Howells 2018-04-04 13:41:28 +0100 1014)
ee1235a9a0681 (David Howells 2018-04-04 13:41:28 +0100 1015) if (object->store_limit_l != object_size)
ee1235a9a0681 (David Howells 2018-04-04 13:41:28 +0100 1016) fscache_set_store_limit(object, object_size);
ee1235a9a0681 (David Howells 2018-04-04 13:41:28 +0100 1017)
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 1018) spin_lock(&cookie->stores_lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1019)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1020) _debug("store limit %llx", (unsigned long long) object->store_limit);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1021)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1022) ret = radix_tree_insert(&cookie->stores, page->index, page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1023) if (ret < 0) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1024) if (ret == -EEXIST)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1025) goto already_queued;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1026) _debug("insert failed %d", ret);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1027) goto nobufs_unlock_obj;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1028) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1029)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1030) trace_fscache_page(cookie, page, fscache_page_radix_insert);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1031) radix_tree_tag_set(&cookie->stores, page->index,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1032) FSCACHE_COOKIE_PENDING_TAG);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1033) trace_fscache_page(cookie, page, fscache_page_radix_set_pend);
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 1034) get_page(page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1035)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1036) /* we only want one writer at a time, but we do need to queue new
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1037) * writers after exclusive ops */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1038) if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1039) goto already_pending;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1040)
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 1041) spin_unlock(&cookie->stores_lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1042) spin_unlock(&object->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1043)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1044) op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1045) op->store_limit = object->store_limit;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1046)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 1047) __fscache_use_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1048) if (fscache_submit_op(object, &op->op) < 0)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1049) goto submit_failed;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1050)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1051) spin_unlock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1052) radix_tree_preload_end();
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1053) fscache_stat(&fscache_n_store_ops);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1054) fscache_stat(&fscache_n_stores_ok);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1055)
8af7c12436803 (Tejun Heo 2010-07-20 22:09:01 +0200 1056) /* the work queue now carries its own ref on the object */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1057) fscache_put_operation(&op->op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1058) _leave(" = 0");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1059) return 0;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1060)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1061) already_queued:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1062) fscache_stat(&fscache_n_stores_again);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1063) already_pending:
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 1064) spin_unlock(&cookie->stores_lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1065) spin_unlock(&object->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1066) spin_unlock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1067) radix_tree_preload_end();
a39caadf06879 (David Howells 2015-02-25 14:22:40 +0000 1068) fscache_put_operation(&op->op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1069) fscache_stat(&fscache_n_stores_ok);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1070) _leave(" = 0");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1071) return 0;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1072)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1073) submit_failed:
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 1074) spin_lock(&cookie->stores_lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1075) radix_tree_delete(&cookie->stores, page->index);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1076) trace_fscache_page(cookie, page, fscache_page_radix_delete);
1bccf513ac49d (David Howells 2009-11-19 18:11:25 +0000 1077) spin_unlock(&cookie->stores_lock);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 1078) wake_cookie = __fscache_unuse_cookie(cookie);
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 1079) put_page(page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1080) ret = -ENOBUFS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1081) goto nobufs;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1082)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1083) nobufs_unlock_obj:
1147d0f915e3b (Dan Carpenter 2010-03-23 14:48:37 +0000 1084) spin_unlock(&cookie->stores_lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1085) spin_unlock(&object->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1086) nobufs:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1087) spin_unlock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1088) radix_tree_preload_end();
a39caadf06879 (David Howells 2015-02-25 14:22:40 +0000 1089) fscache_put_operation(&op->op);
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 1090) if (wake_cookie)
8fb883f3e3006 (David Howells 2013-09-21 00:09:31 +0100 1091) __fscache_wake_unused_cookie(cookie);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1092) fscache_stat(&fscache_n_stores_nobufs);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1093) _leave(" = -ENOBUFS");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1094) return -ENOBUFS;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1095)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1096) nomem_free:
a39caadf06879 (David Howells 2015-02-25 14:22:40 +0000 1097) fscache_put_operation(&op->op);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1098) nomem:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1099) fscache_stat(&fscache_n_stores_oom);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1100) _leave(" = -ENOMEM");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1101) return -ENOMEM;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1102) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1103) EXPORT_SYMBOL(__fscache_write_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1104)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1105) /*
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1106) * remove a page from the cache
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1107) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1108) void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1109) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1110) struct fscache_object *object;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1111)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1112) _enter(",%p", page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1113)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1114) ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1115) ASSERTCMP(page, !=, NULL);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1116)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1117) fscache_stat(&fscache_n_uncaches);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1118)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1119) /* cache withdrawal may beat us to it */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1120) if (!PageFsCache(page))
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1121) goto done;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1122)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1123) trace_fscache_page(cookie, page, fscache_page_uncache);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1124)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1125) /* get the object */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1126) spin_lock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1127)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1128) if (hlist_empty(&cookie->backing_objects)) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1129) ClearPageFsCache(page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1130) goto done_unlock;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1131) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1132)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1133) object = hlist_entry(cookie->backing_objects.first,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1134) struct fscache_object, cookie_link);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1135)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1136) /* there might now be stuff on disk we could read */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1137) clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1138)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1139) /* only invoke the cache backend if we managed to mark the page
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1140) * uncached here; this deals with synchronisation vs withdrawal */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1141) if (TestClearPageFsCache(page) &&
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1142) object->cache->ops->uncache_page) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1143) /* the cache backend releases the cookie lock */
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 1144) fscache_stat(&fscache_n_cop_uncache_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1145) object->cache->ops->uncache_page(object, page);
52bd75fdb135d (David Howells 2009-11-19 18:11:08 +0000 1146) fscache_stat_d(&fscache_n_cop_uncache_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1147) goto done;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1148) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1149)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1150) done_unlock:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1151) spin_unlock(&cookie->lock);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1152) done:
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1153) _leave("");
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1154) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1155) EXPORT_SYMBOL(__fscache_uncache_page);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1156)
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1157) /**
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1158) * fscache_mark_page_cached - Mark a page as being cached
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1159) * @op: The retrieval op pages are being marked for
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1160) * @page: The page to be marked
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1161) *
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1162) * Mark a netfs page as being cached. After this is called, the netfs
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1163) * must call fscache_uncache_page() to remove the mark.
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1164) */
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1165) void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1166) {
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1167) struct fscache_cookie *cookie = op->op.object->cookie;
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1168)
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1169) #ifdef CONFIG_FSCACHE_STATS
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1170) atomic_inc(&fscache_n_marks);
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1171) #endif
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1172)
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1173) trace_fscache_page(cookie, page, fscache_page_cached);
08c2e3d087840 (David Howells 2018-04-04 13:41:27 +0100 1174)
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1175) _debug("- mark %p{%lx}", page, page->index);
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1176) if (TestSetPageFsCache(page)) {
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1177) static bool once_only;
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1178) if (!once_only) {
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1179) once_only = true;
36dfd116edd48 (Fabian Frederick 2014-06-04 16:05:38 -0700 1180) pr_warn("Cookie type %s marked page %lx multiple times\n",
36dfd116edd48 (Fabian Frederick 2014-06-04 16:05:38 -0700 1181) cookie->def->name, page->index);
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1182) }
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1183) }
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1184)
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1185) if (cookie->def->mark_page_cached)
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1186) cookie->def->mark_page_cached(cookie->netfs_data,
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1187) op->mapping, page);
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1188) }
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1189) EXPORT_SYMBOL(fscache_mark_page_cached);
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1190)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1191) /**
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1192) * fscache_mark_pages_cached - Mark pages as being cached
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1193) * @op: The retrieval op pages are being marked for
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1194) * @pagevec: The pages to be marked
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1195) *
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1196) * Mark a bunch of netfs pages as being cached. After this is called,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1197) * the netfs must call fscache_uncache_page() to remove the mark.
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1198) */
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1199) void fscache_mark_pages_cached(struct fscache_retrieval *op,
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1200) struct pagevec *pagevec)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1201) {
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1202) unsigned long loop;
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1203)
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1204) for (loop = 0; loop < pagevec->nr; loop++)
c4d6d8dbf335c (David Howells 2012-12-20 21:52:32 +0000 1205) fscache_mark_page_cached(op, pagevec->pages[loop]);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1206)
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1207) pagevec_reinit(pagevec);
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1208) }
b510882281d56 (David Howells 2009-04-03 16:42:39 +0100 1209) EXPORT_SYMBOL(fscache_mark_pages_cached);
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1210)
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1211) /*
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1212) * Uncache all the pages in an inode that are marked PG_fscache, assuming them
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1213) * to be associated with the given cookie.
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1214) */
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1215) void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1216) struct inode *inode)
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1217) {
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1218) struct address_space *mapping = inode->i_mapping;
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1219) struct pagevec pvec;
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1220) pgoff_t next;
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1221) int i;
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1222)
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1223) _enter("%p,%p", cookie, inode);
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1224)
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1225) if (!mapping || mapping->nrpages == 0) {
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1226) _leave(" [no pages]");
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1227) return;
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1228) }
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1229)
8667982014d60 (Mel Gorman 2017-11-15 17:37:52 -0800 1230) pagevec_init(&pvec);
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1231) next = 0;
b307d4655a717 (Jan Beulich 2011-07-21 15:02:43 +0100 1232) do {
397162ffa2ed1 (Jan Kara 2017-09-06 16:21:43 -0700 1233) if (!pagevec_lookup(&pvec, mapping, &next))
b307d4655a717 (Jan Beulich 2011-07-21 15:02:43 +0100 1234) break;
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1235) for (i = 0; i < pagevec_count(&pvec); i++) {
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1236) struct page *page = pvec.pages[i];
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1237) if (PageFsCache(page)) {
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1238) __fscache_wait_on_page_write(cookie, page);
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1239) __fscache_uncache_page(cookie, page);
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1240) }
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1241) }
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1242) pagevec_release(&pvec);
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1243) cond_resched();
d72dc8a25afc7 (Jan Kara 2017-09-06 16:21:18 -0700 1244) } while (next);
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1245)
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1246) _leave("");
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1247) }
c902ce1bfb40d (David Howells 2011-07-07 12:19:48 +0100 1248) EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags