457c899653991 (Thomas Gleixner 2019-05-19 13:08:55 +0100 1) // SPDX-License-Identifier: GPL-2.0-only
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 2) /*
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 3) * mm/balloon_compaction.c
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 4) *
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 5) * Common interface for making balloon pages movable by compaction.
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 6) *
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 7) * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com>
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 8) */
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 9) #include <linux/mm.h>
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 10) #include <linux/slab.h>
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 11) #include <linux/export.h>
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 12) #include <linux/balloon_compaction.h>
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 13)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 14) static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 15) struct page *page)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 16) {
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 17) /*
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 18) * Block others from accessing the 'page' when we get around to
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 19) * establishing additional references. We should be the only one
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 20) * holding a reference to the 'page' at this point. If we are not, then
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 21) * memory corruption is possible and we should stop execution.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 22) */
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 23) BUG_ON(!trylock_page(page));
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 24) balloon_page_insert(b_dev_info, page);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 25) unlock_page(page);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 26) __count_vm_event(BALLOON_INFLATE);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 27) }
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 28)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 29) /**
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 30) * balloon_page_list_enqueue() - inserts a list of pages into the balloon page
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 31) * list.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 32) * @b_dev_info: balloon device descriptor where we will insert a new page to
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 33) * @pages: pages to enqueue - allocated using balloon_page_alloc.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 34) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 35) * Driver must call this function to properly enqueue balloon pages before
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 36) * definitively removing them from the guest system.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 37) *
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 38) * Return: number of pages that were enqueued.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 39) */
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 40) size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 41) struct list_head *pages)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 42) {
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 43) struct page *page, *tmp;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 44) unsigned long flags;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 45) size_t n_pages = 0;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 46)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 47) spin_lock_irqsave(&b_dev_info->pages_lock, flags);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 48) list_for_each_entry_safe(page, tmp, pages, lru) {
dd422906799f2 (Wei Wang 2019-07-18 17:27:20 +0800 49) list_del(&page->lru);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 50) balloon_page_enqueue_one(b_dev_info, page);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 51) n_pages++;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 52) }
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 53) spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 54) return n_pages;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 55) }
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 56) EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 57)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 58) /**
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 59) * balloon_page_list_dequeue() - removes pages from balloon's page list and
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 60) * returns a list of the pages.
f0953a1bbaca7 (Ingo Molnar 2021-05-06 18:06:47 -0700 61) * @b_dev_info: balloon device descriptor where we will grab a page from.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 62) * @pages: pointer to the list of pages that would be returned to the caller.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 63) * @n_req_pages: number of requested pages.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 64) *
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 65) * Driver must call this function to properly de-allocate a previous enlisted
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 66) * balloon pages before definitively releasing it back to the guest system.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 67) * This function tries to remove @n_req_pages from the ballooned pages and
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 68) * return them to the caller in the @pages list.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 69) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 70) * Note that this function may fail to dequeue some pages even if the balloon
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 71) * isn't empty - since the page list can be temporarily empty due to compaction
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 72) * of isolated pages.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 73) *
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 74) * Return: number of pages that were added to the @pages list.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 75) */
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 76) size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 77) struct list_head *pages, size_t n_req_pages)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 78) {
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 79) struct page *page, *tmp;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 80) unsigned long flags;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 81) size_t n_pages = 0;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 82)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 83) spin_lock_irqsave(&b_dev_info->pages_lock, flags);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 84) list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 85) if (n_pages == n_req_pages)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 86) break;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 87)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 88) /*
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 89) * Block others from accessing the 'page' while we get around to
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 90) * establishing additional references and preparing the 'page'
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 91) * to be released by the balloon driver.
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 92) */
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 93) if (!trylock_page(page))
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 94) continue;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 95)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 96) if (IS_ENABLED(CONFIG_BALLOON_COMPACTION) &&
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 97) PageIsolated(page)) {
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 98) /* raced with isolation */
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 99) unlock_page(page);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 100) continue;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 101) }
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 102) balloon_page_delete(page);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 103) __count_vm_event(BALLOON_DEFLATE);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 104) list_add(&page->lru, pages);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 105) unlock_page(page);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 106) n_pages++;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 107) }
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 108) spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 109)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 110) return n_pages;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 111) }
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 112) EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 113)
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 114) /*
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 115) * balloon_page_alloc - allocates a new page for insertion into the balloon
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 116) * page list.
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 117) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 118) * Driver must call this function to properly allocate a new balloon page.
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 119) * Driver must call balloon_page_enqueue before definitively removing the page
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 120) * from the guest system.
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 121) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 122) * Return: struct page for the allocated page or NULL on allocation failure.
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 123) */
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 124) struct page *balloon_page_alloc(void)
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 125) {
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 126) struct page *page = alloc_page(balloon_mapping_gfp_mask() |
02fa5d7b17a76 (Nadav Amit 2019-08-20 02:16:46 -0700 127) __GFP_NOMEMALLOC | __GFP_NORETRY |
02fa5d7b17a76 (Nadav Amit 2019-08-20 02:16:46 -0700 128) __GFP_NOWARN);
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 129) return page;
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 130) }
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 131) EXPORT_SYMBOL_GPL(balloon_page_alloc);
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 132)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 133) /*
dd422906799f2 (Wei Wang 2019-07-18 17:27:20 +0800 134) * balloon_page_enqueue - inserts a new page into the balloon page list.
dd422906799f2 (Wei Wang 2019-07-18 17:27:20 +0800 135) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 136) * @b_dev_info: balloon device descriptor where we will insert a new page
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 137) * @page: new page to enqueue - allocated using balloon_page_alloc.
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 138) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 139) * Drivers must call this function to properly enqueue a new allocated balloon
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 140) * page before definitively removing the page from the guest system.
dd422906799f2 (Wei Wang 2019-07-18 17:27:20 +0800 141) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 142) * Drivers must not call balloon_page_enqueue on pages that have been pushed to
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 143) * a list with balloon_page_push before removing them with balloon_page_pop. To
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 144) * enqueue a list of pages, use balloon_page_list_enqueue instead.
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 145) */
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 146) void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
c7cdff0e86471 (Michael S. Tsirkin 2017-10-13 16:11:48 +0300 147) struct page *page)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 148) {
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 149) unsigned long flags;
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 150)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 151) spin_lock_irqsave(&b_dev_info->pages_lock, flags);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 152) balloon_page_enqueue_one(b_dev_info, page);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 153) spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 154) }
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 155) EXPORT_SYMBOL_GPL(balloon_page_enqueue);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 156)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 157) /*
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 158) * balloon_page_dequeue - removes a page from balloon's page list and returns
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 159) * its address to allow the driver to release the page.
f0953a1bbaca7 (Ingo Molnar 2021-05-06 18:06:47 -0700 160) * @b_dev_info: balloon device descriptor where we will grab a page from.
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 161) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 162) * Driver must call this function to properly dequeue a previously enqueued page
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 163) * before definitively releasing it back to the guest system.
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 164) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 165) * Caller must perform its own accounting to ensure that this
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 166) * function is called only if some pages are actually enqueued.
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 167) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 168) * Note that this function may fail to dequeue some pages even if there are
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 169) * some enqueued pages - since the page list can be temporarily empty due to
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 170) * the compaction of isolated pages.
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 171) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 172) * TODO: remove the caller accounting requirements, and allow caller to wait
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 173) * until all pages can be dequeued.
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 174) *
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 175) * Return: struct page for the dequeued page, or NULL if no page was dequeued.
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 176) */
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 177) struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 178) {
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 179) unsigned long flags;
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 180) LIST_HEAD(pages);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 181) int n_pages;
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 182)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 183) n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 184)
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 185) if (n_pages != 1) {
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 186) /*
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 187) * If we are unable to dequeue a balloon page because the page
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 188) * list is empty and there are no isolated pages, then something
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 189) * went out of track and some balloon pages are lost.
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 190) * BUG() here, otherwise the balloon driver may get stuck in
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 191) * an infinite loop while attempting to release all its pages.
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 192) */
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 193) spin_lock_irqsave(&b_dev_info->pages_lock, flags);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 194) if (unlikely(list_empty(&b_dev_info->pages) &&
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 195) !b_dev_info->isolated_pages))
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 196) BUG();
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 197) spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 198) return NULL;
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 199) }
418a3ab1e7785 (Nadav Amit 2019-04-25 04:54:42 -0700 200) return list_first_entry(&pages, struct page, lru);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 201) }
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 202) EXPORT_SYMBOL_GPL(balloon_page_dequeue);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 203)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 204) #ifdef CONFIG_BALLOON_COMPACTION
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 205)
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 206) bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 207)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 208) {
9d1ba8056474a (Konstantin Khlebnikov 2014-10-09 15:29:29 -0700 209) struct balloon_dev_info *b_dev_info = balloon_page_device(page);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 210) unsigned long flags;
d6d86c0a7f8dd (Konstantin Khlebnikov 2014-10-09 15:29:27 -0700 211)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 212) spin_lock_irqsave(&b_dev_info->pages_lock, flags);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 213) list_del(&page->lru);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 214) b_dev_info->isolated_pages++;
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 215) spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 216)
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 217) return true;
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 218) }
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 219)
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 220) void balloon_page_putback(struct page *page)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 221) {
9d1ba8056474a (Konstantin Khlebnikov 2014-10-09 15:29:29 -0700 222) struct balloon_dev_info *b_dev_info = balloon_page_device(page);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 223) unsigned long flags;
d6d86c0a7f8dd (Konstantin Khlebnikov 2014-10-09 15:29:27 -0700 224)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 225) spin_lock_irqsave(&b_dev_info->pages_lock, flags);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 226) list_add(&page->lru, &b_dev_info->pages);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 227) b_dev_info->isolated_pages--;
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 228) spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 229) }
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 230)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 231)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 232) /* move_to_new_page() counterpart for a ballooned page */
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 233) int balloon_page_migrate(struct address_space *mapping,
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 234) struct page *newpage, struct page *page,
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 235) enum migrate_mode mode)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 236) {
9d1ba8056474a (Konstantin Khlebnikov 2014-10-09 15:29:29 -0700 237) struct balloon_dev_info *balloon = balloon_page_device(page);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 238)
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 239) /*
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 240) * We can not easily support the no copy case here so ignore it as it
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 241) * is unlikely to be used with balloon pages. See include/linux/hmm.h
cfe61801b0f11 (Michael S. Tsirkin 2019-07-18 08:19:24 -0400 242) * for a user of the MIGRATE_SYNC_NO_COPY mode.
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 243) */
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 244) if (mode == MIGRATE_SYNC_NO_COPY)
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 245) return -EINVAL;
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 246)
7db7671f835cc (Hugh Dickins 2015-11-05 18:49:49 -0800 247) VM_BUG_ON_PAGE(!PageLocked(page), page);
7db7671f835cc (Hugh Dickins 2015-11-05 18:49:49 -0800 248) VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 249)
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 250) return balloon->migratepage(balloon, newpage, page, mode);
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 251) }
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 252)
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 253) const struct address_space_operations balloon_aops = {
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 254) .migratepage = balloon_page_migrate,
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 255) .isolate_page = balloon_page_isolate,
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 256) .putback_page = balloon_page_putback,
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 257) };
b1123ea6d3b3d (Minchan Kim 2016-07-26 15:23:09 -0700 258) EXPORT_SYMBOL_GPL(balloon_aops);
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 259)
18468d93e53b0 (Rafael Aquini 2012-12-11 16:02:38 -0800 260) #endif /* CONFIG_BALLOON_COMPACTION */
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
33 Branches
57 Tags