^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/rmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/swapops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) static inline bool not_found(struct page_vma_mapped_walk *pvmw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) page_vma_mapped_walk_done(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) static bool map_pte(struct page_vma_mapped_walk *pvmw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) pvmw->pte = pte_offset_map(pvmw->pmd, pvmw->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) if (!(pvmw->flags & PVMW_SYNC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) if (pvmw->flags & PVMW_MIGRATION) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) if (!is_swap_pte(*pvmw->pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * We get here when we are trying to unmap a private
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * device page from the process address space. Such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * page is not CPU accessible and thus is mapped as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * a special swap entry, nonetheless it still does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * count as a valid regular mapping for the page (and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * is accounted as such in page maps count).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * So handle this special case as if it was a normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * page mapping ie lock CPU page table and returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * For more details on device private memory see HMM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * (include/linux/hmm.h or mm/hmm.c).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (is_swap_pte(*pvmw->pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) swp_entry_t entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* Handle un-addressable ZONE_DEVICE memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) entry = pte_to_swp_entry(*pvmw->pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) if (!is_device_private_entry(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) } else if (!pte_present(*pvmw->pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) pvmw->ptl = pte_lockptr(pvmw->vma->vm_mm, pvmw->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) spin_lock(pvmw->ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static inline bool pfn_is_match(struct page *page, unsigned long pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) unsigned long page_pfn = page_to_pfn(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /* normal page and hugetlbfs page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (!PageTransCompound(page) || PageHuge(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) return page_pfn == pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* THP can be referenced by any subpage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) return pfn >= page_pfn && pfn - page_pfn < thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * check_pte - check if @pvmw->page is mapped at the @pvmw->pte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * page_vma_mapped_walk() found a place where @pvmw->page is *potentially*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * mapped. check_pte() has to validate this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * @pvmw->pte may point to empty PTE, swap PTE or PTE pointing to arbitrary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * If PVMW_MIGRATION flag is set, returns true if @pvmw->pte contains migration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * entry that points to @pvmw->page or any subpage in case of THP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * If PVMW_MIGRATION flag is not set, returns true if @pvmw->pte points to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * @pvmw->page or any subpage in case of THP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * Otherwise, return false.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static bool check_pte(struct page_vma_mapped_walk *pvmw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (pvmw->flags & PVMW_MIGRATION) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) swp_entry_t entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (!is_swap_pte(*pvmw->pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) entry = pte_to_swp_entry(*pvmw->pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (!is_migration_entry(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) pfn = migration_entry_to_pfn(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) } else if (is_swap_pte(*pvmw->pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) swp_entry_t entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) /* Handle un-addressable ZONE_DEVICE memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) entry = pte_to_swp_entry(*pvmw->pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (!is_device_private_entry(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) pfn = device_private_entry_to_pfn(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (!pte_present(*pvmw->pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) pfn = pte_pfn(*pvmw->pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return pfn_is_match(pvmw->page, pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) pvmw->address = (pvmw->address + size) & ~(size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (!pvmw->address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) pvmw->address = ULONG_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * @pvmw->address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * must be set. pmd, pte and ptl must be NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * Returns true if the page is mapped in the vma. @pvmw->pmd and @pvmw->pte point
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * to relevant page table entries. @pvmw->ptl is locked. @pvmw->address is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * adjusted if needed (for PTE-mapped THPs).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * If @pvmw->pmd is set but @pvmw->pte is not, you have found PMD-mapped page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * (usually THP). For PTE-mapped THP, you should run page_vma_mapped_walk() in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * a loop to find all PTEs that map the THP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * For HugeTLB pages, @pvmw->pte is set to the relevant page table entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * regardless of which page table level the page is mapped at. @pvmw->pmd is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * Retruns false if there are no more page table entries for the page in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * the vma. @pvmw->ptl is unlocked and @pvmw->pte is unmapped.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * If you need to stop the walk before page_vma_mapped_walk() returned false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * use page_vma_mapped_walk_done(). It will do the housekeeping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct mm_struct *mm = pvmw->vma->vm_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) struct page *page = pvmw->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) unsigned long end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) pmd_t pmde;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* The only possible pmd mapping has been handled on last iteration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (pvmw->pmd && !pvmw->pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return not_found(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (unlikely(PageHuge(page))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /* The only possible mapping was handled on last iteration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (pvmw->pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) return not_found(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* when pud is not present, pte will be NULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) pvmw->pte = huge_pte_offset(mm, pvmw->address, page_size(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (!pvmw->pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) spin_lock(pvmw->ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (!check_pte(pvmw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return not_found(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * Seek to next pte only makes sense for THP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * But more important than that optimization, is to filter out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * any PageKsm page: whose page->index misleads vma_address()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * and vma_address_end() to disaster.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) end = PageTransCompound(page) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) vma_address_end(page, pvmw->vma) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) pvmw->address + PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (pvmw->pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) goto next_pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) restart:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) pgd = pgd_offset(mm, pvmw->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) if (!pgd_present(*pgd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) step_forward(pvmw, PGDIR_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) p4d = p4d_offset(pgd, pvmw->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) if (!p4d_present(*p4d)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) step_forward(pvmw, P4D_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) pud = pud_offset(p4d, pvmw->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (!pud_present(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) step_forward(pvmw, PUD_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) pvmw->pmd = pmd_offset(pud, pvmw->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * Make sure the pmd value isn't cached in a register by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * compiler and used as a stale value after we've observed a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * subsequent update.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) pmde = READ_ONCE(*pvmw->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (pmd_trans_huge(pmde) || is_pmd_migration_entry(pmde)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) pvmw->ptl = pmd_lock(mm, pvmw->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) pmde = *pvmw->pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (likely(pmd_trans_huge(pmde))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (pvmw->flags & PVMW_MIGRATION)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return not_found(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (pmd_page(pmde) != page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return not_found(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) if (!pmd_present(pmde)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) swp_entry_t entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (!thp_migration_supported() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) !(pvmw->flags & PVMW_MIGRATION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) return not_found(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) entry = pmd_to_swp_entry(pmde);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (!is_migration_entry(entry) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) migration_entry_to_page(entry) != page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return not_found(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /* THP pmd was split under us: handle on pte level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) spin_unlock(pvmw->ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) pvmw->ptl = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) } else if (!pmd_present(pmde)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * If PVMW_SYNC, take and drop THP pmd lock so that we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * cannot return prematurely, while zap_huge_pmd() has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * cleared *pmd but not decremented compound_mapcount().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if ((pvmw->flags & PVMW_SYNC) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) PageTransCompound(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) spinlock_t *ptl = pmd_lock(mm, pvmw->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) spin_unlock(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) step_forward(pvmw, PMD_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (!map_pte(pvmw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) goto next_pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) this_pte:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (check_pte(pvmw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) next_pte:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) pvmw->address += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (pvmw->address >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) return not_found(pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /* Did we cross page table boundary? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) if (pvmw->ptl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) spin_unlock(pvmw->ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) pvmw->ptl = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) pte_unmap(pvmw->pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) pvmw->pte = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) goto restart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) pvmw->pte++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if ((pvmw->flags & PVMW_SYNC) && !pvmw->ptl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) pvmw->ptl = pte_lockptr(mm, pvmw->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) spin_lock(pvmw->ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) } while (pte_none(*pvmw->pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (!pvmw->ptl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) pvmw->ptl = pte_lockptr(mm, pvmw->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) spin_lock(pvmw->ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) goto this_pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) } while (pvmw->address < end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * page_mapped_in_vma - check whether a page is really mapped in a VMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * @page: the page to test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * @vma: the VMA to test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * Returns 1 if the page is mapped into the page tables of the VMA, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * if the page is not mapped into the page tables of this VMA. Only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * valid for normal file or anonymous VMAs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) struct page_vma_mapped_walk pvmw = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) .page = page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) .vma = vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) .flags = PVMW_SYNC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) pvmw.address = vma_address(page, vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (pvmw.address == -EFAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (!page_vma_mapped_walk(&pvmw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) page_vma_mapped_walk_done(&pvmw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags