524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 1) // SPDX-License-Identifier: GPL-2.0-only
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 2) /*
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 3) * Copyright (C) Gao Xiang <xiang@kernel.org>
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 4) *
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 5) * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 6) * per-CPU virtual memory (in pages) in advance to store such inplace I/O
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 7) * data if inplace decompression is failed (due to unmet inplace margin for
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 8) * example).
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 9) */
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 10) #include "internal.h"
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 11)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 12) struct erofs_pcpubuf {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 13) raw_spinlock_t lock;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 14) void *ptr;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 15) struct page **pages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 16) unsigned int nrpages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 17) };
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 18)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 19) static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 20)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 21) void *erofs_get_pcpubuf(unsigned int requiredpages)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 22) __acquires(pcb->lock)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 23) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 24) struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 25)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 26) raw_spin_lock(&pcb->lock);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 27) /* check if the per-CPU buffer is too small */
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 28) if (requiredpages > pcb->nrpages) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 29) raw_spin_unlock(&pcb->lock);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 30) put_cpu_var(erofs_pcb);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 31) /* (for sparse checker) pretend pcb->lock is still taken */
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 32) __acquire(pcb->lock);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 33) return NULL;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 34) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 35) return pcb->ptr;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 36) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 37)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 38) void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 39) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 40) struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 41)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 42) DBG_BUGON(pcb->ptr != ptr);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 43) raw_spin_unlock(&pcb->lock);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 44) put_cpu_var(erofs_pcb);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 45) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 46)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 47) /* the next step: support per-CPU page buffers hotplug */
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 48) int erofs_pcpubuf_growsize(unsigned int nrpages)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 49) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 50) static DEFINE_MUTEX(pcb_resize_mutex);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 51) static unsigned int pcb_nrpages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 52) LIST_HEAD(pagepool);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 53) int delta, cpu, ret, i;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 54)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 55) mutex_lock(&pcb_resize_mutex);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 56) delta = nrpages - pcb_nrpages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 57) ret = 0;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 58) /* avoid shrinking pcpubuf, since no idea how many fses rely on */
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 59) if (delta <= 0)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 60) goto out;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 61)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 62) for_each_possible_cpu(cpu) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 63) struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 64) struct page **pages, **oldpages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 65) void *ptr, *old_ptr;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 66)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 67) pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 68) if (!pages) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 69) ret = -ENOMEM;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 70) break;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 71) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 72)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 73) for (i = 0; i < nrpages; ++i) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 74) pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 75) if (!pages[i]) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 76) ret = -ENOMEM;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 77) oldpages = pages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 78) goto free_pagearray;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 79) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 80) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 81) ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 82) if (!ptr) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 83) ret = -ENOMEM;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 84) oldpages = pages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 85) goto free_pagearray;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 86) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 87) raw_spin_lock(&pcb->lock);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 88) old_ptr = pcb->ptr;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 89) pcb->ptr = ptr;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 90) oldpages = pcb->pages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 91) pcb->pages = pages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 92) i = pcb->nrpages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 93) pcb->nrpages = nrpages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 94) raw_spin_unlock(&pcb->lock);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 95)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 96) if (!oldpages) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 97) DBG_BUGON(old_ptr);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 98) continue;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 99) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 100)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 101) if (old_ptr)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 102) vunmap(old_ptr);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 103) free_pagearray:
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 104) while (i)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 105) list_add(&oldpages[--i]->lru, &pagepool);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 106) kfree(oldpages);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 107) if (ret)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 108) break;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 109) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 110) pcb_nrpages = nrpages;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 111) put_pages_list(&pagepool);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 112) out:
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 113) mutex_unlock(&pcb_resize_mutex);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 114) return ret;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 115) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 116)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 117) void erofs_pcpubuf_init(void)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 118) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 119) int cpu;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 120)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 121) for_each_possible_cpu(cpu) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 122) struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 123)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 124) raw_spin_lock_init(&pcb->lock);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 125) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 126) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 127)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 128) void erofs_pcpubuf_exit(void)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 129) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 130) int cpu, i;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 131)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 132) for_each_possible_cpu(cpu) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 133) struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 134)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 135) if (pcb->ptr) {
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 136) vunmap(pcb->ptr);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 137) pcb->ptr = NULL;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 138) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 139) if (!pcb->pages)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 140) continue;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 141)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 142) for (i = 0; i < pcb->nrpages; ++i)
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 143) if (pcb->pages[i])
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 144) put_page(pcb->pages[i]);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 145) kfree(pcb->pages);
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 146) pcb->pages = NULL;
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 147) }
524887347fcb6 (Gao Xiang 2021-04-10 03:06:30 +0800 148) }
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags