457c899653991 (Thomas Gleixner 2019-05-19 13:08:55 +0100 1) // SPDX-License-Identifier: GPL-2.0-only
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 2) #include <linux/module.h>
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 3) #include <linux/scatterlist.h>
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 4) #include <linux/mempool.h>
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 5) #include <linux/slab.h>
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 6)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 7) #define SG_MEMPOOL_NR ARRAY_SIZE(sg_pools)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 8) #define SG_MEMPOOL_SIZE 2
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 9)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 10) struct sg_pool {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 11) size_t size;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 12) char *name;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 13) struct kmem_cache *slab;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 14) mempool_t *pool;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 15) };
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 16)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 17) #define SP(x) { .size = x, "sgpool-" __stringify(x) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 18) #if (SG_CHUNK_SIZE < 32)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 19) #error SG_CHUNK_SIZE is too small (must be 32 or greater)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 20) #endif
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 21) static struct sg_pool sg_pools[] = {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 22) SP(8),
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 23) SP(16),
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 24) #if (SG_CHUNK_SIZE > 32)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 25) SP(32),
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 26) #if (SG_CHUNK_SIZE > 64)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 27) SP(64),
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 28) #if (SG_CHUNK_SIZE > 128)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 29) SP(128),
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 30) #if (SG_CHUNK_SIZE > 256)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 31) #error SG_CHUNK_SIZE is too large (256 MAX)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 32) #endif
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 33) #endif
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 34) #endif
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 35) #endif
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 36) SP(SG_CHUNK_SIZE)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 37) };
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 38) #undef SP
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 39)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 40) static inline unsigned int sg_pool_index(unsigned short nents)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 41) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 42) unsigned int index;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 43)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 44) BUG_ON(nents > SG_CHUNK_SIZE);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 45)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 46) if (nents <= 8)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 47) index = 0;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 48) else
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 49) index = get_count_order(nents) - 3;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 50)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 51) return index;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 52) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 53)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 54) static void sg_pool_free(struct scatterlist *sgl, unsigned int nents)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 55) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 56) struct sg_pool *sgp;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 57)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 58) sgp = sg_pools + sg_pool_index(nents);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 59) mempool_free(sgl, sgp->pool);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 60) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 61)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 62) static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 63) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 64) struct sg_pool *sgp;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 65)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 66) sgp = sg_pools + sg_pool_index(nents);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 67) return mempool_alloc(sgp->pool, gfp_mask);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 68) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 69)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 70) /**
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 71) * sg_free_table_chained - Free a previously mapped sg table
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 72) * @table: The sg table header to use
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 73) * @nents_first_chunk: size of the first_chunk SGL passed to
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 74) * sg_alloc_table_chained
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 75) *
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 76) * Description:
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 77) * Free an sg table previously allocated and setup with
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 78) * sg_alloc_table_chained().
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 79) *
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 80) * @nents_first_chunk has to be same with that same parameter passed
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 81) * to sg_alloc_table_chained().
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 82) *
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 83) **/
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 84) void sg_free_table_chained(struct sg_table *table,
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 85) unsigned nents_first_chunk)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 86) {
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 87) if (table->orig_nents <= nents_first_chunk)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 88) return;
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 89)
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 90) if (nents_first_chunk == 1)
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 91) nents_first_chunk = 0;
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 92)
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 93) __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 94) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 95) EXPORT_SYMBOL_GPL(sg_free_table_chained);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 96)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 97) /**
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 98) * sg_alloc_table_chained - Allocate and chain SGLs in an sg table
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 99) * @table: The sg table header to use
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 100) * @nents: Number of entries in sg list
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 101) * @first_chunk: first SGL
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 102) * @nents_first_chunk: number of the SGL of @first_chunk
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 103) *
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 104) * Description:
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 105) * Allocate and chain SGLs in an sg table. If @nents@ is larger than
b79d9a09ae23c (Ming Lei 2019-06-06 16:34:08 +0800 106) * @nents_first_chunk a chained sg table will be setup. @first_chunk is
b79d9a09ae23c (Ming Lei 2019-06-06 16:34:08 +0800 107) * ignored if nents_first_chunk <= 1 because user expects the SGL points
b79d9a09ae23c (Ming Lei 2019-06-06 16:34:08 +0800 108) * non-chain SGL.
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 109) *
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 110) **/
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 111) int sg_alloc_table_chained(struct sg_table *table, int nents,
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 112) struct scatterlist *first_chunk, unsigned nents_first_chunk)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 113) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 114) int ret;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 115)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 116) BUG_ON(!nents);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 117)
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 118) if (first_chunk && nents_first_chunk) {
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 119) if (nents <= nents_first_chunk) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 120) table->nents = table->orig_nents = nents;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 121) sg_init_table(table->sgl, nents);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 122) return 0;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 123) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 124) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 125)
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 126) /* User supposes that the 1st SGL includes real entry */
b79d9a09ae23c (Ming Lei 2019-06-06 16:34:08 +0800 127) if (nents_first_chunk <= 1) {
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 128) first_chunk = NULL;
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 129) nents_first_chunk = 0;
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 130) }
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 131)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 132) ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE,
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 133) first_chunk, nents_first_chunk,
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 134) GFP_ATOMIC, sg_pool_alloc);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 135) if (unlikely(ret))
4635873c561ac (Ming Lei 2019-04-28 15:39:30 +0800 136) sg_free_table_chained(table, nents_first_chunk);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 137) return ret;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 138) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 139) EXPORT_SYMBOL_GPL(sg_alloc_table_chained);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 140)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 141) static __init int sg_pool_init(void)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 142) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 143) int i;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 144)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 145) for (i = 0; i < SG_MEMPOOL_NR; i++) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 146) struct sg_pool *sgp = sg_pools + i;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 147) int size = sgp->size * sizeof(struct scatterlist);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 148)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 149) sgp->slab = kmem_cache_create(sgp->name, size, 0,
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 150) SLAB_HWCACHE_ALIGN, NULL);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 151) if (!sgp->slab) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 152) printk(KERN_ERR "SG_POOL: can't init sg slab %s\n",
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 153) sgp->name);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 154) goto cleanup_sdb;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 155) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 156)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 157) sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 158) sgp->slab);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 159) if (!sgp->pool) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 160) printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n",
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 161) sgp->name);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 162) goto cleanup_sdb;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 163) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 164) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 165)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 166) return 0;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 167)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 168) cleanup_sdb:
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 169) for (i = 0; i < SG_MEMPOOL_NR; i++) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 170) struct sg_pool *sgp = sg_pools + i;
7f476715d05f5 (zhong jiang 2018-10-30 15:05:37 -0700 171)
7f476715d05f5 (zhong jiang 2018-10-30 15:05:37 -0700 172) mempool_destroy(sgp->pool);
7f476715d05f5 (zhong jiang 2018-10-30 15:05:37 -0700 173) kmem_cache_destroy(sgp->slab);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 174) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 175)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 176) return -ENOMEM;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 177) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 178)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 179) static __exit void sg_pool_exit(void)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 180) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 181) int i;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 182)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 183) for (i = 0; i < SG_MEMPOOL_NR; i++) {
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 184) struct sg_pool *sgp = sg_pools + i;
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 185) mempool_destroy(sgp->pool);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 186) kmem_cache_destroy(sgp->slab);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 187) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 188) }
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 189)
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 190) module_init(sg_pool_init);
9b1d6c8950021 (Ming Lin 2016-04-04 14:48:11 -0700 191) module_exit(sg_pool_exit);
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags