3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 2) * Generic infrastructure for lifetime debugging of objects.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 3) *
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 4) * Started by Thomas Gleixner
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 5) *
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 6) * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de>
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 7) *
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 8) * For licencing details see kernel-base/COPYING
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 9) */
719e484396e27 (Fabian Frederick 2014-06-04 16:06:04 -0700 10)
719e484396e27 (Fabian Frederick 2014-06-04 16:06:04 -0700 11) #define pr_fmt(fmt) "ODEBUG: " fmt
719e484396e27 (Fabian Frederick 2014-06-04 16:06:04 -0700 12)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 13) #include <linux/debugobjects.h>
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 14) #include <linux/interrupt.h>
d43c36dc6b357 (Alexey Dobriyan 2009-10-07 17:09:06 +0400 15) #include <linux/sched.h>
68db0cf106786 (Ingo Molnar 2017-02-08 18:51:37 +0100 16) #include <linux/sched/task_stack.h>
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 17) #include <linux/seq_file.h>
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 18) #include <linux/debugfs.h>
5a0e3ad6af866 (Tejun Heo 2010-03-24 17:04:11 +0900 19) #include <linux/slab.h>
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 20) #include <linux/hash.h>
caba4cbbd27d7 (Waiman Long 2017-08-14 09:52:13 -0400 21) #include <linux/kmemleak.h>
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 22) #include <linux/cpu.h>
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 23)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 24) #define ODEBUG_HASH_BITS 14
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 25) #define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 26)
0b6ec8c0a3708 (Christian Borntraeger 2016-01-27 15:37:58 +0100 27) #define ODEBUG_POOL_SIZE 1024
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 28) #define ODEBUG_POOL_MIN_LEVEL 256
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 29) #define ODEBUG_POOL_PERCPU_SIZE 64
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 30) #define ODEBUG_BATCH_SIZE 16
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 31)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 32) #define ODEBUG_CHUNK_SHIFT PAGE_SHIFT
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 33) #define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 34) #define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 35)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 36) /*
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 37) * We limit the freeing of debug objects via workqueue at a maximum
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 38) * frequency of 10Hz and about 1024 objects for each freeing operation.
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 39) * So it is freeing at most 10k debug objects per second.
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 40) */
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 41) #define ODEBUG_FREE_WORK_MAX 1024
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 42) #define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 43)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 44) struct debug_bucket {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 45) struct hlist_head list;
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 46) raw_spinlock_t lock;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 47) };
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 48)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 49) /*
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 50) * Debug object percpu free list
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 51) * Access is protected by disabling irq
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 52) */
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 53) struct debug_percpu_free {
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 54) struct hlist_head free_objs;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 55) int obj_free;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 56) };
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 57)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 58) static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 59)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 60) static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 61)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 62) static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 63)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 64) static DEFINE_RAW_SPINLOCK(pool_lock);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 65)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 66) static HLIST_HEAD(obj_pool);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 67) static HLIST_HEAD(obj_to_free);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 68)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 69) /*
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 70) * Because of the presence of percpu free pools, obj_pool_free will
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 71) * under-count those in the percpu free pools. Similarly, obj_pool_used
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 72) * will over-count those in the percpu free pools. Adjustments will be
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 73) * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 74) * can be off.
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 75) */
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 76) static int obj_pool_min_free = ODEBUG_POOL_SIZE;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 77) static int obj_pool_free = ODEBUG_POOL_SIZE;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 78) static int obj_pool_used;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 79) static int obj_pool_max_used;
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 80) static bool obj_freeing;
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 81) /* The number of objs on the global free list */
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 82) static int obj_nr_tofree;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 83)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 84) static int debug_objects_maxchain __read_mostly;
163cf842f5837 (Arnd Bergmann 2018-03-13 14:18:46 +0100 85) static int __maybe_unused debug_objects_maxchecked __read_mostly;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 86) static int debug_objects_fixups __read_mostly;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 87) static int debug_objects_warnings __read_mostly;
3ae7020543db0 (Ingo Molnar 2008-11-26 10:02:00 +0100 88) static int debug_objects_enabled __read_mostly
3ae7020543db0 (Ingo Molnar 2008-11-26 10:02:00 +0100 89) = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
97dd552eb23c8 (Waiman Long 2017-01-05 15:17:04 -0500 90) static int debug_objects_pool_size __read_mostly
97dd552eb23c8 (Waiman Long 2017-01-05 15:17:04 -0500 91) = ODEBUG_POOL_SIZE;
97dd552eb23c8 (Waiman Long 2017-01-05 15:17:04 -0500 92) static int debug_objects_pool_min_level __read_mostly
97dd552eb23c8 (Waiman Long 2017-01-05 15:17:04 -0500 93) = ODEBUG_POOL_MIN_LEVEL;
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 94) static const struct debug_obj_descr *descr_test __read_mostly;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 95) static struct kmem_cache *obj_cache __read_mostly;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 96)
c4b73aabd0989 (Waiman Long 2017-01-05 15:17:03 -0500 97) /*
0cad93c3453fc (Waiman Long 2017-02-07 16:40:30 -0500 98) * Track numbers of kmem_cache_alloc()/free() calls done.
c4b73aabd0989 (Waiman Long 2017-01-05 15:17:03 -0500 99) */
0cad93c3453fc (Waiman Long 2017-02-07 16:40:30 -0500 100) static int debug_objects_allocated;
c4b73aabd0989 (Waiman Long 2017-01-05 15:17:03 -0500 101) static int debug_objects_freed;
c4b73aabd0989 (Waiman Long 2017-01-05 15:17:03 -0500 102)
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 103) static void free_obj_work(struct work_struct *work);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 104) static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 105)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 106) static int __init enable_object_debug(char *str)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 107) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 108) debug_objects_enabled = 1;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 109) return 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 110) }
3e8ebb5c433f0 (Kyle McMartin 2009-03-01 20:41:41 -0500 111)
3e8ebb5c433f0 (Kyle McMartin 2009-03-01 20:41:41 -0500 112) static int __init disable_object_debug(char *str)
3e8ebb5c433f0 (Kyle McMartin 2009-03-01 20:41:41 -0500 113) {
3e8ebb5c433f0 (Kyle McMartin 2009-03-01 20:41:41 -0500 114) debug_objects_enabled = 0;
3e8ebb5c433f0 (Kyle McMartin 2009-03-01 20:41:41 -0500 115) return 0;
3e8ebb5c433f0 (Kyle McMartin 2009-03-01 20:41:41 -0500 116) }
3e8ebb5c433f0 (Kyle McMartin 2009-03-01 20:41:41 -0500 117)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 118) early_param("debug_objects", enable_object_debug);
3e8ebb5c433f0 (Kyle McMartin 2009-03-01 20:41:41 -0500 119) early_param("no_debug_objects", disable_object_debug);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 120)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 121) static const char *obj_states[ODEBUG_STATE_MAX] = {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 122) [ODEBUG_STATE_NONE] = "none",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 123) [ODEBUG_STATE_INIT] = "initialized",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 124) [ODEBUG_STATE_INACTIVE] = "inactive",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 125) [ODEBUG_STATE_ACTIVE] = "active",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 126) [ODEBUG_STATE_DESTROYED] = "destroyed",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 127) [ODEBUG_STATE_NOTAVAILABLE] = "not available",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 128) };
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 129)
1fda107d44de1 (Thomas Gleixner 2012-04-11 11:52:18 +0200 130) static void fill_pool(void)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 131) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 132) gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 133) struct debug_obj *obj;
50db04dd9c741 (Vegard Nossum 2008-06-15 00:47:36 +0200 134) unsigned long flags;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 135)
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 136) if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
1fda107d44de1 (Thomas Gleixner 2012-04-11 11:52:18 +0200 137) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 138)
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 139) /*
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 140) * Reuse objs from the global free list; they will be reinitialized
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 141) * when allocating.
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 142) *
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 143) * Both obj_nr_tofree and obj_pool_free are checked locklessly; the
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 144) * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 145) * sections.
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 146) */
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 147) while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) {
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 148) raw_spin_lock_irqsave(&pool_lock, flags);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 149) /*
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 150) * Recheck with the lock held as the worker thread might have
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 151) * won the race and freed the global free list already.
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 152) */
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 153) while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 154) obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 155) hlist_del(&obj->node);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 156) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 157) hlist_add_head(&obj->node, &obj_pool);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 158) WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 159) }
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 160) raw_spin_unlock_irqrestore(&pool_lock, flags);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 161) }
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 162)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 163) if (unlikely(!obj_cache))
1fda107d44de1 (Thomas Gleixner 2012-04-11 11:52:18 +0200 164) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 165)
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 166) while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 167) struct debug_obj *new[ODEBUG_BATCH_SIZE];
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 168) int cnt;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 169)
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 170) for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 171) new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 172) if (!new[cnt])
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 173) break;
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 174) }
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 175) if (!cnt)
3340808cf04fa (Dan Carpenter 2012-04-18 14:28:10 +0300 176) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 177)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 178) raw_spin_lock_irqsave(&pool_lock, flags);
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 179) while (cnt) {
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 180) hlist_add_head(&new[--cnt]->node, &obj_pool);
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 181) debug_objects_allocated++;
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 182) WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
d26bf5056fc08 (Waiman Long 2019-05-20 10:14:48 -0400 183) }
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 184) raw_spin_unlock_irqrestore(&pool_lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 185) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 186) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 187)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 188) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 189) * Lookup an object in the hash bucket.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 190) */
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 191) static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 192) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 193) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 194) int cnt = 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 195)
b67bfe0d42cac (Sasha Levin 2013-02-27 17:06:00 -0800 196) hlist_for_each_entry(obj, &b->list, node) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 197) cnt++;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 198) if (obj->object == addr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 199) return obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 200) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 201) if (cnt > debug_objects_maxchain)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 202) debug_objects_maxchain = cnt;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 203)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 204) return NULL;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 205) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 206)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 207) /*
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 208) * Allocate a new object from the hlist
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 209) */
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 210) static struct debug_obj *__alloc_object(struct hlist_head *list)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 211) {
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 212) struct debug_obj *obj = NULL;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 213)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 214) if (list->first) {
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 215) obj = hlist_entry(list->first, typeof(*obj), node);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 216) hlist_del(&obj->node);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 217) }
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 218)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 219) return obj;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 220) }
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 221)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 222) /*
50db04dd9c741 (Vegard Nossum 2008-06-15 00:47:36 +0200 223) * Allocate a new object. If the pool is empty, switch off the debugger.
673d62cc5ea6f (Vegard Nossum 2008-08-31 23:39:21 +0200 224) * Must be called with interrupts disabled.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 225) */
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 226) static struct debug_obj *
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 227) alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 228) {
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 229) struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 230) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 231)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 232) if (likely(obj_cache)) {
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 233) obj = __alloc_object(&percpu_pool->free_objs);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 234) if (obj) {
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 235) percpu_pool->obj_free--;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 236) goto init_obj;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 237) }
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 238) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 239)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 240) raw_spin_lock(&pool_lock);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 241) obj = __alloc_object(&obj_pool);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 242) if (obj) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 243) obj_pool_used++;
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 244) WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 245)
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 246) /*
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 247) * Looking ahead, allocate one batch of debug objects and
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 248) * put them into the percpu free pool.
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 249) */
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 250) if (likely(obj_cache)) {
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 251) int i;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 252)
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 253) for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 254) struct debug_obj *obj2;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 255)
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 256) obj2 = __alloc_object(&obj_pool);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 257) if (!obj2)
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 258) break;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 259) hlist_add_head(&obj2->node,
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 260) &percpu_pool->free_objs);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 261) percpu_pool->obj_free++;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 262) obj_pool_used++;
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 263) WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 264) }
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 265) }
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 266)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 267) if (obj_pool_used > obj_pool_max_used)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 268) obj_pool_max_used = obj_pool_used;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 269)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 270) if (obj_pool_free < obj_pool_min_free)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 271) obj_pool_min_free = obj_pool_free;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 272) }
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 273) raw_spin_unlock(&pool_lock);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 274)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 275) init_obj:
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 276) if (obj) {
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 277) obj->object = addr;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 278) obj->descr = descr;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 279) obj->state = ODEBUG_STATE_NONE;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 280) obj->astate = 0;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 281) hlist_add_head(&obj->node, &b->list);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 282) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 283) return obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 284) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 285)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 286) /*
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 287) * workqueue function to free objects.
858274b6a13b4 (Waiman Long 2017-01-05 15:17:05 -0500 288) *
858274b6a13b4 (Waiman Long 2017-01-05 15:17:05 -0500 289) * To reduce contention on the global pool_lock, the actual freeing of
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 290) * debug objects will be delayed if the pool_lock is busy.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 291) */
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 292) static void free_obj_work(struct work_struct *work)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 293) {
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 294) struct hlist_node *tmp;
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 295) struct debug_obj *obj;
673d62cc5ea6f (Vegard Nossum 2008-08-31 23:39:21 +0200 296) unsigned long flags;
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 297) HLIST_HEAD(tofree);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 298)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 299) WRITE_ONCE(obj_freeing, false);
858274b6a13b4 (Waiman Long 2017-01-05 15:17:05 -0500 300) if (!raw_spin_trylock_irqsave(&pool_lock, flags))
858274b6a13b4 (Waiman Long 2017-01-05 15:17:05 -0500 301) return;
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 302)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 303) if (obj_pool_free >= debug_objects_pool_size)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 304) goto free_objs;
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 305)
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 306) /*
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 307) * The objs on the pool list might be allocated before the work is
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 308) * run, so recheck if pool list it full or not, if not fill pool
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 309) * list from the global free list. As it is likely that a workload
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 310) * may be gearing up to use more and more objects, don't free any
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 311) * of them until the next round.
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 312) */
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 313) while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 314) obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 315) hlist_del(&obj->node);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 316) hlist_add_head(&obj->node, &obj_pool);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 317) WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 318) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 319) }
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 320) raw_spin_unlock_irqrestore(&pool_lock, flags);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 321) return;
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 322)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 323) free_objs:
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 324) /*
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 325) * Pool list is already full and there are still objs on the free
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 326) * list. Move remaining free objs to a temporary list to free the
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 327) * memory outside the pool_lock held region.
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 328) */
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 329) if (obj_nr_tofree) {
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 330) hlist_move_list(&obj_to_free, &tofree);
04148187aa9df (Arnd Bergmann 2018-02-22 16:52:58 +0100 331) debug_objects_freed += obj_nr_tofree;
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 332) WRITE_ONCE(obj_nr_tofree, 0);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 333) }
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 334) raw_spin_unlock_irqrestore(&pool_lock, flags);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 335)
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 336) hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 337) hlist_del(&obj->node);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 338) kmem_cache_free(obj_cache, obj);
36c4ead6f6dfb (Yang Shi 2018-02-06 07:18:26 +0800 339) }
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 340) }
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 341)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 342) static void __free_object(struct debug_obj *obj)
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 343) {
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 344) struct debug_obj *objs[ODEBUG_BATCH_SIZE];
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 345) struct debug_percpu_free *percpu_pool;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 346) int lookahead_count = 0;
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 347) unsigned long flags;
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 348) bool work;
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 349)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 350) local_irq_save(flags);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 351) if (!obj_cache)
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 352) goto free_to_obj_pool;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 353)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 354) /*
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 355) * Try to free it into the percpu pool first.
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 356) */
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 357) percpu_pool = this_cpu_ptr(&percpu_obj_pool);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 358) if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 359) hlist_add_head(&obj->node, &percpu_pool->free_objs);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 360) percpu_pool->obj_free++;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 361) local_irq_restore(flags);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 362) return;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 363) }
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 364)
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 365) /*
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 366) * As the percpu pool is full, look ahead and pull out a batch
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 367) * of objects from the percpu pool and free them as well.
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 368) */
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 369) for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 370) objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 371) if (!objs[lookahead_count])
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 372) break;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 373) percpu_pool->obj_free--;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 374) }
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 375)
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 376) free_to_obj_pool:
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 377) raw_spin_lock(&pool_lock);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 378) work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 379) (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
337fff8b5ed05 (Thomas Gleixner 2009-03-16 10:04:53 +0100 380) obj_pool_used--;
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 381)
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 382) if (work) {
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 383) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 384) hlist_add_head(&obj->node, &obj_to_free);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 385) if (lookahead_count) {
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 386) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 387) obj_pool_used -= lookahead_count;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 388) while (lookahead_count) {
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 389) hlist_add_head(&objs[--lookahead_count]->node,
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 390) &obj_to_free);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 391) }
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 392) }
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 393)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 394) if ((obj_pool_free > debug_objects_pool_size) &&
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 395) (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 396) int i;
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 397)
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 398) /*
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 399) * Free one more batch of objects from obj_pool.
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 400) */
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 401) for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 402) obj = __alloc_object(&obj_pool);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 403) hlist_add_head(&obj->node, &obj_to_free);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 404) WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 405) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 406) }
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 407) }
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 408) } else {
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 409) WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 410) hlist_add_head(&obj->node, &obj_pool);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 411) if (lookahead_count) {
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 412) WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 413) obj_pool_used -= lookahead_count;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 414) while (lookahead_count) {
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 415) hlist_add_head(&objs[--lookahead_count]->node,
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 416) &obj_pool);
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 417) }
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 418) }
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 419) }
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 420) raw_spin_unlock(&pool_lock);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 421) local_irq_restore(flags);
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 422) }
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 423)
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 424) /*
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 425) * Put the object back into the pool and schedule work to free objects
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 426) * if necessary.
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 427) */
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 428) static void free_object(struct debug_obj *obj)
636e1970fd7de (Yang Shi 2018-02-06 07:18:27 +0800 429) {
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 430) __free_object(obj);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 431) if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 432) WRITE_ONCE(obj_freeing, true);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 433) schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 434) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 435) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 436)
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 437) #ifdef CONFIG_HOTPLUG_CPU
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 438) static int object_cpu_offline(unsigned int cpu)
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 439) {
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 440) struct debug_percpu_free *percpu_pool;
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 441) struct hlist_node *tmp;
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 442) struct debug_obj *obj;
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 443)
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 444) /* Remote access is safe as the CPU is dead already */
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 445) percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu);
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 446) hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) {
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 447) hlist_del(&obj->node);
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 448) kmem_cache_free(obj_cache, obj);
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 449) }
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 450) percpu_pool->obj_free = 0;
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 451)
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 452) return 0;
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 453) }
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 454) #endif
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 455)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 456) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 457) * We run out of memory. That means we probably have tons of objects
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 458) * allocated.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 459) */
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 460) static void debug_objects_oom(void)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 461) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 462) struct debug_bucket *db = obj_hash;
b67bfe0d42cac (Sasha Levin 2013-02-27 17:06:00 -0800 463) struct hlist_node *tmp;
673d62cc5ea6f (Vegard Nossum 2008-08-31 23:39:21 +0200 464) HLIST_HEAD(freelist);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 465) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 466) unsigned long flags;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 467) int i;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 468)
719e484396e27 (Fabian Frederick 2014-06-04 16:06:04 -0700 469) pr_warn("Out of memory. ODEBUG disabled\n");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 470)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 471) for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 472) raw_spin_lock_irqsave(&db->lock, flags);
673d62cc5ea6f (Vegard Nossum 2008-08-31 23:39:21 +0200 473) hlist_move_list(&db->list, &freelist);
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 474) raw_spin_unlock_irqrestore(&db->lock, flags);
673d62cc5ea6f (Vegard Nossum 2008-08-31 23:39:21 +0200 475)
673d62cc5ea6f (Vegard Nossum 2008-08-31 23:39:21 +0200 476) /* Now free them */
b67bfe0d42cac (Sasha Levin 2013-02-27 17:06:00 -0800 477) hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 478) hlist_del(&obj->node);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 479) free_object(obj);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 480) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 481) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 482) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 483)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 484) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 485) * We use the pfn of the address for the hash. That way we can check
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 486) * for freed objects simply by checking the affected bucket.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 487) */
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 488) static struct debug_bucket *get_bucket(unsigned long addr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 489) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 490) unsigned long hash;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 491)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 492) hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 493) return &obj_hash[hash];
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 494) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 495)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 496) static void debug_print_object(struct debug_obj *obj, char *msg)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 497) {
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 498) const struct debug_obj_descr *descr = obj->descr;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 499) static int limit;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 500)
997772884036e (Stanislaw Gruszka 2011-03-07 09:58:33 +0100 501) if (limit < 5 && descr != descr_test) {
997772884036e (Stanislaw Gruszka 2011-03-07 09:58:33 +0100 502) void *hint = descr->debug_hint ?
997772884036e (Stanislaw Gruszka 2011-03-07 09:58:33 +0100 503) descr->debug_hint(obj->object) : NULL;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 504) limit++;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 505) WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
997772884036e (Stanislaw Gruszka 2011-03-07 09:58:33 +0100 506) "object type: %s hint: %pS\n",
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 507) msg, obj_states[obj->state], obj->astate,
997772884036e (Stanislaw Gruszka 2011-03-07 09:58:33 +0100 508) descr->name, hint);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 509) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 510) debug_objects_warnings++;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 511) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 512)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 513) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 514) * Try to repair the damage, so we have a better chance to get useful
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 515) * debug output.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 516) */
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 517) static bool
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 518) debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 519) void * addr, enum debug_obj_state state)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 520) {
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 521) if (fixup && fixup(addr, state)) {
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 522) debug_objects_fixups++;
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 523) return true;
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 524) }
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 525) return false;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 526) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 527)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 528) static void debug_object_is_on_stack(void *addr, int onstack)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 529) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 530) int is_on_stack;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 531) static int limit;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 532)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 533) if (limit > 4)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 534) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 535)
8b05c7e6e159d (FUJITA Tomonori 2008-07-23 21:26:53 -0700 536) is_on_stack = object_is_on_stack(addr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 537) if (is_on_stack == onstack)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 538) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 539)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 540) limit++;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 541) if (is_on_stack)
fc91a3c4c27ac (Joel Fernandes (Google) 2018-07-23 14:25:31 -0700 542) pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,
fc91a3c4c27ac (Joel Fernandes (Google) 2018-07-23 14:25:31 -0700 543) task_stack_page(current));
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 544) else
fc91a3c4c27ac (Joel Fernandes (Google) 2018-07-23 14:25:31 -0700 545) pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,
fc91a3c4c27ac (Joel Fernandes (Google) 2018-07-23 14:25:31 -0700 546) task_stack_page(current));
fc91a3c4c27ac (Joel Fernandes (Google) 2018-07-23 14:25:31 -0700 547)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 548) WARN_ON(1);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 549) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 550)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 551) static void
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 552) __debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 553) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 554) enum debug_obj_state state;
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 555) bool check_stack = false;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 556) struct debug_bucket *db;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 557) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 558) unsigned long flags;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 559)
50db04dd9c741 (Vegard Nossum 2008-06-15 00:47:36 +0200 560) fill_pool();
50db04dd9c741 (Vegard Nossum 2008-06-15 00:47:36 +0200 561)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 562) db = get_bucket((unsigned long) addr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 563)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 564) raw_spin_lock_irqsave(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 565)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 566) obj = lookup_object(addr, db);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 567) if (!obj) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 568) obj = alloc_object(addr, db, descr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 569) if (!obj) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 570) debug_objects_enabled = 0;
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 571) raw_spin_unlock_irqrestore(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 572) debug_objects_oom();
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 573) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 574) }
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 575) check_stack = true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 576) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 577)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 578) switch (obj->state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 579) case ODEBUG_STATE_NONE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 580) case ODEBUG_STATE_INIT:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 581) case ODEBUG_STATE_INACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 582) obj->state = ODEBUG_STATE_INIT;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 583) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 584)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 585) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 586) state = obj->state;
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 587) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 588) debug_print_object(obj, "init");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 589) debug_object_fixup(descr->fixup_init, addr, state);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 590) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 591)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 592) case ODEBUG_STATE_DESTROYED:
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 593) raw_spin_unlock_irqrestore(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 594) debug_print_object(obj, "init");
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 595) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 596) default:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 597) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 598) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 599)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 600) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 601) if (check_stack)
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 602) debug_object_is_on_stack(addr, onstack);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 603) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 604)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 605) /**
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 606) * debug_object_init - debug checks when an object is initialized
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 607) * @addr: address of the object
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 608) * @descr: pointer to an object specific debug description structure
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 609) */
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 610) void debug_object_init(void *addr, const struct debug_obj_descr *descr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 611) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 612) if (!debug_objects_enabled)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 613) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 614)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 615) __debug_object_init(addr, descr, 0);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 616) }
f8ff04e2be081 (Chris Wilson 2016-11-30 15:54:10 -0800 617) EXPORT_SYMBOL_GPL(debug_object_init);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 618)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 619) /**
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 620) * debug_object_init_on_stack - debug checks when an object on stack is
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 621) * initialized
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 622) * @addr: address of the object
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 623) * @descr: pointer to an object specific debug description structure
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 624) */
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 625) void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 626) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 627) if (!debug_objects_enabled)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 628) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 629)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 630) __debug_object_init(addr, descr, 1);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 631) }
f8ff04e2be081 (Chris Wilson 2016-11-30 15:54:10 -0800 632) EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 633)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 634) /**
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 635) * debug_object_activate - debug checks when an object is activated
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 636) * @addr: address of the object
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 637) * @descr: pointer to an object specific debug description structure
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 638) * Returns 0 for success, -EINVAL for check failed.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 639) */
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 640) int debug_object_activate(void *addr, const struct debug_obj_descr *descr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 641) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 642) enum debug_obj_state state;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 643) struct debug_bucket *db;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 644) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 645) unsigned long flags;
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 646) int ret;
feac18dda2513 (Stephen Boyd 2011-11-07 19:48:26 -0800 647) struct debug_obj o = { .object = addr,
feac18dda2513 (Stephen Boyd 2011-11-07 19:48:26 -0800 648) .state = ODEBUG_STATE_NOTAVAILABLE,
feac18dda2513 (Stephen Boyd 2011-11-07 19:48:26 -0800 649) .descr = descr };
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 650)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 651) if (!debug_objects_enabled)
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 652) return 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 653)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 654) db = get_bucket((unsigned long) addr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 655)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 656) raw_spin_lock_irqsave(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 657)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 658) obj = lookup_object(addr, db);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 659) if (obj) {
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 660) bool print_object = false;
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 661)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 662) switch (obj->state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 663) case ODEBUG_STATE_INIT:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 664) case ODEBUG_STATE_INACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 665) obj->state = ODEBUG_STATE_ACTIVE;
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 666) ret = 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 667) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 668)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 669) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 670) state = obj->state;
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 671) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 672) debug_print_object(obj, "activate");
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 673) ret = debug_object_fixup(descr->fixup_activate, addr, state);
e7a8e78bd4ad9 (Changbin Du 2016-05-19 17:09:23 -0700 674) return ret ? 0 : -EINVAL;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 675)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 676) case ODEBUG_STATE_DESTROYED:
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 677) print_object = true;
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 678) ret = -EINVAL;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 679) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 680) default:
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 681) ret = 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 682) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 683) }
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 684) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 685) if (print_object)
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 686) debug_print_object(obj, "activate");
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 687) return ret;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 688) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 689)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 690) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 691)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 692) /*
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 693) * We are here when a static object is activated. We
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 694) * let the type specific code confirm whether this is
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 695) * true or not. if true, we just make sure that the
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 696) * static object is tracked in the object tracker. If
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 697) * not, this must be a bug, so we try to fix it up.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 698) */
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 699) if (descr->is_static_object && descr->is_static_object(addr)) {
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 700) /* track this static object */
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 701) debug_object_init(addr, descr);
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 702) debug_object_activate(addr, descr);
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 703) } else {
feac18dda2513 (Stephen Boyd 2011-11-07 19:48:26 -0800 704) debug_print_object(&o, "activate");
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 705) ret = debug_object_fixup(descr->fixup_activate, addr,
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 706) ODEBUG_STATE_NOTAVAILABLE);
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 707) return ret ? 0 : -EINVAL;
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 708) }
b778ae25366e6 (Paul E. McKenney 2013-04-23 12:51:11 -0700 709) return 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 710) }
f8ff04e2be081 (Chris Wilson 2016-11-30 15:54:10 -0800 711) EXPORT_SYMBOL_GPL(debug_object_activate);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 712)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 713) /**
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 714) * debug_object_deactivate - debug checks when an object is deactivated
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 715) * @addr: address of the object
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 716) * @descr: pointer to an object specific debug description structure
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 717) */
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 718) void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 719) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 720) struct debug_bucket *db;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 721) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 722) unsigned long flags;
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 723) bool print_object = false;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 724)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 725) if (!debug_objects_enabled)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 726) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 727)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 728) db = get_bucket((unsigned long) addr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 729)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 730) raw_spin_lock_irqsave(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 731)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 732) obj = lookup_object(addr, db);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 733) if (obj) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 734) switch (obj->state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 735) case ODEBUG_STATE_INIT:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 736) case ODEBUG_STATE_INACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 737) case ODEBUG_STATE_ACTIVE:
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 738) if (!obj->astate)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 739) obj->state = ODEBUG_STATE_INACTIVE;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 740) else
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 741) print_object = true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 742) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 743)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 744) case ODEBUG_STATE_DESTROYED:
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 745) print_object = true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 746) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 747) default:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 748) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 749) }
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 750) }
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 751)
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 752) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 753) if (!obj) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 754) struct debug_obj o = { .object = addr,
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 755) .state = ODEBUG_STATE_NOTAVAILABLE,
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 756) .descr = descr };
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 757)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 758) debug_print_object(&o, "deactivate");
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 759) } else if (print_object) {
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 760) debug_print_object(obj, "deactivate");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 761) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 762) }
f8ff04e2be081 (Chris Wilson 2016-11-30 15:54:10 -0800 763) EXPORT_SYMBOL_GPL(debug_object_deactivate);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 764)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 765) /**
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 766) * debug_object_destroy - debug checks when an object is destroyed
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 767) * @addr: address of the object
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 768) * @descr: pointer to an object specific debug description structure
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 769) */
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 770) void debug_object_destroy(void *addr, const struct debug_obj_descr *descr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 771) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 772) enum debug_obj_state state;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 773) struct debug_bucket *db;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 774) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 775) unsigned long flags;
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 776) bool print_object = false;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 777)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 778) if (!debug_objects_enabled)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 779) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 780)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 781) db = get_bucket((unsigned long) addr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 782)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 783) raw_spin_lock_irqsave(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 784)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 785) obj = lookup_object(addr, db);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 786) if (!obj)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 787) goto out_unlock;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 788)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 789) switch (obj->state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 790) case ODEBUG_STATE_NONE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 791) case ODEBUG_STATE_INIT:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 792) case ODEBUG_STATE_INACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 793) obj->state = ODEBUG_STATE_DESTROYED;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 794) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 795) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 796) state = obj->state;
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 797) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 798) debug_print_object(obj, "destroy");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 799) debug_object_fixup(descr->fixup_destroy, addr, state);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 800) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 801)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 802) case ODEBUG_STATE_DESTROYED:
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 803) print_object = true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 804) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 805) default:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 806) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 807) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 808) out_unlock:
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 809) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 810) if (print_object)
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 811) debug_print_object(obj, "destroy");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 812) }
f8ff04e2be081 (Chris Wilson 2016-11-30 15:54:10 -0800 813) EXPORT_SYMBOL_GPL(debug_object_destroy);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 814)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 815) /**
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 816) * debug_object_free - debug checks when an object is freed
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 817) * @addr: address of the object
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 818) * @descr: pointer to an object specific debug description structure
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 819) */
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 820) void debug_object_free(void *addr, const struct debug_obj_descr *descr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 821) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 822) enum debug_obj_state state;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 823) struct debug_bucket *db;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 824) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 825) unsigned long flags;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 826)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 827) if (!debug_objects_enabled)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 828) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 829)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 830) db = get_bucket((unsigned long) addr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 831)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 832) raw_spin_lock_irqsave(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 833)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 834) obj = lookup_object(addr, db);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 835) if (!obj)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 836) goto out_unlock;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 837)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 838) switch (obj->state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 839) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 840) state = obj->state;
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 841) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 842) debug_print_object(obj, "free");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 843) debug_object_fixup(descr->fixup_free, addr, state);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 844) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 845) default:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 846) hlist_del(&obj->node);
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 847) raw_spin_unlock_irqrestore(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 848) free_object(obj);
673d62cc5ea6f (Vegard Nossum 2008-08-31 23:39:21 +0200 849) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 850) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 851) out_unlock:
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 852) raw_spin_unlock_irqrestore(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 853) }
f8ff04e2be081 (Chris Wilson 2016-11-30 15:54:10 -0800 854) EXPORT_SYMBOL_GPL(debug_object_free);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 855)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 856) /**
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 857) * debug_object_assert_init - debug checks when object should be init-ed
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 858) * @addr: address of the object
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 859) * @descr: pointer to an object specific debug description structure
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 860) */
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 861) void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 862) {
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 863) struct debug_bucket *db;
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 864) struct debug_obj *obj;
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 865) unsigned long flags;
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 866)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 867) if (!debug_objects_enabled)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 868) return;
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 869)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 870) db = get_bucket((unsigned long) addr);
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 871)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 872) raw_spin_lock_irqsave(&db->lock, flags);
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 873)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 874) obj = lookup_object(addr, db);
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 875) if (!obj) {
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 876) struct debug_obj o = { .object = addr,
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 877) .state = ODEBUG_STATE_NOTAVAILABLE,
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 878) .descr = descr };
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 879)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 880) raw_spin_unlock_irqrestore(&db->lock, flags);
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 881) /*
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 882) * Maybe the object is static, and we let the type specific
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 883) * code confirm. Track this static object if true, else invoke
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 884) * fixup.
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 885) */
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 886) if (descr->is_static_object && descr->is_static_object(addr)) {
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 887) /* Track this static object */
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 888) debug_object_init(addr, descr);
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 889) } else {
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 890) debug_print_object(&o, "assert_init");
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 891) debug_object_fixup(descr->fixup_assert_init, addr,
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 892) ODEBUG_STATE_NOTAVAILABLE);
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 893) }
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 894) return;
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 895) }
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 896)
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 897) raw_spin_unlock_irqrestore(&db->lock, flags);
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 898) }
f8ff04e2be081 (Chris Wilson 2016-11-30 15:54:10 -0800 899) EXPORT_SYMBOL_GPL(debug_object_assert_init);
b84d435cc228e (Christine Chan 2011-11-07 19:48:27 -0800 900)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 901) /**
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 902) * debug_object_active_state - debug checks object usage state machine
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 903) * @addr: address of the object
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 904) * @descr: pointer to an object specific debug description structure
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 905) * @expect: expected state
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 906) * @next: state to move to if expected state is found
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 907) */
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 908) void
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 909) debug_object_active_state(void *addr, const struct debug_obj_descr *descr,
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 910) unsigned int expect, unsigned int next)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 911) {
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 912) struct debug_bucket *db;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 913) struct debug_obj *obj;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 914) unsigned long flags;
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 915) bool print_object = false;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 916)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 917) if (!debug_objects_enabled)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 918) return;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 919)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 920) db = get_bucket((unsigned long) addr);
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 921)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 922) raw_spin_lock_irqsave(&db->lock, flags);
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 923)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 924) obj = lookup_object(addr, db);
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 925) if (obj) {
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 926) switch (obj->state) {
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 927) case ODEBUG_STATE_ACTIVE:
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 928) if (obj->astate == expect)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 929) obj->astate = next;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 930) else
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 931) print_object = true;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 932) break;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 933)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 934) default:
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 935) print_object = true;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 936) break;
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 937) }
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 938) }
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 939)
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 940) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 941) if (!obj) {
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 942) struct debug_obj o = { .object = addr,
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 943) .state = ODEBUG_STATE_NOTAVAILABLE,
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 944) .descr = descr };
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 945)
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 946) debug_print_object(&o, "active_state");
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 947) } else if (print_object) {
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 948) debug_print_object(obj, "active_state");
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 949) }
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 950) }
f8ff04e2be081 (Chris Wilson 2016-11-30 15:54:10 -0800 951) EXPORT_SYMBOL_GPL(debug_object_active_state);
a5d8e467f83f6 (Mathieu Desnoyers 2010-04-17 08:48:38 -0400 952)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 953) #ifdef CONFIG_DEBUG_OBJECTS_FREE
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 954) static void __debug_check_no_obj_freed(const void *address, unsigned long size)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 955) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 956) unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 957) const struct debug_obj_descr *descr;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 958) enum debug_obj_state state;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 959) struct debug_bucket *db;
1ea9b98b007a6 (Yang Shi 2018-02-06 07:18:28 +0800 960) struct hlist_node *tmp;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 961) struct debug_obj *obj;
bd9dcd046509c (Yang Shi 2018-02-06 07:18:25 +0800 962) int cnt, objs_checked = 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 963)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 964) saddr = (unsigned long) address;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 965) eaddr = saddr + size;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 966) paddr = saddr & ODEBUG_CHUNK_MASK;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 967) chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 968) chunks >>= ODEBUG_CHUNK_SHIFT;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 969)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 970) for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 971) db = get_bucket(paddr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 972)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 973) repeat:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 974) cnt = 0;
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 975) raw_spin_lock_irqsave(&db->lock, flags);
b67bfe0d42cac (Sasha Levin 2013-02-27 17:06:00 -0800 976) hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 977) cnt++;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 978) oaddr = (unsigned long) obj->object;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 979) if (oaddr < saddr || oaddr >= eaddr)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 980) continue;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 981)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 982) switch (obj->state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 983) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 984) descr = obj->descr;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 985) state = obj->state;
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 986) raw_spin_unlock_irqrestore(&db->lock, flags);
d5f34153e5269 (Waiman Long 2019-05-20 10:14:50 -0400 987) debug_print_object(obj, "free");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 988) debug_object_fixup(descr->fixup_free,
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 989) (void *) oaddr, state);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 990) goto repeat;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 991) default:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 992) hlist_del(&obj->node);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 993) __free_object(obj);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 994) break;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 995) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 996) }
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 997) raw_spin_unlock_irqrestore(&db->lock, flags);
673d62cc5ea6f (Vegard Nossum 2008-08-31 23:39:21 +0200 998)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 999) if (cnt > debug_objects_maxchain)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1000) debug_objects_maxchain = cnt;
bd9dcd046509c (Yang Shi 2018-02-06 07:18:25 +0800 1001)
bd9dcd046509c (Yang Shi 2018-02-06 07:18:25 +0800 1002) objs_checked += cnt;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1003) }
bd9dcd046509c (Yang Shi 2018-02-06 07:18:25 +0800 1004)
bd9dcd046509c (Yang Shi 2018-02-06 07:18:25 +0800 1005) if (objs_checked > debug_objects_maxchecked)
bd9dcd046509c (Yang Shi 2018-02-06 07:18:25 +0800 1006) debug_objects_maxchecked = objs_checked;
1ea9b98b007a6 (Yang Shi 2018-02-06 07:18:28 +0800 1007)
1ea9b98b007a6 (Yang Shi 2018-02-06 07:18:28 +0800 1008) /* Schedule work to actually kmem_cache_free() objects */
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 1009) if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 1010) WRITE_ONCE(obj_freeing, true);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 1011) schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
a7344a68a79ab (Waiman Long 2019-05-20 10:14:49 -0400 1012) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1013) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1014)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1015) void debug_check_no_obj_freed(const void *address, unsigned long size)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1016) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1017) if (debug_objects_enabled)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1018) __debug_check_no_obj_freed(address, size);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1019) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1020) #endif
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1021)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1022) #ifdef CONFIG_DEBUG_FS
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1023)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1024) static int debug_stats_show(struct seq_file *m, void *v)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1025) {
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1026) int cpu, obj_percpu_free = 0;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1027)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1028) for_each_possible_cpu(cpu)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1029) obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1030)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1031) seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);
bd9dcd046509c (Yang Shi 2018-02-06 07:18:25 +0800 1032) seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1033) seq_printf(m, "warnings :%d\n", debug_objects_warnings);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1034) seq_printf(m, "fixups :%d\n", debug_objects_fixups);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 1035) seq_printf(m, "pool_free :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1036) seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1037) seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1038) seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1039) seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
35fd7a637c42b (Marco Elver 2020-01-16 19:55:29 +0100 1040) seq_printf(m, "on_free_list :%d\n", READ_ONCE(obj_nr_tofree));
0cad93c3453fc (Waiman Long 2017-02-07 16:40:30 -0500 1041) seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
0cad93c3453fc (Waiman Long 2017-02-07 16:40:30 -0500 1042) seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1043) return 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1044) }
0f85c48051847 (Qinglang Miao 2020-07-16 16:47:47 +0800 1045) DEFINE_SHOW_ATTRIBUTE(debug_stats);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1046)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1047) static int __init debug_objects_init_debugfs(void)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1048) {
fecb0d95cdf75 (Greg Kroah-Hartman 2019-06-12 17:35:13 +0200 1049) struct dentry *dbgdir;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1050)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1051) if (!debug_objects_enabled)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1052) return 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1053)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1054) dbgdir = debugfs_create_dir("debug_objects", NULL);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1055)
fecb0d95cdf75 (Greg Kroah-Hartman 2019-06-12 17:35:13 +0200 1056) debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1057)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1058) return 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1059) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1060) __initcall(debug_objects_init_debugfs);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1061)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1062) #else
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1063) static inline void debug_objects_init_debugfs(void) { }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1064) #endif
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1065)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1066) #ifdef CONFIG_DEBUG_OBJECTS_SELFTEST
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1067)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1068) /* Random data structure for the self test */
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1069) struct self_test {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1070) unsigned long dummy1[6];
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1071) int static_init;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1072) unsigned long dummy2[3];
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1073) };
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1074)
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 1075) static __initconst const struct debug_obj_descr descr_type_test;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1076)
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1077) static bool __init is_static_object(void *addr)
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1078) {
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1079) struct self_test *obj = addr;
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1080)
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1081) return obj->static_init;
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1082) }
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1083)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1084) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1085) * fixup_init is called when:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1086) * - an active object is initialized
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1087) */
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1088) static bool __init fixup_init(void *addr, enum debug_obj_state state)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1089) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1090) struct self_test *obj = addr;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1091)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1092) switch (state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1093) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1094) debug_object_deactivate(obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1095) debug_object_init(obj, &descr_type_test);
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1096) return true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1097) default:
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1098) return false;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1099) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1100) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1101)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1102) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1103) * fixup_activate is called when:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1104) * - an active object is activated
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1105) * - an unknown non-static object is activated
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1106) */
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1107) static bool __init fixup_activate(void *addr, enum debug_obj_state state)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1108) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1109) struct self_test *obj = addr;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1110)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1111) switch (state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1112) case ODEBUG_STATE_NOTAVAILABLE:
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1113) return true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1114) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1115) debug_object_deactivate(obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1116) debug_object_activate(obj, &descr_type_test);
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1117) return true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1118)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1119) default:
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1120) return false;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1121) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1122) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1123)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1124) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1125) * fixup_destroy is called when:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1126) * - an active object is destroyed
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1127) */
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1128) static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1129) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1130) struct self_test *obj = addr;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1131)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1132) switch (state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1133) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1134) debug_object_deactivate(obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1135) debug_object_destroy(obj, &descr_type_test);
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1136) return true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1137) default:
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1138) return false;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1139) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1140) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1141)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1142) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1143) * fixup_free is called when:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1144) * - an active object is freed
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1145) */
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1146) static bool __init fixup_free(void *addr, enum debug_obj_state state)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1147) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1148) struct self_test *obj = addr;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1149)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1150) switch (state) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1151) case ODEBUG_STATE_ACTIVE:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1152) debug_object_deactivate(obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1153) debug_object_free(obj, &descr_type_test);
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1154) return true;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1155) default:
b1e4d9d82df8a (Changbin Du 2016-05-19 17:09:20 -0700 1156) return false;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1157) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1158) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1159)
1fb2f77c03762 (Henrik Kretzschmar 2010-03-26 20:38:35 +0100 1160) static int __init
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1161) check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1162) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1163) struct debug_bucket *db;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1164) struct debug_obj *obj;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1165) unsigned long flags;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1166) int res = -EINVAL;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1167)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1168) db = get_bucket((unsigned long) addr);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1169)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 1170) raw_spin_lock_irqsave(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1171)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1172) obj = lookup_object(addr, db);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1173) if (!obj && state != ODEBUG_STATE_NONE) {
5cd2b459d326a (Arjan van de Ven 2008-07-25 19:45:39 -0700 1174) WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1175) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1176) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1177) if (obj && obj->state != state) {
5cd2b459d326a (Arjan van de Ven 2008-07-25 19:45:39 -0700 1178) WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1179) obj->state, state);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1180) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1181) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1182) if (fixups != debug_objects_fixups) {
5cd2b459d326a (Arjan van de Ven 2008-07-25 19:45:39 -0700 1183) WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1184) fixups, debug_objects_fixups);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1185) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1186) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1187) if (warnings != debug_objects_warnings) {
5cd2b459d326a (Arjan van de Ven 2008-07-25 19:45:39 -0700 1188) WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1189) warnings, debug_objects_warnings);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1190) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1191) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1192) res = 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1193) out:
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 1194) raw_spin_unlock_irqrestore(&db->lock, flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1195) if (res)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1196) debug_objects_enabled = 0;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1197) return res;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1198) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1199)
aedcade6f4fa9 (Stephen Boyd 2020-08-14 17:40:26 -0700 1200) static __initconst const struct debug_obj_descr descr_type_test = {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1201) .name = "selftest",
b9fdac7f66060 (Changbin Du 2016-05-19 17:09:41 -0700 1202) .is_static_object = is_static_object,
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1203) .fixup_init = fixup_init,
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1204) .fixup_activate = fixup_activate,
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1205) .fixup_destroy = fixup_destroy,
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1206) .fixup_free = fixup_free,
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1207) };
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1208)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1209) static __initdata struct self_test obj = { .static_init = 0 };
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1210)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1211) static void __init debug_objects_selftest(void)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1212) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1213) int fixups, oldfixups, warnings, oldwarnings;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1214) unsigned long flags;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1215)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1216) local_irq_save(flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1217)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1218) fixups = oldfixups = debug_objects_fixups;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1219) warnings = oldwarnings = debug_objects_warnings;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1220) descr_test = &descr_type_test;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1221)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1222) debug_object_init(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1223) if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1224) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1225) debug_object_activate(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1226) if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1227) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1228) debug_object_activate(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1229) if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1230) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1231) debug_object_deactivate(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1232) if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1233) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1234) debug_object_destroy(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1235) if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1236) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1237) debug_object_init(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1238) if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1239) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1240) debug_object_activate(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1241) if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1242) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1243) debug_object_deactivate(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1244) if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1245) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1246) debug_object_free(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1247) if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1248) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1249)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1250) obj.static_init = 1;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1251) debug_object_activate(&obj, &descr_type_test);
9f78ff005a6b6 (Stephen Boyd 2012-03-05 14:59:17 -0800 1252) if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1253) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1254) debug_object_init(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1255) if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1256) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1257) debug_object_free(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1258) if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1259) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1260)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1261) #ifdef CONFIG_DEBUG_OBJECTS_FREE
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1262) debug_object_init(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1263) if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1264) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1265) debug_object_activate(&obj, &descr_type_test);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1266) if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1267) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1268) __debug_check_no_obj_freed(&obj, sizeof(obj));
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1269) if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1270) goto out;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1271) #endif
719e484396e27 (Fabian Frederick 2014-06-04 16:06:04 -0700 1272) pr_info("selftest passed\n");
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1273)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1274) out:
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1275) debug_objects_fixups = oldfixups;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1276) debug_objects_warnings = oldwarnings;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1277) descr_test = NULL;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1278)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1279) local_irq_restore(flags);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1280) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1281) #else
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1282) static inline void debug_objects_selftest(void) { }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1283) #endif
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1284)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1285) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1286) * Called during early boot to initialize the hash buckets and link
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1287) * the static object pool objects into the poll list. After this call
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1288) * the object tracker is fully operational.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1289) */
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1290) void __init debug_objects_early_init(void)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1291) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1292) int i;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1293)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1294) for (i = 0; i < ODEBUG_HASH_SIZE; i++)
aef9cb05247df (Thomas Gleixner 2009-11-17 18:11:28 +0100 1295) raw_spin_lock_init(&obj_hash[i].lock);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1296)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1297) for (i = 0; i < ODEBUG_POOL_SIZE; i++)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1298) hlist_add_head(&obj_static_pool[i].node, &obj_pool);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1299) }
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1300)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1301) /*
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1302) * Convert the statically allocated objects to dynamic ones:
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1303) */
1fb2f77c03762 (Henrik Kretzschmar 2010-03-26 20:38:35 +0100 1304) static int __init debug_objects_replace_static_objects(void)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1305) {
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1306) struct debug_bucket *db = obj_hash;
b67bfe0d42cac (Sasha Levin 2013-02-27 17:06:00 -0800 1307) struct hlist_node *tmp;
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1308) struct debug_obj *obj, *new;
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1309) HLIST_HEAD(objects);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1310) int i, cnt = 0;
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1311)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1312) for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1313) obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1314) if (!obj)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1315) goto free;
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1316) hlist_add_head(&obj->node, &objects);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1317) }
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1318)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1319) /*
a9ee3a63dbfff (Qian Cai 2018-12-28 00:32:32 -0800 1320) * debug_objects_mem_init() is now called early that only one CPU is up
a9ee3a63dbfff (Qian Cai 2018-12-28 00:32:32 -0800 1321) * and interrupts have been disabled, so it is safe to replace the
a9ee3a63dbfff (Qian Cai 2018-12-28 00:32:32 -0800 1322) * active object references.
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1323) */
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1324)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1325) /* Remove the statically allocated objects from the pool */
b67bfe0d42cac (Sasha Levin 2013-02-27 17:06:00 -0800 1326) hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1327) hlist_del(&obj->node);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1328) /* Move the allocated objects to the pool */
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1329) hlist_move_list(&objects, &obj_pool);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1330)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1331) /* Replace the active object references */
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1332) for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1333) hlist_move_list(&db->list, &objects);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1334)
b67bfe0d42cac (Sasha Levin 2013-02-27 17:06:00 -0800 1335) hlist_for_each_entry(obj, &objects, node) {
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1336) new = hlist_entry(obj_pool.first, typeof(*obj), node);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1337) hlist_del(&new->node);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1338) /* copy object data */
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1339) *new = *obj;
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1340) hlist_add_head(&new->node, &db->list);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1341) cnt++;
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1342) }
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1343) }
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1344)
c0f35cc0be0e1 (Fabian Frederick 2014-06-04 16:06:05 -0700 1345) pr_debug("%d of %d active objects replaced\n",
c0f35cc0be0e1 (Fabian Frederick 2014-06-04 16:06:05 -0700 1346) cnt, obj_pool_used);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1347) return 0;
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1348) free:
b67bfe0d42cac (Sasha Levin 2013-02-27 17:06:00 -0800 1349) hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1350) hlist_del(&obj->node);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1351) kmem_cache_free(obj_cache, obj);
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1352) }
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1353) return -ENOMEM;
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1354) }
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1355)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1356) /*
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1357) * Called after the kmem_caches are functional to setup a dedicated
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1358) * cache pool, which has the SLAB_DEBUG_OBJECTS flag set. This flag
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1359) * prevents that the debug code is called on kmem_cache_free() for the
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1360) * debug tracker objects to avoid recursive calls.
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1361) */
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1362) void __init debug_objects_mem_init(void)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1363) {
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1364) int cpu, extras;
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1365)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1366) if (!debug_objects_enabled)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1367) return;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1368)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1369) /*
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1370) * Initialize the percpu object pools
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1371) *
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1372) * Initialization is not strictly necessary, but was done for
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1373) * completeness.
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1374) */
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1375) for_each_possible_cpu(cpu)
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1376) INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
d86998b17a010 (Waiman Long 2019-05-20 10:14:46 -0400 1377)
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1378) obj_cache = kmem_cache_create("debug_objects_cache",
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1379) sizeof (struct debug_obj), 0,
8de456cf87ba8 (Qian Cai 2018-11-30 14:09:48 -0800 1380) SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
8de456cf87ba8 (Qian Cai 2018-11-30 14:09:48 -0800 1381) NULL);
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1382)
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1383) if (!obj_cache || debug_objects_replace_static_objects()) {
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1384) debug_objects_enabled = 0;
3ff4f80a74fd3 (Zhong Jiang 2018-08-01 00:24:58 +0800 1385) kmem_cache_destroy(obj_cache);
719e484396e27 (Fabian Frederick 2014-06-04 16:06:04 -0700 1386) pr_warn("out of memory.\n");
1be1cb7b47f07 (Thomas Gleixner 2009-03-16 18:53:18 +0100 1387) } else
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1388) debug_objects_selftest();
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1389)
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 1390) #ifdef CONFIG_HOTPLUG_CPU
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 1391) cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 1392) object_cpu_offline);
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 1393) #endif
88451f2cd3cec (Zqiang 2020-09-08 14:27:09 +0800 1394)
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1395) /*
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1396) * Increase the thresholds for allocating and freeing objects
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1397) * according to the number of possible CPUs available in the system.
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1398) */
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1399) extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1400) debug_objects_pool_size += extras;
634d61f45d6f6 (Waiman Long 2019-05-20 10:14:47 -0400 1401) debug_objects_pool_min_level += extras;
3ac7fe5a4aab4 (Thomas Gleixner 2008-04-30 00:55:01 -0700 1402) }
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
32 Branches
54 Tags