4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 1) /*
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 2) * cpuidle.c - core cpuidle infrastructure
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 3) *
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 4) * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 5) * Shaohua Li <shaohua.li@intel.com>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 6) * Adam Belay <abelay@novell.com>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 7) *
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 8) * This code is licenced under the GPL.
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 9) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 10)
b60e6a0eb0273 (Daniel Lezcano 2013-03-21 12:21:31 +0000 11) #include <linux/clockchips.h>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 12) #include <linux/kernel.h>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 13) #include <linux/mutex.h>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 14) #include <linux/sched.h>
e601757102cfd (Ingo Molnar 2017-02-01 16:36:40 +0100 15) #include <linux/sched/clock.h>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 16) #include <linux/notifier.h>
e8db0be1245de (Jean Pihet 2011-08-25 15:35:03 +0200 17) #include <linux/pm_qos.h>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 18) #include <linux/cpu.h>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 19) #include <linux/cpuidle.h>
9a0b841586c3c (venkatesh.pallipadi@intel.com 2008-01-31 17:35:06 -0800 20) #include <linux/ktime.h>
2e94d1f71f7e4 (Arjan van de Ven 2008-09-10 16:06:00 -0700 21) #include <linux/hrtimer.h>
884b17e109d61 (Paul Gortmaker 2011-08-29 17:52:39 -0400 22) #include <linux/module.h>
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 23) #include <linux/suspend.h>
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 24) #include <linux/tick.h>
bf9282dc26e7f (Peter Zijlstra 2020-08-12 12:22:17 +0200 25) #include <linux/mmu_context.h>
288f023e708ef (Arjan van de Ven 2009-09-19 13:35:33 +0200 26) #include <trace/events/power.h>
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 27)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 28) #include "cpuidle.h"
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 29)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 30) DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 31) DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 32)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 33) DEFINE_MUTEX(cpuidle_lock);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 34) LIST_HEAD(cpuidle_detected_devices);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 35)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 36) static int enabled_devices;
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 37) static int off __read_mostly;
a0bfa1373859e (Len Brown 2011-04-01 19:34:59 -0400 38) static int initialized __read_mostly;
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 39)
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 40) int cpuidle_disabled(void)
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 41) {
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 42) return off;
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 43) }
d91ee5863b71e (Len Brown 2011-04-01 18:28:35 -0400 44) void disable_cpuidle(void)
d91ee5863b71e (Len Brown 2011-04-01 18:28:35 -0400 45) {
d91ee5863b71e (Len Brown 2011-04-01 18:28:35 -0400 46) off = 1;
d91ee5863b71e (Len Brown 2011-04-01 18:28:35 -0400 47) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 48)
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 49) bool cpuidle_not_available(struct cpuidle_driver *drv,
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 50) struct cpuidle_device *dev)
31a3409065d1d (Rafael J. Wysocki 2015-02-27 00:39:56 +0100 51) {
31a3409065d1d (Rafael J. Wysocki 2015-02-27 00:39:56 +0100 52) return off || !initialized || !drv || !dev || !dev->enabled;
31a3409065d1d (Rafael J. Wysocki 2015-02-27 00:39:56 +0100 53) }
31a3409065d1d (Rafael J. Wysocki 2015-02-27 00:39:56 +0100 54)
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 55) /**
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 56) * cpuidle_play_dead - cpu off-lining
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 57) *
ee01e66337334 (Toshi Kani 2012-03-31 21:37:02 -0600 58) * Returns in case of an error or no driver
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 59) */
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 60) int cpuidle_play_dead(void)
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 61) {
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 62) struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 63) struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
8aef33a7cf40c (Daniel Lezcano 2013-01-15 14:18:04 +0100 64) int i;
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 65)
ee01e66337334 (Toshi Kani 2012-03-31 21:37:02 -0600 66) if (!drv)
ee01e66337334 (Toshi Kani 2012-03-31 21:37:02 -0600 67) return -ENODEV;
ee01e66337334 (Toshi Kani 2012-03-31 21:37:02 -0600 68)
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 69) /* Find lowest-power state that supports long-term idle */
7d51d97925e6c (Rafael J. Wysocki 2015-05-28 04:09:24 +0200 70) for (i = drv->state_count - 1; i >= 0; i--)
8aef33a7cf40c (Daniel Lezcano 2013-01-15 14:18:04 +0100 71) if (drv->states[i].enter_dead)
8aef33a7cf40c (Daniel Lezcano 2013-01-15 14:18:04 +0100 72) return drv->states[i].enter_dead(dev, i);
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 73)
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 74) return -ENODEV;
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 75) }
1a022e3f1be11 (Boris Ostrovsky 2012-03-13 19:55:09 +0100 76)
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 77) static int find_deepest_state(struct cpuidle_driver *drv,
0d94039fabcca (Rafael J. Wysocki 2015-05-10 01:19:52 +0200 78) struct cpuidle_device *dev,
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 79) u64 max_latency_ns,
0d94039fabcca (Rafael J. Wysocki 2015-05-10 01:19:52 +0200 80) unsigned int forbidden_flags,
28ba086ed30fb (Rafael J. Wysocki 2017-08-10 00:14:45 +0200 81) bool s2idle)
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 82) {
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 83) u64 latency_req = 0;
51164251f5c35 (Rafael J. Wysocki 2016-01-16 00:54:53 +0100 84) int i, ret = 0;
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 85)
51164251f5c35 (Rafael J. Wysocki 2016-01-16 00:54:53 +0100 86) for (i = 1; i < drv->state_count; i++) {
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 87) struct cpuidle_state *s = &drv->states[i];
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 88)
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 89) if (dev->states_usage[i].disable ||
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 90) s->exit_latency_ns <= latency_req ||
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 91) s->exit_latency_ns > max_latency_ns ||
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 92) (s->flags & forbidden_flags) ||
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 93) (s2idle && !s->enter_s2idle))
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 94) continue;
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 95)
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 96) latency_req = s->exit_latency_ns;
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 97) ret = i;
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 98) }
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 99) return ret;
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 100) }
a6220fc19afc0 (Rafael J. Wysocki 2014-05-05 00:51:54 +0200 101)
0e7414b7aa8b2 (Rafael J. Wysocki 2016-12-01 23:31:32 +0100 102) /**
c55b51a06b01d (Daniel Lezcano 2019-11-16 14:16:12 +0100 103) * cpuidle_use_deepest_state - Set/unset governor override mode.
c55b51a06b01d (Daniel Lezcano 2019-11-16 14:16:12 +0100 104) * @latency_limit_ns: Idle state exit latency limit (or no override if 0).
0e7414b7aa8b2 (Rafael J. Wysocki 2016-12-01 23:31:32 +0100 105) *
c55b51a06b01d (Daniel Lezcano 2019-11-16 14:16:12 +0100 106) * If @latency_limit_ns is nonzero, set the current CPU to use the deepest idle
c55b51a06b01d (Daniel Lezcano 2019-11-16 14:16:12 +0100 107) * state with exit latency within @latency_limit_ns (override governors going
c55b51a06b01d (Daniel Lezcano 2019-11-16 14:16:12 +0100 108) * forward), or do not override governors if it is zero.
0e7414b7aa8b2 (Rafael J. Wysocki 2016-12-01 23:31:32 +0100 109) */
c55b51a06b01d (Daniel Lezcano 2019-11-16 14:16:12 +0100 110) void cpuidle_use_deepest_state(u64 latency_limit_ns)
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 111) {
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 112) struct cpuidle_device *dev;
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 113)
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 114) preempt_disable();
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 115) dev = cpuidle_get_device();
41dc750ea67f3 (Li, Fei 2017-04-27 01:47:25 +0000 116) if (dev)
c55b51a06b01d (Daniel Lezcano 2019-11-16 14:16:12 +0100 117) dev->forced_idle_latency_limit_ns = latency_limit_ns;
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 118) preempt_enable();
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 119) }
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 120)
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 121) /**
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 122) * cpuidle_find_deepest_state - Find the deepest available idle state.
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 123) * @drv: cpuidle driver for the given CPU.
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 124) * @dev: cpuidle device for the given CPU.
cefb9409ff995 (Benjamin Gaignard 2020-01-21 09:27:58 +0100 125) * @latency_limit_ns: Idle state exit latency limit
cefb9409ff995 (Benjamin Gaignard 2020-01-21 09:27:58 +0100 126) *
cefb9409ff995 (Benjamin Gaignard 2020-01-21 09:27:58 +0100 127) * Return: the index of the deepest available idle state.
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 128) */
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 129) int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
5aa9ba6312e36 (Daniel Lezcano 2019-11-16 14:16:13 +0100 130) struct cpuidle_device *dev,
5aa9ba6312e36 (Daniel Lezcano 2019-11-16 14:16:13 +0100 131) u64 latency_limit_ns)
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 132) {
5aa9ba6312e36 (Daniel Lezcano 2019-11-16 14:16:13 +0100 133) return find_deepest_state(drv, dev, latency_limit_ns, 0, false);
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 134) }
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 135)
bb8313b603eb8 (Jacob Pan 2016-11-28 23:03:04 -0800 136) #ifdef CONFIG_SUSPEND
28ba086ed30fb (Rafael J. Wysocki 2017-08-10 00:14:45 +0200 137) static void enter_s2idle_proper(struct cpuidle_driver *drv,
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 138) struct cpuidle_device *dev, int index)
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 139) {
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 140) ktime_t time_start, time_end;
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 141) struct cpuidle_state *target_state = &drv->states[index];
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 142)
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 143) time_start = ns_to_ktime(local_clock());
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 144)
1098582a0f6c4 (Peter Zijlstra 2020-08-07 20:50:19 +0200 145) tick_freeze();
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 146) /*
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 147) * The state used here cannot be a "coupled" one, because the "coupled"
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 148) * cpuidle mechanism enables interrupts and doing that with timekeeping
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 149) * suspended is generally unsafe.
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 150) */
63caae8480921 (Lucas Stach 2015-07-20 18:34:50 +0200 151) stop_critical_timings();
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 152) if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 153) rcu_idle_enter();
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 154) target_state->enter_s2idle(dev, drv, index);
49d9c5936314e (Peter Zijlstra 2020-08-20 16:47:24 +0200 155) if (WARN_ON_ONCE(!irqs_disabled()))
49d9c5936314e (Peter Zijlstra 2020-08-20 16:47:24 +0200 156) local_irq_disable();
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 157) if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 158) rcu_idle_exit();
1098582a0f6c4 (Peter Zijlstra 2020-08-07 20:50:19 +0200 159) tick_unfreeze();
63caae8480921 (Lucas Stach 2015-07-20 18:34:50 +0200 160) start_critical_timings();
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 161)
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 162) time_end = ns_to_ktime(local_clock());
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 163)
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 164) dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);
64bdff698092a (Rafael J. Wysocki 2018-03-14 12:27:21 +0100 165) dev->states_usage[index].s2idle_usage++;
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 166) }
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 167)
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 168) /**
28ba086ed30fb (Rafael J. Wysocki 2017-08-10 00:14:45 +0200 169) * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle.
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 170) * @drv: cpuidle driver for the given CPU.
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 171) * @dev: cpuidle device for the given CPU.
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 172) *
28ba086ed30fb (Rafael J. Wysocki 2017-08-10 00:14:45 +0200 173) * If there are states with the ->enter_s2idle callback, find the deepest of
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 174) * them and enter it with frozen tick.
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 175) */
28ba086ed30fb (Rafael J. Wysocki 2017-08-10 00:14:45 +0200 176) int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev)
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 177) {
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 178) int index;
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 179)
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 180) /*
28ba086ed30fb (Rafael J. Wysocki 2017-08-10 00:14:45 +0200 181) * Find the deepest state with ->enter_s2idle present, which guarantees
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 182) * that interrupts won't be enabled when it exits and allows the tick to
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 183) * be frozen safely.
124cf9117c5f9 (Rafael J. Wysocki 2015-02-13 23:50:43 +0100 184) */
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 185) index = find_deepest_state(drv, dev, U64_MAX, 0, true);
10e8b11eb3195 (Rafael J. Wysocki 2020-06-25 13:52:53 +0200 186) if (index > 0) {
28ba086ed30fb (Rafael J. Wysocki 2017-08-10 00:14:45 +0200 187) enter_s2idle_proper(drv, dev, index);
10e8b11eb3195 (Rafael J. Wysocki 2020-06-25 13:52:53 +0200 188) local_irq_enable();
10e8b11eb3195 (Rafael J. Wysocki 2020-06-25 13:52:53 +0200 189) }
ef2b22ac540c0 (Rafael J. Wysocki 2015-03-02 22:26:55 +0100 190) return index;
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 191) }
87e9b9f1d86c2 (Rafael J. Wysocki 2015-05-16 01:38:15 +0200 192) #endif /* CONFIG_SUSPEND */
3810631332465 (Rafael J. Wysocki 2015-02-12 23:33:15 +0100 193)
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 194) /**
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 195) * cpuidle_enter_state - enter the state and update stats
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 196) * @dev: cpuidle device for this cpu
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 197) * @drv: cpuidle driver for this cpu
7312280bd2ad9 (Rafael J. Wysocki 2015-05-09 21:50:32 +0200 198) * @index: index into the states table in @drv of the state to enter
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 199) */
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 200) int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
554c06ba3ee29 (Daniel Lezcano 2013-04-23 08:54:31 +0000 201) int index)
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 202) {
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 203) int entered_state;
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 204)
554c06ba3ee29 (Daniel Lezcano 2013-04-23 08:54:31 +0000 205) struct cpuidle_state *target_state = &drv->states[index];
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 206) bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
dbd1b8ea43b17 (Shreyas B. Prabhu 2016-07-01 09:24:14 -0500 207) ktime_t time_start, time_end;
554c06ba3ee29 (Daniel Lezcano 2013-04-23 08:54:31 +0000 208)
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 209) /*
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 210) * Tell the time framework to switch to a broadcast timer because our
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 211) * local timer will be shut down. If a local timer is used from another
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 212) * CPU as a broadcast timer, this call may fail if it is not available.
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 213) */
827a5aefc542b (Rafael J. Wysocki 2015-05-10 01:18:46 +0200 214) if (broadcast && tick_broadcast_enter()) {
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 215) index = find_deepest_state(drv, dev, target_state->exit_latency_ns,
0d94039fabcca (Rafael J. Wysocki 2015-05-10 01:19:52 +0200 216) CPUIDLE_FLAG_TIMER_STOP, false);
0d94039fabcca (Rafael J. Wysocki 2015-05-10 01:19:52 +0200 217) if (index < 0) {
0d94039fabcca (Rafael J. Wysocki 2015-05-10 01:19:52 +0200 218) default_idle_call();
0d94039fabcca (Rafael J. Wysocki 2015-05-10 01:19:52 +0200 219) return -EBUSY;
0d94039fabcca (Rafael J. Wysocki 2015-05-10 01:19:52 +0200 220) }
0d94039fabcca (Rafael J. Wysocki 2015-05-10 01:19:52 +0200 221) target_state = &drv->states[index];
f187851b9b4a7 (Nicholas Piggin 2017-09-01 14:29:56 +1000 222) broadcast = false;
827a5aefc542b (Rafael J. Wysocki 2015-05-10 01:18:46 +0200 223) }
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 224)
bf9282dc26e7f (Peter Zijlstra 2020-08-12 12:22:17 +0200 225) if (target_state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
bf9282dc26e7f (Peter Zijlstra 2020-08-12 12:22:17 +0200 226) leave_mm(dev->cpu);
bf9282dc26e7f (Peter Zijlstra 2020-08-12 12:22:17 +0200 227)
faad384928141 (Rafael J. Wysocki 2015-05-10 01:18:03 +0200 228) /* Take note of the planned idle state. */
faad384928141 (Rafael J. Wysocki 2015-05-10 01:18:03 +0200 229) sched_idle_set_state(target_state);
faad384928141 (Rafael J. Wysocki 2015-05-10 01:18:03 +0200 230)
1098582a0f6c4 (Peter Zijlstra 2020-08-07 20:50:19 +0200 231) trace_cpu_idle(index, dev->cpu);
dbd1b8ea43b17 (Shreyas B. Prabhu 2016-07-01 09:24:14 -0500 232) time_start = ns_to_ktime(local_clock());
554c06ba3ee29 (Daniel Lezcano 2013-04-23 08:54:31 +0000 233)
63caae8480921 (Lucas Stach 2015-07-20 18:34:50 +0200 234) stop_critical_timings();
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 235) if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 236) rcu_idle_enter();
554c06ba3ee29 (Daniel Lezcano 2013-04-23 08:54:31 +0000 237) entered_state = target_state->enter(dev, drv, index);
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 238) if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
8747f2022fe8d (Peter Zijlstra 2020-09-15 12:32:00 +0200 239) rcu_idle_exit();
63caae8480921 (Lucas Stach 2015-07-20 18:34:50 +0200 240) start_critical_timings();
554c06ba3ee29 (Daniel Lezcano 2013-04-23 08:54:31 +0000 241)
f9fccdb9efef6 (Peter Zijlstra 2017-04-21 12:43:59 +0200 242) sched_clock_idle_wakeup_event();
dbd1b8ea43b17 (Shreyas B. Prabhu 2016-07-01 09:24:14 -0500 243) time_end = ns_to_ktime(local_clock());
1098582a0f6c4 (Peter Zijlstra 2020-08-07 20:50:19 +0200 244) trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);
554c06ba3ee29 (Daniel Lezcano 2013-04-23 08:54:31 +0000 245)
faad384928141 (Rafael J. Wysocki 2015-05-10 01:18:03 +0200 246) /* The cpu is no longer idle or about to enter idle. */
faad384928141 (Rafael J. Wysocki 2015-05-10 01:18:03 +0200 247) sched_idle_set_state(NULL);
faad384928141 (Rafael J. Wysocki 2015-05-10 01:18:03 +0200 248)
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 249) if (broadcast) {
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 250) if (WARN_ON_ONCE(!irqs_disabled()))
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 251) local_irq_disable();
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 252)
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 253) tick_broadcast_exit();
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 254) }
df8d9eeadd0f7 (Rafael J. Wysocki 2015-04-29 15:19:21 +0200 255)
e7387da52028b (Daniel Lezcano 2016-05-17 16:54:00 +0200 256) if (!cpuidle_state_is_coupled(drv, index))
0b89e9aa28566 (Paul Burton 2014-03-06 11:02:01 +0000 257) local_irq_enable();
554c06ba3ee29 (Daniel Lezcano 2013-04-23 08:54:31 +0000 258)
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 259) if (entered_state >= 0) {
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 260) s64 diff, delay = drv->states[entered_state].exit_latency_ns;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 261) int i;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 262)
7037b43e0076c (Fieah Lim 2018-09-11 05:47:25 +0800 263) /*
7037b43e0076c (Fieah Lim 2018-09-11 05:47:25 +0800 264) * Update cpuidle counters
7037b43e0076c (Fieah Lim 2018-09-11 05:47:25 +0800 265) * This can be moved to within driver enter routine,
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 266) * but that results in multiple copies of same code.
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 267) */
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 268) diff = ktime_sub(time_end, time_start);
7037b43e0076c (Fieah Lim 2018-09-11 05:47:25 +0800 269)
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 270) dev->last_residency_ns = diff;
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 271) dev->states_usage[entered_state].time_ns += diff;
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 272) dev->states_usage[entered_state].usage++;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 273)
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 274) if (diff < drv->states[entered_state].target_residency_ns) {
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 275) for (i = entered_state - 1; i >= 0; i--) {
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 276) if (dev->states_usage[i].disable)
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 277) continue;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 278)
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 279) /* Shallower states are enabled, so update. */
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 280) dev->states_usage[entered_state].above++;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 281) break;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 282) }
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 283) } else if (diff > delay) {
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 284) for (i = entered_state + 1; i < drv->state_count; i++) {
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 285) if (dev->states_usage[i].disable)
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 286) continue;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 287)
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 288) /*
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 289) * Update if a deeper state would have been a
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 290) * better match for the observed idle duration.
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 291) */
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 292) if (diff - delay >= drv->states[i].target_residency_ns)
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 293) dev->states_usage[entered_state].below++;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 294)
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 295) break;
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 296) }
04dab58a39d40 (Rafael J. Wysocki 2018-12-10 12:30:23 +0100 297) }
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 298) } else {
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 299) dev->last_residency_ns = 0;
f49735f4978f4 (Lina Iyer 2020-09-22 12:34:16 -0600 300) dev->states_usage[index].rejected++;
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 301) }
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 302)
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 303) return entered_state;
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 304) }
56cfbf74a17c4 (Colin Cross 2012-05-07 17:57:39 -0700 305)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 306) /**
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 307) * cpuidle_select - ask the cpuidle framework to choose an idle state
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 308) *
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 309) * @drv: the cpuidle driver
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 310) * @dev: the cpuidle device
45f1ff59e27ca (Rafael J. Wysocki 2018-03-22 17:50:49 +0100 311) * @stop_tick: indication on whether or not to stop the tick
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 312) *
51164251f5c35 (Rafael J. Wysocki 2016-01-16 00:54:53 +0100 313) * Returns the index of the idle state. The return value must not be negative.
45f1ff59e27ca (Rafael J. Wysocki 2018-03-22 17:50:49 +0100 314) *
45f1ff59e27ca (Rafael J. Wysocki 2018-03-22 17:50:49 +0100 315) * The memory location pointed to by @stop_tick is expected to be written the
45f1ff59e27ca (Rafael J. Wysocki 2018-03-22 17:50:49 +0100 316) * 'false' boolean value if the scheduler tick should not be stopped before
45f1ff59e27ca (Rafael J. Wysocki 2018-03-22 17:50:49 +0100 317) * entering the returned state.
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 318) */
45f1ff59e27ca (Rafael J. Wysocki 2018-03-22 17:50:49 +0100 319) int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
45f1ff59e27ca (Rafael J. Wysocki 2018-03-22 17:50:49 +0100 320) bool *stop_tick)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 321) {
45f1ff59e27ca (Rafael J. Wysocki 2018-03-22 17:50:49 +0100 322) return cpuidle_curr_governor->select(drv, dev, stop_tick);
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 323) }
ba8f20c2eb415 (Preeti U Murthy 2014-02-07 13:36:52 +0530 324)
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 325) /**
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 326) * cpuidle_enter - enter into the specified idle state
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 327) *
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 328) * @drv: the cpuidle driver tied with the cpu
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 329) * @dev: the cpuidle device
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 330) * @index: the index in the idle state table
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 331) *
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 332) * Returns the index in the idle state, < 0 in case of error.
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 333) * The error code depends on the backend driver
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 334) */
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 335) int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 336) int index)
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 337) {
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 338) int ret = 0;
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 339)
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 340) /*
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 341) * Store the next hrtimer, which becomes either next tick or the next
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 342) * timer event, whatever expires first. Additionally, to make this data
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 343) * useful for consumers outside cpuidle, we rely on that the governor's
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 344) * ->select() callback have decided, whether to stop the tick or not.
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 345) */
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 346) WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 347)
4c1ed5a607907 (Xunlei Pang 2015-08-04 13:48:56 +0800 348) if (cpuidle_state_is_coupled(drv, index))
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 349) ret = cpuidle_enter_state_coupled(dev, drv, index);
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 350) else
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 351) ret = cpuidle_enter_state(dev, drv, index);
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 352)
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 353) WRITE_ONCE(dev->next_hrtimer, 0);
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 354) return ret;
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 355) }
b60e6a0eb0273 (Daniel Lezcano 2013-03-21 12:21:31 +0000 356)
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 357) /**
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 358) * cpuidle_reflect - tell the underlying governor what was the state
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 359) * we were in
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 360) *
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 361) * @dev : the cpuidle device
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 362) * @index: the index in the idle state table
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 363) *
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 364) */
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 365) void cpuidle_reflect(struct cpuidle_device *dev, int index)
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 366) {
a802ea9645457 (Rafael J. Wysocki 2015-05-04 22:53:28 +0200 367) if (cpuidle_curr_governor->reflect && index >= 0)
907e30f1bb4a9 (Daniel Lezcano 2014-03-03 08:48:50 +0100 368) cpuidle_curr_governor->reflect(dev, index);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 369) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 370)
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 371) /*
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 372) * Min polling interval of 10usec is a guess. It is assuming that
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 373) * for most users, the time for a single ping-pong workload like
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 374) * perf bench pipe would generally complete within 10usec but
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 375) * this is hardware dependant. Actual time can be estimated with
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 376) *
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 377) * perf bench sched pipe -l 10000
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 378) *
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 379) * Run multiple times to avoid cpufreq effects.
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 380) */
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 381) #define CPUIDLE_POLL_MIN 10000
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 382) #define CPUIDLE_POLL_MAX (TICK_NSEC / 16)
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 383)
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 384) /**
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 385) * cpuidle_poll_time - return amount of time to poll for,
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 386) * governors can override dev->poll_limit_ns if necessary
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 387) *
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 388) * @drv: the cpuidle driver tied with the cpu
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 389) * @dev: the cpuidle device
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 390) *
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 391) */
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 392) u64 cpuidle_poll_time(struct cpuidle_driver *drv,
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 393) struct cpuidle_device *dev)
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 394) {
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 395) int i;
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 396) u64 limit_ns;
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 397)
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 398) BUILD_BUG_ON(CPUIDLE_POLL_MIN > CPUIDLE_POLL_MAX);
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 399)
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 400) if (dev->poll_limit_ns)
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 401) return dev->poll_limit_ns;
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 402)
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 403) limit_ns = CPUIDLE_POLL_MAX;
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 404) for (i = 1; i < drv->state_count; i++) {
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 405) u64 state_limit;
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 406)
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 407) if (dev->states_usage[i].disable)
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 408) continue;
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 409)
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 410) state_limit = drv->states[i].target_residency_ns;
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 411) if (state_limit < CPUIDLE_POLL_MIN)
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 412) continue;
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 413)
7a25759eaa04b (Mel Gorman 2020-11-30 22:32:02 +0000 414) limit_ns = min_t(u64, state_limit, CPUIDLE_POLL_MAX);
36fcb4292473c (Marcelo Tosatti 2019-12-06 13:07:41 -0200 415) break;
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 416) }
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 417)
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 418) dev->poll_limit_ns = limit_ns;
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 419)
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 420) return dev->poll_limit_ns;
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 421) }
259231a045616 (Marcelo Tosatti 2019-07-03 20:51:26 -0300 422)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 423) /**
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 424) * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 425) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 426) void cpuidle_install_idle_handler(void)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 427) {
a0bfa1373859e (Len Brown 2011-04-01 19:34:59 -0400 428) if (enabled_devices) {
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 429) /* Make sure all changes finished before we switch to new idle */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 430) smp_wmb();
a0bfa1373859e (Len Brown 2011-04-01 19:34:59 -0400 431) initialized = 1;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 432) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 433) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 434)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 435) /**
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 436) * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 437) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 438) void cpuidle_uninstall_idle_handler(void)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 439) {
a0bfa1373859e (Len Brown 2011-04-01 19:34:59 -0400 440) if (enabled_devices) {
a0bfa1373859e (Len Brown 2011-04-01 19:34:59 -0400 441) initialized = 0;
2ed903c5485ba (Chuansheng Liu 2014-09-04 15:17:55 +0800 442) wake_up_all_idle_cpus();
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 443) }
442bf3aaf55a9 (Daniel Lezcano 2014-09-04 11:32:09 -0400 444)
442bf3aaf55a9 (Daniel Lezcano 2014-09-04 11:32:09 -0400 445) /*
442bf3aaf55a9 (Daniel Lezcano 2014-09-04 11:32:09 -0400 446) * Make sure external observers (such as the scheduler)
442bf3aaf55a9 (Daniel Lezcano 2014-09-04 11:32:09 -0400 447) * are done looking at pointed idle states.
442bf3aaf55a9 (Daniel Lezcano 2014-09-04 11:32:09 -0400 448) */
442bf3aaf55a9 (Daniel Lezcano 2014-09-04 11:32:09 -0400 449) synchronize_rcu();
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 450) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 451)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 452) /**
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 453) * cpuidle_pause_and_lock - temporarily disables CPUIDLE
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 454) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 455) void cpuidle_pause_and_lock(void)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 456) {
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 457) mutex_lock(&cpuidle_lock);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 458) cpuidle_uninstall_idle_handler();
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 459) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 460)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 461) EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 462)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 463) /**
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 464) * cpuidle_resume_and_unlock - resumes CPUIDLE operation
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 465) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 466) void cpuidle_resume_and_unlock(void)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 467) {
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 468) cpuidle_install_idle_handler();
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 469) mutex_unlock(&cpuidle_lock);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 470) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 471)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 472) EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 473)
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 474) /* Currently used in suspend/resume path to suspend cpuidle */
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 475) void cpuidle_pause(void)
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 476) {
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 477) mutex_lock(&cpuidle_lock);
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 478) cpuidle_uninstall_idle_handler();
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 479) mutex_unlock(&cpuidle_lock);
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 480) }
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 481)
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 482) /* Currently used in suspend/resume path to resume cpuidle */
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 483) void cpuidle_resume(void)
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 484) {
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 485) mutex_lock(&cpuidle_lock);
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 486) cpuidle_install_idle_handler();
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 487) mutex_unlock(&cpuidle_lock);
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 488) }
8651f97bd951d (Preeti U Murthy 2012-07-09 10:12:56 +0200 489)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 490) /**
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 491) * cpuidle_enable_device - enables idle PM for a CPU
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 492) * @dev: the CPU
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 493) *
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 494) * This function must be called between cpuidle_pause_and_lock and
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 495) * cpuidle_resume_and_unlock when used externally.
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 496) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 497) int cpuidle_enable_device(struct cpuidle_device *dev)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 498) {
5df0aa7341bd9 (Daniel Lezcano 2013-06-12 15:08:54 +0200 499) int ret;
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 500) struct cpuidle_driver *drv;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 501)
1b0a0e9a15b97 (Srivatsa S. Bhat 2012-05-04 14:06:02 -0700 502) if (!dev)
1b0a0e9a15b97 (Srivatsa S. Bhat 2012-05-04 14:06:02 -0700 503) return -EINVAL;
1b0a0e9a15b97 (Srivatsa S. Bhat 2012-05-04 14:06:02 -0700 504)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 505) if (dev->enabled)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 506) return 0;
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 507)
e7b06a09e7d87 (Gaurav Jindal 2017-09-01 20:37:26 +0530 508) if (!cpuidle_curr_governor)
e7b06a09e7d87 (Gaurav Jindal 2017-09-01 20:37:26 +0530 509) return -EIO;
e7b06a09e7d87 (Gaurav Jindal 2017-09-01 20:37:26 +0530 510)
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 511) drv = cpuidle_get_cpu_driver(dev);
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 512)
e7b06a09e7d87 (Gaurav Jindal 2017-09-01 20:37:26 +0530 513) if (!drv)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 514) return -EIO;
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 515)
10b9d3f8a4d5c (Daniel Lezcano 2013-06-12 15:08:49 +0200 516) if (!dev->registered)
10b9d3f8a4d5c (Daniel Lezcano 2013-06-12 15:08:49 +0200 517) return -EINVAL;
10b9d3f8a4d5c (Daniel Lezcano 2013-06-12 15:08:49 +0200 518)
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 519) ret = cpuidle_add_device_sysfs(dev);
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 520) if (ret)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 521) return ret;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 522)
3fc74bd8a723c (Gaurav Jindal 2017-09-02 00:56:38 +0530 523) if (cpuidle_curr_governor->enable) {
3fc74bd8a723c (Gaurav Jindal 2017-09-02 00:56:38 +0530 524) ret = cpuidle_curr_governor->enable(drv, dev);
3fc74bd8a723c (Gaurav Jindal 2017-09-02 00:56:38 +0530 525) if (ret)
3fc74bd8a723c (Gaurav Jindal 2017-09-02 00:56:38 +0530 526) goto fail_sysfs;
3fc74bd8a723c (Gaurav Jindal 2017-09-02 00:56:38 +0530 527) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 528)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 529) smp_wmb();
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 530)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 531) dev->enabled = 1;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 532)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 533) enabled_devices++;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 534) return 0;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 535)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 536) fail_sysfs:
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 537) cpuidle_remove_device_sysfs(dev);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 538)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 539) return ret;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 540) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 541)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 542) EXPORT_SYMBOL_GPL(cpuidle_enable_device);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 543)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 544) /**
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 545) * cpuidle_disable_device - disables idle PM for a CPU
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 546) * @dev: the CPU
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 547) *
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 548) * This function must be called between cpuidle_pause_and_lock and
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 549) * cpuidle_resume_and_unlock when used externally.
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 550) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 551) void cpuidle_disable_device(struct cpuidle_device *dev)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 552) {
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 553) struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 554)
cf31cd1a0c692 (Srivatsa S. Bhat 2012-10-08 13:43:08 +0530 555) if (!dev || !dev->enabled)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 556) return;
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 557)
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 558) if (!drv || !cpuidle_curr_governor)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 559) return;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 560)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 561) dev->enabled = 0;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 562)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 563) if (cpuidle_curr_governor->disable)
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 564) cpuidle_curr_governor->disable(drv, dev);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 565)
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 566) cpuidle_remove_device_sysfs(dev);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 567) enabled_devices--;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 568) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 569)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 570) EXPORT_SYMBOL_GPL(cpuidle_disable_device);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 571)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 572) static void __cpuidle_unregister_device(struct cpuidle_device *dev)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 573) {
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 574) struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 575)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 576) list_del(&dev->device_list);
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 577) per_cpu(cpuidle_devices, dev->cpu) = NULL;
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 578) module_put(drv->owner);
c998c07836f98 (Dave Gerlach 2016-04-05 14:05:38 -0500 579)
c998c07836f98 (Dave Gerlach 2016-04-05 14:05:38 -0500 580) dev->registered = 0;
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 581) }
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 582)
267d4bf8ee6db (Viresh Kumar 2013-10-03 21:26:43 +0530 583) static void __cpuidle_device_init(struct cpuidle_device *dev)
5df0aa7341bd9 (Daniel Lezcano 2013-06-12 15:08:54 +0200 584) {
5df0aa7341bd9 (Daniel Lezcano 2013-06-12 15:08:54 +0200 585) memset(dev->states_usage, 0, sizeof(dev->states_usage));
c1d51f684c72b (Rafael J. Wysocki 2019-11-07 15:25:12 +0100 586) dev->last_residency_ns = 0;
6f9b83ac877fb (Ulf Hansson 2019-03-27 15:35:47 +0100 587) dev->next_hrtimer = 0;
5df0aa7341bd9 (Daniel Lezcano 2013-06-12 15:08:54 +0200 588) }
5df0aa7341bd9 (Daniel Lezcano 2013-06-12 15:08:54 +0200 589)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 590) /**
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 591) * __cpuidle_register_device - internal register function called before register
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 592) * and enable routines
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 593) * @dev: the cpu
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 594) *
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 595) * cpuidle_lock mutex must be held before this is called
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 596) */
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 597) static int __cpuidle_register_device(struct cpuidle_device *dev)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 598) {
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 599) struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 600) int i, ret;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 601)
bf4d1b5ddb78f (Daniel Lezcano 2012-10-31 16:44:48 +0000 602) if (!try_module_get(drv->owner))
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 603) return -EINVAL;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 604)
75a80267410e3 (Rafael J. Wysocki 2019-12-13 09:56:13 +0100 605) for (i = 0; i < drv->state_count; i++) {
ba1e78a1dc0ca (Rafael J. Wysocki 2019-11-21 19:41:51 +0100 606) if (drv->states[i].flags & CPUIDLE_FLAG_UNUSABLE)
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 607) dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;
99e98d3fb1008 (Rafael J. Wysocki 2019-11-04 12:16:17 +0100 608)
75a80267410e3 (Rafael J. Wysocki 2019-12-13 09:56:13 +0100 609) if (drv->states[i].flags & CPUIDLE_FLAG_OFF)
75a80267410e3 (Rafael J. Wysocki 2019-12-13 09:56:13 +0100 610) dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_USER;
75a80267410e3 (Rafael J. Wysocki 2019-12-13 09:56:13 +0100 611) }
75a80267410e3 (Rafael J. Wysocki 2019-12-13 09:56:13 +0100 612)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 613) per_cpu(cpuidle_devices, dev->cpu) = dev;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 614) list_add(&dev->device_list, &cpuidle_detected_devices);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 615)
4126c0197bc8c (Colin Cross 2012-05-07 17:57:41 -0700 616) ret = cpuidle_coupled_register_device(dev);
47182668ca140 (Viresh Kumar 2013-10-03 21:26:46 +0530 617) if (ret)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 618) __cpuidle_unregister_device(dev);
47182668ca140 (Viresh Kumar 2013-10-03 21:26:46 +0530 619) else
47182668ca140 (Viresh Kumar 2013-10-03 21:26:46 +0530 620) dev->registered = 1;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 621)
47182668ca140 (Viresh Kumar 2013-10-03 21:26:46 +0530 622) return ret;
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 623) }
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 624)
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 625) /**
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 626) * cpuidle_register_device - registers a CPU's idle PM feature
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 627) * @dev: the cpu
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 628) */
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 629) int cpuidle_register_device(struct cpuidle_device *dev)
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 630) {
c878a52d3c7ca (Daniel Lezcano 2013-06-12 15:08:55 +0200 631) int ret = -EBUSY;
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 632)
1b0a0e9a15b97 (Srivatsa S. Bhat 2012-05-04 14:06:02 -0700 633) if (!dev)
1b0a0e9a15b97 (Srivatsa S. Bhat 2012-05-04 14:06:02 -0700 634) return -EINVAL;
1b0a0e9a15b97 (Srivatsa S. Bhat 2012-05-04 14:06:02 -0700 635)
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 636) mutex_lock(&cpuidle_lock);
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 637)
c878a52d3c7ca (Daniel Lezcano 2013-06-12 15:08:55 +0200 638) if (dev->registered)
c878a52d3c7ca (Daniel Lezcano 2013-06-12 15:08:55 +0200 639) goto out_unlock;
c878a52d3c7ca (Daniel Lezcano 2013-06-12 15:08:55 +0200 640)
267d4bf8ee6db (Viresh Kumar 2013-10-03 21:26:43 +0530 641) __cpuidle_device_init(dev);
5df0aa7341bd9 (Daniel Lezcano 2013-06-12 15:08:54 +0200 642)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 643) ret = __cpuidle_register_device(dev);
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 644) if (ret)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 645) goto out_unlock;
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 646)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 647) ret = cpuidle_add_sysfs(dev);
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 648) if (ret)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 649) goto out_unregister;
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 650)
10b9d3f8a4d5c (Daniel Lezcano 2013-06-12 15:08:49 +0200 651) ret = cpuidle_enable_device(dev);
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 652) if (ret)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 653) goto out_sysfs;
10b9d3f8a4d5c (Daniel Lezcano 2013-06-12 15:08:49 +0200 654)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 655) cpuidle_install_idle_handler();
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 656)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 657) out_unlock:
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 658) mutex_unlock(&cpuidle_lock);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 659)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 660) return ret;
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 661)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 662) out_sysfs:
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 663) cpuidle_remove_sysfs(dev);
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 664) out_unregister:
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 665) __cpuidle_unregister_device(dev);
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 666) goto out_unlock;
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 667) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 668)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 669) EXPORT_SYMBOL_GPL(cpuidle_register_device);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 670)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 671) /**
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 672) * cpuidle_unregister_device - unregisters a CPU's idle PM feature
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 673) * @dev: the cpu
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 674) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 675) void cpuidle_unregister_device(struct cpuidle_device *dev)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 676) {
813e8e3d6aaa0 (Konrad Rzeszutek Wilk 2013-12-03 10:59:58 -0500 677) if (!dev || dev->registered == 0)
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 678) return;
dcb84f335bee9 (Venkatesh Pallipadi 2008-05-19 19:09:27 -0400 679)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 680) cpuidle_pause_and_lock();
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 681)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 682) cpuidle_disable_device(dev);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 683)
1aef40e288acf (Daniel Lezcano 2012-10-26 12:26:24 +0200 684) cpuidle_remove_sysfs(dev);
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 685)
f6bb51a53a753 (Daniel Lezcano 2013-06-12 15:08:53 +0200 686) __cpuidle_unregister_device(dev);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 687)
4126c0197bc8c (Colin Cross 2012-05-07 17:57:41 -0700 688) cpuidle_coupled_unregister_device(dev);
4126c0197bc8c (Colin Cross 2012-05-07 17:57:41 -0700 689)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 690) cpuidle_resume_and_unlock();
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 691) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 692)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 693) EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 694)
1c192d047a0dd (Daniel Lezcano 2013-04-23 15:28:44 +0000 695) /**
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 696) * cpuidle_unregister: unregister a driver and the devices. This function
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 697) * can be used only if the driver has been previously registered through
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 698) * the cpuidle_register function.
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 699) *
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 700) * @drv: a valid pointer to a struct cpuidle_driver
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 701) */
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 702) void cpuidle_unregister(struct cpuidle_driver *drv)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 703) {
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 704) int cpu;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 705) struct cpuidle_device *device;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 706)
82467a5a885dd (Daniel Lezcano 2013-06-07 21:53:09 +0000 707) for_each_cpu(cpu, drv->cpumask) {
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 708) device = &per_cpu(cpuidle_dev, cpu);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 709) cpuidle_unregister_device(device);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 710) }
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 711)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 712) cpuidle_unregister_driver(drv);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 713) }
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 714) EXPORT_SYMBOL_GPL(cpuidle_unregister);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 715)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 716) /**
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 717) * cpuidle_register: registers the driver and the cpu devices with the
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 718) * coupled_cpus passed as parameter. This function is used for all common
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 719) * initialization pattern there are in the arch specific drivers. The
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 720) * devices is globally defined in this file.
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 721) *
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 722) * @drv : a valid pointer to a struct cpuidle_driver
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 723) * @coupled_cpus: a cpumask for the coupled states
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 724) *
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 725) * Returns 0 on success, < 0 otherwise
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 726) */
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 727) int cpuidle_register(struct cpuidle_driver *drv,
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 728) const struct cpumask *const coupled_cpus)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 729) {
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 730) int ret, cpu;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 731) struct cpuidle_device *device;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 732)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 733) ret = cpuidle_register_driver(drv);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 734) if (ret) {
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 735) pr_err("failed to register cpuidle driver\n");
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 736) return ret;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 737) }
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 738)
82467a5a885dd (Daniel Lezcano 2013-06-07 21:53:09 +0000 739) for_each_cpu(cpu, drv->cpumask) {
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 740) device = &per_cpu(cpuidle_dev, cpu);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 741) device->cpu = cpu;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 742)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 743) #ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 744) /*
caf4a36e818ba (Viresh Kumar 2013-10-03 21:26:41 +0530 745) * On multiplatform for ARM, the coupled idle states could be
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 746) * enabled in the kernel even if the cpuidle driver does not
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 747) * use it. Note, coupled_cpus is a struct copy.
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 748) */
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 749) if (coupled_cpus)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 750) device->coupled_cpus = *coupled_cpus;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 751) #endif
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 752) ret = cpuidle_register_device(device);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 753) if (!ret)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 754) continue;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 755)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 756) pr_err("Failed to register cpuidle device for cpu%d\n", cpu);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 757)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 758) cpuidle_unregister(drv);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 759) break;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 760) }
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 761)
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 762) return ret;
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 763) }
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 764) EXPORT_SYMBOL_GPL(cpuidle_register);
4c637b2175a0d (Daniel Lezcano 2013-04-23 08:54:33 +0000 765)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 766) /**
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 767) * cpuidle_init - core initializer
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 768) */
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 769) static int __init cpuidle_init(void)
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 770) {
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 771) if (cpuidle_disabled())
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 772) return -ENODEV;
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 773)
3a4a0042228a8 (Rafael J. Wysocki 2020-02-12 00:02:30 +0100 774) return cpuidle_add_interface(cpu_subsys.dev_root);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 775) }
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 776)
62027aea23fcd (Len Brown 2011-04-01 18:13:10 -0400 777) module_param(off, int, 0444);
61cb5758d3c46 (Rafael J. Wysocki 2018-12-05 23:45:34 +0100 778) module_param_string(governor, param_governor, CPUIDLE_NAME_LEN, 0444);
4f86d3a8e2972 (Len Brown 2007-10-03 18:58:00 -0400 779) core_initcall(cpuidle_init);
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
33 Branches
55 Tags