author: Sebastian Andrzej Siewior <bigeasy@linutronix.de> 2021-09-28 14:24:30 +0200
committer: Minda Chen <minda.chen@starfivetech.com> 2023-11-06 19:24:43 +0800
commit: bd35209a783ad436104b51c4eff9757863fe17bd
parent: cb3f59da07e9d326f434e0e766306a385f818b80
Commit Summary:
Diffstat:
3 files changed, 14 insertions, 3 deletions
diff --git a/kernel/exit.c b/kernel/exit.c
index 63851320ae73..1d099609568d 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -172,6 +172,11 @@ static void delayed_put_task_struct(struct rcu_head *rhp)
kprobe_flush_task(tsk);
perf_event_delayed_put(tsk);
trace_sched_process_free(tsk);
+
+ /* RT enabled kernels delay freeing the VMAP'ed task stack */
+ if (IS_ENABLED(CONFIG_PREEMPT_RT))
+ put_task_stack(tsk);
+
put_task_struct(tsk);
}
diff --git a/kernel/fork.c b/kernel/fork.c
index 38681ad44c76..132ed526859a 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -289,7 +289,10 @@ static inline void free_thread_stack(struct task_struct *tsk)
return;
}
- vfree_atomic(tsk->stack);
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+ vfree_atomic(tsk->stack);
+ else
+ vfree(tsk->stack);
return;
}
#endif
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 886cf36bca5a..08134def09ea 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4845,8 +4845,12 @@ static struct rq *finish_task_switch(struct task_struct *prev)
if (prev->sched_class->task_dead)
prev->sched_class->task_dead(prev);
- /* Task is done with its stack. */
- put_task_stack(prev);
+ /*
+ * Release VMAP'ed task stack immediate for reuse. On RT
+ * enabled kernels this is delayed for latency reasons.
+ */
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+ put_task_stack(prev);
put_task_struct_rcu_user(prev);
}