[PATCH v3 4/7] sched: Enable sched_feat callbacks on enable/disable

From: David Vernet
Date: Wed Aug 09 2023 - 18:13:05 EST


When a scheduler feature is enabled or disabled, the sched_feat_enable()
and sched_feat_disable() functions are invoked respectively for that
feature. For features that don't require resetting any state, this works
fine. However, there will be an upcoming feature called shared_runq
which needs to drain all tasks from a set of global shared runqueues in
order to avoid stale tasks from staying in the queues after the feature
has been disabled.

This patch therefore defines a new SCHED_FEAT_CALLBACK macro which
allows scheduler features to specify a callback that should be invoked
when a feature is enabled or disabled respectively. The SCHED_FEAT macro
assumes a NULL callback.

Signed-off-by: David Vernet <void@xxxxxxxxxxxxx>
---
kernel/sched/core.c | 4 ++--
kernel/sched/debug.c | 18 ++++++++++++++----
kernel/sched/sched.h | 16 ++++++++++------
3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index dd6412a49263..385c565da87f 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -124,12 +124,12 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
* sysctl_sched_features, defined in sched.h, to allow constants propagation
* at compile time and compiler optimization based on features default.
*/
-#define SCHED_FEAT(name, enabled) \
+#define SCHED_FEAT_CALLBACK(name, enabled, cb) \
(1UL << __SCHED_FEAT_##name) * enabled |
const_debug unsigned int sysctl_sched_features =
#include "features.h"
0;
-#undef SCHED_FEAT
+#undef SCHED_FEAT_CALLBACK

/*
* Print a warning if need_resched is set for the given duration (if
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index aeeba46a096b..803dff75c56f 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -44,14 +44,14 @@ static unsigned long nsec_low(unsigned long long nsec)

#define SPLIT_NS(x) nsec_high(x), nsec_low(x)

-#define SCHED_FEAT(name, enabled) \
+#define SCHED_FEAT_CALLBACK(name, enabled, cb) \
#name ,

static const char * const sched_feat_names[] = {
#include "features.h"
};

-#undef SCHED_FEAT
+#undef SCHED_FEAT_CALLBACK

static int sched_feat_show(struct seq_file *m, void *v)
{
@@ -72,22 +72,32 @@ static int sched_feat_show(struct seq_file *m, void *v)
#define jump_label_key__true STATIC_KEY_INIT_TRUE
#define jump_label_key__false STATIC_KEY_INIT_FALSE

-#define SCHED_FEAT(name, enabled) \
+#define SCHED_FEAT_CALLBACK(name, enabled, cb) \
jump_label_key__##enabled ,

struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
#include "features.h"
};

-#undef SCHED_FEAT
+#undef SCHED_FEAT_CALLBACK
+
+#define SCHED_FEAT_CALLBACK(name, enabled, cb) cb,
+static const sched_feat_change_f sched_feat_cbs[__SCHED_FEAT_NR] = {
+#include "features.h"
+};
+#undef SCHED_FEAT_CALLBACK

static void sched_feat_disable(int i)
{
+ if (sched_feat_cbs[i])
+ sched_feat_cbs[i](false);
static_key_disable_cpuslocked(&sched_feat_keys[i]);
}

static void sched_feat_enable(int i)
{
+ if (sched_feat_cbs[i])
+ sched_feat_cbs[i](true);
static_key_enable_cpuslocked(&sched_feat_keys[i]);
}
#else
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 88cca7cc00cf..2631da3c8a4d 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2065,6 +2065,8 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
#endif
}

+#define SCHED_FEAT(name, enabled) SCHED_FEAT_CALLBACK(name, enabled, NULL)
+
/*
* Tunables that become constants when CONFIG_SCHED_DEBUG is off:
*/
@@ -2074,7 +2076,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
# define const_debug const
#endif

-#define SCHED_FEAT(name, enabled) \
+#define SCHED_FEAT_CALLBACK(name, enabled, cb) \
__SCHED_FEAT_##name ,

enum {
@@ -2082,7 +2084,7 @@ enum {
__SCHED_FEAT_NR,
};

-#undef SCHED_FEAT
+#undef SCHED_FEAT_CALLBACK

#ifdef CONFIG_SCHED_DEBUG

@@ -2093,14 +2095,14 @@ enum {
extern const_debug unsigned int sysctl_sched_features;

#ifdef CONFIG_JUMP_LABEL
-#define SCHED_FEAT(name, enabled) \
+#define SCHED_FEAT_CALLBACK(name, enabled, cb) \
static __always_inline bool static_branch_##name(struct static_key *key) \
{ \
return static_key_##enabled(key); \
}

#include "features.h"
-#undef SCHED_FEAT
+#undef SCHED_FEAT_CALLBACK

extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
@@ -2118,17 +2120,19 @@ extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
* constants propagation at compile time and compiler optimization based on
* features default.
*/
-#define SCHED_FEAT(name, enabled) \
+#define SCHED_FEAT_CALLBACK(name, enabled, cb) \
(1UL << __SCHED_FEAT_##name) * enabled |
static const_debug __maybe_unused unsigned int sysctl_sched_features =
#include "features.h"
0;
-#undef SCHED_FEAT
+#undef SCHED_FEAT_CALLBACK

#define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x))

#endif /* SCHED_DEBUG */

+typedef void (*sched_feat_change_f)(bool enabling);
+
extern struct static_key_false sched_numa_balancing;
extern struct static_key_false sched_schedstats;

--
2.41.0