cpufreq / sched: Pass flags to cpufreq_update_util()

It is useful to know the reason why cpufreq_update_util() has just
been called and that can be passed as flags to cpufreq_update_util()
and to the ->func() callback in struct update_util_data.  However,
doing that in addition to passing the util and max arguments they
already take would be clumsy, so avoid it.

Instead, use the observation that the schedutil governor is part
of the scheduler proper, so it can access scheduler data directly.
This allows the util and max arguments of cpufreq_update_util()
and the ->func() callback in struct update_util_data to be replaced
with a flags one, but schedutil has to be modified to follow.

Thus make the schedutil governor obtain the CFS utilization
information from the scheduler and use the "RT" and "DL" flags
instead of the special utilization value of ULONG_MAX to track
updates from the RT and DL sched classes.  Make it non-modular
too to avoid having to export scheduler variables to modules at
large.

Next, update all of the other users of cpufreq_update_util()
and the ->func() callback in struct update_util_data accordingly.

Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>

diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index 74919aa81dcb3311765899b976fab44d0ddeb5d2..4dc95250cf4eaf667d1d1de203807f8128011e8d 100644 (file)
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -194,7 +194,7 @@ config CPU_FREQ_GOV_CONSERVATIVE
          If in doubt, say N.
 
 config CPU_FREQ_GOV_SCHEDUTIL
-       tristate "'schedutil' cpufreq policy governor"
+       bool "'schedutil' cpufreq policy governor"
        depends on CPU_FREQ && SMP
        select CPU_FREQ_GOV_ATTR_SET
        select IRQ_WORK
@@ -208,9 +208,6 @@ config CPU_FREQ_GOV_SCHEDUTIL
          frequency tipping point is at utilization/capacity equal to 80% in
          both cases.
 
-         To compile this driver as a module, choose M here: the module will
-         be called cpufreq_schedutil.
-
          If in doubt, say N.
 
 comment "CPU frequency scaling drivers"

diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index e415349ab31b6334f177576e866331fc95264c3f..642dd0f183a8d90f8804b64fd39ffee95e6bdfa8 100644 (file)
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -260,7 +260,7 @@ static void dbs_irq_work(struct irq_work *irq_work)
 }
 
 static void dbs_update_util_handler(struct update_util_data *data, u64 time,
-                                   unsigned long util, unsigned long max)
+                                   unsigned int flags)
 {
        struct cpu_dbs_info *cdbs = container_of(data, struct cpu_dbs_info, update_util);
        struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;

diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index be9eade147f2335c056e0fc3c4297528330961b5..bdbe9369146bb7cc2eec9a7aee84dc5f3e9ae49d 100644 (file)
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -1329,7 +1329,7 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
 }
 
 static void intel_pstate_update_util(struct update_util_data *data, u64 time,
-                                    unsigned long util, unsigned long max)
+                                    unsigned int flags)
 {
        struct cpudata *cpu = container_of(data, struct cpudata, update_util);
        u64 delta_ns = time - cpu->sample.time;

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 62c68e513e391372b28c3bf49ded290634d94189..b0fa726b7f31a625cb25667e55cb4d919a2271d4 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -3469,15 +3469,19 @@ static inline unsigned long rlimit_max(unsigned int limit)
        return task_rlimit_max(current, limit);
 }
 
+#define SCHED_CPUFREQ_RT       (1U << 0)
+#define SCHED_CPUFREQ_DL       (1U << 1)
+
+#define SCHED_CPUFREQ_RT_DL    (SCHED_CPUFREQ_RT | SCHED_CPUFREQ_DL)
+
 #ifdef CONFIG_CPU_FREQ
 struct update_util_data {
-       void (*func)(struct update_util_data *data,
-                    u64 time, unsigned long util, unsigned long max);
+       void (*func)(struct update_util_data *data, u64 time, unsigned int flags);
 };
 
 void cpufreq_add_update_util_hook(int cpu, struct update_util_data *data,
-                       void (*func)(struct update_util_data *data, u64 time,
-                                    unsigned long util, unsigned long max));
+                       void (*func)(struct update_util_data *data, u64 time,
+                                   unsigned int flags));
 void cpufreq_remove_update_util_hook(int cpu);
 #endif /* CONFIG_CPU_FREQ */
 

diff --git a/kernel/sched/cpufreq.c b/kernel/sched/cpufreq.c
index 1141954e73b46cf1104e179e6ec1a7ace530d87e..dbc51442ecbc94f9e24e68fb5a08dd2c8515d00d 100644 (file)
--- a/kernel/sched/cpufreq.c
+++ b/kernel/sched/cpufreq.c
@@ -33,7 +33,7 @@ DEFINE_PER_CPU(struct update_util_data *, cpufreq_update_util_data);
  */
 void cpufreq_add_update_util_hook(int cpu, struct update_util_data *data,
                        void (*func)(struct update_util_data *data, u64 time,
-                                    unsigned long util, unsigned long max))
+                                    unsigned int flags))
 {
        if (WARN_ON(!data || !func))
                return;

diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index a84641b222c1b4769fbceec1bf16236cd5abc010..60d985f4dc47ee960ac18526da5c70f7f1b4b249 100644 (file)
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -12,7 +12,6 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/cpufreq.h>
-#include <linux/module.h>
 #include <linux/slab.h>
 #include <trace/events/power.h>
 
@@ -53,6 +52,7 @@ struct sugov_cpu {
        unsigned long util;
        unsigned long max;
        u64 last_update;
+       unsigned int flags;
 };
 
 static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);
@@ -144,24 +144,39 @@ static unsigned int get_next_freq(struct sugov_cpu *sg_cpu, unsigned long util,
        return cpufreq_driver_resolve_freq(policy, freq);
 }
 
+static void sugov_get_util(unsigned long *util, unsigned long *max)
+{
+       struct rq *rq = this_rq();
+       unsigned long cfs_max = rq->cpu_capacity_orig;
+
+       *util = min(rq->cfs.avg.util_avg, cfs_max);
+       *max = cfs_max;
+}
+
 static void sugov_update_single(struct update_util_data *hook, u64 time,
-                               unsigned long util, unsigned long max)
+                               unsigned int flags)
 {
        struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
        struct sugov_policy *sg_policy = sg_cpu->sg_policy;
        struct cpufreq_policy *policy = sg_policy->policy;
+       unsigned long util, max;
        unsigned int next_f;
 
        if (!sugov_should_update_freq(sg_policy, time))
                return;
 
-       next_f = util == ULONG_MAX ? policy->cpuinfo.max_freq :
-                       get_next_freq(sg_cpu, util, max);
+       if (flags & SCHED_CPUFREQ_RT_DL) {
+               next_f = policy->cpuinfo.max_freq;
+       } else {
+               sugov_get_util(&util, &max);
+               next_f = get_next_freq(sg_cpu, util, max);
+       }
        sugov_update_commit(sg_policy, time, next_f);
 }
 
 static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
-                                          unsigned long util, unsigned long max)
+                                          unsigned long util, unsigned long max,
+                                          unsigned int flags)
 {
        struct sugov_policy *sg_policy = sg_cpu->sg_policy;
        struct cpufreq_policy *policy = sg_policy->policy;
@@ -169,7 +184,7 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
        u64 last_freq_update_time = sg_policy->last_freq_update_time;
        unsigned int j;
 
-       if (util == ULONG_MAX)
+       if (flags & SCHED_CPUFREQ_RT_DL)
                return max_f;
 
        for_each_cpu(j, policy->cpus) {
@@ -192,10 +207,10 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
                if (delta_ns > TICK_NSEC)
                        continue;
 
-               j_util = j_sg_cpu->util;
-               if (j_util == ULONG_MAX)
+               if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL)
                        return max_f;
 
+               j_util = j_sg_cpu->util;
                j_max = j_sg_cpu->max;
                if (j_util * max > j_max * util) {
                        util = j_util;
@@ -207,20 +222,24 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
 }
 
 static void sugov_update_shared(struct update_util_data *hook, u64 time,
-                               unsigned long util, unsigned long max)
+                               unsigned int flags)
 {
        struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
        struct sugov_policy *sg_policy = sg_cpu->sg_policy;
+       unsigned long util, max;
        unsigned int next_f;
 
+       sugov_get_util(&util, &max);
+
        raw_spin_lock(&sg_policy->update_lock);
 
        sg_cpu->util = util;
        sg_cpu->max = max;
+       sg_cpu->flags = flags;
        sg_cpu->last_update = time;
 
        if (sugov_should_update_freq(sg_policy, time)) {
-               next_f = sugov_next_freq_shared(sg_cpu, util, max);
+               next_f = sugov_next_freq_shared(sg_cpu, util, max, flags);
                sugov_update_commit(sg_policy, time, next_f);
        }
 
@@ -444,8 +463,9 @@ static int sugov_start(struct cpufreq_policy *policy)
 
                sg_cpu->sg_policy = sg_policy;
                if (policy_is_shared(policy)) {
-                       sg_cpu->util = ULONG_MAX;
+                       sg_cpu->util = 0;
                        sg_cpu->max = 0;
+                       sg_cpu->flags = SCHED_CPUFREQ_RT;
                        sg_cpu->last_update = 0;
                        sg_cpu->cached_raw_freq = 0;
                        cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
@@ -495,28 +515,15 @@ static struct cpufreq_governor schedutil_gov = {
        .limits = sugov_limits,
 };
 
-static int __init sugov_module_init(void)
-{
-       return cpufreq_register_governor(&schedutil_gov);
-}
-
-static void __exit sugov_module_exit(void)
-{
-       cpufreq_unregister_governor(&schedutil_gov);
-}
-
-MODULE_AUTHOR("Rafael J. Wysocki <rafael.j.wysocki@intel.com>");
-MODULE_DESCRIPTION("Utilization-based CPU frequency selection");
-MODULE_LICENSE("GPL");
-
 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL
 struct cpufreq_governor *cpufreq_default_governor(void)
 {
        return &schedutil_gov;
 }
-
-fs_initcall(sugov_module_init);
-#else
-module_init(sugov_module_init);
 #endif
-module_exit(sugov_module_exit);
+
+static int __init sugov_register(void)
+{
+       return cpufreq_register_governor(&schedutil_gov);
+}
+fs_initcall(sugov_register);

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 1ce8867283dcde6e35ef74a72a1bca968decb918..4464cc3e4f3d29e0135eb3c87391b9a7dff3ac20 100644 (file)
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -735,9 +735,9 @@ static void update_curr_dl(struct rq *rq)
                return;
        }
 
-       /* kick cpufreq (see the comment in linux/cpufreq.h). */
+       /* kick cpufreq (see the comment in kernel/sched/sched.h). */
        if (cpu_of(rq) == smp_processor_id())
-               cpufreq_trigger_update(rq_clock(rq));
+               cpufreq_update_util(rq_clock(rq), SCHED_CPUFREQ_DL);
 
        schedstat_set(curr->se.statistics.exec_max,
                      max(curr->se.statistics.exec_max, delta_exec));

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 039de34f15216d19f61386b6d6c66744660516c9..f91fa5796e502f2606fa9c48a92f26e8825ab4a4 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2875,11 +2875,8 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {}
 
 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq)
 {
-       struct rq *rq = rq_of(cfs_rq);
-       int cpu = cpu_of(rq);
-
-       if (cpu == smp_processor_id() && &rq->cfs == cfs_rq) {
-               unsigned long max = rq->cpu_capacity_orig;
+       if (&this_rq()->cfs == cfs_rq) {
+               struct rq *rq = rq_of(cfs_rq);
 
                /*
                 * There are a few boundary cases this might miss but it should
@@ -2897,8 +2894,7 @@ static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq)
                 *
                 * See cpu_util().
                 */
-               cpufreq_update_util(rq_clock(rq),
-                                   min(cfs_rq->avg.util_avg, max), max);
+               cpufreq_update_util(rq_clock(rq), 0);
        }
 }
 
@@ -3162,7 +3158,7 @@ static inline void update_load_avg(struct sched_entity *se, int not_used)
        struct cfs_rq *cfs_rq = cfs_rq_of(se);
        struct rq *rq = rq_of(cfs_rq);
 
-       cpufreq_trigger_update(rq_clock(rq));
+       cpufreq_update_util(rq_clock(rq), 0);
 }
 
 static inline void

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index d5690b722691b4f7457492eb11d1d9bea7ba7dbe..8a9cd9ba5153a271c5aefb55e08c96990b9c32b1 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -957,9 +957,9 @@ static void update_curr_rt(struct rq *rq)
        if (unlikely((s64)delta_exec <= 0))
                return;
 
-       /* Kick cpufreq (see the comment in linux/cpufreq.h). */
+       /* Kick cpufreq (see the comment in kernel/sched/sched.h). */
        if (cpu_of(rq) == smp_processor_id())
-               cpufreq_trigger_update(rq_clock(rq));
+               cpufreq_update_util(rq_clock(rq), SCHED_CPUFREQ_RT);
 
        schedstat_set(curr->se.statistics.exec_max,
                      max(curr->se.statistics.exec_max, delta_exec));

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index c64fc5114004f6a893a1bf942a49f3df1d6c8fea..82fc5542708c34d53021d8dd40a46535b4c8f42b 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1764,26 +1764,12 @@ DECLARE_PER_CPU(struct update_util_data *, cpufreq_update_util_data);
 /**
  * cpufreq_update_util - Take a note about CPU utilization changes.
  * @time: Current time.
- * @util: Current utilization.
- * @max: Utilization ceiling.
+ * @flags: Update reason flags.
  *
- * This function is called by the scheduler on every invocation of
- * update_load_avg() on the CPU whose utilization is being updated.
+ * This function is called by the scheduler on the CPU whose utilization is
+ * being updated.
  *
  * It can only be called from RCU-sched read-side critical sections.
- */
-static inline void cpufreq_update_util(u64 time, unsigned long util, unsigned long max)
-{
-       struct update_util_data *data;
-
-       data = rcu_dereference_sched(*this_cpu_ptr(&cpufreq_update_util_data));
-       if (data)
-               data->func(data, time, util, max);
-}
-
-/**
- * cpufreq_trigger_update - Trigger CPU performance state evaluation if needed.
- * @time: Current time.
  *
  * The way cpufreq is currently arranged requires it to evaluate the CPU
  * performance state (frequency/voltage) on a regular basis to prevent it from
@@ -1797,13 +1783,16 @@ static inline void cpufreq_update_util(u64 time, unsigned long util, unsigned lo
  * but that really is a band-aid.  Going forward it should be replaced with
  * solutions targeted more specifically at RT and DL tasks.
  */
-static inline void cpufreq_trigger_update(u64 time)
+static inline void cpufreq_update_util(u64 time, unsigned int flags)
 {
-       cpufreq_update_util(time, ULONG_MAX, 0);
+       struct update_util_data *data;
+
+       data = rcu_dereference_sched(*this_cpu_ptr(&cpufreq_update_util_data));
+       if (data)
+               data->func(data, time, flags);
 }
 #else
-static inline void cpufreq_update_util(u64 time, unsigned long util, unsigned long max) {}
-static inline void cpufreq_trigger_update(u64 time) {}
+static inline void cpufreq_update_util(u64 time, unsigned int flags) {}
 #endif /* CONFIG_CPU_FREQ */
 
 #ifdef arch_scale_freq_capacity