ondemand: Add support to consider other cpu load when scaling frequencies
Currently only current cpu load is considered when frequency scaling.
As a side effect of this when scheduler migrates a thread to a slower
core, the execution time increases unexpectedly.
The new algorithm not only considers current cpu load for frequency
scaling, but also takes into account load of other online cpus. The
treshold for this can be controlled using up_threshold_any_cpu_load.
When the up_threshold_any_cpu_load is crossed in any cpu then the
current cpu is set to sync_freq.
Also, when more than one cpu is online a two step ramp-up mechanism
is used. When the up_threshold_multi_core is crossed the cpu is ramped
up to optimal_freq which is less than policy->max and set to a power
friendly value.
Change-Id: I64bed1b142cde1f239e73aa1f78981c78aaf59ce
Signed-off-by: Narayanan Gopalakrishnan <nargop@codeaurora.org>
(cherry picked from commit d3ca78370df00ec7a5b6816321ea1a1a371f2850)
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index 785ba6c..6bab8db 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -92,6 +92,8 @@
unsigned int freq_lo_jiffies;
unsigned int freq_hi_jiffies;
unsigned int rate_mult;
+ unsigned int prev_load;
+ unsigned int max_load;
int cpu;
unsigned int sample_type:1;
/*
@@ -120,17 +122,27 @@
static struct dbs_tuners {
unsigned int sampling_rate;
unsigned int up_threshold;
+ unsigned int up_threshold_multi_core;
unsigned int down_differential;
+ unsigned int down_differential_multi_core;
+ unsigned int optimal_freq;
+ unsigned int up_threshold_any_cpu_load;
+ unsigned int sync_freq;
unsigned int ignore_nice;
unsigned int sampling_down_factor;
int powersave_bias;
unsigned int io_is_busy;
} dbs_tuners_ins = {
+ .up_threshold_multi_core = DEF_FREQUENCY_UP_THRESHOLD,
.up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
.sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR,
.down_differential = DEF_FREQUENCY_DOWN_DIFFERENTIAL,
+ .down_differential_multi_core = MICRO_FREQUENCY_DOWN_DIFFERENTIAL,
+ .up_threshold_any_cpu_load = DEF_FREQUENCY_UP_THRESHOLD,
.ignore_nice = 0,
.powersave_bias = 0,
+ .sync_freq = 0,
+ .optimal_freq = 0,
};
static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
@@ -288,9 +300,13 @@
show_one(sampling_rate, sampling_rate);
show_one(io_is_busy, io_is_busy);
show_one(up_threshold, up_threshold);
+show_one(up_threshold_multi_core, up_threshold_multi_core);
show_one(down_differential, down_differential);
show_one(sampling_down_factor, sampling_down_factor);
show_one(ignore_nice_load, ignore_nice);
+show_one(optimal_freq, optimal_freq);
+show_one(up_threshold_any_cpu_load, up_threshold_any_cpu_load);
+show_one(sync_freq, sync_freq);
static ssize_t show_powersave_bias
(struct kobject *kobj, struct attribute *attr, char *buf)
@@ -366,6 +382,19 @@
return count;
}
+static ssize_t store_sync_freq(struct kobject *a, struct attribute *b,
+ const char *buf, size_t count)
+{
+ unsigned int input;
+ int ret;
+
+ ret = sscanf(buf, "%u", &input);
+ if (ret != 1)
+ return -EINVAL;
+ dbs_tuners_ins.sync_freq = input;
+ return count;
+}
+
static ssize_t store_io_is_busy(struct kobject *a, struct attribute *b,
const char *buf, size_t count)
{
@@ -379,6 +408,19 @@
return count;
}
+static ssize_t store_optimal_freq(struct kobject *a, struct attribute *b,
+ const char *buf, size_t count)
+{
+ unsigned int input;
+ int ret;
+
+ ret = sscanf(buf, "%u", &input);
+ if (ret != 1)
+ return -EINVAL;
+ dbs_tuners_ins.optimal_freq = input;
+ return count;
+}
+
static ssize_t store_up_threshold(struct kobject *a, struct attribute *b,
const char *buf, size_t count)
{
@@ -394,6 +436,36 @@
return count;
}
+static ssize_t store_up_threshold_multi_core(struct kobject *a,
+ struct attribute *b, const char *buf, size_t count)
+{
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
+ input < MIN_FREQUENCY_UP_THRESHOLD) {
+ return -EINVAL;
+ }
+ dbs_tuners_ins.up_threshold_multi_core = input;
+ return count;
+}
+
+static ssize_t store_up_threshold_any_cpu_load(struct kobject *a,
+ struct attribute *b, const char *buf, size_t count)
+{
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
+ input < MIN_FREQUENCY_UP_THRESHOLD) {
+ return -EINVAL;
+ }
+ dbs_tuners_ins.up_threshold_any_cpu_load = input;
+ return count;
+}
+
static ssize_t store_down_differential(struct kobject *a, struct attribute *b,
const char *buf, size_t count)
{
@@ -576,6 +648,10 @@
define_one_global_rw(sampling_down_factor);
define_one_global_rw(ignore_nice_load);
define_one_global_rw(powersave_bias);
+define_one_global_rw(up_threshold_multi_core);
+define_one_global_rw(optimal_freq);
+define_one_global_rw(up_threshold_any_cpu_load);
+define_one_global_rw(sync_freq);
static struct attribute *dbs_attributes[] = {
&sampling_rate_min.attr,
@@ -586,6 +662,10 @@
&ignore_nice_load.attr,
&powersave_bias.attr,
&io_is_busy.attr,
+ &up_threshold_multi_core.attr,
+ &optimal_freq.attr,
+ &up_threshold_any_cpu_load.attr,
+ &sync_freq.attr,
NULL
};
@@ -614,7 +694,7 @@
unsigned int max_load_freq;
/* Current load across this CPU */
unsigned int cur_load = 0;
-
+ unsigned int max_load_other_cpu = 0;
struct cpufreq_policy *policy;
unsigned int j;
@@ -691,7 +771,8 @@
continue;
cur_load = 100 * (wall_time - idle_time) / wall_time;
-
+ j_dbs_info->max_load = max(cur_load, j_dbs_info->prev_load);
+ j_dbs_info->prev_load = cur_load;
freq_avg = __cpufreq_driver_getavg(policy, j);
if (freq_avg <= 0)
freq_avg = policy->cur;
@@ -700,11 +781,37 @@
if (load_freq > max_load_freq)
max_load_freq = load_freq;
}
+
+ for_each_online_cpu(j) {
+ struct cpu_dbs_info_s *j_dbs_info;
+ j_dbs_info = &per_cpu(od_cpu_dbs_info, j);
+
+ if (j == policy->cpu)
+ continue;
+
+ if (max_load_other_cpu < j_dbs_info->max_load)
+ max_load_other_cpu = j_dbs_info->max_load;
+ /*
+ * The other cpu could be running at higher frequency
+ * but may not have completed it's sampling_down_factor.
+ * For that case consider other cpu is loaded so that
+ * frequency imbalance does not occur.
+ */
+
+ if ((j_dbs_info->cur_policy != NULL)
+ && (j_dbs_info->cur_policy->cur ==
+ j_dbs_info->cur_policy->max)) {
+
+ if (policy->cur >= dbs_tuners_ins.optimal_freq)
+ max_load_other_cpu =
+ dbs_tuners_ins.up_threshold_any_cpu_load;
+ }
+ }
+
/* calculate the scaled load across CPU */
load_at_max_freq = (cur_load * policy->cur)/policy->cpuinfo.max_freq;
cpufreq_notify_utilization(policy, load_at_max_freq);
-
/* Check for frequency increase */
if (max_load_freq > dbs_tuners_ins.up_threshold * policy->cur) {
/* If switching to max speed, apply sampling_down_factor */
@@ -715,6 +822,25 @@
return;
}
+ if (num_online_cpus() > 1) {
+
+ if (max_load_other_cpu >
+ dbs_tuners_ins.up_threshold_any_cpu_load) {
+ if (policy->cur < dbs_tuners_ins.sync_freq)
+ dbs_freq_increase(policy,
+ dbs_tuners_ins.sync_freq);
+ return;
+ }
+
+ if (max_load_freq > dbs_tuners_ins.up_threshold_multi_core *
+ policy->cur) {
+ if (policy->cur < dbs_tuners_ins.optimal_freq)
+ dbs_freq_increase(policy,
+ dbs_tuners_ins.optimal_freq);
+ return;
+ }
+ }
+
/* Check for frequency decrease */
/* if we cannot reduce the frequency anymore, break out early */
if (policy->cur == policy->min)
@@ -739,6 +865,20 @@
if (freq_next < policy->min)
freq_next = policy->min;
+ if (num_online_cpus() > 1) {
+ if (max_load_other_cpu >
+ (dbs_tuners_ins.up_threshold_multi_core -
+ dbs_tuners_ins.down_differential) &&
+ freq_next < dbs_tuners_ins.sync_freq)
+ freq_next = dbs_tuners_ins.sync_freq;
+
+ if (max_load_freq >
+ (dbs_tuners_ins.up_threshold_multi_core -
+ dbs_tuners_ins.down_differential_multi_core) *
+ policy->cur)
+ freq_next = dbs_tuners_ins.optimal_freq;
+
+ }
if (!dbs_tuners_ins.powersave_bias) {
__cpufreq_driver_target(policy, freq_next,
CPUFREQ_RELATION_L);
@@ -989,6 +1129,12 @@
max(min_sampling_rate,
latency * LATENCY_MULTIPLIER);
dbs_tuners_ins.io_is_busy = should_io_be_busy();
+
+ if (dbs_tuners_ins.optimal_freq == 0)
+ dbs_tuners_ins.optimal_freq = policy->min;
+
+ if (dbs_tuners_ins.sync_freq == 0)
+ dbs_tuners_ins.sync_freq = policy->min;
}
if (!cpu)
rc = input_register_handler(&dbs_input_handler);