blob: 6a2b036c93894fa3307dd4aebf8505e30c1178ea [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * drivers/cpufreq/cpufreq_ondemand.c
3 *
4 * Copyright (C) 2001 Russell King
5 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
6 * Jun Nakajima <jun.nakajima@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/kernel.h>
14#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/cpufreq.h>
Andrew Morton138a01282006-06-23 03:31:19 -070017#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/jiffies.h>
19#include <linux/kernel_stat.h>
akpm@osdl.org3fc54d32006-01-13 15:54:22 -080020#include <linux/mutex.h>
venkatesh.pallipadi@intel.com80800912008-08-04 11:59:12 -070021#include <linux/hrtimer.h>
22#include <linux/tick.h>
23#include <linux/ktime.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
25/*
26 * dbs is used in this file as a shortform for demandbased switching
27 * It helps to keep variable names smaller, simpler
28 */
29
venkatesh.pallipadi@intel.come9d95bf2008-08-04 11:59:10 -070030#define DEF_FREQUENCY_DOWN_DIFFERENTIAL (10)
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#define DEF_FREQUENCY_UP_THRESHOLD (80)
venkatesh.pallipadi@intel.com80800912008-08-04 11:59:12 -070032#define MICRO_FREQUENCY_DOWN_DIFFERENTIAL (3)
33#define MICRO_FREQUENCY_UP_THRESHOLD (95)
Dave Jonesc29f1402005-05-31 19:03:50 -070034#define MIN_FREQUENCY_UP_THRESHOLD (11)
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#define MAX_FREQUENCY_UP_THRESHOLD (100)
36
Dave Jones32ee8c32006-02-28 00:43:23 -050037/*
38 * The polling frequency of this governor depends on the capability of
Linus Torvalds1da177e2005-04-16 15:20:36 -070039 * the processor. Default polling frequency is 1000 times the transition
Dave Jones32ee8c32006-02-28 00:43:23 -050040 * latency of the processor. The governor will work on any processor with
41 * transition latency <= 10mS, using appropriate sampling
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 * rate.
43 * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
44 * this governor will not work.
45 * All times here are in uS.
46 */
Dave Jones32ee8c32006-02-28 00:43:23 -050047static unsigned int def_sampling_rate;
Dave Jonesdf8b59b2005-09-20 12:39:35 -070048#define MIN_SAMPLING_RATE_RATIO (2)
49/* for correct statistics, we need at least 10 ticks between each measure */
Gautham R Shenoye08f5f52006-10-26 16:20:58 +053050#define MIN_STAT_SAMPLING_RATE \
51 (MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10))
52#define MIN_SAMPLING_RATE \
53 (def_sampling_rate / MIN_SAMPLING_RATE_RATIO)
Linus Torvalds1da177e2005-04-16 15:20:36 -070054#define MAX_SAMPLING_RATE (500 * def_sampling_rate)
55#define DEF_SAMPLING_RATE_LATENCY_MULTIPLIER (1000)
Thomas Renninger1c256242007-10-02 13:28:12 -070056#define TRANSITION_LATENCY_LIMIT (10 * 1000 * 1000)
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
David Howellsc4028952006-11-22 14:57:56 +000058static void do_dbs_timer(struct work_struct *work);
59
60/* Sampling types */
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -080061enum {DBS_NORMAL_SAMPLE, DBS_SUB_SAMPLE};
Linus Torvalds1da177e2005-04-16 15:20:36 -070062
63struct cpu_dbs_info_s {
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -070064 cputime64_t prev_cpu_idle;
65 cputime64_t prev_cpu_wall;
venkatesh.pallipadi@intel.com80800912008-08-04 11:59:12 -070066 cputime64_t prev_cpu_nice;
Dave Jones32ee8c32006-02-28 00:43:23 -050067 struct cpufreq_policy *cur_policy;
David Howellsc4028952006-11-22 14:57:56 +000068 struct delayed_work work;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +040069 struct cpufreq_frequency_table *freq_table;
70 unsigned int freq_lo;
71 unsigned int freq_lo_jiffies;
72 unsigned int freq_hi_jiffies;
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -080073 int cpu;
74 unsigned int enable:1,
75 sample_type:1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070076};
77static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);
78
79static unsigned int dbs_enable; /* number of CPUs using this policy */
80
Venkatesh Pallipadi4ec223d2006-06-21 15:18:34 -070081/*
82 * DEADLOCK ALERT! There is a ordering requirement between cpu_hotplug
83 * lock and dbs_mutex. cpu_hotplug lock should always be held before
84 * dbs_mutex. If any function that can potentially take cpu_hotplug lock
85 * (like __cpufreq_driver_target()) is being called with dbs_mutex taken, then
86 * cpu_hotplug lock should be taken before that. Note that cpu_hotplug lock
87 * is recursive for the same process. -Venki
88 */
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -070089static DEFINE_MUTEX(dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -070090
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -070091static struct workqueue_struct *kondemand_wq;
Andi Kleen6810b542006-05-08 15:17:31 +020092
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +040093static struct dbs_tuners {
Dave Jones32ee8c32006-02-28 00:43:23 -050094 unsigned int sampling_rate;
Dave Jones32ee8c32006-02-28 00:43:23 -050095 unsigned int up_threshold;
venkatesh.pallipadi@intel.come9d95bf2008-08-04 11:59:10 -070096 unsigned int down_differential;
Dave Jones32ee8c32006-02-28 00:43:23 -050097 unsigned int ignore_nice;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +040098 unsigned int powersave_bias;
99} dbs_tuners_ins = {
Dave Jones32ee8c32006-02-28 00:43:23 -0500100 .up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
venkatesh.pallipadi@intel.come9d95bf2008-08-04 11:59:10 -0700101 .down_differential = DEF_FREQUENCY_DOWN_DIFFERENTIAL,
Eric Piel9cbad612006-03-10 11:35:27 +0200102 .ignore_nice = 0,
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400103 .powersave_bias = 0,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104};
105
venkatesh.pallipadi@intel.com80800912008-08-04 11:59:12 -0700106static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
107 cputime64_t *wall)
Dave Jonesdac1c1a2005-05-31 19:03:49 -0700108{
Venki Pallipadiea487612007-06-20 14:26:24 -0700109 cputime64_t idle_time;
venkatesh.pallipadi@intel.com34305022008-08-04 11:59:09 -0700110 cputime64_t cur_wall_time;
Venki Pallipadiea487612007-06-20 14:26:24 -0700111 cputime64_t busy_time;
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700112
venkatesh.pallipadi@intel.com34305022008-08-04 11:59:09 -0700113 cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
Venki Pallipadiea487612007-06-20 14:26:24 -0700114 busy_time = cputime64_add(kstat_cpu(cpu).cpustat.user,
115 kstat_cpu(cpu).cpustat.system);
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700116
Venki Pallipadiea487612007-06-20 14:26:24 -0700117 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.irq);
118 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.softirq);
119 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.steal);
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700120
Venki Pallipadiea487612007-06-20 14:26:24 -0700121 if (!dbs_tuners_ins.ignore_nice) {
122 busy_time = cputime64_add(busy_time,
123 kstat_cpu(cpu).cpustat.nice);
124 }
125
venkatesh.pallipadi@intel.com34305022008-08-04 11:59:09 -0700126 idle_time = cputime64_sub(cur_wall_time, busy_time);
127 if (wall)
128 *wall = cur_wall_time;
129
Venki Pallipadiea487612007-06-20 14:26:24 -0700130 return idle_time;
Dave Jonesdac1c1a2005-05-31 19:03:49 -0700131}
132
venkatesh.pallipadi@intel.com80800912008-08-04 11:59:12 -0700133static inline cputime64_t get_cpu_idle_time(unsigned int cpu, cputime64_t *wall)
134{
135 u64 idle_time = get_cpu_idle_time_us(cpu, wall);
136
137 if (idle_time == -1ULL)
138 return get_cpu_idle_time_jiffy(cpu, wall);
139
140 if (dbs_tuners_ins.ignore_nice) {
141 cputime64_t cur_nice;
142 unsigned long cur_nice_jiffies;
143 struct cpu_dbs_info_s *dbs_info;
144
145 dbs_info = &per_cpu(cpu_dbs_info, cpu);
146 cur_nice = cputime64_sub(kstat_cpu(cpu).cpustat.nice,
147 dbs_info->prev_cpu_nice);
148 /*
149 * Assumption: nice time between sampling periods will be
150 * less than 2^32 jiffies for 32 bit sys
151 */
152 cur_nice_jiffies = (unsigned long)
153 cputime64_to_jiffies64(cur_nice);
154 dbs_info->prev_cpu_nice = kstat_cpu(cpu).cpustat.nice;
155 return idle_time + jiffies_to_usecs(cur_nice_jiffies);
156 }
157 return idle_time;
158}
159
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400160/*
161 * Find right freq to be set now with powersave_bias on.
162 * Returns the freq_hi to be used right now and will set freq_hi_jiffies,
163 * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs.
164 */
Adrian Bunkb5ecf602006-08-13 23:00:08 +0200165static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
166 unsigned int freq_next,
167 unsigned int relation)
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400168{
169 unsigned int freq_req, freq_reduc, freq_avg;
170 unsigned int freq_hi, freq_lo;
171 unsigned int index = 0;
172 unsigned int jiffies_total, jiffies_hi, jiffies_lo;
173 struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, policy->cpu);
174
175 if (!dbs_info->freq_table) {
176 dbs_info->freq_lo = 0;
177 dbs_info->freq_lo_jiffies = 0;
178 return freq_next;
179 }
180
181 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
182 relation, &index);
183 freq_req = dbs_info->freq_table[index].frequency;
184 freq_reduc = freq_req * dbs_tuners_ins.powersave_bias / 1000;
185 freq_avg = freq_req - freq_reduc;
186
187 /* Find freq bounds for freq_avg in freq_table */
188 index = 0;
189 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
190 CPUFREQ_RELATION_H, &index);
191 freq_lo = dbs_info->freq_table[index].frequency;
192 index = 0;
193 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
194 CPUFREQ_RELATION_L, &index);
195 freq_hi = dbs_info->freq_table[index].frequency;
196
197 /* Find out how long we have to be in hi and lo freqs */
198 if (freq_hi == freq_lo) {
199 dbs_info->freq_lo = 0;
200 dbs_info->freq_lo_jiffies = 0;
201 return freq_lo;
202 }
203 jiffies_total = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
204 jiffies_hi = (freq_avg - freq_lo) * jiffies_total;
205 jiffies_hi += ((freq_hi - freq_lo) / 2);
206 jiffies_hi /= (freq_hi - freq_lo);
207 jiffies_lo = jiffies_total - jiffies_hi;
208 dbs_info->freq_lo = freq_lo;
209 dbs_info->freq_lo_jiffies = jiffies_lo;
210 dbs_info->freq_hi_jiffies = jiffies_hi;
211 return freq_hi;
212}
213
214static void ondemand_powersave_bias_init(void)
215{
216 int i;
217 for_each_online_cpu(i) {
218 struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, i);
219 dbs_info->freq_table = cpufreq_frequency_get_table(i);
220 dbs_info->freq_lo = 0;
221 }
222}
223
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224/************************** sysfs interface ************************/
225static ssize_t show_sampling_rate_max(struct cpufreq_policy *policy, char *buf)
226{
227 return sprintf (buf, "%u\n", MAX_SAMPLING_RATE);
228}
229
230static ssize_t show_sampling_rate_min(struct cpufreq_policy *policy, char *buf)
231{
232 return sprintf (buf, "%u\n", MIN_SAMPLING_RATE);
233}
234
Dave Jones32ee8c32006-02-28 00:43:23 -0500235#define define_one_ro(_name) \
236static struct freq_attr _name = \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237__ATTR(_name, 0444, show_##_name, NULL)
238
239define_one_ro(sampling_rate_max);
240define_one_ro(sampling_rate_min);
241
242/* cpufreq_ondemand Governor Tunables */
243#define show_one(file_name, object) \
244static ssize_t show_##file_name \
245(struct cpufreq_policy *unused, char *buf) \
246{ \
247 return sprintf(buf, "%u\n", dbs_tuners_ins.object); \
248}
249show_one(sampling_rate, sampling_rate);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250show_one(up_threshold, up_threshold);
Alexander Clouter001893c2005-12-01 01:09:25 -0800251show_one(ignore_nice_load, ignore_nice);
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400252show_one(powersave_bias, powersave_bias);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253
Dave Jones32ee8c32006-02-28 00:43:23 -0500254static ssize_t store_sampling_rate(struct cpufreq_policy *unused,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 const char *buf, size_t count)
256{
257 unsigned int input;
258 int ret;
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700259 ret = sscanf(buf, "%u", &input);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800261 mutex_lock(&dbs_mutex);
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530262 if (ret != 1 || input > MAX_SAMPLING_RATE
263 || input < MIN_SAMPLING_RATE) {
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800264 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 return -EINVAL;
266 }
267
268 dbs_tuners_ins.sampling_rate = input;
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800269 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270
271 return count;
272}
273
Dave Jones32ee8c32006-02-28 00:43:23 -0500274static ssize_t store_up_threshold(struct cpufreq_policy *unused,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 const char *buf, size_t count)
276{
277 unsigned int input;
278 int ret;
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700279 ret = sscanf(buf, "%u", &input);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800281 mutex_lock(&dbs_mutex);
Dave Jones32ee8c32006-02-28 00:43:23 -0500282 if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
Dave Jonesc29f1402005-05-31 19:03:50 -0700283 input < MIN_FREQUENCY_UP_THRESHOLD) {
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800284 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 return -EINVAL;
286 }
287
288 dbs_tuners_ins.up_threshold = input;
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800289 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290
291 return count;
292}
293
Alexander Clouter001893c2005-12-01 01:09:25 -0800294static ssize_t store_ignore_nice_load(struct cpufreq_policy *policy,
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700295 const char *buf, size_t count)
296{
297 unsigned int input;
298 int ret;
299
300 unsigned int j;
Dave Jones32ee8c32006-02-28 00:43:23 -0500301
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700302 ret = sscanf(buf, "%u", &input);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700303 if ( ret != 1 )
304 return -EINVAL;
305
306 if ( input > 1 )
307 input = 1;
Dave Jones32ee8c32006-02-28 00:43:23 -0500308
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800309 mutex_lock(&dbs_mutex);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700310 if ( input == dbs_tuners_ins.ignore_nice ) { /* nothing to do */
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800311 mutex_unlock(&dbs_mutex);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700312 return count;
313 }
314 dbs_tuners_ins.ignore_nice = input;
315
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700316 /* we need to re-evaluate prev_cpu_idle */
Dave Jonesdac1c1a2005-05-31 19:03:49 -0700317 for_each_online_cpu(j) {
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700318 struct cpu_dbs_info_s *dbs_info;
319 dbs_info = &per_cpu(cpu_dbs_info, j);
venkatesh.pallipadi@intel.com34305022008-08-04 11:59:09 -0700320 dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
321 &dbs_info->prev_cpu_wall);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700322 }
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800323 mutex_unlock(&dbs_mutex);
Dave Jones3d5ee9e2005-05-31 19:03:47 -0700324
325 return count;
326}
327
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400328static ssize_t store_powersave_bias(struct cpufreq_policy *unused,
329 const char *buf, size_t count)
330{
331 unsigned int input;
332 int ret;
333 ret = sscanf(buf, "%u", &input);
334
335 if (ret != 1)
336 return -EINVAL;
337
338 if (input > 1000)
339 input = 1000;
340
341 mutex_lock(&dbs_mutex);
342 dbs_tuners_ins.powersave_bias = input;
343 ondemand_powersave_bias_init();
344 mutex_unlock(&dbs_mutex);
345
346 return count;
347}
348
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349#define define_one_rw(_name) \
350static struct freq_attr _name = \
351__ATTR(_name, 0644, show_##_name, store_##_name)
352
353define_one_rw(sampling_rate);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354define_one_rw(up_threshold);
Alexander Clouter001893c2005-12-01 01:09:25 -0800355define_one_rw(ignore_nice_load);
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400356define_one_rw(powersave_bias);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357
358static struct attribute * dbs_attributes[] = {
359 &sampling_rate_max.attr,
360 &sampling_rate_min.attr,
361 &sampling_rate.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 &up_threshold.attr,
Alexander Clouter001893c2005-12-01 01:09:25 -0800363 &ignore_nice_load.attr,
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400364 &powersave_bias.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 NULL
366};
367
368static struct attribute_group dbs_attr_group = {
369 .attrs = dbs_attributes,
370 .name = "ondemand",
371};
372
373/************************** sysfs end ************************/
374
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -0700375static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376{
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700377 unsigned int max_load_freq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378
379 struct cpufreq_policy *policy;
380 unsigned int j;
381
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 if (!this_dbs_info->enable)
383 return;
384
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400385 this_dbs_info->freq_lo = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 policy = this_dbs_info->cur_policy;
Venki Pallipadiea487612007-06-20 14:26:24 -0700387
Dave Jones32ee8c32006-02-28 00:43:23 -0500388 /*
Dave Jonesc29f1402005-05-31 19:03:50 -0700389 * Every sampling_rate, we check, if current idle time is less
390 * than 20% (default), then we try to increase frequency
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700391 * Every sampling_rate, we look for a the lowest
Dave Jonesc29f1402005-05-31 19:03:50 -0700392 * frequency which can sustain the load while keeping idle time over
393 * 30%. If such a frequency exist, we try to decrease to this frequency.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 *
Dave Jones32ee8c32006-02-28 00:43:23 -0500395 * Any frequency increase takes it to the maximum frequency.
396 * Frequency reduction happens at minimum steps of
397 * 5% (default) of current frequency
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 */
399
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700400 /* Get Absolute Load - in terms of freq */
401 max_load_freq = 0;
402
Rusty Russell835481d2009-01-04 05:18:06 -0800403 for_each_cpu(j, policy->cpus) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 struct cpu_dbs_info_s *j_dbs_info;
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700405 cputime64_t cur_wall_time, cur_idle_time;
406 unsigned int idle_time, wall_time;
407 unsigned int load, load_freq;
408 int freq_avg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 j_dbs_info = &per_cpu(cpu_dbs_info, j);
venkatesh.pallipadi@intel.com34305022008-08-04 11:59:09 -0700411
412 cur_idle_time = get_cpu_idle_time(j, &cur_wall_time);
413
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700414 wall_time = (unsigned int) cputime64_sub(cur_wall_time,
415 j_dbs_info->prev_cpu_wall);
416 j_dbs_info->prev_cpu_wall = cur_wall_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700418 idle_time = (unsigned int) cputime64_sub(cur_idle_time,
419 j_dbs_info->prev_cpu_idle);
420 j_dbs_info->prev_cpu_idle = cur_idle_time;
421
venkatesh.pallipadi@intel.com34305022008-08-04 11:59:09 -0700422 if (unlikely(!wall_time || wall_time < idle_time))
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700423 continue;
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700424
425 load = 100 * (wall_time - idle_time) / wall_time;
426
427 freq_avg = __cpufreq_driver_getavg(policy, j);
428 if (freq_avg <= 0)
429 freq_avg = policy->cur;
430
431 load_freq = load * freq_avg;
432 if (load_freq > max_load_freq)
433 max_load_freq = load_freq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 }
435
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700436 /* Check for frequency increase */
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700437 if (max_load_freq > dbs_tuners_ins.up_threshold * policy->cur) {
Dave Jonesc11420a2005-05-31 19:03:48 -0700438 /* if we are already at full speed then break out early */
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400439 if (!dbs_tuners_ins.powersave_bias) {
440 if (policy->cur == policy->max)
441 return;
Dave Jones32ee8c32006-02-28 00:43:23 -0500442
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400443 __cpufreq_driver_target(policy, policy->max,
444 CPUFREQ_RELATION_H);
445 } else {
446 int freq = powersave_bias_target(policy, policy->max,
447 CPUFREQ_RELATION_H);
448 __cpufreq_driver_target(policy, freq,
449 CPUFREQ_RELATION_L);
450 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451 return;
452 }
453
454 /* Check for frequency decrease */
Dave Jonesc29f1402005-05-31 19:03:50 -0700455 /* if we cannot reduce the frequency anymore, break out early */
456 if (policy->cur == policy->min)
457 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458
Dave Jonesc29f1402005-05-31 19:03:50 -0700459 /*
460 * The optimal frequency is the frequency that is the lowest that
461 * can support the current CPU usage without triggering the up
462 * policy. To be safe, we focus 10 points under the threshold.
463 */
venkatesh.pallipadi@intel.come9d95bf2008-08-04 11:59:10 -0700464 if (max_load_freq <
465 (dbs_tuners_ins.up_threshold - dbs_tuners_ins.down_differential) *
466 policy->cur) {
venkatesh.pallipadi@intel.comc43aa3b2008-08-04 11:59:08 -0700467 unsigned int freq_next;
venkatesh.pallipadi@intel.come9d95bf2008-08-04 11:59:10 -0700468 freq_next = max_load_freq /
469 (dbs_tuners_ins.up_threshold -
470 dbs_tuners_ins.down_differential);
Venkatesh Pallipadidfde5d62006-10-03 12:38:45 -0700471
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400472 if (!dbs_tuners_ins.powersave_bias) {
473 __cpufreq_driver_target(policy, freq_next,
474 CPUFREQ_RELATION_L);
475 } else {
476 int freq = powersave_bias_target(policy, freq_next,
477 CPUFREQ_RELATION_L);
478 __cpufreq_driver_target(policy, freq,
479 CPUFREQ_RELATION_L);
480 }
Venkatesh Pallipadiccb2fe22006-06-28 13:49:52 -0700481 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482}
483
David Howellsc4028952006-11-22 14:57:56 +0000484static void do_dbs_timer(struct work_struct *work)
Dave Jones32ee8c32006-02-28 00:43:23 -0500485{
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800486 struct cpu_dbs_info_s *dbs_info =
487 container_of(work, struct cpu_dbs_info_s, work.work);
488 unsigned int cpu = dbs_info->cpu;
489 int sample_type = dbs_info->sample_type;
490
Alexey Starikovskiy1ce28d62006-07-31 22:25:20 +0400491 /* We want all CPUs to do sampling nearly on same jiffy */
492 int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
David Howellsc4028952006-11-22 14:57:56 +0000493
Alexey Starikovskiy1ce28d62006-07-31 22:25:20 +0400494 delay -= jiffies % delay;
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -0700495
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800496 if (lock_policy_rwsem_write(cpu) < 0)
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700497 return;
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800498
499 if (!dbs_info->enable) {
500 unlock_policy_rwsem_write(cpu);
501 return;
502 }
503
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400504 /* Common NORMAL_SAMPLE setup */
David Howellsc4028952006-11-22 14:57:56 +0000505 dbs_info->sample_type = DBS_NORMAL_SAMPLE;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400506 if (!dbs_tuners_ins.powersave_bias ||
David Howellsc4028952006-11-22 14:57:56 +0000507 sample_type == DBS_NORMAL_SAMPLE) {
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400508 dbs_check_cpu(dbs_info);
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400509 if (dbs_info->freq_lo) {
510 /* Setup timer for SUB_SAMPLE */
David Howellsc4028952006-11-22 14:57:56 +0000511 dbs_info->sample_type = DBS_SUB_SAMPLE;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400512 delay = dbs_info->freq_hi_jiffies;
513 }
514 } else {
515 __cpufreq_driver_target(dbs_info->cur_policy,
516 dbs_info->freq_lo,
517 CPUFREQ_RELATION_H);
518 }
Alexey Starikovskiy1ce28d62006-07-31 22:25:20 +0400519 queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work, delay);
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800520 unlock_policy_rwsem_write(cpu);
Dave Jones32ee8c32006-02-28 00:43:23 -0500521}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800523static inline void dbs_timer_init(struct cpu_dbs_info_s *dbs_info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524{
Alexey Starikovskiy1ce28d62006-07-31 22:25:20 +0400525 /* We want all CPUs to do sampling nearly on same jiffy */
526 int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
527 delay -= jiffies % delay;
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -0700528
Dave Jonesc18a1482007-02-10 20:03:51 -0500529 dbs_info->enable = 1;
Alexey Starikovskiy05ca0352006-07-31 22:28:12 +0400530 ondemand_powersave_bias_init();
David Howellsc4028952006-11-22 14:57:56 +0000531 dbs_info->sample_type = DBS_NORMAL_SAMPLE;
Venki Pallipadi28287032007-05-08 00:27:47 -0700532 INIT_DELAYED_WORK_DEFERRABLE(&dbs_info->work, do_dbs_timer);
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800533 queue_delayed_work_on(dbs_info->cpu, kondemand_wq, &dbs_info->work,
534 delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535}
536
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700537static inline void dbs_timer_exit(struct cpu_dbs_info_s *dbs_info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538{
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700539 dbs_info->enable = 0;
540 cancel_delayed_work(&dbs_info->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541}
542
543static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
544 unsigned int event)
545{
546 unsigned int cpu = policy->cpu;
547 struct cpu_dbs_info_s *this_dbs_info;
548 unsigned int j;
Jeff Garzik914f7c32006-10-20 14:31:00 -0700549 int rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550
551 this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
552
553 switch (event) {
554 case CPUFREQ_GOV_START:
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700555 if ((!cpu_online(cpu)) || (!policy->cur))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 return -EINVAL;
557
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558 if (this_dbs_info->enable) /* Already enabled */
559 break;
Dave Jones32ee8c32006-02-28 00:43:23 -0500560
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800561 mutex_lock(&dbs_mutex);
Venkatesh Pallipadi2f8a8352006-06-28 13:51:19 -0700562 dbs_enable++;
Jeff Garzik914f7c32006-10-20 14:31:00 -0700563
564 rc = sysfs_create_group(&policy->kobj, &dbs_attr_group);
565 if (rc) {
Jeff Garzik914f7c32006-10-20 14:31:00 -0700566 dbs_enable--;
567 mutex_unlock(&dbs_mutex);
568 return rc;
569 }
570
Rusty Russell835481d2009-01-04 05:18:06 -0800571 for_each_cpu(j, policy->cpus) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 struct cpu_dbs_info_s *j_dbs_info;
573 j_dbs_info = &per_cpu(cpu_dbs_info, j);
574 j_dbs_info->cur_policy = policy;
Dave Jones32ee8c32006-02-28 00:43:23 -0500575
venkatesh.pallipadi@intel.com34305022008-08-04 11:59:09 -0700576 j_dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
577 &j_dbs_info->prev_cpu_wall);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 }
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800579 this_dbs_info->cpu = cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 /*
581 * Start the timerschedule work, when this governor
582 * is used for first time
583 */
584 if (dbs_enable == 1) {
585 unsigned int latency;
586 /* policy latency is in nS. Convert it to uS first */
Dave Jonesdf8b59b2005-09-20 12:39:35 -0700587 latency = policy->cpuinfo.transition_latency / 1000;
588 if (latency == 0)
589 latency = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590
Dave Jonesdf8b59b2005-09-20 12:39:35 -0700591 def_sampling_rate = latency *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 DEF_SAMPLING_RATE_LATENCY_MULTIPLIER;
Dave Jonesdf8b59b2005-09-20 12:39:35 -0700593
594 if (def_sampling_rate < MIN_STAT_SAMPLING_RATE)
595 def_sampling_rate = MIN_STAT_SAMPLING_RATE;
596
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 dbs_tuners_ins.sampling_rate = def_sampling_rate;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 }
Venkatesh Pallipadi529af7a2007-02-05 16:12:44 -0800599 dbs_timer_init(this_dbs_info);
Dave Jones32ee8c32006-02-28 00:43:23 -0500600
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800601 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602 break;
603
604 case CPUFREQ_GOV_STOP:
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800605 mutex_lock(&dbs_mutex);
Linus Torvalds2cd7cbd2006-07-23 12:05:00 -0700606 dbs_timer_exit(this_dbs_info);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 sysfs_remove_group(&policy->kobj, &dbs_attr_group);
608 dbs_enable--;
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800609 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610
611 break;
612
613 case CPUFREQ_GOV_LIMITS:
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800614 mutex_lock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 if (policy->max < this_dbs_info->cur_policy->cur)
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700616 __cpufreq_driver_target(this_dbs_info->cur_policy,
617 policy->max,
618 CPUFREQ_RELATION_H);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 else if (policy->min > this_dbs_info->cur_policy->cur)
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700620 __cpufreq_driver_target(this_dbs_info->cur_policy,
621 policy->min,
622 CPUFREQ_RELATION_L);
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800623 mutex_unlock(&dbs_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 break;
625 }
626 return 0;
627}
628
Sven Wegenerc4d14bc2008-09-20 16:50:08 +0200629#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
630static
631#endif
Thomas Renninger1c256242007-10-02 13:28:12 -0700632struct cpufreq_governor cpufreq_gov_ondemand = {
633 .name = "ondemand",
634 .governor = cpufreq_governor_dbs,
635 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
636 .owner = THIS_MODULE,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
639static int __init cpufreq_gov_dbs_init(void)
640{
Akinobu Mita888a7942008-07-14 12:00:45 +0900641 int err;
venkatesh.pallipadi@intel.com80800912008-08-04 11:59:12 -0700642 cputime64_t wall;
Andrea Righi4f6e6b92008-09-18 10:43:40 +0000643 u64 idle_time;
644 int cpu = get_cpu();
venkatesh.pallipadi@intel.com80800912008-08-04 11:59:12 -0700645
Andrea Righi4f6e6b92008-09-18 10:43:40 +0000646 idle_time = get_cpu_idle_time_us(cpu, &wall);
647 put_cpu();
venkatesh.pallipadi@intel.com80800912008-08-04 11:59:12 -0700648 if (idle_time != -1ULL) {
649 /* Idle micro accounting is supported. Use finer thresholds */
650 dbs_tuners_ins.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
651 dbs_tuners_ins.down_differential =
652 MICRO_FREQUENCY_DOWN_DIFFERENTIAL;
653 }
Akinobu Mita888a7942008-07-14 12:00:45 +0900654
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800655 kondemand_wq = create_workqueue("kondemand");
656 if (!kondemand_wq) {
657 printk(KERN_ERR "Creation of kondemand failed\n");
658 return -EFAULT;
659 }
Akinobu Mita888a7942008-07-14 12:00:45 +0900660 err = cpufreq_register_governor(&cpufreq_gov_ondemand);
661 if (err)
662 destroy_workqueue(kondemand_wq);
663
664 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665}
666
667static void __exit cpufreq_gov_dbs_exit(void)
668{
Thomas Renninger1c256242007-10-02 13:28:12 -0700669 cpufreq_unregister_governor(&cpufreq_gov_ondemand);
Venkatesh Pallipadi56463b72007-02-05 16:12:45 -0800670 destroy_workqueue(kondemand_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671}
672
673
Venkatesh Pallipadiffac80e2006-06-28 13:52:18 -0700674MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
675MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
676MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
677 "Low Latency Frequency Transition capable processors");
678MODULE_LICENSE("GPL");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679
Johannes Weiner69157192008-01-17 15:21:08 -0800680#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
681fs_initcall(cpufreq_gov_dbs_init);
682#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683module_init(cpufreq_gov_dbs_init);
Johannes Weiner69157192008-01-17 15:21:08 -0800684#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685module_exit(cpufreq_gov_dbs_exit);