john stultz | 539eb11 | 2006-06-26 00:25:10 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This code largely moved from arch/i386/kernel/timer/timer_tsc.c |
| 3 | * which was originally moved from arch/i386/kernel/time.c. |
| 4 | * See comments there for proper credits. |
| 5 | */ |
| 6 | |
john stultz | 5d0cf41 | 2006-06-26 00:25:12 -0700 | [diff] [blame] | 7 | #include <linux/clocksource.h> |
john stultz | 539eb11 | 2006-06-26 00:25:10 -0700 | [diff] [blame] | 8 | #include <linux/workqueue.h> |
| 9 | #include <linux/cpufreq.h> |
| 10 | #include <linux/jiffies.h> |
| 11 | #include <linux/init.h> |
john stultz | 5d0cf41 | 2006-06-26 00:25:12 -0700 | [diff] [blame] | 12 | #include <linux/dmi.h> |
john stultz | 539eb11 | 2006-06-26 00:25:10 -0700 | [diff] [blame] | 13 | |
john stultz | 5d0cf41 | 2006-06-26 00:25:12 -0700 | [diff] [blame] | 14 | #include <asm/delay.h> |
john stultz | 539eb11 | 2006-06-26 00:25:10 -0700 | [diff] [blame] | 15 | #include <asm/tsc.h> |
john stultz | 6f84fa2 | 2006-06-26 00:25:11 -0700 | [diff] [blame] | 16 | #include <asm/delay.h> |
john stultz | 539eb11 | 2006-06-26 00:25:10 -0700 | [diff] [blame] | 17 | #include <asm/io.h> |
| 18 | |
| 19 | #include "mach_timer.h" |
| 20 | |
| 21 | /* |
| 22 | * On some systems the TSC frequency does not |
| 23 | * change with the cpu frequency. So we need |
| 24 | * an extra value to store the TSC freq |
| 25 | */ |
| 26 | unsigned int tsc_khz; |
| 27 | |
| 28 | int tsc_disable __cpuinitdata = 0; |
| 29 | |
| 30 | #ifdef CONFIG_X86_TSC |
| 31 | static int __init tsc_setup(char *str) |
| 32 | { |
| 33 | printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, " |
| 34 | "cannot disable TSC.\n"); |
| 35 | return 1; |
| 36 | } |
| 37 | #else |
| 38 | /* |
| 39 | * disable flag for tsc. Takes effect by clearing the TSC cpu flag |
| 40 | * in cpu/common.c |
| 41 | */ |
| 42 | static int __init tsc_setup(char *str) |
| 43 | { |
| 44 | tsc_disable = 1; |
| 45 | |
| 46 | return 1; |
| 47 | } |
| 48 | #endif |
| 49 | |
| 50 | __setup("notsc", tsc_setup); |
| 51 | |
john stultz | 539eb11 | 2006-06-26 00:25:10 -0700 | [diff] [blame] | 52 | /* |
| 53 | * code to mark and check if the TSC is unstable |
| 54 | * due to cpufreq or due to unsynced TSCs |
| 55 | */ |
| 56 | static int tsc_unstable; |
| 57 | |
| 58 | static inline int check_tsc_unstable(void) |
| 59 | { |
| 60 | return tsc_unstable; |
| 61 | } |
| 62 | |
| 63 | void mark_tsc_unstable(void) |
| 64 | { |
| 65 | tsc_unstable = 1; |
| 66 | } |
| 67 | EXPORT_SYMBOL_GPL(mark_tsc_unstable); |
| 68 | |
| 69 | /* Accellerators for sched_clock() |
| 70 | * convert from cycles(64bits) => nanoseconds (64bits) |
| 71 | * basic equation: |
| 72 | * ns = cycles / (freq / ns_per_sec) |
| 73 | * ns = cycles * (ns_per_sec / freq) |
| 74 | * ns = cycles * (10^9 / (cpu_khz * 10^3)) |
| 75 | * ns = cycles * (10^6 / cpu_khz) |
| 76 | * |
| 77 | * Then we use scaling math (suggested by george@mvista.com) to get: |
| 78 | * ns = cycles * (10^6 * SC / cpu_khz) / SC |
| 79 | * ns = cycles * cyc2ns_scale / SC |
| 80 | * |
| 81 | * And since SC is a constant power of two, we can convert the div |
| 82 | * into a shift. |
| 83 | * |
| 84 | * We can use khz divisor instead of mhz to keep a better percision, since |
| 85 | * cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. |
| 86 | * (mathieu.desnoyers@polymtl.ca) |
| 87 | * |
| 88 | * -johnstul@us.ibm.com "math is hard, lets go shopping!" |
| 89 | */ |
| 90 | static unsigned long cyc2ns_scale __read_mostly; |
| 91 | |
| 92 | #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */ |
| 93 | |
| 94 | static inline void set_cyc2ns_scale(unsigned long cpu_khz) |
| 95 | { |
| 96 | cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz; |
| 97 | } |
| 98 | |
| 99 | static inline unsigned long long cycles_2_ns(unsigned long long cyc) |
| 100 | { |
| 101 | return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR; |
| 102 | } |
| 103 | |
| 104 | /* |
| 105 | * Scheduler clock - returns current time in nanosec units. |
| 106 | */ |
| 107 | unsigned long long sched_clock(void) |
| 108 | { |
| 109 | unsigned long long this_offset; |
| 110 | |
| 111 | /* |
| 112 | * in the NUMA case we dont use the TSC as they are not |
| 113 | * synchronized across all CPUs. |
| 114 | */ |
| 115 | #ifndef CONFIG_NUMA |
| 116 | if (!cpu_khz || check_tsc_unstable()) |
| 117 | #endif |
| 118 | /* no locking but a rare wrong value is not a big deal */ |
| 119 | return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); |
| 120 | |
| 121 | /* read the Time Stamp Counter: */ |
| 122 | rdtscll(this_offset); |
| 123 | |
| 124 | /* return the value in ns */ |
| 125 | return cycles_2_ns(this_offset); |
| 126 | } |
| 127 | |
| 128 | static unsigned long calculate_cpu_khz(void) |
| 129 | { |
| 130 | unsigned long long start, end; |
| 131 | unsigned long count; |
| 132 | u64 delta64; |
| 133 | int i; |
| 134 | unsigned long flags; |
| 135 | |
| 136 | local_irq_save(flags); |
| 137 | |
| 138 | /* run 3 times to ensure the cache is warm */ |
| 139 | for (i = 0; i < 3; i++) { |
| 140 | mach_prepare_counter(); |
| 141 | rdtscll(start); |
| 142 | mach_countup(&count); |
| 143 | rdtscll(end); |
| 144 | } |
| 145 | /* |
| 146 | * Error: ECTCNEVERSET |
| 147 | * The CTC wasn't reliable: we got a hit on the very first read, |
| 148 | * or the CPU was so fast/slow that the quotient wouldn't fit in |
| 149 | * 32 bits.. |
| 150 | */ |
| 151 | if (count <= 1) |
| 152 | goto err; |
| 153 | |
| 154 | delta64 = end - start; |
| 155 | |
| 156 | /* cpu freq too fast: */ |
| 157 | if (delta64 > (1ULL<<32)) |
| 158 | goto err; |
| 159 | |
| 160 | /* cpu freq too slow: */ |
| 161 | if (delta64 <= CALIBRATE_TIME_MSEC) |
| 162 | goto err; |
| 163 | |
| 164 | delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */ |
| 165 | do_div(delta64,CALIBRATE_TIME_MSEC); |
| 166 | |
| 167 | local_irq_restore(flags); |
| 168 | return (unsigned long)delta64; |
| 169 | err: |
| 170 | local_irq_restore(flags); |
| 171 | return 0; |
| 172 | } |
| 173 | |
| 174 | int recalibrate_cpu_khz(void) |
| 175 | { |
| 176 | #ifndef CONFIG_SMP |
| 177 | unsigned long cpu_khz_old = cpu_khz; |
| 178 | |
| 179 | if (cpu_has_tsc) { |
| 180 | cpu_khz = calculate_cpu_khz(); |
| 181 | tsc_khz = cpu_khz; |
| 182 | cpu_data[0].loops_per_jiffy = |
| 183 | cpufreq_scale(cpu_data[0].loops_per_jiffy, |
| 184 | cpu_khz_old, cpu_khz); |
| 185 | return 0; |
| 186 | } else |
| 187 | return -ENODEV; |
| 188 | #else |
| 189 | return -ENODEV; |
| 190 | #endif |
| 191 | } |
| 192 | |
| 193 | EXPORT_SYMBOL(recalibrate_cpu_khz); |
| 194 | |
Magnus Damm | c0d8374 | 2006-09-26 10:52:35 +0200 | [diff] [blame] | 195 | void __init tsc_init(void) |
john stultz | 539eb11 | 2006-06-26 00:25:10 -0700 | [diff] [blame] | 196 | { |
| 197 | if (!cpu_has_tsc || tsc_disable) |
| 198 | return; |
| 199 | |
| 200 | cpu_khz = calculate_cpu_khz(); |
| 201 | tsc_khz = cpu_khz; |
| 202 | |
| 203 | if (!cpu_khz) |
| 204 | return; |
| 205 | |
| 206 | printk("Detected %lu.%03lu MHz processor.\n", |
| 207 | (unsigned long)cpu_khz / 1000, |
| 208 | (unsigned long)cpu_khz % 1000); |
| 209 | |
| 210 | set_cyc2ns_scale(cpu_khz); |
john stultz | 6f84fa2 | 2006-06-26 00:25:11 -0700 | [diff] [blame] | 211 | use_tsc_delay(); |
john stultz | 539eb11 | 2006-06-26 00:25:10 -0700 | [diff] [blame] | 212 | } |
| 213 | |
| 214 | #ifdef CONFIG_CPU_FREQ |
| 215 | |
| 216 | static unsigned int cpufreq_delayed_issched = 0; |
| 217 | static unsigned int cpufreq_init = 0; |
| 218 | static struct work_struct cpufreq_delayed_get_work; |
| 219 | |
| 220 | static void handle_cpufreq_delayed_get(void *v) |
| 221 | { |
| 222 | unsigned int cpu; |
| 223 | |
| 224 | for_each_online_cpu(cpu) |
| 225 | cpufreq_get(cpu); |
| 226 | |
| 227 | cpufreq_delayed_issched = 0; |
| 228 | } |
| 229 | |
| 230 | /* |
| 231 | * if we notice cpufreq oddness, schedule a call to cpufreq_get() as it tries |
| 232 | * to verify the CPU frequency the timing core thinks the CPU is running |
| 233 | * at is still correct. |
| 234 | */ |
| 235 | static inline void cpufreq_delayed_get(void) |
| 236 | { |
| 237 | if (cpufreq_init && !cpufreq_delayed_issched) { |
| 238 | cpufreq_delayed_issched = 1; |
| 239 | printk(KERN_DEBUG "Checking if CPU frequency changed.\n"); |
| 240 | schedule_work(&cpufreq_delayed_get_work); |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | /* |
| 245 | * if the CPU frequency is scaled, TSC-based delays will need a different |
| 246 | * loops_per_jiffy value to function properly. |
| 247 | */ |
| 248 | static unsigned int ref_freq = 0; |
| 249 | static unsigned long loops_per_jiffy_ref = 0; |
| 250 | static unsigned long cpu_khz_ref = 0; |
| 251 | |
| 252 | static int |
| 253 | time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data) |
| 254 | { |
| 255 | struct cpufreq_freqs *freq = data; |
| 256 | |
| 257 | if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE) |
| 258 | write_seqlock_irq(&xtime_lock); |
| 259 | |
| 260 | if (!ref_freq) { |
| 261 | if (!freq->old){ |
| 262 | ref_freq = freq->new; |
| 263 | goto end; |
| 264 | } |
| 265 | ref_freq = freq->old; |
| 266 | loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy; |
| 267 | cpu_khz_ref = cpu_khz; |
| 268 | } |
| 269 | |
| 270 | if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || |
| 271 | (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || |
| 272 | (val == CPUFREQ_RESUMECHANGE)) { |
| 273 | if (!(freq->flags & CPUFREQ_CONST_LOOPS)) |
| 274 | cpu_data[freq->cpu].loops_per_jiffy = |
| 275 | cpufreq_scale(loops_per_jiffy_ref, |
| 276 | ref_freq, freq->new); |
| 277 | |
| 278 | if (cpu_khz) { |
| 279 | |
| 280 | if (num_online_cpus() == 1) |
| 281 | cpu_khz = cpufreq_scale(cpu_khz_ref, |
| 282 | ref_freq, freq->new); |
| 283 | if (!(freq->flags & CPUFREQ_CONST_LOOPS)) { |
| 284 | tsc_khz = cpu_khz; |
| 285 | set_cyc2ns_scale(cpu_khz); |
| 286 | /* |
| 287 | * TSC based sched_clock turns |
| 288 | * to junk w/ cpufreq |
| 289 | */ |
| 290 | mark_tsc_unstable(); |
| 291 | } |
| 292 | } |
| 293 | } |
| 294 | end: |
| 295 | if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE) |
| 296 | write_sequnlock_irq(&xtime_lock); |
| 297 | |
| 298 | return 0; |
| 299 | } |
| 300 | |
| 301 | static struct notifier_block time_cpufreq_notifier_block = { |
| 302 | .notifier_call = time_cpufreq_notifier |
| 303 | }; |
| 304 | |
| 305 | static int __init cpufreq_tsc(void) |
| 306 | { |
| 307 | int ret; |
| 308 | |
| 309 | INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL); |
| 310 | ret = cpufreq_register_notifier(&time_cpufreq_notifier_block, |
| 311 | CPUFREQ_TRANSITION_NOTIFIER); |
| 312 | if (!ret) |
| 313 | cpufreq_init = 1; |
| 314 | |
| 315 | return ret; |
| 316 | } |
| 317 | |
| 318 | core_initcall(cpufreq_tsc); |
| 319 | |
| 320 | #endif |
john stultz | 5d0cf41 | 2006-06-26 00:25:12 -0700 | [diff] [blame] | 321 | |
| 322 | /* clock source code */ |
| 323 | |
| 324 | static unsigned long current_tsc_khz = 0; |
| 325 | static int tsc_update_callback(void); |
| 326 | |
| 327 | static cycle_t read_tsc(void) |
| 328 | { |
| 329 | cycle_t ret; |
| 330 | |
| 331 | rdtscll(ret); |
| 332 | |
| 333 | return ret; |
| 334 | } |
| 335 | |
| 336 | static struct clocksource clocksource_tsc = { |
| 337 | .name = "tsc", |
| 338 | .rating = 300, |
| 339 | .read = read_tsc, |
Jim Cromie | 7f9f303 | 2006-06-26 00:25:15 -0700 | [diff] [blame] | 340 | .mask = CLOCKSOURCE_MASK(64), |
john stultz | 5d0cf41 | 2006-06-26 00:25:12 -0700 | [diff] [blame] | 341 | .mult = 0, /* to be set */ |
| 342 | .shift = 22, |
| 343 | .update_callback = tsc_update_callback, |
| 344 | .is_continuous = 1, |
| 345 | }; |
| 346 | |
| 347 | static int tsc_update_callback(void) |
| 348 | { |
| 349 | int change = 0; |
| 350 | |
| 351 | /* check to see if we should switch to the safe clocksource: */ |
| 352 | if (clocksource_tsc.rating != 50 && check_tsc_unstable()) { |
| 353 | clocksource_tsc.rating = 50; |
john stultz | a275254 | 2006-06-26 00:25:14 -0700 | [diff] [blame] | 354 | clocksource_reselect(); |
john stultz | 5d0cf41 | 2006-06-26 00:25:12 -0700 | [diff] [blame] | 355 | change = 1; |
| 356 | } |
| 357 | |
| 358 | /* only update if tsc_khz has changed: */ |
| 359 | if (current_tsc_khz != tsc_khz) { |
| 360 | current_tsc_khz = tsc_khz; |
| 361 | clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz, |
| 362 | clocksource_tsc.shift); |
| 363 | change = 1; |
| 364 | } |
| 365 | |
| 366 | return change; |
| 367 | } |
| 368 | |
| 369 | static int __init dmi_mark_tsc_unstable(struct dmi_system_id *d) |
| 370 | { |
| 371 | printk(KERN_NOTICE "%s detected: marking TSC unstable.\n", |
| 372 | d->ident); |
| 373 | mark_tsc_unstable(); |
| 374 | return 0; |
| 375 | } |
| 376 | |
| 377 | /* List of systems that have known TSC problems */ |
| 378 | static struct dmi_system_id __initdata bad_tsc_dmi_table[] = { |
| 379 | { |
| 380 | .callback = dmi_mark_tsc_unstable, |
| 381 | .ident = "IBM Thinkpad 380XD", |
| 382 | .matches = { |
| 383 | DMI_MATCH(DMI_BOARD_VENDOR, "IBM"), |
| 384 | DMI_MATCH(DMI_BOARD_NAME, "2635FA0"), |
| 385 | }, |
| 386 | }, |
| 387 | {} |
| 388 | }; |
| 389 | |
| 390 | #define TSC_FREQ_CHECK_INTERVAL (10*MSEC_PER_SEC) /* 10sec in MS */ |
| 391 | static struct timer_list verify_tsc_freq_timer; |
| 392 | |
| 393 | /* XXX - Probably should add locking */ |
| 394 | static void verify_tsc_freq(unsigned long unused) |
| 395 | { |
| 396 | static u64 last_tsc; |
| 397 | static unsigned long last_jiffies; |
| 398 | |
| 399 | u64 now_tsc, interval_tsc; |
| 400 | unsigned long now_jiffies, interval_jiffies; |
| 401 | |
| 402 | |
| 403 | if (check_tsc_unstable()) |
| 404 | return; |
| 405 | |
| 406 | rdtscll(now_tsc); |
| 407 | now_jiffies = jiffies; |
| 408 | |
| 409 | if (!last_jiffies) { |
| 410 | goto out; |
| 411 | } |
| 412 | |
| 413 | interval_jiffies = now_jiffies - last_jiffies; |
| 414 | interval_tsc = now_tsc - last_tsc; |
| 415 | interval_tsc *= HZ; |
| 416 | do_div(interval_tsc, cpu_khz*1000); |
| 417 | |
| 418 | if (interval_tsc < (interval_jiffies * 3 / 4)) { |
| 419 | printk("TSC appears to be running slowly. " |
| 420 | "Marking it as unstable\n"); |
| 421 | mark_tsc_unstable(); |
| 422 | return; |
| 423 | } |
| 424 | |
| 425 | out: |
| 426 | last_tsc = now_tsc; |
| 427 | last_jiffies = now_jiffies; |
| 428 | /* set us up to go off on the next interval: */ |
| 429 | mod_timer(&verify_tsc_freq_timer, |
| 430 | jiffies + msecs_to_jiffies(TSC_FREQ_CHECK_INTERVAL)); |
| 431 | } |
| 432 | |
| 433 | /* |
| 434 | * Make an educated guess if the TSC is trustworthy and synchronized |
| 435 | * over all CPUs. |
| 436 | */ |
| 437 | static __init int unsynchronized_tsc(void) |
| 438 | { |
| 439 | /* |
| 440 | * Intel systems are normally all synchronized. |
| 441 | * Exceptions must mark TSC as unstable: |
| 442 | */ |
| 443 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) |
| 444 | return 0; |
| 445 | |
| 446 | /* assume multi socket systems are not synchronized: */ |
| 447 | return num_possible_cpus() > 1; |
| 448 | } |
| 449 | |
| 450 | static int __init init_tsc_clocksource(void) |
| 451 | { |
| 452 | |
| 453 | if (cpu_has_tsc && tsc_khz && !tsc_disable) { |
| 454 | /* check blacklist */ |
| 455 | dmi_check_system(bad_tsc_dmi_table); |
| 456 | |
| 457 | if (unsynchronized_tsc()) /* mark unstable if unsynced */ |
| 458 | mark_tsc_unstable(); |
| 459 | current_tsc_khz = tsc_khz; |
| 460 | clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz, |
| 461 | clocksource_tsc.shift); |
| 462 | /* lower the rating if we already know its unstable: */ |
| 463 | if (check_tsc_unstable()) |
| 464 | clocksource_tsc.rating = 50; |
| 465 | |
| 466 | init_timer(&verify_tsc_freq_timer); |
| 467 | verify_tsc_freq_timer.function = verify_tsc_freq; |
| 468 | verify_tsc_freq_timer.expires = |
| 469 | jiffies + msecs_to_jiffies(TSC_FREQ_CHECK_INTERVAL); |
| 470 | add_timer(&verify_tsc_freq_timer); |
| 471 | |
john stultz | a275254 | 2006-06-26 00:25:14 -0700 | [diff] [blame] | 472 | return clocksource_register(&clocksource_tsc); |
john stultz | 5d0cf41 | 2006-06-26 00:25:12 -0700 | [diff] [blame] | 473 | } |
| 474 | |
| 475 | return 0; |
| 476 | } |
| 477 | |
| 478 | module_init(init_tsc_clocksource); |