1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3 #include <linux/kernel.h>
4 #include <linux/sched.h>
6 #include <linux/module.h>
13 #include <linux/timex.h>
16 #include <asm/timer.h>
19 #include <asm/delay.h>
20 #include <asm/hypervisor.h>
65 return __cycles_2_ns(this_offset);
70 #ifdef CONFIG_PARAVIRT
73 return paravirt_sched_clock();
89 pr_warn(
"Kernel compiled with CONFIG_X86_TSC, cannot disable TSC completely\n");
107 static int no_sched_irq_time;
111 if (!
strcmp(str,
"reliable"))
113 if (!
strncmp(str,
"noirqtime", 9))
114 no_sched_irq_time = 1;
120 #define MAX_RETRIES 5
121 #define SMI_TRESHOLD 50000
136 *p = acpi_pm_read_early();
147 static unsigned long calc_hpet_ref(
u64 deltatsc,
u64 hpet1,
u64 hpet2)
152 hpet2 += 0x100000000ULL;
158 return (
unsigned long) deltatsc;
164 static unsigned long calc_pmtimer_ref(
u64 deltatsc,
u64 pm1,
u64 pm2)
174 tmp = pm2 * 1000000000
LL;
178 return (
unsigned long) deltatsc;
182 #define CAL_LATCH (PIT_TICK_RATE / (1000 / CAL_MS))
183 #define CAL_PIT_LOOPS 1000
186 #define CAL2_LATCH (PIT_TICK_RATE / (1000 / CAL2_MS))
187 #define CAL2_PIT_LOOPS 5000
197 static unsigned long pit_calibrate_tsc(
u32 latch,
unsigned long ms,
int loopmin)
200 unsigned long tscmin, tscmax;
204 outb((
inb(0x61) & ~0x02) | 0x01, 0x61);
212 outb(latch & 0xff, 0x42);
213 outb(latch >> 8, 0x42);
220 while ((
inb(0x61) & 0x20) == 0) {
224 if ((
unsigned long) delta < tscmin)
225 tscmin = (
unsigned int) delta;
226 if ((
unsigned long) delta > tscmax)
227 tscmax = (
unsigned int)
delta;
240 if (pitcnt < loopmin || tscmax > 10 * tscmin)
284 static inline int pit_verify_msb(
unsigned char val)
291 static inline int pit_expect_msb(
unsigned char val,
u64 *tscp,
unsigned long *deltap)
294 u64 tsc = 0, prev_tsc = 0;
296 for (count = 0; count < 50000; count++) {
297 if (!pit_verify_msb(val))
318 #define MAX_QUICK_PIT_MS 50
319 #define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
321 static unsigned long quick_pit_calibrate(
void)
325 unsigned long d1,
d2;
328 outb((
inb(0x61) & ~0x02) | 0x01, 0x61);
353 if (pit_expect_msb(0xff, &tsc, &d1)) {
355 if (!pit_expect_msb(0xff-i, &delta, &d2))
362 if (d1+d2 >= delta >> 11)
372 if (!pit_verify_msb(0xfe - i))
377 pr_err(
"Fast TSC calibration failed\n");
395 do_div(delta, i*256*1000);
396 pr_info(
"Fast TSC calibration using PIT\n");
407 unsigned long flags, latch,
ms, fast_calibrate;
411 fast_calibrate = quick_pit_calibrate();
414 return fast_calibrate;
446 for (
i = 0;
i < 3;
i++) {
447 unsigned long tsc_pit_khz;
456 tsc1 = tsc_read_refs(&ref1, hpet);
457 tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin);
458 tsc2 = tsc_read_refs(&ref2, hpet);
462 tsc_pit_min =
min(tsc_pit_min, tsc_pit_khz);
472 tsc2 = (tsc2 - tsc1) * 1000000LL;
474 tsc2 = calc_hpet_ref(tsc2, ref1, ref2);
476 tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2);
478 tsc_ref_min =
min(tsc_ref_min, (
unsigned long) tsc2);
481 delta = ((
u64) tsc_pit_min) * 100;
482 do_div(delta, tsc_ref_min);
490 if (delta >= 90 && delta <= 110) {
491 pr_info(
"PIT calibration matches %s. %d loops\n",
492 hpet ?
"HPET" :
"PMTIMER",
i + 1);
514 pr_warn(
"Unable to calibrate against PIT\n");
517 if (!hpet && !ref1 && !ref2) {
518 pr_notice(
"No reference (HPET/PMTIMER) available\n");
524 pr_warn(
"HPET/PMTIMER calibration failed\n");
529 pr_info(
"using %s reference calibration\n",
530 hpet ?
"HPET" :
"PMTIMER");
536 if (!hpet && !ref1 && !ref2) {
537 pr_info(
"Using PIT calibration value\n");
543 pr_warn(
"HPET/PMTIMER calibration failed. Using PIT calibration.\n");
552 pr_warn(
"PIT calibration deviates from %s: %lu %lu\n",
553 hpet ?
"HPET" :
"PMTIMER", tsc_pit_min, tsc_ref_min);
554 pr_info(
"Using PIT calibration value\n");
561 unsigned long cpu_khz_old =
cpu_khz;
605 static void set_cyc2ns_scale(
unsigned long cpu_khz,
int cpu)
607 unsigned long long tsc_now, ns_now, *
offset;
608 unsigned long flags, *scale;
611 sched_clock_idle_sleep_event();
614 offset = &
per_cpu(cyc2ns_offset, cpu);
617 ns_now = __cycles_2_ns(tsc_now);
621 *offset = ns_now -
mult_frac(tsc_now, *scale,
625 sched_clock_idle_wakeup_event(0);
629 static unsigned long long cyc2ns_suspend;
633 if (!sched_clock_stable)
649 unsigned long long offset;
653 if (!sched_clock_stable)
667 #ifdef CONFIG_CPU_FREQ
680 static unsigned int ref_freq;
681 static unsigned long loops_per_jiffy_ref;
682 static unsigned long tsc_khz_ref;
684 static int time_cpufreq_notifier(
struct notifier_block *nb,
unsigned long val,
700 ref_freq = freq->
old;
701 loops_per_jiffy_ref = *lpj;
707 *lpj = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->
new);
709 tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->
new);
723 static int __init cpufreq_tsc(
void)
758 return ret >= clocksource_tsc.cycle_last ?
759 ret : clocksource_tsc.cycle_last;
764 clocksource_tsc.cycle_last = 0;
771 .resume = resume_tsc,
776 .archdata = { .vclock_mode = VCLOCK_TSC },
784 sched_clock_stable = 0;
785 disable_sched_clock_irqtime();
786 pr_info(
"Marking TSC unstable due to %s\n", reason);
788 if (clocksource_tsc.
mult)
792 clocksource_tsc.
rating = 0;
799 static void __init check_system_tsc_reliable(
void)
801 #ifdef CONFIG_MGEODE_LX
803 #define RTSC_SUSP 0x100
804 unsigned long res_low, res_high;
808 if (res_low & RTSC_SUSP)
821 if (!cpu_has_tsc || tsc_unstable)
825 if (apic_is_clustered_box())
866 static u64 tsc_start = -1, ref_start;
880 if (tsc_start == -1) {
887 tsc_start = tsc_read_refs(&ref_start, hpet);
891 tsc_stop = tsc_read_refs(&ref_stop, hpet);
894 if (ref_start == ref_stop)
901 delta = tsc_stop - tsc_start;
904 freq = calc_hpet_ref(delta, ref_start, ref_stop);
906 freq = calc_pmtimer_ref(delta, ref_start, ref_stop);
913 pr_info(
"Refined TSC clocksource calibration: %lu.%03lu MHz\n",
915 (
unsigned long)
tsc_khz % 1000);
918 clocksource_register_khz(&clocksource_tsc,
tsc_khz);
922 static int __init init_tsc_clocksource(
void)
924 if (!cpu_has_tsc || tsc_disabled > 0 || !
tsc_khz)
931 clocksource_tsc.
rating = 0;
940 clocksource_register_khz(&clocksource_tsc,
tsc_khz);
971 pr_info(
"Detected %lu.%03lu MHz processor\n",
972 (
unsigned long)cpu_khz / 1000,
973 (
unsigned long)cpu_khz % 1000);
982 set_cyc2ns_scale(cpu_khz, cpu);
984 if (tsc_disabled > 0)
990 if (!no_sched_irq_time)
991 enable_sched_clock_irqtime();
1002 check_system_tsc_reliable();