blob: 8cb2b2d35f5dbff7b8b35b3f778a16b1f30af3a4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86-64/kernel/time.c
3 *
4 * "High Precision Event Timer" based timekeeping.
5 *
6 * Copyright (c) 1991,1992,1995 Linus Torvalds
7 * Copyright (c) 1994 Alan Modra
8 * Copyright (c) 1995 Markus Kuhn
9 * Copyright (c) 1996 Ingo Molnar
10 * Copyright (c) 1998 Andrea Arcangeli
Vojtech Pavlik2f82bde42006-06-26 13:58:38 +020011 * Copyright (c) 2002,2006 Vojtech Pavlik
Linus Torvalds1da177e2005-04-16 15:20:36 -070012 * Copyright (c) 2003 Andi Kleen
13 * RTC support code taken from arch/i386/kernel/timers/time_hpet.c
14 */
15
16#include <linux/kernel.h>
17#include <linux/sched.h>
18#include <linux/interrupt.h>
19#include <linux/init.h>
20#include <linux/mc146818rtc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/time.h>
22#include <linux/ioport.h>
23#include <linux/module.h>
24#include <linux/device.h>
25#include <linux/sysdev.h>
26#include <linux/bcd.h>
Vojtech Pavlika670fad2006-09-26 10:52:28 +020027#include <linux/notifier.h>
28#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/kallsyms.h>
Andi Kleen312df5f2005-05-16 21:53:28 -070030#include <linux/acpi.h>
Andi Kleen8d916402005-05-31 14:39:26 -070031#ifdef CONFIG_ACPI
Andi Kleen312df5f2005-05-16 21:53:28 -070032#include <acpi/achware.h> /* for PM timer frequency */
Andi Kleen0e5f61b2006-07-29 21:42:37 +020033#include <acpi/acpi_bus.h>
Andi Kleen8d916402005-05-31 14:39:26 -070034#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <asm/8253pit.h>
36#include <asm/pgtable.h>
37#include <asm/vsyscall.h>
38#include <asm/timex.h>
39#include <asm/proto.h>
40#include <asm/hpet.h>
41#include <asm/sections.h>
42#include <linux/cpufreq.h>
43#include <linux/hpet.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <asm/apic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070045
Linus Torvalds1da177e2005-04-16 15:20:36 -070046#ifdef CONFIG_CPU_FREQ
47static void cpufreq_delayed_get(void);
48#endif
49extern void i8254_timer_resume(void);
50extern int using_apic_timer;
51
Vojtech Pavlika670fad2006-09-26 10:52:28 +020052static char *timename = NULL;
Andi Kleene8b91772006-02-26 04:18:49 +010053
Linus Torvalds1da177e2005-04-16 15:20:36 -070054DEFINE_SPINLOCK(rtc_lock);
Andi Kleen2ee60e172006-06-26 13:59:44 +020055EXPORT_SYMBOL(rtc_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070056DEFINE_SPINLOCK(i8253_lock);
57
Andi Kleen73dea472006-02-03 21:50:50 +010058int nohpet __initdata = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070059static int notsc __initdata = 0;
60
Vojtech Pavlik42211332006-06-26 13:58:32 +020061#define USEC_PER_TICK (USEC_PER_SEC / HZ)
62#define NSEC_PER_TICK (NSEC_PER_SEC / HZ)
63#define FSEC_PER_TICK (FSEC_PER_SEC / HZ)
64
65#define NS_SCALE 10 /* 2^10, carefully chosen */
66#define US_SCALE 32 /* 2^32, arbitralrily chosen */
67
Linus Torvalds1da177e2005-04-16 15:20:36 -070068unsigned int cpu_khz; /* TSC clocks / usec, not used here */
Andi Kleen2ee60e172006-06-26 13:59:44 +020069EXPORT_SYMBOL(cpu_khz);
Linus Torvalds1da177e2005-04-16 15:20:36 -070070static unsigned long hpet_period; /* fsecs / HPET clock */
71unsigned long hpet_tick; /* HPET clocks / interrupt */
Chris McDermott33042a92006-02-11 17:55:50 -080072int hpet_use_timer; /* Use counter of hpet for time keeping, otherwise PIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -070073unsigned long vxtime_hz = PIT_TICK_RATE;
74int report_lost_ticks; /* command line option */
75unsigned long long monotonic_base;
76
77struct vxtime_data __vxtime __section_vxtime; /* for vsyscalls */
78
79volatile unsigned long __jiffies __section_jiffies = INITIAL_JIFFIES;
Linus Torvalds1da177e2005-04-16 15:20:36 -070080struct timespec __xtime __section_xtime;
81struct timezone __sys_tz __section_sys_tz;
82
Linus Torvalds1da177e2005-04-16 15:20:36 -070083/*
84 * do_gettimeoffset() returns microseconds since last timer interrupt was
85 * triggered by hardware. A memory read of HPET is slower than a register read
86 * of TSC, but much more reliable. It's also synchronized to the timer
87 * interrupt. Note that do_gettimeoffset() may return more than hpet_tick, if a
88 * timer interrupt has happened already, but vxtime.trigger wasn't updated yet.
89 * This is not a problem, because jiffies hasn't updated either. They are bound
90 * together by xtime_lock.
91 */
92
93static inline unsigned int do_gettimeoffset_tsc(void)
94{
95 unsigned long t;
96 unsigned long x;
Andi Kleenc818a182006-01-11 22:45:24 +010097 t = get_cycles_sync();
Andi Kleen7351c0b2006-03-25 16:30:34 +010098 if (t < vxtime.last_tsc)
99 t = vxtime.last_tsc; /* hack */
Vojtech Pavlik42211332006-06-26 13:58:32 +0200100 x = ((t - vxtime.last_tsc) * vxtime.tsc_quot) >> US_SCALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 return x;
102}
103
104static inline unsigned int do_gettimeoffset_hpet(void)
105{
john stultza3a00752005-06-23 00:08:36 -0700106 /* cap counter read to one tick to avoid inconsistencies */
107 unsigned long counter = hpet_readl(HPET_COUNTER) - vxtime.last;
Vojtech Pavlik42211332006-06-26 13:58:32 +0200108 return (min(counter,hpet_tick) * vxtime.quot) >> US_SCALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109}
110
111unsigned int (*do_gettimeoffset)(void) = do_gettimeoffset_tsc;
112
113/*
114 * This version of gettimeofday() has microsecond resolution and better than
115 * microsecond precision, as we're using at least a 10 MHz (usually 14.31818
116 * MHz) HPET timer.
117 */
118
119void do_gettimeofday(struct timeval *tv)
120{
Atsushi Nemoto8ef38602006-09-30 23:28:31 -0700121 unsigned long seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 unsigned int sec, usec;
123
124 do {
125 seq = read_seqbegin(&xtime_lock);
126
127 sec = xtime.tv_sec;
Vojtech Pavlik42211332006-06-26 13:58:32 +0200128 usec = xtime.tv_nsec / NSEC_PER_USEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
130 /* i386 does some correction here to keep the clock
131 monotonous even when ntpd is fixing drift.
132 But they didn't work for me, there is a non monotonic
133 clock anyways with ntp.
134 I dropped all corrections now until a real solution can
135 be found. Note when you fix it here you need to do the same
136 in arch/x86_64/kernel/vsyscall.c and export all needed
137 variables in vmlinux.lds. -AK */
Atsushi Nemoto8ef38602006-09-30 23:28:31 -0700138 usec += do_gettimeoffset();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
140 } while (read_seqretry(&xtime_lock, seq));
141
Vojtech Pavlik42211332006-06-26 13:58:32 +0200142 tv->tv_sec = sec + usec / USEC_PER_SEC;
143 tv->tv_usec = usec % USEC_PER_SEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144}
145
146EXPORT_SYMBOL(do_gettimeofday);
147
148/*
149 * settimeofday() first undoes the correction that gettimeofday would do
150 * on the time, and then saves it. This is ugly, but has been like this for
151 * ages already.
152 */
153
154int do_settimeofday(struct timespec *tv)
155{
156 time_t wtm_sec, sec = tv->tv_sec;
157 long wtm_nsec, nsec = tv->tv_nsec;
158
159 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
160 return -EINVAL;
161
162 write_seqlock_irq(&xtime_lock);
163
Atsushi Nemoto8ef38602006-09-30 23:28:31 -0700164 nsec -= do_gettimeoffset() * NSEC_PER_USEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165
166 wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec);
167 wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec);
168
169 set_normalized_timespec(&xtime, sec, nsec);
170 set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec);
171
john stultzb149ee22005-09-06 15:17:46 -0700172 ntp_clear();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173
174 write_sequnlock_irq(&xtime_lock);
175 clock_was_set();
176 return 0;
177}
178
179EXPORT_SYMBOL(do_settimeofday);
180
181unsigned long profile_pc(struct pt_regs *regs)
182{
183 unsigned long pc = instruction_pointer(regs);
184
Andi Kleen31679f32006-09-26 10:52:28 +0200185 /* Assume the lock function has either no stack frame or a copy
186 of eflags from PUSHF
187 Eflags always has bits 22 and up cleared unlike kernel addresses. */
Andi Kleend5a26012006-07-28 14:44:42 +0200188 if (!user_mode(regs) && in_lock_functions(pc)) {
Andi Kleen31679f32006-09-26 10:52:28 +0200189 unsigned long *sp = (unsigned long *)regs->rsp;
190 if (sp[0] >> 22)
191 return sp[0];
192 if (sp[1] >> 22)
193 return sp[1];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 }
195 return pc;
196}
197EXPORT_SYMBOL(profile_pc);
198
199/*
200 * In order to set the CMOS clock precisely, set_rtc_mmss has to be called 500
201 * ms after the second nowtime has started, because when nowtime is written
202 * into the registers of the CMOS clock, it will jump to the next second
203 * precisely 500 ms later. Check the Motorola MC146818A or Dallas DS12887 data
204 * sheet for details.
205 */
206
207static void set_rtc_mmss(unsigned long nowtime)
208{
209 int real_seconds, real_minutes, cmos_minutes;
210 unsigned char control, freq_select;
211
212/*
213 * IRQs are disabled when we're called from the timer interrupt,
214 * no need for spin_lock_irqsave()
215 */
216
217 spin_lock(&rtc_lock);
218
219/*
220 * Tell the clock it's being set and stop it.
221 */
222
223 control = CMOS_READ(RTC_CONTROL);
224 CMOS_WRITE(control | RTC_SET, RTC_CONTROL);
225
226 freq_select = CMOS_READ(RTC_FREQ_SELECT);
227 CMOS_WRITE(freq_select | RTC_DIV_RESET2, RTC_FREQ_SELECT);
228
229 cmos_minutes = CMOS_READ(RTC_MINUTES);
230 BCD_TO_BIN(cmos_minutes);
231
232/*
233 * since we're only adjusting minutes and seconds, don't interfere with hour
234 * overflow. This avoids messing with unknown time zones but requires your RTC
235 * not to be off by more than 15 minutes. Since we're calling it only when
236 * our clock is externally synchronized using NTP, this shouldn't be a problem.
237 */
238
239 real_seconds = nowtime % 60;
240 real_minutes = nowtime / 60;
241 if (((abs(real_minutes - cmos_minutes) + 15) / 30) & 1)
242 real_minutes += 30; /* correct for half hour time zone */
243 real_minutes %= 60;
244
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 if (abs(real_minutes - cmos_minutes) >= 30) {
246 printk(KERN_WARNING "time.c: can't update CMOS clock "
247 "from %d to %d\n", cmos_minutes, real_minutes);
Andi Kleen28456ed2006-03-25 16:30:37 +0100248 } else {
Andi Kleen0b913172006-01-11 22:45:33 +0100249 BIN_TO_BCD(real_seconds);
250 BIN_TO_BCD(real_minutes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 CMOS_WRITE(real_seconds, RTC_SECONDS);
252 CMOS_WRITE(real_minutes, RTC_MINUTES);
253 }
254
255/*
256 * The following flags have to be released exactly in this order, otherwise the
257 * DS12887 (popular MC146818A clone with integrated battery and quartz) will
258 * not reset the oscillator and will not update precisely 500 ms later. You
259 * won't find this mentioned in the Dallas Semiconductor data sheets, but who
260 * believes data sheets anyway ... -- Markus Kuhn
261 */
262
263 CMOS_WRITE(control, RTC_CONTROL);
264 CMOS_WRITE(freq_select, RTC_FREQ_SELECT);
265
266 spin_unlock(&rtc_lock);
267}
268
269
270/* monotonic_clock(): returns # of nanoseconds passed since time_init()
271 * Note: This function is required to return accurate
272 * time even in the absence of multiple timer ticks.
273 */
Dimitri Sivanichcbf9b4b2006-09-26 10:52:34 +0200274static inline unsigned long long cycles_2_ns(unsigned long long cyc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275unsigned long long monotonic_clock(void)
276{
277 unsigned long seq;
278 u32 last_offset, this_offset, offset;
279 unsigned long long base;
280
281 if (vxtime.mode == VXTIME_HPET) {
282 do {
283 seq = read_seqbegin(&xtime_lock);
284
285 last_offset = vxtime.last;
286 base = monotonic_base;
john stultza3a00752005-06-23 00:08:36 -0700287 this_offset = hpet_readl(HPET_COUNTER);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 } while (read_seqretry(&xtime_lock, seq));
289 offset = (this_offset - last_offset);
Vojtech Pavlik42211332006-06-26 13:58:32 +0200290 offset *= NSEC_PER_TICK / hpet_tick;
Andi Kleen0b913172006-01-11 22:45:33 +0100291 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 do {
293 seq = read_seqbegin(&xtime_lock);
294
295 last_offset = vxtime.last_tsc;
296 base = monotonic_base;
297 } while (read_seqretry(&xtime_lock, seq));
Andi Kleenc818a182006-01-11 22:45:24 +0100298 this_offset = get_cycles_sync();
Dimitri Sivanichcbf9b4b2006-09-26 10:52:34 +0200299 offset = cycles_2_ns(this_offset - last_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 }
Andi Kleen7351c0b2006-03-25 16:30:34 +0100301 return base + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302}
303EXPORT_SYMBOL(monotonic_clock);
304
David Howells7d12e782006-10-05 14:55:46 +0100305static noinline void handle_lost_ticks(int lost)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306{
Andi Kleen7351c0b2006-03-25 16:30:34 +0100307 static long lost_count;
308 static int warned;
309 if (report_lost_ticks) {
310 printk(KERN_WARNING "time.c: Lost %d timer tick(s)! ", lost);
David Howells7d12e782006-10-05 14:55:46 +0100311 print_symbol("rip %s)\n", get_irq_regs()->rip);
Andi Kleen7351c0b2006-03-25 16:30:34 +0100312 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
Andi Kleen7351c0b2006-03-25 16:30:34 +0100314 if (lost_count == 1000 && !warned) {
315 printk(KERN_WARNING "warning: many lost ticks.\n"
316 KERN_WARNING "Your time source seems to be instable or "
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 "some driver is hogging interupts\n");
David Howells7d12e782006-10-05 14:55:46 +0100318 print_symbol("rip %s\n", get_irq_regs()->rip);
Andi Kleen7351c0b2006-03-25 16:30:34 +0100319 if (vxtime.mode == VXTIME_TSC && vxtime.hpet_address) {
320 printk(KERN_WARNING "Falling back to HPET\n");
321 if (hpet_use_timer)
322 vxtime.last = hpet_readl(HPET_T0_CMP) -
323 hpet_tick;
324 else
325 vxtime.last = hpet_readl(HPET_COUNTER);
326 vxtime.mode = VXTIME_HPET;
327 do_gettimeoffset = do_gettimeoffset_hpet;
328 }
329 /* else should fall back to PIT, but code missing. */
330 warned = 1;
331 } else
332 lost_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333
334#ifdef CONFIG_CPU_FREQ
Andi Kleen7351c0b2006-03-25 16:30:34 +0100335 /* In some cases the CPU can change frequency without us noticing
336 Give cpufreq a change to catch up. */
337 if ((lost_count+1) % 25 == 0)
338 cpufreq_delayed_get();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339#endif
340}
341
David Howells7d12e782006-10-05 14:55:46 +0100342void main_timer_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343{
344 static unsigned long rtc_update = 0;
345 unsigned long tsc;
Andi Kleen9ede6b02006-03-25 16:29:31 +0100346 int delay = 0, offset = 0, lost = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347
348/*
349 * Here we are in the timer irq handler. We have irqs locally disabled (so we
350 * don't need spin_lock_irqsave()) but we don't know if the timer_bh is running
351 * on the other CPU, so we need a lock. We also need to lock the vsyscall
352 * variables, because both do_timer() and us change them -arca+vojtech
353 */
354
355 write_seqlock(&xtime_lock);
356
john stultza3a00752005-06-23 00:08:36 -0700357 if (vxtime.hpet_address)
358 offset = hpet_readl(HPET_COUNTER);
359
360 if (hpet_use_timer) {
361 /* if we're using the hpet timer functionality,
362 * we can more accurately know the counter value
363 * when the timer interrupt occured.
364 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
366 delay = hpet_readl(HPET_COUNTER) - offset;
Andi Kleen9ede6b02006-03-25 16:29:31 +0100367 } else if (!pmtmr_ioport) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368 spin_lock(&i8253_lock);
369 outb_p(0x00, 0x43);
370 delay = inb_p(0x40);
371 delay |= inb(0x40) << 8;
372 spin_unlock(&i8253_lock);
373 delay = LATCH - 1 - delay;
374 }
375
Andi Kleenc818a182006-01-11 22:45:24 +0100376 tsc = get_cycles_sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377
378 if (vxtime.mode == VXTIME_HPET) {
379 if (offset - vxtime.last > hpet_tick) {
380 lost = (offset - vxtime.last) / hpet_tick - 1;
381 }
382
383 monotonic_base +=
Vojtech Pavlik42211332006-06-26 13:58:32 +0200384 (offset - vxtime.last) * NSEC_PER_TICK / hpet_tick;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385
386 vxtime.last = offset;
Andi Kleen312df5f2005-05-16 21:53:28 -0700387#ifdef CONFIG_X86_PM_TIMER
388 } else if (vxtime.mode == VXTIME_PMTMR) {
389 lost = pmtimer_mark_offset();
390#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 } else {
392 offset = (((tsc - vxtime.last_tsc) *
Vojtech Pavlik42211332006-06-26 13:58:32 +0200393 vxtime.tsc_quot) >> US_SCALE) - USEC_PER_TICK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394
395 if (offset < 0)
396 offset = 0;
397
Vojtech Pavlik42211332006-06-26 13:58:32 +0200398 if (offset > USEC_PER_TICK) {
399 lost = offset / USEC_PER_TICK;
400 offset %= USEC_PER_TICK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 }
402
Dimitri Sivanichcbf9b4b2006-09-26 10:52:34 +0200403 monotonic_base += cycles_2_ns(tsc - vxtime.last_tsc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404
405 vxtime.last_tsc = tsc - vxtime.quot * delay / vxtime.tsc_quot;
406
407 if ((((tsc - vxtime.last_tsc) *
Vojtech Pavlik42211332006-06-26 13:58:32 +0200408 vxtime.tsc_quot) >> US_SCALE) < offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 vxtime.last_tsc = tsc -
Vojtech Pavlik42211332006-06-26 13:58:32 +0200410 (((long) offset << US_SCALE) / vxtime.tsc_quot) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 }
412
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700413 if (lost > 0)
David Howells7d12e782006-10-05 14:55:46 +0100414 handle_lost_ticks(lost);
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700415 else
416 lost = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
418/*
419 * Do the timer stuff.
420 */
421
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700422 do_timer(lost + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423#ifndef CONFIG_SMP
David Howells7d12e782006-10-05 14:55:46 +0100424 update_process_times(user_mode(get_irq_regs()));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425#endif
426
427/*
428 * In the SMP case we use the local APIC timer interrupt to do the profiling,
429 * except when we simulate SMP mode on a uniprocessor system, in that case we
430 * have to call the local interrupt handler.
431 */
432
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433 if (!using_apic_timer)
David Howells7d12e782006-10-05 14:55:46 +0100434 smp_local_timer_interrupt();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435
436/*
437 * If we have an externally synchronized Linux clock, then update CMOS clock
438 * accordingly every ~11 minutes. set_rtc_mmss() will be called in the jiffy
439 * closest to exactly 500 ms before the next second. If the update fails, we
440 * don't care, as it'll be updated on the next turn, and the problem (time way
441 * off) isn't likely to go away much sooner anyway.
442 */
443
john stultzb149ee22005-09-06 15:17:46 -0700444 if (ntp_synced() && xtime.tv_sec > rtc_update &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 abs(xtime.tv_nsec - 500000000) <= tick_nsec / 2) {
446 set_rtc_mmss(xtime.tv_sec);
447 rtc_update = xtime.tv_sec + 660;
448 }
449
450 write_sequnlock(&xtime_lock);
Andi Kleen73dea472006-02-03 21:50:50 +0100451}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452
David Howells7d12e782006-10-05 14:55:46 +0100453static irqreturn_t timer_interrupt(int irq, void *dev_id)
Andi Kleen73dea472006-02-03 21:50:50 +0100454{
455 if (apic_runs_main_timer > 1)
456 return IRQ_HANDLED;
David Howells7d12e782006-10-05 14:55:46 +0100457 main_timer_handler();
Venkatesh Pallipadid25bf7e2006-01-11 22:44:24 +0100458 if (using_apic_timer)
459 smp_send_timer_broadcast_ipi();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 return IRQ_HANDLED;
461}
462
Ravikiran G Thirumalai68ed0042006-03-22 00:07:38 -0800463static unsigned int cyc2ns_scale __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464
Mathieu Desnoyersdacb16b2005-10-30 14:59:25 -0800465static inline void set_cyc2ns_scale(unsigned long cpu_khz)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466{
Vojtech Pavlik42211332006-06-26 13:58:32 +0200467 cyc2ns_scale = (NSEC_PER_MSEC << NS_SCALE) / cpu_khz;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468}
469
470static inline unsigned long long cycles_2_ns(unsigned long long cyc)
471{
Vojtech Pavlik42211332006-06-26 13:58:32 +0200472 return (cyc * cyc2ns_scale) >> NS_SCALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473}
474
475unsigned long long sched_clock(void)
476{
477 unsigned long a = 0;
478
479#if 0
480 /* Don't do a HPET read here. Using TSC always is much faster
481 and HPET may not be mapped yet when the scheduler first runs.
482 Disadvantage is a small drift between CPUs in some configurations,
483 but that should be tolerable. */
484 if (__vxtime.mode == VXTIME_HPET)
Vojtech Pavlik42211332006-06-26 13:58:32 +0200485 return (hpet_readl(HPET_COUNTER) * vxtime.quot) >> US_SCALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486#endif
487
488 /* Could do CPU core sync here. Opteron can execute rdtsc speculatively,
489 which means it is not completely exact and may not be monotonous between
490 CPUs. But the errors should be too small to matter for scheduling
491 purposes. */
492
493 rdtscll(a);
494 return cycles_2_ns(a);
495}
496
Andi Kleenbdf2b1c2006-01-11 22:46:39 +0100497static unsigned long get_cmos_time(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498{
Matt Mackall641f71f2006-03-28 01:56:01 -0800499 unsigned int year, mon, day, hour, min, sec;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 unsigned long flags;
Alexey Starikovskiyad718602007-02-02 19:48:19 +0300501 unsigned century = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 spin_lock_irqsave(&rtc_lock, flags);
504
Matt Mackall641f71f2006-03-28 01:56:01 -0800505 do {
506 sec = CMOS_READ(RTC_SECONDS);
507 min = CMOS_READ(RTC_MINUTES);
508 hour = CMOS_READ(RTC_HOURS);
509 day = CMOS_READ(RTC_DAY_OF_MONTH);
510 mon = CMOS_READ(RTC_MONTH);
511 year = CMOS_READ(RTC_YEAR);
Andi Kleen6954bee2006-03-25 16:30:31 +0100512#ifdef CONFIG_ACPI
Alexey Starikovskiyad718602007-02-02 19:48:19 +0300513 if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID &&
514 acpi_gbl_FADT.century)
515 century = CMOS_READ(acpi_gbl_FADT.century);
Andi Kleen6954bee2006-03-25 16:30:31 +0100516#endif
Matt Mackall641f71f2006-03-28 01:56:01 -0800517 } while (sec != CMOS_READ(RTC_SECONDS));
Andi Kleen6954bee2006-03-25 16:30:31 +0100518
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 spin_unlock_irqrestore(&rtc_lock, flags);
520
Andi Kleen0b913172006-01-11 22:45:33 +0100521 /*
522 * We know that x86-64 always uses BCD format, no need to check the
523 * config register.
Andi Kleen7351c0b2006-03-25 16:30:34 +0100524 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525
Andi Kleen0b913172006-01-11 22:45:33 +0100526 BCD_TO_BIN(sec);
527 BCD_TO_BIN(min);
528 BCD_TO_BIN(hour);
529 BCD_TO_BIN(day);
530 BCD_TO_BIN(mon);
531 BCD_TO_BIN(year);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532
Alexey Starikovskiyad718602007-02-02 19:48:19 +0300533 if (century) {
534 BCD_TO_BIN(century);
535 year += century * 100;
536 printk(KERN_INFO "Extended CMOS year: %d\n", century * 100);
Andi Kleen6954bee2006-03-25 16:30:31 +0100537 } else {
538 /*
539 * x86-64 systems only exists since 2002.
540 * This will work up to Dec 31, 2100
541 */
542 year += 2000;
543 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544
545 return mktime(year, mon, day, hour, min, sec);
546}
547
548#ifdef CONFIG_CPU_FREQ
549
550/* Frequency scaling support. Adjust the TSC based timer when the cpu frequency
551 changes.
552
553 RED-PEN: On SMP we assume all CPUs run with the same frequency. It's
554 not that important because current Opteron setups do not support
555 scaling on SMP anyroads.
556
557 Should fix up last_tsc too. Currently gettimeofday in the
558 first tick after the change will be slightly wrong. */
559
560#include <linux/workqueue.h>
561
562static unsigned int cpufreq_delayed_issched = 0;
563static unsigned int cpufreq_init = 0;
564static struct work_struct cpufreq_delayed_get_work;
565
David Howells65f27f32006-11-22 14:55:48 +0000566static void handle_cpufreq_delayed_get(struct work_struct *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567{
568 unsigned int cpu;
569 for_each_online_cpu(cpu) {
570 cpufreq_get(cpu);
571 }
572 cpufreq_delayed_issched = 0;
573}
574
575/* if we notice lost ticks, schedule a call to cpufreq_get() as it tries
576 * to verify the CPU frequency the timing core thinks the CPU is running
577 * at is still correct.
578 */
579static void cpufreq_delayed_get(void)
580{
581 static int warned;
582 if (cpufreq_init && !cpufreq_delayed_issched) {
583 cpufreq_delayed_issched = 1;
584 if (!warned) {
585 warned = 1;
Andi Kleen7351c0b2006-03-25 16:30:34 +0100586 printk(KERN_DEBUG
587 "Losing some ticks... checking if CPU frequency changed.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 }
589 schedule_work(&cpufreq_delayed_get_work);
590 }
591}
592
593static unsigned int ref_freq = 0;
594static unsigned long loops_per_jiffy_ref = 0;
595
596static unsigned long cpu_khz_ref = 0;
597
598static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
599 void *data)
600{
601 struct cpufreq_freqs *freq = data;
602 unsigned long *lpj, dummy;
603
Andi Kleenc29601e2005-04-16 15:25:05 -0700604 if (cpu_has(&cpu_data[freq->cpu], X86_FEATURE_CONSTANT_TSC))
605 return 0;
606
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 lpj = &dummy;
608 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
609#ifdef CONFIG_SMP
Andi Kleen7351c0b2006-03-25 16:30:34 +0100610 lpj = &cpu_data[freq->cpu].loops_per_jiffy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611#else
Andi Kleen7351c0b2006-03-25 16:30:34 +0100612 lpj = &boot_cpu_data.loops_per_jiffy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613#endif
614
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 if (!ref_freq) {
616 ref_freq = freq->old;
617 loops_per_jiffy_ref = *lpj;
618 cpu_khz_ref = cpu_khz;
619 }
620 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
621 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
622 (val == CPUFREQ_RESUMECHANGE)) {
623 *lpj =
624 cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
625
626 cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new);
627 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
Vojtech Pavlik42211332006-06-26 13:58:32 +0200628 vxtime.tsc_quot = (USEC_PER_MSEC << US_SCALE) / cpu_khz;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 }
630
Mathieu Desnoyersdacb16b2005-10-30 14:59:25 -0800631 set_cyc2ns_scale(cpu_khz_ref);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632
633 return 0;
634}
635
636static struct notifier_block time_cpufreq_notifier_block = {
637 .notifier_call = time_cpufreq_notifier
638};
639
640static int __init cpufreq_tsc(void)
641{
David Howells65f27f32006-11-22 14:55:48 +0000642 INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 if (!cpufreq_register_notifier(&time_cpufreq_notifier_block,
644 CPUFREQ_TRANSITION_NOTIFIER))
645 cpufreq_init = 1;
646 return 0;
647}
648
649core_initcall(cpufreq_tsc);
650
651#endif
652
653/*
654 * calibrate_tsc() calibrates the processor TSC in a very simple way, comparing
655 * it to the HPET timer of known frequency.
656 */
657
658#define TICK_COUNT 100000000
Jack Steinered5316d2007-01-11 01:52:44 +0100659#define TICK_MIN 5000
Jack Steiner2f7a2a72007-02-13 13:26:24 +0100660#define MAX_READ_RETRIES 5
Jack Steinered5316d2007-01-11 01:52:44 +0100661
662/*
663 * Some platforms take periodic SMI interrupts with 5ms duration. Make sure none
664 * occurs between the reads of the hpet & TSC.
665 */
666static void __init read_hpet_tsc(int *hpet, int *tsc)
667{
Jack Steiner2f7a2a72007-02-13 13:26:24 +0100668 int tsc1, tsc2, hpet1, retries = 0;
669 static int msg;
Jack Steinered5316d2007-01-11 01:52:44 +0100670
671 do {
672 tsc1 = get_cycles_sync();
673 hpet1 = hpet_readl(HPET_COUNTER);
674 tsc2 = get_cycles_sync();
Jack Steiner2f7a2a72007-02-13 13:26:24 +0100675 } while (tsc2 - tsc1 > TICK_MIN && retries++ < MAX_READ_RETRIES);
676 if (retries >= MAX_READ_RETRIES && !msg++)
677 printk(KERN_WARNING
678 "hpet.c: exceeded max retries to read HPET & TSC\n");
Jack Steinered5316d2007-01-11 01:52:44 +0100679 *hpet = hpet1;
680 *tsc = tsc2;
681}
682
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
684static unsigned int __init hpet_calibrate_tsc(void)
685{
686 int tsc_start, hpet_start;
687 int tsc_now, hpet_now;
688 unsigned long flags;
689
690 local_irq_save(flags);
691 local_irq_disable();
692
Jack Steinered5316d2007-01-11 01:52:44 +0100693 read_hpet_tsc(&hpet_start, &tsc_start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694
695 do {
696 local_irq_disable();
Jack Steinered5316d2007-01-11 01:52:44 +0100697 read_hpet_tsc(&hpet_now, &tsc_now);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 local_irq_restore(flags);
699 } while ((tsc_now - tsc_start) < TICK_COUNT &&
700 (hpet_now - hpet_start) < TICK_COUNT);
701
702 return (tsc_now - tsc_start) * 1000000000L
703 / ((hpet_now - hpet_start) * hpet_period / 1000);
704}
705
706
707/*
708 * pit_calibrate_tsc() uses the speaker output (channel 2) of
709 * the PIT. This is better than using the timer interrupt output,
710 * because we can read the value of the speaker with just one inb(),
711 * where we need three i/o operations for the interrupt channel.
712 * We count how many ticks the TSC does in 50 ms.
713 */
714
715static unsigned int __init pit_calibrate_tsc(void)
716{
717 unsigned long start, end;
718 unsigned long flags;
719
720 spin_lock_irqsave(&i8253_lock, flags);
721
722 outb((inb(0x61) & ~0x02) | 0x01, 0x61);
723
724 outb(0xb0, 0x43);
725 outb((PIT_TICK_RATE / (1000 / 50)) & 0xff, 0x42);
726 outb((PIT_TICK_RATE / (1000 / 50)) >> 8, 0x42);
Andi Kleenc818a182006-01-11 22:45:24 +0100727 start = get_cycles_sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 while ((inb(0x61) & 0x20) == 0);
Andi Kleenc818a182006-01-11 22:45:24 +0100729 end = get_cycles_sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730
731 spin_unlock_irqrestore(&i8253_lock, flags);
732
733 return (end - start) / 50;
734}
735
736#ifdef CONFIG_HPET
737static __init int late_hpet_init(void)
738{
739 struct hpet_data hd;
740 unsigned int ntimer;
741
742 if (!vxtime.hpet_address)
Andi Kleen3d34ee62006-04-07 19:50:06 +0200743 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744
745 memset(&hd, 0, sizeof (hd));
746
747 ntimer = hpet_readl(HPET_ID);
748 ntimer = (ntimer & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT;
749 ntimer++;
750
751 /*
752 * Register with driver.
753 * Timer0 and Timer1 is used by platform.
754 */
755 hd.hd_phys_address = vxtime.hpet_address;
Al Virodd42b152006-02-01 07:30:33 -0500756 hd.hd_address = (void __iomem *)fix_to_virt(FIX_HPET_BASE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 hd.hd_nirqs = ntimer;
758 hd.hd_flags = HPET_DATA_PLATFORM;
759 hpet_reserve_timer(&hd, 0);
760#ifdef CONFIG_HPET_EMULATE_RTC
761 hpet_reserve_timer(&hd, 1);
762#endif
763 hd.hd_irq[0] = HPET_LEGACY_8254;
764 hd.hd_irq[1] = HPET_LEGACY_RTC;
765 if (ntimer > 2) {
766 struct hpet *hpet;
767 struct hpet_timer *timer;
768 int i;
769
770 hpet = (struct hpet *) fix_to_virt(FIX_HPET_BASE);
Andi Kleen7351c0b2006-03-25 16:30:34 +0100771 timer = &hpet->hpet_timers[2];
772 for (i = 2; i < ntimer; timer++, i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773 hd.hd_irq[i] = (timer->hpet_config &
774 Tn_INT_ROUTE_CNF_MASK) >>
775 Tn_INT_ROUTE_CNF_SHIFT;
776
777 }
778
779 hpet_alloc(&hd);
780 return 0;
781}
782fs_initcall(late_hpet_init);
783#endif
784
785static int hpet_timer_stop_set_go(unsigned long tick)
786{
787 unsigned int cfg;
788
789/*
790 * Stop the timers and reset the main counter.
791 */
792
793 cfg = hpet_readl(HPET_CFG);
794 cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY);
795 hpet_writel(cfg, HPET_CFG);
796 hpet_writel(0, HPET_COUNTER);
797 hpet_writel(0, HPET_COUNTER + 4);
798
799/*
800 * Set up timer 0, as periodic with first interrupt to happen at hpet_tick,
801 * and period also hpet_tick.
802 */
john stultza3a00752005-06-23 00:08:36 -0700803 if (hpet_use_timer) {
804 hpet_writel(HPET_TN_ENABLE | HPET_TN_PERIODIC | HPET_TN_SETVAL |
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805 HPET_TN_32BIT, HPET_T0_CFG);
Vojtech Pavlikb2df3dd2006-06-26 13:58:35 +0200806 hpet_writel(hpet_tick, HPET_T0_CMP); /* next interrupt */
807 hpet_writel(hpet_tick, HPET_T0_CMP); /* period */
john stultza3a00752005-06-23 00:08:36 -0700808 cfg |= HPET_CFG_LEGACY;
809 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810/*
811 * Go!
812 */
813
john stultza3a00752005-06-23 00:08:36 -0700814 cfg |= HPET_CFG_ENABLE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 hpet_writel(cfg, HPET_CFG);
816
817 return 0;
818}
819
820static int hpet_init(void)
821{
822 unsigned int id;
823
824 if (!vxtime.hpet_address)
825 return -1;
826 set_fixmap_nocache(FIX_HPET_BASE, vxtime.hpet_address);
827 __set_fixmap(VSYSCALL_HPET, vxtime.hpet_address, PAGE_KERNEL_VSYSCALL_NOCACHE);
828
829/*
830 * Read the period, compute tick and quotient.
831 */
832
833 id = hpet_readl(HPET_ID);
834
john stultza3a00752005-06-23 00:08:36 -0700835 if (!(id & HPET_ID_VENDOR) || !(id & HPET_ID_NUMBER))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 return -1;
837
838 hpet_period = hpet_readl(HPET_PERIOD);
839 if (hpet_period < 100000 || hpet_period > 100000000)
840 return -1;
841
Vojtech Pavlik42211332006-06-26 13:58:32 +0200842 hpet_tick = (FSEC_PER_TICK + hpet_period / 2) / hpet_period;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843
john stultza3a00752005-06-23 00:08:36 -0700844 hpet_use_timer = (id & HPET_ID_LEGSUP);
845
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 return hpet_timer_stop_set_go(hpet_tick);
847}
848
849static int hpet_reenable(void)
850{
851 return hpet_timer_stop_set_go(hpet_tick);
852}
853
Andi Kleen73dea472006-02-03 21:50:50 +0100854#define PIT_MODE 0x43
855#define PIT_CH0 0x40
856
857static void __init __pit_init(int val, u8 mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858{
859 unsigned long flags;
860
861 spin_lock_irqsave(&i8253_lock, flags);
Andi Kleen73dea472006-02-03 21:50:50 +0100862 outb_p(mode, PIT_MODE);
863 outb_p(val & 0xff, PIT_CH0); /* LSB */
864 outb_p(val >> 8, PIT_CH0); /* MSB */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865 spin_unlock_irqrestore(&i8253_lock, flags);
866}
867
Andi Kleen73dea472006-02-03 21:50:50 +0100868void __init pit_init(void)
869{
870 __pit_init(LATCH, 0x34); /* binary, mode 2, LSB/MSB, ch 0 */
871}
872
873void __init pit_stop_interrupt(void)
874{
875 __pit_init(0, 0x30); /* mode 0 */
876}
877
878void __init stop_timer_interrupt(void)
879{
880 char *name;
881 if (vxtime.hpet_address) {
882 name = "HPET";
883 hpet_timer_stop_set_go(0);
884 } else {
885 name = "PIT";
886 pit_stop_interrupt();
887 }
888 printk(KERN_INFO "timer: %s interrupt stopped.\n", name);
889}
890
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891int __init time_setup(char *str)
892{
893 report_lost_ticks = 1;
894 return 1;
895}
896
897static struct irqaction irq0 = {
Thomas Gleixnerb1e05aa2006-07-01 19:29:29 -0700898 timer_interrupt, IRQF_DISABLED, CPU_MASK_NONE, "timer", NULL, NULL
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899};
900
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901void __init time_init(void)
902{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 if (nohpet)
904 vxtime.hpet_address = 0;
905
906 xtime.tv_sec = get_cmos_time();
907 xtime.tv_nsec = 0;
908
909 set_normalized_timespec(&wall_to_monotonic,
910 -xtime.tv_sec, -xtime.tv_nsec);
911
john stultza3a00752005-06-23 00:08:36 -0700912 if (!hpet_init())
Vojtech Pavlik42211332006-06-26 13:58:32 +0200913 vxtime_hz = (FSEC_PER_SEC + hpet_period / 2) / hpet_period;
Andi Kleen68e18892005-12-12 22:17:07 -0800914 else
915 vxtime.hpet_address = 0;
john stultza3a00752005-06-23 00:08:36 -0700916
917 if (hpet_use_timer) {
Jordan Hargraveb20367a2006-04-07 19:50:18 +0200918 /* set tick_nsec to use the proper rate for HPET */
919 tick_nsec = TICK_NSEC_HPET;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 cpu_khz = hpet_calibrate_tsc();
921 timename = "HPET";
Andi Kleen312df5f2005-05-16 21:53:28 -0700922#ifdef CONFIG_X86_PM_TIMER
john stultzfd495472005-12-12 22:17:13 -0800923 } else if (pmtmr_ioport && !vxtime.hpet_address) {
Andi Kleen312df5f2005-05-16 21:53:28 -0700924 vxtime_hz = PM_TIMER_FREQUENCY;
925 timename = "PM";
926 pit_init();
927 cpu_khz = pit_calibrate_tsc();
928#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 } else {
930 pit_init();
931 cpu_khz = pit_calibrate_tsc();
932 timename = "PIT";
933 }
934
Andi Kleene8b91772006-02-26 04:18:49 +0100935 vxtime.mode = VXTIME_TSC;
Vojtech Pavlik42211332006-06-26 13:58:32 +0200936 vxtime.quot = (USEC_PER_SEC << US_SCALE) / vxtime_hz;
937 vxtime.tsc_quot = (USEC_PER_MSEC << US_SCALE) / cpu_khz;
Andi Kleenc818a182006-01-11 22:45:24 +0100938 vxtime.last_tsc = get_cycles_sync();
Mathieu Desnoyersdacb16b2005-10-30 14:59:25 -0800939 set_cyc2ns_scale(cpu_khz);
Dimitri Sivanichcbf9b4b2006-09-26 10:52:34 +0200940 setup_irq(0, &irq0);
Vojtech Pavlika670fad2006-09-26 10:52:28 +0200941
942#ifndef CONFIG_SMP
943 time_init_gtod();
944#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945}
946
Ingo Molnar95492e42007-02-16 01:27:34 -0800947static int tsc_unstable = 0;
948
949void mark_tsc_unstable(void)
950{
951 tsc_unstable = 1;
952}
953EXPORT_SYMBOL_GPL(mark_tsc_unstable);
954
Andi Kleena8ab26f2005-04-16 15:25:19 -0700955/*
Andi Kleen312df5f2005-05-16 21:53:28 -0700956 * Make an educated guess if the TSC is trustworthy and synchronized
957 * over all CPUs.
958 */
Shaohua Li396bd502006-02-03 21:51:20 +0100959__cpuinit int unsynchronized_tsc(void)
Andi Kleen312df5f2005-05-16 21:53:28 -0700960{
Ingo Molnar95492e42007-02-16 01:27:34 -0800961 if (tsc_unstable)
962 return 1;
963
Andi Kleen312df5f2005-05-16 21:53:28 -0700964#ifdef CONFIG_SMP
Vojtech Pavlikf8bf3c62006-06-26 13:58:23 +0200965 if (apic_is_clustered_box())
Andi Kleen312df5f2005-05-16 21:53:28 -0700966 return 1;
Andi Kleen312df5f2005-05-16 21:53:28 -0700967#endif
Andi Kleen0e5f61b2006-07-29 21:42:37 +0200968 /* Most intel systems have synchronized TSCs except for
969 multi node systems */
970 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
971#ifdef CONFIG_ACPI
972 /* But TSC doesn't tick in C3 so don't use it there */
Alexey Starikovskiyad718602007-02-02 19:48:19 +0300973 if (acpi_gbl_FADT.header.length > 0 && acpi_gbl_FADT.C3latency < 1000)
Andi Kleen0e5f61b2006-07-29 21:42:37 +0200974 return 1;
975#endif
976 return 0;
977 }
978
Andi Kleen312df5f2005-05-16 21:53:28 -0700979 /* Assume multi socket systems are not synchronized */
Andi Kleen737c5c32006-01-11 22:45:15 +0100980 return num_present_cpus() > 1;
Andi Kleen312df5f2005-05-16 21:53:28 -0700981}
982
983/*
Andi Kleene8b91772006-02-26 04:18:49 +0100984 * Decide what mode gettimeofday should use.
Andi Kleena8ab26f2005-04-16 15:25:19 -0700985 */
Vojtech Pavlika670fad2006-09-26 10:52:28 +0200986void time_init_gtod(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987{
988 char *timetype;
989
Andi Kleen312df5f2005-05-16 21:53:28 -0700990 if (unsynchronized_tsc())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991 notsc = 1;
Vojtech Pavlika670fad2006-09-26 10:52:28 +0200992
Vojtech Pavlikc08c8202006-09-26 10:52:28 +0200993 if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP))
994 vgetcpu_mode = VGETCPU_RDTSCP;
995 else
996 vgetcpu_mode = VGETCPU_LSL;
997
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 if (vxtime.hpet_address && notsc) {
john stultza3a00752005-06-23 00:08:36 -0700999 timetype = hpet_use_timer ? "HPET" : "PIT/HPET";
Chris McDermott33042a92006-02-11 17:55:50 -08001000 if (hpet_use_timer)
1001 vxtime.last = hpet_readl(HPET_T0_CMP) - hpet_tick;
1002 else
1003 vxtime.last = hpet_readl(HPET_COUNTER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004 vxtime.mode = VXTIME_HPET;
1005 do_gettimeoffset = do_gettimeoffset_hpet;
Andi Kleen312df5f2005-05-16 21:53:28 -07001006#ifdef CONFIG_X86_PM_TIMER
1007 /* Using PM for gettimeofday is quite slow, but we have no other
1008 choice because the TSC is too unreliable on some systems. */
1009 } else if (pmtmr_ioport && !vxtime.hpet_address && notsc) {
1010 timetype = "PM";
1011 do_gettimeoffset = do_gettimeoffset_pm;
1012 vxtime.mode = VXTIME_PMTMR;
1013 sysctl_vsyscall = 0;
1014 printk(KERN_INFO "Disabling vsyscall due to use of PM timer\n");
1015#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 } else {
john stultza3a00752005-06-23 00:08:36 -07001017 timetype = hpet_use_timer ? "HPET/TSC" : "PIT/TSC";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 vxtime.mode = VXTIME_TSC;
1019 }
Vojtech Pavlika670fad2006-09-26 10:52:28 +02001020
1021 printk(KERN_INFO "time.c: Using %ld.%06ld MHz WALL %s GTOD %s timer.\n",
1022 vxtime_hz / 1000000, vxtime_hz % 1000000, timename, timetype);
1023 printk(KERN_INFO "time.c: Detected %d.%03d MHz processor.\n",
1024 cpu_khz / 1000, cpu_khz % 1000);
1025 vxtime.quot = (USEC_PER_SEC << US_SCALE) / vxtime_hz;
1026 vxtime.tsc_quot = (USEC_PER_MSEC << US_SCALE) / cpu_khz;
1027 vxtime.last_tsc = get_cycles_sync();
1028
1029 set_cyc2ns_scale(cpu_khz);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030}
1031
1032__setup("report_lost_ticks", time_setup);
1033
1034static long clock_cmos_diff;
1035static unsigned long sleep_start;
1036
Andi Kleen0b913172006-01-11 22:45:33 +01001037/*
1038 * sysfs support for the timer.
1039 */
1040
Pavel Machek0b9c33a2005-04-16 15:25:31 -07001041static int timer_suspend(struct sys_device *dev, pm_message_t state)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042{
1043 /*
1044 * Estimate time zone so that set_time can update the clock
1045 */
1046 long cmos_time = get_cmos_time();
1047
1048 clock_cmos_diff = -cmos_time;
1049 clock_cmos_diff += get_seconds();
1050 sleep_start = cmos_time;
1051 return 0;
1052}
1053
1054static int timer_resume(struct sys_device *dev)
1055{
1056 unsigned long flags;
1057 unsigned long sec;
1058 unsigned long ctime = get_cmos_time();
Rafael J. Wysocki34464a52006-09-26 10:52:37 +02001059 long sleep_length = (ctime - sleep_start) * HZ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060
Rafael J. Wysocki34464a52006-09-26 10:52:37 +02001061 if (sleep_length < 0) {
1062 printk(KERN_WARNING "Time skew detected in timer resume!\n");
1063 /* The time after the resume must not be earlier than the time
1064 * before the suspend or some nasty things will happen
1065 */
1066 sleep_length = 0;
1067 ctime = sleep_start;
1068 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 if (vxtime.hpet_address)
1070 hpet_reenable();
1071 else
1072 i8254_timer_resume();
1073
1074 sec = ctime + clock_cmos_diff;
1075 write_seqlock_irqsave(&xtime_lock,flags);
1076 xtime.tv_sec = sec;
1077 xtime.tv_nsec = 0;
Shaohua Li0dd2ea92006-02-03 21:50:56 +01001078 if (vxtime.mode == VXTIME_HPET) {
1079 if (hpet_use_timer)
1080 vxtime.last = hpet_readl(HPET_T0_CMP) - hpet_tick;
1081 else
1082 vxtime.last = hpet_readl(HPET_COUNTER);
1083#ifdef CONFIG_X86_PM_TIMER
1084 } else if (vxtime.mode == VXTIME_PMTMR) {
1085 pmtimer_resume();
1086#endif
1087 } else
1088 vxtime.last_tsc = get_cycles_sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089 write_sequnlock_irqrestore(&xtime_lock,flags);
1090 jiffies += sleep_length;
Shaohua Li0dd2ea92006-02-03 21:50:56 +01001091 monotonic_base += sleep_length * (NSEC_PER_SEC/HZ);
Ingo Molnar8446f1d2005-09-06 15:16:27 -07001092 touch_softlockup_watchdog();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 return 0;
1094}
1095
1096static struct sysdev_class timer_sysclass = {
1097 .resume = timer_resume,
1098 .suspend = timer_suspend,
1099 set_kset_name("timer"),
1100};
1101
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102/* XXX this driverfs stuff should probably go elsewhere later -john */
1103static struct sys_device device_timer = {
1104 .id = 0,
1105 .cls = &timer_sysclass,
1106};
1107
1108static int time_init_device(void)
1109{
1110 int error = sysdev_class_register(&timer_sysclass);
1111 if (!error)
1112 error = sysdev_register(&device_timer);
1113 return error;
1114}
1115
1116device_initcall(time_init_device);
1117
1118#ifdef CONFIG_HPET_EMULATE_RTC
1119/* HPET in LegacyReplacement Mode eats up RTC interrupt line. When, HPET
1120 * is enabled, we support RTC interrupt functionality in software.
1121 * RTC has 3 kinds of interrupts:
1122 * 1) Update Interrupt - generate an interrupt, every sec, when RTC clock
1123 * is updated
1124 * 2) Alarm Interrupt - generate an interrupt at a specific time of day
1125 * 3) Periodic Interrupt - generate periodic interrupt, with frequencies
1126 * 2Hz-8192Hz (2Hz-64Hz for non-root user) (all freqs in powers of 2)
1127 * (1) and (2) above are implemented using polling at a frequency of
1128 * 64 Hz. The exact frequency is a tradeoff between accuracy and interrupt
1129 * overhead. (DEFAULT_RTC_INT_FREQ)
1130 * For (3), we use interrupts at 64Hz or user specified periodic
1131 * frequency, whichever is higher.
1132 */
1133#include <linux/rtc.h>
1134
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135#define DEFAULT_RTC_INT_FREQ 64
1136#define RTC_NUM_INTS 1
1137
1138static unsigned long UIE_on;
1139static unsigned long prev_update_sec;
1140
1141static unsigned long AIE_on;
1142static struct rtc_time alarm_time;
1143
1144static unsigned long PIE_on;
1145static unsigned long PIE_freq = DEFAULT_RTC_INT_FREQ;
1146static unsigned long PIE_count;
1147
1148static unsigned long hpet_rtc_int_freq; /* RTC interrupt frequency */
Clemens Ladisch7811fb82005-10-30 15:03:36 -08001149static unsigned int hpet_t1_cmp; /* cached comparator register */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150
1151int is_hpet_enabled(void)
1152{
1153 return vxtime.hpet_address != 0;
1154}
1155
1156/*
1157 * Timer 1 for RTC, we do not use periodic interrupt feature,
1158 * even if HPET supports periodic interrupts on Timer 1.
1159 * The reason being, to set up a periodic interrupt in HPET, we need to
1160 * stop the main counter. And if we do that everytime someone diables/enables
1161 * RTC, we will have adverse effect on main kernel timer running on Timer 0.
1162 * So, for the time being, simulate the periodic interrupt in software.
1163 *
1164 * hpet_rtc_timer_init() is called for the first time and during subsequent
1165 * interuppts reinit happens through hpet_rtc_timer_reinit().
1166 */
1167int hpet_rtc_timer_init(void)
1168{
1169 unsigned int cfg, cnt;
1170 unsigned long flags;
1171
1172 if (!is_hpet_enabled())
1173 return 0;
1174 /*
1175 * Set the counter 1 and enable the interrupts.
1176 */
1177 if (PIE_on && (PIE_freq > DEFAULT_RTC_INT_FREQ))
1178 hpet_rtc_int_freq = PIE_freq;
1179 else
1180 hpet_rtc_int_freq = DEFAULT_RTC_INT_FREQ;
1181
1182 local_irq_save(flags);
Clemens Ladisch1447c272006-09-25 23:32:17 -07001183
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184 cnt = hpet_readl(HPET_COUNTER);
1185 cnt += ((hpet_tick*HZ)/hpet_rtc_int_freq);
1186 hpet_writel(cnt, HPET_T1_CMP);
Clemens Ladisch7811fb82005-10-30 15:03:36 -08001187 hpet_t1_cmp = cnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188
1189 cfg = hpet_readl(HPET_T1_CFG);
Clemens Ladisch5f819942005-10-30 15:03:36 -08001190 cfg &= ~HPET_TN_PERIODIC;
1191 cfg |= HPET_TN_ENABLE | HPET_TN_32BIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 hpet_writel(cfg, HPET_T1_CFG);
1193
Clemens Ladisch1447c272006-09-25 23:32:17 -07001194 local_irq_restore(flags);
1195
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 return 1;
1197}
1198
1199static void hpet_rtc_timer_reinit(void)
1200{
Clemens Ladisch1447c272006-09-25 23:32:17 -07001201 unsigned int cfg, cnt, ticks_per_int, lost_ints;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202
Clemens Ladischf00c96f2005-10-30 15:03:35 -08001203 if (unlikely(!(PIE_on | AIE_on | UIE_on))) {
1204 cfg = hpet_readl(HPET_T1_CFG);
1205 cfg &= ~HPET_TN_ENABLE;
1206 hpet_writel(cfg, HPET_T1_CFG);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207 return;
Clemens Ladischf00c96f2005-10-30 15:03:35 -08001208 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209
1210 if (PIE_on && (PIE_freq > DEFAULT_RTC_INT_FREQ))
1211 hpet_rtc_int_freq = PIE_freq;
1212 else
1213 hpet_rtc_int_freq = DEFAULT_RTC_INT_FREQ;
1214
1215 /* It is more accurate to use the comparator value than current count.*/
Clemens Ladisch1447c272006-09-25 23:32:17 -07001216 ticks_per_int = hpet_tick * HZ / hpet_rtc_int_freq;
1217 hpet_t1_cmp += ticks_per_int;
1218 hpet_writel(hpet_t1_cmp, HPET_T1_CMP);
1219
1220 /*
1221 * If the interrupt handler was delayed too long, the write above tries
1222 * to schedule the next interrupt in the past and the hardware would
1223 * not interrupt until the counter had wrapped around.
1224 * So we have to check that the comparator wasn't set to a past time.
1225 */
1226 cnt = hpet_readl(HPET_COUNTER);
1227 if (unlikely((int)(cnt - hpet_t1_cmp) > 0)) {
1228 lost_ints = (cnt - hpet_t1_cmp) / ticks_per_int + 1;
1229 /* Make sure that, even with the time needed to execute
1230 * this code, the next scheduled interrupt has been moved
1231 * back to the future: */
1232 lost_ints++;
1233
1234 hpet_t1_cmp += lost_ints * ticks_per_int;
1235 hpet_writel(hpet_t1_cmp, HPET_T1_CMP);
1236
1237 if (PIE_on)
1238 PIE_count += lost_ints;
1239
Roland Dreier3e94fb82007-02-13 13:26:25 +01001240 if (printk_ratelimit())
1241 printk(KERN_WARNING "rtc: lost some interrupts at %ldHz.\n",
1242 hpet_rtc_int_freq);
Clemens Ladisch1447c272006-09-25 23:32:17 -07001243 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244}
1245
1246/*
1247 * The functions below are called from rtc driver.
1248 * Return 0 if HPET is not being used.
1249 * Otherwise do the necessary changes and return 1.
1250 */
1251int hpet_mask_rtc_irq_bit(unsigned long bit_mask)
1252{
1253 if (!is_hpet_enabled())
1254 return 0;
1255
1256 if (bit_mask & RTC_UIE)
1257 UIE_on = 0;
1258 if (bit_mask & RTC_PIE)
1259 PIE_on = 0;
1260 if (bit_mask & RTC_AIE)
1261 AIE_on = 0;
1262
1263 return 1;
1264}
1265
1266int hpet_set_rtc_irq_bit(unsigned long bit_mask)
1267{
1268 int timer_init_reqd = 0;
1269
1270 if (!is_hpet_enabled())
1271 return 0;
1272
1273 if (!(PIE_on | AIE_on | UIE_on))
1274 timer_init_reqd = 1;
1275
1276 if (bit_mask & RTC_UIE) {
1277 UIE_on = 1;
1278 }
1279 if (bit_mask & RTC_PIE) {
1280 PIE_on = 1;
1281 PIE_count = 0;
1282 }
1283 if (bit_mask & RTC_AIE) {
1284 AIE_on = 1;
1285 }
1286
1287 if (timer_init_reqd)
1288 hpet_rtc_timer_init();
1289
1290 return 1;
1291}
1292
1293int hpet_set_alarm_time(unsigned char hrs, unsigned char min, unsigned char sec)
1294{
1295 if (!is_hpet_enabled())
1296 return 0;
1297
1298 alarm_time.tm_hour = hrs;
1299 alarm_time.tm_min = min;
1300 alarm_time.tm_sec = sec;
1301
1302 return 1;
1303}
1304
1305int hpet_set_periodic_freq(unsigned long freq)
1306{
1307 if (!is_hpet_enabled())
1308 return 0;
1309
1310 PIE_freq = freq;
1311 PIE_count = 0;
1312
1313 return 1;
1314}
1315
1316int hpet_rtc_dropped_irq(void)
1317{
1318 if (!is_hpet_enabled())
1319 return 0;
1320
1321 return 1;
1322}
1323
1324irqreturn_t hpet_rtc_interrupt(int irq, void *dev_id, struct pt_regs *regs)
1325{
1326 struct rtc_time curr_time;
1327 unsigned long rtc_int_flag = 0;
1328 int call_rtc_interrupt = 0;
1329
1330 hpet_rtc_timer_reinit();
1331
1332 if (UIE_on | AIE_on) {
1333 rtc_get_rtc_time(&curr_time);
1334 }
1335 if (UIE_on) {
1336 if (curr_time.tm_sec != prev_update_sec) {
1337 /* Set update int info, call real rtc int routine */
1338 call_rtc_interrupt = 1;
1339 rtc_int_flag = RTC_UF;
1340 prev_update_sec = curr_time.tm_sec;
1341 }
1342 }
1343 if (PIE_on) {
1344 PIE_count++;
1345 if (PIE_count >= hpet_rtc_int_freq/PIE_freq) {
1346 /* Set periodic int info, call real rtc int routine */
1347 call_rtc_interrupt = 1;
1348 rtc_int_flag |= RTC_PF;
1349 PIE_count = 0;
1350 }
1351 }
1352 if (AIE_on) {
1353 if ((curr_time.tm_sec == alarm_time.tm_sec) &&
1354 (curr_time.tm_min == alarm_time.tm_min) &&
1355 (curr_time.tm_hour == alarm_time.tm_hour)) {
1356 /* Set alarm int info, call real rtc int routine */
1357 call_rtc_interrupt = 1;
1358 rtc_int_flag |= RTC_AF;
1359 }
1360 }
1361 if (call_rtc_interrupt) {
1362 rtc_int_flag |= (RTC_IRQF | (RTC_NUM_INTS << 8));
David Howells7d12e782006-10-05 14:55:46 +01001363 rtc_interrupt(rtc_int_flag, dev_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364 }
1365 return IRQ_HANDLED;
1366}
1367#endif
1368
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369static int __init nohpet_setup(char *s)
1370{
1371 nohpet = 1;
OGAWA Hirofumi9b410462006-03-31 02:30:33 -08001372 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373}
1374
1375__setup("nohpet", nohpet_setup);
1376
Andi Kleen7fd67842006-02-16 23:42:07 +01001377int __init notsc_setup(char *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378{
1379 notsc = 1;
OGAWA Hirofumi9b410462006-03-31 02:30:33 -08001380 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381}
1382
1383__setup("notsc", notsc_setup);