blob: 9c7fba30dac1209745893cbb382c28bbcbb5baae [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86-64/kernel/time.c
3 *
4 * "High Precision Event Timer" based timekeeping.
5 *
6 * Copyright (c) 1991,1992,1995 Linus Torvalds
7 * Copyright (c) 1994 Alan Modra
8 * Copyright (c) 1995 Markus Kuhn
9 * Copyright (c) 1996 Ingo Molnar
10 * Copyright (c) 1998 Andrea Arcangeli
Vojtech Pavlik2f82bde42006-06-26 13:58:38 +020011 * Copyright (c) 2002,2006 Vojtech Pavlik
Linus Torvalds1da177e2005-04-16 15:20:36 -070012 * Copyright (c) 2003 Andi Kleen
13 * RTC support code taken from arch/i386/kernel/timers/time_hpet.c
14 */
15
16#include <linux/kernel.h>
17#include <linux/sched.h>
18#include <linux/interrupt.h>
19#include <linux/init.h>
20#include <linux/mc146818rtc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/time.h>
22#include <linux/ioport.h>
23#include <linux/module.h>
24#include <linux/device.h>
25#include <linux/sysdev.h>
26#include <linux/bcd.h>
Vojtech Pavlika670fad2006-09-26 10:52:28 +020027#include <linux/notifier.h>
28#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/kallsyms.h>
Andi Kleen312df5f2005-05-16 21:53:28 -070030#include <linux/acpi.h>
Andi Kleen8d916402005-05-31 14:39:26 -070031#ifdef CONFIG_ACPI
Andi Kleen312df5f2005-05-16 21:53:28 -070032#include <acpi/achware.h> /* for PM timer frequency */
Andi Kleen0e5f61b2006-07-29 21:42:37 +020033#include <acpi/acpi_bus.h>
Andi Kleen8d916402005-05-31 14:39:26 -070034#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <asm/8253pit.h>
36#include <asm/pgtable.h>
37#include <asm/vsyscall.h>
38#include <asm/timex.h>
39#include <asm/proto.h>
40#include <asm/hpet.h>
41#include <asm/sections.h>
42#include <linux/cpufreq.h>
43#include <linux/hpet.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <asm/apic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070045
Linus Torvalds1da177e2005-04-16 15:20:36 -070046#ifdef CONFIG_CPU_FREQ
47static void cpufreq_delayed_get(void);
48#endif
49extern void i8254_timer_resume(void);
50extern int using_apic_timer;
51
Vojtech Pavlika670fad2006-09-26 10:52:28 +020052static char *timename = NULL;
Andi Kleene8b91772006-02-26 04:18:49 +010053
Linus Torvalds1da177e2005-04-16 15:20:36 -070054DEFINE_SPINLOCK(rtc_lock);
Andi Kleen2ee60e172006-06-26 13:59:44 +020055EXPORT_SYMBOL(rtc_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070056DEFINE_SPINLOCK(i8253_lock);
57
Andi Kleen73dea472006-02-03 21:50:50 +010058int nohpet __initdata = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070059static int notsc __initdata = 0;
60
Vojtech Pavlik42211332006-06-26 13:58:32 +020061#define USEC_PER_TICK (USEC_PER_SEC / HZ)
62#define NSEC_PER_TICK (NSEC_PER_SEC / HZ)
63#define FSEC_PER_TICK (FSEC_PER_SEC / HZ)
64
65#define NS_SCALE 10 /* 2^10, carefully chosen */
66#define US_SCALE 32 /* 2^32, arbitralrily chosen */
67
Linus Torvalds1da177e2005-04-16 15:20:36 -070068unsigned int cpu_khz; /* TSC clocks / usec, not used here */
Andi Kleen2ee60e172006-06-26 13:59:44 +020069EXPORT_SYMBOL(cpu_khz);
Linus Torvalds1da177e2005-04-16 15:20:36 -070070static unsigned long hpet_period; /* fsecs / HPET clock */
71unsigned long hpet_tick; /* HPET clocks / interrupt */
Chris McDermott33042a92006-02-11 17:55:50 -080072int hpet_use_timer; /* Use counter of hpet for time keeping, otherwise PIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -070073unsigned long vxtime_hz = PIT_TICK_RATE;
74int report_lost_ticks; /* command line option */
75unsigned long long monotonic_base;
76
77struct vxtime_data __vxtime __section_vxtime; /* for vsyscalls */
78
79volatile unsigned long __jiffies __section_jiffies = INITIAL_JIFFIES;
Linus Torvalds1da177e2005-04-16 15:20:36 -070080struct timespec __xtime __section_xtime;
81struct timezone __sys_tz __section_sys_tz;
82
Linus Torvalds1da177e2005-04-16 15:20:36 -070083/*
84 * do_gettimeoffset() returns microseconds since last timer interrupt was
85 * triggered by hardware. A memory read of HPET is slower than a register read
86 * of TSC, but much more reliable. It's also synchronized to the timer
87 * interrupt. Note that do_gettimeoffset() may return more than hpet_tick, if a
88 * timer interrupt has happened already, but vxtime.trigger wasn't updated yet.
89 * This is not a problem, because jiffies hasn't updated either. They are bound
90 * together by xtime_lock.
91 */
92
93static inline unsigned int do_gettimeoffset_tsc(void)
94{
95 unsigned long t;
96 unsigned long x;
Andi Kleenc818a182006-01-11 22:45:24 +010097 t = get_cycles_sync();
Andi Kleen7351c0b2006-03-25 16:30:34 +010098 if (t < vxtime.last_tsc)
99 t = vxtime.last_tsc; /* hack */
Vojtech Pavlik42211332006-06-26 13:58:32 +0200100 x = ((t - vxtime.last_tsc) * vxtime.tsc_quot) >> US_SCALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 return x;
102}
103
104static inline unsigned int do_gettimeoffset_hpet(void)
105{
john stultza3a00752005-06-23 00:08:36 -0700106 /* cap counter read to one tick to avoid inconsistencies */
107 unsigned long counter = hpet_readl(HPET_COUNTER) - vxtime.last;
Vojtech Pavlik42211332006-06-26 13:58:32 +0200108 return (min(counter,hpet_tick) * vxtime.quot) >> US_SCALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109}
110
111unsigned int (*do_gettimeoffset)(void) = do_gettimeoffset_tsc;
112
113/*
114 * This version of gettimeofday() has microsecond resolution and better than
115 * microsecond precision, as we're using at least a 10 MHz (usually 14.31818
116 * MHz) HPET timer.
117 */
118
119void do_gettimeofday(struct timeval *tv)
120{
Atsushi Nemoto8ef38602006-09-30 23:28:31 -0700121 unsigned long seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 unsigned int sec, usec;
123
124 do {
125 seq = read_seqbegin(&xtime_lock);
126
127 sec = xtime.tv_sec;
Vojtech Pavlik42211332006-06-26 13:58:32 +0200128 usec = xtime.tv_nsec / NSEC_PER_USEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
130 /* i386 does some correction here to keep the clock
131 monotonous even when ntpd is fixing drift.
132 But they didn't work for me, there is a non monotonic
133 clock anyways with ntp.
134 I dropped all corrections now until a real solution can
135 be found. Note when you fix it here you need to do the same
136 in arch/x86_64/kernel/vsyscall.c and export all needed
137 variables in vmlinux.lds. -AK */
Atsushi Nemoto8ef38602006-09-30 23:28:31 -0700138 usec += do_gettimeoffset();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
140 } while (read_seqretry(&xtime_lock, seq));
141
Vojtech Pavlik42211332006-06-26 13:58:32 +0200142 tv->tv_sec = sec + usec / USEC_PER_SEC;
143 tv->tv_usec = usec % USEC_PER_SEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144}
145
146EXPORT_SYMBOL(do_gettimeofday);
147
148/*
149 * settimeofday() first undoes the correction that gettimeofday would do
150 * on the time, and then saves it. This is ugly, but has been like this for
151 * ages already.
152 */
153
154int do_settimeofday(struct timespec *tv)
155{
156 time_t wtm_sec, sec = tv->tv_sec;
157 long wtm_nsec, nsec = tv->tv_nsec;
158
159 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
160 return -EINVAL;
161
162 write_seqlock_irq(&xtime_lock);
163
Atsushi Nemoto8ef38602006-09-30 23:28:31 -0700164 nsec -= do_gettimeoffset() * NSEC_PER_USEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165
166 wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec);
167 wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec);
168
169 set_normalized_timespec(&xtime, sec, nsec);
170 set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec);
171
john stultzb149ee22005-09-06 15:17:46 -0700172 ntp_clear();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173
174 write_sequnlock_irq(&xtime_lock);
175 clock_was_set();
176 return 0;
177}
178
179EXPORT_SYMBOL(do_settimeofday);
180
181unsigned long profile_pc(struct pt_regs *regs)
182{
183 unsigned long pc = instruction_pointer(regs);
184
Andi Kleen31679f32006-09-26 10:52:28 +0200185 /* Assume the lock function has either no stack frame or a copy
186 of eflags from PUSHF
187 Eflags always has bits 22 and up cleared unlike kernel addresses. */
Andi Kleend5a26012006-07-28 14:44:42 +0200188 if (!user_mode(regs) && in_lock_functions(pc)) {
Andi Kleen31679f32006-09-26 10:52:28 +0200189 unsigned long *sp = (unsigned long *)regs->rsp;
190 if (sp[0] >> 22)
191 return sp[0];
192 if (sp[1] >> 22)
193 return sp[1];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 }
195 return pc;
196}
197EXPORT_SYMBOL(profile_pc);
198
199/*
200 * In order to set the CMOS clock precisely, set_rtc_mmss has to be called 500
201 * ms after the second nowtime has started, because when nowtime is written
202 * into the registers of the CMOS clock, it will jump to the next second
203 * precisely 500 ms later. Check the Motorola MC146818A or Dallas DS12887 data
204 * sheet for details.
205 */
206
207static void set_rtc_mmss(unsigned long nowtime)
208{
209 int real_seconds, real_minutes, cmos_minutes;
210 unsigned char control, freq_select;
211
212/*
213 * IRQs are disabled when we're called from the timer interrupt,
214 * no need for spin_lock_irqsave()
215 */
216
217 spin_lock(&rtc_lock);
218
219/*
220 * Tell the clock it's being set and stop it.
221 */
222
223 control = CMOS_READ(RTC_CONTROL);
224 CMOS_WRITE(control | RTC_SET, RTC_CONTROL);
225
226 freq_select = CMOS_READ(RTC_FREQ_SELECT);
227 CMOS_WRITE(freq_select | RTC_DIV_RESET2, RTC_FREQ_SELECT);
228
229 cmos_minutes = CMOS_READ(RTC_MINUTES);
230 BCD_TO_BIN(cmos_minutes);
231
232/*
233 * since we're only adjusting minutes and seconds, don't interfere with hour
234 * overflow. This avoids messing with unknown time zones but requires your RTC
235 * not to be off by more than 15 minutes. Since we're calling it only when
236 * our clock is externally synchronized using NTP, this shouldn't be a problem.
237 */
238
239 real_seconds = nowtime % 60;
240 real_minutes = nowtime / 60;
241 if (((abs(real_minutes - cmos_minutes) + 15) / 30) & 1)
242 real_minutes += 30; /* correct for half hour time zone */
243 real_minutes %= 60;
244
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 if (abs(real_minutes - cmos_minutes) >= 30) {
246 printk(KERN_WARNING "time.c: can't update CMOS clock "
247 "from %d to %d\n", cmos_minutes, real_minutes);
Andi Kleen28456ed2006-03-25 16:30:37 +0100248 } else {
Andi Kleen0b913172006-01-11 22:45:33 +0100249 BIN_TO_BCD(real_seconds);
250 BIN_TO_BCD(real_minutes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 CMOS_WRITE(real_seconds, RTC_SECONDS);
252 CMOS_WRITE(real_minutes, RTC_MINUTES);
253 }
254
255/*
256 * The following flags have to be released exactly in this order, otherwise the
257 * DS12887 (popular MC146818A clone with integrated battery and quartz) will
258 * not reset the oscillator and will not update precisely 500 ms later. You
259 * won't find this mentioned in the Dallas Semiconductor data sheets, but who
260 * believes data sheets anyway ... -- Markus Kuhn
261 */
262
263 CMOS_WRITE(control, RTC_CONTROL);
264 CMOS_WRITE(freq_select, RTC_FREQ_SELECT);
265
266 spin_unlock(&rtc_lock);
267}
268
269
270/* monotonic_clock(): returns # of nanoseconds passed since time_init()
271 * Note: This function is required to return accurate
272 * time even in the absence of multiple timer ticks.
273 */
Dimitri Sivanichcbf9b4b2006-09-26 10:52:34 +0200274static inline unsigned long long cycles_2_ns(unsigned long long cyc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275unsigned long long monotonic_clock(void)
276{
277 unsigned long seq;
278 u32 last_offset, this_offset, offset;
279 unsigned long long base;
280
281 if (vxtime.mode == VXTIME_HPET) {
282 do {
283 seq = read_seqbegin(&xtime_lock);
284
285 last_offset = vxtime.last;
286 base = monotonic_base;
john stultza3a00752005-06-23 00:08:36 -0700287 this_offset = hpet_readl(HPET_COUNTER);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 } while (read_seqretry(&xtime_lock, seq));
289 offset = (this_offset - last_offset);
Vojtech Pavlik42211332006-06-26 13:58:32 +0200290 offset *= NSEC_PER_TICK / hpet_tick;
Andi Kleen0b913172006-01-11 22:45:33 +0100291 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 do {
293 seq = read_seqbegin(&xtime_lock);
294
295 last_offset = vxtime.last_tsc;
296 base = monotonic_base;
297 } while (read_seqretry(&xtime_lock, seq));
Andi Kleenc818a182006-01-11 22:45:24 +0100298 this_offset = get_cycles_sync();
Dimitri Sivanichcbf9b4b2006-09-26 10:52:34 +0200299 offset = cycles_2_ns(this_offset - last_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 }
Andi Kleen7351c0b2006-03-25 16:30:34 +0100301 return base + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302}
303EXPORT_SYMBOL(monotonic_clock);
304
David Howells7d12e782006-10-05 14:55:46 +0100305static noinline void handle_lost_ticks(int lost)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306{
Andi Kleen7351c0b2006-03-25 16:30:34 +0100307 static long lost_count;
308 static int warned;
309 if (report_lost_ticks) {
310 printk(KERN_WARNING "time.c: Lost %d timer tick(s)! ", lost);
David Howells7d12e782006-10-05 14:55:46 +0100311 print_symbol("rip %s)\n", get_irq_regs()->rip);
Andi Kleen7351c0b2006-03-25 16:30:34 +0100312 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
Andi Kleen7351c0b2006-03-25 16:30:34 +0100314 if (lost_count == 1000 && !warned) {
315 printk(KERN_WARNING "warning: many lost ticks.\n"
316 KERN_WARNING "Your time source seems to be instable or "
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 "some driver is hogging interupts\n");
David Howells7d12e782006-10-05 14:55:46 +0100318 print_symbol("rip %s\n", get_irq_regs()->rip);
Andi Kleen7351c0b2006-03-25 16:30:34 +0100319 if (vxtime.mode == VXTIME_TSC && vxtime.hpet_address) {
320 printk(KERN_WARNING "Falling back to HPET\n");
321 if (hpet_use_timer)
322 vxtime.last = hpet_readl(HPET_T0_CMP) -
323 hpet_tick;
324 else
325 vxtime.last = hpet_readl(HPET_COUNTER);
326 vxtime.mode = VXTIME_HPET;
327 do_gettimeoffset = do_gettimeoffset_hpet;
328 }
329 /* else should fall back to PIT, but code missing. */
330 warned = 1;
331 } else
332 lost_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333
334#ifdef CONFIG_CPU_FREQ
Andi Kleen7351c0b2006-03-25 16:30:34 +0100335 /* In some cases the CPU can change frequency without us noticing
336 Give cpufreq a change to catch up. */
337 if ((lost_count+1) % 25 == 0)
338 cpufreq_delayed_get();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339#endif
340}
341
David Howells7d12e782006-10-05 14:55:46 +0100342void main_timer_handler(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343{
344 static unsigned long rtc_update = 0;
345 unsigned long tsc;
Andi Kleen9ede6b02006-03-25 16:29:31 +0100346 int delay = 0, offset = 0, lost = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347
348/*
349 * Here we are in the timer irq handler. We have irqs locally disabled (so we
350 * don't need spin_lock_irqsave()) but we don't know if the timer_bh is running
351 * on the other CPU, so we need a lock. We also need to lock the vsyscall
352 * variables, because both do_timer() and us change them -arca+vojtech
353 */
354
355 write_seqlock(&xtime_lock);
356
john stultza3a00752005-06-23 00:08:36 -0700357 if (vxtime.hpet_address)
358 offset = hpet_readl(HPET_COUNTER);
359
360 if (hpet_use_timer) {
361 /* if we're using the hpet timer functionality,
362 * we can more accurately know the counter value
363 * when the timer interrupt occured.
364 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
366 delay = hpet_readl(HPET_COUNTER) - offset;
Andi Kleen9ede6b02006-03-25 16:29:31 +0100367 } else if (!pmtmr_ioport) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368 spin_lock(&i8253_lock);
369 outb_p(0x00, 0x43);
370 delay = inb_p(0x40);
371 delay |= inb(0x40) << 8;
372 spin_unlock(&i8253_lock);
373 delay = LATCH - 1 - delay;
374 }
375
Andi Kleenc818a182006-01-11 22:45:24 +0100376 tsc = get_cycles_sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377
378 if (vxtime.mode == VXTIME_HPET) {
379 if (offset - vxtime.last > hpet_tick) {
380 lost = (offset - vxtime.last) / hpet_tick - 1;
381 }
382
383 monotonic_base +=
Vojtech Pavlik42211332006-06-26 13:58:32 +0200384 (offset - vxtime.last) * NSEC_PER_TICK / hpet_tick;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385
386 vxtime.last = offset;
Andi Kleen312df5f2005-05-16 21:53:28 -0700387#ifdef CONFIG_X86_PM_TIMER
388 } else if (vxtime.mode == VXTIME_PMTMR) {
389 lost = pmtimer_mark_offset();
390#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 } else {
392 offset = (((tsc - vxtime.last_tsc) *
Vojtech Pavlik42211332006-06-26 13:58:32 +0200393 vxtime.tsc_quot) >> US_SCALE) - USEC_PER_TICK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394
395 if (offset < 0)
396 offset = 0;
397
Vojtech Pavlik42211332006-06-26 13:58:32 +0200398 if (offset > USEC_PER_TICK) {
399 lost = offset / USEC_PER_TICK;
400 offset %= USEC_PER_TICK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 }
402
Dimitri Sivanichcbf9b4b2006-09-26 10:52:34 +0200403 monotonic_base += cycles_2_ns(tsc - vxtime.last_tsc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404
405 vxtime.last_tsc = tsc - vxtime.quot * delay / vxtime.tsc_quot;
406
407 if ((((tsc - vxtime.last_tsc) *
Vojtech Pavlik42211332006-06-26 13:58:32 +0200408 vxtime.tsc_quot) >> US_SCALE) < offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 vxtime.last_tsc = tsc -
Vojtech Pavlik42211332006-06-26 13:58:32 +0200410 (((long) offset << US_SCALE) / vxtime.tsc_quot) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 }
412
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700413 if (lost > 0)
David Howells7d12e782006-10-05 14:55:46 +0100414 handle_lost_ticks(lost);
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700415 else
416 lost = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
418/*
419 * Do the timer stuff.
420 */
421
Atsushi Nemoto3171a032006-09-29 02:00:32 -0700422 do_timer(lost + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423#ifndef CONFIG_SMP
David Howells7d12e782006-10-05 14:55:46 +0100424 update_process_times(user_mode(get_irq_regs()));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425#endif
426
427/*
428 * In the SMP case we use the local APIC timer interrupt to do the profiling,
429 * except when we simulate SMP mode on a uniprocessor system, in that case we
430 * have to call the local interrupt handler.
431 */
432
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433 if (!using_apic_timer)
David Howells7d12e782006-10-05 14:55:46 +0100434 smp_local_timer_interrupt();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435
436/*
437 * If we have an externally synchronized Linux clock, then update CMOS clock
438 * accordingly every ~11 minutes. set_rtc_mmss() will be called in the jiffy
439 * closest to exactly 500 ms before the next second. If the update fails, we
440 * don't care, as it'll be updated on the next turn, and the problem (time way
441 * off) isn't likely to go away much sooner anyway.
442 */
443
john stultzb149ee22005-09-06 15:17:46 -0700444 if (ntp_synced() && xtime.tv_sec > rtc_update &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 abs(xtime.tv_nsec - 500000000) <= tick_nsec / 2) {
446 set_rtc_mmss(xtime.tv_sec);
447 rtc_update = xtime.tv_sec + 660;
448 }
449
450 write_sequnlock(&xtime_lock);
Andi Kleen73dea472006-02-03 21:50:50 +0100451}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452
David Howells7d12e782006-10-05 14:55:46 +0100453static irqreturn_t timer_interrupt(int irq, void *dev_id)
Andi Kleen73dea472006-02-03 21:50:50 +0100454{
455 if (apic_runs_main_timer > 1)
456 return IRQ_HANDLED;
David Howells7d12e782006-10-05 14:55:46 +0100457 main_timer_handler();
Venkatesh Pallipadid25bf7e2006-01-11 22:44:24 +0100458 if (using_apic_timer)
459 smp_send_timer_broadcast_ipi();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 return IRQ_HANDLED;
461}
462
Ravikiran G Thirumalai68ed0042006-03-22 00:07:38 -0800463static unsigned int cyc2ns_scale __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464
Mathieu Desnoyersdacb16b2005-10-30 14:59:25 -0800465static inline void set_cyc2ns_scale(unsigned long cpu_khz)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466{
Vojtech Pavlik42211332006-06-26 13:58:32 +0200467 cyc2ns_scale = (NSEC_PER_MSEC << NS_SCALE) / cpu_khz;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468}
469
470static inline unsigned long long cycles_2_ns(unsigned long long cyc)
471{
Vojtech Pavlik42211332006-06-26 13:58:32 +0200472 return (cyc * cyc2ns_scale) >> NS_SCALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473}
474
475unsigned long long sched_clock(void)
476{
477 unsigned long a = 0;
478
479#if 0
480 /* Don't do a HPET read here. Using TSC always is much faster
481 and HPET may not be mapped yet when the scheduler first runs.
482 Disadvantage is a small drift between CPUs in some configurations,
483 but that should be tolerable. */
484 if (__vxtime.mode == VXTIME_HPET)
Vojtech Pavlik42211332006-06-26 13:58:32 +0200485 return (hpet_readl(HPET_COUNTER) * vxtime.quot) >> US_SCALE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486#endif
487
488 /* Could do CPU core sync here. Opteron can execute rdtsc speculatively,
489 which means it is not completely exact and may not be monotonous between
490 CPUs. But the errors should be too small to matter for scheduling
491 purposes. */
492
493 rdtscll(a);
494 return cycles_2_ns(a);
495}
496
Andi Kleenbdf2b1c2006-01-11 22:46:39 +0100497static unsigned long get_cmos_time(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498{
Matt Mackall641f71f2006-03-28 01:56:01 -0800499 unsigned int year, mon, day, hour, min, sec;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 unsigned long flags;
Alexey Starikovskiyad718602007-02-02 19:48:19 +0300501 unsigned century = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 spin_lock_irqsave(&rtc_lock, flags);
504
Matt Mackall641f71f2006-03-28 01:56:01 -0800505 do {
506 sec = CMOS_READ(RTC_SECONDS);
507 min = CMOS_READ(RTC_MINUTES);
508 hour = CMOS_READ(RTC_HOURS);
509 day = CMOS_READ(RTC_DAY_OF_MONTH);
510 mon = CMOS_READ(RTC_MONTH);
511 year = CMOS_READ(RTC_YEAR);
Andi Kleen6954bee2006-03-25 16:30:31 +0100512#ifdef CONFIG_ACPI
Alexey Starikovskiyad718602007-02-02 19:48:19 +0300513 if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID &&
514 acpi_gbl_FADT.century)
515 century = CMOS_READ(acpi_gbl_FADT.century);
Andi Kleen6954bee2006-03-25 16:30:31 +0100516#endif
Matt Mackall641f71f2006-03-28 01:56:01 -0800517 } while (sec != CMOS_READ(RTC_SECONDS));
Andi Kleen6954bee2006-03-25 16:30:31 +0100518
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 spin_unlock_irqrestore(&rtc_lock, flags);
520
Andi Kleen0b913172006-01-11 22:45:33 +0100521 /*
522 * We know that x86-64 always uses BCD format, no need to check the
523 * config register.
Andi Kleen7351c0b2006-03-25 16:30:34 +0100524 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525
Andi Kleen0b913172006-01-11 22:45:33 +0100526 BCD_TO_BIN(sec);
527 BCD_TO_BIN(min);
528 BCD_TO_BIN(hour);
529 BCD_TO_BIN(day);
530 BCD_TO_BIN(mon);
531 BCD_TO_BIN(year);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532
Alexey Starikovskiyad718602007-02-02 19:48:19 +0300533 if (century) {
534 BCD_TO_BIN(century);
535 year += century * 100;
536 printk(KERN_INFO "Extended CMOS year: %d\n", century * 100);
Andi Kleen6954bee2006-03-25 16:30:31 +0100537 } else {
538 /*
539 * x86-64 systems only exists since 2002.
540 * This will work up to Dec 31, 2100
541 */
542 year += 2000;
543 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544
545 return mktime(year, mon, day, hour, min, sec);
546}
547
548#ifdef CONFIG_CPU_FREQ
549
550/* Frequency scaling support. Adjust the TSC based timer when the cpu frequency
551 changes.
552
553 RED-PEN: On SMP we assume all CPUs run with the same frequency. It's
554 not that important because current Opteron setups do not support
555 scaling on SMP anyroads.
556
557 Should fix up last_tsc too. Currently gettimeofday in the
558 first tick after the change will be slightly wrong. */
559
560#include <linux/workqueue.h>
561
562static unsigned int cpufreq_delayed_issched = 0;
563static unsigned int cpufreq_init = 0;
564static struct work_struct cpufreq_delayed_get_work;
565
David Howells65f27f32006-11-22 14:55:48 +0000566static void handle_cpufreq_delayed_get(struct work_struct *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567{
568 unsigned int cpu;
569 for_each_online_cpu(cpu) {
570 cpufreq_get(cpu);
571 }
572 cpufreq_delayed_issched = 0;
573}
574
575/* if we notice lost ticks, schedule a call to cpufreq_get() as it tries
576 * to verify the CPU frequency the timing core thinks the CPU is running
577 * at is still correct.
578 */
579static void cpufreq_delayed_get(void)
580{
581 static int warned;
582 if (cpufreq_init && !cpufreq_delayed_issched) {
583 cpufreq_delayed_issched = 1;
584 if (!warned) {
585 warned = 1;
Andi Kleen7351c0b2006-03-25 16:30:34 +0100586 printk(KERN_DEBUG
587 "Losing some ticks... checking if CPU frequency changed.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 }
589 schedule_work(&cpufreq_delayed_get_work);
590 }
591}
592
593static unsigned int ref_freq = 0;
594static unsigned long loops_per_jiffy_ref = 0;
595
596static unsigned long cpu_khz_ref = 0;
597
598static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
599 void *data)
600{
601 struct cpufreq_freqs *freq = data;
602 unsigned long *lpj, dummy;
603
Andi Kleenc29601e2005-04-16 15:25:05 -0700604 if (cpu_has(&cpu_data[freq->cpu], X86_FEATURE_CONSTANT_TSC))
605 return 0;
606
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 lpj = &dummy;
608 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
609#ifdef CONFIG_SMP
Andi Kleen7351c0b2006-03-25 16:30:34 +0100610 lpj = &cpu_data[freq->cpu].loops_per_jiffy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611#else
Andi Kleen7351c0b2006-03-25 16:30:34 +0100612 lpj = &boot_cpu_data.loops_per_jiffy;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613#endif
614
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 if (!ref_freq) {
616 ref_freq = freq->old;
617 loops_per_jiffy_ref = *lpj;
618 cpu_khz_ref = cpu_khz;
619 }
620 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
621 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
622 (val == CPUFREQ_RESUMECHANGE)) {
623 *lpj =
624 cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
625
626 cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new);
627 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
Vojtech Pavlik42211332006-06-26 13:58:32 +0200628 vxtime.tsc_quot = (USEC_PER_MSEC << US_SCALE) / cpu_khz;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 }
630
Mathieu Desnoyersdacb16b2005-10-30 14:59:25 -0800631 set_cyc2ns_scale(cpu_khz_ref);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632
633 return 0;
634}
635
636static struct notifier_block time_cpufreq_notifier_block = {
637 .notifier_call = time_cpufreq_notifier
638};
639
640static int __init cpufreq_tsc(void)
641{
David Howells65f27f32006-11-22 14:55:48 +0000642 INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 if (!cpufreq_register_notifier(&time_cpufreq_notifier_block,
644 CPUFREQ_TRANSITION_NOTIFIER))
645 cpufreq_init = 1;
646 return 0;
647}
648
649core_initcall(cpufreq_tsc);
650
651#endif
652
653/*
654 * calibrate_tsc() calibrates the processor TSC in a very simple way, comparing
655 * it to the HPET timer of known frequency.
656 */
657
658#define TICK_COUNT 100000000
Jack Steinered5316d2007-01-11 01:52:44 +0100659#define TICK_MIN 5000
Jack Steiner2f7a2a72007-02-13 13:26:24 +0100660#define MAX_READ_RETRIES 5
Jack Steinered5316d2007-01-11 01:52:44 +0100661
662/*
663 * Some platforms take periodic SMI interrupts with 5ms duration. Make sure none
664 * occurs between the reads of the hpet & TSC.
665 */
666static void __init read_hpet_tsc(int *hpet, int *tsc)
667{
Jack Steiner2f7a2a72007-02-13 13:26:24 +0100668 int tsc1, tsc2, hpet1, retries = 0;
669 static int msg;
Jack Steinered5316d2007-01-11 01:52:44 +0100670
671 do {
672 tsc1 = get_cycles_sync();
673 hpet1 = hpet_readl(HPET_COUNTER);
674 tsc2 = get_cycles_sync();
Jack Steiner2f7a2a72007-02-13 13:26:24 +0100675 } while (tsc2 - tsc1 > TICK_MIN && retries++ < MAX_READ_RETRIES);
676 if (retries >= MAX_READ_RETRIES && !msg++)
677 printk(KERN_WARNING
678 "hpet.c: exceeded max retries to read HPET & TSC\n");
Jack Steinered5316d2007-01-11 01:52:44 +0100679 *hpet = hpet1;
680 *tsc = tsc2;
681}
682
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
684static unsigned int __init hpet_calibrate_tsc(void)
685{
686 int tsc_start, hpet_start;
687 int tsc_now, hpet_now;
688 unsigned long flags;
689
690 local_irq_save(flags);
691 local_irq_disable();
692
Jack Steinered5316d2007-01-11 01:52:44 +0100693 read_hpet_tsc(&hpet_start, &tsc_start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694
695 do {
696 local_irq_disable();
Jack Steinered5316d2007-01-11 01:52:44 +0100697 read_hpet_tsc(&hpet_now, &tsc_now);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 local_irq_restore(flags);
699 } while ((tsc_now - tsc_start) < TICK_COUNT &&
700 (hpet_now - hpet_start) < TICK_COUNT);
701
702 return (tsc_now - tsc_start) * 1000000000L
703 / ((hpet_now - hpet_start) * hpet_period / 1000);
704}
705
706
707/*
708 * pit_calibrate_tsc() uses the speaker output (channel 2) of
709 * the PIT. This is better than using the timer interrupt output,
710 * because we can read the value of the speaker with just one inb(),
711 * where we need three i/o operations for the interrupt channel.
712 * We count how many ticks the TSC does in 50 ms.
713 */
714
715static unsigned int __init pit_calibrate_tsc(void)
716{
717 unsigned long start, end;
718 unsigned long flags;
719
720 spin_lock_irqsave(&i8253_lock, flags);
721
722 outb((inb(0x61) & ~0x02) | 0x01, 0x61);
723
724 outb(0xb0, 0x43);
725 outb((PIT_TICK_RATE / (1000 / 50)) & 0xff, 0x42);
726 outb((PIT_TICK_RATE / (1000 / 50)) >> 8, 0x42);
Andi Kleenc818a182006-01-11 22:45:24 +0100727 start = get_cycles_sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 while ((inb(0x61) & 0x20) == 0);
Andi Kleenc818a182006-01-11 22:45:24 +0100729 end = get_cycles_sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730
731 spin_unlock_irqrestore(&i8253_lock, flags);
732
733 return (end - start) / 50;
734}
735
736#ifdef CONFIG_HPET
737static __init int late_hpet_init(void)
738{
739 struct hpet_data hd;
740 unsigned int ntimer;
741
742 if (!vxtime.hpet_address)
Andi Kleen3d34ee62006-04-07 19:50:06 +0200743 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744
745 memset(&hd, 0, sizeof (hd));
746
747 ntimer = hpet_readl(HPET_ID);
748 ntimer = (ntimer & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT;
749 ntimer++;
750
751 /*
752 * Register with driver.
753 * Timer0 and Timer1 is used by platform.
754 */
755 hd.hd_phys_address = vxtime.hpet_address;
Al Virodd42b152006-02-01 07:30:33 -0500756 hd.hd_address = (void __iomem *)fix_to_virt(FIX_HPET_BASE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 hd.hd_nirqs = ntimer;
758 hd.hd_flags = HPET_DATA_PLATFORM;
759 hpet_reserve_timer(&hd, 0);
760#ifdef CONFIG_HPET_EMULATE_RTC
761 hpet_reserve_timer(&hd, 1);
762#endif
763 hd.hd_irq[0] = HPET_LEGACY_8254;
764 hd.hd_irq[1] = HPET_LEGACY_RTC;
765 if (ntimer > 2) {
766 struct hpet *hpet;
767 struct hpet_timer *timer;
768 int i;
769
770 hpet = (struct hpet *) fix_to_virt(FIX_HPET_BASE);
Andi Kleen7351c0b2006-03-25 16:30:34 +0100771 timer = &hpet->hpet_timers[2];
772 for (i = 2; i < ntimer; timer++, i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773 hd.hd_irq[i] = (timer->hpet_config &
774 Tn_INT_ROUTE_CNF_MASK) >>
775 Tn_INT_ROUTE_CNF_SHIFT;
776
777 }
778
779 hpet_alloc(&hd);
780 return 0;
781}
782fs_initcall(late_hpet_init);
783#endif
784
785static int hpet_timer_stop_set_go(unsigned long tick)
786{
787 unsigned int cfg;
788
789/*
790 * Stop the timers and reset the main counter.
791 */
792
793 cfg = hpet_readl(HPET_CFG);
794 cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY);
795 hpet_writel(cfg, HPET_CFG);
796 hpet_writel(0, HPET_COUNTER);
797 hpet_writel(0, HPET_COUNTER + 4);
798
799/*
800 * Set up timer 0, as periodic with first interrupt to happen at hpet_tick,
801 * and period also hpet_tick.
802 */
john stultza3a00752005-06-23 00:08:36 -0700803 if (hpet_use_timer) {
804 hpet_writel(HPET_TN_ENABLE | HPET_TN_PERIODIC | HPET_TN_SETVAL |
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805 HPET_TN_32BIT, HPET_T0_CFG);
Vojtech Pavlikb2df3dd2006-06-26 13:58:35 +0200806 hpet_writel(hpet_tick, HPET_T0_CMP); /* next interrupt */
807 hpet_writel(hpet_tick, HPET_T0_CMP); /* period */
john stultza3a00752005-06-23 00:08:36 -0700808 cfg |= HPET_CFG_LEGACY;
809 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810/*
811 * Go!
812 */
813
john stultza3a00752005-06-23 00:08:36 -0700814 cfg |= HPET_CFG_ENABLE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 hpet_writel(cfg, HPET_CFG);
816
817 return 0;
818}
819
820static int hpet_init(void)
821{
822 unsigned int id;
823
824 if (!vxtime.hpet_address)
825 return -1;
826 set_fixmap_nocache(FIX_HPET_BASE, vxtime.hpet_address);
827 __set_fixmap(VSYSCALL_HPET, vxtime.hpet_address, PAGE_KERNEL_VSYSCALL_NOCACHE);
828
829/*
830 * Read the period, compute tick and quotient.
831 */
832
833 id = hpet_readl(HPET_ID);
834
john stultza3a00752005-06-23 00:08:36 -0700835 if (!(id & HPET_ID_VENDOR) || !(id & HPET_ID_NUMBER))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 return -1;
837
838 hpet_period = hpet_readl(HPET_PERIOD);
839 if (hpet_period < 100000 || hpet_period > 100000000)
840 return -1;
841
Vojtech Pavlik42211332006-06-26 13:58:32 +0200842 hpet_tick = (FSEC_PER_TICK + hpet_period / 2) / hpet_period;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843
john stultza3a00752005-06-23 00:08:36 -0700844 hpet_use_timer = (id & HPET_ID_LEGSUP);
845
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 return hpet_timer_stop_set_go(hpet_tick);
847}
848
849static int hpet_reenable(void)
850{
851 return hpet_timer_stop_set_go(hpet_tick);
852}
853
Andi Kleen73dea472006-02-03 21:50:50 +0100854#define PIT_MODE 0x43
855#define PIT_CH0 0x40
856
857static void __init __pit_init(int val, u8 mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858{
859 unsigned long flags;
860
861 spin_lock_irqsave(&i8253_lock, flags);
Andi Kleen73dea472006-02-03 21:50:50 +0100862 outb_p(mode, PIT_MODE);
863 outb_p(val & 0xff, PIT_CH0); /* LSB */
864 outb_p(val >> 8, PIT_CH0); /* MSB */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865 spin_unlock_irqrestore(&i8253_lock, flags);
866}
867
Andi Kleen73dea472006-02-03 21:50:50 +0100868void __init pit_init(void)
869{
870 __pit_init(LATCH, 0x34); /* binary, mode 2, LSB/MSB, ch 0 */
871}
872
873void __init pit_stop_interrupt(void)
874{
875 __pit_init(0, 0x30); /* mode 0 */
876}
877
878void __init stop_timer_interrupt(void)
879{
880 char *name;
881 if (vxtime.hpet_address) {
882 name = "HPET";
883 hpet_timer_stop_set_go(0);
884 } else {
885 name = "PIT";
886 pit_stop_interrupt();
887 }
888 printk(KERN_INFO "timer: %s interrupt stopped.\n", name);
889}
890
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891int __init time_setup(char *str)
892{
893 report_lost_ticks = 1;
894 return 1;
895}
896
897static struct irqaction irq0 = {
Thomas Gleixnerb1e05aa2006-07-01 19:29:29 -0700898 timer_interrupt, IRQF_DISABLED, CPU_MASK_NONE, "timer", NULL, NULL
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899};
900
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901void __init time_init(void)
902{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 if (nohpet)
904 vxtime.hpet_address = 0;
905
906 xtime.tv_sec = get_cmos_time();
907 xtime.tv_nsec = 0;
908
909 set_normalized_timespec(&wall_to_monotonic,
910 -xtime.tv_sec, -xtime.tv_nsec);
911
john stultza3a00752005-06-23 00:08:36 -0700912 if (!hpet_init())
Vojtech Pavlik42211332006-06-26 13:58:32 +0200913 vxtime_hz = (FSEC_PER_SEC + hpet_period / 2) / hpet_period;
Andi Kleen68e18892005-12-12 22:17:07 -0800914 else
915 vxtime.hpet_address = 0;
john stultza3a00752005-06-23 00:08:36 -0700916
917 if (hpet_use_timer) {
Jordan Hargraveb20367a2006-04-07 19:50:18 +0200918 /* set tick_nsec to use the proper rate for HPET */
919 tick_nsec = TICK_NSEC_HPET;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 cpu_khz = hpet_calibrate_tsc();
921 timename = "HPET";
Andi Kleen312df5f2005-05-16 21:53:28 -0700922#ifdef CONFIG_X86_PM_TIMER
john stultzfd495472005-12-12 22:17:13 -0800923 } else if (pmtmr_ioport && !vxtime.hpet_address) {
Andi Kleen312df5f2005-05-16 21:53:28 -0700924 vxtime_hz = PM_TIMER_FREQUENCY;
925 timename = "PM";
926 pit_init();
927 cpu_khz = pit_calibrate_tsc();
928#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 } else {
930 pit_init();
931 cpu_khz = pit_calibrate_tsc();
932 timename = "PIT";
933 }
934
Andi Kleene8b91772006-02-26 04:18:49 +0100935 vxtime.mode = VXTIME_TSC;
Vojtech Pavlik42211332006-06-26 13:58:32 +0200936 vxtime.quot = (USEC_PER_SEC << US_SCALE) / vxtime_hz;
937 vxtime.tsc_quot = (USEC_PER_MSEC << US_SCALE) / cpu_khz;
Andi Kleenc818a182006-01-11 22:45:24 +0100938 vxtime.last_tsc = get_cycles_sync();
Mathieu Desnoyersdacb16b2005-10-30 14:59:25 -0800939 set_cyc2ns_scale(cpu_khz);
Dimitri Sivanichcbf9b4b2006-09-26 10:52:34 +0200940 setup_irq(0, &irq0);
Vojtech Pavlika670fad2006-09-26 10:52:28 +0200941
942#ifndef CONFIG_SMP
943 time_init_gtod();
944#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945}
946
Andi Kleena8ab26f2005-04-16 15:25:19 -0700947/*
Andi Kleen312df5f2005-05-16 21:53:28 -0700948 * Make an educated guess if the TSC is trustworthy and synchronized
949 * over all CPUs.
950 */
Shaohua Li396bd502006-02-03 21:51:20 +0100951__cpuinit int unsynchronized_tsc(void)
Andi Kleen312df5f2005-05-16 21:53:28 -0700952{
953#ifdef CONFIG_SMP
Vojtech Pavlikf8bf3c62006-06-26 13:58:23 +0200954 if (apic_is_clustered_box())
Andi Kleen312df5f2005-05-16 21:53:28 -0700955 return 1;
Andi Kleen312df5f2005-05-16 21:53:28 -0700956#endif
Andi Kleen0e5f61b2006-07-29 21:42:37 +0200957 /* Most intel systems have synchronized TSCs except for
958 multi node systems */
959 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
960#ifdef CONFIG_ACPI
961 /* But TSC doesn't tick in C3 so don't use it there */
Alexey Starikovskiyad718602007-02-02 19:48:19 +0300962 if (acpi_gbl_FADT.header.length > 0 && acpi_gbl_FADT.C3latency < 1000)
Andi Kleen0e5f61b2006-07-29 21:42:37 +0200963 return 1;
964#endif
965 return 0;
966 }
967
Andi Kleen312df5f2005-05-16 21:53:28 -0700968 /* Assume multi socket systems are not synchronized */
Andi Kleen737c5c32006-01-11 22:45:15 +0100969 return num_present_cpus() > 1;
Andi Kleen312df5f2005-05-16 21:53:28 -0700970}
971
972/*
Andi Kleene8b91772006-02-26 04:18:49 +0100973 * Decide what mode gettimeofday should use.
Andi Kleena8ab26f2005-04-16 15:25:19 -0700974 */
Vojtech Pavlika670fad2006-09-26 10:52:28 +0200975void time_init_gtod(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976{
977 char *timetype;
978
Andi Kleen312df5f2005-05-16 21:53:28 -0700979 if (unsynchronized_tsc())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 notsc = 1;
Vojtech Pavlika670fad2006-09-26 10:52:28 +0200981
Vojtech Pavlikc08c8202006-09-26 10:52:28 +0200982 if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP))
983 vgetcpu_mode = VGETCPU_RDTSCP;
984 else
985 vgetcpu_mode = VGETCPU_LSL;
986
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987 if (vxtime.hpet_address && notsc) {
john stultza3a00752005-06-23 00:08:36 -0700988 timetype = hpet_use_timer ? "HPET" : "PIT/HPET";
Chris McDermott33042a92006-02-11 17:55:50 -0800989 if (hpet_use_timer)
990 vxtime.last = hpet_readl(HPET_T0_CMP) - hpet_tick;
991 else
992 vxtime.last = hpet_readl(HPET_COUNTER);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 vxtime.mode = VXTIME_HPET;
994 do_gettimeoffset = do_gettimeoffset_hpet;
Andi Kleen312df5f2005-05-16 21:53:28 -0700995#ifdef CONFIG_X86_PM_TIMER
996 /* Using PM for gettimeofday is quite slow, but we have no other
997 choice because the TSC is too unreliable on some systems. */
998 } else if (pmtmr_ioport && !vxtime.hpet_address && notsc) {
999 timetype = "PM";
1000 do_gettimeoffset = do_gettimeoffset_pm;
1001 vxtime.mode = VXTIME_PMTMR;
1002 sysctl_vsyscall = 0;
1003 printk(KERN_INFO "Disabling vsyscall due to use of PM timer\n");
1004#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 } else {
john stultza3a00752005-06-23 00:08:36 -07001006 timetype = hpet_use_timer ? "HPET/TSC" : "PIT/TSC";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 vxtime.mode = VXTIME_TSC;
1008 }
Vojtech Pavlika670fad2006-09-26 10:52:28 +02001009
1010 printk(KERN_INFO "time.c: Using %ld.%06ld MHz WALL %s GTOD %s timer.\n",
1011 vxtime_hz / 1000000, vxtime_hz % 1000000, timename, timetype);
1012 printk(KERN_INFO "time.c: Detected %d.%03d MHz processor.\n",
1013 cpu_khz / 1000, cpu_khz % 1000);
1014 vxtime.quot = (USEC_PER_SEC << US_SCALE) / vxtime_hz;
1015 vxtime.tsc_quot = (USEC_PER_MSEC << US_SCALE) / cpu_khz;
1016 vxtime.last_tsc = get_cycles_sync();
1017
1018 set_cyc2ns_scale(cpu_khz);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019}
1020
1021__setup("report_lost_ticks", time_setup);
1022
1023static long clock_cmos_diff;
1024static unsigned long sleep_start;
1025
Andi Kleen0b913172006-01-11 22:45:33 +01001026/*
1027 * sysfs support for the timer.
1028 */
1029
Pavel Machek0b9c33a2005-04-16 15:25:31 -07001030static int timer_suspend(struct sys_device *dev, pm_message_t state)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031{
1032 /*
1033 * Estimate time zone so that set_time can update the clock
1034 */
1035 long cmos_time = get_cmos_time();
1036
1037 clock_cmos_diff = -cmos_time;
1038 clock_cmos_diff += get_seconds();
1039 sleep_start = cmos_time;
1040 return 0;
1041}
1042
1043static int timer_resume(struct sys_device *dev)
1044{
1045 unsigned long flags;
1046 unsigned long sec;
1047 unsigned long ctime = get_cmos_time();
Rafael J. Wysocki34464a52006-09-26 10:52:37 +02001048 long sleep_length = (ctime - sleep_start) * HZ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049
Rafael J. Wysocki34464a52006-09-26 10:52:37 +02001050 if (sleep_length < 0) {
1051 printk(KERN_WARNING "Time skew detected in timer resume!\n");
1052 /* The time after the resume must not be earlier than the time
1053 * before the suspend or some nasty things will happen
1054 */
1055 sleep_length = 0;
1056 ctime = sleep_start;
1057 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 if (vxtime.hpet_address)
1059 hpet_reenable();
1060 else
1061 i8254_timer_resume();
1062
1063 sec = ctime + clock_cmos_diff;
1064 write_seqlock_irqsave(&xtime_lock,flags);
1065 xtime.tv_sec = sec;
1066 xtime.tv_nsec = 0;
Shaohua Li0dd2ea92006-02-03 21:50:56 +01001067 if (vxtime.mode == VXTIME_HPET) {
1068 if (hpet_use_timer)
1069 vxtime.last = hpet_readl(HPET_T0_CMP) - hpet_tick;
1070 else
1071 vxtime.last = hpet_readl(HPET_COUNTER);
1072#ifdef CONFIG_X86_PM_TIMER
1073 } else if (vxtime.mode == VXTIME_PMTMR) {
1074 pmtimer_resume();
1075#endif
1076 } else
1077 vxtime.last_tsc = get_cycles_sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078 write_sequnlock_irqrestore(&xtime_lock,flags);
1079 jiffies += sleep_length;
Shaohua Li0dd2ea92006-02-03 21:50:56 +01001080 monotonic_base += sleep_length * (NSEC_PER_SEC/HZ);
Ingo Molnar8446f1d2005-09-06 15:16:27 -07001081 touch_softlockup_watchdog();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082 return 0;
1083}
1084
1085static struct sysdev_class timer_sysclass = {
1086 .resume = timer_resume,
1087 .suspend = timer_suspend,
1088 set_kset_name("timer"),
1089};
1090
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091/* XXX this driverfs stuff should probably go elsewhere later -john */
1092static struct sys_device device_timer = {
1093 .id = 0,
1094 .cls = &timer_sysclass,
1095};
1096
1097static int time_init_device(void)
1098{
1099 int error = sysdev_class_register(&timer_sysclass);
1100 if (!error)
1101 error = sysdev_register(&device_timer);
1102 return error;
1103}
1104
1105device_initcall(time_init_device);
1106
1107#ifdef CONFIG_HPET_EMULATE_RTC
1108/* HPET in LegacyReplacement Mode eats up RTC interrupt line. When, HPET
1109 * is enabled, we support RTC interrupt functionality in software.
1110 * RTC has 3 kinds of interrupts:
1111 * 1) Update Interrupt - generate an interrupt, every sec, when RTC clock
1112 * is updated
1113 * 2) Alarm Interrupt - generate an interrupt at a specific time of day
1114 * 3) Periodic Interrupt - generate periodic interrupt, with frequencies
1115 * 2Hz-8192Hz (2Hz-64Hz for non-root user) (all freqs in powers of 2)
1116 * (1) and (2) above are implemented using polling at a frequency of
1117 * 64 Hz. The exact frequency is a tradeoff between accuracy and interrupt
1118 * overhead. (DEFAULT_RTC_INT_FREQ)
1119 * For (3), we use interrupts at 64Hz or user specified periodic
1120 * frequency, whichever is higher.
1121 */
1122#include <linux/rtc.h>
1123
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124#define DEFAULT_RTC_INT_FREQ 64
1125#define RTC_NUM_INTS 1
1126
1127static unsigned long UIE_on;
1128static unsigned long prev_update_sec;
1129
1130static unsigned long AIE_on;
1131static struct rtc_time alarm_time;
1132
1133static unsigned long PIE_on;
1134static unsigned long PIE_freq = DEFAULT_RTC_INT_FREQ;
1135static unsigned long PIE_count;
1136
1137static unsigned long hpet_rtc_int_freq; /* RTC interrupt frequency */
Clemens Ladisch7811fb82005-10-30 15:03:36 -08001138static unsigned int hpet_t1_cmp; /* cached comparator register */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139
1140int is_hpet_enabled(void)
1141{
1142 return vxtime.hpet_address != 0;
1143}
1144
1145/*
1146 * Timer 1 for RTC, we do not use periodic interrupt feature,
1147 * even if HPET supports periodic interrupts on Timer 1.
1148 * The reason being, to set up a periodic interrupt in HPET, we need to
1149 * stop the main counter. And if we do that everytime someone diables/enables
1150 * RTC, we will have adverse effect on main kernel timer running on Timer 0.
1151 * So, for the time being, simulate the periodic interrupt in software.
1152 *
1153 * hpet_rtc_timer_init() is called for the first time and during subsequent
1154 * interuppts reinit happens through hpet_rtc_timer_reinit().
1155 */
1156int hpet_rtc_timer_init(void)
1157{
1158 unsigned int cfg, cnt;
1159 unsigned long flags;
1160
1161 if (!is_hpet_enabled())
1162 return 0;
1163 /*
1164 * Set the counter 1 and enable the interrupts.
1165 */
1166 if (PIE_on && (PIE_freq > DEFAULT_RTC_INT_FREQ))
1167 hpet_rtc_int_freq = PIE_freq;
1168 else
1169 hpet_rtc_int_freq = DEFAULT_RTC_INT_FREQ;
1170
1171 local_irq_save(flags);
Clemens Ladisch1447c272006-09-25 23:32:17 -07001172
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173 cnt = hpet_readl(HPET_COUNTER);
1174 cnt += ((hpet_tick*HZ)/hpet_rtc_int_freq);
1175 hpet_writel(cnt, HPET_T1_CMP);
Clemens Ladisch7811fb82005-10-30 15:03:36 -08001176 hpet_t1_cmp = cnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177
1178 cfg = hpet_readl(HPET_T1_CFG);
Clemens Ladisch5f819942005-10-30 15:03:36 -08001179 cfg &= ~HPET_TN_PERIODIC;
1180 cfg |= HPET_TN_ENABLE | HPET_TN_32BIT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 hpet_writel(cfg, HPET_T1_CFG);
1182
Clemens Ladisch1447c272006-09-25 23:32:17 -07001183 local_irq_restore(flags);
1184
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185 return 1;
1186}
1187
1188static void hpet_rtc_timer_reinit(void)
1189{
Clemens Ladisch1447c272006-09-25 23:32:17 -07001190 unsigned int cfg, cnt, ticks_per_int, lost_ints;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001191
Clemens Ladischf00c96f2005-10-30 15:03:35 -08001192 if (unlikely(!(PIE_on | AIE_on | UIE_on))) {
1193 cfg = hpet_readl(HPET_T1_CFG);
1194 cfg &= ~HPET_TN_ENABLE;
1195 hpet_writel(cfg, HPET_T1_CFG);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 return;
Clemens Ladischf00c96f2005-10-30 15:03:35 -08001197 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198
1199 if (PIE_on && (PIE_freq > DEFAULT_RTC_INT_FREQ))
1200 hpet_rtc_int_freq = PIE_freq;
1201 else
1202 hpet_rtc_int_freq = DEFAULT_RTC_INT_FREQ;
1203
1204 /* It is more accurate to use the comparator value than current count.*/
Clemens Ladisch1447c272006-09-25 23:32:17 -07001205 ticks_per_int = hpet_tick * HZ / hpet_rtc_int_freq;
1206 hpet_t1_cmp += ticks_per_int;
1207 hpet_writel(hpet_t1_cmp, HPET_T1_CMP);
1208
1209 /*
1210 * If the interrupt handler was delayed too long, the write above tries
1211 * to schedule the next interrupt in the past and the hardware would
1212 * not interrupt until the counter had wrapped around.
1213 * So we have to check that the comparator wasn't set to a past time.
1214 */
1215 cnt = hpet_readl(HPET_COUNTER);
1216 if (unlikely((int)(cnt - hpet_t1_cmp) > 0)) {
1217 lost_ints = (cnt - hpet_t1_cmp) / ticks_per_int + 1;
1218 /* Make sure that, even with the time needed to execute
1219 * this code, the next scheduled interrupt has been moved
1220 * back to the future: */
1221 lost_ints++;
1222
1223 hpet_t1_cmp += lost_ints * ticks_per_int;
1224 hpet_writel(hpet_t1_cmp, HPET_T1_CMP);
1225
1226 if (PIE_on)
1227 PIE_count += lost_ints;
1228
1229 printk(KERN_WARNING "rtc: lost some interrupts at %ldHz.\n",
1230 hpet_rtc_int_freq);
1231 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232}
1233
1234/*
1235 * The functions below are called from rtc driver.
1236 * Return 0 if HPET is not being used.
1237 * Otherwise do the necessary changes and return 1.
1238 */
1239int hpet_mask_rtc_irq_bit(unsigned long bit_mask)
1240{
1241 if (!is_hpet_enabled())
1242 return 0;
1243
1244 if (bit_mask & RTC_UIE)
1245 UIE_on = 0;
1246 if (bit_mask & RTC_PIE)
1247 PIE_on = 0;
1248 if (bit_mask & RTC_AIE)
1249 AIE_on = 0;
1250
1251 return 1;
1252}
1253
1254int hpet_set_rtc_irq_bit(unsigned long bit_mask)
1255{
1256 int timer_init_reqd = 0;
1257
1258 if (!is_hpet_enabled())
1259 return 0;
1260
1261 if (!(PIE_on | AIE_on | UIE_on))
1262 timer_init_reqd = 1;
1263
1264 if (bit_mask & RTC_UIE) {
1265 UIE_on = 1;
1266 }
1267 if (bit_mask & RTC_PIE) {
1268 PIE_on = 1;
1269 PIE_count = 0;
1270 }
1271 if (bit_mask & RTC_AIE) {
1272 AIE_on = 1;
1273 }
1274
1275 if (timer_init_reqd)
1276 hpet_rtc_timer_init();
1277
1278 return 1;
1279}
1280
1281int hpet_set_alarm_time(unsigned char hrs, unsigned char min, unsigned char sec)
1282{
1283 if (!is_hpet_enabled())
1284 return 0;
1285
1286 alarm_time.tm_hour = hrs;
1287 alarm_time.tm_min = min;
1288 alarm_time.tm_sec = sec;
1289
1290 return 1;
1291}
1292
1293int hpet_set_periodic_freq(unsigned long freq)
1294{
1295 if (!is_hpet_enabled())
1296 return 0;
1297
1298 PIE_freq = freq;
1299 PIE_count = 0;
1300
1301 return 1;
1302}
1303
1304int hpet_rtc_dropped_irq(void)
1305{
1306 if (!is_hpet_enabled())
1307 return 0;
1308
1309 return 1;
1310}
1311
1312irqreturn_t hpet_rtc_interrupt(int irq, void *dev_id, struct pt_regs *regs)
1313{
1314 struct rtc_time curr_time;
1315 unsigned long rtc_int_flag = 0;
1316 int call_rtc_interrupt = 0;
1317
1318 hpet_rtc_timer_reinit();
1319
1320 if (UIE_on | AIE_on) {
1321 rtc_get_rtc_time(&curr_time);
1322 }
1323 if (UIE_on) {
1324 if (curr_time.tm_sec != prev_update_sec) {
1325 /* Set update int info, call real rtc int routine */
1326 call_rtc_interrupt = 1;
1327 rtc_int_flag = RTC_UF;
1328 prev_update_sec = curr_time.tm_sec;
1329 }
1330 }
1331 if (PIE_on) {
1332 PIE_count++;
1333 if (PIE_count >= hpet_rtc_int_freq/PIE_freq) {
1334 /* Set periodic int info, call real rtc int routine */
1335 call_rtc_interrupt = 1;
1336 rtc_int_flag |= RTC_PF;
1337 PIE_count = 0;
1338 }
1339 }
1340 if (AIE_on) {
1341 if ((curr_time.tm_sec == alarm_time.tm_sec) &&
1342 (curr_time.tm_min == alarm_time.tm_min) &&
1343 (curr_time.tm_hour == alarm_time.tm_hour)) {
1344 /* Set alarm int info, call real rtc int routine */
1345 call_rtc_interrupt = 1;
1346 rtc_int_flag |= RTC_AF;
1347 }
1348 }
1349 if (call_rtc_interrupt) {
1350 rtc_int_flag |= (RTC_IRQF | (RTC_NUM_INTS << 8));
David Howells7d12e782006-10-05 14:55:46 +01001351 rtc_interrupt(rtc_int_flag, dev_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352 }
1353 return IRQ_HANDLED;
1354}
1355#endif
1356
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357static int __init nohpet_setup(char *s)
1358{
1359 nohpet = 1;
OGAWA Hirofumi9b410462006-03-31 02:30:33 -08001360 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361}
1362
1363__setup("nohpet", nohpet_setup);
1364
Andi Kleen7fd67842006-02-16 23:42:07 +01001365int __init notsc_setup(char *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366{
1367 notsc = 1;
OGAWA Hirofumi9b410462006-03-31 02:30:33 -08001368 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369}
1370
1371__setup("notsc", notsc_setup);