blob: 7e8983aecf830b37ac9255f9b0919f15c53a4bec [file] [log] [blame]
john stultz85240702007-05-08 00:27:59 -07001/*
2 * linux/kernel/time/timekeeping.c
3 *
4 * Kernel timekeeping code and accessor functions
5 *
6 * This code was moved from linux/kernel/timer.c.
7 * Please see that file for copyright and history logs.
8 *
9 */
10
11#include <linux/module.h>
12#include <linux/interrupt.h>
13#include <linux/percpu.h>
14#include <linux/init.h>
15#include <linux/mm.h>
16#include <linux/sysdev.h>
17#include <linux/clocksource.h>
18#include <linux/jiffies.h>
19#include <linux/time.h>
20#include <linux/tick.h>
21
22
23/*
24 * This read-write spinlock protects us from races in SMP while
25 * playing with xtime and avenrun.
26 */
27__attribute__((weak)) __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
28
29EXPORT_SYMBOL(xtime_lock);
30
31
32/*
33 * The current time
34 * wall_to_monotonic is what we need to add to xtime (or xtime corrected
35 * for sub jiffie times) to get to monotonic time. Monotonic is pegged
36 * at zero at system boot time, so wall_to_monotonic will be negative,
37 * however, we will ALWAYS keep the tv_nsec part positive so we can use
38 * the usual normalization.
Tomas Janousek7c3f1a52007-07-15 23:39:41 -070039 *
40 * wall_to_monotonic is moved after resume from suspend for the monotonic
41 * time not to jump. We need to add total_sleep_time to wall_to_monotonic
42 * to get the real boot based time offset.
43 *
44 * - wall_to_monotonic is no longer the boot time, getboottime must be
45 * used instead.
john stultz85240702007-05-08 00:27:59 -070046 */
47struct timespec xtime __attribute__ ((aligned (16)));
48struct timespec wall_to_monotonic __attribute__ ((aligned (16)));
Tomas Janousek7c3f1a52007-07-15 23:39:41 -070049static unsigned long total_sleep_time; /* seconds */
john stultz85240702007-05-08 00:27:59 -070050EXPORT_SYMBOL(xtime);
51
john stultz17c38b72007-07-24 18:38:34 -070052static struct timespec xtime_cache __attribute__ ((aligned (16)));
53static inline void update_xtime_cache(u64 nsec)
54{
55 xtime_cache = xtime;
56 timespec_add_ns(&xtime_cache, nsec);
57}
john stultz17c38b72007-07-24 18:38:34 -070058
john stultz85240702007-05-08 00:27:59 -070059static struct clocksource *clock; /* pointer to current clocksource */
60
61
62#ifdef CONFIG_GENERIC_TIME
63/**
64 * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook
65 *
66 * private function, must hold xtime_lock lock when being
67 * called. Returns the number of nanoseconds since the
68 * last call to update_wall_time() (adjusted by NTP scaling)
69 */
70static inline s64 __get_nsec_offset(void)
71{
72 cycle_t cycle_now, cycle_delta;
73 s64 ns_offset;
74
75 /* read clocksource: */
76 cycle_now = clocksource_read(clock);
77
78 /* calculate the delta since the last update_wall_time: */
79 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
80
81 /* convert to nanoseconds: */
82 ns_offset = cyc2ns(clock, cycle_delta);
83
84 return ns_offset;
85}
86
87/**
88 * __get_realtime_clock_ts - Returns the time of day in a timespec
89 * @ts: pointer to the timespec to be set
90 *
91 * Returns the time of day in a timespec. Used by
92 * do_gettimeofday() and get_realtime_clock_ts().
93 */
94static inline void __get_realtime_clock_ts(struct timespec *ts)
95{
96 unsigned long seq;
97 s64 nsecs;
98
99 do {
100 seq = read_seqbegin(&xtime_lock);
101
102 *ts = xtime;
103 nsecs = __get_nsec_offset();
104
105 } while (read_seqretry(&xtime_lock, seq));
106
107 timespec_add_ns(ts, nsecs);
108}
109
110/**
111 * getnstimeofday - Returns the time of day in a timespec
112 * @ts: pointer to the timespec to be set
113 *
114 * Returns the time of day in a timespec.
115 */
116void getnstimeofday(struct timespec *ts)
117{
118 __get_realtime_clock_ts(ts);
119}
120
121EXPORT_SYMBOL(getnstimeofday);
122
123/**
124 * do_gettimeofday - Returns the time of day in a timeval
125 * @tv: pointer to the timeval to be set
126 *
127 * NOTE: Users should be converted to using get_realtime_clock_ts()
128 */
129void do_gettimeofday(struct timeval *tv)
130{
131 struct timespec now;
132
133 __get_realtime_clock_ts(&now);
134 tv->tv_sec = now.tv_sec;
135 tv->tv_usec = now.tv_nsec/1000;
136}
137
138EXPORT_SYMBOL(do_gettimeofday);
139/**
140 * do_settimeofday - Sets the time of day
141 * @tv: pointer to the timespec variable containing the new time
142 *
143 * Sets the time of day to the new time and update NTP and notify hrtimers
144 */
145int do_settimeofday(struct timespec *tv)
146{
147 unsigned long flags;
148 time_t wtm_sec, sec = tv->tv_sec;
149 long wtm_nsec, nsec = tv->tv_nsec;
150
151 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
152 return -EINVAL;
153
154 write_seqlock_irqsave(&xtime_lock, flags);
155
156 nsec -= __get_nsec_offset();
157
158 wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec);
159 wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec);
160
161 set_normalized_timespec(&xtime, sec, nsec);
162 set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec);
163
164 clock->error = 0;
165 ntp_clear();
166
167 update_vsyscall(&xtime, clock);
168
169 write_sequnlock_irqrestore(&xtime_lock, flags);
170
171 /* signal hrtimers about time change */
172 clock_was_set();
173
174 return 0;
175}
176
177EXPORT_SYMBOL(do_settimeofday);
178
179/**
180 * change_clocksource - Swaps clocksources if a new one is available
181 *
182 * Accumulates current time interval and initializes new clocksource
183 */
184static void change_clocksource(void)
185{
186 struct clocksource *new;
187 cycle_t now;
188 u64 nsec;
189
190 new = clocksource_get_next();
191
192 if (clock == new)
193 return;
194
195 now = clocksource_read(new);
196 nsec = __get_nsec_offset();
197 timespec_add_ns(&xtime, nsec);
198
199 clock = new;
200 clock->cycle_last = now;
201
202 clock->error = 0;
203 clock->xtime_nsec = 0;
204 clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH);
205
206 tick_clock_notify();
207
208 printk(KERN_INFO "Time: %s clocksource has been installed.\n",
209 clock->name);
210}
211#else
212static inline void change_clocksource(void) { }
Thomas Gleixner6a669ee2007-09-16 15:36:43 +0200213static inline s64 __get_nsec_offset(void) { return 0; }
john stultz85240702007-05-08 00:27:59 -0700214#endif
215
216/**
217 * timekeeping_is_continuous - check to see if timekeeping is free running
218 */
219int timekeeping_is_continuous(void)
220{
221 unsigned long seq;
222 int ret;
223
224 do {
225 seq = read_seqbegin(&xtime_lock);
226
227 ret = clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
228
229 } while (read_seqretry(&xtime_lock, seq));
230
231 return ret;
232}
233
234/**
235 * read_persistent_clock - Return time in seconds from the persistent clock.
236 *
237 * Weak dummy function for arches that do not yet support it.
238 * Returns seconds from epoch using the battery backed persistent clock.
239 * Returns zero if unsupported.
240 *
241 * XXX - Do be sure to remove it once all arches implement it.
242 */
243unsigned long __attribute__((weak)) read_persistent_clock(void)
244{
245 return 0;
246}
247
248/*
249 * timekeeping_init - Initializes the clocksource and common timekeeping values
250 */
251void __init timekeeping_init(void)
252{
253 unsigned long flags;
254 unsigned long sec = read_persistent_clock();
255
256 write_seqlock_irqsave(&xtime_lock, flags);
257
258 ntp_clear();
259
260 clock = clocksource_get_next();
261 clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH);
262 clock->cycle_last = clocksource_read(clock);
263
264 xtime.tv_sec = sec;
265 xtime.tv_nsec = 0;
266 set_normalized_timespec(&wall_to_monotonic,
267 -xtime.tv_sec, -xtime.tv_nsec);
Tomas Janousek7c3f1a52007-07-15 23:39:41 -0700268 total_sleep_time = 0;
john stultz85240702007-05-08 00:27:59 -0700269
270 write_sequnlock_irqrestore(&xtime_lock, flags);
271}
272
273/* flag for if timekeeping is suspended */
274static int timekeeping_suspended;
275/* time in seconds when suspend began */
276static unsigned long timekeeping_suspend_time;
Thomas Gleixner6a669ee2007-09-16 15:36:43 +0200277/* xtime offset when we went into suspend */
278static s64 timekeeping_suspend_nsecs;
john stultz85240702007-05-08 00:27:59 -0700279
280/**
281 * timekeeping_resume - Resumes the generic timekeeping subsystem.
282 * @dev: unused
283 *
284 * This is for the generic clocksource timekeeping.
285 * xtime/wall_to_monotonic/jiffies/etc are
286 * still managed by arch specific suspend/resume code.
287 */
288static int timekeeping_resume(struct sys_device *dev)
289{
290 unsigned long flags;
291 unsigned long now = read_persistent_clock();
292
Thomas Gleixnerd10ff3f2007-05-14 11:10:02 +0200293 clocksource_resume();
294
john stultz85240702007-05-08 00:27:59 -0700295 write_seqlock_irqsave(&xtime_lock, flags);
296
297 if (now && (now > timekeeping_suspend_time)) {
298 unsigned long sleep_length = now - timekeeping_suspend_time;
299
300 xtime.tv_sec += sleep_length;
301 wall_to_monotonic.tv_sec -= sleep_length;
Tomas Janousek7c3f1a52007-07-15 23:39:41 -0700302 total_sleep_time += sleep_length;
john stultz85240702007-05-08 00:27:59 -0700303 }
Thomas Gleixner6a669ee2007-09-16 15:36:43 +0200304 /* Make sure that we have the correct xtime reference */
305 timespec_add_ns(&xtime, timekeeping_suspend_nsecs);
john stultz85240702007-05-08 00:27:59 -0700306 /* re-base the last cycle value */
307 clock->cycle_last = clocksource_read(clock);
308 clock->error = 0;
309 timekeeping_suspended = 0;
310 write_sequnlock_irqrestore(&xtime_lock, flags);
311
312 touch_softlockup_watchdog();
313
314 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
315
316 /* Resume hrtimers */
317 hres_timers_resume();
318
319 return 0;
320}
321
322static int timekeeping_suspend(struct sys_device *dev, pm_message_t state)
323{
324 unsigned long flags;
325
Thomas Gleixner3be90952007-09-16 15:36:43 +0200326 timekeeping_suspend_time = read_persistent_clock();
327
john stultz85240702007-05-08 00:27:59 -0700328 write_seqlock_irqsave(&xtime_lock, flags);
Thomas Gleixner6a669ee2007-09-16 15:36:43 +0200329 /* Get the current xtime offset */
330 timekeeping_suspend_nsecs = __get_nsec_offset();
john stultz85240702007-05-08 00:27:59 -0700331 timekeeping_suspended = 1;
john stultz85240702007-05-08 00:27:59 -0700332 write_sequnlock_irqrestore(&xtime_lock, flags);
333
334 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
335
336 return 0;
337}
338
339/* sysfs resume/suspend bits for timekeeping */
340static struct sysdev_class timekeeping_sysclass = {
341 .resume = timekeeping_resume,
342 .suspend = timekeeping_suspend,
343 set_kset_name("timekeeping"),
344};
345
346static struct sys_device device_timer = {
347 .id = 0,
348 .cls = &timekeeping_sysclass,
349};
350
351static int __init timekeeping_init_device(void)
352{
353 int error = sysdev_class_register(&timekeeping_sysclass);
354 if (!error)
355 error = sysdev_register(&device_timer);
356 return error;
357}
358
359device_initcall(timekeeping_init_device);
360
361/*
362 * If the error is already larger, we look ahead even further
363 * to compensate for late or lost adjustments.
364 */
365static __always_inline int clocksource_bigadjust(s64 error, s64 *interval,
366 s64 *offset)
367{
368 s64 tick_error, i;
369 u32 look_ahead, adj;
370 s32 error2, mult;
371
372 /*
373 * Use the current error value to determine how much to look ahead.
374 * The larger the error the slower we adjust for it to avoid problems
375 * with losing too many ticks, otherwise we would overadjust and
376 * produce an even larger error. The smaller the adjustment the
377 * faster we try to adjust for it, as lost ticks can do less harm
378 * here. This is tuned so that an error of about 1 msec is adusted
379 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
380 */
381 error2 = clock->error >> (TICK_LENGTH_SHIFT + 22 - 2 * SHIFT_HZ);
382 error2 = abs(error2);
383 for (look_ahead = 0; error2 > 0; look_ahead++)
384 error2 >>= 2;
385
386 /*
387 * Now calculate the error in (1 << look_ahead) ticks, but first
388 * remove the single look ahead already included in the error.
389 */
390 tick_error = current_tick_length() >>
391 (TICK_LENGTH_SHIFT - clock->shift + 1);
392 tick_error -= clock->xtime_interval >> 1;
393 error = ((error - tick_error) >> look_ahead) + tick_error;
394
395 /* Finally calculate the adjustment shift value. */
396 i = *interval;
397 mult = 1;
398 if (error < 0) {
399 error = -error;
400 *interval = -*interval;
401 *offset = -*offset;
402 mult = -1;
403 }
404 for (adj = 0; error > i; adj++)
405 error >>= 1;
406
407 *interval <<= adj;
408 *offset <<= adj;
409 return mult << adj;
410}
411
412/*
413 * Adjust the multiplier to reduce the error value,
414 * this is optimized for the most common adjustments of -1,0,1,
415 * for other values we can do a bit more work.
416 */
Thomas Gleixner71120f12007-07-19 01:49:16 -0700417static void clocksource_adjust(s64 offset)
john stultz85240702007-05-08 00:27:59 -0700418{
419 s64 error, interval = clock->cycle_interval;
420 int adj;
421
422 error = clock->error >> (TICK_LENGTH_SHIFT - clock->shift - 1);
423 if (error > interval) {
424 error >>= 2;
425 if (likely(error <= interval))
426 adj = 1;
427 else
428 adj = clocksource_bigadjust(error, &interval, &offset);
429 } else if (error < -interval) {
430 error >>= 2;
431 if (likely(error >= -interval)) {
432 adj = -1;
433 interval = -interval;
434 offset = -offset;
435 } else
436 adj = clocksource_bigadjust(error, &interval, &offset);
437 } else
438 return;
439
440 clock->mult += adj;
441 clock->xtime_interval += interval;
442 clock->xtime_nsec -= offset;
443 clock->error -= (interval - offset) <<
444 (TICK_LENGTH_SHIFT - clock->shift);
445}
446
447/**
448 * update_wall_time - Uses the current clocksource to increment the wall time
449 *
450 * Called from the timer interrupt, must hold a write on xtime_lock.
451 */
452void update_wall_time(void)
453{
454 cycle_t offset;
455
456 /* Make sure we're fully resumed: */
457 if (unlikely(timekeeping_suspended))
458 return;
459
460#ifdef CONFIG_GENERIC_TIME
461 offset = (clocksource_read(clock) - clock->cycle_last) & clock->mask;
462#else
463 offset = clock->cycle_interval;
464#endif
465 clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift;
466
467 /* normally this loop will run just once, however in the
468 * case of lost or late ticks, it will accumulate correctly.
469 */
470 while (offset >= clock->cycle_interval) {
471 /* accumulate one interval */
472 clock->xtime_nsec += clock->xtime_interval;
473 clock->cycle_last += clock->cycle_interval;
474 offset -= clock->cycle_interval;
475
476 if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) {
477 clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift;
478 xtime.tv_sec++;
479 second_overflow();
480 }
481
john stultz85240702007-05-08 00:27:59 -0700482 /* accumulate error between NTP and clock interval */
483 clock->error += current_tick_length();
484 clock->error -= clock->xtime_interval << (TICK_LENGTH_SHIFT - clock->shift);
485 }
486
487 /* correct the clock when NTP error is too big */
Thomas Gleixner71120f12007-07-19 01:49:16 -0700488 clocksource_adjust(offset);
john stultz85240702007-05-08 00:27:59 -0700489
490 /* store full nanoseconds into xtime */
491 xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift;
492 clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
493
john stultz17c38b72007-07-24 18:38:34 -0700494 update_xtime_cache(cyc2ns(clock, offset));
495
john stultz85240702007-05-08 00:27:59 -0700496 /* check to see if there is a new clocksource to use */
497 change_clocksource();
498 update_vsyscall(&xtime, clock);
499}
Tomas Janousek7c3f1a52007-07-15 23:39:41 -0700500
501/**
502 * getboottime - Return the real time of system boot.
503 * @ts: pointer to the timespec to be set
504 *
505 * Returns the time of day in a timespec.
506 *
507 * This is based on the wall_to_monotonic offset and the total suspend
508 * time. Calls to settimeofday will affect the value returned (which
509 * basically means that however wrong your real time clock is at boot time,
510 * you get the right time here).
511 */
512void getboottime(struct timespec *ts)
513{
514 set_normalized_timespec(ts,
515 - (wall_to_monotonic.tv_sec + total_sleep_time),
516 - wall_to_monotonic.tv_nsec);
517}
518
519/**
520 * monotonic_to_bootbased - Convert the monotonic time to boot based.
521 * @ts: pointer to the timespec to be converted
522 */
523void monotonic_to_bootbased(struct timespec *ts)
524{
525 ts->tv_sec += total_sleep_time;
526}
john stultz2c6b47d2007-07-24 17:47:43 -0700527
john stultz17c38b72007-07-24 18:38:34 -0700528unsigned long get_seconds(void)
529{
530 return xtime_cache.tv_sec;
531}
532EXPORT_SYMBOL(get_seconds);
533
534
john stultz2c6b47d2007-07-24 17:47:43 -0700535struct timespec current_kernel_time(void)
536{
537 struct timespec now;
538 unsigned long seq;
539
540 do {
541 seq = read_seqbegin(&xtime_lock);
542
john stultz17c38b72007-07-24 18:38:34 -0700543 now = xtime_cache;
john stultz2c6b47d2007-07-24 17:47:43 -0700544 } while (read_seqretry(&xtime_lock, seq));
545
546 return now;
547}
john stultz2c6b47d2007-07-24 17:47:43 -0700548EXPORT_SYMBOL(current_kernel_time);