blob: 3479f1b0d4e0d86f90ca5bb974bf09814dc1c987 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
4 *
5 * S390 version
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
8 */
9
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/module.h>
11#include <linux/kernel.h>
12#include <linux/time.h>
13#include <linux/delay.h>
14#include <linux/init.h>
15#include <linux/smp.h>
16#include <linux/types.h>
17#include <linux/timex.h>
18#include <linux/notifier.h>
19#include <linux/kernel_stat.h>
20#include <linux/rcupdate.h>
21#include <linux/posix-timers.h>
22
23#include <asm/s390_ext.h>
24#include <asm/timer.h>
Heiko Carstens5a489b92006-10-06 16:38:35 +020025#include <asm/irq_regs.h>
Martin Schwidefsky76d4e002009-06-12 10:26:21 +020026#include <asm/cputime.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Heiko Carstens2b67fc42007-02-05 21:16:47 +010028static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +020030DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +010031
32static inline __u64 get_vtimer(void)
33{
34 __u64 timer;
35
36 asm volatile("STPT %0" : "=m" (timer));
37 return timer;
38}
39
40static inline void set_vtimer(__u64 expires)
41{
42 __u64 timer;
43
44 asm volatile (" STPT %0\n" /* Store current cpu timer value */
45 " SPT %1" /* Set new value immediatly afterwards */
46 : "=m" (timer) : "m" (expires) );
47 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
48 S390_lowcore.last_update_timer = expires;
49}
50
Linus Torvalds1da177e2005-04-16 15:20:36 -070051/*
52 * Update process times based on virtual cpu times stored by entry.S
53 * to the lowcore fields user_timer, system_timer & steal_clock.
54 */
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010055static void do_account_vtime(struct task_struct *tsk, int hardirq_offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -070056{
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010057 struct thread_info *ti = task_thread_info(tsk);
58 __u64 timer, clock, user, system, steal;
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
60 timer = S390_lowcore.last_update_timer;
61 clock = S390_lowcore.last_update_clock;
62 asm volatile (" STPT %0\n" /* Store current cpu timer value */
63 " STCK %1" /* Store current tod clock value */
64 : "=m" (S390_lowcore.last_update_timer),
65 "=m" (S390_lowcore.last_update_clock) );
66 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010067 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
Linus Torvalds1da177e2005-04-16 15:20:36 -070068
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010069 user = S390_lowcore.user_timer - ti->user_timer;
70 S390_lowcore.steal_timer -= user;
71 ti->user_timer = S390_lowcore.user_timer;
72 account_user_time(tsk, user, user);
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010074 system = S390_lowcore.system_timer - ti->system_timer;
75 S390_lowcore.steal_timer -= system;
76 ti->system_timer = S390_lowcore.system_timer;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +010077 account_system_time(tsk, hardirq_offset, system, system);
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010079 steal = S390_lowcore.steal_timer;
80 if ((s64) steal > 0) {
81 S390_lowcore.steal_timer = 0;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +010082 account_steal_time(steal);
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070084}
85
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010086void account_vtime(struct task_struct *prev, struct task_struct *next)
Martin Schwidefsky1f1c12a2006-01-14 13:21:03 -080087{
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010088 struct thread_info *ti;
Martin Schwidefsky1f1c12a2006-01-14 13:21:03 -080089
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010090 do_account_vtime(prev, 0);
91 ti = task_thread_info(prev);
92 ti->user_timer = S390_lowcore.user_timer;
93 ti->system_timer = S390_lowcore.system_timer;
94 ti = task_thread_info(next);
95 S390_lowcore.user_timer = ti->user_timer;
96 S390_lowcore.system_timer = ti->system_timer;
97}
Martin Schwidefsky1f1c12a2006-01-14 13:21:03 -080098
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +010099void account_process_tick(struct task_struct *tsk, int user_tick)
100{
101 do_account_vtime(tsk, HARDIRQ_OFFSET);
Martin Schwidefsky1f1c12a2006-01-14 13:21:03 -0800102}
103
104/*
105 * Update process times based on virtual cpu times stored by entry.S
106 * to the lowcore fields user_timer, system_timer & steal_clock.
107 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108void account_system_vtime(struct task_struct *tsk)
109{
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +0100110 struct thread_info *ti = task_thread_info(tsk);
111 __u64 timer, system;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
113 timer = S390_lowcore.last_update_timer;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100114 S390_lowcore.last_update_timer = get_vtimer();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
116
Martin Schwidefskyaa5e97c2008-12-31 15:11:39 +0100117 system = S390_lowcore.system_timer - ti->system_timer;
118 S390_lowcore.steal_timer -= system;
119 ti->system_timer = S390_lowcore.system_timer;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100120 account_system_time(tsk, 0, system, system);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121}
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100122EXPORT_SYMBOL_GPL(account_system_vtime);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123
Martin Schwidefsky63779812010-05-17 10:00:03 +0200124void vtime_start_cpu(__u64 int_clock, __u64 enter_timer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125{
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100126 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
Martin Schwidefsky6f430922008-12-31 15:11:40 +0100127 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100128 __u64 idle_time, expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
Martin Schwidefsky63779812010-05-17 10:00:03 +0200130 if (idle->idle_enter == 0ULL)
131 return;
132
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100133 /* Account time spent with enabled wait psw loaded as idle time. */
Martin Schwidefsky63779812010-05-17 10:00:03 +0200134 idle_time = int_clock - idle->idle_enter;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100135 account_idle_time(idle_time);
Martin Schwidefskyb86ccca2009-04-14 15:36:26 +0200136 S390_lowcore.steal_timer +=
137 idle->idle_enter - S390_lowcore.last_update_clock;
Martin Schwidefsky63779812010-05-17 10:00:03 +0200138 S390_lowcore.last_update_clock = int_clock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100140 /* Account system time spent going idle. */
141 S390_lowcore.system_timer += S390_lowcore.last_update_timer - vq->idle;
Martin Schwidefsky63779812010-05-17 10:00:03 +0200142 S390_lowcore.last_update_timer = enter_timer;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100144 /* Restart vtime CPU timer */
145 if (vq->do_spt) {
146 /* Program old expire value but first save progress. */
Martin Schwidefsky63779812010-05-17 10:00:03 +0200147 expires = vq->idle - enter_timer;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100148 expires += get_vtimer();
149 set_vtimer(expires);
150 } else {
151 /* Don't account the CPU timer delta while the cpu was idle. */
Martin Schwidefsky63779812010-05-17 10:00:03 +0200152 vq->elapsed -= vq->idle - enter_timer;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 }
154
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +0200155 idle->sequence++;
156 smp_wmb();
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100157 idle->idle_time += idle_time;
158 idle->idle_enter = 0ULL;
159 idle->idle_count++;
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +0200160 smp_wmb();
161 idle->sequence++;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100162}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100164void vtime_stop_cpu(void)
165{
166 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
167 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
168 psw_t psw;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100170 /* Wait for external, I/O or machine check interrupt. */
171 psw.mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_IO | PSW_MASK_EXT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172
Martin Schwidefsky3c5d92a2009-09-29 14:25:16 +0200173 idle->nohz_delay = 0;
174
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100175 /* Check if the CPU timer needs to be reprogrammed. */
176 if (vq->do_spt) {
177 __u64 vmax = VTIMER_MAX_SLICE;
178 /*
179 * The inline assembly is equivalent to
180 * vq->idle = get_cpu_timer();
181 * set_cpu_timer(VTIMER_MAX_SLICE);
182 * idle->idle_enter = get_clock();
183 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
184 * PSW_MASK_IO | PSW_MASK_EXT);
185 * The difference is that the inline assembly makes sure that
186 * the last three instruction are stpt, stck and lpsw in that
187 * order. This is done to increase the precision.
188 */
189 asm volatile(
190#ifndef CONFIG_64BIT
191 " basr 1,0\n"
192 "0: ahi 1,1f-0b\n"
193 " st 1,4(%2)\n"
194#else /* CONFIG_64BIT */
195 " larl 1,1f\n"
196 " stg 1,8(%2)\n"
197#endif /* CONFIG_64BIT */
198 " stpt 0(%4)\n"
199 " spt 0(%5)\n"
200 " stck 0(%3)\n"
201#ifndef CONFIG_64BIT
202 " lpsw 0(%2)\n"
203#else /* CONFIG_64BIT */
204 " lpswe 0(%2)\n"
205#endif /* CONFIG_64BIT */
206 "1:"
207 : "=m" (idle->idle_enter), "=m" (vq->idle)
208 : "a" (&psw), "a" (&idle->idle_enter),
209 "a" (&vq->idle), "a" (&vmax), "m" (vmax), "m" (psw)
210 : "memory", "cc", "1");
211 } else {
212 /*
213 * The inline assembly is equivalent to
214 * vq->idle = get_cpu_timer();
215 * idle->idle_enter = get_clock();
216 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
217 * PSW_MASK_IO | PSW_MASK_EXT);
218 * The difference is that the inline assembly makes sure that
219 * the last three instruction are stpt, stck and lpsw in that
220 * order. This is done to increase the precision.
221 */
222 asm volatile(
223#ifndef CONFIG_64BIT
224 " basr 1,0\n"
225 "0: ahi 1,1f-0b\n"
226 " st 1,4(%2)\n"
227#else /* CONFIG_64BIT */
228 " larl 1,1f\n"
229 " stg 1,8(%2)\n"
230#endif /* CONFIG_64BIT */
231 " stpt 0(%4)\n"
232 " stck 0(%3)\n"
233#ifndef CONFIG_64BIT
234 " lpsw 0(%2)\n"
235#else /* CONFIG_64BIT */
236 " lpswe 0(%2)\n"
237#endif /* CONFIG_64BIT */
238 "1:"
239 : "=m" (idle->idle_enter), "=m" (vq->idle)
240 : "a" (&psw), "a" (&idle->idle_enter),
241 "a" (&vq->idle), "m" (psw)
242 : "memory", "cc", "1");
243 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244}
245
Martin Schwidefskye1c80532009-04-23 13:58:08 +0200246cputime64_t s390_get_idle_time(int cpu)
247{
248 struct s390_idle_data *idle;
249 unsigned long long now, idle_time, idle_enter;
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +0200250 unsigned int sequence;
Martin Schwidefskye1c80532009-04-23 13:58:08 +0200251
252 idle = &per_cpu(s390_idle, cpu);
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +0200253
Martin Schwidefskye1c80532009-04-23 13:58:08 +0200254 now = get_clock();
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +0200255repeat:
256 sequence = idle->sequence;
257 smp_rmb();
258 if (sequence & 1)
259 goto repeat;
Martin Schwidefskye1c80532009-04-23 13:58:08 +0200260 idle_time = 0;
261 idle_enter = idle->idle_enter;
262 if (idle_enter != 0ULL && idle_enter < now)
263 idle_time = now - idle_enter;
Martin Schwidefskye98bbaa2009-06-22 12:08:20 +0200264 smp_rmb();
265 if (idle->sequence != sequence)
266 goto repeat;
Martin Schwidefskye1c80532009-04-23 13:58:08 +0200267 return idle_time;
268}
269
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270/*
271 * Sorted add to a list. List is linear searched until first bigger
272 * element is found.
273 */
274static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
275{
276 struct vtimer_list *event;
277
278 list_for_each_entry(event, head, entry) {
279 if (event->expires > timer->expires) {
280 list_add_tail(&timer->entry, &event->entry);
281 return;
282 }
283 }
284 list_add_tail(&timer->entry, head);
285}
286
287/*
288 * Do the callback functions of expired vtimer events.
289 * Called from within the interrupt handler.
290 */
Heiko Carstens9d0a57c2006-10-11 15:31:26 +0200291static void do_callbacks(struct list_head *cb_list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292{
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100293 struct vtimer_queue *vq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 struct vtimer_list *event, *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295
296 if (list_empty(cb_list))
297 return;
298
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100299 vq = &__get_cpu_var(virt_cpu_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300
301 list_for_each_entry_safe(event, tmp, cb_list, entry) {
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100302 list_del_init(&event->entry);
303 (event->function)(event->data);
304 if (event->interval) {
305 /* Recharge interval timer */
306 event->expires = event->interval + vq->elapsed;
307 spin_lock(&vq->lock);
308 list_add_sorted(event, &vq->list);
309 spin_unlock(&vq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 }
311 }
312}
313
314/*
315 * Handler for the virtual CPU timer.
316 */
Heiko Carstens5a489b92006-10-06 16:38:35 +0200317static void do_cpu_timer_interrupt(__u16 error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318{
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100319 struct vtimer_queue *vq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320 struct vtimer_list *event, *tmp;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100321 struct list_head cb_list; /* the callback queue */
322 __u64 elapsed, next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323
324 INIT_LIST_HEAD(&cb_list);
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100325 vq = &__get_cpu_var(virt_cpu_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326
327 /* walk timer list, fire all expired events */
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100328 spin_lock(&vq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100330 elapsed = vq->elapsed + (vq->timer - S390_lowcore.async_enter_timer);
331 BUG_ON((s64) elapsed < 0);
332 vq->elapsed = 0;
333 list_for_each_entry_safe(event, tmp, &vq->list, entry) {
334 if (event->expires < elapsed)
335 /* move expired timer to the callback queue */
336 list_move_tail(&event->entry, &cb_list);
337 else
338 event->expires -= elapsed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 }
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100340 spin_unlock(&vq->lock);
341
342 vq->do_spt = list_empty(&cb_list);
Heiko Carstens9d0a57c2006-10-11 15:31:26 +0200343 do_callbacks(&cb_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344
345 /* next event is first in list */
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100346 next = VTIMER_MAX_SLICE;
347 spin_lock(&vq->lock);
348 if (!list_empty(&vq->list)) {
349 event = list_first_entry(&vq->list, struct vtimer_list, entry);
350 next = event->expires;
351 } else
352 vq->do_spt = 0;
353 spin_unlock(&vq->lock);
354 /*
355 * To improve precision add the time spent by the
356 * interrupt handler to the elapsed time.
357 * Note: CPU timer counts down and we got an interrupt,
358 * the current content is negative
359 */
360 elapsed = S390_lowcore.async_enter_timer - get_vtimer();
361 set_vtimer(next - elapsed);
362 vq->timer = next - elapsed;
363 vq->elapsed = elapsed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364}
365
366void init_virt_timer(struct vtimer_list *timer)
367{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368 timer->function = NULL;
369 INIT_LIST_HEAD(&timer->entry);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370}
371EXPORT_SYMBOL(init_virt_timer);
372
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373static inline int vtimer_pending(struct vtimer_list *timer)
374{
375 return (!list_empty(&timer->entry));
376}
377
378/*
379 * this function should only run on the specified CPU
380 */
381static void internal_add_vtimer(struct vtimer_list *timer)
382{
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100383 struct vtimer_queue *vq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 unsigned long flags;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100385 __u64 left, expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100387 vq = &per_cpu(virt_cpu_timer, timer->cpu);
388 spin_lock_irqsave(&vq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389
Martin Schwidefskyca366a32008-07-14 09:59:23 +0200390 BUG_ON(timer->cpu != smp_processor_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100392 if (list_empty(&vq->list)) {
393 /* First timer on this cpu, just program it. */
394 list_add(&timer->entry, &vq->list);
395 set_vtimer(timer->expires);
396 vq->timer = timer->expires;
397 vq->elapsed = 0;
398 } else {
399 /* Check progress of old timers. */
400 expires = timer->expires;
401 left = get_vtimer();
402 if (likely((s64) expires < (s64) left)) {
403 /* The new timer expires before the current timer. */
404 set_vtimer(expires);
405 vq->elapsed += vq->timer - left;
406 vq->timer = expires;
407 } else {
408 vq->elapsed += vq->timer - left;
409 vq->timer = left;
410 }
411 /* Insert new timer into per cpu list. */
412 timer->expires += vq->elapsed;
413 list_add_sorted(timer, &vq->list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 }
415
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100416 spin_unlock_irqrestore(&vq->lock, flags);
Andreas Mohrd6e05ed2006-06-26 18:35:02 +0200417 /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 put_cpu();
419}
420
Martin Schwidefskyca366a32008-07-14 09:59:23 +0200421static inline void prepare_vtimer(struct vtimer_list *timer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422{
Martin Schwidefskyca366a32008-07-14 09:59:23 +0200423 BUG_ON(!timer->function);
424 BUG_ON(!timer->expires || timer->expires > VTIMER_MAX_SLICE);
425 BUG_ON(vtimer_pending(timer));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 timer->cpu = get_cpu();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427}
428
429/*
430 * add_virt_timer - add an oneshot virtual CPU timer
431 */
432void add_virt_timer(void *new)
433{
434 struct vtimer_list *timer;
435
436 timer = (struct vtimer_list *)new;
Martin Schwidefskyca366a32008-07-14 09:59:23 +0200437 prepare_vtimer(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438 timer->interval = 0;
439 internal_add_vtimer(timer);
440}
441EXPORT_SYMBOL(add_virt_timer);
442
443/*
444 * add_virt_timer_int - add an interval virtual CPU timer
445 */
446void add_virt_timer_periodic(void *new)
447{
448 struct vtimer_list *timer;
449
450 timer = (struct vtimer_list *)new;
Martin Schwidefskyca366a32008-07-14 09:59:23 +0200451 prepare_vtimer(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 timer->interval = timer->expires;
453 internal_add_vtimer(timer);
454}
455EXPORT_SYMBOL(add_virt_timer_periodic);
456
Jan Glauberb6ecfa92009-04-14 15:36:20 +0200457int __mod_vtimer(struct vtimer_list *timer, __u64 expires, int periodic)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458{
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100459 struct vtimer_queue *vq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 unsigned long flags;
461 int cpu;
462
Martin Schwidefskyca366a32008-07-14 09:59:23 +0200463 BUG_ON(!timer->function);
464 BUG_ON(!expires || expires > VTIMER_MAX_SLICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 if (timer->expires == expires && vtimer_pending(timer))
467 return 1;
468
469 cpu = get_cpu();
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100470 vq = &per_cpu(virt_cpu_timer, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471
472 /* disable interrupts before test if timer is pending */
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100473 spin_lock_irqsave(&vq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474
475 /* if timer isn't pending add it on the current CPU */
476 if (!vtimer_pending(timer)) {
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100477 spin_unlock_irqrestore(&vq->lock, flags);
Jan Glauberb6ecfa92009-04-14 15:36:20 +0200478
479 if (periodic)
480 timer->interval = expires;
481 else
482 timer->interval = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 timer->expires = expires;
484 timer->cpu = cpu;
485 internal_add_vtimer(timer);
486 return 0;
487 }
488
Jan Glauberb6ecfa92009-04-14 15:36:20 +0200489 /* check if we run on the right CPU */
490 BUG_ON(timer->cpu != cpu);
491
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492 list_del_init(&timer->entry);
493 timer->expires = expires;
Jan Glauberb6ecfa92009-04-14 15:36:20 +0200494 if (periodic)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 timer->interval = expires;
496
497 /* the timer can't expire anymore so we can release the lock */
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100498 spin_unlock_irqrestore(&vq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 internal_add_vtimer(timer);
500 return 1;
501}
Jan Glauberb6ecfa92009-04-14 15:36:20 +0200502
503/*
504 * If we change a pending timer the function must be called on the CPU
505 * where the timer is running on.
506 *
507 * returns whether it has modified a pending timer (1) or not (0)
508 */
509int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
510{
511 return __mod_vtimer(timer, expires, 0);
512}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513EXPORT_SYMBOL(mod_virt_timer);
514
515/*
Jan Glauberb6ecfa92009-04-14 15:36:20 +0200516 * If we change a pending timer the function must be called on the CPU
517 * where the timer is running on.
518 *
519 * returns whether it has modified a pending timer (1) or not (0)
520 */
521int mod_virt_timer_periodic(struct vtimer_list *timer, __u64 expires)
522{
523 return __mod_vtimer(timer, expires, 1);
524}
525EXPORT_SYMBOL(mod_virt_timer_periodic);
526
527/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 * delete a virtual timer
529 *
530 * returns whether the deleted timer was pending (1) or not (0)
531 */
532int del_virt_timer(struct vtimer_list *timer)
533{
534 unsigned long flags;
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100535 struct vtimer_queue *vq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 /* check if timer is pending */
538 if (!vtimer_pending(timer))
539 return 0;
540
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100541 vq = &per_cpu(virt_cpu_timer, timer->cpu);
542 spin_lock_irqsave(&vq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543
544 /* we don't interrupt a running timer, just let it expire! */
545 list_del_init(&timer->entry);
546
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100547 spin_unlock_irqrestore(&vq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 return 1;
549}
550EXPORT_SYMBOL(del_virt_timer);
551
552/*
553 * Start the virtual CPU timer on the current CPU.
554 */
555void init_cpu_vtimer(void)
556{
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100557 struct vtimer_queue *vq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558
Martin Schwidefsky9cfb9b32008-12-31 15:11:41 +0100559 /* initialize per cpu vtimer structure */
560 vq = &__get_cpu_var(virt_cpu_timer);
561 INIT_LIST_HEAD(&vq->list);
562 spin_lock_init(&vq->lock);
Martin Schwidefskyd54853e2007-02-05 21:18:19 +0100563
564 /* enable cpu timer interrupts */
565 __ctl_set_bit(0,10);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566}
567
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568void __init vtime_init(void)
569{
570 /* request the cpu timer external interrupt */
Heiko Carstensd7d11042009-06-22 12:08:03 +0200571 if (register_external_interrupt(0x1005, do_cpu_timer_interrupt))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 panic("Couldn't request external interrupt 0x1005");
573
Martin Schwidefskyd54853e2007-02-05 21:18:19 +0100574 /* Enable cpu timer interrupts on the boot cpu. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 init_cpu_vtimer();
576}
577