Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * arch/s390/kernel/vtime.c |
| 3 | * Virtual cpu timer based timer functions. |
| 4 | * |
| 5 | * S390 version |
| 6 | * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation |
| 7 | * Author(s): Jan Glauber <jan.glauber@de.ibm.com> |
| 8 | */ |
| 9 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | #include <linux/module.h> |
| 11 | #include <linux/kernel.h> |
| 12 | #include <linux/time.h> |
| 13 | #include <linux/delay.h> |
| 14 | #include <linux/init.h> |
| 15 | #include <linux/smp.h> |
| 16 | #include <linux/types.h> |
| 17 | #include <linux/timex.h> |
| 18 | #include <linux/notifier.h> |
| 19 | #include <linux/kernel_stat.h> |
| 20 | #include <linux/rcupdate.h> |
| 21 | #include <linux/posix-timers.h> |
| 22 | |
| 23 | #include <asm/s390_ext.h> |
| 24 | #include <asm/timer.h> |
Heiko Carstens | 5a489b9 | 2006-10-06 16:38:35 +0200 | [diff] [blame] | 25 | #include <asm/irq_regs.h> |
Martin Schwidefsky | 76d4e00 | 2009-06-12 10:26:21 +0200 | [diff] [blame] | 26 | #include <asm/cputime.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 27 | |
Heiko Carstens | 2b67fc4 | 2007-02-05 21:16:47 +0100 | [diff] [blame] | 28 | static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 30 | DEFINE_PER_CPU(struct s390_idle_data, s390_idle); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 31 | |
| 32 | static inline __u64 get_vtimer(void) |
| 33 | { |
| 34 | __u64 timer; |
| 35 | |
| 36 | asm volatile("STPT %0" : "=m" (timer)); |
| 37 | return timer; |
| 38 | } |
| 39 | |
| 40 | static inline void set_vtimer(__u64 expires) |
| 41 | { |
| 42 | __u64 timer; |
| 43 | |
| 44 | asm volatile (" STPT %0\n" /* Store current cpu timer value */ |
| 45 | " SPT %1" /* Set new value immediatly afterwards */ |
| 46 | : "=m" (timer) : "m" (expires) ); |
| 47 | S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; |
| 48 | S390_lowcore.last_update_timer = expires; |
| 49 | } |
| 50 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 51 | /* |
| 52 | * Update process times based on virtual cpu times stored by entry.S |
| 53 | * to the lowcore fields user_timer, system_timer & steal_clock. |
| 54 | */ |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 55 | static void do_account_vtime(struct task_struct *tsk, int hardirq_offset) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 56 | { |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 57 | struct thread_info *ti = task_thread_info(tsk); |
| 58 | __u64 timer, clock, user, system, steal; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 59 | |
| 60 | timer = S390_lowcore.last_update_timer; |
| 61 | clock = S390_lowcore.last_update_clock; |
| 62 | asm volatile (" STPT %0\n" /* Store current cpu timer value */ |
| 63 | " STCK %1" /* Store current tod clock value */ |
| 64 | : "=m" (S390_lowcore.last_update_timer), |
| 65 | "=m" (S390_lowcore.last_update_clock) ); |
| 66 | S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 67 | S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 68 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 69 | user = S390_lowcore.user_timer - ti->user_timer; |
| 70 | S390_lowcore.steal_timer -= user; |
| 71 | ti->user_timer = S390_lowcore.user_timer; |
| 72 | account_user_time(tsk, user, user); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 74 | system = S390_lowcore.system_timer - ti->system_timer; |
| 75 | S390_lowcore.steal_timer -= system; |
| 76 | ti->system_timer = S390_lowcore.system_timer; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 77 | account_system_time(tsk, hardirq_offset, system, system); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 79 | steal = S390_lowcore.steal_timer; |
| 80 | if ((s64) steal > 0) { |
| 81 | S390_lowcore.steal_timer = 0; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 82 | account_steal_time(steal); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 83 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | } |
| 85 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 86 | void account_vtime(struct task_struct *prev, struct task_struct *next) |
Martin Schwidefsky | 1f1c12a | 2006-01-14 13:21:03 -0800 | [diff] [blame] | 87 | { |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 88 | struct thread_info *ti; |
Martin Schwidefsky | 1f1c12a | 2006-01-14 13:21:03 -0800 | [diff] [blame] | 89 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 90 | do_account_vtime(prev, 0); |
| 91 | ti = task_thread_info(prev); |
| 92 | ti->user_timer = S390_lowcore.user_timer; |
| 93 | ti->system_timer = S390_lowcore.system_timer; |
| 94 | ti = task_thread_info(next); |
| 95 | S390_lowcore.user_timer = ti->user_timer; |
| 96 | S390_lowcore.system_timer = ti->system_timer; |
| 97 | } |
Martin Schwidefsky | 1f1c12a | 2006-01-14 13:21:03 -0800 | [diff] [blame] | 98 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 99 | void account_process_tick(struct task_struct *tsk, int user_tick) |
| 100 | { |
| 101 | do_account_vtime(tsk, HARDIRQ_OFFSET); |
Martin Schwidefsky | 1f1c12a | 2006-01-14 13:21:03 -0800 | [diff] [blame] | 102 | } |
| 103 | |
| 104 | /* |
| 105 | * Update process times based on virtual cpu times stored by entry.S |
| 106 | * to the lowcore fields user_timer, system_timer & steal_clock. |
| 107 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | void account_system_vtime(struct task_struct *tsk) |
| 109 | { |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 110 | struct thread_info *ti = task_thread_info(tsk); |
| 111 | __u64 timer, system; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | |
| 113 | timer = S390_lowcore.last_update_timer; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 114 | S390_lowcore.last_update_timer = get_vtimer(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 115 | S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; |
| 116 | |
Martin Schwidefsky | aa5e97c | 2008-12-31 15:11:39 +0100 | [diff] [blame] | 117 | system = S390_lowcore.system_timer - ti->system_timer; |
| 118 | S390_lowcore.steal_timer -= system; |
| 119 | ti->system_timer = S390_lowcore.system_timer; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 120 | account_system_time(tsk, 0, system, system); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | } |
Heiko Carstens | b0c632d | 2008-03-25 18:47:20 +0100 | [diff] [blame] | 122 | EXPORT_SYMBOL_GPL(account_system_vtime); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | |
Martin Schwidefsky | 6377981 | 2010-05-17 10:00:03 +0200 | [diff] [blame] | 124 | void vtime_start_cpu(__u64 int_clock, __u64 enter_timer) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 125 | { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 126 | struct s390_idle_data *idle = &__get_cpu_var(s390_idle); |
Martin Schwidefsky | 6f43092 | 2008-12-31 15:11:40 +0100 | [diff] [blame] | 127 | struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 128 | __u64 idle_time, expires; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | |
Martin Schwidefsky | 6377981 | 2010-05-17 10:00:03 +0200 | [diff] [blame] | 130 | if (idle->idle_enter == 0ULL) |
| 131 | return; |
| 132 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 133 | /* Account time spent with enabled wait psw loaded as idle time. */ |
Martin Schwidefsky | 6377981 | 2010-05-17 10:00:03 +0200 | [diff] [blame] | 134 | idle_time = int_clock - idle->idle_enter; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 135 | account_idle_time(idle_time); |
Martin Schwidefsky | b86ccca | 2009-04-14 15:36:26 +0200 | [diff] [blame] | 136 | S390_lowcore.steal_timer += |
| 137 | idle->idle_enter - S390_lowcore.last_update_clock; |
Martin Schwidefsky | 6377981 | 2010-05-17 10:00:03 +0200 | [diff] [blame] | 138 | S390_lowcore.last_update_clock = int_clock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 140 | /* Account system time spent going idle. */ |
| 141 | S390_lowcore.system_timer += S390_lowcore.last_update_timer - vq->idle; |
Martin Schwidefsky | 6377981 | 2010-05-17 10:00:03 +0200 | [diff] [blame] | 142 | S390_lowcore.last_update_timer = enter_timer; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 143 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 144 | /* Restart vtime CPU timer */ |
| 145 | if (vq->do_spt) { |
| 146 | /* Program old expire value but first save progress. */ |
Martin Schwidefsky | 6377981 | 2010-05-17 10:00:03 +0200 | [diff] [blame] | 147 | expires = vq->idle - enter_timer; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 148 | expires += get_vtimer(); |
| 149 | set_vtimer(expires); |
| 150 | } else { |
| 151 | /* Don't account the CPU timer delta while the cpu was idle. */ |
Martin Schwidefsky | 6377981 | 2010-05-17 10:00:03 +0200 | [diff] [blame] | 152 | vq->elapsed -= vq->idle - enter_timer; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 153 | } |
| 154 | |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 155 | idle->sequence++; |
| 156 | smp_wmb(); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 157 | idle->idle_time += idle_time; |
| 158 | idle->idle_enter = 0ULL; |
| 159 | idle->idle_count++; |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 160 | smp_wmb(); |
| 161 | idle->sequence++; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 162 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 163 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 164 | void vtime_stop_cpu(void) |
| 165 | { |
| 166 | struct s390_idle_data *idle = &__get_cpu_var(s390_idle); |
| 167 | struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer); |
| 168 | psw_t psw; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 169 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 170 | /* Wait for external, I/O or machine check interrupt. */ |
| 171 | psw.mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_IO | PSW_MASK_EXT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 172 | |
Martin Schwidefsky | 3c5d92a | 2009-09-29 14:25:16 +0200 | [diff] [blame] | 173 | idle->nohz_delay = 0; |
| 174 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 175 | /* Check if the CPU timer needs to be reprogrammed. */ |
| 176 | if (vq->do_spt) { |
| 177 | __u64 vmax = VTIMER_MAX_SLICE; |
| 178 | /* |
| 179 | * The inline assembly is equivalent to |
| 180 | * vq->idle = get_cpu_timer(); |
| 181 | * set_cpu_timer(VTIMER_MAX_SLICE); |
| 182 | * idle->idle_enter = get_clock(); |
| 183 | * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT | |
| 184 | * PSW_MASK_IO | PSW_MASK_EXT); |
| 185 | * The difference is that the inline assembly makes sure that |
| 186 | * the last three instruction are stpt, stck and lpsw in that |
| 187 | * order. This is done to increase the precision. |
| 188 | */ |
| 189 | asm volatile( |
| 190 | #ifndef CONFIG_64BIT |
| 191 | " basr 1,0\n" |
| 192 | "0: ahi 1,1f-0b\n" |
| 193 | " st 1,4(%2)\n" |
| 194 | #else /* CONFIG_64BIT */ |
| 195 | " larl 1,1f\n" |
| 196 | " stg 1,8(%2)\n" |
| 197 | #endif /* CONFIG_64BIT */ |
| 198 | " stpt 0(%4)\n" |
| 199 | " spt 0(%5)\n" |
| 200 | " stck 0(%3)\n" |
| 201 | #ifndef CONFIG_64BIT |
| 202 | " lpsw 0(%2)\n" |
| 203 | #else /* CONFIG_64BIT */ |
| 204 | " lpswe 0(%2)\n" |
| 205 | #endif /* CONFIG_64BIT */ |
| 206 | "1:" |
| 207 | : "=m" (idle->idle_enter), "=m" (vq->idle) |
| 208 | : "a" (&psw), "a" (&idle->idle_enter), |
| 209 | "a" (&vq->idle), "a" (&vmax), "m" (vmax), "m" (psw) |
| 210 | : "memory", "cc", "1"); |
| 211 | } else { |
| 212 | /* |
| 213 | * The inline assembly is equivalent to |
| 214 | * vq->idle = get_cpu_timer(); |
| 215 | * idle->idle_enter = get_clock(); |
| 216 | * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT | |
| 217 | * PSW_MASK_IO | PSW_MASK_EXT); |
| 218 | * The difference is that the inline assembly makes sure that |
| 219 | * the last three instruction are stpt, stck and lpsw in that |
| 220 | * order. This is done to increase the precision. |
| 221 | */ |
| 222 | asm volatile( |
| 223 | #ifndef CONFIG_64BIT |
| 224 | " basr 1,0\n" |
| 225 | "0: ahi 1,1f-0b\n" |
| 226 | " st 1,4(%2)\n" |
| 227 | #else /* CONFIG_64BIT */ |
| 228 | " larl 1,1f\n" |
| 229 | " stg 1,8(%2)\n" |
| 230 | #endif /* CONFIG_64BIT */ |
| 231 | " stpt 0(%4)\n" |
| 232 | " stck 0(%3)\n" |
| 233 | #ifndef CONFIG_64BIT |
| 234 | " lpsw 0(%2)\n" |
| 235 | #else /* CONFIG_64BIT */ |
| 236 | " lpswe 0(%2)\n" |
| 237 | #endif /* CONFIG_64BIT */ |
| 238 | "1:" |
| 239 | : "=m" (idle->idle_enter), "=m" (vq->idle) |
| 240 | : "a" (&psw), "a" (&idle->idle_enter), |
| 241 | "a" (&vq->idle), "m" (psw) |
| 242 | : "memory", "cc", "1"); |
| 243 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 244 | } |
| 245 | |
Martin Schwidefsky | e1c8053 | 2009-04-23 13:58:08 +0200 | [diff] [blame] | 246 | cputime64_t s390_get_idle_time(int cpu) |
| 247 | { |
| 248 | struct s390_idle_data *idle; |
| 249 | unsigned long long now, idle_time, idle_enter; |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 250 | unsigned int sequence; |
Martin Schwidefsky | e1c8053 | 2009-04-23 13:58:08 +0200 | [diff] [blame] | 251 | |
| 252 | idle = &per_cpu(s390_idle, cpu); |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 253 | |
Martin Schwidefsky | e1c8053 | 2009-04-23 13:58:08 +0200 | [diff] [blame] | 254 | now = get_clock(); |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 255 | repeat: |
| 256 | sequence = idle->sequence; |
| 257 | smp_rmb(); |
| 258 | if (sequence & 1) |
| 259 | goto repeat; |
Martin Schwidefsky | e1c8053 | 2009-04-23 13:58:08 +0200 | [diff] [blame] | 260 | idle_time = 0; |
| 261 | idle_enter = idle->idle_enter; |
| 262 | if (idle_enter != 0ULL && idle_enter < now) |
| 263 | idle_time = now - idle_enter; |
Martin Schwidefsky | e98bbaa | 2009-06-22 12:08:20 +0200 | [diff] [blame] | 264 | smp_rmb(); |
| 265 | if (idle->sequence != sequence) |
| 266 | goto repeat; |
Martin Schwidefsky | e1c8053 | 2009-04-23 13:58:08 +0200 | [diff] [blame] | 267 | return idle_time; |
| 268 | } |
| 269 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 270 | /* |
| 271 | * Sorted add to a list. List is linear searched until first bigger |
| 272 | * element is found. |
| 273 | */ |
| 274 | static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) |
| 275 | { |
| 276 | struct vtimer_list *event; |
| 277 | |
| 278 | list_for_each_entry(event, head, entry) { |
| 279 | if (event->expires > timer->expires) { |
| 280 | list_add_tail(&timer->entry, &event->entry); |
| 281 | return; |
| 282 | } |
| 283 | } |
| 284 | list_add_tail(&timer->entry, head); |
| 285 | } |
| 286 | |
| 287 | /* |
| 288 | * Do the callback functions of expired vtimer events. |
| 289 | * Called from within the interrupt handler. |
| 290 | */ |
Heiko Carstens | 9d0a57c | 2006-10-11 15:31:26 +0200 | [diff] [blame] | 291 | static void do_callbacks(struct list_head *cb_list) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 292 | { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 293 | struct vtimer_queue *vq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 294 | struct vtimer_list *event, *tmp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 295 | |
| 296 | if (list_empty(cb_list)) |
| 297 | return; |
| 298 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 299 | vq = &__get_cpu_var(virt_cpu_timer); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 300 | |
| 301 | list_for_each_entry_safe(event, tmp, cb_list, entry) { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 302 | list_del_init(&event->entry); |
| 303 | (event->function)(event->data); |
| 304 | if (event->interval) { |
| 305 | /* Recharge interval timer */ |
| 306 | event->expires = event->interval + vq->elapsed; |
| 307 | spin_lock(&vq->lock); |
| 308 | list_add_sorted(event, &vq->list); |
| 309 | spin_unlock(&vq->lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 310 | } |
| 311 | } |
| 312 | } |
| 313 | |
| 314 | /* |
| 315 | * Handler for the virtual CPU timer. |
| 316 | */ |
Heiko Carstens | 5a489b9 | 2006-10-06 16:38:35 +0200 | [diff] [blame] | 317 | static void do_cpu_timer_interrupt(__u16 error_code) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 318 | { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 319 | struct vtimer_queue *vq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 320 | struct vtimer_list *event, *tmp; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 321 | struct list_head cb_list; /* the callback queue */ |
| 322 | __u64 elapsed, next; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 323 | |
| 324 | INIT_LIST_HEAD(&cb_list); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 325 | vq = &__get_cpu_var(virt_cpu_timer); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 326 | |
| 327 | /* walk timer list, fire all expired events */ |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 328 | spin_lock(&vq->lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 329 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 330 | elapsed = vq->elapsed + (vq->timer - S390_lowcore.async_enter_timer); |
| 331 | BUG_ON((s64) elapsed < 0); |
| 332 | vq->elapsed = 0; |
| 333 | list_for_each_entry_safe(event, tmp, &vq->list, entry) { |
| 334 | if (event->expires < elapsed) |
| 335 | /* move expired timer to the callback queue */ |
| 336 | list_move_tail(&event->entry, &cb_list); |
| 337 | else |
| 338 | event->expires -= elapsed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 339 | } |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 340 | spin_unlock(&vq->lock); |
| 341 | |
| 342 | vq->do_spt = list_empty(&cb_list); |
Heiko Carstens | 9d0a57c | 2006-10-11 15:31:26 +0200 | [diff] [blame] | 343 | do_callbacks(&cb_list); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 344 | |
| 345 | /* next event is first in list */ |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 346 | next = VTIMER_MAX_SLICE; |
| 347 | spin_lock(&vq->lock); |
| 348 | if (!list_empty(&vq->list)) { |
| 349 | event = list_first_entry(&vq->list, struct vtimer_list, entry); |
| 350 | next = event->expires; |
| 351 | } else |
| 352 | vq->do_spt = 0; |
| 353 | spin_unlock(&vq->lock); |
| 354 | /* |
| 355 | * To improve precision add the time spent by the |
| 356 | * interrupt handler to the elapsed time. |
| 357 | * Note: CPU timer counts down and we got an interrupt, |
| 358 | * the current content is negative |
| 359 | */ |
| 360 | elapsed = S390_lowcore.async_enter_timer - get_vtimer(); |
| 361 | set_vtimer(next - elapsed); |
| 362 | vq->timer = next - elapsed; |
| 363 | vq->elapsed = elapsed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | } |
| 365 | |
| 366 | void init_virt_timer(struct vtimer_list *timer) |
| 367 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 368 | timer->function = NULL; |
| 369 | INIT_LIST_HEAD(&timer->entry); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | } |
| 371 | EXPORT_SYMBOL(init_virt_timer); |
| 372 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 373 | static inline int vtimer_pending(struct vtimer_list *timer) |
| 374 | { |
| 375 | return (!list_empty(&timer->entry)); |
| 376 | } |
| 377 | |
| 378 | /* |
| 379 | * this function should only run on the specified CPU |
| 380 | */ |
| 381 | static void internal_add_vtimer(struct vtimer_list *timer) |
| 382 | { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 383 | struct vtimer_queue *vq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 384 | unsigned long flags; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 385 | __u64 left, expires; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 386 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 387 | vq = &per_cpu(virt_cpu_timer, timer->cpu); |
| 388 | spin_lock_irqsave(&vq->lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 389 | |
Martin Schwidefsky | ca366a3 | 2008-07-14 09:59:23 +0200 | [diff] [blame] | 390 | BUG_ON(timer->cpu != smp_processor_id()); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 391 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 392 | if (list_empty(&vq->list)) { |
| 393 | /* First timer on this cpu, just program it. */ |
| 394 | list_add(&timer->entry, &vq->list); |
| 395 | set_vtimer(timer->expires); |
| 396 | vq->timer = timer->expires; |
| 397 | vq->elapsed = 0; |
| 398 | } else { |
| 399 | /* Check progress of old timers. */ |
| 400 | expires = timer->expires; |
| 401 | left = get_vtimer(); |
| 402 | if (likely((s64) expires < (s64) left)) { |
| 403 | /* The new timer expires before the current timer. */ |
| 404 | set_vtimer(expires); |
| 405 | vq->elapsed += vq->timer - left; |
| 406 | vq->timer = expires; |
| 407 | } else { |
| 408 | vq->elapsed += vq->timer - left; |
| 409 | vq->timer = left; |
| 410 | } |
| 411 | /* Insert new timer into per cpu list. */ |
| 412 | timer->expires += vq->elapsed; |
| 413 | list_add_sorted(timer, &vq->list); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 414 | } |
| 415 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 416 | spin_unlock_irqrestore(&vq->lock, flags); |
Andreas Mohr | d6e05ed | 2006-06-26 18:35:02 +0200 | [diff] [blame] | 417 | /* release CPU acquired in prepare_vtimer or mod_virt_timer() */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 418 | put_cpu(); |
| 419 | } |
| 420 | |
Martin Schwidefsky | ca366a3 | 2008-07-14 09:59:23 +0200 | [diff] [blame] | 421 | static inline void prepare_vtimer(struct vtimer_list *timer) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 422 | { |
Martin Schwidefsky | ca366a3 | 2008-07-14 09:59:23 +0200 | [diff] [blame] | 423 | BUG_ON(!timer->function); |
| 424 | BUG_ON(!timer->expires || timer->expires > VTIMER_MAX_SLICE); |
| 425 | BUG_ON(vtimer_pending(timer)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 426 | timer->cpu = get_cpu(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 427 | } |
| 428 | |
| 429 | /* |
| 430 | * add_virt_timer - add an oneshot virtual CPU timer |
| 431 | */ |
| 432 | void add_virt_timer(void *new) |
| 433 | { |
| 434 | struct vtimer_list *timer; |
| 435 | |
| 436 | timer = (struct vtimer_list *)new; |
Martin Schwidefsky | ca366a3 | 2008-07-14 09:59:23 +0200 | [diff] [blame] | 437 | prepare_vtimer(timer); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 438 | timer->interval = 0; |
| 439 | internal_add_vtimer(timer); |
| 440 | } |
| 441 | EXPORT_SYMBOL(add_virt_timer); |
| 442 | |
| 443 | /* |
| 444 | * add_virt_timer_int - add an interval virtual CPU timer |
| 445 | */ |
| 446 | void add_virt_timer_periodic(void *new) |
| 447 | { |
| 448 | struct vtimer_list *timer; |
| 449 | |
| 450 | timer = (struct vtimer_list *)new; |
Martin Schwidefsky | ca366a3 | 2008-07-14 09:59:23 +0200 | [diff] [blame] | 451 | prepare_vtimer(timer); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 452 | timer->interval = timer->expires; |
| 453 | internal_add_vtimer(timer); |
| 454 | } |
| 455 | EXPORT_SYMBOL(add_virt_timer_periodic); |
| 456 | |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 457 | int __mod_vtimer(struct vtimer_list *timer, __u64 expires, int periodic) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 458 | { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 459 | struct vtimer_queue *vq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 460 | unsigned long flags; |
| 461 | int cpu; |
| 462 | |
Martin Schwidefsky | ca366a3 | 2008-07-14 09:59:23 +0200 | [diff] [blame] | 463 | BUG_ON(!timer->function); |
| 464 | BUG_ON(!expires || expires > VTIMER_MAX_SLICE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 465 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 466 | if (timer->expires == expires && vtimer_pending(timer)) |
| 467 | return 1; |
| 468 | |
| 469 | cpu = get_cpu(); |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 470 | vq = &per_cpu(virt_cpu_timer, cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 471 | |
| 472 | /* disable interrupts before test if timer is pending */ |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 473 | spin_lock_irqsave(&vq->lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 474 | |
| 475 | /* if timer isn't pending add it on the current CPU */ |
| 476 | if (!vtimer_pending(timer)) { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 477 | spin_unlock_irqrestore(&vq->lock, flags); |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 478 | |
| 479 | if (periodic) |
| 480 | timer->interval = expires; |
| 481 | else |
| 482 | timer->interval = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 483 | timer->expires = expires; |
| 484 | timer->cpu = cpu; |
| 485 | internal_add_vtimer(timer); |
| 486 | return 0; |
| 487 | } |
| 488 | |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 489 | /* check if we run on the right CPU */ |
| 490 | BUG_ON(timer->cpu != cpu); |
| 491 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 492 | list_del_init(&timer->entry); |
| 493 | timer->expires = expires; |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 494 | if (periodic) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 495 | timer->interval = expires; |
| 496 | |
| 497 | /* the timer can't expire anymore so we can release the lock */ |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 498 | spin_unlock_irqrestore(&vq->lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 499 | internal_add_vtimer(timer); |
| 500 | return 1; |
| 501 | } |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 502 | |
| 503 | /* |
| 504 | * If we change a pending timer the function must be called on the CPU |
| 505 | * where the timer is running on. |
| 506 | * |
| 507 | * returns whether it has modified a pending timer (1) or not (0) |
| 508 | */ |
| 509 | int mod_virt_timer(struct vtimer_list *timer, __u64 expires) |
| 510 | { |
| 511 | return __mod_vtimer(timer, expires, 0); |
| 512 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 513 | EXPORT_SYMBOL(mod_virt_timer); |
| 514 | |
| 515 | /* |
Jan Glauber | b6ecfa9 | 2009-04-14 15:36:20 +0200 | [diff] [blame] | 516 | * If we change a pending timer the function must be called on the CPU |
| 517 | * where the timer is running on. |
| 518 | * |
| 519 | * returns whether it has modified a pending timer (1) or not (0) |
| 520 | */ |
| 521 | int mod_virt_timer_periodic(struct vtimer_list *timer, __u64 expires) |
| 522 | { |
| 523 | return __mod_vtimer(timer, expires, 1); |
| 524 | } |
| 525 | EXPORT_SYMBOL(mod_virt_timer_periodic); |
| 526 | |
| 527 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 528 | * delete a virtual timer |
| 529 | * |
| 530 | * returns whether the deleted timer was pending (1) or not (0) |
| 531 | */ |
| 532 | int del_virt_timer(struct vtimer_list *timer) |
| 533 | { |
| 534 | unsigned long flags; |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 535 | struct vtimer_queue *vq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 536 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 537 | /* check if timer is pending */ |
| 538 | if (!vtimer_pending(timer)) |
| 539 | return 0; |
| 540 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 541 | vq = &per_cpu(virt_cpu_timer, timer->cpu); |
| 542 | spin_lock_irqsave(&vq->lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 543 | |
| 544 | /* we don't interrupt a running timer, just let it expire! */ |
| 545 | list_del_init(&timer->entry); |
| 546 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 547 | spin_unlock_irqrestore(&vq->lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 548 | return 1; |
| 549 | } |
| 550 | EXPORT_SYMBOL(del_virt_timer); |
| 551 | |
| 552 | /* |
| 553 | * Start the virtual CPU timer on the current CPU. |
| 554 | */ |
| 555 | void init_cpu_vtimer(void) |
| 556 | { |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 557 | struct vtimer_queue *vq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 558 | |
Martin Schwidefsky | 9cfb9b3 | 2008-12-31 15:11:41 +0100 | [diff] [blame] | 559 | /* initialize per cpu vtimer structure */ |
| 560 | vq = &__get_cpu_var(virt_cpu_timer); |
| 561 | INIT_LIST_HEAD(&vq->list); |
| 562 | spin_lock_init(&vq->lock); |
Martin Schwidefsky | d54853e | 2007-02-05 21:18:19 +0100 | [diff] [blame] | 563 | |
| 564 | /* enable cpu timer interrupts */ |
| 565 | __ctl_set_bit(0,10); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 566 | } |
| 567 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 568 | void __init vtime_init(void) |
| 569 | { |
| 570 | /* request the cpu timer external interrupt */ |
Heiko Carstens | d7d1104 | 2009-06-22 12:08:03 +0200 | [diff] [blame] | 571 | if (register_external_interrupt(0x1005, do_cpu_timer_interrupt)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 572 | panic("Couldn't request external interrupt 0x1005"); |
| 573 | |
Martin Schwidefsky | d54853e | 2007-02-05 21:18:19 +0100 | [diff] [blame] | 574 | /* Enable cpu timer interrupts on the boot cpu. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 575 | init_cpu_vtimer(); |
| 576 | } |
| 577 | |