blob: ce6650eb64e976e120cbefa87adae9b4675e94ed [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Copyright (C) 1991, 1992 Linus Torvalds
Alexander van Heukeluma8c1be92008-07-02 01:29:44 +02003 * Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * Pentium III FXSR, SSE support
6 * Gareth Hughes <gareth@valinux.com>, May 2000
7 */
8
9/*
Alexander van Heukelumc1d518c2008-10-03 23:17:11 +020010 * Handle hardware traps and faults.
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/interrupt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/kallsyms.h>
Ingo Molnarb5964402008-02-26 11:15:50 +010014#include <linux/spinlock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/kprobes.h>
Andrew Morton1e2af922006-09-27 01:51:15 -070016#include <linux/uaccess.h>
Ingo Molnarb5964402008-02-26 11:15:50 +010017#include <linux/utsname.h>
18#include <linux/kdebug.h>
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/ptrace.h>
22#include <linux/string.h>
23#include <linux/unwind.h>
24#include <linux/delay.h>
25#include <linux/errno.h>
26#include <linux/kexec.h>
27#include <linux/sched.h>
28#include <linux/timer.h>
29#include <linux/init.h>
Jeremy Fitzhardinge91768d62006-12-08 02:36:21 -080030#include <linux/bug.h>
Ingo Molnarb5964402008-02-26 11:15:50 +010031#include <linux/nmi.h>
32#include <linux/mm.h>
Alexander van Heukelumc1d518c2008-10-03 23:17:11 +020033#include <linux/smp.h>
34#include <linux/io.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
36#ifdef CONFIG_EISA
37#include <linux/ioport.h>
38#include <linux/eisa.h>
39#endif
40
41#ifdef CONFIG_MCA
42#include <linux/mca.h>
43#endif
44
Dave Jiangc0d12172007-07-19 01:49:46 -070045#if defined(CONFIG_EDAC)
46#include <linux/edac.h>
47#endif
48
Ingo Molnarb5964402008-02-26 11:15:50 +010049#include <asm/stacktrace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050#include <asm/processor.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070051#include <asm/debugreg.h>
Ingo Molnarb5964402008-02-26 11:15:50 +010052#include <asm/atomic.h>
53#include <asm/system.h>
54#include <asm/unwind.h>
Alexander van Heukelumc1d518c2008-10-03 23:17:11 +020055#include <asm/traps.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056#include <asm/desc.h>
57#include <asm/i387.h>
Alexander van Heukelumc1d518c2008-10-03 23:17:11 +020058
59#include <mach_traps.h>
60
Alexander van Heukelum081f75b2008-10-03 22:00:39 +020061#ifdef CONFIG_X86_64
62#include <asm/pgalloc.h>
63#include <asm/proto.h>
64#include <asm/pda.h>
65#else
Alexander van Heukelumc1d518c2008-10-03 23:17:11 +020066#include <asm/processor-flags.h>
67#include <asm/arch_hooks.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070068#include <asm/nmi.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070069#include <asm/smp.h>
Ingo Molnarb5964402008-02-26 11:15:50 +010070#include <asm/io.h>
Jaswinder Singh6ac8d512008-07-15 21:09:13 +053071#include <asm/traps.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
Alexander van Heukelumeb642f62008-09-09 21:56:11 +020073#include "cpu/mcheck/mce.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
75asmlinkage int system_call(void);
76
Linus Torvalds1da177e2005-04-16 15:20:36 -070077/* Do we ignore FPU interrupts ? */
Ingo Molnarb5964402008-02-26 11:15:50 +010078char ignore_fpu_irq;
Linus Torvalds1da177e2005-04-16 15:20:36 -070079
80/*
81 * The IDT has to be page-aligned to simplify the Pentium
82 * F0 0F bug workaround.. We have a special link segment
83 * for this.
84 */
Glauber de Oliveira Costa010d4f82008-01-30 13:31:12 +010085gate_desc idt_table[256]
Glauber de Oliveira Costa6842ef02008-01-30 13:31:11 +010086 __attribute__((__section__(".data.idt"))) = { { { { 0, 0 } } }, };
Alexander van Heukelum081f75b2008-10-03 22:00:39 +020087#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Yinghai Lub77b8812008-12-19 15:23:44 -080089DECLARE_BITMAP(used_vectors, NR_VECTORS);
90EXPORT_SYMBOL_GPL(used_vectors);
91
Alexander van Heukelumbadc7652008-07-02 01:30:30 +020092static int ignore_nmis;
Alan Sterne041c682006-03-27 01:16:30 -080093
Alexander van Heukelum762db432008-09-09 21:55:55 +020094static inline void conditional_sti(struct pt_regs *regs)
95{
96 if (regs->flags & X86_EFLAGS_IF)
97 local_irq_enable();
98}
99
Alexander van Heukelum3d2a71a2008-09-30 18:41:37 +0200100static inline void preempt_conditional_sti(struct pt_regs *regs)
101{
102 inc_preempt_count();
103 if (regs->flags & X86_EFLAGS_IF)
104 local_irq_enable();
105}
106
107static inline void preempt_conditional_cli(struct pt_regs *regs)
108{
109 if (regs->flags & X86_EFLAGS_IF)
110 local_irq_disable();
111 dec_preempt_count();
112}
113
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200114#ifdef CONFIG_X86_32
Ingo Molnarb5964402008-02-26 11:15:50 +0100115static inline void
116die_if_kernel(const char *str, struct pt_regs *regs, long err)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117{
Vincent Hanquez717b5942005-06-23 00:08:45 -0700118 if (!user_mode_vm(regs))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119 die(str, regs, err);
120}
121
Alexander van Heukelumae821572008-09-30 18:41:35 +0200122/*
123 * Perform the lazy TSS's I/O bitmap copy. If the TSS has an
124 * invalid offset set (the LAZY one) and the faulting thread has
125 * a valid I/O bitmap pointer, we copy the I/O bitmap in the TSS,
126 * we set the offset field correctly and return 1.
127 */
128static int lazy_iobitmap_copy(void)
129{
130 struct thread_struct *thread;
131 struct tss_struct *tss;
132 int cpu;
133
134 cpu = get_cpu();
135 tss = &per_cpu(init_tss, cpu);
136 thread = &current->thread;
137
138 if (tss->x86_tss.io_bitmap_base == INVALID_IO_BITMAP_OFFSET_LAZY &&
139 thread->io_bitmap_ptr) {
140 memcpy(tss->io_bitmap, thread->io_bitmap_ptr,
141 thread->io_bitmap_max);
142 /*
143 * If the previously set map was extending to higher ports
144 * than the current one, pad extra space with 0xff (no access).
145 */
146 if (thread->io_bitmap_max < tss->io_bitmap_max) {
147 memset((char *) tss->io_bitmap +
148 thread->io_bitmap_max, 0xff,
149 tss->io_bitmap_max - thread->io_bitmap_max);
150 }
151 tss->io_bitmap_max = thread->io_bitmap_max;
152 tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
153 tss->io_bitmap_owner = thread;
154 put_cpu();
155
156 return 1;
157 }
158 put_cpu();
159
160 return 0;
161}
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200162#endif
Alexander van Heukelumae821572008-09-30 18:41:35 +0200163
Ingo Molnarb5964402008-02-26 11:15:50 +0100164static void __kprobes
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200165do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
Ingo Molnarb5964402008-02-26 11:15:50 +0100166 long error_code, siginfo_t *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167{
Alexander Nyberg4f339ec2005-06-25 14:58:27 -0700168 struct task_struct *tsk = current;
Alexander Nyberg4f339ec2005-06-25 14:58:27 -0700169
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200170#ifdef CONFIG_X86_32
gorcunov@gmail.com6b6891f2008-03-28 17:56:57 +0300171 if (regs->flags & X86_VM_MASK) {
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200172 /*
173 * traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
174 * On nmi (interrupt 2), do_trap should not be called.
175 */
176 if (trapnr < 6)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 goto vm86_trap;
178 goto trap_signal;
179 }
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200180#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
Vincent Hanquez717b5942005-06-23 00:08:45 -0700182 if (!user_mode(regs))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 goto kernel_trap;
184
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200185#ifdef CONFIG_X86_32
Ingo Molnarb5964402008-02-26 11:15:50 +0100186trap_signal:
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200187#endif
Ingo Molnarb5964402008-02-26 11:15:50 +0100188 /*
189 * We want error_code and trap_no set for userspace faults and
190 * kernelspace faults which result in die(), but not
191 * kernelspace faults which are fixed up. die() gives the
192 * process no chance to handle the signal and notice the
193 * kernel fault information, so that won't result in polluting
194 * the information about previously queued, but not yet
195 * delivered, faults. See also do_general_protection below.
196 */
197 tsk->thread.error_code = error_code;
198 tsk->thread.trap_no = trapnr;
199
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200200#ifdef CONFIG_X86_64
201 if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
202 printk_ratelimit()) {
203 printk(KERN_INFO
204 "%s[%d] trap %s ip:%lx sp:%lx error:%lx",
205 tsk->comm, tsk->pid, str,
206 regs->ip, regs->sp, error_code);
207 print_vma_addr(" in ", regs->ip);
208 printk("\n");
209 }
210#endif
211
Ingo Molnarb5964402008-02-26 11:15:50 +0100212 if (info)
213 force_sig_info(signr, info, tsk);
214 else
215 force_sig(signr, tsk);
216 return;
217
218kernel_trap:
219 if (!fixup_exception(regs)) {
Andi Kleend1895182007-05-02 19:27:05 +0200220 tsk->thread.error_code = error_code;
221 tsk->thread.trap_no = trapnr;
Ingo Molnarb5964402008-02-26 11:15:50 +0100222 die(str, regs, error_code);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 }
Ingo Molnarb5964402008-02-26 11:15:50 +0100224 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200226#ifdef CONFIG_X86_32
Ingo Molnarb5964402008-02-26 11:15:50 +0100227vm86_trap:
228 if (handle_vm86_trap((struct kernel_vm86_regs *) regs,
229 error_code, trapnr))
230 goto trap_signal;
231 return;
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200232#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233}
234
Ingo Molnarb5964402008-02-26 11:15:50 +0100235#define DO_ERROR(trapnr, signr, str, name) \
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200236dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
Ingo Molnarb5964402008-02-26 11:15:50 +0100237{ \
238 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
Alexander van Heukeluma8c1be92008-07-02 01:29:44 +0200239 == NOTIFY_STOP) \
Ingo Molnarb5964402008-02-26 11:15:50 +0100240 return; \
Alexander van Heukelum61aef7d2008-09-09 21:55:56 +0200241 conditional_sti(regs); \
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200242 do_trap(trapnr, signr, str, regs, error_code, NULL); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243}
244
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200245#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200246dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
Ingo Molnarb5964402008-02-26 11:15:50 +0100247{ \
248 siginfo_t info; \
249 info.si_signo = signr; \
250 info.si_errno = 0; \
251 info.si_code = sicode; \
252 info.si_addr = (void __user *)siaddr; \
Ingo Molnarb5964402008-02-26 11:15:50 +0100253 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
Alexander van Heukeluma8c1be92008-07-02 01:29:44 +0200254 == NOTIFY_STOP) \
Ingo Molnarb5964402008-02-26 11:15:50 +0100255 return; \
Alexander van Heukelum61aef7d2008-09-09 21:55:56 +0200256 conditional_sti(regs); \
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200257 do_trap(trapnr, signr, str, regs, error_code, &info); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258}
259
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200260DO_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip)
261DO_ERROR(4, SIGSEGV, "overflow", overflow)
262DO_ERROR(5, SIGSEGV, "bounds", bounds)
263DO_ERROR_INFO(6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip)
Alexander van Heukelum51bc1ed2008-09-09 21:56:03 +0200264DO_ERROR(9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun)
Alexander van Heukelum6bf77bf2008-09-09 21:56:04 +0200265DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS)
Alexander van Heukelum36d936c2008-09-09 21:56:05 +0200266DO_ERROR(11, SIGBUS, "segment not present", segment_not_present)
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200267#ifdef CONFIG_X86_32
Alexander van Heukelumf5ca8182008-09-09 21:56:06 +0200268DO_ERROR(12, SIGBUS, "stack segment", stack_segment)
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200269#endif
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200270DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200272#ifdef CONFIG_X86_64
273/* Runs on IST stack */
274dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code)
275{
276 if (notify_die(DIE_TRAP, "stack segment", regs, error_code,
277 12, SIGBUS) == NOTIFY_STOP)
278 return;
279 preempt_conditional_sti(regs);
280 do_trap(12, SIGBUS, "stack segment", regs, error_code, NULL);
281 preempt_conditional_cli(regs);
282}
283
284dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
285{
286 static const char str[] = "double fault";
287 struct task_struct *tsk = current;
288
289 /* Return not checked because double check cannot be ignored */
290 notify_die(DIE_TRAP, str, regs, error_code, 8, SIGSEGV);
291
292 tsk->thread.error_code = error_code;
293 tsk->thread.trap_no = 8;
294
Ingo Molnarbd8b96d2008-12-26 09:20:22 +0100295 /*
296 * This is always a kernel trap and never fixable (and thus must
297 * never return).
298 */
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200299 for (;;)
300 die(str, regs, error_code);
301}
302#endif
303
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200304dotraplinkage void __kprobes
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200305do_general_protection(struct pt_regs *regs, long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306{
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200307 struct task_struct *tsk;
Ingo Molnarb5964402008-02-26 11:15:50 +0100308
Alexander van Heukelumc6df0d72008-09-09 21:56:07 +0200309 conditional_sti(regs);
310
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200311#ifdef CONFIG_X86_32
Alexander van Heukelumae821572008-09-30 18:41:35 +0200312 if (lazy_iobitmap_copy()) {
313 /* restart the faulting instruction */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 return;
315 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316
gorcunov@gmail.com6b6891f2008-03-28 17:56:57 +0300317 if (regs->flags & X86_VM_MASK)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 goto gp_in_vm86;
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200319#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200321 tsk = current;
Vincent Hanquez717b5942005-06-23 00:08:45 -0700322 if (!user_mode(regs))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 goto gp_in_kernel;
324
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200325 tsk->thread.error_code = error_code;
326 tsk->thread.trap_no = 13;
Ingo Molnarb5964402008-02-26 11:15:50 +0100327
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200328 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
329 printk_ratelimit()) {
Masoud Asgharifard Sharbianiabd4f752007-07-22 11:12:28 +0200330 printk(KERN_INFO
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200331 "%s[%d] general protection ip:%lx sp:%lx error:%lx",
332 tsk->comm, task_pid_nr(tsk),
333 regs->ip, regs->sp, error_code);
Andi Kleen03252912008-01-30 13:33:18 +0100334 print_vma_addr(" in ", regs->ip);
335 printk("\n");
336 }
Masoud Asgharifard Sharbianiabd4f752007-07-22 11:12:28 +0200337
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200338 force_sig(SIGSEGV, tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 return;
340
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200341#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342gp_in_vm86:
343 local_irq_enable();
344 handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
345 return;
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200346#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347
348gp_in_kernel:
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200349 if (fixup_exception(regs))
350 return;
351
352 tsk->thread.error_code = error_code;
353 tsk->thread.trap_no = 13;
354 if (notify_die(DIE_GPF, "general protection fault", regs,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 error_code, 13, SIGSEGV) == NOTIFY_STOP)
Alexander van Heukelum13485ab2008-07-02 01:32:04 +0200356 return;
357 die("general protection fault", regs, error_code);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358}
359
Steven Rostedt5deb45e2008-04-19 19:19:55 +0200360static notrace __kprobes void
Ingo Molnarb5964402008-02-26 11:15:50 +0100361mem_parity_error(unsigned char reason, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362{
Ingo Molnarb5964402008-02-26 11:15:50 +0100363 printk(KERN_EMERG
364 "Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
365 reason, smp_processor_id());
366
367 printk(KERN_EMERG
368 "You have some hardware problem, likely on the PCI bus.\n");
Dave Jiangc0d12172007-07-19 01:49:46 -0700369
370#if defined(CONFIG_EDAC)
Ingo Molnarb5964402008-02-26 11:15:50 +0100371 if (edac_handler_set()) {
Dave Jiangc0d12172007-07-19 01:49:46 -0700372 edac_atomic_assert_error();
373 return;
374 }
375#endif
376
Don Zickus8da5add2006-09-26 10:52:27 +0200377 if (panic_on_unrecovered_nmi)
Ingo Molnarb5964402008-02-26 11:15:50 +0100378 panic("NMI: Not continuing");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379
Don Zickusc41c5cd2006-09-26 10:52:27 +0200380 printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381
382 /* Clear and disable the memory parity error line. */
Alexander van Heukelum79704792008-10-03 22:00:36 +0200383 reason = (reason & 0xf) | 4;
384 outb(reason, 0x61);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385}
386
Steven Rostedt5deb45e2008-04-19 19:19:55 +0200387static notrace __kprobes void
Ingo Molnarb5964402008-02-26 11:15:50 +0100388io_check_error(unsigned char reason, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389{
390 unsigned long i;
391
Dave Jones9c107802006-01-09 20:51:32 -0800392 printk(KERN_EMERG "NMI: IOCK error (debug interrupt?)\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 show_registers(regs);
394
395 /* Re-enable the IOCK line, wait for a few seconds */
396 reason = (reason & 0xf) | 8;
397 outb(reason, 0x61);
Ingo Molnarb5964402008-02-26 11:15:50 +0100398
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 i = 2000;
Ingo Molnarb5964402008-02-26 11:15:50 +0100400 while (--i)
401 udelay(1000);
402
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403 reason &= ~8;
404 outb(reason, 0x61);
405}
406
Steven Rostedt5deb45e2008-04-19 19:19:55 +0200407static notrace __kprobes void
Ingo Molnarb5964402008-02-26 11:15:50 +0100408unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409{
Alexander van Heukelumc1d518c2008-10-03 23:17:11 +0200410 if (notify_die(DIE_NMIUNKNOWN, "nmi", regs, reason, 2, SIGINT) ==
411 NOTIFY_STOP)
Jason Wesseld3597522008-02-15 14:55:53 -0600412 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413#ifdef CONFIG_MCA
Ingo Molnarb5964402008-02-26 11:15:50 +0100414 /*
415 * Might actually be able to figure out what the guilty party
416 * is:
417 */
418 if (MCA_bus) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 mca_handle_nmi();
420 return;
421 }
422#endif
Ingo Molnarb5964402008-02-26 11:15:50 +0100423 printk(KERN_EMERG
424 "Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
425 reason, smp_processor_id());
426
Don Zickusc41c5cd2006-09-26 10:52:27 +0200427 printk(KERN_EMERG "Do you have a strange power saving mode enabled?\n");
Don Zickus8da5add2006-09-26 10:52:27 +0200428 if (panic_on_unrecovered_nmi)
Ingo Molnarb5964402008-02-26 11:15:50 +0100429 panic("NMI: Not continuing");
Don Zickus8da5add2006-09-26 10:52:27 +0200430
Don Zickusc41c5cd2006-09-26 10:52:27 +0200431 printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432}
433
Steven Rostedt5deb45e2008-04-19 19:19:55 +0200434static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435{
436 unsigned char reason = 0;
Alexander van Heukelumabd34802008-07-02 18:39:01 +0200437 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438
Alexander van Heukelumabd34802008-07-02 18:39:01 +0200439 cpu = smp_processor_id();
440
441 /* Only the BSP gets external NMIs from the system. */
442 if (!cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 reason = get_nmi_reason();
Ingo Molnarb5964402008-02-26 11:15:50 +0100444
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 if (!(reason & 0xc0)) {
Jan Beulich20c0d2d2006-03-26 01:37:01 -0800446 if (notify_die(DIE_NMI_IPI, "nmi_ipi", regs, reason, 2, SIGINT)
Alexander van Heukeluma8c1be92008-07-02 01:29:44 +0200447 == NOTIFY_STOP)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 return;
449#ifdef CONFIG_X86_LOCAL_APIC
450 /*
451 * Ok, so this is none of the documented NMI sources,
452 * so it must be the NMI watchdog.
453 */
Don Zickus3adbbcce2006-09-26 10:52:26 +0200454 if (nmi_watchdog_tick(regs, reason))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 return;
Alexander van Heukelumabd34802008-07-02 18:39:01 +0200456 if (!do_nmi_callback(regs, cpu))
Don Zickus3adbbcce2006-09-26 10:52:26 +0200457 unknown_nmi_error(reason, regs);
Ingo Molnarb5964402008-02-26 11:15:50 +0100458#else
459 unknown_nmi_error(reason, regs);
460#endif
Don Zickus2fbe7b22006-09-26 10:52:27 +0200461
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 return;
463 }
Jan Beulich20c0d2d2006-03-26 01:37:01 -0800464 if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 return;
Alexander van Heukeluma8c1be92008-07-02 01:29:44 +0200466
467 /* AK: following checks seem to be broken on modern chipsets. FIXME */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 if (reason & 0x80)
469 mem_parity_error(reason, regs);
470 if (reason & 0x40)
471 io_check_error(reason, regs);
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200472#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 /*
474 * Reassert NMI in case it became active meanwhile
Ingo Molnarb5964402008-02-26 11:15:50 +0100475 * as it's edge-triggered:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 */
477 reassert_nmi();
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200478#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479}
480
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200481dotraplinkage notrace __kprobes void
482do_nmi(struct pt_regs *regs, long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 nmi_enter();
485
Hiroshi Shimamoto915b0d02008-12-08 19:19:26 -0800486 inc_irq_stat(__nmi_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487
Andi Kleen8f4e9562007-07-22 11:12:32 +0200488 if (!ignore_nmis)
489 default_do_nmi(regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490
491 nmi_exit();
492}
493
Andi Kleen8f4e9562007-07-22 11:12:32 +0200494void stop_nmi(void)
495{
496 acpi_nmi_disable();
497 ignore_nmis++;
498}
499
500void restart_nmi(void)
501{
502 ignore_nmis--;
503 acpi_nmi_enable();
504}
505
Alexander van Heukelumc1d518c2008-10-03 23:17:11 +0200506/* May run on IST stack. */
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200507dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508{
Alexander van Heukelumb94da1e2008-09-09 21:55:58 +0200509#ifdef CONFIG_KPROBES
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP)
511 == NOTIFY_STOP)
Stas Sergeev48c88212005-05-01 08:58:49 -0700512 return;
Alexander van Heukelumb94da1e2008-09-09 21:55:58 +0200513#else
514 if (notify_die(DIE_TRAP, "int3", regs, error_code, 3, SIGTRAP)
515 == NOTIFY_STOP)
516 return;
517#endif
Ingo Molnarb5964402008-02-26 11:15:50 +0100518
Alexander van Heukelum4915a352008-10-03 22:00:34 +0200519 preempt_conditional_sti(regs);
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200520 do_trap(3, SIGTRAP, "int3", regs, error_code, NULL);
Alexander van Heukelum4915a352008-10-03 22:00:34 +0200521 preempt_conditional_cli(regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200524#ifdef CONFIG_X86_64
Ingo Molnarbd8b96d2008-12-26 09:20:22 +0100525/*
526 * Help handler running on IST stack to switch back to user stack
527 * for scheduling or signal handling. The actual stack switch is done in
528 * entry.S
529 */
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200530asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
531{
532 struct pt_regs *regs = eregs;
533 /* Did already sync */
534 if (eregs == (struct pt_regs *)eregs->sp)
535 ;
536 /* Exception from user space */
537 else if (user_mode(eregs))
538 regs = task_pt_regs(current);
Ingo Molnarbd8b96d2008-12-26 09:20:22 +0100539 /*
540 * Exception from kernel and interrupts are enabled. Move to
541 * kernel process stack.
542 */
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200543 else if (eregs->flags & X86_EFLAGS_IF)
544 regs = (struct pt_regs *)(eregs->sp -= sizeof(struct pt_regs));
545 if (eregs != regs)
546 *regs = *eregs;
547 return regs;
548}
549#endif
550
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551/*
552 * Our handling of the processor debug registers is non-trivial.
553 * We do not clear them on entry and exit from the kernel. Therefore
554 * it is possible to get a watchpoint trap here from inside the kernel.
555 * However, the code in ./ptrace.c has ensured that the user can
556 * only set watchpoints on userspace addresses. Therefore the in-kernel
557 * watchpoint trap can only occur in code which is reading/writing
558 * from user space. Such code must not hold kernel locks (since it
559 * can equally take a page fault), therefore it is safe to call
560 * force_sig_info even though that claims and releases locks.
Ingo Molnarb5964402008-02-26 11:15:50 +0100561 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 * Code in ./signal.c ensures that the debug control register
563 * is restored before we deliver any signal, and therefore that
564 * user code runs with the correct debug control register even though
565 * we clear it here.
566 *
567 * Being careful here means that we don't have to be as careful in a
568 * lot of more complicated places (task switching can be a bit lazy
569 * about restoring all the debug state, and ptrace doesn't have to
570 * find every occurrence of the TF bit that could be saved away even
571 * by user code)
Alexander van Heukelumc1d518c2008-10-03 23:17:11 +0200572 *
573 * May run on IST stack.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 */
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200575dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577 struct task_struct *tsk = current;
Alexander van Heukelum3d2a71a2008-09-30 18:41:37 +0200578 unsigned long condition;
Srinivasa Dsda654b72008-09-23 15:23:52 +0530579 int si_code;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580
Vincent Hanquez1cc6f122005-06-23 00:08:43 -0700581 get_debugreg(condition, 6);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582
Roland McGrath10faa812008-01-30 13:30:54 +0100583 /*
584 * The processor cleared BTF, so don't mark that we need it set.
585 */
586 clear_tsk_thread_flag(tsk, TIF_DEBUGCTLMSR);
587 tsk->thread.debugctlmsr = 0;
588
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 if (notify_die(DIE_DEBUG, "debug", regs, condition, error_code,
Alexander van Heukeluma8c1be92008-07-02 01:29:44 +0200590 SIGTRAP) == NOTIFY_STOP)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 return;
Alexander van Heukelum3d2a71a2008-09-30 18:41:37 +0200592
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 /* It's safe to allow irq's after DR6 has been saved */
Alexander van Heukelum3d2a71a2008-09-30 18:41:37 +0200594 preempt_conditional_sti(regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
596 /* Mask out spurious debug traps due to lazy DR7 setting */
597 if (condition & (DR_TRAP0|DR_TRAP1|DR_TRAP2|DR_TRAP3)) {
Roland McGrath0f534092008-01-30 13:30:59 +0100598 if (!tsk->thread.debugreg7)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599 goto clear_dr7;
600 }
601
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200602#ifdef CONFIG_X86_32
gorcunov@gmail.com6b6891f2008-03-28 17:56:57 +0300603 if (regs->flags & X86_VM_MASK)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 goto debug_vm86;
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200605#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606
607 /* Save debug status register where ptrace can see it */
Roland McGrath0f534092008-01-30 13:30:59 +0100608 tsk->thread.debugreg6 = condition;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609
610 /*
611 * Single-stepping through TF: make sure we ignore any events in
612 * kernel space (but re-enable TF when returning to user mode).
613 */
614 if (condition & DR_STEP) {
Vincent Hanquez717b5942005-06-23 00:08:45 -0700615 if (!user_mode(regs))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 goto clear_TF_reenable;
617 }
618
Alexander van Heukelum3d2a71a2008-09-30 18:41:37 +0200619 si_code = get_si_code(condition);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 /* Ok, finally something we can handle */
Srinivasa Dsda654b72008-09-23 15:23:52 +0530621 send_sigtrap(tsk, regs, error_code, si_code);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622
Ingo Molnarb5964402008-02-26 11:15:50 +0100623 /*
624 * Disable additional traps. They'll be re-enabled when
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 * the signal is delivered.
626 */
627clear_dr7:
Vincent Hanquez1cc6f122005-06-23 00:08:43 -0700628 set_debugreg(0, 7);
Alexander van Heukelum3d2a71a2008-09-30 18:41:37 +0200629 preempt_conditional_cli(regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 return;
631
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200632#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633debug_vm86:
634 handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code, 1);
Alexander van Heukelum3d2a71a2008-09-30 18:41:37 +0200635 preempt_conditional_cli(regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 return;
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200637#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
639clear_TF_reenable:
640 set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
Ingo Molnar60930152008-03-30 11:45:23 +0200641 regs->flags &= ~X86_EFLAGS_TF;
Alexander van Heukelum3d2a71a2008-09-30 18:41:37 +0200642 preempt_conditional_cli(regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 return;
644}
645
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200646#ifdef CONFIG_X86_64
647static int kernel_math_error(struct pt_regs *regs, const char *str, int trapnr)
648{
649 if (fixup_exception(regs))
650 return 1;
651
652 notify_die(DIE_GPF, str, regs, 0, trapnr, SIGFPE);
653 /* Illegal floating point operation in the kernel */
654 current->thread.trap_no = trapnr;
655 die(str, regs, 0);
656 return 0;
657}
658#endif
659
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660/*
661 * Note that we play around with the 'TS' bit in an attempt to get
662 * the correct behaviour even in the presence of the asynchronous
663 * IRQ13 behaviour
664 */
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100665void math_error(void __user *ip)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666{
Ingo Molnarb5964402008-02-26 11:15:50 +0100667 struct task_struct *task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 siginfo_t info;
H. Peter Anvinadf77ba2008-12-22 17:56:05 -0800669 unsigned short cwd, swd, err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670
671 /*
672 * Save the info for the exception handler and clear the error.
673 */
674 task = current;
675 save_init_fpu(task);
676 task->thread.trap_no = 16;
677 task->thread.error_code = 0;
678 info.si_signo = SIGFPE;
679 info.si_errno = 0;
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100680 info.si_addr = ip;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 /*
682 * (~cwd & swd) will mask out exceptions that are not set to unmasked
683 * status. 0x3f is the exception bits in these regs, 0x200 is the
684 * C1 reg you need in case of a stack fault, 0x040 is the stack
685 * fault bit. We should only be taking one exception at a time,
686 * so if this combination doesn't produce any single exception,
Alexander van Heukeluma8c1be92008-07-02 01:29:44 +0200687 * then we have a bad program that isn't synchronizing its FPU usage
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 * and it will suffer the consequences since we won't be able to
689 * fully reproduce the context of the exception
690 */
691 cwd = get_fpu_cwd(task);
692 swd = get_fpu_swd(task);
H. Peter Anvinadf77ba2008-12-22 17:56:05 -0800693
H. Peter Anvina73ad332008-12-25 10:39:01 -0800694 err = swd & ~cwd;
H. Peter Anvinadf77ba2008-12-22 17:56:05 -0800695
696 if (err & 0x001) { /* Invalid op */
Ingo Molnarb5964402008-02-26 11:15:50 +0100697 /*
698 * swd & 0x240 == 0x040: Stack Underflow
699 * swd & 0x240 == 0x240: Stack Overflow
700 * User must clear the SF bit (0x40) if set
701 */
702 info.si_code = FPE_FLTINV;
H. Peter Anvinadf77ba2008-12-22 17:56:05 -0800703 } else if (err & 0x004) { /* Divide by Zero */
Ingo Molnarb5964402008-02-26 11:15:50 +0100704 info.si_code = FPE_FLTDIV;
H. Peter Anvinadf77ba2008-12-22 17:56:05 -0800705 } else if (err & 0x008) { /* Overflow */
Ingo Molnarb5964402008-02-26 11:15:50 +0100706 info.si_code = FPE_FLTOVF;
H. Peter Anvinadf77ba2008-12-22 17:56:05 -0800707 } else if (err & 0x012) { /* Denormal, Underflow */
708 info.si_code = FPE_FLTUND;
709 } else if (err & 0x020) { /* Precision */
Ingo Molnarb5964402008-02-26 11:15:50 +0100710 info.si_code = FPE_FLTRES;
H. Peter Anvinadf77ba2008-12-22 17:56:05 -0800711 } else {
Ingo Molnarbd8b96d2008-12-26 09:20:22 +0100712 /*
713 * If we're using IRQ 13, or supposedly even some trap 16
714 * implementations, it's possible we get a spurious trap...
715 */
H. Peter Anvina73ad332008-12-25 10:39:01 -0800716 return; /* Spurious trap, no error */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 }
718 force_sig_info(SIGFPE, &info, task);
719}
720
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200721dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722{
Alexander van Heukelum252d28f2008-09-09 21:56:09 +0200723 conditional_sti(regs);
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200724
725#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 ignore_fpu_irq = 1;
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200727#else
728 if (!user_mode(regs) &&
729 kernel_math_error(regs, "kernel x87 math error", 16))
730 return;
731#endif
732
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100733 math_error((void __user *)regs->ip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734}
735
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100736static void simd_math_error(void __user *ip)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737{
Ingo Molnarb5964402008-02-26 11:15:50 +0100738 struct task_struct *task;
Ingo Molnarb5964402008-02-26 11:15:50 +0100739 siginfo_t info;
Alexander van Heukelum7b4fd4b2008-07-02 01:33:14 +0200740 unsigned short mxcsr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741
742 /*
743 * Save the info for the exception handler and clear the error.
744 */
745 task = current;
746 save_init_fpu(task);
747 task->thread.trap_no = 19;
748 task->thread.error_code = 0;
749 info.si_signo = SIGFPE;
750 info.si_errno = 0;
751 info.si_code = __SI_FAULT;
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100752 info.si_addr = ip;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 /*
754 * The SIMD FPU exceptions are handled a little differently, as there
755 * is only a single status/control register. Thus, to determine which
756 * unmasked exception was caught we must mask the exception mask bits
757 * at 0x1f80, and then use these to mask the exception bits at 0x3f.
758 */
759 mxcsr = get_fpu_mxcsr(task);
760 switch (~((mxcsr & 0x1f80) >> 7) & (mxcsr & 0x3f)) {
Ingo Molnarb5964402008-02-26 11:15:50 +0100761 case 0x000:
762 default:
763 break;
764 case 0x001: /* Invalid Op */
765 info.si_code = FPE_FLTINV;
766 break;
767 case 0x002: /* Denormalize */
768 case 0x010: /* Underflow */
769 info.si_code = FPE_FLTUND;
770 break;
771 case 0x004: /* Zero Divide */
772 info.si_code = FPE_FLTDIV;
773 break;
774 case 0x008: /* Overflow */
775 info.si_code = FPE_FLTOVF;
776 break;
777 case 0x020: /* Precision */
778 info.si_code = FPE_FLTRES;
779 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 }
781 force_sig_info(SIGFPE, &info, task);
782}
783
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200784dotraplinkage void
785do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786{
Alexander van Heukelumb939bde2008-09-09 21:56:12 +0200787 conditional_sti(regs);
788
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200789#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 if (cpu_has_xmm) {
791 /* Handle SIMD FPU exceptions on PIII+ processors. */
792 ignore_fpu_irq = 1;
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100793 simd_math_error((void __user *)regs->ip);
Ingo Molnarb5964402008-02-26 11:15:50 +0100794 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 }
Ingo Molnarb5964402008-02-26 11:15:50 +0100796 /*
797 * Handle strange cache flush from user space exception
798 * in all other cases. This is undocumented behaviour.
799 */
gorcunov@gmail.com6b6891f2008-03-28 17:56:57 +0300800 if (regs->flags & X86_VM_MASK) {
Ingo Molnarb5964402008-02-26 11:15:50 +0100801 handle_vm86_fault((struct kernel_vm86_regs *)regs, error_code);
802 return;
803 }
804 current->thread.trap_no = 19;
805 current->thread.error_code = error_code;
806 die_if_kernel("cache flush denied", regs, error_code);
807 force_sig(SIGSEGV, current);
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200808#else
809 if (!user_mode(regs) &&
810 kernel_math_error(regs, "kernel simd math error", 19))
811 return;
812 simd_math_error((void __user *)regs->ip);
813#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814}
815
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200816dotraplinkage void
817do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818{
Alexander van Heukelumcf819782008-09-09 21:56:08 +0200819 conditional_sti(regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820#if 0
821 /* No need to warn about this any longer. */
Ingo Molnarb5964402008-02-26 11:15:50 +0100822 printk(KERN_INFO "Ignoring P6 Local APIC Spurious Interrupt Bug...\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823#endif
824}
825
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200826#ifdef CONFIG_X86_32
Ingo Molnarb5964402008-02-26 11:15:50 +0100827unsigned long patch_espfix_desc(unsigned long uesp, unsigned long kesp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828{
Glauber Costa736f12b2008-05-27 20:14:51 -0700829 struct desc_struct *gdt = get_cpu_gdt_table(smp_processor_id());
Stas Sergeevbe44d2a2006-12-07 02:14:01 +0100830 unsigned long base = (kesp - uesp) & -THREAD_SIZE;
831 unsigned long new_kesp = kesp - base;
832 unsigned long lim_pages = (new_kesp | (THREAD_SIZE - 1)) >> PAGE_SHIFT;
833 __u64 desc = *(__u64 *)&gdt[GDT_ENTRY_ESPFIX_SS];
Ingo Molnarb5964402008-02-26 11:15:50 +0100834
Stas Sergeevbe44d2a2006-12-07 02:14:01 +0100835 /* Set up base for espfix segment */
Ingo Molnarb5964402008-02-26 11:15:50 +0100836 desc &= 0x00f0ff0000000000ULL;
837 desc |= ((((__u64)base) << 16) & 0x000000ffffff0000ULL) |
Stas Sergeevbe44d2a2006-12-07 02:14:01 +0100838 ((((__u64)base) << 32) & 0xff00000000000000ULL) |
839 ((((__u64)lim_pages) << 32) & 0x000f000000000000ULL) |
840 (lim_pages & 0xffff);
841 *(__u64 *)&gdt[GDT_ENTRY_ESPFIX_SS] = desc;
Ingo Molnarb5964402008-02-26 11:15:50 +0100842
Stas Sergeevbe44d2a2006-12-07 02:14:01 +0100843 return new_kesp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844}
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200845#else
846asmlinkage void __attribute__((weak)) smp_thermal_interrupt(void)
847{
848}
849
850asmlinkage void __attribute__((weak)) mce_threshold_interrupt(void)
851{
852}
853#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854
855/*
Ingo Molnarb5964402008-02-26 11:15:50 +0100856 * 'math_state_restore()' saves the current math information in the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 * old math state array, and gets the new ones from the current task
858 *
859 * Careful.. There are problems with IBM-designed IRQ13 behaviour.
860 * Don't touch unless you *really* know how it works.
861 *
862 * Must be called with kernel preemption disabled (in this case,
863 * local interrupts are disabled at the call-site in entry.S).
864 */
Chuck Ebbertacc20762006-12-07 02:14:01 +0100865asmlinkage void math_state_restore(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866{
867 struct thread_info *thread = current_thread_info();
868 struct task_struct *tsk = thread->task;
869
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700870 if (!tsk_used_math(tsk)) {
871 local_irq_enable();
872 /*
873 * does a slab alloc which can sleep
874 */
875 if (init_fpu(tsk)) {
876 /*
877 * ran out of memory!
878 */
879 do_group_exit(SIGKILL);
880 return;
881 }
882 local_irq_disable();
883 }
884
Ingo Molnarb5964402008-02-26 11:15:50 +0100885 clts(); /* Allow maths ops (or we recurse) */
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200886#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 restore_fpu(tsk);
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200888#else
889 /*
890 * Paranoid restore. send a SIGSEGV if we fail to restore the state.
891 */
892 if (unlikely(restore_fpu_checking(tsk))) {
893 stts();
894 force_sig(SIGSEGV, tsk);
895 return;
896 }
897#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898 thread->status |= TS_USEDFPU; /* So we fnsave on switch_to() */
Chuck Ebbertacc20762006-12-07 02:14:01 +0100899 tsk->fpu_counter++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900}
Rusty Russell5992b6d2007-07-19 01:49:21 -0700901EXPORT_SYMBOL_GPL(math_state_restore);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902
903#ifndef CONFIG_MATH_EMULATION
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904asmlinkage void math_emulate(long arg)
905{
Ingo Molnarb5964402008-02-26 11:15:50 +0100906 printk(KERN_EMERG
907 "math-emulation not enabled and no coprocessor found.\n");
908 printk(KERN_EMERG "killing %s.\n", current->comm);
909 force_sig(SIGFPE, current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910 schedule();
911}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912#endif /* CONFIG_MATH_EMULATION */
913
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200914dotraplinkage void __kprobes
915do_device_not_available(struct pt_regs *regs, long error)
Alexander van Heukelum7643e9b2008-09-09 21:56:02 +0200916{
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200917#ifdef CONFIG_X86_32
Alexander van Heukelum7643e9b2008-09-09 21:56:02 +0200918 if (read_cr0() & X86_CR0_EM) {
919 conditional_sti(regs);
920 math_emulate(0);
921 } else {
922 math_state_restore(); /* interrupts still off */
923 conditional_sti(regs);
924 }
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200925#else
926 math_state_restore();
927#endif
Alexander van Heukelum7643e9b2008-09-09 21:56:02 +0200928}
929
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200930#ifdef CONFIG_X86_32
Alexander van Heukelume407d6202008-09-30 18:41:36 +0200931dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
Alexander van Heukelumf8e08702008-09-09 21:56:13 +0200932{
933 siginfo_t info;
934 local_irq_enable();
935
936 info.si_signo = SIGILL;
937 info.si_errno = 0;
938 info.si_code = ILL_BADSTK;
939 info.si_addr = 0;
940 if (notify_die(DIE_TRAP, "iret exception",
941 regs, error_code, 32, SIGILL) == NOTIFY_STOP)
942 return;
Alexander van Heukelum3c1326f2008-09-26 14:03:08 +0200943 do_trap(32, SIGILL, "iret exception", regs, error_code, &info);
Alexander van Heukelumf8e08702008-09-09 21:56:13 +0200944}
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200945#endif
Alexander van Heukelumf8e08702008-09-09 21:56:13 +0200946
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947void __init trap_init(void)
948{
Rusty Russelldbeb2be2007-10-19 20:35:03 +0200949 int i;
950
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951#ifdef CONFIG_EISA
Ingo Molnar927222b2008-01-30 13:33:49 +0100952 void __iomem *p = early_ioremap(0x0FFFD9, 4);
Ingo Molnarb5964402008-02-26 11:15:50 +0100953
954 if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955 EISA_bus = 1;
Ingo Molnar927222b2008-01-30 13:33:49 +0100956 early_iounmap(p, 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957#endif
958
Alexander van Heukelum976382d2008-09-09 21:55:57 +0200959 set_intr_gate(0, &divide_error);
Alexander van Heukelum699d2932008-10-03 22:00:32 +0200960 set_intr_gate_ist(1, &debug, DEBUG_STACK);
961 set_intr_gate_ist(2, &nmi, NMI_STACK);
962 /* int3 can be called from all */
963 set_system_intr_gate_ist(3, &int3, DEBUG_STACK);
964 /* int4 can be called from all */
965 set_system_intr_gate(4, &overflow);
Alexander van Heukelum64f644c2008-09-09 21:56:00 +0200966 set_intr_gate(5, &bounds);
Alexander van Heukelum12394cf2008-09-09 21:56:01 +0200967 set_intr_gate(6, &invalid_op);
Alexander van Heukelum7643e9b2008-09-09 21:56:02 +0200968 set_intr_gate(7, &device_not_available);
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200969#ifdef CONFIG_X86_32
Alexander van Heukeluma8c1be92008-07-02 01:29:44 +0200970 set_task_gate(8, GDT_ENTRY_DOUBLEFAULT_TSS);
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200971#else
972 set_intr_gate_ist(8, &double_fault, DOUBLEFAULT_STACK);
973#endif
Alexander van Heukelum51bc1ed2008-09-09 21:56:03 +0200974 set_intr_gate(9, &coprocessor_segment_overrun);
Alexander van Heukelum6bf77bf2008-09-09 21:56:04 +0200975 set_intr_gate(10, &invalid_TSS);
Alexander van Heukelum36d936c2008-09-09 21:56:05 +0200976 set_intr_gate(11, &segment_not_present);
Alexander van Heukelum699d2932008-10-03 22:00:32 +0200977 set_intr_gate_ist(12, &stack_segment, STACKFAULT_STACK);
Alexander van Heukelumc6df0d72008-09-09 21:56:07 +0200978 set_intr_gate(13, &general_protection);
Ingo Molnarb5964402008-02-26 11:15:50 +0100979 set_intr_gate(14, &page_fault);
Alexander van Heukelumcf819782008-09-09 21:56:08 +0200980 set_intr_gate(15, &spurious_interrupt_bug);
Alexander van Heukelum252d28f2008-09-09 21:56:09 +0200981 set_intr_gate(16, &coprocessor_error);
Alexander van Heukelum5feedfd2008-09-09 21:56:10 +0200982 set_intr_gate(17, &alignment_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983#ifdef CONFIG_X86_MCE
Alexander van Heukelum699d2932008-10-03 22:00:32 +0200984 set_intr_gate_ist(18, &machine_check, MCE_STACK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985#endif
Alexander van Heukelumb939bde2008-09-09 21:56:12 +0200986 set_intr_gate(19, &simd_coprocessor_error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987
Alexander van Heukelum081f75b2008-10-03 22:00:39 +0200988#ifdef CONFIG_IA32_EMULATION
989 set_system_intr_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
990#endif
991
992#ifdef CONFIG_X86_32
Jan Beulichd43c6e82006-01-06 00:11:49 -0800993 if (cpu_has_fxsr) {
Jan Beulichd43c6e82006-01-06 00:11:49 -0800994 printk(KERN_INFO "Enabling fast FPU save and restore... ");
995 set_in_cr4(X86_CR4_OSFXSR);
996 printk("done.\n");
997 }
998 if (cpu_has_xmm) {
Ingo Molnarb5964402008-02-26 11:15:50 +0100999 printk(KERN_INFO
1000 "Enabling unmasked SIMD FPU exception support... ");
Jan Beulichd43c6e82006-01-06 00:11:49 -08001001 set_in_cr4(X86_CR4_OSXMMEXCPT);
1002 printk("done.\n");
1003 }
1004
Alexander van Heukelum699d2932008-10-03 22:00:32 +02001005 set_system_trap_gate(SYSCALL_VECTOR, &system_call);
Yinghai Lub77b8812008-12-19 15:23:44 -08001006#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007
Ingo Molnarb5964402008-02-26 11:15:50 +01001008 /* Reserve all the builtin and the syscall vector: */
Rusty Russelldbeb2be2007-10-19 20:35:03 +02001009 for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
1010 set_bit(i, used_vectors);
Ingo Molnarb5964402008-02-26 11:15:50 +01001011
Yinghai Lub77b8812008-12-19 15:23:44 -08001012#ifdef CONFIG_X86_64
1013 set_bit(IA32_SYSCALL_VECTOR, used_vectors);
1014#else
Rusty Russelldbeb2be2007-10-19 20:35:03 +02001015 set_bit(SYSCALL_VECTOR, used_vectors);
Alexander van Heukelum081f75b2008-10-03 22:00:39 +02001016#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 /*
Ingo Molnarb5964402008-02-26 11:15:50 +01001018 * Should be a barrier for any external CPU state:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019 */
1020 cpu_init();
1021
Alexander van Heukelum081f75b2008-10-03 22:00:39 +02001022#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023 trap_init_hook();
Alexander van Heukelum081f75b2008-10-03 22:00:39 +02001024#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025}