blob: fb903e65e079a26034f76a30ef6f54b40613453d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86-64/kernel/process.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 *
6 * Pentium III FXSR, SSE support
7 * Gareth Hughes <gareth@valinux.com>, May 2000
8 *
9 * X86-64 port
10 * Andi Kleen.
Ashok Raj76e4f662005-06-25 14:55:00 -070011 *
12 * CPU hotplug support - ashok.raj@intel.com
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * $Id: process.c,v 1.38 2002/01/15 10:08:03 ak Exp $
14 */
15
16/*
17 * This file handles the architecture-dependent parts of process handling..
18 */
19
20#include <stdarg.h>
21
Ashok Raj76e4f662005-06-25 14:55:00 -070022#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/errno.h>
24#include <linux/sched.h>
25#include <linux/kernel.h>
26#include <linux/mm.h>
27#include <linux/elfcore.h>
28#include <linux/smp.h>
29#include <linux/slab.h>
30#include <linux/user.h>
31#include <linux/module.h>
32#include <linux/a.out.h>
33#include <linux/interrupt.h>
34#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/ptrace.h>
36#include <linux/utsname.h>
37#include <linux/random.h>
Andi Kleen95833c82006-01-11 22:44:36 +010038#include <linux/notifier.h>
bibo maoc6fd91f2006-03-26 01:38:20 -080039#include <linux/kprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
41#include <asm/uaccess.h>
42#include <asm/pgtable.h>
43#include <asm/system.h>
44#include <asm/io.h>
45#include <asm/processor.h>
46#include <asm/i387.h>
47#include <asm/mmu_context.h>
48#include <asm/pda.h>
49#include <asm/prctl.h>
50#include <asm/kdebug.h>
51#include <asm/desc.h>
52#include <asm/proto.h>
53#include <asm/ia32.h>
Andi Kleen95833c82006-01-11 22:44:36 +010054#include <asm/idle.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
56asmlinkage extern void ret_from_fork(void);
57
58unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
59
Linus Torvalds1da177e2005-04-16 15:20:36 -070060unsigned long boot_option_idle_override = 0;
61EXPORT_SYMBOL(boot_option_idle_override);
62
63/*
64 * Powermanagement idle function, if any..
65 */
66void (*pm_idle)(void);
67static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
68
Alan Sterne041c682006-03-27 01:16:30 -080069static ATOMIC_NOTIFIER_HEAD(idle_notifier);
Andi Kleen95833c82006-01-11 22:44:36 +010070
71void idle_notifier_register(struct notifier_block *n)
72{
Alan Sterne041c682006-03-27 01:16:30 -080073 atomic_notifier_chain_register(&idle_notifier, n);
Andi Kleen95833c82006-01-11 22:44:36 +010074}
75EXPORT_SYMBOL_GPL(idle_notifier_register);
76
77void idle_notifier_unregister(struct notifier_block *n)
78{
Alan Sterne041c682006-03-27 01:16:30 -080079 atomic_notifier_chain_unregister(&idle_notifier, n);
Andi Kleen95833c82006-01-11 22:44:36 +010080}
81EXPORT_SYMBOL(idle_notifier_unregister);
82
83enum idle_state { CPU_IDLE, CPU_NOT_IDLE };
84static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE;
85
86void enter_idle(void)
87{
88 __get_cpu_var(idle_state) = CPU_IDLE;
Alan Sterne041c682006-03-27 01:16:30 -080089 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
Andi Kleen95833c82006-01-11 22:44:36 +010090}
91
92static void __exit_idle(void)
93{
94 __get_cpu_var(idle_state) = CPU_NOT_IDLE;
Alan Sterne041c682006-03-27 01:16:30 -080095 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
Andi Kleen95833c82006-01-11 22:44:36 +010096}
97
98/* Called from interrupts to signify idle end */
99void exit_idle(void)
100{
101 if (current->pid | read_pda(irqcount))
102 return;
103 __exit_idle();
104}
105
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106/*
107 * We use this if we don't have any better
108 * idle routine..
109 */
Adrian Bunkcdb04522006-03-24 03:15:57 -0800110static void default_idle(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111{
Nick Piggin64c7c8f2005-11-08 21:39:04 -0800112 local_irq_enable();
113
Andi Kleen2d52ede2006-01-11 22:42:42 +0100114 clear_thread_flag(TIF_POLLING_NRFLAG);
115 smp_mb__after_clear_bit();
116 while (!need_resched()) {
117 local_irq_disable();
118 if (!need_resched())
119 safe_halt();
120 else
121 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 }
Andi Kleen2d52ede2006-01-11 22:42:42 +0100123 set_thread_flag(TIF_POLLING_NRFLAG);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124}
125
126/*
127 * On SMP it's slightly faster (but much more power-consuming!)
128 * to poll the ->need_resched flag instead of waiting for the
129 * cross-CPU IPI to arrive. Use this option with caution.
130 */
131static void poll_idle (void)
132{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133 local_irq_enable();
134
Nick Piggin64c7c8f2005-11-08 21:39:04 -0800135 asm volatile(
136 "2:"
137 "testl %0,%1;"
138 "rep; nop;"
139 "je 2b;"
140 : :
141 "i" (_TIF_NEED_RESCHED),
142 "m" (current_thread_info()->flags));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143}
144
145void cpu_idle_wait(void)
146{
147 unsigned int cpu, this_cpu = get_cpu();
148 cpumask_t map;
149
150 set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
151 put_cpu();
152
153 cpus_clear(map);
154 for_each_online_cpu(cpu) {
155 per_cpu(cpu_idle_state, cpu) = 1;
156 cpu_set(cpu, map);
157 }
158
159 __get_cpu_var(cpu_idle_state) = 0;
160
161 wmb();
162 do {
163 ssleep(1);
164 for_each_online_cpu(cpu) {
Andi Kleena88cde12005-11-05 17:25:54 +0100165 if (cpu_isset(cpu, map) &&
166 !per_cpu(cpu_idle_state, cpu))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 cpu_clear(cpu, map);
168 }
169 cpus_and(map, map, cpu_online_map);
170 } while (!cpus_empty(map));
171}
172EXPORT_SYMBOL_GPL(cpu_idle_wait);
173
Ashok Raj76e4f662005-06-25 14:55:00 -0700174#ifdef CONFIG_HOTPLUG_CPU
175DECLARE_PER_CPU(int, cpu_state);
176
177#include <asm/nmi.h>
Shaohua Li1fa744e2006-01-06 00:12:20 -0800178/* We halt the CPU with physical CPU hotplug */
Ashok Raj76e4f662005-06-25 14:55:00 -0700179static inline void play_dead(void)
180{
181 idle_task_exit();
182 wbinvd();
183 mb();
184 /* Ack it */
185 __get_cpu_var(cpu_state) = CPU_DEAD;
186
Shaohua Li1fa744e2006-01-06 00:12:20 -0800187 local_irq_disable();
Ashok Raj76e4f662005-06-25 14:55:00 -0700188 while (1)
Shaohua Li1fa744e2006-01-06 00:12:20 -0800189 halt();
Ashok Raj76e4f662005-06-25 14:55:00 -0700190}
191#else
192static inline void play_dead(void)
193{
194 BUG();
195}
196#endif /* CONFIG_HOTPLUG_CPU */
197
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198/*
199 * The idle thread. There's no useful work to be
200 * done, so just try to conserve power and have a
201 * low exit latency (ie sit in a loop waiting for
202 * somebody to say that they'd like to reschedule)
203 */
204void cpu_idle (void)
205{
Nick Piggin64c7c8f2005-11-08 21:39:04 -0800206 set_thread_flag(TIF_POLLING_NRFLAG);
207
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 /* endless idle loop with no priority at all */
209 while (1) {
210 while (!need_resched()) {
211 void (*idle)(void);
212
213 if (__get_cpu_var(cpu_idle_state))
214 __get_cpu_var(cpu_idle_state) = 0;
215
216 rmb();
217 idle = pm_idle;
218 if (!idle)
219 idle = default_idle;
Ashok Raj76e4f662005-06-25 14:55:00 -0700220 if (cpu_is_offline(smp_processor_id()))
221 play_dead();
Andi Kleen95833c82006-01-11 22:44:36 +0100222 enter_idle();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 idle();
Andi Kleen95833c82006-01-11 22:44:36 +0100224 __exit_idle();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 }
226
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800227 preempt_enable_no_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 schedule();
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800229 preempt_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 }
231}
232
233/*
234 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
235 * which can obviate IPI to trigger checking of need_resched.
236 * We execute MONITOR against need_resched and enter optimized wait state
237 * through MWAIT. Whenever someone changes need_resched, we would be woken
238 * up from MWAIT (without an IPI).
239 */
240static void mwait_idle(void)
241{
242 local_irq_enable();
243
Nick Piggin64c7c8f2005-11-08 21:39:04 -0800244 while (!need_resched()) {
245 __monitor((void *)&current_thread_info()->flags, 0, 0);
246 smp_mb();
247 if (need_resched())
248 break;
249 __mwait(0, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 }
251}
252
Ashok Raje6982c62005-06-25 14:54:58 -0700253void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254{
255 static int printed;
256 if (cpu_has(c, X86_FEATURE_MWAIT)) {
257 /*
258 * Skip, if setup has overridden idle.
259 * One CPU supports mwait => All CPUs supports mwait
260 */
261 if (!pm_idle) {
262 if (!printed) {
263 printk("using mwait in idle threads.\n");
264 printed = 1;
265 }
266 pm_idle = mwait_idle;
267 }
268 }
269}
270
271static int __init idle_setup (char *str)
272{
273 if (!strncmp(str, "poll", 4)) {
274 printk("using polling idle threads.\n");
275 pm_idle = poll_idle;
276 }
277
278 boot_option_idle_override = 1;
279 return 1;
280}
281
282__setup("idle=", idle_setup);
283
284/* Prints also some state that isn't saved in the pt_regs */
285void __show_regs(struct pt_regs * regs)
286{
287 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
288 unsigned int fsindex,gsindex;
289 unsigned int ds,cs,es;
290
291 printk("\n");
292 print_modules();
Andi Kleen9acf23c2005-09-12 18:49:24 +0200293 printk("Pid: %d, comm: %.20s %s %s %.*s\n",
294 current->pid, current->comm, print_tainted(),
295 system_utsname.release,
296 (int)strcspn(system_utsname.version, " "),
297 system_utsname.version);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip);
299 printk_address(regs->rip);
Andi Kleena88cde12005-11-05 17:25:54 +0100300 printk("\nRSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->rsp,
301 regs->eflags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
303 regs->rax, regs->rbx, regs->rcx);
304 printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
305 regs->rdx, regs->rsi, regs->rdi);
306 printk("RBP: %016lx R08: %016lx R09: %016lx\n",
307 regs->rbp, regs->r8, regs->r9);
308 printk("R10: %016lx R11: %016lx R12: %016lx\n",
309 regs->r10, regs->r11, regs->r12);
310 printk("R13: %016lx R14: %016lx R15: %016lx\n",
311 regs->r13, regs->r14, regs->r15);
312
313 asm("movl %%ds,%0" : "=r" (ds));
314 asm("movl %%cs,%0" : "=r" (cs));
315 asm("movl %%es,%0" : "=r" (es));
316 asm("movl %%fs,%0" : "=r" (fsindex));
317 asm("movl %%gs,%0" : "=r" (gsindex));
318
319 rdmsrl(MSR_FS_BASE, fs);
320 rdmsrl(MSR_GS_BASE, gs);
321 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
322
323 asm("movq %%cr0, %0": "=r" (cr0));
324 asm("movq %%cr2, %0": "=r" (cr2));
325 asm("movq %%cr3, %0": "=r" (cr3));
326 asm("movq %%cr4, %0": "=r" (cr4));
327
328 printk("FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
329 fs,fsindex,gs,gsindex,shadowgs);
330 printk("CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0);
331 printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
332}
333
334void show_regs(struct pt_regs *regs)
335{
Zwane Mwaikamboc078d322005-09-06 15:16:16 -0700336 printk("CPU %d:", smp_processor_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 __show_regs(regs);
338 show_trace(&regs->rsp);
339}
340
341/*
342 * Free current thread data structures etc..
343 */
344void exit_thread(void)
345{
346 struct task_struct *me = current;
347 struct thread_struct *t = &me->thread;
Rusty Lynch73649da2005-06-23 00:09:23 -0700348
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 if (me->thread.io_bitmap_ptr) {
350 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
351
352 kfree(t->io_bitmap_ptr);
353 t->io_bitmap_ptr = NULL;
354 /*
355 * Careful, clear this in the TSS too:
356 */
357 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
358 t->io_bitmap_max = 0;
359 put_cpu();
360 }
361}
362
363void flush_thread(void)
364{
365 struct task_struct *tsk = current;
366 struct thread_info *t = current_thread_info();
367
368 if (t->flags & _TIF_ABI_PENDING)
369 t->flags ^= (_TIF_ABI_PENDING | _TIF_IA32);
370
371 tsk->thread.debugreg0 = 0;
372 tsk->thread.debugreg1 = 0;
373 tsk->thread.debugreg2 = 0;
374 tsk->thread.debugreg3 = 0;
375 tsk->thread.debugreg6 = 0;
376 tsk->thread.debugreg7 = 0;
377 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
378 /*
379 * Forget coprocessor state..
380 */
381 clear_fpu(tsk);
382 clear_used_math();
383}
384
385void release_thread(struct task_struct *dead_task)
386{
387 if (dead_task->mm) {
388 if (dead_task->mm->context.size) {
389 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
390 dead_task->comm,
391 dead_task->mm->context.ldt,
392 dead_task->mm->context.size);
393 BUG();
394 }
395 }
396}
397
398static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
399{
400 struct user_desc ud = {
401 .base_addr = addr,
402 .limit = 0xfffff,
403 .seg_32bit = 1,
404 .limit_in_pages = 1,
405 .useable = 1,
406 };
407 struct n_desc_struct *desc = (void *)t->thread.tls_array;
408 desc += tls;
409 desc->a = LDT_entry_a(&ud);
410 desc->b = LDT_entry_b(&ud);
411}
412
413static inline u32 read_32bit_tls(struct task_struct *t, int tls)
414{
415 struct desc_struct *desc = (void *)t->thread.tls_array;
416 desc += tls;
417 return desc->base0 |
418 (((u32)desc->base1) << 16) |
419 (((u32)desc->base2) << 24);
420}
421
422/*
423 * This gets called before we allocate a new thread and copy
424 * the current task into it.
425 */
426void prepare_to_copy(struct task_struct *tsk)
427{
428 unlazy_fpu(tsk);
429}
430
431int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
432 unsigned long unused,
433 struct task_struct * p, struct pt_regs * regs)
434{
435 int err;
436 struct pt_regs * childregs;
437 struct task_struct *me = current;
438
Andi Kleena88cde12005-11-05 17:25:54 +0100439 childregs = ((struct pt_regs *)
Al Viro57eafdc2006-01-12 01:05:39 -0800440 (THREAD_SIZE + task_stack_page(p))) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 *childregs = *regs;
442
443 childregs->rax = 0;
444 childregs->rsp = rsp;
Andi Kleena88cde12005-11-05 17:25:54 +0100445 if (rsp == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 childregs->rsp = (unsigned long)childregs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447
448 p->thread.rsp = (unsigned long) childregs;
449 p->thread.rsp0 = (unsigned long) (childregs+1);
450 p->thread.userrsp = me->thread.userrsp;
451
Al Viroe4f17c42006-01-12 01:05:38 -0800452 set_tsk_thread_flag(p, TIF_FORK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453
454 p->thread.fs = me->thread.fs;
455 p->thread.gs = me->thread.gs;
456
H. J. Lufd51f662005-05-01 08:58:48 -0700457 asm("mov %%gs,%0" : "=m" (p->thread.gsindex));
458 asm("mov %%fs,%0" : "=m" (p->thread.fsindex));
459 asm("mov %%es,%0" : "=m" (p->thread.es));
460 asm("mov %%ds,%0" : "=m" (p->thread.ds));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461
462 if (unlikely(me->thread.io_bitmap_ptr != NULL)) {
463 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
464 if (!p->thread.io_bitmap_ptr) {
465 p->thread.io_bitmap_max = 0;
466 return -ENOMEM;
467 }
Andi Kleena88cde12005-11-05 17:25:54 +0100468 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
469 IO_BITMAP_BYTES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 }
471
472 /*
473 * Set a new TLS for the child thread?
474 */
475 if (clone_flags & CLONE_SETTLS) {
476#ifdef CONFIG_IA32_EMULATION
477 if (test_thread_flag(TIF_IA32))
478 err = ia32_child_tls(p, childregs);
479 else
480#endif
481 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
482 if (err)
483 goto out;
484 }
485 err = 0;
486out:
487 if (err && p->thread.io_bitmap_ptr) {
488 kfree(p->thread.io_bitmap_ptr);
489 p->thread.io_bitmap_max = 0;
490 }
491 return err;
492}
493
494/*
495 * This special macro can be used to load a debugging register
496 */
Jan Beulich2b514e72006-03-25 16:29:22 +0100497#define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
499/*
500 * switch_to(x,y) should switch tasks from x to y.
501 *
502 * This could still be optimized:
503 * - fold all the options into a flag word and test it with a single test.
504 * - could test fs/gs bitsliced
Andi Kleen099f3182006-02-03 21:51:38 +0100505 *
506 * Kprobes not supported here. Set the probe on schedule instead.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 */
Andi Kleen099f3182006-02-03 21:51:38 +0100508__kprobes struct task_struct *
Andi Kleena88cde12005-11-05 17:25:54 +0100509__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510{
511 struct thread_struct *prev = &prev_p->thread,
512 *next = &next_p->thread;
513 int cpu = smp_processor_id();
514 struct tss_struct *tss = &per_cpu(init_tss, cpu);
515
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 /*
517 * Reload esp0, LDT and the page table pointer:
518 */
519 tss->rsp0 = next->rsp0;
520
521 /*
522 * Switch DS and ES.
523 * This won't pick up thread selector changes, but I guess that is ok.
524 */
H. J. Lufd51f662005-05-01 08:58:48 -0700525 asm volatile("mov %%es,%0" : "=m" (prev->es));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 if (unlikely(next->es | prev->es))
527 loadsegment(es, next->es);
528
H. J. Lufd51f662005-05-01 08:58:48 -0700529 asm volatile ("mov %%ds,%0" : "=m" (prev->ds));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 if (unlikely(next->ds | prev->ds))
531 loadsegment(ds, next->ds);
532
533 load_TLS(next, cpu);
534
535 /*
536 * Switch FS and GS.
537 */
538 {
539 unsigned fsindex;
540 asm volatile("movl %%fs,%0" : "=r" (fsindex));
541 /* segment register != 0 always requires a reload.
542 also reload when it has changed.
543 when prev process used 64bit base always reload
544 to avoid an information leak. */
545 if (unlikely(fsindex | next->fsindex | prev->fs)) {
546 loadsegment(fs, next->fsindex);
547 /* check if the user used a selector != 0
548 * if yes clear 64bit base, since overloaded base
549 * is always mapped to the Null selector
550 */
551 if (fsindex)
552 prev->fs = 0;
553 }
554 /* when next process has a 64bit base use it */
555 if (next->fs)
556 wrmsrl(MSR_FS_BASE, next->fs);
557 prev->fsindex = fsindex;
558 }
559 {
560 unsigned gsindex;
561 asm volatile("movl %%gs,%0" : "=r" (gsindex));
562 if (unlikely(gsindex | next->gsindex | prev->gs)) {
563 load_gs_index(next->gsindex);
564 if (gsindex)
565 prev->gs = 0;
566 }
567 if (next->gs)
568 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
569 prev->gsindex = gsindex;
570 }
571
572 /*
Jan Beulich45948d72006-03-25 16:29:25 +0100573 * Switch the PDA and FPU contexts.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 */
575 prev->userrsp = read_pda(oldrsp);
576 write_pda(oldrsp, next->userrsp);
577 write_pda(pcurrent, next_p);
Andi Kleen18bd0572006-04-20 02:36:45 +0200578
Jan Beulich45948d72006-03-25 16:29:25 +0100579 /* This must be here to ensure both math_state_restore() and
Andi Kleen18bd0572006-04-20 02:36:45 +0200580 kernel_fpu_begin() work consistently.
581 And the AMD workaround requires it to be after DS reload. */
Jan Beulich45948d72006-03-25 16:29:25 +0100582 unlazy_fpu(prev_p);
Andi Kleena88cde12005-11-05 17:25:54 +0100583 write_pda(kernelstack,
Al Viro57eafdc2006-01-12 01:05:39 -0800584 task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
586 /*
587 * Now maybe reload the debug registers
588 */
589 if (unlikely(next->debugreg7)) {
590 loaddebug(next, 0);
591 loaddebug(next, 1);
592 loaddebug(next, 2);
593 loaddebug(next, 3);
594 /* no 4 and 5 */
595 loaddebug(next, 6);
596 loaddebug(next, 7);
597 }
598
599
600 /*
601 * Handle the IO bitmap
602 */
603 if (unlikely(prev->io_bitmap_ptr || next->io_bitmap_ptr)) {
604 if (next->io_bitmap_ptr)
605 /*
606 * Copy the relevant range of the IO bitmap.
607 * Normally this is 128 bytes or less:
608 */
609 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
610 max(prev->io_bitmap_max, next->io_bitmap_max));
611 else {
612 /*
613 * Clear any possible leftover bits:
614 */
615 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
616 }
617 }
618
619 return prev_p;
620}
621
622/*
623 * sys_execve() executes a new program.
624 */
625asmlinkage
626long sys_execve(char __user *name, char __user * __user *argv,
627 char __user * __user *envp, struct pt_regs regs)
628{
629 long error;
630 char * filename;
631
632 filename = getname(name);
633 error = PTR_ERR(filename);
634 if (IS_ERR(filename))
635 return error;
636 error = do_execve(filename, argv, envp, &regs);
637 if (error == 0) {
638 task_lock(current);
639 current->ptrace &= ~PT_DTRACE;
640 task_unlock(current);
641 }
642 putname(filename);
643 return error;
644}
645
646void set_personality_64bit(void)
647{
648 /* inherit personality from parent */
649
650 /* Make sure to be in 64bit mode */
651 clear_thread_flag(TIF_IA32);
652
653 /* TBD: overwrites user setup. Should have two bits.
654 But 64bit processes have always behaved this way,
655 so it's not too bad. The main problem is just that
656 32bit childs are affected again. */
657 current->personality &= ~READ_IMPLIES_EXEC;
658}
659
660asmlinkage long sys_fork(struct pt_regs *regs)
661{
662 return do_fork(SIGCHLD, regs->rsp, regs, 0, NULL, NULL);
663}
664
Andi Kleena88cde12005-11-05 17:25:54 +0100665asmlinkage long
666sys_clone(unsigned long clone_flags, unsigned long newsp,
667 void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668{
669 if (!newsp)
670 newsp = regs->rsp;
671 return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
672}
673
674/*
675 * This is trivial, and on the face of it looks like it
676 * could equally well be done in user mode.
677 *
678 * Not so, for quite unobvious reasons - register pressure.
679 * In user mode vfork() cannot have a stack frame, and if
680 * done by calling the "clone()" system call directly, you
681 * do not have enough call-clobbered registers to hold all
682 * the information you need.
683 */
684asmlinkage long sys_vfork(struct pt_regs *regs)
685{
686 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->rsp, regs, 0,
687 NULL, NULL);
688}
689
690unsigned long get_wchan(struct task_struct *p)
691{
692 unsigned long stack;
693 u64 fp,rip;
694 int count = 0;
695
696 if (!p || p == current || p->state==TASK_RUNNING)
697 return 0;
Al Viro57eafdc2006-01-12 01:05:39 -0800698 stack = (unsigned long)task_stack_page(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 if (p->thread.rsp < stack || p->thread.rsp > stack+THREAD_SIZE)
700 return 0;
701 fp = *(u64 *)(p->thread.rsp);
702 do {
Andi Kleena88cde12005-11-05 17:25:54 +0100703 if (fp < (unsigned long)stack ||
704 fp > (unsigned long)stack+THREAD_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 return 0;
706 rip = *(u64 *)(fp+8);
707 if (!in_sched_functions(rip))
708 return rip;
709 fp = *(u64 *)fp;
710 } while (count++ < 16);
711 return 0;
712}
713
714long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
715{
716 int ret = 0;
717 int doit = task == current;
718 int cpu;
719
720 switch (code) {
721 case ARCH_SET_GS:
Suresh Siddha84929802005-06-21 17:14:32 -0700722 if (addr >= TASK_SIZE_OF(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 return -EPERM;
724 cpu = get_cpu();
725 /* handle small bases via the GDT because that's faster to
726 switch. */
727 if (addr <= 0xffffffff) {
728 set_32bit_tls(task, GS_TLS, addr);
729 if (doit) {
730 load_TLS(&task->thread, cpu);
731 load_gs_index(GS_TLS_SEL);
732 }
733 task->thread.gsindex = GS_TLS_SEL;
734 task->thread.gs = 0;
735 } else {
736 task->thread.gsindex = 0;
737 task->thread.gs = addr;
738 if (doit) {
Andi Kleena88cde12005-11-05 17:25:54 +0100739 load_gs_index(0);
740 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 }
742 }
743 put_cpu();
744 break;
745 case ARCH_SET_FS:
746 /* Not strictly needed for fs, but do it for symmetry
747 with gs */
Suresh Siddha84929802005-06-21 17:14:32 -0700748 if (addr >= TASK_SIZE_OF(task))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 return -EPERM;
750 cpu = get_cpu();
751 /* handle small bases via the GDT because that's faster to
752 switch. */
753 if (addr <= 0xffffffff) {
754 set_32bit_tls(task, FS_TLS, addr);
755 if (doit) {
756 load_TLS(&task->thread, cpu);
Andi Kleena88cde12005-11-05 17:25:54 +0100757 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758 }
759 task->thread.fsindex = FS_TLS_SEL;
760 task->thread.fs = 0;
761 } else {
762 task->thread.fsindex = 0;
763 task->thread.fs = addr;
764 if (doit) {
765 /* set the selector to 0 to not confuse
766 __switch_to */
Andi Kleena88cde12005-11-05 17:25:54 +0100767 asm volatile("movl %0,%%fs" :: "r" (0));
768 ret = checking_wrmsrl(MSR_FS_BASE, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 }
770 }
771 put_cpu();
772 break;
773 case ARCH_GET_FS: {
774 unsigned long base;
775 if (task->thread.fsindex == FS_TLS_SEL)
776 base = read_32bit_tls(task, FS_TLS);
Andi Kleena88cde12005-11-05 17:25:54 +0100777 else if (doit)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 rdmsrl(MSR_FS_BASE, base);
Andi Kleena88cde12005-11-05 17:25:54 +0100779 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 base = task->thread.fs;
781 ret = put_user(base, (unsigned long __user *)addr);
782 break;
783 }
784 case ARCH_GET_GS: {
785 unsigned long base;
John Blackwood97c28032006-04-07 19:50:25 +0200786 unsigned gsindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787 if (task->thread.gsindex == GS_TLS_SEL)
788 base = read_32bit_tls(task, GS_TLS);
John Blackwood97c28032006-04-07 19:50:25 +0200789 else if (doit) {
790 asm("movl %%gs,%0" : "=r" (gsindex));
791 if (gsindex)
792 rdmsrl(MSR_KERNEL_GS_BASE, base);
793 else
794 base = task->thread.gs;
795 }
Andi Kleena88cde12005-11-05 17:25:54 +0100796 else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 base = task->thread.gs;
798 ret = put_user(base, (unsigned long __user *)addr);
799 break;
800 }
801
802 default:
803 ret = -EINVAL;
804 break;
805 }
806
807 return ret;
808}
809
810long sys_arch_prctl(int code, unsigned long addr)
811{
812 return do_arch_prctl(current, code, addr);
813}
814
815/*
816 * Capture the user space registers if the task is not running (in user space)
817 */
818int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
819{
820 struct pt_regs *pp, ptregs;
821
Al Virobb049232006-01-12 01:05:38 -0800822 pp = task_pt_regs(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823
824 ptregs = *pp;
825 ptregs.cs &= 0xffff;
826 ptregs.ss &= 0xffff;
827
828 elf_core_copy_regs(regs, &ptregs);
829
830 return 1;
831}
832
833unsigned long arch_align_stack(unsigned long sp)
834{
835 if (randomize_va_space)
836 sp -= get_random_int() % 8192;
837 return sp & ~0xf;
838}