blob: 42e9d908914a0e47117cfe7f3dcc5fe50d215d98 [file] [log] [blame]
Paul Mackerras9994a332005-10-10 22:36:14 +10001/*
Paul Mackerras9994a332005-10-10 22:36:14 +10002 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
20
Paul Mackerras9994a332005-10-10 22:36:14 +100021#include <linux/errno.h>
22#include <asm/unistd.h>
23#include <asm/processor.h>
24#include <asm/page.h>
25#include <asm/mmu.h>
26#include <asm/thread_info.h>
27#include <asm/ppc_asm.h>
28#include <asm/asm-offsets.h>
29#include <asm/cputable.h>
Stephen Rothwell3f639ee2006-09-25 18:19:00 +100030#include <asm/firmware.h>
David Woodhouse007d88d2007-01-01 18:45:34 +000031#include <asm/bug.h>
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +100032#include <asm/ptrace.h>
Benjamin Herrenschmidt945feb12008-04-17 14:35:01 +100033#include <asm/irqflags.h>
Abhishek Sagar395a59d2008-06-21 23:47:27 +053034#include <asm/ftrace.h>
Paul Mackerras9994a332005-10-10 22:36:14 +100035
36/*
37 * System calls.
38 */
39 .section ".toc","aw"
40.SYS_CALL_TABLE:
41 .tc .sys_call_table[TC],.sys_call_table
42
43/* This value is used to mark exception frames on the stack. */
44exception_marker:
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +100045 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
Paul Mackerras9994a332005-10-10 22:36:14 +100046
47 .section ".text"
48 .align 7
49
50#undef SHOW_SYSCALLS
51
52 .globl system_call_common
53system_call_common:
54 andi. r10,r12,MSR_PR
55 mr r10,r1
56 addi r1,r1,-INT_FRAME_SIZE
57 beq- 1f
58 ld r1,PACAKSAVE(r13)
591: std r10,0(r1)
60 std r11,_NIP(r1)
61 std r12,_MSR(r1)
62 std r0,GPR0(r1)
63 std r10,GPR1(r1)
Paul Mackerrasc6622f62006-02-24 10:06:59 +110064 ACCOUNT_CPU_USER_ENTRY(r10, r11)
Paul Mackerrasab598b62008-11-30 11:49:45 +000065 /*
66 * This "crclr so" clears CR0.SO, which is the error indication on
67 * return from this system call. There must be no cmp instruction
68 * between it and the "mfcr r9" below, otherwise if XER.SO is set,
69 * CR0.SO will get set, causing all system calls to appear to fail.
70 */
71 crclr so
Paul Mackerras9994a332005-10-10 22:36:14 +100072 std r2,GPR2(r1)
73 std r3,GPR3(r1)
74 std r4,GPR4(r1)
75 std r5,GPR5(r1)
76 std r6,GPR6(r1)
77 std r7,GPR7(r1)
78 std r8,GPR8(r1)
79 li r11,0
80 std r11,GPR9(r1)
81 std r11,GPR10(r1)
82 std r11,GPR11(r1)
83 std r11,GPR12(r1)
84 std r9,GPR13(r1)
Paul Mackerras9994a332005-10-10 22:36:14 +100085 mfcr r9
86 mflr r10
87 li r11,0xc01
88 std r9,_CCR(r1)
89 std r10,_LINK(r1)
90 std r11,_TRAP(r1)
91 mfxer r9
92 mfctr r10
93 std r9,_XER(r1)
94 std r10,_CTR(r1)
95 std r3,ORIG_GPR3(r1)
96 ld r2,PACATOC(r13)
97 addi r9,r1,STACK_FRAME_OVERHEAD
98 ld r11,exception_marker@toc(r2)
99 std r11,-16(r9) /* "regshere" marker */
Benjamin Herrenschmidt945feb12008-04-17 14:35:01 +1000100#ifdef CONFIG_TRACE_IRQFLAGS
101 bl .trace_hardirqs_on
102 REST_GPR(0,r1)
103 REST_4GPRS(3,r1)
104 REST_2GPRS(7,r1)
105 addi r9,r1,STACK_FRAME_OVERHEAD
106 ld r12,_MSR(r1)
107#endif /* CONFIG_TRACE_IRQFLAGS */
Paul Mackerrasd04c56f2006-10-04 16:47:49 +1000108 li r10,1
109 stb r10,PACASOFTIRQEN(r13)
110 stb r10,PACAHARDIRQEN(r13)
111 std r10,SOFTE(r1)
Paul Mackerras9994a332005-10-10 22:36:14 +1000112#ifdef CONFIG_PPC_ISERIES
Stephen Rothwell3f639ee2006-09-25 18:19:00 +1000113BEGIN_FW_FTR_SECTION
Paul Mackerras9994a332005-10-10 22:36:14 +1000114 /* Hack for handling interrupts when soft-enabling on iSeries */
115 cmpdi cr1,r0,0x5555 /* syscall 0x5555 */
116 andi. r10,r12,MSR_PR /* from kernel */
117 crand 4*cr0+eq,4*cr1+eq,4*cr0+eq
Stephen Rothwellc7056772006-11-27 14:59:50 +1100118 bne 2f
119 b hardware_interrupt_entry
1202:
Stephen Rothwell3f639ee2006-09-25 18:19:00 +1000121END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
Benjamin Herrenschmidt945feb12008-04-17 14:35:01 +1000122#endif /* CONFIG_PPC_ISERIES */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000123
124 /* Hard enable interrupts */
125#ifdef CONFIG_PPC_BOOK3E
126 wrteei 1
127#else
Paul Mackerras9994a332005-10-10 22:36:14 +1000128 mfmsr r11
129 ori r11,r11,MSR_EE
130 mtmsrd r11,1
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000131#endif /* CONFIG_PPC_BOOK3E */
Paul Mackerras9994a332005-10-10 22:36:14 +1000132
133#ifdef SHOW_SYSCALLS
134 bl .do_show_syscall
135 REST_GPR(0,r1)
136 REST_4GPRS(3,r1)
137 REST_2GPRS(7,r1)
138 addi r9,r1,STACK_FRAME_OVERHEAD
139#endif
140 clrrdi r11,r1,THREAD_SHIFT
Paul Mackerras9994a332005-10-10 22:36:14 +1000141 ld r10,TI_FLAGS(r11)
Paul Mackerras9994a332005-10-10 22:36:14 +1000142 andi. r11,r10,_TIF_SYSCALL_T_OR_A
143 bne- syscall_dotrace
144syscall_dotrace_cont:
145 cmpldi 0,r0,NR_syscalls
146 bge- syscall_enosys
147
148system_call: /* label this so stack traces look sane */
149/*
150 * Need to vector to 32 Bit or default sys_call_table here,
151 * based on caller's run-mode / personality.
152 */
153 ld r11,.SYS_CALL_TABLE@toc(2)
154 andi. r10,r10,_TIF_32BIT
155 beq 15f
156 addi r11,r11,8 /* use 32-bit syscall entries */
157 clrldi r3,r3,32
158 clrldi r4,r4,32
159 clrldi r5,r5,32
160 clrldi r6,r6,32
161 clrldi r7,r7,32
162 clrldi r8,r8,32
16315:
164 slwi r0,r0,4
165 ldx r10,r11,r0 /* Fetch system call handler [ptr] */
166 mtctr r10
167 bctrl /* Call handler */
168
169syscall_exit:
Paul Mackerras9994a332005-10-10 22:36:14 +1000170 std r3,RESULT(r1)
David Woodhouse401d1f02005-11-15 18:52:18 +0000171#ifdef SHOW_SYSCALLS
172 bl .do_show_syscall_exit
173 ld r3,RESULT(r1)
174#endif
Paul Mackerras9994a332005-10-10 22:36:14 +1000175 clrrdi r12,r1,THREAD_SHIFT
Paul Mackerras9994a332005-10-10 22:36:14 +1000176
Paul Mackerras9994a332005-10-10 22:36:14 +1000177 ld r8,_MSR(r1)
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000178#ifdef CONFIG_PPC_BOOK3S
179 /* No MSR:RI on BookE */
Paul Mackerras9994a332005-10-10 22:36:14 +1000180 andi. r10,r8,MSR_RI
181 beq- unrecov_restore
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000182#endif
183
184 /* Disable interrupts so current_thread_info()->flags can't change,
185 * and so that we don't get interrupted after loading SRR0/1.
186 */
187#ifdef CONFIG_PPC_BOOK3E
188 wrteei 0
189#else
Paul Mackerras9994a332005-10-10 22:36:14 +1000190 mfmsr r10
191 rldicl r10,r10,48,1
192 rotldi r10,r10,16
193 mtmsrd r10,1
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000194#endif /* CONFIG_PPC_BOOK3E */
195
Paul Mackerras9994a332005-10-10 22:36:14 +1000196 ld r9,TI_FLAGS(r12)
David Woodhouse401d1f02005-11-15 18:52:18 +0000197 li r11,-_LAST_ERRNO
Paul Mackerras1bd79332006-03-08 13:24:22 +1100198 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
Paul Mackerras9994a332005-10-10 22:36:14 +1000199 bne- syscall_exit_work
David Woodhouse401d1f02005-11-15 18:52:18 +0000200 cmpld r3,r11
201 ld r5,_CCR(r1)
202 bge- syscall_error
203syscall_error_cont:
Paul Mackerras9994a332005-10-10 22:36:14 +1000204 ld r7,_NIP(r1)
205 stdcx. r0,0,r1 /* to clear the reservation */
206 andi. r6,r8,MSR_PR
207 ld r4,_LINK(r1)
Paul Mackerrase56a6e22007-02-07 13:13:26 +1100208 /*
209 * Clear RI before restoring r13. If we are returning to
210 * userspace and we take an exception after restoring r13,
211 * we end up corrupting the userspace r13 value.
212 */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000213#ifdef CONFIG_PPC_BOOK3S
214 /* No MSR:RI on BookE */
Paul Mackerrase56a6e22007-02-07 13:13:26 +1100215 li r12,MSR_RI
216 andc r11,r10,r12
217 mtmsrd r11,1 /* clear MSR.RI */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000218#endif /* CONFIG_PPC_BOOK3S */
219
Paul Mackerrasc6622f62006-02-24 10:06:59 +1100220 beq- 1f
221 ACCOUNT_CPU_USER_EXIT(r11, r12)
222 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
Paul Mackerras9994a332005-10-10 22:36:14 +10002231: ld r2,GPR2(r1)
Paul Mackerras9994a332005-10-10 22:36:14 +1000224 ld r1,GPR1(r1)
225 mtlr r4
226 mtcr r5
227 mtspr SPRN_SRR0,r7
228 mtspr SPRN_SRR1,r8
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000229 RFI
Paul Mackerras9994a332005-10-10 22:36:14 +1000230 b . /* prevent speculative execution */
231
David Woodhouse401d1f02005-11-15 18:52:18 +0000232syscall_error:
Paul Mackerras9994a332005-10-10 22:36:14 +1000233 oris r5,r5,0x1000 /* Set SO bit in CR */
David Woodhouse401d1f02005-11-15 18:52:18 +0000234 neg r3,r3
Paul Mackerras9994a332005-10-10 22:36:14 +1000235 std r5,_CCR(r1)
236 b syscall_error_cont
David Woodhouse401d1f02005-11-15 18:52:18 +0000237
Paul Mackerras9994a332005-10-10 22:36:14 +1000238/* Traced system call support */
239syscall_dotrace:
240 bl .save_nvgprs
241 addi r3,r1,STACK_FRAME_OVERHEAD
242 bl .do_syscall_trace_enter
Roland McGrath4f72c422008-07-27 16:51:03 +1000243 /*
244 * Restore argument registers possibly just changed.
245 * We use the return value of do_syscall_trace_enter
246 * for the call number to look up in the table (r0).
247 */
248 mr r0,r3
Paul Mackerras9994a332005-10-10 22:36:14 +1000249 ld r3,GPR3(r1)
250 ld r4,GPR4(r1)
251 ld r5,GPR5(r1)
252 ld r6,GPR6(r1)
253 ld r7,GPR7(r1)
254 ld r8,GPR8(r1)
255 addi r9,r1,STACK_FRAME_OVERHEAD
256 clrrdi r10,r1,THREAD_SHIFT
257 ld r10,TI_FLAGS(r10)
258 b syscall_dotrace_cont
259
David Woodhouse401d1f02005-11-15 18:52:18 +0000260syscall_enosys:
261 li r3,-ENOSYS
262 b syscall_exit
263
264syscall_exit_work:
265 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
266 If TIF_NOERROR is set, just save r3 as it is. */
267
268 andi. r0,r9,_TIF_RESTOREALL
Paul Mackerras1bd79332006-03-08 13:24:22 +1100269 beq+ 0f
270 REST_NVGPRS(r1)
271 b 2f
2720: cmpld r3,r11 /* r10 is -LAST_ERRNO */
David Woodhouse401d1f02005-11-15 18:52:18 +0000273 blt+ 1f
274 andi. r0,r9,_TIF_NOERROR
275 bne- 1f
276 ld r5,_CCR(r1)
277 neg r3,r3
278 oris r5,r5,0x1000 /* Set SO bit in CR */
279 std r5,_CCR(r1)
2801: std r3,GPR3(r1)
2812: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
282 beq 4f
283
Paul Mackerras1bd79332006-03-08 13:24:22 +1100284 /* Clear per-syscall TIF flags if any are set. */
David Woodhouse401d1f02005-11-15 18:52:18 +0000285
286 li r11,_TIF_PERSYSCALL_MASK
287 addi r12,r12,TI_FLAGS
2883: ldarx r10,0,r12
289 andc r10,r10,r11
290 stdcx. r10,0,r12
291 bne- 3b
292 subi r12,r12,TI_FLAGS
Paul Mackerras1bd79332006-03-08 13:24:22 +1100293
2944: /* Anything else left to do? */
295 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
David Woodhouse401d1f02005-11-15 18:52:18 +0000296 beq .ret_from_except_lite
297
298 /* Re-enable interrupts */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000299#ifdef CONFIG_PPC_BOOK3E
300 wrteei 1
301#else
David Woodhouse401d1f02005-11-15 18:52:18 +0000302 mfmsr r10
303 ori r10,r10,MSR_EE
304 mtmsrd r10,1
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000305#endif /* CONFIG_PPC_BOOK3E */
David Woodhouse401d1f02005-11-15 18:52:18 +0000306
Paul Mackerras1bd79332006-03-08 13:24:22 +1100307 bl .save_nvgprs
Paul Mackerras9994a332005-10-10 22:36:14 +1000308 addi r3,r1,STACK_FRAME_OVERHEAD
309 bl .do_syscall_trace_leave
Paul Mackerras1bd79332006-03-08 13:24:22 +1100310 b .ret_from_except
Paul Mackerras9994a332005-10-10 22:36:14 +1000311
312/* Save non-volatile GPRs, if not already saved. */
313_GLOBAL(save_nvgprs)
314 ld r11,_TRAP(r1)
315 andi. r0,r11,1
316 beqlr-
317 SAVE_NVGPRS(r1)
318 clrrdi r0,r11,1
319 std r0,_TRAP(r1)
320 blr
321
David Woodhouse401d1f02005-11-15 18:52:18 +0000322
Paul Mackerras9994a332005-10-10 22:36:14 +1000323/*
324 * The sigsuspend and rt_sigsuspend system calls can call do_signal
325 * and thus put the process into the stopped state where we might
326 * want to examine its user state with ptrace. Therefore we need
327 * to save all the nonvolatile registers (r14 - r31) before calling
328 * the C code. Similarly, fork, vfork and clone need the full
329 * register state on the stack so that it can be copied to the child.
330 */
Paul Mackerras9994a332005-10-10 22:36:14 +1000331
332_GLOBAL(ppc_fork)
333 bl .save_nvgprs
334 bl .sys_fork
335 b syscall_exit
336
337_GLOBAL(ppc_vfork)
338 bl .save_nvgprs
339 bl .sys_vfork
340 b syscall_exit
341
342_GLOBAL(ppc_clone)
343 bl .save_nvgprs
344 bl .sys_clone
345 b syscall_exit
346
Paul Mackerras1bd79332006-03-08 13:24:22 +1100347_GLOBAL(ppc32_swapcontext)
348 bl .save_nvgprs
349 bl .compat_sys_swapcontext
350 b syscall_exit
351
352_GLOBAL(ppc64_swapcontext)
353 bl .save_nvgprs
354 bl .sys_swapcontext
355 b syscall_exit
356
Paul Mackerras9994a332005-10-10 22:36:14 +1000357_GLOBAL(ret_from_fork)
358 bl .schedule_tail
359 REST_NVGPRS(r1)
360 li r3,0
361 b syscall_exit
362
363/*
364 * This routine switches between two different tasks. The process
365 * state of one is saved on its kernel stack. Then the state
366 * of the other is restored from its kernel stack. The memory
367 * management hardware is updated to the second process's state.
368 * Finally, we can return to the second process, via ret_from_except.
369 * On entry, r3 points to the THREAD for the current task, r4
370 * points to the THREAD for the new task.
371 *
372 * Note: there are two ways to get to the "going out" portion
373 * of this code; either by coming in via the entry (_switch)
374 * or via "fork" which must set up an environment equivalent
375 * to the "_switch" path. If you change this you'll have to change
376 * the fork code also.
377 *
378 * The code which creates the new task context is in 'copy_thread'
Jon Mason2ef94812006-01-23 10:58:20 -0600379 * in arch/powerpc/kernel/process.c
Paul Mackerras9994a332005-10-10 22:36:14 +1000380 */
381 .align 7
382_GLOBAL(_switch)
383 mflr r0
384 std r0,16(r1)
385 stdu r1,-SWITCH_FRAME_SIZE(r1)
386 /* r3-r13 are caller saved -- Cort */
387 SAVE_8GPRS(14, r1)
388 SAVE_10GPRS(22, r1)
389 mflr r20 /* Return to switch caller */
390 mfmsr r22
391 li r0, MSR_FP
Michael Neulingce48b212008-06-25 14:07:18 +1000392#ifdef CONFIG_VSX
393BEGIN_FTR_SECTION
394 oris r0,r0,MSR_VSX@h /* Disable VSX */
395END_FTR_SECTION_IFSET(CPU_FTR_VSX)
396#endif /* CONFIG_VSX */
Paul Mackerras9994a332005-10-10 22:36:14 +1000397#ifdef CONFIG_ALTIVEC
398BEGIN_FTR_SECTION
399 oris r0,r0,MSR_VEC@h /* Disable altivec */
400 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
401 std r24,THREAD_VRSAVE(r3)
402END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
403#endif /* CONFIG_ALTIVEC */
404 and. r0,r0,r22
405 beq+ 1f
406 andc r22,r22,r0
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000407 MTMSRD(r22)
Paul Mackerras9994a332005-10-10 22:36:14 +1000408 isync
4091: std r20,_NIP(r1)
410 mfcr r23
411 std r23,_CCR(r1)
412 std r1,KSP(r3) /* Set old stack pointer */
413
414#ifdef CONFIG_SMP
415 /* We need a sync somewhere here to make sure that if the
416 * previous task gets rescheduled on another CPU, it sees all
417 * stores it has performed on this one.
418 */
419 sync
420#endif /* CONFIG_SMP */
421
422 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
423 std r6,PACACURRENT(r13) /* Set new 'current' */
424
425 ld r8,KSP(r4) /* new stack pointer */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000426#ifdef CONFIG_PPC_BOOK3S
Paul Mackerras9994a332005-10-10 22:36:14 +1000427BEGIN_FTR_SECTION
Michael Ellermanc2303282008-06-24 11:33:05 +1000428 BEGIN_FTR_SECTION_NESTED(95)
Paul Mackerras9994a332005-10-10 22:36:14 +1000429 clrrdi r6,r8,28 /* get its ESID */
430 clrrdi r9,r1,28 /* get current sp ESID */
Michael Ellermanc2303282008-06-24 11:33:05 +1000431 FTR_SECTION_ELSE_NESTED(95)
Paul Mackerras1189be62007-10-11 20:37:10 +1000432 clrrdi r6,r8,40 /* get its 1T ESID */
433 clrrdi r9,r1,40 /* get current sp 1T ESID */
Michael Ellermanc2303282008-06-24 11:33:05 +1000434 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_1T_SEGMENT, 95)
435FTR_SECTION_ELSE
436 b 2f
437ALT_FTR_SECTION_END_IFSET(CPU_FTR_SLB)
Paul Mackerras9994a332005-10-10 22:36:14 +1000438 clrldi. r0,r6,2 /* is new ESID c00000000? */
439 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
440 cror eq,4*cr1+eq,eq
441 beq 2f /* if yes, don't slbie it */
442
443 /* Bolt in the new stack SLB entry */
444 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
445 oris r0,r6,(SLB_ESID_V)@h
446 ori r0,r0,(SLB_NUM_BOLTED-1)@l
Paul Mackerras1189be62007-10-11 20:37:10 +1000447BEGIN_FTR_SECTION
448 li r9,MMU_SEGSIZE_1T /* insert B field */
449 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
450 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
451END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
Michael Neuling2f6093c2006-08-07 16:19:19 +1000452
Michael Neuling00efee72007-08-24 16:58:37 +1000453 /* Update the last bolted SLB. No write barriers are needed
454 * here, provided we only update the current CPU's SLB shadow
455 * buffer.
456 */
Michael Neuling2f6093c2006-08-07 16:19:19 +1000457 ld r9,PACA_SLBSHADOWPTR(r13)
Michael Neuling11a27ad2006-08-09 17:00:30 +1000458 li r12,0
459 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
460 std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
461 std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
Michael Neuling2f6093c2006-08-07 16:19:19 +1000462
Olof Johanssonf66bce52007-10-16 00:58:59 +1000463 /* No need to check for CPU_FTR_NO_SLBIE_B here, since when
464 * we have 1TB segments, the only CPUs known to have the errata
465 * only support less than 1TB of system memory and we'll never
466 * actually hit this code path.
467 */
468
Paul Mackerras9994a332005-10-10 22:36:14 +1000469 slbie r6
470 slbie r6 /* Workaround POWER5 < DD2.1 issue */
471 slbmte r7,r0
472 isync
Paul Mackerras9994a332005-10-10 22:36:14 +10004732:
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000474#endif /* !CONFIG_PPC_BOOK3S */
475
Paul Mackerras9994a332005-10-10 22:36:14 +1000476 clrrdi r7,r8,THREAD_SHIFT /* base of new stack */
477 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
478 because we don't need to leave the 288-byte ABI gap at the
479 top of the kernel stack. */
480 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
481
482 mr r1,r8 /* start using new stack pointer */
483 std r7,PACAKSAVE(r13)
484
485 ld r6,_CCR(r1)
486 mtcrf 0xFF,r6
487
488#ifdef CONFIG_ALTIVEC
489BEGIN_FTR_SECTION
490 ld r0,THREAD_VRSAVE(r4)
491 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
492END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
493#endif /* CONFIG_ALTIVEC */
494
495 /* r3-r13 are destroyed -- Cort */
496 REST_8GPRS(14, r1)
497 REST_10GPRS(22, r1)
498
499 /* convert old thread to its task_struct for return value */
500 addi r3,r3,-THREAD
501 ld r7,_NIP(r1) /* Return to _switch caller in new task */
502 mtlr r7
503 addi r1,r1,SWITCH_FRAME_SIZE
504 blr
505
506 .align 7
507_GLOBAL(ret_from_except)
508 ld r11,_TRAP(r1)
509 andi. r0,r11,1
510 bne .ret_from_except_lite
511 REST_NVGPRS(r1)
512
513_GLOBAL(ret_from_except_lite)
514 /*
515 * Disable interrupts so that current_thread_info()->flags
516 * can't change between when we test it and when we return
517 * from the interrupt.
518 */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000519#ifdef CONFIG_PPC_BOOK3E
520 wrteei 0
521#else
Paul Mackerras9994a332005-10-10 22:36:14 +1000522 mfmsr r10 /* Get current interrupt state */
523 rldicl r9,r10,48,1 /* clear MSR_EE */
524 rotldi r9,r9,16
525 mtmsrd r9,1 /* Update machine state */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000526#endif /* CONFIG_PPC_BOOK3E */
Paul Mackerras9994a332005-10-10 22:36:14 +1000527
528#ifdef CONFIG_PREEMPT
529 clrrdi r9,r1,THREAD_SHIFT /* current_thread_info() */
530 li r0,_TIF_NEED_RESCHED /* bits to check */
531 ld r3,_MSR(r1)
532 ld r4,TI_FLAGS(r9)
533 /* Move MSR_PR bit in r3 to _TIF_SIGPENDING position in r0 */
534 rlwimi r0,r3,32+TIF_SIGPENDING-MSR_PR_LG,_TIF_SIGPENDING
535 and. r0,r4,r0 /* check NEED_RESCHED and maybe SIGPENDING */
536 bne do_work
537
538#else /* !CONFIG_PREEMPT */
539 ld r3,_MSR(r1) /* Returning to user mode? */
540 andi. r3,r3,MSR_PR
541 beq restore /* if not, just restore regs and return */
542
543 /* Check current_thread_info()->flags */
544 clrrdi r9,r1,THREAD_SHIFT
545 ld r4,TI_FLAGS(r9)
546 andi. r0,r4,_TIF_USER_WORK_MASK
547 bne do_work
548#endif
549
550restore:
Stephen Rothwell3f639ee2006-09-25 18:19:00 +1000551BEGIN_FW_FTR_SECTION
Michael Ellerman01f38802008-07-16 14:21:34 +1000552 ld r5,SOFTE(r1)
553FW_FTR_SECTION_ELSE
Anton Blanchard917e4072009-10-18 01:24:29 +0000554 b .Liseries_check_pending_irqs
Michael Ellerman01f38802008-07-16 14:21:34 +1000555ALT_FW_FTR_SECTION_END_IFCLR(FW_FEATURE_ISERIES)
5562:
Benjamin Herrenschmidt945feb12008-04-17 14:35:01 +1000557 TRACE_AND_RESTORE_IRQ(r5);
Paul Mackerras9994a332005-10-10 22:36:14 +1000558
Paul Mackerrasb0a779d2006-10-18 10:11:22 +1000559 /* extract EE bit and use it to restore paca->hard_enabled */
Paul Mackerrase56a6e22007-02-07 13:13:26 +1100560 ld r3,_MSR(r1)
Paul Mackerrasb0a779d2006-10-18 10:11:22 +1000561 rldicl r4,r3,49,63 /* r0 = (r3 >> 15) & 1 */
562 stb r4,PACAHARDIRQEN(r13)
563
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000564#ifdef CONFIG_PPC_BOOK3E
565 b .exception_return_book3e
566#else
Paul Mackerrase56a6e22007-02-07 13:13:26 +1100567 ld r4,_CTR(r1)
568 ld r0,_LINK(r1)
569 mtctr r4
570 mtlr r0
571 ld r4,_XER(r1)
572 mtspr SPRN_XER,r4
573
574 REST_8GPRS(5, r1)
575
576 andi. r0,r3,MSR_RI
577 beq- unrecov_restore
578
579 stdcx. r0,0,r1 /* to clear the reservation */
580
581 /*
582 * Clear RI before restoring r13. If we are returning to
583 * userspace and we take an exception after restoring r13,
584 * we end up corrupting the userspace r13 value.
585 */
586 mfmsr r4
587 andc r4,r4,r0 /* r0 contains MSR_RI here */
588 mtmsrd r4,1
Paul Mackerras9994a332005-10-10 22:36:14 +1000589
590 /*
591 * r13 is our per cpu area, only restore it if we are returning to
592 * userspace
593 */
Paul Mackerrase56a6e22007-02-07 13:13:26 +1100594 andi. r0,r3,MSR_PR
Paul Mackerras9994a332005-10-10 22:36:14 +1000595 beq 1f
Paul Mackerrase56a6e22007-02-07 13:13:26 +1100596 ACCOUNT_CPU_USER_EXIT(r2, r4)
Paul Mackerras9994a332005-10-10 22:36:14 +1000597 REST_GPR(13, r1)
5981:
Paul Mackerrase56a6e22007-02-07 13:13:26 +1100599 mtspr SPRN_SRR1,r3
Paul Mackerras9994a332005-10-10 22:36:14 +1000600
601 ld r2,_CCR(r1)
602 mtcrf 0xFF,r2
603 ld r2,_NIP(r1)
604 mtspr SPRN_SRR0,r2
605
606 ld r0,GPR0(r1)
607 ld r2,GPR2(r1)
608 ld r3,GPR3(r1)
609 ld r4,GPR4(r1)
610 ld r1,GPR1(r1)
611
612 rfid
613 b . /* prevent speculative execution */
614
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000615#endif /* CONFIG_PPC_BOOK3E */
616
Anton Blanchard917e4072009-10-18 01:24:29 +0000617.Liseries_check_pending_irqs:
Michael Ellerman01f38802008-07-16 14:21:34 +1000618#ifdef CONFIG_PPC_ISERIES
619 ld r5,SOFTE(r1)
620 cmpdi 0,r5,0
621 beq 2b
622 /* Check for pending interrupts (iSeries) */
623 ld r3,PACALPPACAPTR(r13)
624 ld r3,LPPACAANYINT(r3)
625 cmpdi r3,0
626 beq+ 2b /* skip do_IRQ if no interrupts */
627
628 li r3,0
629 stb r3,PACASOFTIRQEN(r13) /* ensure we are soft-disabled */
630#ifdef CONFIG_TRACE_IRQFLAGS
631 bl .trace_hardirqs_off
632 mfmsr r10
633#endif
634 ori r10,r10,MSR_EE
635 mtmsrd r10 /* hard-enable again */
636 addi r3,r1,STACK_FRAME_OVERHEAD
637 bl .do_IRQ
638 b .ret_from_except_lite /* loop back and handle more */
639#endif
640
Paul Mackerras9994a332005-10-10 22:36:14 +1000641do_work:
642#ifdef CONFIG_PREEMPT
643 andi. r0,r3,MSR_PR /* Returning to user mode? */
644 bne user_work
645 /* Check that preempt_count() == 0 and interrupts are enabled */
646 lwz r8,TI_PREEMPT(r9)
647 cmpwi cr1,r8,0
Paul Mackerras9994a332005-10-10 22:36:14 +1000648 ld r0,SOFTE(r1)
649 cmpdi r0,0
Paul Mackerras9994a332005-10-10 22:36:14 +1000650 crandc eq,cr1*4+eq,eq
651 bne restore
Benjamin Herrenschmidt4f917ba2009-10-26 19:41:17 +0000652
653 /* Here we are preempting the current task.
654 *
655 * Ensure interrupts are soft-disabled. We also properly mark
656 * the PACA to reflect the fact that they are hard-disabled
657 * and trace the change
Benjamin Herrenschmidt945feb12008-04-17 14:35:01 +1000658 */
Benjamin Herrenschmidt4f917ba2009-10-26 19:41:17 +0000659 li r0,0
Paul Mackerrasd04c56f2006-10-04 16:47:49 +1000660 stb r0,PACASOFTIRQEN(r13)
661 stb r0,PACAHARDIRQEN(r13)
Benjamin Herrenschmidt4f917ba2009-10-26 19:41:17 +0000662 TRACE_DISABLE_INTS
663
664 /* Call the scheduler with soft IRQs off */
6651: bl .preempt_schedule_irq
666
667 /* Hard-disable interrupts again (and update PACA) */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000668#ifdef CONFIG_PPC_BOOK3E
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000669 wrteei 0
670#else
Paul Mackerras9994a332005-10-10 22:36:14 +1000671 mfmsr r10
Benjamin Herrenschmidt4f917ba2009-10-26 19:41:17 +0000672 rldicl r10,r10,48,1
Paul Mackerras9994a332005-10-10 22:36:14 +1000673 rotldi r10,r10,16
674 mtmsrd r10,1
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000675#endif /* CONFIG_PPC_BOOK3E */
Benjamin Herrenschmidt4f917ba2009-10-26 19:41:17 +0000676 li r0,0
677 stb r0,PACAHARDIRQEN(r13)
678
679 /* Re-test flags and eventually loop */
680 clrrdi r9,r1,THREAD_SHIFT
Paul Mackerras9994a332005-10-10 22:36:14 +1000681 ld r4,TI_FLAGS(r9)
682 andi. r0,r4,_TIF_NEED_RESCHED
683 bne 1b
684 b restore
685
686user_work:
Benjamin Herrenschmidt4f917ba2009-10-26 19:41:17 +0000687#endif /* CONFIG_PREEMPT */
688
Paul Mackerras9994a332005-10-10 22:36:14 +1000689 /* Enable interrupts */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000690#ifdef CONFIG_PPC_BOOK3E
691 wrteei 1
692#else
Paul Mackerras9994a332005-10-10 22:36:14 +1000693 ori r10,r10,MSR_EE
694 mtmsrd r10,1
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000695#endif /* CONFIG_PPC_BOOK3E */
Paul Mackerras9994a332005-10-10 22:36:14 +1000696
697 andi. r0,r4,_TIF_NEED_RESCHED
698 beq 1f
699 bl .schedule
700 b .ret_from_except_lite
701
7021: bl .save_nvgprs
Roland McGrath7d6d6372008-07-27 16:52:52 +1000703 addi r3,r1,STACK_FRAME_OVERHEAD
Paul Mackerras9994a332005-10-10 22:36:14 +1000704 bl .do_signal
705 b .ret_from_except
706
707unrecov_restore:
708 addi r3,r1,STACK_FRAME_OVERHEAD
709 bl .unrecoverable_exception
710 b unrecov_restore
711
712#ifdef CONFIG_PPC_RTAS
713/*
714 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
715 * called with the MMU off.
716 *
717 * In addition, we need to be in 32b mode, at least for now.
718 *
719 * Note: r3 is an input parameter to rtas, so don't trash it...
720 */
721_GLOBAL(enter_rtas)
722 mflr r0
723 std r0,16(r1)
724 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
725
726 /* Because RTAS is running in 32b mode, it clobbers the high order half
727 * of all registers that it saves. We therefore save those registers
728 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
729 */
730 SAVE_GPR(2, r1) /* Save the TOC */
731 SAVE_GPR(13, r1) /* Save paca */
732 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
733 SAVE_10GPRS(22, r1) /* ditto */
734
735 mfcr r4
736 std r4,_CCR(r1)
737 mfctr r5
738 std r5,_CTR(r1)
739 mfspr r6,SPRN_XER
740 std r6,_XER(r1)
741 mfdar r7
742 std r7,_DAR(r1)
743 mfdsisr r8
744 std r8,_DSISR(r1)
Paul Mackerras9994a332005-10-10 22:36:14 +1000745
Mike Kravetz9fe901d2006-03-27 15:20:00 -0800746 /* Temporary workaround to clear CR until RTAS can be modified to
747 * ignore all bits.
748 */
749 li r0,0
750 mtcr r0
751
David Woodhouse007d88d2007-01-01 18:45:34 +0000752#ifdef CONFIG_BUG
Paul Mackerras9994a332005-10-10 22:36:14 +1000753 /* There is no way it is acceptable to get here with interrupts enabled,
754 * check it with the asm equivalent of WARN_ON
755 */
Paul Mackerrasd04c56f2006-10-04 16:47:49 +1000756 lbz r0,PACASOFTIRQEN(r13)
Paul Mackerras9994a332005-10-10 22:36:14 +10007571: tdnei r0,0
David Woodhouse007d88d2007-01-01 18:45:34 +0000758 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
759#endif
760
Paul Mackerrasd04c56f2006-10-04 16:47:49 +1000761 /* Hard-disable interrupts */
762 mfmsr r6
763 rldicl r7,r6,48,1
764 rotldi r7,r7,16
765 mtmsrd r7,1
766
Paul Mackerras9994a332005-10-10 22:36:14 +1000767 /* Unfortunately, the stack pointer and the MSR are also clobbered,
768 * so they are saved in the PACA which allows us to restore
769 * our original state after RTAS returns.
770 */
771 std r1,PACAR1(r13)
772 std r6,PACASAVEDMSR(r13)
773
774 /* Setup our real return addr */
David Gibsone58c3492006-01-13 14:56:25 +1100775 LOAD_REG_ADDR(r4,.rtas_return_loc)
776 clrldi r4,r4,2 /* convert to realmode address */
Paul Mackerras9994a332005-10-10 22:36:14 +1000777 mtlr r4
778
779 li r0,0
780 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
781 andc r0,r6,r0
782
783 li r9,1
784 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
Anton Blanchard44c9f3c2010-02-07 19:37:29 +0000785 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI
Paul Mackerras9994a332005-10-10 22:36:14 +1000786 andc r6,r0,r9
Paul Mackerras9994a332005-10-10 22:36:14 +1000787 sync /* disable interrupts so SRR0/1 */
788 mtmsrd r0 /* don't get trashed */
789
David Gibsone58c3492006-01-13 14:56:25 +1100790 LOAD_REG_ADDR(r4, rtas)
Paul Mackerras9994a332005-10-10 22:36:14 +1000791 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
792 ld r4,RTASBASE(r4) /* get the rtas->base value */
793
794 mtspr SPRN_SRR0,r5
795 mtspr SPRN_SRR1,r6
796 rfid
797 b . /* prevent speculative execution */
798
799_STATIC(rtas_return_loc)
800 /* relocation is off at this point */
Benjamin Herrenschmidtee43eb72009-07-14 20:52:54 +0000801 mfspr r4,SPRN_SPRG_PACA /* Get PACA */
David Gibsone58c3492006-01-13 14:56:25 +1100802 clrldi r4,r4,2 /* convert to realmode address */
Paul Mackerras9994a332005-10-10 22:36:14 +1000803
Paul Mackerrase31aa452008-08-30 11:41:12 +1000804 bcl 20,31,$+4
8050: mflr r3
806 ld r3,(1f-0b)(r3) /* get &.rtas_restore_regs */
807
Paul Mackerras9994a332005-10-10 22:36:14 +1000808 mfmsr r6
809 li r0,MSR_RI
810 andc r6,r6,r0
811 sync
812 mtmsrd r6
813
814 ld r1,PACAR1(r4) /* Restore our SP */
Paul Mackerras9994a332005-10-10 22:36:14 +1000815 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
816
817 mtspr SPRN_SRR0,r3
818 mtspr SPRN_SRR1,r4
819 rfid
820 b . /* prevent speculative execution */
821
Paul Mackerrase31aa452008-08-30 11:41:12 +1000822 .align 3
8231: .llong .rtas_restore_regs
824
Paul Mackerras9994a332005-10-10 22:36:14 +1000825_STATIC(rtas_restore_regs)
826 /* relocation is on at this point */
827 REST_GPR(2, r1) /* Restore the TOC */
828 REST_GPR(13, r1) /* Restore paca */
829 REST_8GPRS(14, r1) /* Restore the non-volatiles */
830 REST_10GPRS(22, r1) /* ditto */
831
Benjamin Herrenschmidtee43eb72009-07-14 20:52:54 +0000832 mfspr r13,SPRN_SPRG_PACA
Paul Mackerras9994a332005-10-10 22:36:14 +1000833
834 ld r4,_CCR(r1)
835 mtcr r4
836 ld r5,_CTR(r1)
837 mtctr r5
838 ld r6,_XER(r1)
839 mtspr SPRN_XER,r6
840 ld r7,_DAR(r1)
841 mtdar r7
842 ld r8,_DSISR(r1)
843 mtdsisr r8
Paul Mackerras9994a332005-10-10 22:36:14 +1000844
845 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
846 ld r0,16(r1) /* get return address */
847
848 mtlr r0
849 blr /* return to caller */
850
851#endif /* CONFIG_PPC_RTAS */
852
Paul Mackerras9994a332005-10-10 22:36:14 +1000853_GLOBAL(enter_prom)
854 mflr r0
855 std r0,16(r1)
856 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
857
858 /* Because PROM is running in 32b mode, it clobbers the high order half
859 * of all registers that it saves. We therefore save those registers
860 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
861 */
Benjamin Herrenschmidt6c171992009-07-23 23:15:07 +0000862 SAVE_GPR(2, r1)
Paul Mackerras9994a332005-10-10 22:36:14 +1000863 SAVE_GPR(13, r1)
864 SAVE_8GPRS(14, r1)
865 SAVE_10GPRS(22, r1)
Benjamin Herrenschmidt6c171992009-07-23 23:15:07 +0000866 mfcr r10
Paul Mackerras9994a332005-10-10 22:36:14 +1000867 mfmsr r11
Benjamin Herrenschmidt6c171992009-07-23 23:15:07 +0000868 std r10,_CCR(r1)
Paul Mackerras9994a332005-10-10 22:36:14 +1000869 std r11,_MSR(r1)
870
871 /* Get the PROM entrypoint */
Benjamin Herrenschmidt6c171992009-07-23 23:15:07 +0000872 mtlr r4
Paul Mackerras9994a332005-10-10 22:36:14 +1000873
874 /* Switch MSR to 32 bits mode
875 */
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000876#ifdef CONFIG_PPC_BOOK3E
877 rlwinm r11,r11,0,1,31
878 mtmsr r11
879#else /* CONFIG_PPC_BOOK3E */
Paul Mackerras9994a332005-10-10 22:36:14 +1000880 mfmsr r11
881 li r12,1
882 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
883 andc r11,r11,r12
884 li r12,1
885 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
886 andc r11,r11,r12
887 mtmsrd r11
Benjamin Herrenschmidt2d27cfd2009-07-23 23:15:59 +0000888#endif /* CONFIG_PPC_BOOK3E */
Paul Mackerras9994a332005-10-10 22:36:14 +1000889 isync
890
Benjamin Herrenschmidt6c171992009-07-23 23:15:07 +0000891 /* Enter PROM here... */
Paul Mackerras9994a332005-10-10 22:36:14 +1000892 blrl
893
894 /* Just make sure that r1 top 32 bits didn't get
895 * corrupt by OF
896 */
897 rldicl r1,r1,0,32
898
899 /* Restore the MSR (back to 64 bits) */
900 ld r0,_MSR(r1)
Benjamin Herrenschmidt6c171992009-07-23 23:15:07 +0000901 MTMSRD(r0)
Paul Mackerras9994a332005-10-10 22:36:14 +1000902 isync
903
904 /* Restore other registers */
905 REST_GPR(2, r1)
906 REST_GPR(13, r1)
907 REST_8GPRS(14, r1)
908 REST_10GPRS(22, r1)
909 ld r4,_CCR(r1)
910 mtcr r4
Paul Mackerras9994a332005-10-10 22:36:14 +1000911
912 addi r1,r1,PROM_FRAME_SIZE
913 ld r0,16(r1)
914 mtlr r0
915 blr
Steven Rostedt4e491d12008-05-14 23:49:44 -0400916
Steven Rostedt606576c2008-10-06 19:06:12 -0400917#ifdef CONFIG_FUNCTION_TRACER
Steven Rostedt4e491d12008-05-14 23:49:44 -0400918#ifdef CONFIG_DYNAMIC_FTRACE
919_GLOBAL(mcount)
920_GLOBAL(_mcount)
Steven Rostedt4e491d12008-05-14 23:49:44 -0400921 blr
922
923_GLOBAL(ftrace_caller)
924 /* Taken from output of objdump from lib64/glibc */
925 mflr r3
926 ld r11, 0(r1)
927 stdu r1, -112(r1)
928 std r3, 128(r1)
929 ld r4, 16(r11)
Abhishek Sagar395a59d2008-06-21 23:47:27 +0530930 subi r3, r3, MCOUNT_INSN_SIZE
Steven Rostedt4e491d12008-05-14 23:49:44 -0400931.globl ftrace_call
932ftrace_call:
933 bl ftrace_stub
934 nop
Steven Rostedt46542882009-02-10 22:19:54 -0800935#ifdef CONFIG_FUNCTION_GRAPH_TRACER
936.globl ftrace_graph_call
937ftrace_graph_call:
938 b ftrace_graph_stub
939_GLOBAL(ftrace_graph_stub)
940#endif
Steven Rostedt4e491d12008-05-14 23:49:44 -0400941 ld r0, 128(r1)
942 mtlr r0
943 addi r1, r1, 112
944_GLOBAL(ftrace_stub)
945 blr
946#else
947_GLOBAL(mcount)
948 blr
949
950_GLOBAL(_mcount)
951 /* Taken from output of objdump from lib64/glibc */
952 mflr r3
953 ld r11, 0(r1)
954 stdu r1, -112(r1)
955 std r3, 128(r1)
956 ld r4, 16(r11)
957
Abhishek Sagar395a59d2008-06-21 23:47:27 +0530958 subi r3, r3, MCOUNT_INSN_SIZE
Steven Rostedt4e491d12008-05-14 23:49:44 -0400959 LOAD_REG_ADDR(r5,ftrace_trace_function)
960 ld r5,0(r5)
961 ld r5,0(r5)
962 mtctr r5
963 bctrl
Steven Rostedt4e491d12008-05-14 23:49:44 -0400964 nop
Steven Rostedt6794c782009-02-09 21:10:27 -0800965
966
967#ifdef CONFIG_FUNCTION_GRAPH_TRACER
968 b ftrace_graph_caller
969#endif
Steven Rostedt4e491d12008-05-14 23:49:44 -0400970 ld r0, 128(r1)
971 mtlr r0
972 addi r1, r1, 112
973_GLOBAL(ftrace_stub)
974 blr
975
Steven Rostedt6794c782009-02-09 21:10:27 -0800976#endif /* CONFIG_DYNAMIC_FTRACE */
977
978#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Steven Rostedt46542882009-02-10 22:19:54 -0800979_GLOBAL(ftrace_graph_caller)
Steven Rostedt6794c782009-02-09 21:10:27 -0800980 /* load r4 with local address */
981 ld r4, 128(r1)
982 subi r4, r4, MCOUNT_INSN_SIZE
983
984 /* get the parent address */
985 ld r11, 112(r1)
986 addi r3, r11, 16
987
988 bl .prepare_ftrace_return
989 nop
990
991 ld r0, 128(r1)
992 mtlr r0
993 addi r1, r1, 112
994 blr
995
996_GLOBAL(return_to_handler)
997 /* need to save return values */
Steven Rostedtbb725342009-02-11 12:45:49 -0800998 std r4, -24(r1)
999 std r3, -16(r1)
1000 std r31, -8(r1)
1001 mr r31, r1
1002 stdu r1, -112(r1)
1003
1004 bl .ftrace_return_to_handler
1005 nop
1006
1007 /* return value has real return address */
1008 mtlr r3
1009
1010 ld r1, 0(r1)
1011 ld r4, -24(r1)
1012 ld r3, -16(r1)
1013 ld r31, -8(r1)
1014
1015 /* Jump back to real return address */
1016 blr
1017
1018_GLOBAL(mod_return_to_handler)
1019 /* need to save return values */
Steven Rostedt6794c782009-02-09 21:10:27 -08001020 std r4, -32(r1)
1021 std r3, -24(r1)
1022 /* save TOC */
1023 std r2, -16(r1)
1024 std r31, -8(r1)
1025 mr r31, r1
1026 stdu r1, -112(r1)
1027
Steven Rostedtbb725342009-02-11 12:45:49 -08001028 /*
1029 * We are in a module using the module's TOC.
1030 * Switch to our TOC to run inside the core kernel.
1031 */
Steven Rostedtbe10ab12009-09-15 08:30:14 -07001032 ld r2, PACATOC(r13)
Steven Rostedt6794c782009-02-09 21:10:27 -08001033
1034 bl .ftrace_return_to_handler
1035 nop
1036
1037 /* return value has real return address */
1038 mtlr r3
1039
1040 ld r1, 0(r1)
1041 ld r4, -32(r1)
1042 ld r3, -24(r1)
1043 ld r2, -16(r1)
1044 ld r31, -8(r1)
1045
1046 /* Jump back to real return address */
1047 blr
1048#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1049#endif /* CONFIG_FUNCTION_TRACER */