blob: bf99cfa6bbfe3a29240094eb5fe2db695ed78e66 [file] [log] [blame]
Steven Rostedt4e491d12008-05-14 23:49:44 -04001/*
2 * Code for replacing ftrace calls with jumps.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 *
6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
7 *
Steven Rostedt6794c782009-02-09 21:10:27 -08008 * Added function graph tracer code, taken from x86 that was written
9 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
10 *
Steven Rostedt4e491d12008-05-14 23:49:44 -040011 */
12
13#include <linux/spinlock.h>
14#include <linux/hardirq.h>
Steven Rostedte4486fe2008-11-14 16:21:20 -080015#include <linux/uaccess.h>
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080016#include <linux/module.h>
Steven Rostedt4e491d12008-05-14 23:49:44 -040017#include <linux/ftrace.h>
18#include <linux/percpu.h>
19#include <linux/init.h>
20#include <linux/list.h>
21
22#include <asm/cacheflush.h>
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080023#include <asm/code-patching.h>
Abhishek Sagar395a59d2008-06-21 23:47:27 +053024#include <asm/ftrace.h>
Ian Munsie02424d82011-02-02 17:27:24 +000025#include <asm/syscall.h>
Steven Rostedt4e491d12008-05-14 23:49:44 -040026
Steven Rostedt4e491d12008-05-14 23:49:44 -040027
Steven Rostedt6794c782009-02-09 21:10:27 -080028#ifdef CONFIG_DYNAMIC_FTRACE
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080029static unsigned int
Steven Rostedt46542882009-02-10 22:19:54 -080030ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
Steven Rostedt4e491d12008-05-14 23:49:44 -040031{
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080032 unsigned int op;
Steven Rostedt4e491d12008-05-14 23:49:44 -040033
Michael Ellerman4a9e3f82009-05-28 19:33:34 +000034 addr = ppc_function_entry((void *)addr);
Steven Rostedt4e491d12008-05-14 23:49:44 -040035
Steven Rostedt46542882009-02-10 22:19:54 -080036 /* if (link) set op to 'bl' else 'b' */
Steven Rostedtbb9b9032009-02-13 06:45:27 -080037 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
Steven Rostedt4e491d12008-05-14 23:49:44 -040038
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080039 return op;
Steven Rostedt4e491d12008-05-14 23:49:44 -040040}
41
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080042static int
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080043ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
Steven Rostedt4e491d12008-05-14 23:49:44 -040044{
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080045 unsigned int replaced;
Steven Rostedt4e491d12008-05-14 23:49:44 -040046
Steven Rostedt4e491d12008-05-14 23:49:44 -040047 /*
48 * Note: Due to modules and __init, code can
49 * disappear and change, we need to protect against faulting
Steven Rostedte4486fe2008-11-14 16:21:20 -080050 * as well as code changing. We do this by using the
51 * probe_kernel_* functions.
Steven Rostedt4e491d12008-05-14 23:49:44 -040052 *
53 * No real locking needed, this code is run through
Steven Rostedte4486fe2008-11-14 16:21:20 -080054 * kstop_machine, or before SMP starts.
Steven Rostedt4e491d12008-05-14 23:49:44 -040055 */
Steven Rostedt4e491d12008-05-14 23:49:44 -040056
Steven Rostedte4486fe2008-11-14 16:21:20 -080057 /* read the text we want to modify */
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080058 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
Steven Rostedte4486fe2008-11-14 16:21:20 -080059 return -EFAULT;
Steven Rostedt4e491d12008-05-14 23:49:44 -040060
Steven Rostedte4486fe2008-11-14 16:21:20 -080061 /* Make sure it is what we expect it to be */
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080062 if (replaced != old)
Steven Rostedte4486fe2008-11-14 16:21:20 -080063 return -EINVAL;
Steven Rostedt4e491d12008-05-14 23:49:44 -040064
Steven Rostedte4486fe2008-11-14 16:21:20 -080065 /* replace the text with the new text */
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080066 if (probe_kernel_write((void *)ip, &new, MCOUNT_INSN_SIZE))
Steven Rostedte4486fe2008-11-14 16:21:20 -080067 return -EPERM;
68
69 flush_icache_range(ip, ip + 8);
70
71 return 0;
Steven Rostedt4e491d12008-05-14 23:49:44 -040072}
73
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080074/*
75 * Helper functions that are the same for both PPC64 and PPC32.
76 */
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080077static int test_24bit_addr(unsigned long ip, unsigned long addr)
78{
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080079
Steven Rostedt0029ff82008-11-25 14:06:19 -080080 /* use the create_branch to verify that this offset can be branched */
81 return create_branch((unsigned int *)ip, addr, 0);
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080082}
83
Steven Rostedt17be5b32009-02-05 21:33:09 -080084#ifdef CONFIG_MODULES
85
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080086static int is_bl_op(unsigned int op)
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080087{
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080088 return (op & 0xfc000003) == 0x48000001;
89}
90
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080091static unsigned long find_bl_target(unsigned long ip, unsigned int op)
92{
93 static int offset;
94
95 offset = (op & 0x03fffffc);
96 /* make it signed */
97 if (offset & 0x02000000)
98 offset |= 0xfe000000;
99
100 return ip + (long)offset;
101}
102
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800103#ifdef CONFIG_PPC64
104static int
105__ftrace_make_nop(struct module *mod,
106 struct dyn_ftrace *rec, unsigned long addr)
107{
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800108 unsigned int op;
109 unsigned int jmp[5];
110 unsigned long ptr;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800111 unsigned long ip = rec->ip;
112 unsigned long tramp;
113 int offset;
114
115 /* read where this goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800116 if (probe_kernel_read(&op, (void *)ip, sizeof(int)))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800117 return -EFAULT;
118
119 /* Make sure that that this is still a 24bit jump */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800120 if (!is_bl_op(op)) {
121 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800122 return -EINVAL;
123 }
124
125 /* lets find where the pointer goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800126 tramp = find_bl_target(ip, op);
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800127
128 /*
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800129 * On PPC64 the trampoline looks like:
130 * 0x3d, 0x82, 0x00, 0x00, addis r12,r2, <high>
131 * 0x39, 0x8c, 0x00, 0x00, addi r12,r12, <low>
132 * Where the bytes 2,3,6 and 7 make up the 32bit offset
133 * to the TOC that holds the pointer.
134 * to jump to.
135 * 0xf8, 0x41, 0x00, 0x28, std r2,40(r1)
136 * 0xe9, 0x6c, 0x00, 0x20, ld r11,32(r12)
137 * The actually address is 32 bytes from the offset
138 * into the TOC.
139 * 0xe8, 0x4c, 0x00, 0x28, ld r2,40(r12)
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800140 */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800141
Michael Ellerman021376a2009-05-13 20:30:24 +0000142 pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800143
144 /* Find where the trampoline jumps to */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800145 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800146 printk(KERN_ERR "Failed to read %lx\n", tramp);
147 return -EFAULT;
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800148 }
149
Michael Ellerman021376a2009-05-13 20:30:24 +0000150 pr_devel(" %08x %08x", jmp[0], jmp[1]);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800151
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800152 /* verify that this is what we expect it to be */
153 if (((jmp[0] & 0xffff0000) != 0x3d820000) ||
154 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
155 (jmp[2] != 0xf8410028) ||
156 (jmp[3] != 0xe96c0020) ||
157 (jmp[4] != 0xe84c0028)) {
158 printk(KERN_ERR "Not a trampoline\n");
159 return -EINVAL;
160 }
161
Steven Rostedtf25f9072009-02-07 20:22:40 +0000162 /* The bottom half is signed extended */
163 offset = ((unsigned)((unsigned short)jmp[0]) << 16) +
164 (int)((short)jmp[1]);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800165
Michael Ellerman021376a2009-05-13 20:30:24 +0000166 pr_devel(" %x ", offset);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800167
168 /* get the address this jumps too */
169 tramp = mod->arch.toc + offset + 32;
Michael Ellerman021376a2009-05-13 20:30:24 +0000170 pr_devel("toc: %lx", tramp);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800171
172 if (probe_kernel_read(jmp, (void *)tramp, 8)) {
173 printk(KERN_ERR "Failed to read %lx\n", tramp);
174 return -EFAULT;
175 }
176
Michael Ellerman021376a2009-05-13 20:30:24 +0000177 pr_devel(" %08x %08x\n", jmp[0], jmp[1]);
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800178
179 ptr = ((unsigned long)jmp[0] << 32) + jmp[1];
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800180
181 /* This should match what was called */
Michael Ellerman4a9e3f82009-05-28 19:33:34 +0000182 if (ptr != ppc_function_entry((void *)addr)) {
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800183 printk(KERN_ERR "addr does not match %lx\n", ptr);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800184 return -EINVAL;
185 }
186
187 /*
188 * We want to nop the line, but the next line is
189 * 0xe8, 0x41, 0x00, 0x28 ld r2,40(r1)
190 * This needs to be turned to a nop too.
191 */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800192 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800193 return -EFAULT;
194
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800195 if (op != 0xe8410028) {
196 printk(KERN_ERR "Next line is not ld! (%08x)\n", op);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800197 return -EINVAL;
198 }
199
200 /*
201 * Milton Miller pointed out that we can not blindly do nops.
202 * If a task was preempted when calling a trace function,
203 * the nops will remove the way to restore the TOC in r2
204 * and the r2 TOC will get corrupted.
205 */
206
207 /*
208 * Replace:
209 * bl <tramp> <==== will be replaced with "b 1f"
210 * ld r2,40(r1)
211 * 1:
212 */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800213 op = 0x48000008; /* b +8 */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800214
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800215 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800216 return -EPERM;
217
Steven Rostedtec682ce2008-11-25 10:22:48 -0800218
219 flush_icache_range(ip, ip + 8);
220
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800221 return 0;
222}
223
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800224#else /* !PPC64 */
225static int
226__ftrace_make_nop(struct module *mod,
227 struct dyn_ftrace *rec, unsigned long addr)
228{
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800229 unsigned int op;
230 unsigned int jmp[4];
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500231 unsigned long ip = rec->ip;
232 unsigned long tramp;
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500233
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800234 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500235 return -EFAULT;
236
237 /* Make sure that that this is still a 24bit jump */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800238 if (!is_bl_op(op)) {
239 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500240 return -EINVAL;
241 }
242
243 /* lets find where the pointer goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800244 tramp = find_bl_target(ip, op);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500245
246 /*
247 * On PPC32 the trampoline looks like:
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800248 * 0x3d, 0x60, 0x00, 0x00 lis r11,sym@ha
249 * 0x39, 0x6b, 0x00, 0x00 addi r11,r11,sym@l
250 * 0x7d, 0x69, 0x03, 0xa6 mtctr r11
251 * 0x4e, 0x80, 0x04, 0x20 bctr
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500252 */
253
Michael Ellerman021376a2009-05-13 20:30:24 +0000254 pr_devel("ip:%lx jumps to %lx", ip, tramp);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500255
256 /* Find where the trampoline jumps to */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800257 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500258 printk(KERN_ERR "Failed to read %lx\n", tramp);
259 return -EFAULT;
260 }
261
Michael Ellerman021376a2009-05-13 20:30:24 +0000262 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500263
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800264 /* verify that this is what we expect it to be */
265 if (((jmp[0] & 0xffff0000) != 0x3d600000) ||
266 ((jmp[1] & 0xffff0000) != 0x396b0000) ||
267 (jmp[2] != 0x7d6903a6) ||
268 (jmp[3] != 0x4e800420)) {
269 printk(KERN_ERR "Not a trampoline\n");
270 return -EINVAL;
271 }
272
273 tramp = (jmp[1] & 0xffff) |
274 ((jmp[0] & 0xffff) << 16);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500275 if (tramp & 0x8000)
276 tramp -= 0x10000;
277
Michael Ellerman021376a2009-05-13 20:30:24 +0000278 pr_devel(" %lx ", tramp);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500279
280 if (tramp != addr) {
281 printk(KERN_ERR
282 "Trampoline location %08lx does not match addr\n",
283 tramp);
284 return -EINVAL;
285 }
286
Kumar Gala16c57b32009-02-10 20:10:44 +0000287 op = PPC_INST_NOP;
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500288
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800289 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500290 return -EPERM;
291
Steven Rostedtec682ce2008-11-25 10:22:48 -0800292 flush_icache_range(ip, ip + 8);
293
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800294 return 0;
295}
296#endif /* PPC64 */
Steven Rostedt17be5b32009-02-05 21:33:09 -0800297#endif /* CONFIG_MODULES */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800298
299int ftrace_make_nop(struct module *mod,
300 struct dyn_ftrace *rec, unsigned long addr)
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800301{
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800302 unsigned long ip = rec->ip;
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800303 unsigned int old, new;
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800304
305 /*
306 * If the calling address is more that 24 bits away,
307 * then we had to use a trampoline to make the call.
308 * Otherwise just update the call site.
309 */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800310 if (test_24bit_addr(ip, addr)) {
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800311 /* within range */
Steven Rostedt46542882009-02-10 22:19:54 -0800312 old = ftrace_call_replace(ip, addr, 1);
Michael Ellerman92e02a52009-05-28 19:33:36 +0000313 new = PPC_INST_NOP;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800314 return ftrace_modify_code(ip, old, new);
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800315 }
316
Steven Rostedt17be5b32009-02-05 21:33:09 -0800317#ifdef CONFIG_MODULES
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800318 /*
319 * Out of range jumps are called from modules.
320 * We should either already have a pointer to the module
321 * or it has been passed in.
322 */
323 if (!rec->arch.mod) {
324 if (!mod) {
325 printk(KERN_ERR "No module loaded addr=%lx\n",
326 addr);
327 return -EFAULT;
328 }
329 rec->arch.mod = mod;
330 } else if (mod) {
331 if (mod != rec->arch.mod) {
332 printk(KERN_ERR
333 "Record mod %p not equal to passed in mod %p\n",
334 rec->arch.mod, mod);
335 return -EINVAL;
336 }
337 /* nothing to do if mod == rec->arch.mod */
338 } else
339 mod = rec->arch.mod;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800340
341 return __ftrace_make_nop(mod, rec, addr);
Steven Rostedt17be5b32009-02-05 21:33:09 -0800342#else
343 /* We should not get here without modules */
344 return -EINVAL;
345#endif /* CONFIG_MODULES */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800346}
347
Steven Rostedt17be5b32009-02-05 21:33:09 -0800348#ifdef CONFIG_MODULES
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800349#ifdef CONFIG_PPC64
350static int
351__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
352{
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800353 unsigned int op[2];
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800354 unsigned long ip = rec->ip;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800355
356 /* read where this goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800357 if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800358 return -EFAULT;
359
360 /*
361 * It should be pointing to two nops or
362 * b +8; ld r2,40(r1)
363 */
364 if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
Kumar Gala16c57b32009-02-10 20:10:44 +0000365 ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) {
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800366 printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
367 return -EINVAL;
368 }
369
370 /* If we never set up a trampoline to ftrace_caller, then bail */
371 if (!rec->arch.mod->arch.tramp) {
372 printk(KERN_ERR "No ftrace trampoline\n");
373 return -EINVAL;
374 }
375
Steven Rostedt0029ff82008-11-25 14:06:19 -0800376 /* create the branch to the trampoline */
377 op[0] = create_branch((unsigned int *)ip,
378 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
379 if (!op[0]) {
380 printk(KERN_ERR "REL24 out of range!\n");
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800381 return -EINVAL;
382 }
383
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800384 /* ld r2,40(r1) */
385 op[1] = 0xe8410028;
386
Michael Ellerman021376a2009-05-13 20:30:24 +0000387 pr_devel("write to %lx\n", rec->ip);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800388
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800389 if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800390 return -EPERM;
391
Steven Rostedtec682ce2008-11-25 10:22:48 -0800392 flush_icache_range(ip, ip + 8);
393
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800394 return 0;
395}
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800396#else
397static int
398__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
399{
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800400 unsigned int op;
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500401 unsigned long ip = rec->ip;
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500402
403 /* read where this goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800404 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500405 return -EFAULT;
406
407 /* It should be pointing to a nop */
Kumar Gala16c57b32009-02-10 20:10:44 +0000408 if (op != PPC_INST_NOP) {
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800409 printk(KERN_ERR "Expected NOP but have %x\n", op);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500410 return -EINVAL;
411 }
412
413 /* If we never set up a trampoline to ftrace_caller, then bail */
414 if (!rec->arch.mod->arch.tramp) {
415 printk(KERN_ERR "No ftrace trampoline\n");
416 return -EINVAL;
417 }
418
Steven Rostedt0029ff82008-11-25 14:06:19 -0800419 /* create the branch to the trampoline */
420 op = create_branch((unsigned int *)ip,
421 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
422 if (!op) {
423 printk(KERN_ERR "REL24 out of range!\n");
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500424 return -EINVAL;
425 }
426
Michael Ellerman021376a2009-05-13 20:30:24 +0000427 pr_devel("write to %lx\n", rec->ip);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500428
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800429 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500430 return -EPERM;
431
Steven Rostedtec682ce2008-11-25 10:22:48 -0800432 flush_icache_range(ip, ip + 8);
433
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800434 return 0;
435}
436#endif /* CONFIG_PPC64 */
Steven Rostedt17be5b32009-02-05 21:33:09 -0800437#endif /* CONFIG_MODULES */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800438
439int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
440{
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800441 unsigned long ip = rec->ip;
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800442 unsigned int old, new;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800443
444 /*
445 * If the calling address is more that 24 bits away,
446 * then we had to use a trampoline to make the call.
447 * Otherwise just update the call site.
448 */
449 if (test_24bit_addr(ip, addr)) {
450 /* within range */
Michael Ellerman92e02a52009-05-28 19:33:36 +0000451 old = PPC_INST_NOP;
Steven Rostedt46542882009-02-10 22:19:54 -0800452 new = ftrace_call_replace(ip, addr, 1);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800453 return ftrace_modify_code(ip, old, new);
454 }
455
Steven Rostedt17be5b32009-02-05 21:33:09 -0800456#ifdef CONFIG_MODULES
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800457 /*
458 * Out of range jumps are called from modules.
459 * Being that we are converting from nop, it had better
460 * already have a module defined.
461 */
462 if (!rec->arch.mod) {
463 printk(KERN_ERR "No module loaded\n");
464 return -EINVAL;
465 }
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800466
467 return __ftrace_make_call(rec, addr);
Steven Rostedt17be5b32009-02-05 21:33:09 -0800468#else
469 /* We should not get here without modules */
470 return -EINVAL;
471#endif /* CONFIG_MODULES */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800472}
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800473
Steven Rostedt15adc042008-10-23 09:33:08 -0400474int ftrace_update_ftrace_func(ftrace_func_t func)
Steven Rostedt4e491d12008-05-14 23:49:44 -0400475{
476 unsigned long ip = (unsigned long)(&ftrace_call);
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800477 unsigned int old, new;
Steven Rostedt4e491d12008-05-14 23:49:44 -0400478 int ret;
479
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800480 old = *(unsigned int *)&ftrace_call;
Steven Rostedt46542882009-02-10 22:19:54 -0800481 new = ftrace_call_replace(ip, (unsigned long)func, 1);
Steven Rostedt4e491d12008-05-14 23:49:44 -0400482 ret = ftrace_modify_code(ip, old, new);
483
484 return ret;
485}
486
Steven Rostedt4e491d12008-05-14 23:49:44 -0400487int __init ftrace_dyn_arch_init(void *data)
488{
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800489 /* caller expects data to be zero */
490 unsigned long *p = data;
Steven Rostedt4e491d12008-05-14 23:49:44 -0400491
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800492 *p = 0;
Steven Rostedt4e491d12008-05-14 23:49:44 -0400493
494 return 0;
495}
Steven Rostedt6794c782009-02-09 21:10:27 -0800496#endif /* CONFIG_DYNAMIC_FTRACE */
497
498#ifdef CONFIG_FUNCTION_GRAPH_TRACER
499
Steven Rostedt46542882009-02-10 22:19:54 -0800500#ifdef CONFIG_DYNAMIC_FTRACE
501extern void ftrace_graph_call(void);
502extern void ftrace_graph_stub(void);
503
504int ftrace_enable_ftrace_graph_caller(void)
505{
506 unsigned long ip = (unsigned long)(&ftrace_graph_call);
507 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
508 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800509 unsigned int old, new;
Steven Rostedt46542882009-02-10 22:19:54 -0800510
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800511 old = ftrace_call_replace(ip, stub, 0);
Steven Rostedt46542882009-02-10 22:19:54 -0800512 new = ftrace_call_replace(ip, addr, 0);
513
514 return ftrace_modify_code(ip, old, new);
515}
516
517int ftrace_disable_ftrace_graph_caller(void)
518{
519 unsigned long ip = (unsigned long)(&ftrace_graph_call);
520 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
521 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800522 unsigned int old, new;
Steven Rostedt46542882009-02-10 22:19:54 -0800523
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800524 old = ftrace_call_replace(ip, addr, 0);
Steven Rostedt46542882009-02-10 22:19:54 -0800525 new = ftrace_call_replace(ip, stub, 0);
526
527 return ftrace_modify_code(ip, old, new);
528}
529#endif /* CONFIG_DYNAMIC_FTRACE */
530
Steven Rostedtbb725342009-02-11 12:45:49 -0800531#ifdef CONFIG_PPC64
532extern void mod_return_to_handler(void);
533#endif
534
Steven Rostedt6794c782009-02-09 21:10:27 -0800535/*
536 * Hook the return address and push it in the stack of return addrs
537 * in current thread info.
538 */
539void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
540{
541 unsigned long old;
Steven Rostedt6794c782009-02-09 21:10:27 -0800542 int faulted;
543 struct ftrace_graph_ent trace;
Steven Rostedtbb725342009-02-11 12:45:49 -0800544 unsigned long return_hooker = (unsigned long)&return_to_handler;
Steven Rostedt6794c782009-02-09 21:10:27 -0800545
546 if (unlikely(atomic_read(&current->tracing_graph_pause)))
547 return;
548
Michael Ellermanf4952f62009-04-06 04:40:45 +0000549#ifdef CONFIG_PPC64
Steven Rostedtbb725342009-02-11 12:45:49 -0800550 /* non core kernel code needs to save and restore the TOC */
551 if (REGION_ID(self_addr) != KERNEL_REGION_ID)
552 return_hooker = (unsigned long)&mod_return_to_handler;
553#endif
554
Michael Ellerman4a9e3f82009-05-28 19:33:34 +0000555 return_hooker = ppc_function_entry((void *)return_hooker);
Steven Rostedt6794c782009-02-09 21:10:27 -0800556
557 /*
558 * Protect against fault, even if it shouldn't
559 * happen. This tool is too much intrusive to
560 * ignore such a protection.
561 */
562 asm volatile(
563 "1: " PPC_LL "%[old], 0(%[parent])\n"
564 "2: " PPC_STL "%[return_hooker], 0(%[parent])\n"
565 " li %[faulted], 0\n"
Steven Rostedtfad4f472009-02-11 19:10:57 -0500566 "3:\n"
Steven Rostedt6794c782009-02-09 21:10:27 -0800567
568 ".section .fixup, \"ax\"\n"
569 "4: li %[faulted], 1\n"
570 " b 3b\n"
571 ".previous\n"
572
573 ".section __ex_table,\"a\"\n"
574 PPC_LONG_ALIGN "\n"
575 PPC_LONG "1b,4b\n"
576 PPC_LONG "2b,4b\n"
577 ".previous"
578
Steven Rostedtc3cf8662009-05-15 04:33:54 +0000579 : [old] "=&r" (old), [faulted] "=r" (faulted)
Steven Rostedt6794c782009-02-09 21:10:27 -0800580 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
581 : "memory"
582 );
583
584 if (unlikely(faulted)) {
585 ftrace_graph_stop();
586 WARN_ON(1);
587 return;
588 }
589
Steven Rostedt71e308a2009-06-18 12:45:08 -0400590 if (ftrace_push_return_trace(old, self_addr, &trace.depth, 0) == -EBUSY) {
Steven Rostedt6794c782009-02-09 21:10:27 -0800591 *parent = old;
592 return;
593 }
594
595 trace.func = self_addr;
596
597 /* Only trace if the calling function expects to */
598 if (!ftrace_graph_entry(&trace)) {
599 current->curr_ret_stack--;
600 *parent = old;
601 }
602}
603#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
Ian Munsie02424d82011-02-02 17:27:24 +0000604
605#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
606unsigned long __init arch_syscall_addr(int nr)
607{
608 return sys_call_table[nr*2];
609}
610#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */