Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 1 | /* |
| 2 | * Dynamic function tracing support. |
| 3 | * |
| 4 | * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 5 | * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 6 | * |
| 7 | * For licencing details, see COPYING. |
| 8 | * |
| 9 | * Defines low-level handling of mcount calls when the kernel |
| 10 | * is compiled with the -pg flag. When using dynamic ftrace, the |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 11 | * mcount call-sites get patched with NOP till they are enabled. |
| 12 | * All code mutation routines here are called under stop_machine(). |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 13 | */ |
| 14 | |
| 15 | #include <linux/ftrace.h> |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 16 | #include <linux/uaccess.h> |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 17 | |
Abhishek Sagar | 395a59d | 2008-06-21 23:47:27 +0530 | [diff] [blame] | 18 | #include <asm/cacheflush.h> |
| 19 | #include <asm/ftrace.h> |
| 20 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 21 | #define NOP 0xe8bd4000 /* pop {lr} */ |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 22 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 23 | #ifdef CONFIG_OLD_MCOUNT |
| 24 | #define OLD_MCOUNT_ADDR ((unsigned long) mcount) |
| 25 | #define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 26 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 27 | #define OLD_NOP 0xe1a00000 /* mov r0, r0 */ |
| 28 | |
| 29 | static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 30 | { |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 31 | return rec->arch.old_mcount ? OLD_NOP : NOP; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 32 | } |
| 33 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 34 | static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) |
| 35 | { |
| 36 | if (!rec->arch.old_mcount) |
| 37 | return addr; |
| 38 | |
| 39 | if (addr == MCOUNT_ADDR) |
| 40 | addr = OLD_MCOUNT_ADDR; |
| 41 | else if (addr == FTRACE_ADDR) |
| 42 | addr = OLD_FTRACE_ADDR; |
| 43 | |
| 44 | return addr; |
| 45 | } |
| 46 | #else |
| 47 | static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) |
| 48 | { |
| 49 | return NOP; |
| 50 | } |
| 51 | |
| 52 | static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) |
| 53 | { |
| 54 | return addr; |
| 55 | } |
| 56 | #endif |
| 57 | |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 58 | /* construct a branch (BL) instruction to addr */ |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 59 | static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 60 | { |
| 61 | long offset; |
| 62 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 63 | offset = (long)addr - (long)(pc + 8); |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 64 | if (unlikely(offset < -33554432 || offset > 33554428)) { |
| 65 | /* Can't generate branches that far (from ARM ARM). Ftrace |
Abhishek Sagar | 395a59d | 2008-06-21 23:47:27 +0530 | [diff] [blame] | 66 | * doesn't generate branches outside of kernel text. |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 67 | */ |
| 68 | WARN_ON_ONCE(1); |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 69 | return 0; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 70 | } |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 71 | |
| 72 | offset = (offset >> 2) & 0x00ffffff; |
| 73 | |
| 74 | return 0xeb000000 | offset; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 75 | } |
| 76 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 77 | static int ftrace_modify_code(unsigned long pc, unsigned long old, |
| 78 | unsigned long new) |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 79 | { |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 80 | unsigned long replaced; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 81 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 82 | if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) |
| 83 | return -EFAULT; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 84 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 85 | if (replaced != old) |
| 86 | return -EINVAL; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 87 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 88 | if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) |
| 89 | return -EPERM; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 90 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 91 | flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 92 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 93 | return 0; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 94 | } |
| 95 | |
| 96 | int ftrace_update_ftrace_func(ftrace_func_t func) |
| 97 | { |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 98 | unsigned long pc, old; |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 99 | unsigned long new; |
| 100 | int ret; |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 101 | |
| 102 | pc = (unsigned long)&ftrace_call; |
Abhishek Sagar | 395a59d | 2008-06-21 23:47:27 +0530 | [diff] [blame] | 103 | memcpy(&old, &ftrace_call, MCOUNT_INSN_SIZE); |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 104 | new = ftrace_call_replace(pc, (unsigned long)func); |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 105 | |
| 106 | ret = ftrace_modify_code(pc, old, new); |
| 107 | |
| 108 | #ifdef CONFIG_OLD_MCOUNT |
| 109 | if (!ret) { |
| 110 | pc = (unsigned long)&ftrace_call_old; |
| 111 | memcpy(&old, &ftrace_call_old, MCOUNT_INSN_SIZE); |
| 112 | new = ftrace_call_replace(pc, (unsigned long)func); |
| 113 | |
| 114 | ret = ftrace_modify_code(pc, old, new); |
| 115 | } |
| 116 | #endif |
| 117 | |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 118 | return ret; |
| 119 | } |
| 120 | |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 121 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
| 122 | { |
| 123 | unsigned long new, old; |
| 124 | unsigned long ip = rec->ip; |
| 125 | |
| 126 | old = ftrace_nop_replace(rec); |
| 127 | new = ftrace_call_replace(ip, adjust_address(rec, addr)); |
| 128 | |
| 129 | return ftrace_modify_code(rec->ip, old, new); |
| 130 | } |
| 131 | |
| 132 | int ftrace_make_nop(struct module *mod, |
| 133 | struct dyn_ftrace *rec, unsigned long addr) |
| 134 | { |
| 135 | unsigned long ip = rec->ip; |
| 136 | unsigned long old; |
| 137 | unsigned long new; |
| 138 | int ret; |
| 139 | |
| 140 | old = ftrace_call_replace(ip, adjust_address(rec, addr)); |
| 141 | new = ftrace_nop_replace(rec); |
| 142 | ret = ftrace_modify_code(ip, old, new); |
| 143 | |
| 144 | #ifdef CONFIG_OLD_MCOUNT |
| 145 | if (ret == -EINVAL && addr == MCOUNT_ADDR) { |
| 146 | rec->arch.old_mcount = true; |
| 147 | |
| 148 | old = ftrace_call_replace(ip, adjust_address(rec, addr)); |
| 149 | new = ftrace_nop_replace(rec); |
| 150 | ret = ftrace_modify_code(ip, old, new); |
| 151 | } |
| 152 | #endif |
| 153 | |
| 154 | return ret; |
| 155 | } |
| 156 | |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 157 | int __init ftrace_dyn_arch_init(void *data) |
| 158 | { |
Rabin Vincent | 3b6c223 | 2010-08-10 19:43:28 +0100 | [diff] [blame^] | 159 | *(unsigned long *)data = 0; |
| 160 | |
Abhishek Sagar | 014c257 | 2008-05-31 14:23:50 +0530 | [diff] [blame] | 161 | return 0; |
| 162 | } |