blob: f4fe15ddcf947643e6c0a444aa852ff90ee956af [file] [log] [blame]
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -08001#include <linux/module.h>
Al Virof6a57032006-10-18 01:47:25 -04002#include <linux/sched.h>
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +02003#include <linux/mutex.h>
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -08004#include <linux/list.h>
Jan Beulich8b5a10f2009-08-19 08:40:48 +01005#include <linux/stringify.h>
Andi Kleen19d36cc2007-07-22 11:12:31 +02006#include <linux/kprobes.h>
7#include <linux/mm.h>
8#include <linux/vmalloc.h>
Masami Hiramatsu3945dab2009-03-06 10:37:22 -05009#include <linux/memory.h>
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -050010#include <linux/stop_machine.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090011#include <linux/slab.h>
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080012#include <asm/alternative.h>
13#include <asm/sections.h>
Andi Kleen19d36cc2007-07-22 11:12:31 +020014#include <asm/pgtable.h>
Andi Kleen8f4e9562007-07-22 11:12:32 +020015#include <asm/mce.h>
16#include <asm/nmi.h>
Dave Jonesb0979762007-10-14 22:57:45 +020017#include <asm/vsyscall.h>
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -050018#include <asm/cacheflush.h>
Masami Hiramatsu78ff7fa2009-03-06 10:37:54 -050019#include <asm/tlbflush.h>
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -050020#include <asm/io.h>
Masami Hiramatsu78ff7fa2009-03-06 10:37:54 -050021#include <asm/fixmap.h>
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080022
Andi Kleenab144f52007-08-10 22:31:03 +020023#define MAX_PATCH_LEN (255-1)
24
Jan Beulich09488162007-07-21 17:10:25 +020025#ifdef CONFIG_HOTPLUG_CPU
26static int smp_alt_once;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080027
Gerd Hoffmannd167a512006-06-26 13:56:16 +020028static int __init bootonly(char *str)
29{
30 smp_alt_once = 1;
31 return 1;
32}
Jeremy Fitzhardingeb7fb4af2007-05-02 19:27:13 +020033__setup("smp-alt-boot", bootonly);
Jan Beulich09488162007-07-21 17:10:25 +020034#else
35#define smp_alt_once 1
36#endif
37
Jan Beulich8b5a10f2009-08-19 08:40:48 +010038static int __initdata_or_module debug_alternative;
Jeremy Fitzhardingeb7fb4af2007-05-02 19:27:13 +020039
Gerd Hoffmannd167a512006-06-26 13:56:16 +020040static int __init debug_alt(char *str)
41{
42 debug_alternative = 1;
43 return 1;
44}
Gerd Hoffmannd167a512006-06-26 13:56:16 +020045__setup("debug-alternative", debug_alt);
46
Jan Beulich09488162007-07-21 17:10:25 +020047static int noreplace_smp;
48
Jeremy Fitzhardingeb7fb4af2007-05-02 19:27:13 +020049static int __init setup_noreplace_smp(char *str)
50{
51 noreplace_smp = 1;
52 return 1;
53}
54__setup("noreplace-smp", setup_noreplace_smp);
55
Jeremy Fitzhardinge959b4fd2007-05-02 19:27:16 +020056#ifdef CONFIG_PARAVIRT
Jan Beulich8b5a10f2009-08-19 08:40:48 +010057static int __initdata_or_module noreplace_paravirt = 0;
Jeremy Fitzhardinge959b4fd2007-05-02 19:27:16 +020058
59static int __init setup_noreplace_paravirt(char *str)
60{
61 noreplace_paravirt = 1;
62 return 1;
63}
64__setup("noreplace-paravirt", setup_noreplace_paravirt);
65#endif
Jeremy Fitzhardingeb7fb4af2007-05-02 19:27:13 +020066
Gerd Hoffmannd167a512006-06-26 13:56:16 +020067#define DPRINTK(fmt, args...) if (debug_alternative) \
68 printk(KERN_DEBUG fmt, args)
69
Jan Beulich8b5a10f2009-08-19 08:40:48 +010070#if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080071/* Use inline assembly to define this because the nops are defined
72 as inline assembly strings in the include files and we cannot
73 get them easily into strings. */
Jan Beulich8b5a10f2009-08-19 08:40:48 +010074asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: "
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080075 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
Steven Rostedtf4be31e2008-04-09 19:04:07 -040076 GENERIC_NOP7 GENERIC_NOP8
77 "\t.previous");
Jan Beulich121d7bf2007-10-17 18:04:37 +020078extern const unsigned char intelnops[];
Jan Beulich8b5a10f2009-08-19 08:40:48 +010079static const unsigned char *const __initconst_or_module
80intel_nops[ASM_NOP_MAX+1] = {
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080081 NULL,
82 intelnops,
83 intelnops + 1,
84 intelnops + 1 + 2,
85 intelnops + 1 + 2 + 3,
86 intelnops + 1 + 2 + 3 + 4,
87 intelnops + 1 + 2 + 3 + 4 + 5,
88 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
89 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
90};
Gerd Hoffmannd167a512006-06-26 13:56:16 +020091#endif
92
93#ifdef K8_NOP1
Jan Beulich8b5a10f2009-08-19 08:40:48 +010094asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: "
Gerd Hoffmannd167a512006-06-26 13:56:16 +020095 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
Steven Rostedtf4be31e2008-04-09 19:04:07 -040096 K8_NOP7 K8_NOP8
97 "\t.previous");
Jan Beulich121d7bf2007-10-17 18:04:37 +020098extern const unsigned char k8nops[];
Jan Beulich8b5a10f2009-08-19 08:40:48 +010099static const unsigned char *const __initconst_or_module
100k8_nops[ASM_NOP_MAX+1] = {
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800101 NULL,
102 k8nops,
103 k8nops + 1,
104 k8nops + 1 + 2,
105 k8nops + 1 + 2 + 3,
106 k8nops + 1 + 2 + 3 + 4,
107 k8nops + 1 + 2 + 3 + 4 + 5,
108 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
109 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
110};
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200111#endif
112
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100113#if defined(K7_NOP1) && !defined(CONFIG_X86_64)
114asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: "
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200115 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
Steven Rostedtf4be31e2008-04-09 19:04:07 -0400116 K7_NOP7 K7_NOP8
117 "\t.previous");
Jan Beulich121d7bf2007-10-17 18:04:37 +0200118extern const unsigned char k7nops[];
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100119static const unsigned char *const __initconst_or_module
120k7_nops[ASM_NOP_MAX+1] = {
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800121 NULL,
122 k7nops,
123 k7nops + 1,
124 k7nops + 1 + 2,
125 k7nops + 1 + 2 + 3,
126 k7nops + 1 + 2 + 3 + 4,
127 k7nops + 1 + 2 + 3 + 4 + 5,
128 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
129 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
130};
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200131#endif
132
Jan Beulich32c464f2007-10-17 18:04:41 +0200133#ifdef P6_NOP1
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100134asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: "
Jan Beulich32c464f2007-10-17 18:04:41 +0200135 P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6
Steven Rostedtf4be31e2008-04-09 19:04:07 -0400136 P6_NOP7 P6_NOP8
137 "\t.previous");
Jan Beulich32c464f2007-10-17 18:04:41 +0200138extern const unsigned char p6nops[];
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100139static const unsigned char *const __initconst_or_module
140p6_nops[ASM_NOP_MAX+1] = {
Jan Beulich32c464f2007-10-17 18:04:41 +0200141 NULL,
142 p6nops,
143 p6nops + 1,
144 p6nops + 1 + 2,
145 p6nops + 1 + 2 + 3,
146 p6nops + 1 + 2 + 3 + 4,
147 p6nops + 1 + 2 + 3 + 4 + 5,
148 p6nops + 1 + 2 + 3 + 4 + 5 + 6,
149 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
150};
151#endif
152
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200153#ifdef CONFIG_X86_64
154
155extern char __vsyscall_0;
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100156static const unsigned char *const *__init_or_module find_nop_table(void)
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200157{
H. Peter Anvinf31d7312008-08-18 17:50:33 -0700158 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
159 boot_cpu_has(X86_FEATURE_NOPL))
160 return p6_nops;
161 else
162 return k8_nops;
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200163}
164
165#else /* CONFIG_X86_64 */
166
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100167static const unsigned char *const *__init_or_module find_nop_table(void)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800168{
H. Peter Anvinf31d7312008-08-18 17:50:33 -0700169 if (boot_cpu_has(X86_FEATURE_K8))
170 return k8_nops;
171 else if (boot_cpu_has(X86_FEATURE_K7))
172 return k7_nops;
173 else if (boot_cpu_has(X86_FEATURE_NOPL))
174 return p6_nops;
175 else
176 return intel_nops;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800177}
178
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200179#endif /* CONFIG_X86_64 */
180
Andi Kleenab144f52007-08-10 22:31:03 +0200181/* Use this to add nops to a buffer, then text_poke the whole buffer. */
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100182static void __init_or_module add_nops(void *insns, unsigned int len)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100183{
Jan Beulich121d7bf2007-10-17 18:04:37 +0200184 const unsigned char *const *noptable = find_nop_table();
Rusty Russell139ec7c2006-12-07 02:14:08 +0100185
186 while (len > 0) {
187 unsigned int noplen = len;
188 if (noplen > ASM_NOP_MAX)
189 noplen = ASM_NOP_MAX;
Andi Kleenab144f52007-08-10 22:31:03 +0200190 memcpy(insns, noptable[noplen], noplen);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100191 insns += noplen;
192 len -= noplen;
193 }
194}
195
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200196extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
Jan Beulich5967ed82010-04-21 16:08:14 +0100197extern s32 __smp_locks[], __smp_locks_end[];
Jason Baronfa6f2cc2010-09-17 11:08:56 -0400198void *text_poke_early(void *addr, const void *opcode, size_t len);
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200199
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800200/* Replace instructions with better alternatives for this CPU type.
201 This runs before SMP is initialized to avoid SMP problems with
Lucas De Marchi0d2eb442011-03-17 16:24:16 -0300202 self modifying code. This implies that asymmetric systems where
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800203 APs have less capabilities than the boot processor are not handled.
204 Tough. Make sure you disable such features by hand. */
205
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100206void __init_or_module apply_alternatives(struct alt_instr *start,
207 struct alt_instr *end)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800208{
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800209 struct alt_instr *a;
Jan Beulich1b1d9252009-12-18 16:12:56 +0000210 u8 insnbuf[MAX_PATCH_LEN];
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800211
Harvey Harrison77bf90e2008-03-03 11:37:23 -0800212 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
Fenghua Yu50973132011-05-17 15:29:12 -0700213 /*
214 * The scan order should be from start to end. A later scanned
215 * alternative code can overwrite a previous scanned alternative code.
216 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
217 * patch code.
218 *
219 * So be careful if you want to change the scan order to any other
220 * order.
221 */
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800222 for (a = start; a < end; a++) {
Andi Kleenab144f52007-08-10 22:31:03 +0200223 u8 *instr = a->instr;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800224 BUG_ON(a->replacementlen > a->instrlen);
Andi Kleenab144f52007-08-10 22:31:03 +0200225 BUG_ON(a->instrlen > sizeof(insnbuf));
H. Peter Anvin3b770a22010-07-13 14:57:50 -0700226 BUG_ON(a->cpuid >= NCAPINTS*32);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800227 if (!boot_cpu_has(a->cpuid))
228 continue;
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200229#ifdef CONFIG_X86_64
230 /* vsyscall code is not mapped yet. resolve it manually. */
231 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
232 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
233 DPRINTK("%s: vsyscall fixup: %p => %p\n",
Harvey Harrison77bf90e2008-03-03 11:37:23 -0800234 __func__, a->instr, instr);
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200235 }
236#endif
Andi Kleenab144f52007-08-10 22:31:03 +0200237 memcpy(insnbuf, a->replacement, a->replacementlen);
Jan Beulich1b1d9252009-12-18 16:12:56 +0000238 if (*insnbuf == 0xe8 && a->replacementlen == 5)
239 *(s32 *)(insnbuf + 1) += a->replacement - a->instr;
Andi Kleenab144f52007-08-10 22:31:03 +0200240 add_nops(insnbuf + a->replacementlen,
241 a->instrlen - a->replacementlen);
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500242 text_poke_early(instr, insnbuf, a->instrlen);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800243 }
244}
245
Gerd Hoffmann8ec4d412006-07-01 04:36:18 -0700246#ifdef CONFIG_SMP
247
Jan Beulich5967ed82010-04-21 16:08:14 +0100248static void alternatives_smp_lock(const s32 *start, const s32 *end,
249 u8 *text, u8 *text_end)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800250{
Jan Beulich5967ed82010-04-21 16:08:14 +0100251 const s32 *poff;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800252
Masami Hiramatsu3945dab2009-03-06 10:37:22 -0500253 mutex_lock(&text_mutex);
Jan Beulich5967ed82010-04-21 16:08:14 +0100254 for (poff = start; poff < end; poff++) {
255 u8 *ptr = (u8 *)poff + *poff;
256
257 if (!*poff || ptr < text || ptr >= text_end)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800258 continue;
Mathieu Desnoyersf88f07e2008-08-14 16:58:15 -0400259 /* turn DS segment override prefix into lock prefix */
H. Peter Anvind9c58412010-04-29 16:53:17 -0700260 if (*ptr == 0x3e)
261 text_poke(ptr, ((unsigned char []){0xf0}), 1);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800262 };
Masami Hiramatsu3945dab2009-03-06 10:37:22 -0500263 mutex_unlock(&text_mutex);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800264}
265
Jan Beulich5967ed82010-04-21 16:08:14 +0100266static void alternatives_smp_unlock(const s32 *start, const s32 *end,
267 u8 *text, u8 *text_end)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800268{
Jan Beulich5967ed82010-04-21 16:08:14 +0100269 const s32 *poff;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800270
Jeremy Fitzhardingeb7fb4af2007-05-02 19:27:13 +0200271 if (noreplace_smp)
272 return;
273
Masami Hiramatsu3945dab2009-03-06 10:37:22 -0500274 mutex_lock(&text_mutex);
Jan Beulich5967ed82010-04-21 16:08:14 +0100275 for (poff = start; poff < end; poff++) {
276 u8 *ptr = (u8 *)poff + *poff;
277
278 if (!*poff || ptr < text || ptr >= text_end)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800279 continue;
Mathieu Desnoyersf88f07e2008-08-14 16:58:15 -0400280 /* turn lock prefix into DS segment override prefix */
H. Peter Anvind9c58412010-04-29 16:53:17 -0700281 if (*ptr == 0xf0)
282 text_poke(ptr, ((unsigned char []){0x3E}), 1);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800283 };
Masami Hiramatsu3945dab2009-03-06 10:37:22 -0500284 mutex_unlock(&text_mutex);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800285}
286
287struct smp_alt_module {
288 /* what is this ??? */
289 struct module *mod;
290 char *name;
291
292 /* ptrs to lock prefixes */
Jan Beulich5967ed82010-04-21 16:08:14 +0100293 const s32 *locks;
294 const s32 *locks_end;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800295
296 /* .text segment, needed to avoid patching init code ;) */
297 u8 *text;
298 u8 *text_end;
299
300 struct list_head next;
301};
302static LIST_HEAD(smp_alt_modules);
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +0200303static DEFINE_MUTEX(smp_alt);
Andi Kleenca74a6f2008-01-30 13:33:17 +0100304static int smp_mode = 1; /* protected by smp_alt */
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800305
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100306void __init_or_module alternatives_smp_module_add(struct module *mod,
307 char *name,
308 void *locks, void *locks_end,
309 void *text, void *text_end)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800310{
311 struct smp_alt_module *smp;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800312
Jeremy Fitzhardingeb7fb4af2007-05-02 19:27:13 +0200313 if (noreplace_smp)
314 return;
315
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800316 if (smp_alt_once) {
317 if (boot_cpu_has(X86_FEATURE_UP))
318 alternatives_smp_unlock(locks, locks_end,
319 text, text_end);
320 return;
321 }
322
323 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
324 if (NULL == smp)
325 return; /* we'll run the (safe but slow) SMP code then ... */
326
327 smp->mod = mod;
328 smp->name = name;
329 smp->locks = locks;
330 smp->locks_end = locks_end;
331 smp->text = text;
332 smp->text_end = text_end;
333 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
Harvey Harrison77bf90e2008-03-03 11:37:23 -0800334 __func__, smp->locks, smp->locks_end,
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800335 smp->text, smp->text_end, smp->name);
336
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +0200337 mutex_lock(&smp_alt);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800338 list_add_tail(&smp->next, &smp_alt_modules);
339 if (boot_cpu_has(X86_FEATURE_UP))
340 alternatives_smp_unlock(smp->locks, smp->locks_end,
341 smp->text, smp->text_end);
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +0200342 mutex_unlock(&smp_alt);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800343}
344
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100345void __init_or_module alternatives_smp_module_del(struct module *mod)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800346{
347 struct smp_alt_module *item;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800348
Jeremy Fitzhardingeb7fb4af2007-05-02 19:27:13 +0200349 if (smp_alt_once || noreplace_smp)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800350 return;
351
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +0200352 mutex_lock(&smp_alt);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800353 list_for_each_entry(item, &smp_alt_modules, next) {
354 if (mod != item->mod)
355 continue;
356 list_del(&item->next);
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +0200357 mutex_unlock(&smp_alt);
Harvey Harrison77bf90e2008-03-03 11:37:23 -0800358 DPRINTK("%s: %s\n", __func__, item->name);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800359 kfree(item);
360 return;
361 }
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +0200362 mutex_unlock(&smp_alt);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800363}
364
Suresh Siddha3fb82d52010-11-23 16:11:40 -0800365bool skip_smp_alternatives;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800366void alternatives_smp_switch(int smp)
367{
368 struct smp_alt_module *mod;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800369
Ingo Molnar3047e992006-07-03 00:24:57 -0700370#ifdef CONFIG_LOCKDEP
371 /*
Ingo Molnar17abecf2008-01-30 13:33:24 +0100372 * Older binutils section handling bug prevented
373 * alternatives-replacement from working reliably.
374 *
375 * If this still occurs then you should see a hang
376 * or crash shortly after this line:
Ingo Molnar3047e992006-07-03 00:24:57 -0700377 */
Ingo Molnar17abecf2008-01-30 13:33:24 +0100378 printk("lockdep: fixing up alternatives.\n");
Ingo Molnar3047e992006-07-03 00:24:57 -0700379#endif
380
Suresh Siddha3fb82d52010-11-23 16:11:40 -0800381 if (noreplace_smp || smp_alt_once || skip_smp_alternatives)
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800382 return;
383 BUG_ON(!smp && (num_online_cpus() > 1));
384
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +0200385 mutex_lock(&smp_alt);
Andi Kleenca74a6f2008-01-30 13:33:17 +0100386
387 /*
388 * Avoid unnecessary switches because it forces JIT based VMs to
389 * throw away all cached translations, which can be quite costly.
390 */
391 if (smp == smp_mode) {
392 /* nothing */
393 } else if (smp) {
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800394 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
Jeremy Fitzhardinge53756d32008-01-30 13:30:55 +0100395 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
396 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800397 list_for_each_entry(mod, &smp_alt_modules, next)
398 alternatives_smp_lock(mod->locks, mod->locks_end,
399 mod->text, mod->text_end);
400 } else {
401 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
Jeremy Fitzhardinge53756d32008-01-30 13:30:55 +0100402 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
403 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800404 list_for_each_entry(mod, &smp_alt_modules, next)
405 alternatives_smp_unlock(mod->locks, mod->locks_end,
406 mod->text, mod->text_end);
407 }
Andi Kleenca74a6f2008-01-30 13:33:17 +0100408 smp_mode = smp;
Pekka Paalanen2f1dafe2008-05-12 21:21:01 +0200409 mutex_unlock(&smp_alt);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800410}
411
Masami Hiramatsu2cfa1972010-02-02 16:49:11 -0500412/* Return 1 if the address range is reserved for smp-alternatives */
413int alternatives_text_reserved(void *start, void *end)
414{
415 struct smp_alt_module *mod;
Jan Beulich5967ed82010-04-21 16:08:14 +0100416 const s32 *poff;
Masami Hiramatsu076dc4a62010-02-05 12:16:47 -0500417 u8 *text_start = start;
418 u8 *text_end = end;
Masami Hiramatsu2cfa1972010-02-02 16:49:11 -0500419
420 list_for_each_entry(mod, &smp_alt_modules, next) {
Masami Hiramatsu076dc4a62010-02-05 12:16:47 -0500421 if (mod->text > text_end || mod->text_end < text_start)
Masami Hiramatsu2cfa1972010-02-02 16:49:11 -0500422 continue;
Jan Beulich5967ed82010-04-21 16:08:14 +0100423 for (poff = mod->locks; poff < mod->locks_end; poff++) {
424 const u8 *ptr = (const u8 *)poff + *poff;
425
426 if (text_start <= ptr && text_end > ptr)
Masami Hiramatsu2cfa1972010-02-02 16:49:11 -0500427 return 1;
Jan Beulich5967ed82010-04-21 16:08:14 +0100428 }
Masami Hiramatsu2cfa1972010-02-02 16:49:11 -0500429 }
430
431 return 0;
432}
Gerd Hoffmann8ec4d412006-07-01 04:36:18 -0700433#endif
434
Rusty Russell139ec7c2006-12-07 02:14:08 +0100435#ifdef CONFIG_PARAVIRT
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100436void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
437 struct paravirt_patch_site *end)
Rusty Russell139ec7c2006-12-07 02:14:08 +0100438{
Jeremy Fitzhardinge98de0322007-05-02 19:27:14 +0200439 struct paravirt_patch_site *p;
Andi Kleenab144f52007-08-10 22:31:03 +0200440 char insnbuf[MAX_PATCH_LEN];
Rusty Russell139ec7c2006-12-07 02:14:08 +0100441
Jeremy Fitzhardinge959b4fd2007-05-02 19:27:16 +0200442 if (noreplace_paravirt)
443 return;
444
Rusty Russell139ec7c2006-12-07 02:14:08 +0100445 for (p = start; p < end; p++) {
446 unsigned int used;
447
Andi Kleenab144f52007-08-10 22:31:03 +0200448 BUG_ON(p->len > MAX_PATCH_LEN);
Chris Wrightd34fda42007-08-18 14:31:41 -0700449 /* prep the buffer with the original instructions */
450 memcpy(insnbuf, p->instr, p->len);
Jeremy Fitzhardinge93b1eab2007-10-16 11:51:29 -0700451 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf,
452 (unsigned long)p->instr, p->len);
Jeremy Fitzhardinge7f63c412007-05-02 19:27:13 +0200453
Jeremy Fitzhardinge63f70272007-05-02 19:27:14 +0200454 BUG_ON(used > p->len);
455
Rusty Russell139ec7c2006-12-07 02:14:08 +0100456 /* Pad the rest with nops */
Andi Kleenab144f52007-08-10 22:31:03 +0200457 add_nops(insnbuf + used, p->len - used);
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500458 text_poke_early(p->instr, insnbuf, p->len);
Rusty Russell139ec7c2006-12-07 02:14:08 +0100459 }
Rusty Russell139ec7c2006-12-07 02:14:08 +0100460}
Jeremy Fitzhardinge98de0322007-05-02 19:27:14 +0200461extern struct paravirt_patch_site __start_parainstructions[],
Rusty Russell139ec7c2006-12-07 02:14:08 +0100462 __stop_parainstructions[];
463#endif /* CONFIG_PARAVIRT */
464
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800465void __init alternative_instructions(void)
466{
Andi Kleen8f4e9562007-07-22 11:12:32 +0200467 /* The patching is not fully atomic, so try to avoid local interruptions
468 that might execute the to be patched code.
469 Other CPUs are not running. */
470 stop_nmi();
Andi Kleen123aa762009-02-12 13:39:27 +0100471
472 /*
473 * Don't stop machine check exceptions while patching.
474 * MCEs only happen when something got corrupted and in this
475 * case we must do something about the corruption.
476 * Ignoring it is worse than a unlikely patching race.
477 * Also machine checks tend to be broadcast and if one CPU
478 * goes into machine check the others follow quickly, so we don't
479 * expect a machine check to cause undue problems during to code
480 * patching.
481 */
Andi Kleen8f4e9562007-07-22 11:12:32 +0200482
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800483 apply_alternatives(__alt_instructions, __alt_instructions_end);
484
485 /* switch to patch-once-at-boottime-only mode and free the
486 * tables in case we know the number of CPUs will never ever
487 * change */
488#ifdef CONFIG_HOTPLUG_CPU
489 if (num_possible_cpus() < 2)
490 smp_alt_once = 1;
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800491#endif
492
Gerd Hoffmann8ec4d412006-07-01 04:36:18 -0700493#ifdef CONFIG_SMP
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800494 if (smp_alt_once) {
495 if (1 == num_possible_cpus()) {
496 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
Jeremy Fitzhardinge53756d32008-01-30 13:30:55 +0100497 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
498 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
499
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800500 alternatives_smp_unlock(__smp_locks, __smp_locks_end,
501 _text, _etext);
502 }
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800503 } else {
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800504 alternatives_smp_module_add(NULL, "core kernel",
505 __smp_locks, __smp_locks_end,
506 _text, _etext);
Andi Kleenca74a6f2008-01-30 13:33:17 +0100507
508 /* Only switch to UP mode if we don't immediately boot others */
Thomas Gleixner649c6652008-10-05 16:52:24 +0200509 if (num_present_cpus() == 1 || setup_max_cpus <= 1)
Andi Kleenca74a6f2008-01-30 13:33:17 +0100510 alternatives_smp_switch(0);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800511 }
Gerd Hoffmann8ec4d412006-07-01 04:36:18 -0700512#endif
Jeremy Fitzhardinge441d40d2007-05-02 19:27:16 +0200513 apply_paravirt(__parainstructions, __parainstructions_end);
Andi Kleen8f4e9562007-07-22 11:12:32 +0200514
Fengguang Wuf68fd5f2007-10-17 18:04:34 +0200515 if (smp_alt_once)
516 free_init_pages("SMP alternatives",
517 (unsigned long)__smp_locks,
518 (unsigned long)__smp_locks_end);
519
Andi Kleen8f4e9562007-07-22 11:12:32 +0200520 restart_nmi();
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800521}
Andi Kleen19d36cc2007-07-22 11:12:31 +0200522
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500523/**
524 * text_poke_early - Update instructions on a live kernel at boot time
525 * @addr: address to modify
526 * @opcode: source of the copy
527 * @len: length to copy
528 *
Andi Kleen19d36cc2007-07-22 11:12:31 +0200529 * When you use this code to patch more than one byte of an instruction
530 * you need to make sure that other CPUs cannot execute this code in parallel.
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500531 * Also no thread must be currently preempted in the middle of these
532 * instructions. And on the local CPU you need to be protected again NMI or MCE
533 * handlers seeing an inconsistent instruction while you patch.
Andi Kleen19d36cc2007-07-22 11:12:31 +0200534 */
Jason Baronfa6f2cc2010-09-17 11:08:56 -0400535void *__init_or_module text_poke_early(void *addr, const void *opcode,
Jan Beulich8b5a10f2009-08-19 08:40:48 +0100536 size_t len)
Andi Kleen19d36cc2007-07-22 11:12:31 +0200537{
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500538 unsigned long flags;
539 local_irq_save(flags);
Andi Kleen19d36cc2007-07-22 11:12:31 +0200540 memcpy(addr, opcode, len);
541 sync_core();
Ben Hutchings5367b682009-09-10 02:53:50 +0100542 local_irq_restore(flags);
Andi Kleena534b672007-09-06 16:59:52 +0200543 /* Could also do a CLFLUSH here to speed up CPU recovery; but
544 that causes hangs on some VIA CPUs. */
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500545 return addr;
546}
547
548/**
549 * text_poke - Update instructions on a live kernel
550 * @addr: address to modify
551 * @opcode: source of the copy
552 * @len: length to copy
553 *
554 * Only atomic text poke/set should be allowed when not doing early patching.
555 * It means the size must be writable atomically and the address must be aligned
556 * in a way that permits an atomic write. It also makes sure we fit on a single
557 * page.
Masami Hiramatsu78ff7fa2009-03-06 10:37:54 -0500558 *
559 * Note: Must be called under text_mutex.
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500560 */
561void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
562{
Masami Hiramatsu78ff7fa2009-03-06 10:37:54 -0500563 unsigned long flags;
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500564 char *vaddr;
Mathieu Desnoyersb7b66ba2008-04-24 11:03:33 -0400565 struct page *pages[2];
566 int i;
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500567
Mathieu Desnoyersb7b66ba2008-04-24 11:03:33 -0400568 if (!core_kernel_text((unsigned long)addr)) {
569 pages[0] = vmalloc_to_page(addr);
570 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
Mathieu Desnoyers15a601e2008-03-12 11:54:16 -0400571 } else {
Mathieu Desnoyersb7b66ba2008-04-24 11:03:33 -0400572 pages[0] = virt_to_page(addr);
Ingo Molnar00c6b2d2008-04-25 17:07:03 +0200573 WARN_ON(!PageReserved(pages[0]));
Mathieu Desnoyersb7b66ba2008-04-24 11:03:33 -0400574 pages[1] = virt_to_page(addr + PAGE_SIZE);
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500575 }
Mathieu Desnoyersb7b66ba2008-04-24 11:03:33 -0400576 BUG_ON(!pages[0]);
Masami Hiramatsu7cf49422009-03-09 12:40:40 -0400577 local_irq_save(flags);
Masami Hiramatsu78ff7fa2009-03-06 10:37:54 -0500578 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0]));
579 if (pages[1])
580 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1]));
581 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0);
Mathieu Desnoyersb7b66ba2008-04-24 11:03:33 -0400582 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
Masami Hiramatsu78ff7fa2009-03-06 10:37:54 -0500583 clear_fixmap(FIX_TEXT_POKE0);
584 if (pages[1])
585 clear_fixmap(FIX_TEXT_POKE1);
586 local_flush_tlb();
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500587 sync_core();
588 /* Could also do a CLFLUSH here to speed up CPU recovery; but
589 that causes hangs on some VIA CPUs. */
Mathieu Desnoyersb7b66ba2008-04-24 11:03:33 -0400590 for (i = 0; i < len; i++)
591 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
Masami Hiramatsu7cf49422009-03-09 12:40:40 -0400592 local_irq_restore(flags);
Mathieu Desnoyerse587cad2008-03-06 08:48:49 -0500593 return addr;
Andi Kleen19d36cc2007-07-22 11:12:31 +0200594}
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500595
596/*
597 * Cross-modifying kernel text with stop_machine().
598 * This code originally comes from immediate value.
599 */
600static atomic_t stop_machine_first;
601static int wrote_text;
602
603struct text_poke_params {
Masami Hiramatsu7deb18d2010-12-03 18:54:22 +0900604 struct text_poke_param *params;
605 int nparams;
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500606};
607
608static int __kprobes stop_machine_text_poke(void *data)
609{
610 struct text_poke_params *tpp = data;
Masami Hiramatsu7deb18d2010-12-03 18:54:22 +0900611 struct text_poke_param *p;
612 int i;
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500613
614 if (atomic_dec_and_test(&stop_machine_first)) {
Masami Hiramatsu7deb18d2010-12-03 18:54:22 +0900615 for (i = 0; i < tpp->nparams; i++) {
616 p = &tpp->params[i];
617 text_poke(p->addr, p->opcode, p->len);
618 }
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500619 smp_wmb(); /* Make sure other cpus see that this has run */
620 wrote_text = 1;
621 } else {
622 while (!wrote_text)
Masami Hiramatsue5a11012010-03-03 22:38:50 -0500623 cpu_relax();
624 smp_mb(); /* Load wrote_text before following execution */
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500625 }
626
Masami Hiramatsu7deb18d2010-12-03 18:54:22 +0900627 for (i = 0; i < tpp->nparams; i++) {
628 p = &tpp->params[i];
629 flush_icache_range((unsigned long)p->addr,
630 (unsigned long)p->addr + p->len);
631 }
Mathieu Desnoyers0e00f7a2011-03-03 11:01:37 -0500632 /*
633 * Intel Archiecture Software Developer's Manual section 7.1.3 specifies
634 * that a core serializing instruction such as "cpuid" should be
635 * executed on _each_ core before the new instruction is made visible.
636 */
637 sync_core();
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500638 return 0;
639}
640
641/**
642 * text_poke_smp - Update instructions on a live kernel on SMP
643 * @addr: address to modify
644 * @opcode: source of the copy
645 * @len: length to copy
646 *
647 * Modify multi-byte instruction by using stop_machine() on SMP. This allows
648 * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying
649 * should be allowed, since stop_machine() does _not_ protect code against
650 * NMI and MCE.
651 *
652 * Note: Must be called under get_online_cpus() and text_mutex.
653 */
654void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len)
655{
656 struct text_poke_params tpp;
Masami Hiramatsu7deb18d2010-12-03 18:54:22 +0900657 struct text_poke_param p;
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500658
Masami Hiramatsu7deb18d2010-12-03 18:54:22 +0900659 p.addr = addr;
660 p.opcode = opcode;
661 p.len = len;
662 tpp.params = &p;
663 tpp.nparams = 1;
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500664 atomic_set(&stop_machine_first, 1);
665 wrote_text = 0;
Masami Hiramatsu3caa37512010-10-14 12:10:36 +0900666 /* Use __stop_machine() because the caller already got online_cpus. */
Jason Baron404ba5d2010-10-28 11:20:27 -0400667 __stop_machine(stop_machine_text_poke, (void *)&tpp, cpu_online_mask);
Masami Hiramatsu3d55cc82010-02-25 08:34:38 -0500668 return addr;
669}
670
Masami Hiramatsu7deb18d2010-12-03 18:54:22 +0900671/**
672 * text_poke_smp_batch - Update instructions on a live kernel on SMP
673 * @params: an array of text_poke parameters
674 * @n: the number of elements in params.
675 *
676 * Modify multi-byte instruction by using stop_machine() on SMP. Since the
677 * stop_machine() is heavy task, it is better to aggregate text_poke requests
678 * and do it once if possible.
679 *
680 * Note: Must be called under get_online_cpus() and text_mutex.
681 */
682void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n)
683{
684 struct text_poke_params tpp = {.params = params, .nparams = n};
685
686 atomic_set(&stop_machine_first, 1);
687 wrote_text = 0;
Peter Zijlstrad91309f2011-02-11 22:07:46 +0100688 __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);
Masami Hiramatsu7deb18d2010-12-03 18:54:22 +0900689}
690
Jason Baronbf5438fc2010-09-17 11:09:00 -0400691#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
Jason Baronf49aa442010-09-17 11:08:51 -0400692
H. Peter Anvin2d1d7122010-10-27 21:09:15 -0700693#ifdef CONFIG_X86_64
694unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 };
695#else
696unsigned char ideal_nop5[5] = { 0x3e, 0x8d, 0x74, 0x26, 0x00 };
697#endif
Jason Baronf49aa442010-09-17 11:08:51 -0400698
699void __init arch_init_ideal_nop5(void)
700{
Jason Baronf49aa442010-09-17 11:08:51 -0400701 /*
H. Peter Anvin2d1d7122010-10-27 21:09:15 -0700702 * There is no good nop for all x86 archs. This selection
703 * algorithm should be unified with the one in find_nop_table(),
704 * but this should be good enough for now.
Jason Baronf49aa442010-09-17 11:08:51 -0400705 *
H. Peter Anvin2d1d7122010-10-27 21:09:15 -0700706 * For cases other than the ones below, use the safe (as in
707 * always functional) defaults above.
Jason Baronf49aa442010-09-17 11:08:51 -0400708 */
H. Peter Anvin2d1d7122010-10-27 21:09:15 -0700709#ifdef CONFIG_X86_64
710 /* Don't use these on 32 bits due to broken virtualizers */
711 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
712 memcpy(ideal_nop5, p6_nops[5], 5);
713#endif
Jason Baronf49aa442010-09-17 11:08:51 -0400714}
715#endif