Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 1 | /* align.c - handle alignment exceptions for the Power PC. |
| 2 | * |
| 3 | * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au> |
| 4 | * Copyright (c) 1998-1999 TiVo, Inc. |
| 5 | * PowerPC 403GCX modifications. |
| 6 | * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu> |
| 7 | * PowerPC 403GCX/405GP modifications. |
| 8 | * Copyright (c) 2001-2002 PPC64 team, IBM Corp |
| 9 | * 64-bit and Power4 support |
| 10 | * Copyright (c) 2005 Benjamin Herrenschmidt, IBM Corp |
| 11 | * <benh@kernel.crashing.org> |
| 12 | * Merge ppc32 and ppc64 implementations |
| 13 | * |
| 14 | * This program is free software; you can redistribute it and/or |
| 15 | * modify it under the terms of the GNU General Public License |
| 16 | * as published by the Free Software Foundation; either version |
| 17 | * 2 of the License, or (at your option) any later version. |
| 18 | */ |
| 19 | |
| 20 | #include <linux/kernel.h> |
| 21 | #include <linux/mm.h> |
| 22 | #include <asm/processor.h> |
| 23 | #include <asm/uaccess.h> |
| 24 | #include <asm/system.h> |
| 25 | #include <asm/cache.h> |
| 26 | #include <asm/cputable.h> |
| 27 | |
| 28 | struct aligninfo { |
| 29 | unsigned char len; |
| 30 | unsigned char flags; |
| 31 | }; |
| 32 | |
| 33 | #define IS_XFORM(inst) (((inst) >> 26) == 31) |
| 34 | #define IS_DSFORM(inst) (((inst) >> 26) >= 56) |
| 35 | |
| 36 | #define INVALID { 0, 0 } |
| 37 | |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 38 | /* Bits in the flags field */ |
| 39 | #define LD 0 /* load */ |
| 40 | #define ST 1 /* store */ |
Paul Mackerras | c6d4267 | 2007-08-10 14:07:38 +1000 | [diff] [blame^] | 41 | #define SE 2 /* sign-extend value, or FP ld/st as word */ |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 42 | #define F 4 /* to/from fp regs */ |
| 43 | #define U 8 /* update index register */ |
| 44 | #define M 0x10 /* multiple load/store */ |
| 45 | #define SW 0x20 /* byte swap */ |
| 46 | #define S 0x40 /* single-precision fp or... */ |
| 47 | #define SX 0x40 /* ... byte count in XER */ |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 48 | #define HARD 0x80 /* string, stwcx. */ |
| 49 | |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 50 | /* DSISR bits reported for a DCBZ instruction: */ |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 51 | #define DCBZ 0x5f /* 8xx/82xx dcbz faults when cache not enabled */ |
| 52 | |
| 53 | #define SWAP(a, b) (t = (a), (a) = (b), (b) = t) |
| 54 | |
| 55 | /* |
| 56 | * The PowerPC stores certain bits of the instruction that caused the |
| 57 | * alignment exception in the DSISR register. This array maps those |
| 58 | * bits to information about the operand length and what the |
| 59 | * instruction would do. |
| 60 | */ |
| 61 | static struct aligninfo aligninfo[128] = { |
| 62 | { 4, LD }, /* 00 0 0000: lwz / lwarx */ |
| 63 | INVALID, /* 00 0 0001 */ |
| 64 | { 4, ST }, /* 00 0 0010: stw */ |
| 65 | INVALID, /* 00 0 0011 */ |
| 66 | { 2, LD }, /* 00 0 0100: lhz */ |
| 67 | { 2, LD+SE }, /* 00 0 0101: lha */ |
| 68 | { 2, ST }, /* 00 0 0110: sth */ |
| 69 | { 4, LD+M }, /* 00 0 0111: lmw */ |
| 70 | { 4, LD+F+S }, /* 00 0 1000: lfs */ |
| 71 | { 8, LD+F }, /* 00 0 1001: lfd */ |
| 72 | { 4, ST+F+S }, /* 00 0 1010: stfs */ |
| 73 | { 8, ST+F }, /* 00 0 1011: stfd */ |
| 74 | INVALID, /* 00 0 1100 */ |
| 75 | { 8, LD }, /* 00 0 1101: ld/ldu/lwa */ |
| 76 | INVALID, /* 00 0 1110 */ |
| 77 | { 8, ST }, /* 00 0 1111: std/stdu */ |
| 78 | { 4, LD+U }, /* 00 1 0000: lwzu */ |
| 79 | INVALID, /* 00 1 0001 */ |
| 80 | { 4, ST+U }, /* 00 1 0010: stwu */ |
| 81 | INVALID, /* 00 1 0011 */ |
| 82 | { 2, LD+U }, /* 00 1 0100: lhzu */ |
| 83 | { 2, LD+SE+U }, /* 00 1 0101: lhau */ |
| 84 | { 2, ST+U }, /* 00 1 0110: sthu */ |
| 85 | { 4, ST+M }, /* 00 1 0111: stmw */ |
| 86 | { 4, LD+F+S+U }, /* 00 1 1000: lfsu */ |
| 87 | { 8, LD+F+U }, /* 00 1 1001: lfdu */ |
| 88 | { 4, ST+F+S+U }, /* 00 1 1010: stfsu */ |
| 89 | { 8, ST+F+U }, /* 00 1 1011: stfdu */ |
Paul Mackerras | c6d4267 | 2007-08-10 14:07:38 +1000 | [diff] [blame^] | 90 | { 16, LD+F }, /* 00 1 1100: lfdp */ |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 91 | INVALID, /* 00 1 1101 */ |
Paul Mackerras | c6d4267 | 2007-08-10 14:07:38 +1000 | [diff] [blame^] | 92 | { 16, ST+F }, /* 00 1 1110: stfdp */ |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 93 | INVALID, /* 00 1 1111 */ |
| 94 | { 8, LD }, /* 01 0 0000: ldx */ |
| 95 | INVALID, /* 01 0 0001 */ |
| 96 | { 8, ST }, /* 01 0 0010: stdx */ |
| 97 | INVALID, /* 01 0 0011 */ |
| 98 | INVALID, /* 01 0 0100 */ |
| 99 | { 4, LD+SE }, /* 01 0 0101: lwax */ |
| 100 | INVALID, /* 01 0 0110 */ |
| 101 | INVALID, /* 01 0 0111 */ |
| 102 | { 4, LD+M+HARD+SX }, /* 01 0 1000: lswx */ |
| 103 | { 4, LD+M+HARD }, /* 01 0 1001: lswi */ |
| 104 | { 4, ST+M+HARD+SX }, /* 01 0 1010: stswx */ |
| 105 | { 4, ST+M+HARD }, /* 01 0 1011: stswi */ |
| 106 | INVALID, /* 01 0 1100 */ |
| 107 | { 8, LD+U }, /* 01 0 1101: ldu */ |
| 108 | INVALID, /* 01 0 1110 */ |
| 109 | { 8, ST+U }, /* 01 0 1111: stdu */ |
| 110 | { 8, LD+U }, /* 01 1 0000: ldux */ |
| 111 | INVALID, /* 01 1 0001 */ |
| 112 | { 8, ST+U }, /* 01 1 0010: stdux */ |
| 113 | INVALID, /* 01 1 0011 */ |
| 114 | INVALID, /* 01 1 0100 */ |
| 115 | { 4, LD+SE+U }, /* 01 1 0101: lwaux */ |
| 116 | INVALID, /* 01 1 0110 */ |
| 117 | INVALID, /* 01 1 0111 */ |
| 118 | INVALID, /* 01 1 1000 */ |
| 119 | INVALID, /* 01 1 1001 */ |
| 120 | INVALID, /* 01 1 1010 */ |
| 121 | INVALID, /* 01 1 1011 */ |
| 122 | INVALID, /* 01 1 1100 */ |
| 123 | INVALID, /* 01 1 1101 */ |
| 124 | INVALID, /* 01 1 1110 */ |
| 125 | INVALID, /* 01 1 1111 */ |
| 126 | INVALID, /* 10 0 0000 */ |
| 127 | INVALID, /* 10 0 0001 */ |
| 128 | INVALID, /* 10 0 0010: stwcx. */ |
| 129 | INVALID, /* 10 0 0011 */ |
| 130 | INVALID, /* 10 0 0100 */ |
| 131 | INVALID, /* 10 0 0101 */ |
| 132 | INVALID, /* 10 0 0110 */ |
| 133 | INVALID, /* 10 0 0111 */ |
| 134 | { 4, LD+SW }, /* 10 0 1000: lwbrx */ |
| 135 | INVALID, /* 10 0 1001 */ |
| 136 | { 4, ST+SW }, /* 10 0 1010: stwbrx */ |
| 137 | INVALID, /* 10 0 1011 */ |
| 138 | { 2, LD+SW }, /* 10 0 1100: lhbrx */ |
| 139 | { 4, LD+SE }, /* 10 0 1101 lwa */ |
| 140 | { 2, ST+SW }, /* 10 0 1110: sthbrx */ |
| 141 | INVALID, /* 10 0 1111 */ |
| 142 | INVALID, /* 10 1 0000 */ |
| 143 | INVALID, /* 10 1 0001 */ |
| 144 | INVALID, /* 10 1 0010 */ |
| 145 | INVALID, /* 10 1 0011 */ |
| 146 | INVALID, /* 10 1 0100 */ |
| 147 | INVALID, /* 10 1 0101 */ |
| 148 | INVALID, /* 10 1 0110 */ |
| 149 | INVALID, /* 10 1 0111 */ |
| 150 | INVALID, /* 10 1 1000 */ |
| 151 | INVALID, /* 10 1 1001 */ |
| 152 | INVALID, /* 10 1 1010 */ |
| 153 | INVALID, /* 10 1 1011 */ |
| 154 | INVALID, /* 10 1 1100 */ |
| 155 | INVALID, /* 10 1 1101 */ |
| 156 | INVALID, /* 10 1 1110 */ |
| 157 | { 0, ST+HARD }, /* 10 1 1111: dcbz */ |
| 158 | { 4, LD }, /* 11 0 0000: lwzx */ |
| 159 | INVALID, /* 11 0 0001 */ |
| 160 | { 4, ST }, /* 11 0 0010: stwx */ |
| 161 | INVALID, /* 11 0 0011 */ |
| 162 | { 2, LD }, /* 11 0 0100: lhzx */ |
| 163 | { 2, LD+SE }, /* 11 0 0101: lhax */ |
| 164 | { 2, ST }, /* 11 0 0110: sthx */ |
| 165 | INVALID, /* 11 0 0111 */ |
| 166 | { 4, LD+F+S }, /* 11 0 1000: lfsx */ |
| 167 | { 8, LD+F }, /* 11 0 1001: lfdx */ |
| 168 | { 4, ST+F+S }, /* 11 0 1010: stfsx */ |
| 169 | { 8, ST+F }, /* 11 0 1011: stfdx */ |
Paul Mackerras | c6d4267 | 2007-08-10 14:07:38 +1000 | [diff] [blame^] | 170 | { 16, LD+F }, /* 11 0 1100: lfdpx */ |
| 171 | { 4, LD+F+SE }, /* 11 0 1101: lfiwax */ |
| 172 | { 16, ST+F }, /* 11 0 1110: stfdpx */ |
| 173 | { 4, ST+F }, /* 11 0 1111: stfiwx */ |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 174 | { 4, LD+U }, /* 11 1 0000: lwzux */ |
| 175 | INVALID, /* 11 1 0001 */ |
| 176 | { 4, ST+U }, /* 11 1 0010: stwux */ |
| 177 | INVALID, /* 11 1 0011 */ |
| 178 | { 2, LD+U }, /* 11 1 0100: lhzux */ |
| 179 | { 2, LD+SE+U }, /* 11 1 0101: lhaux */ |
| 180 | { 2, ST+U }, /* 11 1 0110: sthux */ |
| 181 | INVALID, /* 11 1 0111 */ |
| 182 | { 4, LD+F+S+U }, /* 11 1 1000: lfsux */ |
| 183 | { 8, LD+F+U }, /* 11 1 1001: lfdux */ |
| 184 | { 4, ST+F+S+U }, /* 11 1 1010: stfsux */ |
| 185 | { 8, ST+F+U }, /* 11 1 1011: stfdux */ |
| 186 | INVALID, /* 11 1 1100 */ |
| 187 | INVALID, /* 11 1 1101 */ |
| 188 | INVALID, /* 11 1 1110 */ |
| 189 | INVALID, /* 11 1 1111 */ |
| 190 | }; |
| 191 | |
| 192 | /* |
| 193 | * Create a DSISR value from the instruction |
| 194 | */ |
| 195 | static inline unsigned make_dsisr(unsigned instr) |
| 196 | { |
| 197 | unsigned dsisr; |
| 198 | |
| 199 | |
| 200 | /* bits 6:15 --> 22:31 */ |
| 201 | dsisr = (instr & 0x03ff0000) >> 16; |
| 202 | |
| 203 | if (IS_XFORM(instr)) { |
| 204 | /* bits 29:30 --> 15:16 */ |
| 205 | dsisr |= (instr & 0x00000006) << 14; |
| 206 | /* bit 25 --> 17 */ |
| 207 | dsisr |= (instr & 0x00000040) << 8; |
| 208 | /* bits 21:24 --> 18:21 */ |
| 209 | dsisr |= (instr & 0x00000780) << 3; |
| 210 | } else { |
| 211 | /* bit 5 --> 17 */ |
| 212 | dsisr |= (instr & 0x04000000) >> 12; |
| 213 | /* bits 1: 4 --> 18:21 */ |
| 214 | dsisr |= (instr & 0x78000000) >> 17; |
| 215 | /* bits 30:31 --> 12:13 */ |
| 216 | if (IS_DSFORM(instr)) |
| 217 | dsisr |= (instr & 0x00000003) << 18; |
| 218 | } |
| 219 | |
| 220 | return dsisr; |
| 221 | } |
| 222 | |
| 223 | /* |
| 224 | * The dcbz (data cache block zero) instruction |
| 225 | * gives an alignment fault if used on non-cacheable |
| 226 | * memory. We handle the fault mainly for the |
| 227 | * case when we are running with the cache disabled |
| 228 | * for debugging. |
| 229 | */ |
| 230 | static int emulate_dcbz(struct pt_regs *regs, unsigned char __user *addr) |
| 231 | { |
| 232 | long __user *p; |
| 233 | int i, size; |
| 234 | |
| 235 | #ifdef __powerpc64__ |
| 236 | size = ppc64_caches.dline_size; |
| 237 | #else |
| 238 | size = L1_CACHE_BYTES; |
| 239 | #endif |
| 240 | p = (long __user *) (regs->dar & -size); |
| 241 | if (user_mode(regs) && !access_ok(VERIFY_WRITE, p, size)) |
| 242 | return -EFAULT; |
| 243 | for (i = 0; i < size / sizeof(long); ++i) |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 244 | if (__put_user_inatomic(0, p+i)) |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 245 | return -EFAULT; |
| 246 | return 1; |
| 247 | } |
| 248 | |
| 249 | /* |
| 250 | * Emulate load & store multiple instructions |
| 251 | * On 64-bit machines, these instructions only affect/use the |
| 252 | * bottom 4 bytes of each register, and the loads clear the |
| 253 | * top 4 bytes of the affected register. |
| 254 | */ |
| 255 | #ifdef CONFIG_PPC64 |
| 256 | #define REG_BYTE(rp, i) *((u8 *)((rp) + ((i) >> 2)) + ((i) & 3) + 4) |
| 257 | #else |
| 258 | #define REG_BYTE(rp, i) *((u8 *)(rp) + (i)) |
| 259 | #endif |
| 260 | |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 261 | #define SWIZ_PTR(p) ((unsigned char __user *)((p) ^ swiz)) |
| 262 | |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 263 | static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr, |
| 264 | unsigned int reg, unsigned int nb, |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 265 | unsigned int flags, unsigned int instr, |
| 266 | unsigned long swiz) |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 267 | { |
| 268 | unsigned long *rptr; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 269 | unsigned int nb0, i, bswiz; |
| 270 | unsigned long p; |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 271 | |
| 272 | /* |
| 273 | * We do not try to emulate 8 bytes multiple as they aren't really |
| 274 | * available in our operating environments and we don't try to |
| 275 | * emulate multiples operations in kernel land as they should never |
| 276 | * be used/generated there at least not on unaligned boundaries |
| 277 | */ |
| 278 | if (unlikely((nb > 4) || !user_mode(regs))) |
| 279 | return 0; |
| 280 | |
| 281 | /* lmw, stmw, lswi/x, stswi/x */ |
| 282 | nb0 = 0; |
| 283 | if (flags & HARD) { |
| 284 | if (flags & SX) { |
| 285 | nb = regs->xer & 127; |
| 286 | if (nb == 0) |
| 287 | return 1; |
| 288 | } else { |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 289 | unsigned long pc = regs->nip ^ (swiz & 4); |
| 290 | |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 291 | if (__get_user_inatomic(instr, |
| 292 | (unsigned int __user *)pc)) |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 293 | return -EFAULT; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 294 | if (swiz == 0 && (flags & SW)) |
| 295 | instr = cpu_to_le32(instr); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 296 | nb = (instr >> 11) & 0x1f; |
| 297 | if (nb == 0) |
| 298 | nb = 32; |
| 299 | } |
| 300 | if (nb + reg * 4 > 128) { |
| 301 | nb0 = nb + reg * 4 - 128; |
| 302 | nb = 128 - reg * 4; |
| 303 | } |
| 304 | } else { |
| 305 | /* lwm, stmw */ |
| 306 | nb = (32 - reg) * 4; |
| 307 | } |
| 308 | |
| 309 | if (!access_ok((flags & ST ? VERIFY_WRITE: VERIFY_READ), addr, nb+nb0)) |
| 310 | return -EFAULT; /* bad address */ |
| 311 | |
| 312 | rptr = ®s->gpr[reg]; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 313 | p = (unsigned long) addr; |
| 314 | bswiz = (flags & SW)? 3: 0; |
| 315 | |
| 316 | if (!(flags & ST)) { |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 317 | /* |
| 318 | * This zeroes the top 4 bytes of the affected registers |
| 319 | * in 64-bit mode, and also zeroes out any remaining |
| 320 | * bytes of the last register for lsw*. |
| 321 | */ |
| 322 | memset(rptr, 0, ((nb + 3) / 4) * sizeof(unsigned long)); |
| 323 | if (nb0 > 0) |
| 324 | memset(®s->gpr[0], 0, |
| 325 | ((nb0 + 3) / 4) * sizeof(unsigned long)); |
| 326 | |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 327 | for (i = 0; i < nb; ++i, ++p) |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 328 | if (__get_user_inatomic(REG_BYTE(rptr, i ^ bswiz), |
| 329 | SWIZ_PTR(p))) |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 330 | return -EFAULT; |
| 331 | if (nb0 > 0) { |
| 332 | rptr = ®s->gpr[0]; |
| 333 | addr += nb; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 334 | for (i = 0; i < nb0; ++i, ++p) |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 335 | if (__get_user_inatomic(REG_BYTE(rptr, |
| 336 | i ^ bswiz), |
| 337 | SWIZ_PTR(p))) |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 338 | return -EFAULT; |
| 339 | } |
| 340 | |
| 341 | } else { |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 342 | for (i = 0; i < nb; ++i, ++p) |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 343 | if (__put_user_inatomic(REG_BYTE(rptr, i ^ bswiz), |
| 344 | SWIZ_PTR(p))) |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 345 | return -EFAULT; |
| 346 | if (nb0 > 0) { |
| 347 | rptr = ®s->gpr[0]; |
| 348 | addr += nb; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 349 | for (i = 0; i < nb0; ++i, ++p) |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 350 | if (__put_user_inatomic(REG_BYTE(rptr, |
| 351 | i ^ bswiz), |
| 352 | SWIZ_PTR(p))) |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 353 | return -EFAULT; |
| 354 | } |
| 355 | } |
| 356 | return 1; |
| 357 | } |
| 358 | |
Paul Mackerras | c6d4267 | 2007-08-10 14:07:38 +1000 | [diff] [blame^] | 359 | /* |
| 360 | * Emulate floating-point pair loads and stores. |
| 361 | * Only POWER6 has these instructions, and it does true little-endian, |
| 362 | * so we don't need the address swizzling. |
| 363 | */ |
| 364 | static int emulate_fp_pair(struct pt_regs *regs, unsigned char __user *addr, |
| 365 | unsigned int reg, unsigned int flags) |
| 366 | { |
| 367 | char *ptr = (char *) ¤t->thread.fpr[reg]; |
| 368 | int i, ret; |
| 369 | |
| 370 | if (!(flags & F)) |
| 371 | return 0; |
| 372 | if (reg & 1) |
| 373 | return 0; /* invalid form: FRS/FRT must be even */ |
| 374 | if (!(flags & SW)) { |
| 375 | /* not byte-swapped - easy */ |
| 376 | if (!(flags & ST)) |
| 377 | ret = __copy_from_user(ptr, addr, 16); |
| 378 | else |
| 379 | ret = __copy_to_user(addr, ptr, 16); |
| 380 | } else { |
| 381 | /* each FPR value is byte-swapped separately */ |
| 382 | ret = 0; |
| 383 | for (i = 0; i < 16; ++i) { |
| 384 | if (!(flags & ST)) |
| 385 | ret |= __get_user(ptr[i^7], addr + i); |
| 386 | else |
| 387 | ret |= __put_user(ptr[i^7], addr + i); |
| 388 | } |
| 389 | } |
| 390 | if (ret) |
| 391 | return -EFAULT; |
| 392 | return 1; /* exception handled and fixed up */ |
| 393 | } |
| 394 | |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 395 | |
| 396 | /* |
| 397 | * Called on alignment exception. Attempts to fixup |
| 398 | * |
| 399 | * Return 1 on success |
| 400 | * Return 0 if unable to handle the interrupt |
| 401 | * Return -EFAULT if data address is bad |
| 402 | */ |
| 403 | |
| 404 | int fix_alignment(struct pt_regs *regs) |
| 405 | { |
| 406 | unsigned int instr, nb, flags; |
| 407 | unsigned int reg, areg; |
| 408 | unsigned int dsisr; |
| 409 | unsigned char __user *addr; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 410 | unsigned long p, swiz; |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 411 | int ret, t; |
| 412 | union { |
| 413 | u64 ll; |
| 414 | double dd; |
| 415 | unsigned char v[8]; |
| 416 | struct { |
| 417 | unsigned hi32; |
| 418 | int low32; |
| 419 | } x32; |
| 420 | struct { |
| 421 | unsigned char hi48[6]; |
| 422 | short low16; |
| 423 | } x16; |
| 424 | } data; |
| 425 | |
| 426 | /* |
| 427 | * We require a complete register set, if not, then our assembly |
| 428 | * is broken |
| 429 | */ |
| 430 | CHECK_FULL_REGS(regs); |
| 431 | |
| 432 | dsisr = regs->dsisr; |
| 433 | |
| 434 | /* Some processors don't provide us with a DSISR we can use here, |
| 435 | * let's make one up from the instruction |
| 436 | */ |
| 437 | if (cpu_has_feature(CPU_FTR_NODSISRALIGN)) { |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 438 | unsigned long pc = regs->nip; |
| 439 | |
| 440 | if (cpu_has_feature(CPU_FTR_PPC_LE) && (regs->msr & MSR_LE)) |
| 441 | pc ^= 4; |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 442 | if (unlikely(__get_user_inatomic(instr, |
| 443 | (unsigned int __user *)pc))) |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 444 | return -EFAULT; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 445 | if (cpu_has_feature(CPU_FTR_REAL_LE) && (regs->msr & MSR_LE)) |
| 446 | instr = cpu_to_le32(instr); |
| 447 | dsisr = make_dsisr(instr); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 448 | } |
| 449 | |
| 450 | /* extract the operation and registers from the dsisr */ |
| 451 | reg = (dsisr >> 5) & 0x1f; /* source/dest register */ |
| 452 | areg = dsisr & 0x1f; /* register to update */ |
| 453 | instr = (dsisr >> 10) & 0x7f; |
| 454 | instr |= (dsisr >> 13) & 0x60; |
| 455 | |
| 456 | /* Lookup the operation in our table */ |
| 457 | nb = aligninfo[instr].len; |
| 458 | flags = aligninfo[instr].flags; |
| 459 | |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 460 | /* Byteswap little endian loads and stores */ |
| 461 | swiz = 0; |
| 462 | if (regs->msr & MSR_LE) { |
| 463 | flags ^= SW; |
| 464 | /* |
| 465 | * So-called "PowerPC little endian" mode works by |
| 466 | * swizzling addresses rather than by actually doing |
| 467 | * any byte-swapping. To emulate this, we XOR each |
| 468 | * byte address with 7. We also byte-swap, because |
| 469 | * the processor's address swizzling depends on the |
| 470 | * operand size (it xors the address with 7 for bytes, |
| 471 | * 6 for halfwords, 4 for words, 0 for doublewords) but |
| 472 | * we will xor with 7 and load/store each byte separately. |
| 473 | */ |
| 474 | if (cpu_has_feature(CPU_FTR_PPC_LE)) |
| 475 | swiz = 7; |
| 476 | } |
| 477 | |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 478 | /* DAR has the operand effective address */ |
| 479 | addr = (unsigned char __user *)regs->dar; |
| 480 | |
| 481 | /* A size of 0 indicates an instruction we don't support, with |
| 482 | * the exception of DCBZ which is handled as a special case here |
| 483 | */ |
| 484 | if (instr == DCBZ) |
| 485 | return emulate_dcbz(regs, addr); |
| 486 | if (unlikely(nb == 0)) |
| 487 | return 0; |
| 488 | |
| 489 | /* Load/Store Multiple instructions are handled in their own |
| 490 | * function |
| 491 | */ |
| 492 | if (flags & M) |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 493 | return emulate_multiple(regs, addr, reg, nb, |
| 494 | flags, instr, swiz); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 495 | |
| 496 | /* Verify the address of the operand */ |
| 497 | if (unlikely(user_mode(regs) && |
| 498 | !access_ok((flags & ST ? VERIFY_WRITE : VERIFY_READ), |
| 499 | addr, nb))) |
| 500 | return -EFAULT; |
| 501 | |
| 502 | /* Force the fprs into the save area so we can reference them */ |
| 503 | if (flags & F) { |
| 504 | /* userland only */ |
| 505 | if (unlikely(!user_mode(regs))) |
| 506 | return 0; |
| 507 | flush_fp_to_thread(current); |
| 508 | } |
| 509 | |
Paul Mackerras | c6d4267 | 2007-08-10 14:07:38 +1000 | [diff] [blame^] | 510 | /* Special case for 16-byte FP loads and stores */ |
| 511 | if (nb == 16) |
| 512 | return emulate_fp_pair(regs, addr, reg, flags); |
| 513 | |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 514 | /* If we are loading, get the data from user space, else |
| 515 | * get it from register values |
| 516 | */ |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 517 | if (!(flags & ST)) { |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 518 | data.ll = 0; |
| 519 | ret = 0; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 520 | p = (unsigned long) addr; |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 521 | switch (nb) { |
| 522 | case 8: |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 523 | ret |= __get_user_inatomic(data.v[0], SWIZ_PTR(p++)); |
| 524 | ret |= __get_user_inatomic(data.v[1], SWIZ_PTR(p++)); |
| 525 | ret |= __get_user_inatomic(data.v[2], SWIZ_PTR(p++)); |
| 526 | ret |= __get_user_inatomic(data.v[3], SWIZ_PTR(p++)); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 527 | case 4: |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 528 | ret |= __get_user_inatomic(data.v[4], SWIZ_PTR(p++)); |
| 529 | ret |= __get_user_inatomic(data.v[5], SWIZ_PTR(p++)); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 530 | case 2: |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 531 | ret |= __get_user_inatomic(data.v[6], SWIZ_PTR(p++)); |
| 532 | ret |= __get_user_inatomic(data.v[7], SWIZ_PTR(p++)); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 533 | if (unlikely(ret)) |
| 534 | return -EFAULT; |
| 535 | } |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 536 | } else if (flags & F) { |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 537 | data.dd = current->thread.fpr[reg]; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 538 | if (flags & S) { |
| 539 | /* Single-precision FP store requires conversion... */ |
| 540 | #ifdef CONFIG_PPC_FPU |
| 541 | preempt_disable(); |
| 542 | enable_kernel_fp(); |
| 543 | cvt_df(&data.dd, (float *)&data.v[4], ¤t->thread); |
| 544 | preempt_enable(); |
| 545 | #else |
| 546 | return 0; |
| 547 | #endif |
| 548 | } |
| 549 | } else |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 550 | data.ll = regs->gpr[reg]; |
| 551 | |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 552 | if (flags & SW) { |
| 553 | switch (nb) { |
| 554 | case 8: |
| 555 | SWAP(data.v[0], data.v[7]); |
| 556 | SWAP(data.v[1], data.v[6]); |
| 557 | SWAP(data.v[2], data.v[5]); |
| 558 | SWAP(data.v[3], data.v[4]); |
| 559 | break; |
| 560 | case 4: |
| 561 | SWAP(data.v[4], data.v[7]); |
| 562 | SWAP(data.v[5], data.v[6]); |
| 563 | break; |
| 564 | case 2: |
| 565 | SWAP(data.v[6], data.v[7]); |
| 566 | break; |
| 567 | } |
| 568 | } |
| 569 | |
| 570 | /* Perform other misc operations like sign extension |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 571 | * or floating point single precision conversion |
| 572 | */ |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 573 | switch (flags & ~(U|SW)) { |
Paul Mackerras | c6d4267 | 2007-08-10 14:07:38 +1000 | [diff] [blame^] | 574 | case LD+SE: /* sign extending integer loads */ |
| 575 | case LD+F+SE: /* sign extend for lfiwax */ |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 576 | if ( nb == 2 ) |
| 577 | data.ll = data.x16.low16; |
| 578 | else /* nb must be 4 */ |
| 579 | data.ll = data.x32.low32; |
| 580 | break; |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 581 | |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 582 | /* Single-precision FP load requires conversion... */ |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 583 | case LD+F+S: |
| 584 | #ifdef CONFIG_PPC_FPU |
| 585 | preempt_disable(); |
| 586 | enable_kernel_fp(); |
| 587 | cvt_fd((float *)&data.v[4], &data.dd, ¤t->thread); |
| 588 | preempt_enable(); |
| 589 | #else |
| 590 | return 0; |
| 591 | #endif |
| 592 | break; |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 593 | } |
| 594 | |
| 595 | /* Store result to memory or update registers */ |
| 596 | if (flags & ST) { |
| 597 | ret = 0; |
Paul Mackerras | fab5db9 | 2006-06-07 16:14:40 +1000 | [diff] [blame] | 598 | p = (unsigned long) addr; |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 599 | switch (nb) { |
| 600 | case 8: |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 601 | ret |= __put_user_inatomic(data.v[0], SWIZ_PTR(p++)); |
| 602 | ret |= __put_user_inatomic(data.v[1], SWIZ_PTR(p++)); |
| 603 | ret |= __put_user_inatomic(data.v[2], SWIZ_PTR(p++)); |
| 604 | ret |= __put_user_inatomic(data.v[3], SWIZ_PTR(p++)); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 605 | case 4: |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 606 | ret |= __put_user_inatomic(data.v[4], SWIZ_PTR(p++)); |
| 607 | ret |= __put_user_inatomic(data.v[5], SWIZ_PTR(p++)); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 608 | case 2: |
Benjamin Herrenschmidt | e4ee3891 | 2007-04-11 16:13:19 +1000 | [diff] [blame] | 609 | ret |= __put_user_inatomic(data.v[6], SWIZ_PTR(p++)); |
| 610 | ret |= __put_user_inatomic(data.v[7], SWIZ_PTR(p++)); |
Benjamin Herrenschmidt | 5daf907 | 2005-11-18 14:09:41 +1100 | [diff] [blame] | 611 | } |
| 612 | if (unlikely(ret)) |
| 613 | return -EFAULT; |
| 614 | } else if (flags & F) |
| 615 | current->thread.fpr[reg] = data.dd; |
| 616 | else |
| 617 | regs->gpr[reg] = data.ll; |
| 618 | |
| 619 | /* Update RA as needed */ |
| 620 | if (flags & U) |
| 621 | regs->gpr[areg] = regs->dar; |
| 622 | |
| 623 | return 1; |
| 624 | } |