Paul Mundt | 510c72ad | 2006-11-27 12:06:26 +0900 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * copy_page, __copy_user_page, __copy_user implementation of SuperH |
| 3 | * |
| 4 | * Copyright (C) 2001 Niibe Yutaka & Kaz Kojima |
| 5 | * Copyright (C) 2002 Toshinobu Sugioka |
Paul Mundt | 510c72ad | 2006-11-27 12:06:26 +0900 | [diff] [blame] | 6 | * Copyright (C) 2006 Paul Mundt |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | */ |
| 8 | #include <linux/linkage.h> |
Paul Mundt | 510c72ad | 2006-11-27 12:06:26 +0900 | [diff] [blame] | 9 | #include <asm/page.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | |
| 11 | /* |
| 12 | * copy_page_slow |
| 13 | * @to: P1 address |
| 14 | * @from: P1 address |
| 15 | * |
| 16 | * void copy_page_slow(void *to, void *from) |
| 17 | */ |
| 18 | |
| 19 | /* |
| 20 | * r0, r1, r2, r3, r4, r5, r6, r7 --- scratch |
Paul Mundt | 510c72ad | 2006-11-27 12:06:26 +0900 | [diff] [blame] | 21 | * r8 --- from + PAGE_SIZE |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 22 | * r9 --- not used |
| 23 | * r10 --- to |
| 24 | * r11 --- from |
| 25 | */ |
| 26 | ENTRY(copy_page_slow) |
| 27 | mov.l r8,@-r15 |
| 28 | mov.l r10,@-r15 |
| 29 | mov.l r11,@-r15 |
| 30 | mov r4,r10 |
| 31 | mov r5,r11 |
| 32 | mov r5,r8 |
Paul Mundt | 510c72ad | 2006-11-27 12:06:26 +0900 | [diff] [blame] | 33 | mov.l .Lpsz,r0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | add r0,r8 |
| 35 | ! |
| 36 | 1: mov.l @r11+,r0 |
| 37 | mov.l @r11+,r1 |
| 38 | mov.l @r11+,r2 |
| 39 | mov.l @r11+,r3 |
| 40 | mov.l @r11+,r4 |
| 41 | mov.l @r11+,r5 |
| 42 | mov.l @r11+,r6 |
| 43 | mov.l @r11+,r7 |
| 44 | #if defined(CONFIG_CPU_SH3) |
| 45 | mov.l r0,@r10 |
| 46 | #elif defined(CONFIG_CPU_SH4) |
| 47 | movca.l r0,@r10 |
| 48 | mov r10,r0 |
| 49 | #endif |
| 50 | add #32,r10 |
| 51 | mov.l r7,@-r10 |
| 52 | mov.l r6,@-r10 |
| 53 | mov.l r5,@-r10 |
| 54 | mov.l r4,@-r10 |
| 55 | mov.l r3,@-r10 |
| 56 | mov.l r2,@-r10 |
| 57 | mov.l r1,@-r10 |
| 58 | #if defined(CONFIG_CPU_SH4) |
| 59 | ocbwb @r0 |
| 60 | #endif |
| 61 | cmp/eq r11,r8 |
| 62 | bf/s 1b |
| 63 | add #28,r10 |
| 64 | ! |
| 65 | mov.l @r15+,r11 |
| 66 | mov.l @r15+,r10 |
| 67 | mov.l @r15+,r8 |
| 68 | rts |
| 69 | nop |
| 70 | |
| 71 | #if defined(CONFIG_CPU_SH4) |
| 72 | /* |
| 73 | * __copy_user_page |
| 74 | * @to: P1 address (with same color) |
| 75 | * @from: P1 address |
| 76 | * @orig_to: P1 address |
| 77 | * |
| 78 | * void __copy_user_page(void *to, void *from, void *orig_to) |
| 79 | */ |
| 80 | |
| 81 | /* |
| 82 | * r0, r1, r2, r3, r4, r5, r6, r7 --- scratch |
Paul Mundt | 510c72ad | 2006-11-27 12:06:26 +0900 | [diff] [blame] | 83 | * r8 --- from + PAGE_SIZE |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | * r9 --- orig_to |
| 85 | * r10 --- to |
| 86 | * r11 --- from |
| 87 | */ |
| 88 | ENTRY(__copy_user_page) |
| 89 | mov.l r8,@-r15 |
| 90 | mov.l r9,@-r15 |
| 91 | mov.l r10,@-r15 |
| 92 | mov.l r11,@-r15 |
| 93 | mov r4,r10 |
| 94 | mov r5,r11 |
| 95 | mov r6,r9 |
| 96 | mov r5,r8 |
Paul Mundt | 510c72ad | 2006-11-27 12:06:26 +0900 | [diff] [blame] | 97 | mov.l .Lpsz,r0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 98 | add r0,r8 |
| 99 | ! |
| 100 | 1: ocbi @r9 |
| 101 | add #32,r9 |
| 102 | mov.l @r11+,r0 |
| 103 | mov.l @r11+,r1 |
| 104 | mov.l @r11+,r2 |
| 105 | mov.l @r11+,r3 |
| 106 | mov.l @r11+,r4 |
| 107 | mov.l @r11+,r5 |
| 108 | mov.l @r11+,r6 |
| 109 | mov.l @r11+,r7 |
| 110 | movca.l r0,@r10 |
| 111 | mov r10,r0 |
| 112 | add #32,r10 |
| 113 | mov.l r7,@-r10 |
| 114 | mov.l r6,@-r10 |
| 115 | mov.l r5,@-r10 |
| 116 | mov.l r4,@-r10 |
| 117 | mov.l r3,@-r10 |
| 118 | mov.l r2,@-r10 |
| 119 | mov.l r1,@-r10 |
| 120 | ocbwb @r0 |
| 121 | cmp/eq r11,r8 |
| 122 | bf/s 1b |
| 123 | add #28,r10 |
| 124 | ! |
| 125 | mov.l @r15+,r11 |
| 126 | mov.l @r15+,r10 |
| 127 | mov.l @r15+,r9 |
| 128 | mov.l @r15+,r8 |
| 129 | rts |
| 130 | nop |
| 131 | #endif |
Paul Mundt | 0d08b5f | 2007-05-14 10:49:40 +0900 | [diff] [blame] | 132 | .align 2 |
Paul Mundt | 510c72ad | 2006-11-27 12:06:26 +0900 | [diff] [blame] | 133 | .Lpsz: .long PAGE_SIZE |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | /* |
| 135 | * __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n); |
| 136 | * Return the number of bytes NOT copied |
| 137 | */ |
| 138 | #define EX(...) \ |
| 139 | 9999: __VA_ARGS__ ; \ |
| 140 | .section __ex_table, "a"; \ |
| 141 | .long 9999b, 6000f ; \ |
| 142 | .previous |
| 143 | ENTRY(__copy_user) |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 144 | ! Check if small number of bytes |
| 145 | mov #11,r0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | mov r4,r3 |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 147 | cmp/gt r0,r6 ! r6 (len) > r0 (11) |
| 148 | bf/s .L_cleanup_loop_no_pop |
| 149 | add r6,r3 ! last destination address |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 150 | |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 151 | ! Calculate bytes needed to align to src |
| 152 | mov.l r11,@-r15 |
| 153 | neg r5,r0 |
| 154 | mov.l r10,@-r15 |
| 155 | add #4,r0 |
| 156 | mov.l r9,@-r15 |
| 157 | and #3,r0 |
| 158 | mov.l r8,@-r15 |
| 159 | tst r0,r0 |
| 160 | bt 2f |
| 161 | |
| 162 | 1: |
| 163 | ! Copy bytes to long word align src |
| 164 | EX( mov.b @r5+,r1 ) |
| 165 | dt r0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | add #-1,r6 |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 167 | EX( mov.b r1,@r4 ) |
| 168 | bf/s 1b |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 169 | add #1,r4 |
| 170 | |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 171 | ! Jump to appropriate routine depending on dest |
| 172 | 2: mov #3,r1 |
| 173 | mov r6, r2 |
| 174 | and r4,r1 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | shlr2 r2 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 176 | shll2 r1 |
| 177 | mova .L_jump_tbl,r0 |
| 178 | mov.l @(r0,r1),r1 |
| 179 | jmp @r1 |
| 180 | nop |
| 181 | |
| 182 | .align 2 |
| 183 | .L_jump_tbl: |
| 184 | .long .L_dest00 |
| 185 | .long .L_dest01 |
| 186 | .long .L_dest10 |
| 187 | .long .L_dest11 |
| 188 | |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 189 | /* |
| 190 | * Come here if there are less than 12 bytes to copy |
| 191 | * |
| 192 | * Keep the branch target close, so the bf/s callee doesn't overflow |
| 193 | * and result in a more expensive branch being inserted. This is the |
| 194 | * fast-path for small copies, the jump via the jump table will hit the |
| 195 | * default slow-path cleanup. -PFM. |
| 196 | */ |
| 197 | .L_cleanup_loop_no_pop: |
| 198 | tst r6,r6 ! Check explicitly for zero |
| 199 | bt 1f |
| 200 | |
| 201 | 2: |
| 202 | EX( mov.b @r5+,r0 ) |
| 203 | dt r6 |
| 204 | EX( mov.b r0,@r4 ) |
| 205 | bf/s 2b |
| 206 | add #1,r4 |
| 207 | |
| 208 | 1: mov #0,r0 ! normal return |
| 209 | 5000: |
| 210 | |
| 211 | # Exception handler: |
| 212 | .section .fixup, "ax" |
| 213 | 6000: |
| 214 | mov.l 8000f,r1 |
| 215 | mov r3,r0 |
| 216 | jmp @r1 |
| 217 | sub r4,r0 |
| 218 | .align 2 |
| 219 | 8000: .long 5000b |
| 220 | |
| 221 | .previous |
| 222 | rts |
| 223 | nop |
| 224 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 225 | ! Destination = 00 |
| 226 | |
| 227 | .L_dest00: |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 228 | ! Skip the large copy for small transfers |
| 229 | mov #(32+32-4), r0 |
| 230 | cmp/gt r6, r0 ! r0 (60) > r6 (len) |
| 231 | bt 1f |
| 232 | |
| 233 | ! Align dest to a 32 byte boundary |
| 234 | neg r4,r0 |
| 235 | add #0x20, r0 |
| 236 | and #0x1f, r0 |
| 237 | tst r0, r0 |
| 238 | bt 2f |
| 239 | |
| 240 | sub r0, r6 |
| 241 | shlr2 r0 |
| 242 | 3: |
| 243 | EX( mov.l @r5+,r1 ) |
| 244 | dt r0 |
| 245 | EX( mov.l r1,@r4 ) |
| 246 | bf/s 3b |
| 247 | add #4,r4 |
| 248 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 249 | 2: |
| 250 | EX( mov.l @r5+,r0 ) |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 251 | EX( mov.l @r5+,r1 ) |
| 252 | EX( mov.l @r5+,r2 ) |
| 253 | EX( mov.l @r5+,r7 ) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 254 | EX( mov.l @r5+,r8 ) |
| 255 | EX( mov.l @r5+,r9 ) |
| 256 | EX( mov.l @r5+,r10 ) |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 257 | EX( mov.l @r5+,r11 ) |
| 258 | EX( movca.l r0,@r4 ) |
| 259 | add #-32, r6 |
| 260 | EX( mov.l r1,@(4,r4) ) |
| 261 | mov #32, r0 |
| 262 | EX( mov.l r2,@(8,r4) ) |
| 263 | cmp/gt r6, r0 ! r0 (32) > r6 (len) |
| 264 | EX( mov.l r7,@(12,r4) ) |
| 265 | EX( mov.l r8,@(16,r4) ) |
| 266 | EX( mov.l r9,@(20,r4) ) |
| 267 | EX( mov.l r10,@(24,r4) ) |
| 268 | EX( mov.l r11,@(28,r4) ) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 269 | bf/s 2b |
| 270 | add #32,r4 |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 271 | |
| 272 | 1: mov r6, r0 |
| 273 | shlr2 r0 |
| 274 | tst r0, r0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 275 | bt .L_cleanup |
| 276 | 1: |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 277 | EX( mov.l @r5+,r1 ) |
| 278 | dt r0 |
| 279 | EX( mov.l r1,@r4 ) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 280 | bf/s 1b |
| 281 | add #4,r4 |
| 282 | |
| 283 | bra .L_cleanup |
| 284 | nop |
| 285 | |
| 286 | ! Destination = 10 |
| 287 | |
| 288 | .L_dest10: |
| 289 | mov r2,r7 |
| 290 | shlr2 r7 |
| 291 | shlr r7 |
| 292 | tst r7,r7 |
| 293 | mov #7,r0 |
| 294 | bt/s 1f |
| 295 | and r0,r2 |
| 296 | 2: |
| 297 | dt r7 |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 298 | #ifdef CONFIG_CPU_LITTLE_ENDIAN |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | EX( mov.l @r5+,r0 ) |
| 300 | EX( mov.l @r5+,r1 ) |
| 301 | EX( mov.l @r5+,r8 ) |
| 302 | EX( mov.l @r5+,r9 ) |
| 303 | EX( mov.l @r5+,r10 ) |
| 304 | EX( mov.w r0,@r4 ) |
| 305 | add #2,r4 |
| 306 | xtrct r1,r0 |
| 307 | xtrct r8,r1 |
| 308 | xtrct r9,r8 |
| 309 | xtrct r10,r9 |
| 310 | |
| 311 | EX( mov.l r0,@r4 ) |
| 312 | EX( mov.l r1,@(4,r4) ) |
| 313 | EX( mov.l r8,@(8,r4) ) |
| 314 | EX( mov.l r9,@(12,r4) ) |
| 315 | |
| 316 | EX( mov.l @r5+,r1 ) |
| 317 | EX( mov.l @r5+,r8 ) |
| 318 | EX( mov.l @r5+,r0 ) |
| 319 | xtrct r1,r10 |
| 320 | xtrct r8,r1 |
| 321 | xtrct r0,r8 |
| 322 | shlr16 r0 |
| 323 | EX( mov.l r10,@(16,r4) ) |
| 324 | EX( mov.l r1,@(20,r4) ) |
| 325 | EX( mov.l r8,@(24,r4) ) |
| 326 | EX( mov.w r0,@(28,r4) ) |
| 327 | bf/s 2b |
| 328 | add #30,r4 |
| 329 | #else |
| 330 | EX( mov.l @(28,r5),r0 ) |
| 331 | EX( mov.l @(24,r5),r8 ) |
| 332 | EX( mov.l @(20,r5),r9 ) |
| 333 | EX( mov.l @(16,r5),r10 ) |
| 334 | EX( mov.w r0,@(30,r4) ) |
| 335 | add #-2,r4 |
| 336 | xtrct r8,r0 |
| 337 | xtrct r9,r8 |
| 338 | xtrct r10,r9 |
| 339 | EX( mov.l r0,@(28,r4) ) |
| 340 | EX( mov.l r8,@(24,r4) ) |
| 341 | EX( mov.l r9,@(20,r4) ) |
| 342 | |
| 343 | EX( mov.l @(12,r5),r0 ) |
| 344 | EX( mov.l @(8,r5),r8 ) |
| 345 | xtrct r0,r10 |
| 346 | EX( mov.l @(4,r5),r9 ) |
| 347 | mov.l r10,@(16,r4) |
| 348 | EX( mov.l @r5,r10 ) |
| 349 | xtrct r8,r0 |
| 350 | xtrct r9,r8 |
| 351 | xtrct r10,r9 |
| 352 | EX( mov.l r0,@(12,r4) ) |
| 353 | EX( mov.l r8,@(8,r4) ) |
| 354 | swap.w r10,r0 |
| 355 | EX( mov.l r9,@(4,r4) ) |
| 356 | EX( mov.w r0,@(2,r4) ) |
| 357 | |
| 358 | add #32,r5 |
| 359 | bf/s 2b |
| 360 | add #34,r4 |
| 361 | #endif |
| 362 | tst r2,r2 |
| 363 | bt .L_cleanup |
| 364 | |
| 365 | 1: ! Read longword, write two words per iteration |
| 366 | EX( mov.l @r5+,r0 ) |
| 367 | dt r2 |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 368 | #ifdef CONFIG_CPU_LITTLE_ENDIAN |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 369 | EX( mov.w r0,@r4 ) |
| 370 | shlr16 r0 |
| 371 | EX( mov.w r0,@(2,r4) ) |
| 372 | #else |
| 373 | EX( mov.w r0,@(2,r4) ) |
| 374 | shlr16 r0 |
| 375 | EX( mov.w r0,@r4 ) |
| 376 | #endif |
| 377 | bf/s 1b |
| 378 | add #4,r4 |
| 379 | |
| 380 | bra .L_cleanup |
| 381 | nop |
| 382 | |
| 383 | ! Destination = 01 or 11 |
| 384 | |
| 385 | .L_dest01: |
| 386 | .L_dest11: |
| 387 | ! Read longword, write byte, word, byte per iteration |
| 388 | EX( mov.l @r5+,r0 ) |
| 389 | dt r2 |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 390 | #ifdef CONFIG_CPU_LITTLE_ENDIAN |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 391 | EX( mov.b r0,@r4 ) |
| 392 | shlr8 r0 |
| 393 | add #1,r4 |
| 394 | EX( mov.w r0,@r4 ) |
| 395 | shlr16 r0 |
| 396 | EX( mov.b r0,@(2,r4) ) |
| 397 | bf/s .L_dest01 |
| 398 | add #3,r4 |
| 399 | #else |
| 400 | EX( mov.b r0,@(3,r4) ) |
| 401 | shlr8 r0 |
| 402 | swap.w r0,r7 |
| 403 | EX( mov.b r7,@r4 ) |
| 404 | add #1,r4 |
| 405 | EX( mov.w r0,@r4 ) |
| 406 | bf/s .L_dest01 |
| 407 | add #3,r4 |
| 408 | #endif |
| 409 | |
| 410 | ! Cleanup last few bytes |
| 411 | .L_cleanup: |
| 412 | mov r6,r0 |
| 413 | and #3,r0 |
| 414 | tst r0,r0 |
| 415 | bt .L_exit |
| 416 | mov r0,r6 |
| 417 | |
| 418 | .L_cleanup_loop: |
| 419 | EX( mov.b @r5+,r0 ) |
| 420 | dt r6 |
| 421 | EX( mov.b r0,@r4 ) |
| 422 | bf/s .L_cleanup_loop |
| 423 | add #1,r4 |
| 424 | |
| 425 | .L_exit: |
| 426 | mov #0,r0 ! normal return |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 427 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 428 | 5000: |
| 429 | |
| 430 | # Exception handler: |
| 431 | .section .fixup, "ax" |
| 432 | 6000: |
| 433 | mov.l 8000f,r1 |
| 434 | mov r3,r0 |
| 435 | jmp @r1 |
| 436 | sub r4,r0 |
| 437 | .align 2 |
| 438 | 8000: .long 5000b |
| 439 | |
| 440 | .previous |
| 441 | mov.l @r15+,r8 |
| 442 | mov.l @r15+,r9 |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 443 | mov.l @r15+,r10 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 444 | rts |
Stuart Menefy | 023ef18 | 2007-09-28 12:36:35 +0900 | [diff] [blame^] | 445 | mov.l @r15+,r11 |