blob: a81dbdb05596993a580e9fb69d876133c1990d9c [file] [log] [blame]
Paul Mundt510c72ad2006-11-27 12:06:26 +09001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * copy_page, __copy_user_page, __copy_user implementation of SuperH
3 *
4 * Copyright (C) 2001 Niibe Yutaka & Kaz Kojima
5 * Copyright (C) 2002 Toshinobu Sugioka
Paul Mundt510c72ad2006-11-27 12:06:26 +09006 * Copyright (C) 2006 Paul Mundt
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 */
8#include <linux/linkage.h>
Paul Mundt510c72ad2006-11-27 12:06:26 +09009#include <asm/page.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010
11/*
12 * copy_page_slow
13 * @to: P1 address
14 * @from: P1 address
15 *
16 * void copy_page_slow(void *to, void *from)
17 */
18
19/*
20 * r0, r1, r2, r3, r4, r5, r6, r7 --- scratch
Paul Mundt510c72ad2006-11-27 12:06:26 +090021 * r8 --- from + PAGE_SIZE
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * r9 --- not used
23 * r10 --- to
24 * r11 --- from
25 */
26ENTRY(copy_page_slow)
27 mov.l r8,@-r15
28 mov.l r10,@-r15
29 mov.l r11,@-r15
30 mov r4,r10
31 mov r5,r11
32 mov r5,r8
Paul Mundt510c72ad2006-11-27 12:06:26 +090033 mov.l .Lpsz,r0
Linus Torvalds1da177e2005-04-16 15:20:36 -070034 add r0,r8
35 !
361: mov.l @r11+,r0
37 mov.l @r11+,r1
38 mov.l @r11+,r2
39 mov.l @r11+,r3
40 mov.l @r11+,r4
41 mov.l @r11+,r5
42 mov.l @r11+,r6
43 mov.l @r11+,r7
44#if defined(CONFIG_CPU_SH3)
45 mov.l r0,@r10
46#elif defined(CONFIG_CPU_SH4)
47 movca.l r0,@r10
48 mov r10,r0
49#endif
50 add #32,r10
51 mov.l r7,@-r10
52 mov.l r6,@-r10
53 mov.l r5,@-r10
54 mov.l r4,@-r10
55 mov.l r3,@-r10
56 mov.l r2,@-r10
57 mov.l r1,@-r10
58#if defined(CONFIG_CPU_SH4)
59 ocbwb @r0
60#endif
61 cmp/eq r11,r8
62 bf/s 1b
63 add #28,r10
64 !
65 mov.l @r15+,r11
66 mov.l @r15+,r10
67 mov.l @r15+,r8
68 rts
69 nop
70
71#if defined(CONFIG_CPU_SH4)
72/*
73 * __copy_user_page
74 * @to: P1 address (with same color)
75 * @from: P1 address
76 * @orig_to: P1 address
77 *
78 * void __copy_user_page(void *to, void *from, void *orig_to)
79 */
80
81/*
82 * r0, r1, r2, r3, r4, r5, r6, r7 --- scratch
Paul Mundt510c72ad2006-11-27 12:06:26 +090083 * r8 --- from + PAGE_SIZE
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 * r9 --- orig_to
85 * r10 --- to
86 * r11 --- from
87 */
88ENTRY(__copy_user_page)
89 mov.l r8,@-r15
90 mov.l r9,@-r15
91 mov.l r10,@-r15
92 mov.l r11,@-r15
93 mov r4,r10
94 mov r5,r11
95 mov r6,r9
96 mov r5,r8
Paul Mundt510c72ad2006-11-27 12:06:26 +090097 mov.l .Lpsz,r0
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 add r0,r8
99 !
1001: ocbi @r9
101 add #32,r9
102 mov.l @r11+,r0
103 mov.l @r11+,r1
104 mov.l @r11+,r2
105 mov.l @r11+,r3
106 mov.l @r11+,r4
107 mov.l @r11+,r5
108 mov.l @r11+,r6
109 mov.l @r11+,r7
110 movca.l r0,@r10
111 mov r10,r0
112 add #32,r10
113 mov.l r7,@-r10
114 mov.l r6,@-r10
115 mov.l r5,@-r10
116 mov.l r4,@-r10
117 mov.l r3,@-r10
118 mov.l r2,@-r10
119 mov.l r1,@-r10
120 ocbwb @r0
121 cmp/eq r11,r8
122 bf/s 1b
123 add #28,r10
124 !
125 mov.l @r15+,r11
126 mov.l @r15+,r10
127 mov.l @r15+,r9
128 mov.l @r15+,r8
129 rts
130 nop
131#endif
Paul Mundt0d08b5f2007-05-14 10:49:40 +0900132 .align 2
Paul Mundt510c72ad2006-11-27 12:06:26 +0900133.Lpsz: .long PAGE_SIZE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134/*
135 * __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n);
136 * Return the number of bytes NOT copied
137 */
138#define EX(...) \
139 9999: __VA_ARGS__ ; \
140 .section __ex_table, "a"; \
141 .long 9999b, 6000f ; \
142 .previous
143ENTRY(__copy_user)
Stuart Menefy023ef182007-09-28 12:36:35 +0900144 ! Check if small number of bytes
145 mov #11,r0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 mov r4,r3
Stuart Menefy023ef182007-09-28 12:36:35 +0900147 cmp/gt r0,r6 ! r6 (len) > r0 (11)
148 bf/s .L_cleanup_loop_no_pop
149 add r6,r3 ! last destination address
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150
Stuart Menefy023ef182007-09-28 12:36:35 +0900151 ! Calculate bytes needed to align to src
152 mov.l r11,@-r15
153 neg r5,r0
154 mov.l r10,@-r15
155 add #4,r0
156 mov.l r9,@-r15
157 and #3,r0
158 mov.l r8,@-r15
159 tst r0,r0
160 bt 2f
161
1621:
163 ! Copy bytes to long word align src
164EX( mov.b @r5+,r1 )
165 dt r0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 add #-1,r6
Stuart Menefy023ef182007-09-28 12:36:35 +0900167EX( mov.b r1,@r4 )
168 bf/s 1b
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 add #1,r4
170
Stuart Menefy023ef182007-09-28 12:36:35 +0900171 ! Jump to appropriate routine depending on dest
1722: mov #3,r1
173 mov r6, r2
174 and r4,r1
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 shlr2 r2
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176 shll2 r1
177 mova .L_jump_tbl,r0
178 mov.l @(r0,r1),r1
179 jmp @r1
180 nop
181
182 .align 2
183.L_jump_tbl:
184 .long .L_dest00
185 .long .L_dest01
186 .long .L_dest10
187 .long .L_dest11
188
Stuart Menefy023ef182007-09-28 12:36:35 +0900189/*
190 * Come here if there are less than 12 bytes to copy
191 *
192 * Keep the branch target close, so the bf/s callee doesn't overflow
193 * and result in a more expensive branch being inserted. This is the
194 * fast-path for small copies, the jump via the jump table will hit the
195 * default slow-path cleanup. -PFM.
196 */
197.L_cleanup_loop_no_pop:
198 tst r6,r6 ! Check explicitly for zero
199 bt 1f
200
2012:
202EX( mov.b @r5+,r0 )
203 dt r6
204EX( mov.b r0,@r4 )
205 bf/s 2b
206 add #1,r4
207
2081: mov #0,r0 ! normal return
2095000:
210
211# Exception handler:
212.section .fixup, "ax"
2136000:
214 mov.l 8000f,r1
215 mov r3,r0
216 jmp @r1
217 sub r4,r0
218 .align 2
2198000: .long 5000b
220
221.previous
222 rts
223 nop
224
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225! Destination = 00
226
227.L_dest00:
Stuart Menefy023ef182007-09-28 12:36:35 +0900228 ! Skip the large copy for small transfers
229 mov #(32+32-4), r0
230 cmp/gt r6, r0 ! r0 (60) > r6 (len)
231 bt 1f
232
233 ! Align dest to a 32 byte boundary
234 neg r4,r0
235 add #0x20, r0
236 and #0x1f, r0
237 tst r0, r0
238 bt 2f
239
240 sub r0, r6
241 shlr2 r0
2423:
243EX( mov.l @r5+,r1 )
244 dt r0
245EX( mov.l r1,@r4 )
246 bf/s 3b
247 add #4,r4
248
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492:
250EX( mov.l @r5+,r0 )
Stuart Menefy023ef182007-09-28 12:36:35 +0900251EX( mov.l @r5+,r1 )
252EX( mov.l @r5+,r2 )
253EX( mov.l @r5+,r7 )
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254EX( mov.l @r5+,r8 )
255EX( mov.l @r5+,r9 )
256EX( mov.l @r5+,r10 )
Stuart Menefy023ef182007-09-28 12:36:35 +0900257EX( mov.l @r5+,r11 )
258EX( movca.l r0,@r4 )
259 add #-32, r6
260EX( mov.l r1,@(4,r4) )
261 mov #32, r0
262EX( mov.l r2,@(8,r4) )
263 cmp/gt r6, r0 ! r0 (32) > r6 (len)
264EX( mov.l r7,@(12,r4) )
265EX( mov.l r8,@(16,r4) )
266EX( mov.l r9,@(20,r4) )
267EX( mov.l r10,@(24,r4) )
268EX( mov.l r11,@(28,r4) )
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 bf/s 2b
270 add #32,r4
Stuart Menefy023ef182007-09-28 12:36:35 +0900271
2721: mov r6, r0
273 shlr2 r0
274 tst r0, r0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 bt .L_cleanup
2761:
Stuart Menefy023ef182007-09-28 12:36:35 +0900277EX( mov.l @r5+,r1 )
278 dt r0
279EX( mov.l r1,@r4 )
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 bf/s 1b
281 add #4,r4
282
283 bra .L_cleanup
284 nop
285
286! Destination = 10
287
288.L_dest10:
289 mov r2,r7
290 shlr2 r7
291 shlr r7
292 tst r7,r7
293 mov #7,r0
294 bt/s 1f
295 and r0,r2
2962:
297 dt r7
Stuart Menefy023ef182007-09-28 12:36:35 +0900298#ifdef CONFIG_CPU_LITTLE_ENDIAN
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299EX( mov.l @r5+,r0 )
300EX( mov.l @r5+,r1 )
301EX( mov.l @r5+,r8 )
302EX( mov.l @r5+,r9 )
303EX( mov.l @r5+,r10 )
304EX( mov.w r0,@r4 )
305 add #2,r4
306 xtrct r1,r0
307 xtrct r8,r1
308 xtrct r9,r8
309 xtrct r10,r9
310
311EX( mov.l r0,@r4 )
312EX( mov.l r1,@(4,r4) )
313EX( mov.l r8,@(8,r4) )
314EX( mov.l r9,@(12,r4) )
315
316EX( mov.l @r5+,r1 )
317EX( mov.l @r5+,r8 )
318EX( mov.l @r5+,r0 )
319 xtrct r1,r10
320 xtrct r8,r1
321 xtrct r0,r8
322 shlr16 r0
323EX( mov.l r10,@(16,r4) )
324EX( mov.l r1,@(20,r4) )
325EX( mov.l r8,@(24,r4) )
326EX( mov.w r0,@(28,r4) )
327 bf/s 2b
328 add #30,r4
329#else
330EX( mov.l @(28,r5),r0 )
331EX( mov.l @(24,r5),r8 )
332EX( mov.l @(20,r5),r9 )
333EX( mov.l @(16,r5),r10 )
334EX( mov.w r0,@(30,r4) )
335 add #-2,r4
336 xtrct r8,r0
337 xtrct r9,r8
338 xtrct r10,r9
339EX( mov.l r0,@(28,r4) )
340EX( mov.l r8,@(24,r4) )
341EX( mov.l r9,@(20,r4) )
342
343EX( mov.l @(12,r5),r0 )
344EX( mov.l @(8,r5),r8 )
345 xtrct r0,r10
346EX( mov.l @(4,r5),r9 )
347 mov.l r10,@(16,r4)
348EX( mov.l @r5,r10 )
349 xtrct r8,r0
350 xtrct r9,r8
351 xtrct r10,r9
352EX( mov.l r0,@(12,r4) )
353EX( mov.l r8,@(8,r4) )
354 swap.w r10,r0
355EX( mov.l r9,@(4,r4) )
356EX( mov.w r0,@(2,r4) )
357
358 add #32,r5
359 bf/s 2b
360 add #34,r4
361#endif
362 tst r2,r2
363 bt .L_cleanup
364
3651: ! Read longword, write two words per iteration
366EX( mov.l @r5+,r0 )
367 dt r2
Stuart Menefy023ef182007-09-28 12:36:35 +0900368#ifdef CONFIG_CPU_LITTLE_ENDIAN
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369EX( mov.w r0,@r4 )
370 shlr16 r0
371EX( mov.w r0,@(2,r4) )
372#else
373EX( mov.w r0,@(2,r4) )
374 shlr16 r0
375EX( mov.w r0,@r4 )
376#endif
377 bf/s 1b
378 add #4,r4
379
380 bra .L_cleanup
381 nop
382
383! Destination = 01 or 11
384
385.L_dest01:
386.L_dest11:
387 ! Read longword, write byte, word, byte per iteration
388EX( mov.l @r5+,r0 )
389 dt r2
Stuart Menefy023ef182007-09-28 12:36:35 +0900390#ifdef CONFIG_CPU_LITTLE_ENDIAN
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391EX( mov.b r0,@r4 )
392 shlr8 r0
393 add #1,r4
394EX( mov.w r0,@r4 )
395 shlr16 r0
396EX( mov.b r0,@(2,r4) )
397 bf/s .L_dest01
398 add #3,r4
399#else
400EX( mov.b r0,@(3,r4) )
401 shlr8 r0
402 swap.w r0,r7
403EX( mov.b r7,@r4 )
404 add #1,r4
405EX( mov.w r0,@r4 )
406 bf/s .L_dest01
407 add #3,r4
408#endif
409
410! Cleanup last few bytes
411.L_cleanup:
412 mov r6,r0
413 and #3,r0
414 tst r0,r0
415 bt .L_exit
416 mov r0,r6
417
418.L_cleanup_loop:
419EX( mov.b @r5+,r0 )
420 dt r6
421EX( mov.b r0,@r4 )
422 bf/s .L_cleanup_loop
423 add #1,r4
424
425.L_exit:
426 mov #0,r0 ! normal return
Stuart Menefy023ef182007-09-28 12:36:35 +0900427
Linus Torvalds1da177e2005-04-16 15:20:36 -07004285000:
429
430# Exception handler:
431.section .fixup, "ax"
4326000:
433 mov.l 8000f,r1
434 mov r3,r0
435 jmp @r1
436 sub r4,r0
437 .align 2
4388000: .long 5000b
439
440.previous
441 mov.l @r15+,r8
442 mov.l @r15+,r9
Stuart Menefy023ef182007-09-28 12:36:35 +0900443 mov.l @r15+,r10
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 rts
Stuart Menefy023ef182007-09-28 12:36:35 +0900445 mov.l @r15+,r11