| Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * |
| 3 | * Optimized version of the standard copy_page() function |
| 4 | * |
| 5 | * Inputs: |
| 6 | * in0: address of target page |
| 7 | * in1: address of source page |
| 8 | * Output: |
| 9 | * no return value |
| 10 | * |
| 11 | * Copyright (C) 1999, 2001 Hewlett-Packard Co |
| 12 | * Stephane Eranian <eranian@hpl.hp.com> |
| 13 | * David Mosberger <davidm@hpl.hp.com> |
| 14 | * |
| 15 | * 4/06/01 davidm Tuned to make it perform well both for cached and uncached copies. |
| 16 | */ |
| 17 | #include <asm/asmmacro.h> |
| 18 | #include <asm/page.h> |
| 19 | |
| 20 | #define PIPE_DEPTH 3 |
| 21 | #define EPI p[PIPE_DEPTH-1] |
| 22 | |
| 23 | #define lcount r16 |
| 24 | #define saved_pr r17 |
| 25 | #define saved_lc r18 |
| 26 | #define saved_pfs r19 |
| 27 | #define src1 r20 |
| 28 | #define src2 r21 |
| 29 | #define tgt1 r22 |
| 30 | #define tgt2 r23 |
| 31 | #define srcf r24 |
| 32 | #define tgtf r25 |
| 33 | #define tgt_last r26 |
| 34 | |
| 35 | #define Nrot ((8*PIPE_DEPTH+7)&~7) |
| 36 | |
| 37 | GLOBAL_ENTRY(copy_page) |
| 38 | .prologue |
| 39 | .save ar.pfs, saved_pfs |
| 40 | alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot |
| 41 | |
| 42 | .rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \ |
| 43 | t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH] |
| 44 | .rotp p[PIPE_DEPTH] |
| 45 | |
| 46 | .save ar.lc, saved_lc |
| 47 | mov saved_lc=ar.lc |
| 48 | mov ar.ec=PIPE_DEPTH |
| 49 | |
| 50 | mov lcount=PAGE_SIZE/64-1 |
| 51 | .save pr, saved_pr |
| 52 | mov saved_pr=pr |
| 53 | mov pr.rot=1<<16 |
| 54 | |
| 55 | .body |
| 56 | |
| 57 | mov src1=in1 |
| 58 | adds src2=8,in1 |
| 59 | mov tgt_last = PAGE_SIZE |
| 60 | ;; |
| 61 | adds tgt2=8,in0 |
| 62 | add srcf=512,in1 |
| 63 | mov ar.lc=lcount |
| 64 | mov tgt1=in0 |
| 65 | add tgtf=512,in0 |
| 66 | add tgt_last = tgt_last, in0 |
| 67 | ;; |
| 68 | 1: |
| 69 | (p[0]) ld8 t1[0]=[src1],16 |
| 70 | (EPI) st8 [tgt1]=t1[PIPE_DEPTH-1],16 |
| 71 | (p[0]) ld8 t2[0]=[src2],16 |
| 72 | (EPI) st8 [tgt2]=t2[PIPE_DEPTH-1],16 |
| 73 | cmp.ltu p6,p0 = tgtf, tgt_last |
| 74 | ;; |
| 75 | (p[0]) ld8 t3[0]=[src1],16 |
| 76 | (EPI) st8 [tgt1]=t3[PIPE_DEPTH-1],16 |
| 77 | (p[0]) ld8 t4[0]=[src2],16 |
| 78 | (EPI) st8 [tgt2]=t4[PIPE_DEPTH-1],16 |
| 79 | ;; |
| 80 | (p[0]) ld8 t5[0]=[src1],16 |
| 81 | (EPI) st8 [tgt1]=t5[PIPE_DEPTH-1],16 |
| 82 | (p[0]) ld8 t6[0]=[src2],16 |
| 83 | (EPI) st8 [tgt2]=t6[PIPE_DEPTH-1],16 |
| 84 | ;; |
| 85 | (p[0]) ld8 t7[0]=[src1],16 |
| 86 | (EPI) st8 [tgt1]=t7[PIPE_DEPTH-1],16 |
| 87 | (p[0]) ld8 t8[0]=[src2],16 |
| 88 | (EPI) st8 [tgt2]=t8[PIPE_DEPTH-1],16 |
| 89 | |
| 90 | (p6) lfetch [srcf], 64 |
| 91 | (p6) lfetch [tgtf], 64 |
| 92 | br.ctop.sptk.few 1b |
| 93 | ;; |
| 94 | mov pr=saved_pr,0xffffffffffff0000 // restore predicates |
| 95 | mov ar.pfs=saved_pfs |
| 96 | mov ar.lc=saved_lc |
| 97 | br.ret.sptk.many rp |
| 98 | END(copy_page) |