Gitiles
Code Review Sign In
review.evervolv.com / android_kernel_oneplus_msm8996 / 854af9578cb84e4ca3cb1551a6be40c4e81bb455 / . / arch / sparc64 / lib / copy_page.S
blob: 23ebf2c970b71d950c2bea623e60b58c36b7b46c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]1/* clear_page.S: UltraSparc optimized copy page.
2 *
3 * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com)
4 * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
5 */
6
7#include <asm/visasm.h>
8#include <asm/thread_info.h>
9#include <asm/page.h>
10#include <asm/pgtable.h>
11#include <asm/spitfire.h>
12#include <asm/head.h>
13
14 /* What we used to do was lock a TLB entry into a specific
15 * TLB slot, clear the page with interrupts disabled, then
16 * restore the original TLB entry. This was great for
17 * disturbing the TLB as little as possible, but it meant
18 * we had to keep interrupts disabled for a long time.
19 *
20 * Now, we simply use the normal TLB loading mechanism,
21 * and this makes the cpu choose a slot all by itself.
22 * Then we do a normal TLB flush on exit. We need only
23 * disable preemption during the clear.
24 */
25
26#define TTE_BITS_TOP (_PAGE_VALID | _PAGE_SZBITS)
27#define TTE_BITS_BOTTOM (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W)
28#define DCACHE_SIZE (PAGE_SIZE * 2)
29
30#if (PAGE_SHIFT == 13) || (PAGE_SHIFT == 19)
31#define PAGE_SIZE_REM 0x80
32#elif (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
33#define PAGE_SIZE_REM 0x100
34#else
35#error Wrong PAGE_SHIFT specified
36#endif
37
38#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
39 fmovd %reg0, %f48; fmovd %reg1, %f50; \
40 fmovd %reg2, %f52; fmovd %reg3, %f54; \
41 fmovd %reg4, %f56; fmovd %reg5, %f58; \
42 fmovd %reg6, %f60; fmovd %reg7, %f62;
43
44 .text
45
46 .align 32
47 .globl copy_user_page
48 .type copy_user_page,#function
49copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
50 lduw [%g6 + TI_PRE_COUNT], %o4
51 sethi %uhi(PAGE_OFFSET), %g2
52 sethi %hi(PAGE_SIZE), %o3
53
54 sllx %g2, 32, %g2
55 sethi %uhi(TTE_BITS_TOP), %g3
56
57 sllx %g3, 32, %g3
58 sub %o0, %g2, %g1 ! dest paddr
59
60 sub %o1, %g2, %g2 ! src paddr
61 or %g3, TTE_BITS_BOTTOM, %g3
62
63 and %o2, %o3, %o0 ! vaddr D-cache alias bit
64 or %g1, %g3, %g1 ! dest TTE data
65
66 or %g2, %g3, %g2 ! src TTE data
67 sethi %hi(TLBTEMP_BASE), %o3
68
69 sethi %hi(DCACHE_SIZE), %o1
70 add %o0, %o3, %o0 ! dest TTE vaddr
71
72 add %o4, 1, %o2
73 add %o0, %o1, %o1 ! src TTE vaddr
74
75 /* Disable preemption. */
76 mov TLB_TAG_ACCESS, %g3
77 stw %o2, [%g6 + TI_PRE_COUNT]
78
79 /* Load TLB entries. */
80 rdpr %pstate, %o2
81 wrpr %o2, PSTATE_IE, %pstate
82 stxa %o0, [%g3] ASI_DMMU
83 stxa %g1, [%g0] ASI_DTLB_DATA_IN
84 membar #Sync
85 stxa %o1, [%g3] ASI_DMMU
86 stxa %g2, [%g0] ASI_DTLB_DATA_IN
87 membar #Sync
88 wrpr %o2, 0x0, %pstate
89
90 BRANCH_IF_ANY_CHEETAH(g3,o2,1f)
91 ba,pt %xcc, 9f
92 nop
93
941:
95 VISEntryHalf
96 membar #StoreLoad | #StoreStore | #LoadStore
97 sethi %hi((PAGE_SIZE/64)-2), %o2
98 mov %o0, %g1
99 prefetch [%o1 + 0x000], #one_read
100 or %o2, %lo((PAGE_SIZE/64)-2), %o2
101 prefetch [%o1 + 0x040], #one_read
102 prefetch [%o1 + 0x080], #one_read
103 prefetch [%o1 + 0x0c0], #one_read
104 ldd [%o1 + 0x000], %f0
105 prefetch [%o1 + 0x100], #one_read
106 ldd [%o1 + 0x008], %f2
107 prefetch [%o1 + 0x140], #one_read
108 ldd [%o1 + 0x010], %f4
109 prefetch [%o1 + 0x180], #one_read
110 fmovd %f0, %f16
111 ldd [%o1 + 0x018], %f6
112 fmovd %f2, %f18
113 ldd [%o1 + 0x020], %f8
114 fmovd %f4, %f20
115 ldd [%o1 + 0x028], %f10
116 fmovd %f6, %f22
117 ldd [%o1 + 0x030], %f12
118 fmovd %f8, %f24
119 ldd [%o1 + 0x038], %f14
120 fmovd %f10, %f26
121 ldd [%o1 + 0x040], %f0
1221: ldd [%o1 + 0x048], %f2
123 fmovd %f12, %f28
124 ldd [%o1 + 0x050], %f4
125 fmovd %f14, %f30
126 stda %f16, [%o0] ASI_BLK_P
127 ldd [%o1 + 0x058], %f6
128 fmovd %f0, %f16
129 ldd [%o1 + 0x060], %f8
130 fmovd %f2, %f18
131 ldd [%o1 + 0x068], %f10
132 fmovd %f4, %f20
133 ldd [%o1 + 0x070], %f12
134 fmovd %f6, %f22
135 ldd [%o1 + 0x078], %f14
136 fmovd %f8, %f24
137 ldd [%o1 + 0x080], %f0
138 prefetch [%o1 + 0x180], #one_read
139 fmovd %f10, %f26
140 subcc %o2, 1, %o2
141 add %o0, 0x40, %o0
142 bne,pt %xcc, 1b
143 add %o1, 0x40, %o1
144
145 ldd [%o1 + 0x048], %f2
146 fmovd %f12, %f28
147 ldd [%o1 + 0x050], %f4
148 fmovd %f14, %f30
149 stda %f16, [%o0] ASI_BLK_P
150 ldd [%o1 + 0x058], %f6
151 fmovd %f0, %f16
152 ldd [%o1 + 0x060], %f8
153 fmovd %f2, %f18
154 ldd [%o1 + 0x068], %f10
155 fmovd %f4, %f20
156 ldd [%o1 + 0x070], %f12
157 fmovd %f6, %f22
158 add %o0, 0x40, %o0
159 ldd [%o1 + 0x078], %f14
160 fmovd %f8, %f24
161 fmovd %f10, %f26
162 fmovd %f12, %f28
163 fmovd %f14, %f30
164 stda %f16, [%o0] ASI_BLK_P
165 membar #Sync
166 VISExitHalf
167 ba,pt %xcc, 5f
168 nop
169
1709:
171 VISEntry
172 ldub [%g6 + TI_FAULT_CODE], %g3
173 mov %o0, %g1
174 cmp %g3, 0
175 rd %asi, %g3
176 be,a,pt %icc, 1f
177 wr %g0, ASI_BLK_P, %asi
178 wr %g0, ASI_BLK_COMMIT_P, %asi
1791: ldda [%o1] ASI_BLK_P, %f0
180 add %o1, 0x40, %o1
181 ldda [%o1] ASI_BLK_P, %f16
182 add %o1, 0x40, %o1
183 sethi %hi(PAGE_SIZE), %o2
1841: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
185 ldda [%o1] ASI_BLK_P, %f32
186 stda %f48, [%o0] %asi
187 add %o1, 0x40, %o1
188 sub %o2, 0x40, %o2
189 add %o0, 0x40, %o0
190 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
191 ldda [%o1] ASI_BLK_P, %f0
192 stda %f48, [%o0] %asi
193 add %o1, 0x40, %o1
194 sub %o2, 0x40, %o2
195 add %o0, 0x40, %o0
196 TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
197 ldda [%o1] ASI_BLK_P, %f16
198 stda %f48, [%o0] %asi
199 sub %o2, 0x40, %o2
200 add %o1, 0x40, %o1
201 cmp %o2, PAGE_SIZE_REM
202 bne,pt %xcc, 1b
203 add %o0, 0x40, %o0
204#if (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
205 TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
206 ldda [%o1] ASI_BLK_P, %f32
207 stda %f48, [%o0] %asi
208 add %o1, 0x40, %o1
209 sub %o2, 0x40, %o2
210 add %o0, 0x40, %o0
211 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
212 ldda [%o1] ASI_BLK_P, %f0
213 stda %f48, [%o0] %asi
214 add %o1, 0x40, %o1
215 sub %o2, 0x40, %o2
216 add %o0, 0x40, %o0
217 membar #Sync
218 stda %f32, [%o0] %asi
219 add %o0, 0x40, %o0
220 stda %f0, [%o0] %asi
221#else
222 membar #Sync
223 stda %f0, [%o0] %asi
224 add %o0, 0x40, %o0
225 stda %f16, [%o0] %asi
226#endif
227 membar #Sync
228 wr %g3, 0x0, %asi
229 VISExit
230
2315:
232 stxa %g0, [%g1] ASI_DMMU_DEMAP
233 membar #Sync
234
235 sethi %hi(DCACHE_SIZE), %g2
236 stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP
237 membar #Sync
238
239 retl
240 stw %o4, [%g6 + TI_PRE_COUNT]
241
242 .size copy_user_page, .-copy_user_page