Gitiles
Code Review Sign In
review.evervolv.com / android_bionic / ed777145e7fe6278f9139d2409857ec81b51fa52 / . / libc / arch-x86 / string / sse2-memset-slm.S
blob: 5cff141ada8f22e366c6baac5a1a5ea8854435a4 [file] [log] [blame]
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]1/*
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]2Copyright (c) 2014, Intel Corporation
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]3All rights reserved.
4
5Redistribution and use in source and binary forms, with or without
6modification, are permitted provided that the following conditions are met:
7
8 * Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10
11 * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14
15 * Neither the name of Intel Corporation nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18
19THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29*/
30
Elliott Hughes81d6a182016-03-03 16:10:33 -0800[diff] [blame]31#include <private/bionic_asm.h>
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]32
Elliott Hughesed777142022-07-25 16:25:11 +0000[diff] [blame^]33#define FOR_SILVERMONT
Elliott Hughes81d6a182016-03-03 16:10:33 -0800[diff] [blame]34#include "cache.h"
Liubov Dmitrieva0a490662012-01-17 12:55:46 +0400[diff] [blame]35
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]36#ifndef L
37# define L(label) .L##label
38#endif
39
40#ifndef ALIGN
41# define ALIGN(n) .p2align n
42#endif
43
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]44#define CFI_PUSH(REG) \
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]45 .cfi_adjust_cfa_offset 4; \
46 .cfi_rel_offset REG, 0
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]47
48#define CFI_POP(REG) \
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]49 .cfi_adjust_cfa_offset -4; \
50 .cfi_restore REG
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]51
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]52#define PUSH(REG) pushl REG; CFI_PUSH(REG)
53#define POP(REG) popl REG; CFI_POP(REG)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]54
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]55#define PARMS 8 /* Preserve EBX. */
56#define DST PARMS
57#define CHR (DST+4)
58#define LEN (CHR+4)
Elliott Hughes81d6a182016-03-03 16:10:33 -0800[diff] [blame]59#define CHK_DST_LEN (LEN+4)
60#define SETRTNVAL movl DST(%esp), %eax
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]61
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]62# define ENTRANCE PUSH(%ebx);
63# define RETURN_END POP(%ebx); ret
64# define RETURN RETURN_END; CFI_PUSH(%ebx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]65# define JMPTBL(I, B) I - B
66
67/* Load an entry in a jump table into EBX and branch to it. TABLE is a
68 jump table with relative offsets. */
69# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \
70 /* We first load PC into EBX. */ \
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]71 call __x86.get_pc_thunk.bx; \
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]72 /* Get the address of the jump table. */ \
73 add $(TABLE - .), %ebx; \
74 /* Get the entry and convert the relative offset to the \
75 absolute address. */ \
76 add (%ebx,%ecx,4), %ebx; \
77 add %ecx, %edx; \
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]78 /* We loaded the jump table and adjusted EDX. Go. */ \
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]79 jmp *%ebx
80
Haibo Huangb9244ff2018-08-11 10:12:13 -0700[diff] [blame]81ENTRY(__memset_chk_generic)
Elliott Hughes204990c2016-03-24 22:34:47 -0700[diff] [blame]82 ENTRANCE
Elliott Hughes81d6a182016-03-03 16:10:33 -0800[diff] [blame]83
Elliott Hughes204990c2016-03-24 22:34:47 -0700[diff] [blame]84 movl LEN(%esp), %ecx
85 cmpl CHK_DST_LEN(%esp), %ecx
Lev Rumyantsev3a528f12016-07-29 17:55:42 -0700[diff] [blame]86 jna L(memset_length_loaded)
87
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]88 POP(%ebx) // Undo ENTRANCE without returning.
Lev Rumyantsev3a528f12016-07-29 17:55:42 -0700[diff] [blame]89 jmp __memset_chk_fail
Haibo Huangb9244ff2018-08-11 10:12:13 -0700[diff] [blame]90END(__memset_chk_generic)
Elliott Hughes81d6a182016-03-03 16:10:33 -0800[diff] [blame]91
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]92 .section .text.sse2,"ax",@progbits
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]93 ALIGN(4)
Haibo Huangb9244ff2018-08-11 10:12:13 -0700[diff] [blame]94ENTRY(memset_generic)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]95 ENTRANCE
96
97 movl LEN(%esp), %ecx
Elliott Hughes204990c2016-03-24 22:34:47 -0700[diff] [blame]98L(memset_length_loaded):
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]99 cmp $0, %ecx
100 ja L(1byteormore)
101 SETRTNVAL
102 RETURN
103
104L(1byteormore):
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]105 movzbl CHR(%esp), %eax
106 movb %al, %ah
107 /* Fill the whole EAX with pattern. */
108 movl %eax, %edx
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]109 shl $16, %eax
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]110 or %edx, %eax
Elliott Hughes81d6a182016-03-03 16:10:33 -0800[diff] [blame]111 movl DST(%esp), %edx
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]112 cmp $1, %ecx
113 je L(1byte)
114 cmp $16, %ecx
115 jae L(16bytesormore)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]116
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]117 cmp $4, %ecx
118 jb L(4bytesless)
119 movl %eax, (%edx)
120 movl %eax, -4(%edx, %ecx)
121 cmp $8, %ecx
122 jb L(8bytesless)
123 movl %eax, 4(%edx)
124 movl %eax, -8(%edx, %ecx)
125L(8bytesless):
126 SETRTNVAL
127 RETURN
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]128
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]129L(4bytesless):
130 movw %ax, (%edx)
131 movw %ax, -2(%edx, %ecx)
132 SETRTNVAL
133 RETURN
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]134
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]135L(1byte):
136 movb %al, (%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]137 SETRTNVAL
138 RETURN
139
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]140 ALIGN(4)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]141L(16bytesormore):
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]142 movd %eax, %xmm0
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]143 pshufd $0, %xmm0, %xmm0
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]144
145 cmp $64, %ecx
146 ja L(64bytesmore)
147 movdqu %xmm0, (%edx)
148 movdqu %xmm0, -16(%edx, %ecx)
149 cmp $32, %ecx
150 jbe L(32bytesless)
151 movdqu %xmm0, 16(%edx)
152 movdqu %xmm0, -32(%edx, %ecx)
153L(32bytesless):
154 SETRTNVAL
155 RETURN
156
157L(64bytesmore):
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]158 testl $0xf, %edx
159 jz L(aligned_16)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]160L(not_aligned_16):
161 movdqu %xmm0, (%edx)
162 movl %edx, %eax
163 and $-16, %edx
164 add $16, %edx
165 sub %edx, %eax
166 add %eax, %ecx
167 movd %xmm0, %eax
168
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]169 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]170L(aligned_16):
171 cmp $128, %ecx
172 jae L(128bytesormore)
173
174L(aligned_16_less128bytes):
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]175 BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]176
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]177 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]178L(128bytesormore):
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]179 PUSH(%ebx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]180 mov $SHARED_CACHE_SIZE, %ebx
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]181 cmp %ebx, %ecx
182 jae L(128bytesormore_nt_start)
183
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]184 POP(%ebx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]185
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]186 PUSH(%ebx)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]187 mov $DATA_CACHE_SIZE, %ebx
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]188
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]189 cmp %ebx, %ecx
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]190 jae L(128bytes_L2_normal)
191 subl $128, %ecx
192L(128bytesormore_normal):
193 sub $128, %ecx
194 movdqa %xmm0, (%edx)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]195 movaps %xmm0, 0x10(%edx)
196 movaps %xmm0, 0x20(%edx)
197 movaps %xmm0, 0x30(%edx)
198 movaps %xmm0, 0x40(%edx)
199 movaps %xmm0, 0x50(%edx)
200 movaps %xmm0, 0x60(%edx)
201 movaps %xmm0, 0x70(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]202 lea 128(%edx), %edx
203 jb L(128bytesless_normal)
204
205
206 sub $128, %ecx
207 movdqa %xmm0, (%edx)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]208 movaps %xmm0, 0x10(%edx)
209 movaps %xmm0, 0x20(%edx)
210 movaps %xmm0, 0x30(%edx)
211 movaps %xmm0, 0x40(%edx)
212 movaps %xmm0, 0x50(%edx)
213 movaps %xmm0, 0x60(%edx)
214 movaps %xmm0, 0x70(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]215 lea 128(%edx), %edx
216 jae L(128bytesormore_normal)
217
218L(128bytesless_normal):
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]219 lea 128(%ecx), %ecx
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]220 POP(%ebx)
221 BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]222
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]223 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]224L(128bytes_L2_normal):
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]225 prefetchnta 0x380(%edx)
226 prefetchnta 0x3c0(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]227 sub $128, %ecx
228 movdqa %xmm0, (%edx)
229 movaps %xmm0, 0x10(%edx)
230 movaps %xmm0, 0x20(%edx)
231 movaps %xmm0, 0x30(%edx)
232 movaps %xmm0, 0x40(%edx)
233 movaps %xmm0, 0x50(%edx)
234 movaps %xmm0, 0x60(%edx)
235 movaps %xmm0, 0x70(%edx)
236 add $128, %edx
237 cmp $128, %ecx
238 jae L(128bytes_L2_normal)
239
240L(128bytesless_L2_normal):
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]241 POP(%ebx)
242 BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]243
244L(128bytesormore_nt_start):
245 sub %ebx, %ecx
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]246 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]247L(128bytesormore_shared_cache_loop):
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]248 prefetchnta 0x3c0(%edx)
249 prefetchnta 0x380(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]250 sub $0x80, %ebx
251 movdqa %xmm0, (%edx)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400[diff] [blame]252 movaps %xmm0, 0x10(%edx)
253 movaps %xmm0, 0x20(%edx)
254 movaps %xmm0, 0x30(%edx)
255 movaps %xmm0, 0x40(%edx)
256 movaps %xmm0, 0x50(%edx)
257 movaps %xmm0, 0x60(%edx)
258 movaps %xmm0, 0x70(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]259 add $0x80, %edx
260 cmp $0x80, %ebx
261 jae L(128bytesormore_shared_cache_loop)
262 cmp $0x80, %ecx
263 jb L(shared_cache_loop_end)
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]264 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]265L(128bytesormore_nt):
266 sub $0x80, %ecx
267 movntdq %xmm0, (%edx)
268 movntdq %xmm0, 0x10(%edx)
269 movntdq %xmm0, 0x20(%edx)
270 movntdq %xmm0, 0x30(%edx)
271 movntdq %xmm0, 0x40(%edx)
272 movntdq %xmm0, 0x50(%edx)
273 movntdq %xmm0, 0x60(%edx)
274 movntdq %xmm0, 0x70(%edx)
275 add $0x80, %edx
276 cmp $0x80, %ecx
277 jae L(128bytesormore_nt)
278 sfence
279L(shared_cache_loop_end):
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]280 POP(%ebx)
281 BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]282
283
284 .pushsection .rodata.sse2,"a",@progbits
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]285 ALIGN(2)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]286L(table_16_128bytes):
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]287 .int JMPTBL(L(aligned_16_0bytes), L(table_16_128bytes))
288 .int JMPTBL(L(aligned_16_1bytes), L(table_16_128bytes))
289 .int JMPTBL(L(aligned_16_2bytes), L(table_16_128bytes))
290 .int JMPTBL(L(aligned_16_3bytes), L(table_16_128bytes))
291 .int JMPTBL(L(aligned_16_4bytes), L(table_16_128bytes))
292 .int JMPTBL(L(aligned_16_5bytes), L(table_16_128bytes))
293 .int JMPTBL(L(aligned_16_6bytes), L(table_16_128bytes))
294 .int JMPTBL(L(aligned_16_7bytes), L(table_16_128bytes))
295 .int JMPTBL(L(aligned_16_8bytes), L(table_16_128bytes))
296 .int JMPTBL(L(aligned_16_9bytes), L(table_16_128bytes))
297 .int JMPTBL(L(aligned_16_10bytes), L(table_16_128bytes))
298 .int JMPTBL(L(aligned_16_11bytes), L(table_16_128bytes))
299 .int JMPTBL(L(aligned_16_12bytes), L(table_16_128bytes))
300 .int JMPTBL(L(aligned_16_13bytes), L(table_16_128bytes))
301 .int JMPTBL(L(aligned_16_14bytes), L(table_16_128bytes))
302 .int JMPTBL(L(aligned_16_15bytes), L(table_16_128bytes))
303 .int JMPTBL(L(aligned_16_16bytes), L(table_16_128bytes))
304 .int JMPTBL(L(aligned_16_17bytes), L(table_16_128bytes))
305 .int JMPTBL(L(aligned_16_18bytes), L(table_16_128bytes))
306 .int JMPTBL(L(aligned_16_19bytes), L(table_16_128bytes))
307 .int JMPTBL(L(aligned_16_20bytes), L(table_16_128bytes))
308 .int JMPTBL(L(aligned_16_21bytes), L(table_16_128bytes))
309 .int JMPTBL(L(aligned_16_22bytes), L(table_16_128bytes))
310 .int JMPTBL(L(aligned_16_23bytes), L(table_16_128bytes))
311 .int JMPTBL(L(aligned_16_24bytes), L(table_16_128bytes))
312 .int JMPTBL(L(aligned_16_25bytes), L(table_16_128bytes))
313 .int JMPTBL(L(aligned_16_26bytes), L(table_16_128bytes))
314 .int JMPTBL(L(aligned_16_27bytes), L(table_16_128bytes))
315 .int JMPTBL(L(aligned_16_28bytes), L(table_16_128bytes))
316 .int JMPTBL(L(aligned_16_29bytes), L(table_16_128bytes))
317 .int JMPTBL(L(aligned_16_30bytes), L(table_16_128bytes))
318 .int JMPTBL(L(aligned_16_31bytes), L(table_16_128bytes))
319 .int JMPTBL(L(aligned_16_32bytes), L(table_16_128bytes))
320 .int JMPTBL(L(aligned_16_33bytes), L(table_16_128bytes))
321 .int JMPTBL(L(aligned_16_34bytes), L(table_16_128bytes))
322 .int JMPTBL(L(aligned_16_35bytes), L(table_16_128bytes))
323 .int JMPTBL(L(aligned_16_36bytes), L(table_16_128bytes))
324 .int JMPTBL(L(aligned_16_37bytes), L(table_16_128bytes))
325 .int JMPTBL(L(aligned_16_38bytes), L(table_16_128bytes))
326 .int JMPTBL(L(aligned_16_39bytes), L(table_16_128bytes))
327 .int JMPTBL(L(aligned_16_40bytes), L(table_16_128bytes))
328 .int JMPTBL(L(aligned_16_41bytes), L(table_16_128bytes))
329 .int JMPTBL(L(aligned_16_42bytes), L(table_16_128bytes))
330 .int JMPTBL(L(aligned_16_43bytes), L(table_16_128bytes))
331 .int JMPTBL(L(aligned_16_44bytes), L(table_16_128bytes))
332 .int JMPTBL(L(aligned_16_45bytes), L(table_16_128bytes))
333 .int JMPTBL(L(aligned_16_46bytes), L(table_16_128bytes))
334 .int JMPTBL(L(aligned_16_47bytes), L(table_16_128bytes))
335 .int JMPTBL(L(aligned_16_48bytes), L(table_16_128bytes))
336 .int JMPTBL(L(aligned_16_49bytes), L(table_16_128bytes))
337 .int JMPTBL(L(aligned_16_50bytes), L(table_16_128bytes))
338 .int JMPTBL(L(aligned_16_51bytes), L(table_16_128bytes))
339 .int JMPTBL(L(aligned_16_52bytes), L(table_16_128bytes))
340 .int JMPTBL(L(aligned_16_53bytes), L(table_16_128bytes))
341 .int JMPTBL(L(aligned_16_54bytes), L(table_16_128bytes))
342 .int JMPTBL(L(aligned_16_55bytes), L(table_16_128bytes))
343 .int JMPTBL(L(aligned_16_56bytes), L(table_16_128bytes))
344 .int JMPTBL(L(aligned_16_57bytes), L(table_16_128bytes))
345 .int JMPTBL(L(aligned_16_58bytes), L(table_16_128bytes))
346 .int JMPTBL(L(aligned_16_59bytes), L(table_16_128bytes))
347 .int JMPTBL(L(aligned_16_60bytes), L(table_16_128bytes))
348 .int JMPTBL(L(aligned_16_61bytes), L(table_16_128bytes))
349 .int JMPTBL(L(aligned_16_62bytes), L(table_16_128bytes))
350 .int JMPTBL(L(aligned_16_63bytes), L(table_16_128bytes))
351 .int JMPTBL(L(aligned_16_64bytes), L(table_16_128bytes))
352 .int JMPTBL(L(aligned_16_65bytes), L(table_16_128bytes))
353 .int JMPTBL(L(aligned_16_66bytes), L(table_16_128bytes))
354 .int JMPTBL(L(aligned_16_67bytes), L(table_16_128bytes))
355 .int JMPTBL(L(aligned_16_68bytes), L(table_16_128bytes))
356 .int JMPTBL(L(aligned_16_69bytes), L(table_16_128bytes))
357 .int JMPTBL(L(aligned_16_70bytes), L(table_16_128bytes))
358 .int JMPTBL(L(aligned_16_71bytes), L(table_16_128bytes))
359 .int JMPTBL(L(aligned_16_72bytes), L(table_16_128bytes))
360 .int JMPTBL(L(aligned_16_73bytes), L(table_16_128bytes))
361 .int JMPTBL(L(aligned_16_74bytes), L(table_16_128bytes))
362 .int JMPTBL(L(aligned_16_75bytes), L(table_16_128bytes))
363 .int JMPTBL(L(aligned_16_76bytes), L(table_16_128bytes))
364 .int JMPTBL(L(aligned_16_77bytes), L(table_16_128bytes))
365 .int JMPTBL(L(aligned_16_78bytes), L(table_16_128bytes))
366 .int JMPTBL(L(aligned_16_79bytes), L(table_16_128bytes))
367 .int JMPTBL(L(aligned_16_80bytes), L(table_16_128bytes))
368 .int JMPTBL(L(aligned_16_81bytes), L(table_16_128bytes))
369 .int JMPTBL(L(aligned_16_82bytes), L(table_16_128bytes))
370 .int JMPTBL(L(aligned_16_83bytes), L(table_16_128bytes))
371 .int JMPTBL(L(aligned_16_84bytes), L(table_16_128bytes))
372 .int JMPTBL(L(aligned_16_85bytes), L(table_16_128bytes))
373 .int JMPTBL(L(aligned_16_86bytes), L(table_16_128bytes))
374 .int JMPTBL(L(aligned_16_87bytes), L(table_16_128bytes))
375 .int JMPTBL(L(aligned_16_88bytes), L(table_16_128bytes))
376 .int JMPTBL(L(aligned_16_89bytes), L(table_16_128bytes))
377 .int JMPTBL(L(aligned_16_90bytes), L(table_16_128bytes))
378 .int JMPTBL(L(aligned_16_91bytes), L(table_16_128bytes))
379 .int JMPTBL(L(aligned_16_92bytes), L(table_16_128bytes))
380 .int JMPTBL(L(aligned_16_93bytes), L(table_16_128bytes))
381 .int JMPTBL(L(aligned_16_94bytes), L(table_16_128bytes))
382 .int JMPTBL(L(aligned_16_95bytes), L(table_16_128bytes))
383 .int JMPTBL(L(aligned_16_96bytes), L(table_16_128bytes))
384 .int JMPTBL(L(aligned_16_97bytes), L(table_16_128bytes))
385 .int JMPTBL(L(aligned_16_98bytes), L(table_16_128bytes))
386 .int JMPTBL(L(aligned_16_99bytes), L(table_16_128bytes))
387 .int JMPTBL(L(aligned_16_100bytes), L(table_16_128bytes))
388 .int JMPTBL(L(aligned_16_101bytes), L(table_16_128bytes))
389 .int JMPTBL(L(aligned_16_102bytes), L(table_16_128bytes))
390 .int JMPTBL(L(aligned_16_103bytes), L(table_16_128bytes))
391 .int JMPTBL(L(aligned_16_104bytes), L(table_16_128bytes))
392 .int JMPTBL(L(aligned_16_105bytes), L(table_16_128bytes))
393 .int JMPTBL(L(aligned_16_106bytes), L(table_16_128bytes))
394 .int JMPTBL(L(aligned_16_107bytes), L(table_16_128bytes))
395 .int JMPTBL(L(aligned_16_108bytes), L(table_16_128bytes))
396 .int JMPTBL(L(aligned_16_109bytes), L(table_16_128bytes))
397 .int JMPTBL(L(aligned_16_110bytes), L(table_16_128bytes))
398 .int JMPTBL(L(aligned_16_111bytes), L(table_16_128bytes))
399 .int JMPTBL(L(aligned_16_112bytes), L(table_16_128bytes))
400 .int JMPTBL(L(aligned_16_113bytes), L(table_16_128bytes))
401 .int JMPTBL(L(aligned_16_114bytes), L(table_16_128bytes))
402 .int JMPTBL(L(aligned_16_115bytes), L(table_16_128bytes))
403 .int JMPTBL(L(aligned_16_116bytes), L(table_16_128bytes))
404 .int JMPTBL(L(aligned_16_117bytes), L(table_16_128bytes))
405 .int JMPTBL(L(aligned_16_118bytes), L(table_16_128bytes))
406 .int JMPTBL(L(aligned_16_119bytes), L(table_16_128bytes))
407 .int JMPTBL(L(aligned_16_120bytes), L(table_16_128bytes))
408 .int JMPTBL(L(aligned_16_121bytes), L(table_16_128bytes))
409 .int JMPTBL(L(aligned_16_122bytes), L(table_16_128bytes))
410 .int JMPTBL(L(aligned_16_123bytes), L(table_16_128bytes))
411 .int JMPTBL(L(aligned_16_124bytes), L(table_16_128bytes))
412 .int JMPTBL(L(aligned_16_125bytes), L(table_16_128bytes))
413 .int JMPTBL(L(aligned_16_126bytes), L(table_16_128bytes))
414 .int JMPTBL(L(aligned_16_127bytes), L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]415 .popsection
416
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]417 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]418L(aligned_16_112bytes):
419 movdqa %xmm0, -112(%edx)
420L(aligned_16_96bytes):
421 movdqa %xmm0, -96(%edx)
422L(aligned_16_80bytes):
423 movdqa %xmm0, -80(%edx)
424L(aligned_16_64bytes):
425 movdqa %xmm0, -64(%edx)
426L(aligned_16_48bytes):
427 movdqa %xmm0, -48(%edx)
428L(aligned_16_32bytes):
429 movdqa %xmm0, -32(%edx)
430L(aligned_16_16bytes):
431 movdqa %xmm0, -16(%edx)
432L(aligned_16_0bytes):
433 SETRTNVAL
434 RETURN
435
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]436 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]437L(aligned_16_113bytes):
438 movdqa %xmm0, -113(%edx)
439L(aligned_16_97bytes):
440 movdqa %xmm0, -97(%edx)
441L(aligned_16_81bytes):
442 movdqa %xmm0, -81(%edx)
443L(aligned_16_65bytes):
444 movdqa %xmm0, -65(%edx)
445L(aligned_16_49bytes):
446 movdqa %xmm0, -49(%edx)
447L(aligned_16_33bytes):
448 movdqa %xmm0, -33(%edx)
449L(aligned_16_17bytes):
450 movdqa %xmm0, -17(%edx)
451L(aligned_16_1bytes):
452 movb %al, -1(%edx)
453 SETRTNVAL
454 RETURN
455
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]456 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]457L(aligned_16_114bytes):
458 movdqa %xmm0, -114(%edx)
459L(aligned_16_98bytes):
460 movdqa %xmm0, -98(%edx)
461L(aligned_16_82bytes):
462 movdqa %xmm0, -82(%edx)
463L(aligned_16_66bytes):
464 movdqa %xmm0, -66(%edx)
465L(aligned_16_50bytes):
466 movdqa %xmm0, -50(%edx)
467L(aligned_16_34bytes):
468 movdqa %xmm0, -34(%edx)
469L(aligned_16_18bytes):
470 movdqa %xmm0, -18(%edx)
471L(aligned_16_2bytes):
472 movw %ax, -2(%edx)
473 SETRTNVAL
474 RETURN
475
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]476 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]477L(aligned_16_115bytes):
478 movdqa %xmm0, -115(%edx)
479L(aligned_16_99bytes):
480 movdqa %xmm0, -99(%edx)
481L(aligned_16_83bytes):
482 movdqa %xmm0, -83(%edx)
483L(aligned_16_67bytes):
484 movdqa %xmm0, -67(%edx)
485L(aligned_16_51bytes):
486 movdqa %xmm0, -51(%edx)
487L(aligned_16_35bytes):
488 movdqa %xmm0, -35(%edx)
489L(aligned_16_19bytes):
490 movdqa %xmm0, -19(%edx)
491L(aligned_16_3bytes):
492 movw %ax, -3(%edx)
493 movb %al, -1(%edx)
494 SETRTNVAL
495 RETURN
496
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]497 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]498L(aligned_16_116bytes):
499 movdqa %xmm0, -116(%edx)
500L(aligned_16_100bytes):
501 movdqa %xmm0, -100(%edx)
502L(aligned_16_84bytes):
503 movdqa %xmm0, -84(%edx)
504L(aligned_16_68bytes):
505 movdqa %xmm0, -68(%edx)
506L(aligned_16_52bytes):
507 movdqa %xmm0, -52(%edx)
508L(aligned_16_36bytes):
509 movdqa %xmm0, -36(%edx)
510L(aligned_16_20bytes):
511 movdqa %xmm0, -20(%edx)
512L(aligned_16_4bytes):
513 movl %eax, -4(%edx)
514 SETRTNVAL
515 RETURN
516
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]517 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]518L(aligned_16_117bytes):
519 movdqa %xmm0, -117(%edx)
520L(aligned_16_101bytes):
521 movdqa %xmm0, -101(%edx)
522L(aligned_16_85bytes):
523 movdqa %xmm0, -85(%edx)
524L(aligned_16_69bytes):
525 movdqa %xmm0, -69(%edx)
526L(aligned_16_53bytes):
527 movdqa %xmm0, -53(%edx)
528L(aligned_16_37bytes):
529 movdqa %xmm0, -37(%edx)
530L(aligned_16_21bytes):
531 movdqa %xmm0, -21(%edx)
532L(aligned_16_5bytes):
533 movl %eax, -5(%edx)
534 movb %al, -1(%edx)
535 SETRTNVAL
536 RETURN
537
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]538 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]539L(aligned_16_118bytes):
540 movdqa %xmm0, -118(%edx)
541L(aligned_16_102bytes):
542 movdqa %xmm0, -102(%edx)
543L(aligned_16_86bytes):
544 movdqa %xmm0, -86(%edx)
545L(aligned_16_70bytes):
546 movdqa %xmm0, -70(%edx)
547L(aligned_16_54bytes):
548 movdqa %xmm0, -54(%edx)
549L(aligned_16_38bytes):
550 movdqa %xmm0, -38(%edx)
551L(aligned_16_22bytes):
552 movdqa %xmm0, -22(%edx)
553L(aligned_16_6bytes):
554 movl %eax, -6(%edx)
555 movw %ax, -2(%edx)
556 SETRTNVAL
557 RETURN
558
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]559 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]560L(aligned_16_119bytes):
561 movdqa %xmm0, -119(%edx)
562L(aligned_16_103bytes):
563 movdqa %xmm0, -103(%edx)
564L(aligned_16_87bytes):
565 movdqa %xmm0, -87(%edx)
566L(aligned_16_71bytes):
567 movdqa %xmm0, -71(%edx)
568L(aligned_16_55bytes):
569 movdqa %xmm0, -55(%edx)
570L(aligned_16_39bytes):
571 movdqa %xmm0, -39(%edx)
572L(aligned_16_23bytes):
573 movdqa %xmm0, -23(%edx)
574L(aligned_16_7bytes):
575 movl %eax, -7(%edx)
576 movw %ax, -3(%edx)
577 movb %al, -1(%edx)
578 SETRTNVAL
579 RETURN
580
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]581 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]582L(aligned_16_120bytes):
583 movdqa %xmm0, -120(%edx)
584L(aligned_16_104bytes):
585 movdqa %xmm0, -104(%edx)
586L(aligned_16_88bytes):
587 movdqa %xmm0, -88(%edx)
588L(aligned_16_72bytes):
589 movdqa %xmm0, -72(%edx)
590L(aligned_16_56bytes):
591 movdqa %xmm0, -56(%edx)
592L(aligned_16_40bytes):
593 movdqa %xmm0, -40(%edx)
594L(aligned_16_24bytes):
595 movdqa %xmm0, -24(%edx)
596L(aligned_16_8bytes):
597 movq %xmm0, -8(%edx)
598 SETRTNVAL
599 RETURN
600
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]601 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]602L(aligned_16_121bytes):
603 movdqa %xmm0, -121(%edx)
604L(aligned_16_105bytes):
605 movdqa %xmm0, -105(%edx)
606L(aligned_16_89bytes):
607 movdqa %xmm0, -89(%edx)
608L(aligned_16_73bytes):
609 movdqa %xmm0, -73(%edx)
610L(aligned_16_57bytes):
611 movdqa %xmm0, -57(%edx)
612L(aligned_16_41bytes):
613 movdqa %xmm0, -41(%edx)
614L(aligned_16_25bytes):
615 movdqa %xmm0, -25(%edx)
616L(aligned_16_9bytes):
617 movq %xmm0, -9(%edx)
618 movb %al, -1(%edx)
619 SETRTNVAL
620 RETURN
621
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]622 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]623L(aligned_16_122bytes):
624 movdqa %xmm0, -122(%edx)
625L(aligned_16_106bytes):
626 movdqa %xmm0, -106(%edx)
627L(aligned_16_90bytes):
628 movdqa %xmm0, -90(%edx)
629L(aligned_16_74bytes):
630 movdqa %xmm0, -74(%edx)
631L(aligned_16_58bytes):
632 movdqa %xmm0, -58(%edx)
633L(aligned_16_42bytes):
634 movdqa %xmm0, -42(%edx)
635L(aligned_16_26bytes):
636 movdqa %xmm0, -26(%edx)
637L(aligned_16_10bytes):
638 movq %xmm0, -10(%edx)
639 movw %ax, -2(%edx)
640 SETRTNVAL
641 RETURN
642
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]643 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]644L(aligned_16_123bytes):
645 movdqa %xmm0, -123(%edx)
646L(aligned_16_107bytes):
647 movdqa %xmm0, -107(%edx)
648L(aligned_16_91bytes):
649 movdqa %xmm0, -91(%edx)
650L(aligned_16_75bytes):
651 movdqa %xmm0, -75(%edx)
652L(aligned_16_59bytes):
653 movdqa %xmm0, -59(%edx)
654L(aligned_16_43bytes):
655 movdqa %xmm0, -43(%edx)
656L(aligned_16_27bytes):
657 movdqa %xmm0, -27(%edx)
658L(aligned_16_11bytes):
659 movq %xmm0, -11(%edx)
660 movw %ax, -3(%edx)
661 movb %al, -1(%edx)
662 SETRTNVAL
663 RETURN
664
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]665 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]666L(aligned_16_124bytes):
667 movdqa %xmm0, -124(%edx)
668L(aligned_16_108bytes):
669 movdqa %xmm0, -108(%edx)
670L(aligned_16_92bytes):
671 movdqa %xmm0, -92(%edx)
672L(aligned_16_76bytes):
673 movdqa %xmm0, -76(%edx)
674L(aligned_16_60bytes):
675 movdqa %xmm0, -60(%edx)
676L(aligned_16_44bytes):
677 movdqa %xmm0, -44(%edx)
678L(aligned_16_28bytes):
679 movdqa %xmm0, -28(%edx)
680L(aligned_16_12bytes):
681 movq %xmm0, -12(%edx)
682 movl %eax, -4(%edx)
683 SETRTNVAL
684 RETURN
685
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]686 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]687L(aligned_16_125bytes):
688 movdqa %xmm0, -125(%edx)
689L(aligned_16_109bytes):
690 movdqa %xmm0, -109(%edx)
691L(aligned_16_93bytes):
692 movdqa %xmm0, -93(%edx)
693L(aligned_16_77bytes):
694 movdqa %xmm0, -77(%edx)
695L(aligned_16_61bytes):
696 movdqa %xmm0, -61(%edx)
697L(aligned_16_45bytes):
698 movdqa %xmm0, -45(%edx)
699L(aligned_16_29bytes):
700 movdqa %xmm0, -29(%edx)
701L(aligned_16_13bytes):
702 movq %xmm0, -13(%edx)
703 movl %eax, -5(%edx)
704 movb %al, -1(%edx)
705 SETRTNVAL
706 RETURN
707
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]708 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]709L(aligned_16_126bytes):
710 movdqa %xmm0, -126(%edx)
711L(aligned_16_110bytes):
712 movdqa %xmm0, -110(%edx)
713L(aligned_16_94bytes):
714 movdqa %xmm0, -94(%edx)
715L(aligned_16_78bytes):
716 movdqa %xmm0, -78(%edx)
717L(aligned_16_62bytes):
718 movdqa %xmm0, -62(%edx)
719L(aligned_16_46bytes):
720 movdqa %xmm0, -46(%edx)
721L(aligned_16_30bytes):
722 movdqa %xmm0, -30(%edx)
723L(aligned_16_14bytes):
724 movq %xmm0, -14(%edx)
725 movl %eax, -6(%edx)
726 movw %ax, -2(%edx)
727 SETRTNVAL
728 RETURN
729
Elliott Hughese412f892016-08-01 14:00:45 -0700[diff] [blame]730 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800[diff] [blame]731L(aligned_16_127bytes):
732 movdqa %xmm0, -127(%edx)
733L(aligned_16_111bytes):
734 movdqa %xmm0, -111(%edx)
735L(aligned_16_95bytes):
736 movdqa %xmm0, -95(%edx)
737L(aligned_16_79bytes):
738 movdqa %xmm0, -79(%edx)
739L(aligned_16_63bytes):
740 movdqa %xmm0, -63(%edx)
741L(aligned_16_47bytes):
742 movdqa %xmm0, -47(%edx)
743L(aligned_16_31bytes):
744 movdqa %xmm0, -31(%edx)
745L(aligned_16_15bytes):
746 movq %xmm0, -15(%edx)
747 movl %eax, -7(%edx)
748 movw %ax, -3(%edx)
749 movb %al, -1(%edx)
750 SETRTNVAL
751 RETURN_END
752
Haibo Huangb9244ff2018-08-11 10:12:13 -0700[diff] [blame]753END(memset_generic)