Gitiles
Code Review Sign In
review.evervolv.com / android_kernel_oneplus_msm8996 / ad5cf46b4e3b29f2bc08543e8ef0bf8b13d3f19e / . / arch / x86 / lib / memset_64.S
blob: 2c5948116bd21731d4a85712fddd00d3943fd728 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]1/* Copyright 2002 Andi Kleen, SuSE Labs */
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]2
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]3#include <linux/linkage.h>
4#include <asm/dwarf2.h>
5
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]6/*
7 * ISO C memset - set a memory block to a byte value.
8 *
9 * rdi destination
10 * rsi value (char)
11 * rdx count (bytes)
12 *
13 * rax original destination
14 */
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]15 ALIGN
16memset_c:
17 CFI_STARTPROC
18 movq %rdi,%r9
19 movl %edx,%r8d
20 andl $7,%r8d
21 movl %edx,%ecx
22 shrl $3,%ecx
23 /* expand byte value */
24 movzbl %sil,%esi
25 movabs $0x0101010101010101,%rax
26 mulq %rsi /* with rax, clobbers rdx */
27 rep stosq
28 movl %r8d,%ecx
29 rep stosb
30 movq %r9,%rax
31 ret
32 CFI_ENDPROC
33ENDPROC(memset_c)
34
35ENTRY(memset)
36ENTRY(__memset)
37 CFI_STARTPROC
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]38 movq %rdi,%r10
39 movq %rdx,%r11
40
41 /* expand byte value */
42 movzbl %sil,%ecx
43 movabs $0x0101010101010101,%rax
44 mul %rcx /* with rax, clobbers rdx */
45
46 /* align dst */
47 movl %edi,%r9d
48 andl $7,%r9d
49 jnz .Lbad_alignment
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]50 CFI_REMEMBER_STATE
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]51.Lafter_bad_alignment:
52
53 movl %r11d,%ecx
54 shrl $6,%ecx
55 jz .Lhandle_tail
56
57 .p2align 4
58.Lloop_64:
59 decl %ecx
60 movq %rax,(%rdi)
61 movq %rax,8(%rdi)
62 movq %rax,16(%rdi)
63 movq %rax,24(%rdi)
64 movq %rax,32(%rdi)
65 movq %rax,40(%rdi)
66 movq %rax,48(%rdi)
67 movq %rax,56(%rdi)
68 leaq 64(%rdi),%rdi
69 jnz .Lloop_64
70
71 /* Handle tail in loops. The loops should be faster than hard
72 to predict jump tables. */
73 .p2align 4
74.Lhandle_tail:
75 movl %r11d,%ecx
76 andl $63&(~7),%ecx
77 jz .Lhandle_7
78 shrl $3,%ecx
79 .p2align 4
80.Lloop_8:
81 decl %ecx
82 movq %rax,(%rdi)
83 leaq 8(%rdi),%rdi
84 jnz .Lloop_8
85
86.Lhandle_7:
87 movl %r11d,%ecx
88 andl $7,%ecx
89 jz .Lende
90 .p2align 4
91.Lloop_1:
92 decl %ecx
93 movb %al,(%rdi)
94 leaq 1(%rdi),%rdi
95 jnz .Lloop_1
96
97.Lende:
98 movq %r10,%rax
99 ret
100
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]101 CFI_RESTORE_STATE
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]102.Lbad_alignment:
103 cmpq $7,%r11
104 jbe .Lhandle_7
105 movq %rax,(%rdi) /* unaligned store */
106 movq $8,%r8
107 subq %r9,%r8
108 addq %r8,%rdi
109 subq %r8,%r11
110 jmp .Lafter_bad_alignment
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]111.Lfinal:
112 CFI_ENDPROC
113ENDPROC(memset)
114ENDPROC(__memset)
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]115
116 /* Some CPUs run faster using the string instructions.
117 It is also a lot simpler. Use this when possible */
118
119#include <asm/cpufeature.h>
120
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]121 .section .altinstr_replacement,"ax"
1221: .byte 0xeb /* jmp <disp8> */
123 .byte (memset_c - memset) - (2f - 1b) /* offset */
1242:
125 .previous
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]126 .section .altinstructions,"a"
127 .align 8
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]128 .quad memset
129 .quad 1b
130 .byte X86_FEATURE_REP_GOOD
131 .byte .Lfinal - memset
132 .byte 2b - 1b
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]133 .previous