| /* Copyright 2002 Andi Kleen, SuSE Labs */ |
| |
| #include <linux/linkage.h> |
| #include <asm/dwarf2.h> |
| |
| /* |
| * ISO C memset - set a memory block to a byte value. |
| * |
| * rdi destination |
| * rsi value (char) |
| * rdx count (bytes) |
| * |
| * rax original destination |
| */ |
| ALIGN |
| memset_c: |
| CFI_STARTPROC |
| movq %rdi,%r9 |
| movl %edx,%r8d |
| andl $7,%r8d |
| movl %edx,%ecx |
| shrl $3,%ecx |
| /* expand byte value */ |
| movzbl %sil,%esi |
| movabs $0x0101010101010101,%rax |
| mulq %rsi /* with rax, clobbers rdx */ |
| rep stosq |
| movl %r8d,%ecx |
| rep stosb |
| movq %r9,%rax |
| ret |
| CFI_ENDPROC |
| ENDPROC(memset_c) |
| |
| ENTRY(memset) |
| ENTRY(__memset) |
| CFI_STARTPROC |
| movq %rdi,%r10 |
| movq %rdx,%r11 |
| |
| /* expand byte value */ |
| movzbl %sil,%ecx |
| movabs $0x0101010101010101,%rax |
| mul %rcx /* with rax, clobbers rdx */ |
| |
| /* align dst */ |
| movl %edi,%r9d |
| andl $7,%r9d |
| jnz .Lbad_alignment |
| CFI_REMEMBER_STATE |
| .Lafter_bad_alignment: |
| |
| movl %r11d,%ecx |
| shrl $6,%ecx |
| jz .Lhandle_tail |
| |
| .p2align 4 |
| .Lloop_64: |
| decl %ecx |
| movq %rax,(%rdi) |
| movq %rax,8(%rdi) |
| movq %rax,16(%rdi) |
| movq %rax,24(%rdi) |
| movq %rax,32(%rdi) |
| movq %rax,40(%rdi) |
| movq %rax,48(%rdi) |
| movq %rax,56(%rdi) |
| leaq 64(%rdi),%rdi |
| jnz .Lloop_64 |
| |
| /* Handle tail in loops. The loops should be faster than hard |
| to predict jump tables. */ |
| .p2align 4 |
| .Lhandle_tail: |
| movl %r11d,%ecx |
| andl $63&(~7),%ecx |
| jz .Lhandle_7 |
| shrl $3,%ecx |
| .p2align 4 |
| .Lloop_8: |
| decl %ecx |
| movq %rax,(%rdi) |
| leaq 8(%rdi),%rdi |
| jnz .Lloop_8 |
| |
| .Lhandle_7: |
| movl %r11d,%ecx |
| andl $7,%ecx |
| jz .Lende |
| .p2align 4 |
| .Lloop_1: |
| decl %ecx |
| movb %al,(%rdi) |
| leaq 1(%rdi),%rdi |
| jnz .Lloop_1 |
| |
| .Lende: |
| movq %r10,%rax |
| ret |
| |
| CFI_RESTORE_STATE |
| .Lbad_alignment: |
| cmpq $7,%r11 |
| jbe .Lhandle_7 |
| movq %rax,(%rdi) /* unaligned store */ |
| movq $8,%r8 |
| subq %r9,%r8 |
| addq %r8,%rdi |
| subq %r8,%r11 |
| jmp .Lafter_bad_alignment |
| .Lfinal: |
| CFI_ENDPROC |
| ENDPROC(memset) |
| ENDPROC(__memset) |
| |
| /* Some CPUs run faster using the string instructions. |
| It is also a lot simpler. Use this when possible */ |
| |
| #include <asm/cpufeature.h> |
| |
| .section .altinstr_replacement,"ax" |
| 1: .byte 0xeb /* jmp <disp8> */ |
| .byte (memset_c - memset) - (2f - 1b) /* offset */ |
| 2: |
| .previous |
| .section .altinstructions,"a" |
| .align 8 |
| .quad memset |
| .quad 1b |
| .byte X86_FEATURE_REP_GOOD |
| .byte .Lfinal - memset |
| .byte 2b - 1b |
| .previous |