| /* Copyright 2002 Andi Kleen */ |
| |
| #include <linux/linkage.h> |
| |
| #include <asm/cpufeature.h> |
| #include <asm/dwarf2.h> |
| |
| /* |
| * memcpy - Copy a memory block. |
| * |
| * Input: |
| * rdi destination |
| * rsi source |
| * rdx count |
| * |
| * Output: |
| * rax original destination |
| */ |
| |
| /* |
| * memcpy_c() - fast string ops (REP MOVSQ) based variant. |
| * |
| * Calls to this get patched into the kernel image via the |
| * alternative instructions framework: |
| */ |
| ALIGN |
| memcpy_c: |
| CFI_STARTPROC |
| movq %rdi, %rax |
| |
| movl %edx, %ecx |
| shrl $3, %ecx |
| andl $7, %edx |
| rep movsq |
| movl %edx, %ecx |
| rep movsb |
| ret |
| CFI_ENDPROC |
| ENDPROC(memcpy_c) |
| |
| ENTRY(__memcpy) |
| ENTRY(memcpy) |
| CFI_STARTPROC |
| |
| /* |
| * Put the number of full 64-byte blocks into %ecx. |
| * Tail portion is handled at the end: |
| */ |
| movq %rdi, %rax |
| movl %edx, %ecx |
| shrl $6, %ecx |
| jz .Lhandle_tail |
| |
| .p2align 4 |
| .Lloop_64: |
| /* |
| * We decrement the loop index here - and the zero-flag is |
| * checked at the end of the loop (instructions inbetween do |
| * not change the zero flag): |
| */ |
| decl %ecx |
| |
| /* |
| * Move in blocks of 4x16 bytes: |
| */ |
| movq 0*8(%rsi), %r11 |
| movq 1*8(%rsi), %r8 |
| movq %r11, 0*8(%rdi) |
| movq %r8, 1*8(%rdi) |
| |
| movq 2*8(%rsi), %r9 |
| movq 3*8(%rsi), %r10 |
| movq %r9, 2*8(%rdi) |
| movq %r10, 3*8(%rdi) |
| |
| movq 4*8(%rsi), %r11 |
| movq 5*8(%rsi), %r8 |
| movq %r11, 4*8(%rdi) |
| movq %r8, 5*8(%rdi) |
| |
| movq 6*8(%rsi), %r9 |
| movq 7*8(%rsi), %r10 |
| movq %r9, 6*8(%rdi) |
| movq %r10, 7*8(%rdi) |
| |
| leaq 64(%rsi), %rsi |
| leaq 64(%rdi), %rdi |
| |
| jnz .Lloop_64 |
| |
| .Lhandle_tail: |
| movl %edx, %ecx |
| andl $63, %ecx |
| shrl $3, %ecx |
| jz .Lhandle_7 |
| |
| .p2align 4 |
| .Lloop_8: |
| decl %ecx |
| movq (%rsi), %r8 |
| movq %r8, (%rdi) |
| leaq 8(%rdi), %rdi |
| leaq 8(%rsi), %rsi |
| jnz .Lloop_8 |
| |
| .Lhandle_7: |
| movl %edx, %ecx |
| andl $7, %ecx |
| jz .Lend |
| |
| .p2align 4 |
| .Lloop_1: |
| movb (%rsi), %r8b |
| movb %r8b, (%rdi) |
| incq %rdi |
| incq %rsi |
| decl %ecx |
| jnz .Lloop_1 |
| |
| .Lend: |
| ret |
| CFI_ENDPROC |
| ENDPROC(memcpy) |
| ENDPROC(__memcpy) |
| |
| /* |
| * Some CPUs run faster using the string copy instructions. |
| * It is also a lot simpler. Use this when possible: |
| */ |
| |
| .section .altinstr_replacement, "ax" |
| 1: .byte 0xeb /* jmp <disp8> */ |
| .byte (memcpy_c - memcpy) - (2f - 1b) /* offset */ |
| 2: |
| .previous |
| |
| .section .altinstructions, "a" |
| .align 8 |
| .quad memcpy |
| .quad 1b |
| .byte X86_FEATURE_REP_GOOD |
| |
| /* |
| * Replace only beginning, memcpy is used to apply alternatives, |
| * so it is silly to overwrite itself with nops - reboot is the |
| * only outcome... |
| */ |
| .byte 2b - 1b |
| .byte 2b - 1b |
| .previous |