diff --git a/lib/libc/powerpc64/string/bcopy.S b/lib/libc/powerpc64/string/bcopy.S index bb860c098feb..4dc80c264362 100644 --- a/lib/libc/powerpc64/string/bcopy.S +++ b/lib/libc/powerpc64/string/bcopy.S @@ -1,306 +1,340 @@ /*- * Copyright (c) 2018 Instituto de Pesquisas Eldorado * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the author nor the names of its contributors may * be used to endorse or promote products derived from this software * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #define BLOCK_SIZE_BITS 6 #define BLOCK_SIZE (1 << BLOCK_SIZE_BITS) #define BLOCK_SIZE_MASK (BLOCK_SIZE - 1) +/* Minimum 8 byte alignment, to avoid cache-inhibited alignment faults.*/ +#ifndef ALIGN_MASK +#define ALIGN_MASK 0x7 +#endif + #define MULTI_PHASE_THRESHOLD 512 #ifndef FN_NAME #ifdef MEMMOVE #define FN_NAME __memmove WEAK_REFERENCE(__memmove, memmove); #else #define FN_NAME __bcopy WEAK_REFERENCE(__bcopy, bcopy); #endif #endif /* * r3: dst * r4: src * r5: len */ ENTRY(FN_NAME) cmpld %r3, %r4 /* src == dst? nothing to do */ beqlr- cmpdi %r5, 0 /* len == 0? nothing to do */ beqlr- #ifdef MEMMOVE std %r3, -8(%r1) /* save dst */ #else /* bcopy: swap src/dst */ mr %r0, %r3 mr %r3, %r4 mr %r4, %r0 #endif + /* First check for relative alignment, if unaligned copy one byte at a time */ + andi. %r8, %r3, ALIGN_MASK + andi. %r7, %r4, ALIGN_MASK + cmpd %r7, %r8 + bne .Lunaligned + + cmpldi %r5, MULTI_PHASE_THRESHOLD bge .Lmulti_phase + b .Lfast_copy + +.Lunaligned: + /* forward or backward copy? */ + cmpd %r4, %r3 + blt .Lbackward_unaligned + + /* Just need to setup increment and jump to copy */ + li %r0, 1 + mtctr %r5 + b .Lsingle_1_loop + +.Lbackward_unaligned: + /* advance src and dst to last byte, set decrement and jump to copy */ + add %r3, %r3, %r5 + addi %r3, %r3, -1 + add %r4, %r4, %r5 + addi %r4, %r4, -1 + li %r0, -1 + mtctr %r5 + b .Lsingle_1_loop +.Lfast_copy: /* align src */ cmpd %r4, %r3 /* forward or backward copy? */ blt .Lbackward_align .align 5 .Lalign: andi. %r0, %r4, 15 beq .Lsingle_copy lbz %r0, 0(%r4) addi %r4, %r4, 1 stb %r0, 0(%r3) addi %r3, %r3, 1 addi %r5, %r5, -1 cmpdi %r5, 0 beq- .Ldone b .Lalign .Lbackward_align: /* advance src and dst to end (past last byte) */ add %r3, %r3, %r5 add %r4, %r4, %r5 .align 5 .Lbackward_align_loop: andi. %r0, %r4, 15 beq .Lbackward_single_copy lbzu %r0, -1(%r4) addi %r5, %r5, -1 stbu %r0, -1(%r3) cmpdi %r5, 0 beq- .Ldone b .Lbackward_align_loop .Lsingle_copy: /* forward copy */ li %r0, 1 li %r8, 16 li %r9, 0 b .Lsingle_phase .Lbackward_single_copy: /* backward copy */ li %r0, -1 li %r8, -16 li %r9, -15 /* point src and dst to last byte */ addi %r3, %r3, -1 addi %r4, %r4, -1 .Lsingle_phase: srdi. %r6, %r5, 4 /* number of 16-bytes */ beq .Lsingle_1 /* pre-adjustment */ add %r3, %r3, %r9 add %r4, %r4, %r9 mtctr %r6 .align 5 .Lsingle_16_loop: ld %r6, 0(%r4) ld %r7, 8(%r4) add %r4, %r4, %r8 std %r6, 0(%r3) std %r7, 8(%r3) add %r3, %r3, %r8 bdnz .Lsingle_16_loop /* post-adjustment */ sub %r3, %r3, %r9 sub %r4, %r4, %r9 .Lsingle_1: andi. %r6, %r5, 0x0f /* number of 1-bytes */ beq .Ldone /* 1-bytes == 0? done */ mtctr %r6 .align 5 .Lsingle_1_loop: lbz %r6, 0(%r4) add %r4, %r4, %r0 /* increment */ stb %r6, 0(%r3) add %r3, %r3, %r0 /* increment */ bdnz .Lsingle_1_loop .Ldone: #ifdef MEMMOVE ld %r3, -8(%r1) /* restore dst */ #endif blr .Lmulti_phase: /* set up multi-phase copy parameters */ /* r7 = bytes before the aligned section of the buffer */ andi. %r6, %r4, 15 subfic %r7, %r6, 16 /* r8 = bytes in and after the aligned section of the buffer */ sub %r8, %r5, %r7 /* r9 = bytes after the aligned section of the buffer */ andi. %r9, %r8, BLOCK_SIZE_MASK /* r10 = BLOCKS in the aligned section of the buffer */ srdi %r10, %r8, BLOCK_SIZE_BITS /* forward or backward copy? */ cmpd %r4, %r3 blt .Lbackward_multi_copy /* set up forward copy parameters */ std %r7, -32(%r1) /* bytes to copy in phase 1 */ std %r10, -40(%r1) /* BLOCKS to copy in phase 2 */ std %r9, -48(%r1) /* bytes to copy in phase 3 */ li %r0, 1 /* increment for phases 1 and 3 */ li %r5, BLOCK_SIZE /* increment for phase 2 */ /* op offsets for phase 2 */ li %r7, 0 li %r8, 16 li %r9, 32 li %r10, 48 std %r8, -16(%r1) /* 16-byte increment (16) */ std %r7, -24(%r1) /* 16-byte pre/post adjustment (0) */ b .Lphase1 .Lbackward_multi_copy: /* set up backward copy parameters */ std %r9, -32(%r1) /* bytes to copy in phase 1 */ std %r10, -40(%r1) /* BLOCKS to copy in phase 2 */ std %r7, -48(%r1) /* bytes to copy in phase 3 */ li %r0, -1 /* increment for phases 1 and 3 */ add %r6, %r5, %r0 /* r6 = len - 1 */ li %r5, -BLOCK_SIZE /* increment for phase 2 */ /* advance src and dst to the last position */ add %r3, %r3, %r6 add %r4, %r4, %r6 /* op offsets for phase 2 */ li %r7, -15 li %r8, -31 li %r9, -47 li %r10, -63 add %r6, %r7, %r0 /* r6 = -16 */ std %r6, -16(%r1) /* 16-byte increment (-16) */ std %r7, -24(%r1) /* 16-byte pre/post adjustment (-15) */ .Lphase1: ld %r6, -32(%r1) /* bytes to copy in phase 1 */ cmpldi %r6, 0 /* r6 == 0? skip phase 1 */ beq+ .Lphase2 mtctr %r6 .align 5 .Lphase1_loop: lbz %r6, 0(%r4) add %r4, %r4, %r0 /* phase 1 increment */ stb %r6, 0(%r3) add %r3, %r3, %r0 /* phase 1 increment */ bdnz .Lphase1_loop .Lphase2: ld %r6, -40(%r1) /* BLOCKS to copy in phase 2 */ cmpldi %r6, 0 /* %r6 == 0? skip phase 2 */ beq .Lphase3 #ifdef FN_PHASE2 FN_PHASE2 #else /* save registers */ std %r14, -56(%r1) std %r15, -64(%r1) std %r16, -72(%r1) std %r17, -80(%r1) std %r18, -88(%r1) std %r19, -96(%r1) std %r20, -104(%r1) std %r21, -112(%r1) addi %r18, %r7, 8 addi %r19, %r8, 8 addi %r20, %r9, 8 addi %r21, %r10, 8 mtctr %r6 .align 5 .Lphase2_loop: ldx %r14, %r7, %r4 ldx %r15, %r18, %r4 ldx %r16, %r8, %r4 ldx %r17, %r19, %r4 stdx %r14, %r7, %r3 stdx %r15, %r18, %r3 stdx %r16, %r8, %r3 stdx %r17, %r19, %r3 ldx %r14, %r9, %r4 ldx %r15, %r20, %r4 ldx %r16, %r10, %r4 ldx %r17, %r21, %r4 stdx %r14, %r9, %r3 stdx %r15, %r20, %r3 stdx %r16, %r10, %r3 stdx %r17, %r21, %r3 add %r4, %r4, %r5 /* phase 2 increment */ add %r3, %r3, %r5 /* phase 2 increment */ bdnz .Lphase2_loop /* restore registers */ ld %r14, -56(%r1) ld %r15, -64(%r1) ld %r16, -72(%r1) ld %r17, -80(%r1) ld %r18, -88(%r1) ld %r19, -96(%r1) ld %r20, -104(%r1) ld %r21, -112(%r1) #endif .Lphase3: /* load registers for transitioning into the single-phase logic */ ld %r5, -48(%r1) /* bytes to copy in phase 3 */ ld %r8, -16(%r1) /* 16-byte increment */ ld %r9, -24(%r1) /* 16-byte pre/post adjustment */ b .Lsingle_phase END(FN_NAME) .section .note.GNU-stack,"",%progbits