Index: head/sys/arm64/arm64/copyinout.S =================================================================== --- head/sys/arm64/arm64/copyinout.S (revision 316731) +++ head/sys/arm64/arm64/copyinout.S (revision 316732) @@ -1,220 +1,221 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include "assym.s" /* * Fault handler for the copy{in,out} functions below. */ ENTRY(copyio_fault) SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */ copyio_fault_nopcb: mov x0, #EFAULT ret END(copyio_fault) /* * Copies from a kernel to user address * * int copyout(const void *kaddr, void *udaddr, size_t len) */ ENTRY(copyout) cbz x2, 1f adds x3, x1, x2 b.cs copyio_fault_nopcb ldr x4, =VM_MAXUSER_ADDRESS cmp x3, x4 b.hi copyio_fault_nopcb b copycommon 1: mov x0, xzr /* return 0 */ ret END(copyout) /* * Copies from a user to kernel address * * int copyin(const void *uaddr, void *kdaddr, size_t len) */ ENTRY(copyin) cbz x2, 1f adds x3, x0, x2 b.cs copyio_fault_nopcb ldr x4, =VM_MAXUSER_ADDRESS cmp x3, x4 b.hi copyio_fault_nopcb b copycommon 1: mov x0, xzr /* return 0 */ ret END(copyin) /* * Copies a string from a user to kernel address * * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) */ ENTRY(copyinstr) mov x5, xzr /* count = 0 */ mov w4, #1 /* If zero return faulure */ cbz x2, 3f /* If len == 0 then skip loop */ adr x6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(x6, x7) /* Set the handler */ ldr x7, =VM_MAXUSER_ADDRESS 1: cmp x0, x7 b.cs copyio_fault - ldrb w4, [x0], #1 /* Load from uaddr */ + ldtrb w4, [x0] /* Load from uaddr */ + add x0, x0, #1 /* Next char */ strb w4, [x1], #1 /* Store in kaddr */ add x5, x5, #1 /* count++ */ cbz w4, 2f /* Break when NUL-terminated */ sub x2, x2, #1 /* len-- */ cbnz x2, 1b 2: SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 3: cbz x3, 4f /* Check if done != NULL */ str x5, [x3] /* done = count */ 4: mov w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */ cmp w4, #0 /* Check if we saved the NUL-terminator */ csel w0, wzr, w1, eq /* If so return success, else failure */ ret END(copyinstr) /* * Local helper * * x0 - src pointer * x1 - dst pointer * x2 - size * lr - the return address, so jump here instead of calling * * This function is optimized to minimize concurrent memory accesses. In * present form it is suited for cores with a single memory prefetching * unit. * ARM64TODO: * Consider using separate functions for each ARM64 core. Adding memory * access interleaving might increase a total throughput on A57 or A72. */ .text .align 4 .local copycommon .type copycommon,@function copycommon: adr x6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(x6, x7) /* Set the handler */ /* Check alignment */ orr x3, x0, x1 ands x3, x3, 0x07 b.eq aligned /* Unaligned is byte by byte copy */ byte_by_byte: ldrb w3, [x0], #0x01 strb w3, [x1], #0x01 subs x2, x2, #0x01 b.ne byte_by_byte b ending aligned: cmp x2, #0x10 b.lt lead_out cmp x2, #0x40 b.lt by_dwords_start /* Block copy */ lsr x15, x2, #0x06 by_blocks: ldp x3, x4, [x0], #0x10 ldp x5, x6, [x0], #0x10 ldp x7, x8, [x0], #0x10 ldp x9, x10, [x0], #0x10 stp x3, x4, [x1], #0x10 stp x5, x6, [x1], #0x10 stp x7, x8, [x1], #0x10 stp x9, x10, [x1], #0x10 subs x15, x15, #0x01 b.ne by_blocks and x2, x2, #0x3f by_dwords_start: lsr x15, x2, #0x04 cbz x15, lead_out by_dwords: ldp x3, x4, [x0], #0x10 stp x3, x4, [x1], #0x10 subs x15, x15, #0x01 b.ne by_dwords /* Less than 16 bytes to copy */ lead_out: tbz x2, #0x03, last_word ldr x3, [x0], #0x08 str x3, [x1], #0x08 last_word: tbz x2, #0x02, last_hword ldr w3, [x0], #0x04 str w3, [x1], #0x04 last_hword: tbz x2, #0x01, last_byte ldrh w3, [x0], #0x02 strh w3, [x1], #0x02 last_byte: tbz x2, #0x00, ending ldrb w3, [x0] strb w3, [x1] ending: SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ mov x0, xzr /* return 0 */ ret .size copycommon, . - copycommon Index: head/sys/arm64/arm64/support.S =================================================================== --- head/sys/arm64/arm64/support.S (revision 316731) +++ head/sys/arm64/arm64/support.S (revision 316732) @@ -1,329 +1,329 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2014-2015 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by Andrew Turner * under sponsorship from the FreeBSD Foundation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include "assym.s" /* * One of the fu* or su* functions failed, return -1. */ ENTRY(fsu_fault) SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ fsu_fault_nopcb: mov x0, #-1 ret END(fsu_fault) /* * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) */ ENTRY(casueword32) ldr x4, =(VM_MAXUSER_ADDRESS-3) cmp x0, x4 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x4) /* And set it */ 1: ldxr w4, [x0] /* Load-exclusive the data */ cmp w4, w1 /* Compare */ b.ne 2f /* Not equal, exit */ stxr w5, w3, [x0] /* Store the new data */ cbnz w5, 1b /* Retry on failure */ 2: SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */ str w4, [x2] /* Store the read data */ mov x0, #0 /* Success */ ret /* Return */ END(casueword32) /* * int casueword(volatile u_long *, u_long, u_long *, u_long) */ ENTRY(casueword) ldr x4, =(VM_MAXUSER_ADDRESS-7) cmp x0, x4 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x4) /* And set it */ 1: ldxr x4, [x0] /* Load-exclusive the data */ cmp x4, x1 /* Compare */ b.ne 2f /* Not equal, exit */ stxr w5, x3, [x0] /* Store the new data */ cbnz w5, 1b /* Retry on failure */ 2: SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */ str x4, [x2] /* Store the read data */ mov x0, #0 /* Success */ ret /* Return */ END(casueword) /* * int fubyte(volatile const void *) */ ENTRY(fubyte) ldr x1, =VM_MAXUSER_ADDRESS cmp x0, x1 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x1) /* And set it */ - ldrb w0, [x0] /* Try loading the data */ + ldtrb w0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */ ret /* Return */ END(fubyte) /* * int fuword(volatile const void *) */ ENTRY(fuword16) ldr x1, =(VM_MAXUSER_ADDRESS-1) cmp x0, x1 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x1) /* And set it */ - ldrh w0, [x0] /* Try loading the data */ + ldtrh w0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */ ret /* Return */ END(fuword16) /* * int32_t fueword32(volatile const void *, int32_t *) */ ENTRY(fueword32) ldr x2, =(VM_MAXUSER_ADDRESS-3) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ - ldr w0, [x0] /* Try loading the data */ + ldtr w0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ str w0, [x1] /* Save the data in kernel space */ mov w0, #0 /* Success */ ret /* Return */ END(fueword32) /* * long fueword(volatile const void *, int64_t *) * int64_t fueword64(volatile const void *, int64_t *) */ ENTRY(fueword) EENTRY(fueword64) ldr x2, =(VM_MAXUSER_ADDRESS-7) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ - ldr x0, [x0] /* Try loading the data */ + ldtr x0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ str x0, [x1] /* Save the data in kernel space */ mov x0, #0 /* Success */ ret /* Return */ EEND(fueword64) END(fueword) /* * int subyte(volatile void *, int) */ ENTRY(subyte) ldr x2, =VM_MAXUSER_ADDRESS cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ - strb w1, [x0] /* Try storing the data */ + sttrb w1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ END(subyte) /* * int suword16(volatile void *, int) */ ENTRY(suword16) ldr x2, =(VM_MAXUSER_ADDRESS-1) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ - strh w1, [x0] /* Try storing the data */ + sttrh w1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ END(suword16) /* * int suword32(volatile void *, int) */ ENTRY(suword32) ldr x2, =(VM_MAXUSER_ADDRESS-3) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ - str w1, [x0] /* Try storing the data */ + sttr w1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ END(suword32) /* * int suword(volatile void *, long) */ ENTRY(suword) EENTRY(suword64) ldr x2, =(VM_MAXUSER_ADDRESS-7) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ - str x1, [x0] /* Try storing the data */ + sttr x1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ EEND(suword64) END(suword) /* * fuswintr and suswintr are just like fusword and susword except that if * the page is not in memory or would cause a trap, then we return an error. * The important thing is to prevent sleep() and switch(). */ /* * Special handler so the trap code knows not to sleep. */ ENTRY(fsu_intr_fault) SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ mov x0, #-1 ret END(fsu_fault) /* * int fuswintr(void *) */ ENTRY(fuswintr) ldr x1, =(VM_MAXUSER_ADDRESS-3) cmp x0, x1 b.cs fsu_fault_nopcb adr x6, fsu_intr_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x1) /* And set it */ - ldr w0, [x0] /* Try loading the data */ + ldtr w0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */ ret /* Return */ END(fuswintr) /* * int suswintr(void *base, int word) */ ENTRY(suswintr) ldr x2, =(VM_MAXUSER_ADDRESS-3) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_intr_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ - str w1, [x0] /* Try storing the data */ + sttr w1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ END(suswintr) ENTRY(setjmp) /* Store the stack pointer */ mov x8, sp str x8, [x0], #8 /* Store the general purpose registers and lr */ stp x19, x20, [x0], #16 stp x21, x22, [x0], #16 stp x23, x24, [x0], #16 stp x25, x26, [x0], #16 stp x27, x28, [x0], #16 stp x29, lr, [x0], #16 /* Return value */ mov x0, #0 ret END(setjmp) ENTRY(longjmp) /* Restore the stack pointer */ ldr x8, [x0], #8 mov sp, x8 /* Restore the general purpose registers and lr */ ldp x19, x20, [x0], #16 ldp x21, x22, [x0], #16 ldp x23, x24, [x0], #16 ldp x25, x26, [x0], #16 ldp x27, x28, [x0], #16 ldp x29, lr, [x0], #16 /* Load the return value */ mov x0, x1 ret END(longjmp) /* * pagezero, simple implementation */ ENTRY(pagezero_simple) add x1, x0, #PAGE_SIZE 1: stp xzr, xzr, [x0], #0x10 stp xzr, xzr, [x0], #0x10 stp xzr, xzr, [x0], #0x10 stp xzr, xzr, [x0], #0x10 cmp x0, x1 b.ne 1b ret END(pagezero_simple) /* * pagezero, cache assisted */ ENTRY(pagezero_cache) add x1, x0, #PAGE_SIZE ldr x2, =dczva_line_size ldr x2, [x2] 1: dc zva, x0 add x0, x0, x2 cmp x0, x1 b.ne 1b ret END(pagezero_cache)