Index: head/sys/arm64/arm64/db_interface.c =================================================================== --- head/sys/arm64/arm64/db_interface.c (revision 284195) +++ head/sys/arm64/arm64/db_interface.c (revision 284196) @@ -1,168 +1,166 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include static int db_frame(struct db_variable *vp, db_expr_t *valuep, int op) { long *reg; if (kdb_frame == NULL) return (0); reg = (long *)((uintptr_t)kdb_frame + (db_expr_t)vp->valuep); if (op == DB_VAR_GET) *valuep = *reg; else *reg = *valuep; return (1); } #define DB_OFFSET(x) (db_expr_t *)offsetof(struct trapframe, x) struct db_variable db_regs[] = { { "spsr", DB_OFFSET(tf_spsr), db_frame }, { "x0", DB_OFFSET(tf_x[0]), db_frame }, { "x1", DB_OFFSET(tf_x[1]), db_frame }, { "x2", DB_OFFSET(tf_x[2]), db_frame }, { "x3", DB_OFFSET(tf_x[3]), db_frame }, { "x4", DB_OFFSET(tf_x[4]), db_frame }, { "x5", DB_OFFSET(tf_x[5]), db_frame }, { "x6", DB_OFFSET(tf_x[6]), db_frame }, { "x7", DB_OFFSET(tf_x[7]), db_frame }, { "x8", DB_OFFSET(tf_x[8]), db_frame }, { "x9", DB_OFFSET(tf_x[9]), db_frame }, { "x10", DB_OFFSET(tf_x[10]), db_frame }, { "x11", DB_OFFSET(tf_x[11]), db_frame }, { "x12", DB_OFFSET(tf_x[12]), db_frame }, { "x13", DB_OFFSET(tf_x[13]), db_frame }, { "x14", DB_OFFSET(tf_x[14]), db_frame }, { "x15", DB_OFFSET(tf_x[15]), db_frame }, { "x16", DB_OFFSET(tf_x[16]), db_frame }, { "x17", DB_OFFSET(tf_x[17]), db_frame }, { "x18", DB_OFFSET(tf_x[18]), db_frame }, { "x19", DB_OFFSET(tf_x[19]), db_frame }, { "x20", DB_OFFSET(tf_x[20]), db_frame }, { "x21", DB_OFFSET(tf_x[21]), db_frame }, { "x22", DB_OFFSET(tf_x[22]), db_frame }, { "x23", DB_OFFSET(tf_x[23]), db_frame }, { "x24", DB_OFFSET(tf_x[24]), db_frame }, { "x25", DB_OFFSET(tf_x[25]), db_frame }, { "x26", DB_OFFSET(tf_x[26]), db_frame }, { "x27", DB_OFFSET(tf_x[27]), db_frame }, { "x28", DB_OFFSET(tf_x[28]), db_frame }, { "x29", DB_OFFSET(tf_x[29]), db_frame }, { "lr", DB_OFFSET(tf_lr), db_frame }, { "elr", DB_OFFSET(tf_elr), db_frame }, { "sp", DB_OFFSET(tf_sp), db_frame }, }; struct db_variable *db_eregs = db_regs + sizeof(db_regs)/sizeof(db_regs[0]); void db_show_mdpcpu(struct pcpu *pc) { } static int db_validate_address(vm_offset_t addr) { struct proc *p = curproc; struct pmap *pmap; if (!p || !p->p_vmspace || !p->p_vmspace->vm_map.pmap || addr >= VM_MAXUSER_ADDRESS) pmap = pmap_kernel(); else pmap = p->p_vmspace->vm_map.pmap; return (pmap_extract(pmap, addr) == FALSE); } /* * Read bytes from kernel address space for debugger. */ int db_read_bytes(vm_offset_t addr, size_t size, char *data) { const char *src = (const char *)addr; while (size-- > 0) { if (db_validate_address((vm_offset_t)src)) { db_printf("address %p is invalid\n", src); return (-1); } *data++ = *src++; } return (0); } /* * Write bytes to kernel address space for debugger. */ int db_write_bytes(vm_offset_t addr, size_t size, char *data) { char *dst; dst = (char *)addr; while (size-- > 0) { if (db_validate_address((vm_offset_t)dst)) { db_printf("address %p is invalid\n", dst); return (-1); } *dst++ = *data++; } + dsb(ish); - dsb(); /* Clean D-cache and invalidate I-cache */ cpu_dcache_wb_range(addr, (vm_size_t)size); cpu_icache_sync_range(addr, (vm_size_t)size); - dsb(); - isb(); return (0); } Index: head/sys/arm64/arm64/vfp.c =================================================================== --- head/sys/arm64/arm64/vfp.c (revision 284195) +++ head/sys/arm64/arm64/vfp.c (revision 284196) @@ -1,195 +1,195 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifdef VFP #include #include #include #include #include #include #include #include /* Sanity check we can store all the VFP registers */ CTASSERT(sizeof(((struct pcb *)0)->pcb_vfp) == 16 * 32); static void vfp_enable(void) { uint32_t cpacr; cpacr = READ_SPECIALREG(cpacr_el1); cpacr = (cpacr & ~CPACR_FPEN_MASK) | CPACR_FPEN_TRAP_NONE; WRITE_SPECIALREG(cpacr_el1, cpacr); isb(); } static void vfp_disable(void) { uint32_t cpacr; cpacr = READ_SPECIALREG(cpacr_el1); cpacr = (cpacr & ~CPACR_FPEN_MASK) | CPACR_FPEN_TRAP_ALL1; WRITE_SPECIALREG(cpacr_el1, cpacr); isb(); } /* * Called when the thread is dying. If the thread was the last to use the * VFP unit mark it as unused to tell the kernel the fp state is unowned. * Ensure the VFP unit is off so we get an exception on the next access. */ void vfp_discard(struct thread *td) { if (PCPU_GET(fpcurthread) == td) PCPU_SET(fpcurthread, NULL); vfp_disable(); } void vfp_save_state(struct thread *td) { __int128_t *vfp_state; uint64_t fpcr, fpsr; uint32_t cpacr; critical_enter(); /* * Only store the registers if the VFP is enabled, * i.e. return if we are trapping on FP access. */ cpacr = READ_SPECIALREG(cpacr_el1); if ((cpacr & CPACR_FPEN_MASK) == CPACR_FPEN_TRAP_NONE) { vfp_state = td->td_pcb->pcb_vfp; __asm __volatile( "mrs %0, fpcr \n" "mrs %1, fpsr \n" "stp q0, q1, [%2, #16 * 0]\n" "stp q2, q3, [%2, #16 * 2]\n" "stp q4, q5, [%2, #16 * 4]\n" "stp q6, q7, [%2, #16 * 6]\n" "stp q8, q9, [%2, #16 * 8]\n" "stp q10, q11, [%2, #16 * 10]\n" "stp q12, q13, [%2, #16 * 12]\n" "stp q14, q15, [%2, #16 * 14]\n" "stp q16, q17, [%2, #16 * 16]\n" "stp q18, q19, [%2, #16 * 18]\n" "stp q20, q21, [%2, #16 * 20]\n" "stp q22, q23, [%2, #16 * 22]\n" "stp q24, q25, [%2, #16 * 24]\n" "stp q26, q27, [%2, #16 * 26]\n" "stp q28, q29, [%2, #16 * 28]\n" "stp q30, q31, [%2, #16 * 30]\n" : "=&r"(fpcr), "=&r"(fpsr) : "r"(vfp_state)); td->td_pcb->pcb_fpcr = fpcr; td->td_pcb->pcb_fpsr = fpsr; - dsb(); + dsb(ish); vfp_disable(); } critical_exit(); } void vfp_restore_state(void) { __int128_t *vfp_state; uint64_t fpcr, fpsr; struct pcb *curpcb; u_int cpu; critical_enter(); cpu = PCPU_GET(cpuid); curpcb = curthread->td_pcb; curpcb->pcb_fpflags |= PCB_FP_STARTED; vfp_enable(); if (PCPU_GET(fpcurthread) != curthread && cpu != curpcb->pcb_vfpcpu) { vfp_state = curthread->td_pcb->pcb_vfp; fpcr = curthread->td_pcb->pcb_fpcr; fpsr = curthread->td_pcb->pcb_fpsr; __asm __volatile( "ldp q0, q1, [%2, #16 * 0]\n" "ldp q2, q3, [%2, #16 * 2]\n" "ldp q4, q5, [%2, #16 * 4]\n" "ldp q6, q7, [%2, #16 * 6]\n" "ldp q8, q9, [%2, #16 * 8]\n" "ldp q10, q11, [%2, #16 * 10]\n" "ldp q12, q13, [%2, #16 * 12]\n" "ldp q14, q15, [%2, #16 * 14]\n" "ldp q16, q17, [%2, #16 * 16]\n" "ldp q18, q19, [%2, #16 * 18]\n" "ldp q20, q21, [%2, #16 * 20]\n" "ldp q22, q23, [%2, #16 * 22]\n" "ldp q24, q25, [%2, #16 * 24]\n" "ldp q26, q27, [%2, #16 * 26]\n" "ldp q28, q29, [%2, #16 * 28]\n" "ldp q30, q31, [%2, #16 * 30]\n" "msr fpcr, %0 \n" "msr fpsr, %1 \n" : : "r"(fpcr), "r"(fpsr), "r"(vfp_state)); PCPU_SET(fpcurthread, curthread); curpcb->pcb_vfpcpu = cpu; } critical_exit(); } void vfp_init(void) { uint64_t pfr; /* Check if there is a vfp unit present */ pfr = READ_SPECIALREG(id_aa64pfr0_el1); if ((pfr & ID_AA64PFR0_FP_MASK) == ID_AA64PFR0_FP_NONE) return; /* Disable to be enabled when it's used */ vfp_disable(); } SYSINIT(vfp, SI_SUB_CPU, SI_ORDER_ANY, vfp_init, NULL); #endif Index: head/sys/arm64/include/atomic.h =================================================================== --- head/sys/arm64/include/atomic.h (revision 284195) +++ head/sys/arm64/include/atomic.h (revision 284196) @@ -1,730 +1,746 @@ /*- * Copyright (c) 2013 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ -#define isb() __asm __volatile("isb" : : : "memory") -#define dsb() __asm __volatile("dsb sy" : : : "memory") -#define dmb() __asm __volatile("dmb sy" : : : "memory") +#define isb() __asm __volatile("isb" : : : "memory") -#define mb() dmb() -#define wmb() dmb() -#define rmb() dmb() +/* + * Options for DMB and DSB: + * oshld Outer Shareable, load + * oshst Outer Shareable, store + * osh Outer Shareable, all + * nshld Non-shareable, load + * nshst Non-shareable, store + * nsh Non-shareable, all + * ishld Inner Shareable, load + * ishst Inner Shareable, store + * ish Inner Shareable, all + * ld Full system, load + * st Full system, store + * sy Full system, all + */ +#define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") +#define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") + +#define mb() dmb(sy) /* Full system memory barrier all */ +#define wmb() dmb(st) /* Full system memory barrier store */ +#define rmb() dmb(ld) /* Full system memory barrier load */ static __inline void atomic_add_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " add %w0, %w0, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" ); } static __inline void atomic_clear_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " bic %w0, %w0, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" ); } static __inline int atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t tmp; int res; __asm __volatile( "1: mov %w1, #1 \n" " ldxr %w0, [%2] \n" " cmp %w0, %w3 \n" " b.ne 2f \n" " stxr %w1, %w4, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) : : "cc" ); return (!res); } static __inline uint32_t atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp, ret; int res; __asm __volatile( "1: ldxr %w4, [%2] \n" " add %w0, %w4, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val), "=&r"(ret) : : "cc" ); return (ret); } static __inline uint32_t atomic_readandclear_32(volatile uint32_t *p) { uint32_t tmp, ret; int res; __asm __volatile( " mov %w0, #0 \n" "1: ldxr %w3, [%2] \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "=&r"(ret) : : "cc" ); return (ret); } static __inline void atomic_set_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " orr %w0, %w0, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" ); } static __inline void atomic_subtract_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " sub %w0, %w0, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" ); } #define atomic_add_int atomic_add_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_int atomic_set_32 #define atomic_subtract_int atomic_subtract_32 static __inline void atomic_add_acq_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldaxr %w0, [%2] \n" " add %w0, %w0, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline void atomic_clear_acq_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldaxr %w0, [%2] \n" " bic %w0, %w0, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline int atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t tmp; int res; __asm __volatile( "1: mov %w1, #1 \n" " ldaxr %w0, [%2] \n" " cmp %w0, %w3 \n" " b.ne 2f \n" " stxr %w1, %w4, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory" ); return (!res); } static __inline uint32_t atomic_load_acq_32(volatile uint32_t *p) { uint32_t ret; __asm __volatile( "ldar %w0, [%1] \n" : "=&r" (ret) : "r" (p) : "memory"); return (ret); } static __inline void atomic_set_acq_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldaxr %w0, [%2] \n" " orr %w0, %w0, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline void atomic_subtract_acq_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldaxr %w0, [%2] \n" " sub %w0, %w0, %w3 \n" " stxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } #define atomic_add_acq_int atomic_add_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 /* The atomic functions currently are both acq and rel, we should fix this. */ static __inline void atomic_add_rel_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " add %w0, %w0, %w3 \n" " stlxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline void atomic_clear_rel_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " bic %w0, %w0, %w3 \n" " stlxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline int atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t tmp; int res; __asm __volatile( "1: mov %w1, #1 \n" " ldxr %w0, [%2] \n" " cmp %w0, %w3 \n" " b.ne 2f \n" " stlxr %w1, %w4, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory" ); return (!res); } static __inline void atomic_set_rel_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " orr %w0, %w0, %w3 \n" " stlxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline void atomic_store_rel_32(volatile uint32_t *p, uint32_t val) { __asm __volatile( "stlr %w0, [%1] \n" : : "r" (val), "r" (p) : "memory"); } static __inline void atomic_subtract_rel_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " sub %w0, %w0, %w3 \n" " stlxr %w1, %w0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } #define atomic_add_rel_int atomic_add_rel_32 #define atomic_clear_rel_int atomic_add_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_store_rel_int atomic_store_rel_32 static __inline void atomic_add_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " add %0, %0, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (val) : : "cc" ); } static __inline void atomic_clear_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " bic %0, %0, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" ); } static __inline int atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { uint64_t tmp; int res; __asm __volatile( "1: mov %w1, #1 \n" " ldxr %0, [%2] \n" " cmp %0, %3 \n" " b.ne 2f \n" " stxr %w1, %4, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r" (tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory" ); return (!res); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp, ret; int res; __asm __volatile( "1: ldxr %4, [%2] \n" " add %0, %4, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val), "=&r"(ret) : : "cc" ); return (ret); } static __inline uint64_t atomic_readandclear_64(volatile uint64_t *p) { uint64_t tmp, ret; int res; __asm __volatile( " mov %0, #0 \n" "1: ldxr %3, [%2] \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "=&r"(ret) : : "cc" ); return (ret); } static __inline void atomic_set_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " orr %0, %0, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" ); } static __inline void atomic_subtract_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " sub %0, %0, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" ); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t val) { uint64_t old; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " stxr %w1, %3, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(old), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); return (old); } #define atomic_add_long atomic_add_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #define atomic_set_long atomic_set_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_add_ptr atomic_add_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 #define atomic_readandclear_ptr atomic_readandclear_64 #define atomic_set_ptr atomic_set_64 #define atomic_subtract_ptr atomic_subtract_64 static __inline void atomic_add_acq_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldaxr %0, [%2] \n" " add %0, %0, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline void atomic_clear_acq_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldaxr %0, [%2] \n" " bic %0, %0, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline int atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { uint64_t tmp; int res; __asm __volatile( "1: mov %w1, #1 \n" " ldaxr %0, [%2] \n" " cmp %0, %3 \n" " b.ne 2f \n" " stxr %w1, %4, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory" ); return (!res); } static __inline uint64_t atomic_load_acq_64(volatile uint64_t *p) { uint64_t ret; __asm __volatile( "ldar %0, [%1] \n" : "=&r" (ret) : "r" (p) : "memory"); return (ret); } static __inline void atomic_set_acq_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldaxr %0, [%2] \n" " orr %0, %0, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline void atomic_subtract_acq_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldaxr %0, [%2] \n" " sub %0, %0, %3 \n" " stxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } #define atomic_add_acq_long atomic_add_acq_64 #define atomic_clear_acq_long atomic_add_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 #define atomic_clear_acq_ptr atomic_add_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 #define atomic_set_acq_ptr atomic_set_acq_64 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 /* * TODO: The atomic functions currently are both acq and rel, we should fix * this. */ static __inline void atomic_add_rel_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " add %0, %0, %3 \n" " stlxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline void atomic_clear_rel_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " bic %0, %0, %3 \n" " stlxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline int atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { uint64_t tmp; int res; __asm __volatile( "1: mov %w1, #1 \n" " ldxr %0, [%2] \n" " cmp %0, %3 \n" " b.ne 2f \n" " stlxr %w1, %4, [%2] \n" " cbnz %w1, 1b \n" "2:" : "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory" ); return (!res); } static __inline void atomic_set_rel_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " orr %0, %0, %3 \n" " stlxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t val) { __asm __volatile( "stlr %0, [%1] \n" : : "r" (val), "r" (p) : "memory"); } static __inline void atomic_subtract_rel_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " sub %0, %0, %3 \n" " stlxr %w1, %0, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" ); } #define atomic_add_rel_long atomic_add_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 #define atomic_store_rel_ptr atomic_store_rel_64 #endif /* _MACHINE_ATOMIC_H_ */