Index: sys/conf/files.powerpc =================================================================== --- sys/conf/files.powerpc +++ sys/conf/files.powerpc @@ -239,7 +239,7 @@ powerpc/powerpc/bus_machdep.c standard powerpc/powerpc/busdma_machdep.c standard powerpc/powerpc/clock.c standard -powerpc/powerpc/copyinout.c standard +powerpc/powerpc/copyinout.c optional aim powerpc/powerpc/cpu.c standard powerpc/powerpc/cpu_subr64.S optional powerpc64 powerpc/powerpc/db_disasm.c optional ddb @@ -274,6 +274,7 @@ powerpc/powerpc/swtch32.S optional powerpc | powerpcspe powerpc/powerpc/swtch64.S optional powerpc64 powerpc/powerpc/stack_machdep.c optional ddb | stack +powerpc/powerpc/support.S optional powerpc64 | booke powerpc/powerpc/syncicache.c standard powerpc/powerpc/sys_machdep.c standard powerpc/powerpc/trap.c standard Index: sys/powerpc/aim/mmu_radix.c =================================================================== --- sys/powerpc/aim/mmu_radix.c +++ sys/powerpc/aim/mmu_radix.c @@ -431,8 +431,6 @@ vm_size_t); void mmu_radix_clear_modify(vm_page_t); void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); -int mmu_radix_map_user_ptr(pmap_t pm, - volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen); int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *); int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, @@ -557,7 +555,6 @@ .kextract = mmu_radix_kextract, .kremove = mmu_radix_kremove, .change_attr = mmu_radix_change_attr, - .map_user_ptr = mmu_radix_map_user_ptr, .decode_kernel_ptr = mmu_radix_decode_kernel_ptr, .tlbie_all = mmu_radix_tlbie_all, @@ -6026,20 +6023,6 @@ pte_clear(pte); } -int mmu_radix_map_user_ptr(pmap_t pm, - volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen) -{ - if ((uintptr_t)uaddr + ulen >= VM_MAXUSER_ADDRESS || - (uintptr_t)uaddr + ulen < (uintptr_t)uaddr) - return (EFAULT); - - *kaddr = (void *)(uintptr_t)uaddr; - if (klen) - *klen = ulen; - - return (0); -} - int mmu_radix_decode_kernel_ptr(vm_offset_t addr, int *is_user, vm_offset_t *decoded) Index: sys/powerpc/booke/pmap.c =================================================================== --- sys/powerpc/booke/pmap.c +++ sys/powerpc/booke/pmap.c @@ -349,8 +349,6 @@ static void mmu_booke_quick_remove_page(vm_offset_t addr); static int mmu_booke_change_attr(vm_offset_t addr, vm_size_t sz, vm_memattr_t mode); -static int mmu_booke_map_user_ptr(pmap_t pm, - volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen); static int mmu_booke_decode_kernel_ptr(vm_offset_t addr, int *is_user, vm_offset_t *decoded_addr); static void mmu_booke_page_array_startup(long); @@ -410,7 +408,6 @@ .kremove = mmu_booke_kremove, .unmapdev = mmu_booke_unmapdev, .change_attr = mmu_booke_change_attr, - .map_user_ptr = mmu_booke_map_user_ptr, .decode_kernel_ptr = mmu_booke_decode_kernel_ptr, /* dumpsys() support */ @@ -1208,26 +1205,6 @@ mtx_unlock_spin(&tlbivax_mutex); } -/* - * Provide a kernel pointer corresponding to a given userland pointer. - * The returned pointer is valid until the next time this function is - * called in this thread. This is used internally in copyin/copyout. - */ -int -mmu_booke_map_user_ptr(pmap_t pm, volatile const void *uaddr, - void **kaddr, size_t ulen, size_t *klen) -{ - - if (trunc_page((uintptr_t)uaddr + ulen) > VM_MAXUSER_ADDRESS) - return (EFAULT); - - *kaddr = (void *)(uintptr_t)uaddr; - if (klen) - *klen = ulen; - - return (0); -} - /* * Figure out where a given kernel pointer (usually in a fault) points * to from the VM's perspective, potentially remapping into userland's Index: sys/powerpc/include/asm.h =================================================================== --- sys/powerpc/include/asm.h +++ sys/powerpc/include/asm.h @@ -114,6 +114,7 @@ .p2align 4; \ TYPE_ENTRY(name) \ DOT_LABEL(name): +#define _NAKED_ENTRY(name) _ENTRY(name) #else #define _ENTRY(name) \ .text; \ @@ -124,6 +125,15 @@ addis %r2, %r12, (.TOC.-name)@ha; \ addi %r2, %r2, (.TOC.-name)@l; \ .localentry name, .-name; + +/* "Naked" function entry. No TOC prologue for ELFv2. */ +#define _NAKED_ENTRY(name) \ + .text; \ + .p2align 4; \ + .globl name; \ + .type name,@function; \ +name: \ + .localentry name, .-name; #endif #define _END(name) \ @@ -146,6 +156,8 @@ name: #define _END(name) +#define _NAKED_ENTRY(name) _ENTRY(name) + #define LOAD_ADDR(reg, var) \ lis reg, var@ha; \ ori reg, reg, var@l; Index: sys/powerpc/include/mmuvar.h =================================================================== --- sys/powerpc/include/mmuvar.h +++ sys/powerpc/include/mmuvar.h @@ -182,6 +182,25 @@ typedef struct mmu_kobj *mmu_t; +/* The currently installed pmap object. */ +extern mmu_t mmu_obj; + +/* + * Resolve a given pmap function. + * 'func' is the function name less the 'pmap_' * prefix. + */ +#define PMAP_RESOLVE_FUNC(func) \ + ({ \ + pmap_##func##_t f; \ + const struct mmu_kobj *mmu = mmu_obj; \ + do { \ + f = mmu->funcs->func; \ + if (f != NULL) break; \ + mmu = mmu->base; \ + } while (mmu != NULL); \ + f;}) + + #define MMU_DEF(name, ident, methods) \ \ const struct mmu_kobj name = { \ Index: sys/powerpc/include/param.h =================================================================== --- sys/powerpc/include/param.h +++ sys/powerpc/include/param.h @@ -134,6 +134,9 @@ #define KSTACK_GUARD_PAGES 1 /* pages of kstack guard; 0 disables */ #define USPACE (kstack_pages * PAGE_SIZE) /* total size of pcb */ +#define COPYFAULT 0x1 +#define FUSUFAULT 0x2 + /* * Mach derived conversion macros */ Index: sys/powerpc/powerpc/copyinout.c =================================================================== --- sys/powerpc/powerpc/copyinout.c +++ sys/powerpc/powerpc/copyinout.c @@ -66,13 +66,102 @@ #include #include +#include #include +#include #include #include +#include + +/* + * On powerpc64 (AIM only) the copy functions are IFUNcs, selecting the best + * option based on the PMAP in use. + * + * There are two options for copy functions on powerpc64: + * - 'remap' copies, which remap userspace segments into kernel space for + * copying. This is used by the 'oea64' pmap. + * - 'direct' copies, which copy directly from userspace. This does not require + * remapping user segments into kernel. This is used by the 'radix' pmap for + * performance. + * + * Book-E does not use the C functions, opting instead to use the 'direct' + * copies, directly, avoiding the IFUNC overhead. + * + * On 32-bit AIM these functions are direct, not IFUNCs, for performance. + */ +#ifdef __powerpc64__ +int subyte_remap(volatile void *addr, int byte); +int subyte_direct(volatile void *addr, int byte); +int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done); +int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done); +int copyout_remap(const void *kaddr, void *udaddr, size_t len); +int copyout_direct(const void *kaddr, void *udaddr, size_t len); +int copyin_remap(const void *uaddr, void *kaddr, size_t len); +int copyin_direct(const void *uaddr, void *kaddr, size_t len); +int suword32_remap(volatile void *addr, int word); +int suword32_direct(volatile void *addr, int word); +int suword_remap(volatile void *addr, long word); +int suword_direct(volatile void *addr, long word); +int suword64_remap(volatile void *addr, int64_t word); +int suword64_direct(volatile void *addr, int64_t word); +int fubyte_remap(volatile const void *addr); +int fubyte_direct(volatile const void *addr); +int fuword16_remap(volatile const void *addr); +int fuword16_direct(volatile const void *addr); +int fueword32_remap(volatile const void *addr, int32_t *val); +int fueword32_direct(volatile const void *addr, int32_t *val); +int fueword64_remap(volatile const void *addr, int64_t *val); +int fueword64_direct(volatile const void *addr, int64_t *val); +int fueword_remap(volatile const void *addr, long *val); +int fueword_direct(volatile const void *addr, long *val); +int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, + uint32_t new); +int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, + uint32_t new); +int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp, + u_long new); +int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp, + u_long new); + +/* + * The IFUNC resolver determines the copy based on if the PMAP implementation + * includes a pmap_map_user_ptr function. + */ +#define DEFINE_COPY_FUNC(ret, func, args) \ + DEFINE_IFUNC(, ret, func, args) \ + { \ + return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \ + func##_remap : func##_direct); \ + } +DEFINE_COPY_FUNC(int, subyte, (volatile void *, int)) +DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *)) +DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t)) +DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t)) +DEFINE_COPY_FUNC(int, suword, (volatile void *, long)) +DEFINE_COPY_FUNC(int, suword32, (volatile void *, int)) +#ifdef __powerpc64__ +DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t)) +#endif +DEFINE_COPY_FUNC(int, fubyte, (volatile const void *)) +DEFINE_COPY_FUNC(int, fuword16, (volatile const void *)) +DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *)) +#ifdef __powerpc64__ +DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *)) +#endif +DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *)) +DEFINE_COPY_FUNC(int, casueword32, + (volatile uint32_t *, uint32_t, uint32_t *, uint32_t)) +DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long)) + +#define REMAP(x) x##_remap +#else +#define REMAP(x) x +#endif + int -copyout(const void *kaddr, void *udaddr, size_t len) +REMAP(copyout)(const void *kaddr, void *udaddr, size_t len) { struct thread *td; pmap_t pm; @@ -111,7 +200,7 @@ } int -copyin(const void *udaddr, void *kaddr, size_t len) +REMAP(copyin)(const void *udaddr, void *kaddr, size_t len) { struct thread *td; pmap_t pm; @@ -150,7 +239,7 @@ } int -copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) +REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done) { const char *up; char *kp; @@ -183,7 +272,7 @@ } int -subyte(volatile void *addr, int byte) +REMAP(subyte)(volatile void *addr, int byte) { struct thread *td; pmap_t pm; @@ -212,7 +301,7 @@ #ifdef __powerpc64__ int -suword32(volatile void *addr, int word) +REMAP(suword32)(volatile void *addr, int word) { struct thread *td; pmap_t pm; @@ -238,10 +327,16 @@ td->td_pcb->pcb_onfault = NULL; return (0); } +#else +int +REMAP(suword32)(volatile void *addr, int32_t word) +{ +REMAP( return (suword)(addr, (long)word)); +} #endif int -suword(volatile void *addr, long word) +REMAP(suword)(volatile void *addr, long word) { struct thread *td; pmap_t pm; @@ -270,20 +365,14 @@ #ifdef __powerpc64__ int -suword64(volatile void *addr, int64_t word) -{ - return (suword(addr, (long)word)); -} -#else -int -suword32(volatile void *addr, int32_t word) +REMAP(suword64)(volatile void *addr, int64_t word) { - return (suword(addr, (long)word)); + return (REMAP(suword)(addr, (long)word)); } #endif int -fubyte(volatile const void *addr) +REMAP(fubyte)(volatile const void *addr) { struct thread *td; pmap_t pm; @@ -312,7 +401,7 @@ } int -fuword16(volatile const void *addr) +REMAP(fuword16)(volatile const void *addr) { struct thread *td; pmap_t pm; @@ -340,7 +429,7 @@ } int -fueword32(volatile const void *addr, int32_t *val) +REMAP(fueword32)(volatile const void *addr, int32_t *val) { struct thread *td; pmap_t pm; @@ -369,7 +458,7 @@ #ifdef __powerpc64__ int -fueword64(volatile const void *addr, int64_t *val) +REMAP(fueword64)(volatile const void *addr, int64_t *val) { struct thread *td; pmap_t pm; @@ -398,7 +487,7 @@ #endif int -fueword(volatile const void *addr, long *val) +REMAP(fueword)(volatile const void *addr, long *val) { struct thread *td; pmap_t pm; @@ -426,7 +515,7 @@ } int -casueword32(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, +REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, uint32_t new) { struct thread *td; @@ -474,7 +563,7 @@ #ifndef __powerpc64__ int -casueword(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) +REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) { return (casueword32((volatile uint32_t *)addr, old, @@ -482,7 +571,7 @@ } #else int -casueword(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) +REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) { struct thread *td; pmap_t pm; Index: sys/powerpc/powerpc/pmap_dispatch.c =================================================================== --- sys/powerpc/powerpc/pmap_dispatch.c +++ sys/powerpc/powerpc/pmap_dispatch.c @@ -62,7 +62,7 @@ #include #include -static mmu_t mmu_obj; +mmu_t mmu_obj; /* * pmap globals @@ -99,24 +99,14 @@ #define DEFINE_PMAP_IFUNC(ret, func, args) \ DEFINE_IFUNC(, ret, pmap_##func, args) { \ - const struct mmu_kobj *mmu = mmu_obj; \ pmap_##func##_t f; \ - do { \ - f = mmu->funcs->func; \ - if (f != NULL) break; \ - mmu = mmu->base; \ - } while (mmu != NULL); \ + f = PMAP_RESOLVE_FUNC(func); \ return (f != NULL ? f : (pmap_##func##_t)pmap_nomethod);\ } #define DEFINE_DUMPSYS_IFUNC(ret, func, args) \ DEFINE_IFUNC(, ret, dumpsys_##func, args) { \ - const struct mmu_kobj *mmu = mmu_obj; \ pmap_dumpsys_##func##_t f; \ - do { \ - f = mmu->funcs->dumpsys_##func; \ - if (f != NULL) break; \ - mmu = mmu->base; \ - } while (mmu != NULL); \ + f = PMAP_RESOLVE_FUNC(dumpsys_##func); \ return (f != NULL ? f : (pmap_dumpsys_##func##_t)pmap_nomethod);\ } Index: sys/powerpc/powerpc/support.S =================================================================== --- /dev/null +++ sys/powerpc/powerpc/support.S @@ -0,0 +1,539 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018, Matthew Macy + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* + * Assembly variants of various functions, for those that don't need generic C + * implementations. Currently this includes: + * + * - Direct-access versions of copyin/copyout methods. + * - These are used by Radix AIM pmap (ISA 3.0), and Book-E, to avoid + * unnecessary pmap_map_usr_ptr() calls. + */ + +#include "assym.inc" +#include "opt_sched.h" + +#include +#include + +#include +#include +#include +#include +#include + +#ifdef _CALL_ELF +.abiversion _CALL_ELF +#endif + +#ifdef __powerpc64__ +#define LOAD ld +#define STORE std +#define WORD 8 +#define CMPI cmpdi +#define CMPLI cmpldi +/* log_2(8 * WORD) */ +#define LOOP_LOG 6 +#define LOG_WORD 3 +#else +#define LOAD lwz +#define STORE stw +#define WORD 4 +#define CMPI cmpwi +#define CMPLI cmplwi +/* log_2(8 * WORD) */ +#define LOOP_LOG 5 +#define LOG_WORD 2 +#endif + +#ifdef AIM +#define ENTRY_DIRECT(x) ENTRY(x ## _direct) +#else +#define ENTRY_DIRECT(x) ENTRY(x) +#endif + +#ifdef __powerpc64__ +#define PROLOGUE ;\ + mflr %r0 ;\ + std %r0, 16(%r1) ;\ + +#define EPILOGUE ;\ + ld %r0, 16(%r1) ;\ + mtlr %r0 ;\ + blr ;\ + nop + +#define VALIDATE_TRUNCATE_ADDR_COPY VALIDATE_ADDR_COPY +#define VALIDATE_ADDR_COPY(raddr, len) \ + srdi %r0, raddr, 52 ;\ + cmpwi %r0, 1 ;\ + bge- copy_fault ;\ + nop + +#define VALIDATE_ADDR_FUSU(raddr) ;\ + srdi %r0, raddr, 52 ;\ + cmpwi %r0, 1 ;\ + bge- fusufault ;\ + nop + +#else +#define PROLOGUE ;\ + mflr %r0 ;\ + stw %r0, 4(%r1) ;\ + +#define EPILOGUE ;\ + lwz %r0, 4(%r1) ;\ + mtlr %r0 ;\ + blr ;\ + nop + +/* %r0 is temporary */ +/* + * Validate address and length are valid. + * For VALIDATE_ADDR_COPY() have to account for wraparound. + */ +#define VALIDATE_ADDR_COPY(raddr, len) \ + lis %r0, VM_MAXUSER_ADDRESS@h ;\ + ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ + cmplw %r0, raddr ;\ + blt- copy_fault ;\ + add %r0, raddr, len ;\ + cmplw 7, %r0, raddr ;\ + blt- 7, copy_fault ;\ + mtcrf 0x80, %r0 ;\ + bt- 0, copy_fault ;\ + nop + +#define VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) \ + lis %r0, VM_MAXUSER_ADDRESS@h ;\ + ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ + cmplw %r0, raddr ;\ + blt- copy_fault ;\ + sub %r0, %r0, raddr ;\ + cmplw len, %r0 ;\ + isel len, len, %r0, 0 ;\ + +#define VALIDATE_ADDR_FUSU(raddr) \ + lis %r0, VM_MAXUSER_ADDRESS@h ;\ + ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ + cmplw %r0, raddr ;\ + ble- fusufault + +#endif + +#define PCPU(reg) mfsprg reg, 0 + +#define SET_COPYFAULT(raddr, rpcb, len) \ + VALIDATE_ADDR_COPY(raddr, len) ;\ + PCPU(%r9) ;\ + li %r0, COPYFAULT ;\ + LOAD rpcb, PC_CURPCB(%r9) ;\ + STORE %r0, PCB_ONFAULT(rpcb) ;\ + +#define SET_COPYFAULT_TRUNCATE(raddr, rpcb, len)\ + VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) ;\ + PCPU(%r9) ;\ + li %r0, COPYFAULT ;\ + LOAD rpcb, PC_CURPCB(%r9) ;\ + STORE %r0, PCB_ONFAULT(rpcb) + +#define SET_FUSUFAULT(raddr, rpcb) \ + VALIDATE_ADDR_FUSU(raddr) ;\ + PCPU(%r9) ;\ + li %r0, FUSUFAULT ;\ + LOAD rpcb, PC_CURPCB(%r9) ;\ + STORE %r0, PCB_ONFAULT(rpcb) + +#define CLEAR_FAULT_NO_CLOBBER(rpcb) \ + PCPU(%r9) ;\ + LOAD rpcb, PC_CURPCB(%r9) ;\ + li %r0, 0 ;\ + STORE %r0, PCB_ONFAULT(rpcb) + +#define CLEAR_FAULT(rpcb) \ + CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ + li %r3, 0 + +/* + * bcopy(src, dst, len) + * %r3 %r4 %r5 + * + * %r7 is the pcb pointer + * + * %r0 and %r8-%r10 are volatile + * %r11 and %r12 are generally volatile, used in linking and exception + * handling. Can be clobbered here. + * + * Does not allocate or use stack space, but clobbers all volatile registers. + */ + +#define rs %r3 +#define rd %r4 +#define rl %r5 + +#define t1 %r6 +#define t2 %r7 +#define t3 %r8 +#define t4 %r9 +#define t5 %r10 +#define t6 %r11 +#define t7 %r12 +#define t8 %r0 + +#define Thresh WORD * 8 +#define W4 3 +#define W2 2 +#define W1 1 +#define WORDS(n) (32 - LOG_WORD - W##n) +.text +ENTRY(bcopy_generic) + CMPLI 0, %r5, 0 + beq .Lend + dcbtst 0, rd + dcbt 0, rs + CMPLI rl, Thresh + blt .Lsmall + b .Llarge +/* memcpy */ +/* ... */ +.Lsmall: /* < 8 words remaining */ + mtcrf 0x3, rl +.Lsmall_start: + bf WORDS(4), 0f + LOAD t1, 0(rs) + LOAD t2, WORD*1(rs) + LOAD t3, WORD*2(rs) + LOAD t4, WORD*3(rs) + addi rs, rs, WORD*4 + STORE t1, 0(rd) + STORE t2, WORD*1(rd) + STORE t3, WORD*2(rd) + STORE t4, WORD*3(rd) + addi rd, rd, WORD*4 +0: /* < 4 words remaining */ + bf WORDS(2), 1f + LOAD t1, 0(rs) + LOAD t2, WORD*1(rs) + addi rs, rs, WORD*2 + STORE t1, 0(rd) + STORE t2, WORD*1(rd) + addi rd, rd, WORD*2 +1: /* < 2 words remaining */ + bf WORDS(1), 2f + LOAD t1, 0(rs) + addi rs, rs, WORD + STORE t1, 0(rd) + addi rd, rd, WORD +2: /* < 1 word remaining */ +#ifdef __powerpc64__ + bf 29, 3f + lwz t1, 0(rs) + addi rs, rs, 4 + stw t1, 0(rd) + addi rd, rd, 4 +3: /* < 4 bytes remaining */ +#endif + bf 30, 4f + lhz t1, 0(rs) + addi rs, rs, 2 + sth t1, 0(rd) + addi rd, rd, 2 +4: /* < 2 bytes remaining */ + bf 31, .Lout + lbz t1, 0(rs) + addi rs, rs, 1 + stb t1, 0(rd) + addi rd, rd, 1 + b .Lout + + .align 4 +.Llarge: + neg t3, rd + andi. t6, t3, WORD-1 /* Align rd to word size */ + mtctr t6 + sub rl, rl, t6 + beq+ .Llargealigned +1: + lbz t1, 0(rs) + addi rs, rs, 1 + stb t1, 0(rd) + addi rd, rd, 1 + bdnz 1b + +.Llargealigned: + srwi. t2, rl, LOOP_LOG /* length >> log_2(loop_size) => 8W iterations */ + mtcrf 0x3, rl + beq .Lsmall_start + mtctr t2 + b 1f + + .align 5 +1: + LOAD t1, 0(rs) + LOAD t2, WORD(rs) + LOAD t3, WORD*2(rs) + LOAD t4, WORD*3(rs) + LOAD t5, WORD*4(rs) + LOAD t6, WORD*5(rs) + LOAD t7, WORD*6(rs) + LOAD t8, WORD*7(rs) + addi rs, rs, WORD*8 + STORE t1, 0(rd) + STORE t2, WORD*1(rd) + STORE t3, WORD*2(rd) + STORE t4, WORD*3(rd) + STORE t5, WORD*4(rd) + STORE t6, WORD*5(rd) + STORE t7, WORD*6(rd) + STORE t8, WORD*7(rd) + addi rd, rd, WORD*8 + bdnz 1b + + b .Lsmall_start +.Lout: +/* done */ +.Lend: + blr + +/* + * copyout(from_kernel, to_user, len) + * %r3, %r4, %r5 + */ +ENTRY_DIRECT(copyout) + PROLOGUE + SET_COPYFAULT(%r4, %r7, %r5) + bl bcopy_generic + nop + CLEAR_FAULT(%r7) + EPILOGUE + +/* + * copyin(from_user, to_kernel, len) + * %r3, %r4, %r5 + */ +ENTRY_DIRECT(copyin) + PROLOGUE + SET_COPYFAULT(%r3, %r7, %r5) + bl bcopy_generic + nop + CLEAR_FAULT(%r7) + EPILOGUE +/* + * copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) + * %r3 %r4 %r5 %r6 + */ + +ENTRY_DIRECT(copyinstr) + PROLOGUE + SET_COPYFAULT_TRUNCATE(%r3, %r7, %r5) + addi %r9, %r5, 1 + mtctr %r9 + mr %r8, %r3 + addi %r8, %r8, -1 + addi %r4, %r4, -1 + li %r3, ENAMETOOLONG +0: + bdz- 2f + lbzu %r0, 1(%r8) + stbu %r0, 1(%r4) + + // NULL byte reached ? + CMPI %r0, 0 + beq- 1f + b 0b +1: + li %r3, 0 +2: + /* skip storing length if done is NULL */ + CMPI %r6, 0 + beq- 3f + mfctr %r0 + sub %r0, %r9, %r0 + STORE %r0, 0(%r6) +3: + CLEAR_FAULT_NO_CLOBBER(%r7) + EPILOGUE + +ENTRY_DIRECT(subyte) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + stb %r4, 0(%r3) + CLEAR_FAULT(%r7) + EPILOGUE + +#ifndef __powerpc64__ +ENTRY_DIRECT(suword) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + stw %r4, 0(%r3) + CLEAR_FAULT(%r7) + EPILOGUE +#endif + +ENTRY_DIRECT(suword32) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + stw %r4, 0(%r3) + CLEAR_FAULT(%r7) + EPILOGUE + +#ifdef __powerpc64__ +ENTRY_DIRECT(suword64) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + std %r4, 0(%r3) + CLEAR_FAULT(%r7) + EPILOGUE +ENTRY_DIRECT(suword) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + std %r4, 0(%r3) + CLEAR_FAULT(%r7) + EPILOGUE +#endif + +ENTRY_DIRECT(fubyte) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + lbz %r3, 0(%r3) + CLEAR_FAULT_NO_CLOBBER(%r7) + EPILOGUE + +ENTRY_DIRECT(fuword16) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + lhz %r3, 0(%r3) + CLEAR_FAULT_NO_CLOBBER(%r7) + EPILOGUE + +#ifndef __powerpc64__ +ENTRY_DIRECT(fueword) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + lwz %r0, 0(%r3) + stw %r0, 0(%r4) + CLEAR_FAULT(%r7) + EPILOGUE +#endif +ENTRY_DIRECT(fueword32) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + lwz %r0, 0(%r3) + stw %r0, 0(%r4) + CLEAR_FAULT(%r7) + EPILOGUE + +#ifdef __powerpc64__ +ENTRY_DIRECT(fueword) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + ld %r0, 0(%r3) + std %r0, 0(%r4) + CLEAR_FAULT(%r7) + EPILOGUE + +ENTRY_DIRECT(fueword64) + PROLOGUE + SET_FUSUFAULT(%r3, %r7) + ld %r0, 0(%r3) + std %r0, 0(%r4) + CLEAR_FAULT(%r7) + EPILOGUE +#endif + +/* + * casueword(volatile u_long *base, u_long old, u_long *oldp, u_long new) + * %r3 %r4 %r5 %r6 + */ + +#define CASUEWORD32(raddr, rpcb) ;\ + PROLOGUE ;\ + SET_FUSUFAULT(raddr, rpcb) ;\ + li %r8, 0 ;\ +1: ;\ + lwarx %r0, 0, %r3 ;\ + cmplw %r4, %r0 ;\ + bne 2f ;\ + stwcx. %r6, 0, %r3 ;\ + bne- 3f ;\ + b 4f ;\ +2: ;\ + stwcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\ +3: ;\ + li %r8, 1 ;\ +4: ;\ + stw %r0, 0(%r5) ;\ + CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ + mr %r3, %r8 ;\ + EPILOGUE + +ENTRY_DIRECT(casueword32) + CASUEWORD32(%r3, %r7) + +#ifdef __powerpc64__ +#define CASUEWORD64(raddr, rpcb) ;\ + PROLOGUE ;\ + SET_FUSUFAULT(raddr, rpcb) ;\ + li %r8, 0 ;\ +1: ;\ + ldarx %r0, 0, %r3 ;\ + cmpld %r4, %r0 ;\ + bne 2f ;\ + stdcx. %r6, 0, %r3 ;\ + bne- 3f ;\ + b 4f ;\ +2: ;\ + stdcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\ +3: ;\ + li %r8, 1 ;\ +4: ;\ + std %r0, 0(%r5) ;\ + CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ + mr %r3, %r8 ;\ + EPILOGUE + +ENTRY_DIRECT(casueword) + CASUEWORD64(%r3, %r7) + +ENTRY_DIRECT(casueword64) + CASUEWORD64(%r3, %r7) +#else +ENTRY_DIRECT(casueword) + CASUEWORD32(%r3, %r7) +#endif + +_NAKED_ENTRY(fusufault) + CLEAR_FAULT_NO_CLOBBER(%r7) + li %r3, -1 + EPILOGUE + +_NAKED_ENTRY(copy_fault) + CLEAR_FAULT_NO_CLOBBER(%r7) + li %r3, EFAULT + EPILOGUE Index: sys/powerpc/powerpc/trap.c =================================================================== --- sys/powerpc/powerpc/trap.c +++ sys/powerpc/powerpc/trap.c @@ -98,6 +98,9 @@ extern vm_offset_t __startkernel; +extern int copy_fault(void); +extern int fusufault(void); + #ifdef KDB int db_trap_glue(struct trapframe *); /* Called from trap_subr.S */ #endif @@ -590,6 +593,23 @@ jmp_buf *fb; td = curthread; +#if defined(__powerpc64__) || defined(BOOKE) + uintptr_t dispatch = (uintptr_t)td->td_pcb->pcb_onfault; + + if (dispatch == 0) + return (0); + /* Short-circuit radix and Book-E paths. */ + switch (dispatch) { + case COPYFAULT: + frame->srr0 = (uintptr_t)copy_fault; + return (1); + case FUSUFAULT: + frame->srr0 = (uintptr_t)fusufault; + return (1); + default: + break; + } +#endif fb = td->td_pcb->pcb_onfault; if (fb != NULL) { frame->srr0 = (*fb)->_jb[FAULTBUF_LR];