Index: sys/conf/files.powerpc =================================================================== --- sys/conf/files.powerpc +++ sys/conf/files.powerpc @@ -194,6 +194,7 @@ powerpc/powernv/powernv_centaur.c optional powernv powerpc/powernv/powernv_xscom.c optional powernv powerpc/powerpc/altivec.c optional powerpc | powerpc64 +powerpc/powerpc/htm.c optional powerpc64 powerpc/powerpc/autoconf.c standard powerpc/powerpc/bus_machdep.c standard powerpc/powerpc/busdma_machdep.c standard Index: sys/conf/options.powerpc =================================================================== --- sys/conf/options.powerpc +++ sys/conf/options.powerpc @@ -35,3 +35,5 @@ AGP_DEBUG opt_agp.h MIKROTIK + +HTM opt_platform.h Index: sys/powerpc/aim/aim_machdep.c =================================================================== --- sys/powerpc/aim/aim_machdep.c +++ sys/powerpc/aim/aim_machdep.c @@ -105,6 +105,7 @@ #include #include +#include #ifndef __powerpc64__ #include #endif @@ -630,6 +631,7 @@ jmp_buf resetjb; struct thread *fputd; struct thread *vectd; + struct thread *htmtd; register_t hid0; register_t msr; register_t saved_msr; @@ -641,10 +643,13 @@ saved_msr = mfmsr(); fputd = PCPU_GET(fputhread); vectd = PCPU_GET(vecthread); + htmtd = PCPU_GET(htmthread); if (fputd != NULL) save_fpu(fputd); if (vectd != NULL) save_vec(vectd); + if (htmtd != NULL) + save_htm(htmtd); if (setjmp(resetjb) == 0) { sprgs[0] = mfspr(SPR_SPRG0); sprgs[1] = mfspr(SPR_SPRG1); Index: sys/powerpc/conf/GENERIC64 =================================================================== --- sys/powerpc/conf/GENERIC64 +++ sys/powerpc/conf/GENERIC64 @@ -82,6 +82,7 @@ options KDTRACE_HOOKS # Kernel DTrace hooks options DDB_CTF # Kernel ELF linker loads CTF data options INCLUDE_CONFIG_FILE # Include this file in kernel +options HTM # Hardware Transactional Memory Support # Debugging support. Always need this: options KDB # Enable kernel debugger support. Index: sys/powerpc/conf/NOTES =================================================================== --- sys/powerpc/conf/NOTES +++ sys/powerpc/conf/NOTES @@ -40,6 +40,9 @@ options FPU_EMU +# Enable hardware/restricted transactional memory support in userspace +options HTM + #options MPC85XX options POWERMAC #NewWorld Apple PowerMacs #options PS3 #Sony Playstation 3 Index: sys/powerpc/include/htm.h =================================================================== --- /dev/null +++ sys/powerpc/include/htm.h @@ -0,0 +1,58 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018 Breno Leitao + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_HTM_H_ +#define _MACHINE_HTM_H_ + +/* Enable HTM for the very first time, through a HTM Facility Unavailable */ +void enable_htm_thread(struct thread *td); + +/* Save the HTM states prior to a process leaving the CPU */ +void save_htm(struct thread *td); + +/* Restore the HTM state before the process is scheduled */ +void restore_htm(struct thread *td); + +/* Check if HTM is enabled in the MSR */ +bool htm_enabled(register_t msr); + +/* Check if HTM is in transactional state*/ +bool htm_transactional(register_t msr); + +/* Check if HTM is in suspended state*/ +bool htm_suspended(register_t msr); + +/* enable MSR[HTM] on the current CPU */ +void enable_htm_current_cpu(void); + +void tabort(void); + +#endif /* _MACHINE_HTM_H_ */ + Index: sys/powerpc/include/pcb.h =================================================================== --- sys/powerpc/include/pcb.h +++ sys/powerpc/include/pcb.h @@ -56,6 +56,7 @@ #define PCB_VEC 0x4 /* Process had Altivec initialized */ #define PCB_VSX 0x8 /* Process had VSX initialized */ #define PCB_CDSCR 0x10 /* Process had Custom DSCR initialized */ +#define PCB_HTM 0x20 /* Process had HTM initialized */ struct fpu { union { double fpr; @@ -73,6 +74,11 @@ } pcb_vec __aligned(16); /* Vector processor */ unsigned int pcb_veccpu; /* which CPU had our vector stuff. */ + struct htm { + uint64_t tfhar; + uint64_t texasr; + uint64_t tfiar; + } pcb_htm; union { struct { Index: sys/powerpc/include/pcpu.h =================================================================== --- sys/powerpc/include/pcpu.h +++ sys/powerpc/include/pcpu.h @@ -45,6 +45,7 @@ struct pmap *pc_curpmap; /* current pmap */ \ struct thread *pc_fputhread; /* current fpu user */ \ struct thread *pc_vecthread; /* current vec user */ \ + struct thread *pc_htmthread; /* current htm user */ \ uintptr_t pc_hwref; \ int pc_bsp; \ volatile int pc_awake; \ Index: sys/powerpc/include/psl.h =================================================================== --- sys/powerpc/include/psl.h +++ sys/powerpc/include/psl.h @@ -70,6 +70,8 @@ #ifdef __powerpc64__ #define PSL_SF 0x8000000000000000UL /* 64-bit addressing */ #define PSL_HV 0x1000000000000000UL /* hyper-privileged mode */ +#define PSL_HTM 0x0000000100000000UL /* Hardware Transactional Memory available */ +#define PSL_HTM_TS 0x0000000600000000UL /* Hardware Transactional Memory State */ #endif #define PSL_POW 0x00040000UL /* power management */ @@ -90,6 +92,14 @@ #define PSL_FE_PREC (PSL_FE0 | PSL_FE1) /* precise */ #define PSL_FE_DFLT PSL_FE_DIS /* default == none */ +/* + * Hardware Transactional Memory States + */ +#define PSL_HTM_TS_NT 0x0000000000000000UL /* Non Trasactional */ +#define PSL_HTM_TS_SU 0x0000000200000000UL /* Suspended */ +#define PSL_HTM_TS_TR 0x0000000400000000UL /* Transactional */ +#define PSL_HTM_TS_RE 0x0000000600000000UL /* Reserved */ + #ifndef LOCORE extern register_t psl_kernset; /* Default MSR values for kernel */ extern register_t psl_userset; /* Default MSR values for userland */ Index: sys/powerpc/include/spr.h =================================================================== --- sys/powerpc/include/spr.h +++ sys/powerpc/include/spr.h @@ -121,6 +121,9 @@ #define SPR_EIE 0x050 /* ..8 Exception Interrupt ??? */ #define SPR_EID 0x051 /* ..8 Exception Interrupt ??? */ #define SPR_NRI 0x052 /* ..8 Exception Interrupt ??? */ +#define SPR_TFHAR 0x080 /* Transaction Failure Handler Address Register */ +#define SPR_TFIAR 0x081 /* Transaction Failure Instruction Address Register */ +#define SPR_TEXASR 0x082 /* Transaction EXception And Status Register */ #define SPR_FSCR 0x099 /* Facility Status and Control Register */ #define FSCR_IC_MASK 0xFF00000000000000ULL /* FSCR[0:7] is Interrupt Cause */ #define FSCR_IC_FP 0x0000000000000000ULL /* FP unavailable */ Index: sys/powerpc/include/trap.h =================================================================== --- sys/powerpc/include/trap.h +++ sys/powerpc/include/trap.h @@ -126,6 +126,7 @@ * 2.05. */ +#define EXC_PGM_BAD_THING (1UL << 21) #define EXC_PGM_FPENABLED (1UL << 20) #define EXC_PGM_ILLEGAL (1UL << 19) #define EXC_PGM_PRIV (1UL << 18) Index: sys/powerpc/powerpc/genassym.c =================================================================== --- sys/powerpc/powerpc/genassym.c +++ sys/powerpc/powerpc/genassym.c @@ -204,6 +204,7 @@ ASSYM(PCB_FPU, PCB_FPU); ASSYM(PCB_VEC, PCB_VEC); ASSYM(PCB_CDSCR, PCB_CDSCR); +ASSYM(PCB_HTM, PCB_HTM); ASSYM(PCB_AIM_USR_VSID, offsetof(struct pcb, pcb_cpu.aim.usr_vsid)); ASSYM(PCB_BOOKE_DBCR0, offsetof(struct pcb, pcb_cpu.booke.dbcr0)); @@ -242,6 +243,7 @@ #if defined(AIM) && defined(__powerpc64__) ASSYM(PSL_SF, PSL_SF); ASSYM(PSL_HV, PSL_HV); +ASSYM(PSL_HTM, PSL_HTM); #endif ASSYM(PSL_POW, PSL_POW); Index: sys/powerpc/powerpc/htm.c =================================================================== --- /dev/null +++ sys/powerpc/powerpc/htm.c @@ -0,0 +1,207 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (C) 2018 Breno Leitao + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "opt_platform.h" + +static void htm_restore_sprs(struct pcb *pcb); + +bool +htm_enabled(register_t msr) +{ + if ((msr & PSL_HTM) == PSL_HTM) + return true; + + return false; +} + +bool +htm_transactional(register_t msr) +{ + if ((msr & PSL_HTM_TS_TR) == PSL_HTM_TS_TR) + return true; + + return false; +} + +bool +htm_suspended(register_t msr) +{ + if ((msr & PSL_HTM_TS_SU) == PSL_HTM_TS_SU) + return true; + + return false; +} + +/* Enable HTM flag on current CPU MSR */ +void +enable_htm_current_cpu() +{ + register_t msr; + + msr = mfmsr(); + msr |= PSL_HTM; + mtmsrd(msr); +} + +/* Save HTM Special registers to PCB */ +static void +save_htm_sprs(struct pcb *pcb) +{ + pcb->pcb_htm.tfhar = mfspr(SPR_TFHAR); + pcb->pcb_htm.texasr = mfspr(SPR_TEXASR); + pcb->pcb_htm.tfiar = mfspr(SPR_TFIAR); +} + +/* External function that save HTM SPRs for a specific thread */ +void +save_htm(struct thread *td) +{ + struct pcb *pcb; + + /* This check might be expensive. */ + if (!htm_enabled(mfmsr())) { + enable_htm_current_cpu(); + } + + pcb = td->td_pcb; + save_htm_sprs(pcb); + + /* + * Clear the current htm thread and pcb's CPU id + */ + PCPU_SET(htmthread, NULL); +} + +/* Enable HTM on a thread for the very first time*/ +void +enable_htm_thread(struct thread *td) +{ + struct pcb *pcb; + struct trapframe *tf; + + /* Transaction already active */ + if (htm_transactional(mfmsr())) { + return; + } + + pcb = td->td_pcb; + tf = trapframe(td); + + /* + * Save the thread's HTM CPU number, and set the CPU's current + * vector thread + */ + PCPU_SET(htmthread, td); + + /* + * Enable the HTM feature unit for when the thread returns from the + * exception. If this is the first time the unit has been used by + * the thread, initialise the SPR registers to 0, and + * set the flag to indicate that the vector unit is in use. + */ + tf->srr1 |= PSL_HTM; + + if (!(pcb->pcb_flags & PCB_HTM)) { + memset(&pcb->pcb_htm, 0, sizeof pcb->pcb_htm); + pcb->pcb_flags |= PCB_HTM; + } + + /* + * Enable HTM on current MSR since we are going to access HTM SPRs + */ + enable_htm_current_cpu(); + + htm_restore_sprs(pcb); +} + + +/* Restore HTM on a task that is being scheduled */ +void +restore_htm(struct thread *td) +{ + struct pcb *pcb; + register_t msr; + + msr = mfmsr(); + + /* Transaction active. No need to restore SPRs */ + if (htm_transactional(msr)) { + panic("ERROR! Restoring on HTM active\n"); + } + + /* On context switch, MSR[HTM] will be disabled, enable it */ + if (!htm_enabled(msr)){ + enable_htm_current_cpu(); + } + + pcb = td->td_pcb; + + htm_restore_sprs(pcb); +} + +/* Restore HTM Special registers to CPU from PCB area */ +static void +htm_restore_sprs(struct pcb *pcb) +{ + if (htm_transactional(mfmsr())) { + panic("HTM: Cannot restore HTM SPRs on active transaction.\n"); + } + + mtspr(SPR_TFHAR, pcb->pcb_htm.tfhar); + mtspr(SPR_TEXASR, pcb->pcb_htm.texasr); + mtspr(SPR_TFIAR, pcb->pcb_htm.tfiar); + + isync(); +} + +void +tabort() +{ + register_t msr; + + msr = mfmsr(); + + if (!htm_enabled(msr)){ + enable_htm_current_cpu(); + } + + /* Calling instruction "tabort. r0" */ + __asm __volatile(".long 0x7c00071d\n\t"); +} Index: sys/powerpc/powerpc/swtch64.S =================================================================== --- sys/powerpc/powerpc/swtch64.S +++ sys/powerpc/powerpc/swtch64.S @@ -145,10 +145,20 @@ lwz %r7,PCB_FLAGS(%r17) /* Save Altivec context if needed */ andi. %r7, %r7, PCB_VEC - beq .L2 + beq .L11 bl save_vec nop +.L11: +#if defined(HTM) + mr %r3,%r14 /* restore old thread ptr */ + lwz %r7,PCB_FLAGS(%r17) + /* Save HTM SPR context if needed */ + andi. %r7, %r7, PCB_HTM + beq .L2 + bl save_htm + nop +#endif .L2: mr %r3,%r14 /* restore old thread ptr */ bl pmap_deactivate /* Deactivate the current pmap */ @@ -206,10 +216,21 @@ lwz %r6, PCB_FLAGS(%r17) /* Restore Custom DSCR if needed */ andi. %r6, %r6, PCB_CDSCR - beq .L4 + beq .L32 ld %r6, PCB_DSCR(%r17) /* Load the DSCR register*/ mtspr SPR_DSCR, %r6 +.L32: +#if defined(HTM) + lwz %r6, PCB_FLAGS(%r17) + /* Restore HTM context if needed */ + andi. %r6, %r6, PCB_HTM + beq .L4 + mr %r3, %r13 /* Pass curthread to restore_htm*/ + bl restore_htm + nop + +#endif /* thread to restore is in r3 */ .L4: addi %r1,%r1,48 Index: sys/powerpc/powerpc/trap.c =================================================================== --- sys/powerpc/powerpc/trap.c +++ sys/powerpc/powerpc/trap.c @@ -63,6 +63,7 @@ #include #include +#include #include #include #include @@ -73,6 +74,8 @@ #include #include +#include "opt_platform.h" + /* Below matches setjmp.S */ #define FAULTBUF_LR 21 #define FAULTBUF_R1 1 @@ -169,6 +172,16 @@ "\020L3DAT\017APE\016DPE\015TEA\014b20\013b21\012b22\011b23" \ "\010b24\007b25\006b26\005b27\004b28\003b29\002b30\001b31" +static inline bool +frame_is_bad_thing(struct trapframe *frame) +{ +#ifdef AIM + return (frame->exc == EXC_PGM && frame->srr1 & EXC_PGM_BAD_THING); +#else + return 0; +#endif +} + static const char * trapname(u_int vector) @@ -225,6 +238,15 @@ CTR3(KTR_TRAP, "trap: %s type=%s (%s)", td->td_name, trapname(type), user ? "user" : "kernel"); +#ifdef HTM + if (td->td_pcb->pcb_flags & PCB_HTM) + enable_htm_current_cpu(); + + /* There is a trap inside the transaction. Dooming it */ + if (htm_transactional(frame->srr1)) + tabort(); +#endif + #ifdef KDTRACE_HOOKS /* * A trap can occur while DTrace executes a probe. Before @@ -275,6 +297,14 @@ break; case EXC_SC: +#ifdef HTM + /* It is not allowed to call SC inside a transaction */ + if (htm_transactional(frame->srr1)) { + sig = SIGILL; + ucode = ILL_ILLTRP; + break; + } +#endif syscall(frame); break; @@ -303,9 +333,12 @@ case EXC_FAC: fscr = mfspr(SPR_FSCR); +#ifdef HTM if ((fscr & FSCR_IC_MASK) == FSCR_IC_HTM) { - CTR0(KTR_TRAP, "Hardware Transactional Memory subsystem disabled"); + enable_htm_thread(td); + break; } +#endif sig = SIGILL; ucode = ILL_ILLOPC; break; @@ -350,6 +383,13 @@ case EXC_PGM: /* Identify the trap reason */ if (frame_is_trap_inst(frame)) { +#ifdef HTM + if (htm_transactional(frame->srr1)) { + sig = SIGILL; + ucode = ILL_ILLTRP; + break; + } +#endif #ifdef KDTRACE_HOOKS inst = fuword32((const void *)frame->srr0); if (inst == 0x0FFFDDDD && @@ -360,6 +400,10 @@ #endif sig = SIGTRAP; ucode = TRAP_BRKPT; + } else if (frame_is_bad_thing(frame)) { + CTR1(KTR_TRAP, "%s: Bad thing exception from userspace\n", td->td_name); + sig = SIGILL; + ucode = ILL_ILLOPC; } else { sig = ppc_instr_emulate(frame, td->td_pcb); if (sig == SIGILL) { @@ -392,6 +436,8 @@ ("kernel trap doesn't have ucred")); switch (type) { case EXC_PGM: + if (frame_is_bad_thing(frame)) + panic("Bad thing exception in kernel space\n"); #ifdef KDTRACE_HOOKS if (frame_is_trap_inst(frame)) { if (*(uint32_t *)frame->srr0 == EXC_DTRACE) { @@ -426,6 +472,9 @@ if (handle_onfault(frame)) return; break; + case EXC_FAC: + fscr = mfspr(SPR_FSCR); + panic("Unavailability Facility in Kernel space. FSCR = 0x%" PRIxPTR, fscr); default: break; } @@ -915,6 +964,7 @@ { if (!(frame->srr1 & PSL_PR) + && !htm_suspended(frame->srr1) && (frame->exc == EXC_TRC || frame->exc == EXC_RUNMODETRC || frame_is_trap_inst(frame) || frame->exc == EXC_BPT