diff --git a/sys/cddl/dev/kinst/riscv/kinst_isa.c b/sys/cddl/dev/kinst/riscv/kinst_isa.c --- a/sys/cddl/dev/kinst/riscv/kinst_isa.c +++ b/sys/cddl/dev/kinst/riscv/kinst_isa.c @@ -14,18 +14,10 @@ #include "kinst.h" -/* - * Per-CPU trampolines used when the interrupted thread is executing with - * interrupts disabled. If an interrupt is raised while executing a trampoline, - * the interrupt thread cannot safely overwrite its trampoline if it hits a - * kinst probe while executing the interrupt handler. - */ -DPCPU_DEFINE_STATIC(uint8_t *, intr_tramp); - /* * The double-breakpoint mechanism needs to save the current probe for the next - * call to kinst_invop(). As with per-CPU trampolines, this also has to be done - * per-CPU when interrupts are disabled. + * call to kinst_invop(). Per-CPU probes are used instead of per-thread probes + * when the thread was executing with interrupts disabled. */ DPCPU_DEFINE_STATIC(struct kinst_probe *, intr_probe); @@ -253,14 +245,14 @@ } static void -kinst_trampoline_populate(struct kinst_probe *kp, uint8_t *tramp) +kinst_trampoline_populate(struct kinst_probe *kp) { static uint16_t nop = MATCH_C_NOP; static uint32_t ebreak = MATCH_EBREAK; int ilen; ilen = kp->kp_md.instlen; - kinst_memcpy(tramp, &kp->kp_savedval, ilen); + kinst_memcpy(kp->kp_tramp, &kp->kp_savedval, ilen); /* * Since we cannot encode large displacements in a single instruction @@ -273,9 +265,9 @@ * Add a NOP after a compressed instruction for padding. */ if (ilen == INSN_C_SIZE) - kinst_memcpy(&tramp[ilen], &nop, INSN_C_SIZE); + kinst_memcpy(&kp->kp_tramp[ilen], &nop, INSN_C_SIZE); - kinst_memcpy(&tramp[INSN_SIZE], &ebreak, INSN_SIZE); + kinst_memcpy(&kp->kp_tramp[INSN_SIZE], &ebreak, INSN_SIZE); fence_i(); } @@ -305,28 +297,44 @@ int kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch) { + static register_t savedstatus[MAXCPU] = {0}; solaris_cpu_t *cpu; struct kinst_probe *kp; - uint8_t *tramp; + register_t sstatus; /* - * Use per-CPU trampolines and probes if the thread executing the - * instruction was executing with interrupts disabled. + * If savedstatus is not 0, we have cached SSTATUS during the first + * breakpoint, so use that instead of the frame's SSTATUS. */ - if ((frame->tf_sstatus & SSTATUS_SPIE) == 0) { - tramp = DPCPU_GET(intr_tramp); + if (savedstatus[curcpu]) + sstatus = savedstatus[curcpu]; + else + sstatus = frame->tf_sstatus; + + if ((sstatus & SSTATUS_SPIE) == 0) { kp = DPCPU_GET(intr_probe); + + /* + * Detect if the breakpoint was triggered by the trampoline, + * and manually set the PC to the next instruction. + */ + if (kp != NULL && addr == (uintptr_t)(kp->kp_tramp + INSN_SIZE)) { + savedstatus[curcpu] = 0; + return (kinst_jump_next_instr(frame, kp)); + } } else { - tramp = curthread->t_kinst_tramp; kp = curthread->t_kinst_curprobe; - } - /* - * Detect if the breakpoint was triggered by the trampoline, and - * manually set the PC to the next instruction. - */ - if (addr == (uintptr_t)(tramp + INSN_SIZE)) - return (kinst_jump_next_instr(frame, kp)); + if (kp != NULL && addr == (uintptr_t)(kp->kp_tramp + INSN_SIZE)) { + /* + * Restore interrupts if they were enabled prior to the + * first breakpoint. + */ + frame->tf_sstatus |= SSTATUS_SPIE; + savedstatus[curcpu] = 0; + return (kinst_jump_next_instr(frame, kp)); + } + } LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) { if ((uintptr_t)kp->kp_patchpoint == addr) @@ -343,26 +351,19 @@ if (kp->kp_md.emulate) return (kinst_emulate(frame, kp)); - if (tramp == NULL) { - /* - * A trampoline allocation failed, so this probe is - * effectively disabled. Restore the original - * instruction. - * - * We can't safely print anything here, but the - * trampoline allocator should have left a breadcrumb in - * the dmesg. - */ - kinst_patch_tracepoint(kp, kp->kp_savedval); - frame->tf_sepc = (register_t)kp->kp_patchpoint; - } else { - kinst_trampoline_populate(kp, tramp); - frame->tf_sepc = (register_t)tramp; - if ((frame->tf_sstatus & SSTATUS_SPIE) == 0) - DPCPU_SET(intr_probe, kp); - else - curthread->t_kinst_curprobe = kp; - } + kinst_trampoline_populate(kp); + frame->tf_sepc = (register_t)kp->kp_tramp; + if ((frame->tf_sstatus & SSTATUS_SPIE) == 0) + DPCPU_SET(intr_probe, kp); + else + curthread->t_kinst_curprobe = kp; + + /* + * Cache the current SSTATUS and clear interrupts for the + * duration of the double breakpoint. + */ + savedstatus[curcpu] = frame->tf_sstatus; + frame->tf_sstatus &= ~SSTATUS_SPIE; return (MATCH_C_NOP); } @@ -556,6 +557,10 @@ kp->kp_patchval = KINST_PATCHVAL; else kp->kp_patchval = KINST_C_PATCHVAL; + if ((kp->kp_tramp = kinst_trampoline_alloc(M_WAITOK)) == NULL) { + KINST_LOG("cannot allocate trampoline for: %p", instr); + return (ENOMEM); + } kinst_instr_dissect(kp, instrsize); kinst_probe_create(kp, lf); @@ -571,32 +576,12 @@ int kinst_md_init(void) { - uint8_t *tramp; - int cpu; - - CPU_FOREACH(cpu) { - tramp = kinst_trampoline_alloc(M_WAITOK); - if (tramp == NULL) - return (ENOMEM); - DPCPU_ID_SET(cpu, intr_tramp, tramp); - } - return (0); } void kinst_md_deinit(void) { - uint8_t *tramp; - int cpu; - - CPU_FOREACH(cpu) { - tramp = DPCPU_ID_GET(cpu, intr_tramp); - if (tramp != NULL) { - kinst_trampoline_dealloc(tramp); - DPCPU_ID_SET(cpu, intr_tramp, NULL); - } - } } /*