Index: head/sys/riscv/include/counter.h =================================================================== --- head/sys/riscv/include/counter.h (revision 317150) +++ head/sys/riscv/include/counter.h (revision 317151) @@ -1,93 +1,97 @@ /*- * Copyright (c) 2012 Konstantin Belousov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_COUNTER_H_ #define _MACHINE_COUNTER_H_ #include #ifdef INVARIANTS #include #endif +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #define counter_enter() critical_enter() #define counter_exit() critical_exit() #ifdef IN_SUBR_COUNTER_C static inline uint64_t counter_u64_read_one(uint64_t *p, int cpu) { return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu)); } static inline uint64_t counter_u64_fetch_inline(uint64_t *p) { uint64_t r; int i; r = 0; for (i = 0; i < mp_ncpus; i++) r += counter_u64_read_one((uint64_t *)p, i); return (r); } /* XXXKIB might interrupt increment */ static void counter_u64_zero_one_cpu(void *arg) { *((uint64_t *)((char *)arg + sizeof(struct pcpu) * PCPU_GET(cpuid))) = 0; } static inline void counter_u64_zero_inline(counter_u64_t c) { smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu, smp_no_rendezvous_barrier, c); } #endif #define counter_u64_add_protected(c, inc) do { \ CRITICAL_ASSERT(curthread); \ *(uint64_t *)zpcpu_get(c) += (inc); \ } while (0) static inline void counter_u64_add(counter_u64_t c, int64_t inc) { counter_enter(); counter_u64_add_protected(c, inc); counter_exit(); } #endif /* ! _MACHINE_COUNTER_H_ */ Index: head/sys/riscv/include/pcpu.h =================================================================== --- head/sys/riscv/include/pcpu.h (revision 317150) +++ head/sys/riscv/include/pcpu.h (revision 317151) @@ -1,87 +1,87 @@ /*- * Copyright (c) 1999 Luoqi Chen * Copyright (c) 2015-2016 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: FreeBSD: src/sys/i386/include/globaldata.h,v 1.27 2001/04/27 * $FreeBSD$ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #include #include #define ALT_STACK_SIZE 128 #define PCPU_MD_FIELDS \ uint32_t pc_pending_ipis; /* IPIs pending to this CPU */ \ - char __pad[125] + char __pad[61] #ifdef _KERNEL struct pcb; struct pcpu; extern struct pcpu *pcpup; static inline struct pcpu * get_pcpu(void) { struct pcpu *pcpu; __asm __volatile("mv %0, gp" : "=&r"(pcpu)); return (pcpu); } static inline struct thread * get_curthread(void) { struct thread *td; __asm __volatile("ld %0, 0(gp)" : "=&r"(td)); return (td); } #define curthread get_curthread() #define PCPU_GET(member) (get_pcpu()->pc_ ## member) #define PCPU_ADD(member, value) (get_pcpu()->pc_ ## member += (value)) #define PCPU_INC(member) PCPU_ADD(member, 1) #define PCPU_PTR(member) (&get_pcpu()->pc_ ## member) #define PCPU_SET(member,value) (get_pcpu()->pc_ ## member = (value)) #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */ Index: head/sys/riscv/riscv/intr_machdep.c =================================================================== --- head/sys/riscv/riscv/intr_machdep.c (revision 317150) +++ head/sys/riscv/riscv/intr_machdep.c (revision 317151) @@ -1,316 +1,317 @@ /*- * Copyright (c) 2015-2016 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include +#include #include #include #include #include #include #include #ifdef SMP #include #endif u_long intrcnt[NIRQS]; size_t sintrcnt = sizeof(intrcnt); char intrnames[NIRQS * (MAXCOMLEN + 1) * 2]; size_t sintrnames = sizeof(intrnames); static struct intr_event *intr_events[NIRQS]; static riscv_intrcnt_t riscv_intr_counters[NIRQS]; static int intrcnt_index; riscv_intrcnt_t riscv_intrcnt_create(const char* name) { riscv_intrcnt_t counter; counter = &intrcnt[intrcnt_index++]; riscv_intrcnt_setname(counter, name); return (counter); } void riscv_intrcnt_setname(riscv_intrcnt_t counter, const char *name) { int i; i = (counter - intrcnt); KASSERT(counter != NULL, ("riscv_intrcnt_setname: NULL counter")); snprintf(intrnames + (MAXCOMLEN + 1) * i, MAXCOMLEN + 1, "%-*s", MAXCOMLEN, name); } static void riscv_mask_irq(void *source) { uintptr_t irq; irq = (uintptr_t)source; switch (irq) { case IRQ_TIMER_SUPERVISOR: csr_clear(sie, SIE_STIE); break; case IRQ_SOFTWARE_USER: csr_clear(sie, SIE_USIE); case IRQ_SOFTWARE_SUPERVISOR: csr_clear(sie, SIE_SSIE); break; #if 0 /* lowRISC TODO */ case IRQ_UART: machine_command(ECALL_IO_IRQ_MASK, 0); break; #endif default: panic("Unknown irq %d\n", irq); } } static void riscv_unmask_irq(void *source) { uintptr_t irq; irq = (uintptr_t)source; switch (irq) { case IRQ_TIMER_SUPERVISOR: csr_set(sie, SIE_STIE); break; case IRQ_SOFTWARE_USER: csr_set(sie, SIE_USIE); break; case IRQ_SOFTWARE_SUPERVISOR: csr_set(sie, SIE_SSIE); break; #if 0 /* lowRISC TODO */ case IRQ_UART: machine_command(ECALL_IO_IRQ_MASK, 1); break; #endif default: panic("Unknown irq %d\n", irq); } } void riscv_init_interrupts(void) { char name[MAXCOMLEN + 1]; int i; for (i = 0; i < NIRQS; i++) { snprintf(name, MAXCOMLEN + 1, "int%d:", i); riscv_intr_counters[i] = riscv_intrcnt_create(name); } } int riscv_setup_intr(const char *name, driver_filter_t *filt, void (*handler)(void*), void *arg, int irq, int flags, void **cookiep) { struct intr_event *event; int error; if (irq < 0 || irq >= NIRQS) panic("%s: unknown intr %d", __func__, irq); event = intr_events[irq]; if (event == NULL) { error = intr_event_create(&event, (void *)(uintptr_t)irq, 0, irq, riscv_mask_irq, riscv_unmask_irq, NULL, NULL, "int%d", irq); if (error) return (error); intr_events[irq] = event; riscv_unmask_irq((void*)(uintptr_t)irq); } error = intr_event_add_handler(event, name, filt, handler, arg, intr_priority(flags), flags, cookiep); if (error) { printf("Failed to setup intr: %d\n", irq); return (error); } riscv_intrcnt_setname(riscv_intr_counters[irq], event->ie_fullname); return (0); } int riscv_teardown_intr(void *ih) { /* TODO */ return (0); } int riscv_config_intr(u_int irq, enum intr_trigger trig, enum intr_polarity pol) { /* There is no configuration for interrupts */ return (0); } void riscv_cpu_intr(struct trapframe *frame) { struct intr_event *event; int active_irq; critical_enter(); KASSERT(frame->tf_scause & EXCP_INTR, ("riscv_cpu_intr: wrong frame passed")); active_irq = (frame->tf_scause & EXCP_MASK); switch (active_irq) { #if 0 /* lowRISC TODO */ case IRQ_UART: #endif case IRQ_SOFTWARE_USER: case IRQ_SOFTWARE_SUPERVISOR: case IRQ_TIMER_SUPERVISOR: event = intr_events[active_irq]; /* Update counters */ atomic_add_long(riscv_intr_counters[active_irq], 1); - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); break; default: event = NULL; } if (!event || TAILQ_EMPTY(&event->ie_handlers) || (intr_event_handle(event, frame) != 0)) printf("stray interrupt %d\n", active_irq); critical_exit(); } #ifdef SMP void riscv_setup_ipihandler(driver_filter_t *filt) { riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE_SUPERVISOR, INTR_TYPE_MISC, NULL); } void riscv_unmask_ipi(void) { csr_set(sie, SIE_SSIE); } /* Sending IPI */ static void ipi_send(struct pcpu *pc, int ipi) { CTR3(KTR_SMP, "%s: cpu=%d, ipi=%x", __func__, pc->pc_cpuid, ipi); atomic_set_32(&pc->pc_pending_ipis, ipi); sbi_send_ipi(pc->pc_cpuid); CTR1(KTR_SMP, "%s: sent", __func__); } void ipi_all_but_self(u_int ipi) { cpuset_t other_cpus; other_cpus = all_cpus; CPU_CLR(PCPU_GET(cpuid), &other_cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); ipi_selected(other_cpus, ipi); } void ipi_cpu(int cpu, u_int ipi) { cpuset_t cpus; CPU_ZERO(&cpus); CPU_SET(cpu, &cpus); CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x\n", __func__, cpu, ipi); ipi_send(cpuid_to_pcpu[cpu], ipi); } void ipi_selected(cpuset_t cpus, u_int ipi) { struct pcpu *pc; CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { if (CPU_ISSET(pc->pc_cpuid, &cpus)) { CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc, ipi); ipi_send(pc, ipi); } } } #endif Index: head/sys/riscv/riscv/minidump_machdep.c =================================================================== --- head/sys/riscv/riscv/minidump_machdep.c (revision 317150) +++ head/sys/riscv/riscv/minidump_machdep.c (revision 317151) @@ -1,58 +1,61 @@ /*- * Copyright (c) 2006 Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_watchdog.h" #include #include #include #include #include #include #include #include +#include + #include #include #include #include #include + #include #include #include #include CTASSERT(sizeof(struct kerneldumpheader) == 512); CTASSERT(sizeof(*vm_page_dump) == 8); int minidumpsys(struct dumperinfo *di) { panic("minidumpsys"); }