diff --git a/sys/riscv/include/asm.h b/sys/riscv/include/asm.h index 4b7ab0765272..c119e299a0a5 100644 --- a/sys/riscv/include/asm.h +++ b/sys/riscv/include/asm.h @@ -1,72 +1,77 @@ /*- * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ASM_H_ #define _MACHINE_ASM_H_ #undef __FBSDID #if !defined(lint) && !defined(STRIP_FBSDID) #define __FBSDID(s) .ident s #else #define __FBSDID(s) /* nothing */ #endif /* not lint and not STRIP_FBSDID */ #define _C_LABEL(x) x #define ENTRY(sym) \ .text; .globl sym; .type sym,@function; .align 4; sym: .cfi_startproc; #define END(sym) .cfi_endproc; .size sym, . - sym #define EENTRY(sym) \ .globl sym; sym: #define EEND(sym) #define WEAK_REFERENCE(sym, alias) \ .weak alias; \ .set alias,sym #define SET_FAULT_HANDLER(handler, tmp) \ ld tmp, PC_CURTHREAD(tp); \ ld tmp, TD_PCB(tmp); /* Load the pcb */ \ sd handler, PCB_ONFAULT(tmp) /* Set the handler */ #define ENTER_USER_ACCESS(tmp) \ li tmp, SSTATUS_SUM; \ csrs sstatus, tmp #define EXIT_USER_ACCESS(tmp) \ li tmp, SSTATUS_SUM; \ csrc sstatus, tmp +#define SBI_CALL(ext, func) \ + li a7, ext; \ + li a6, func; \ + ecall + #endif /* _MACHINE_ASM_H_ */ diff --git a/sys/riscv/include/cpu.h b/sys/riscv/include/cpu.h index 0c33adb2abcd..a204b21a4a74 100644 --- a/sys/riscv/include/cpu.h +++ b/sys/riscv/include/cpu.h @@ -1,110 +1,114 @@ /*- * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_CPU_H_ #define _MACHINE_CPU_H_ +#ifndef LOCORE #include #include #include +#endif #define TRAPF_PC(tfp) ((tfp)->tf_sepc) #define TRAPF_USERMODE(tfp) (((tfp)->tf_sstatus & SSTATUS_SPP) == 0) #define cpu_getstack(td) ((td)->td_frame->tf_sp) #define cpu_setstack(td, sp) ((td)->td_frame->tf_sp = (sp)) #define cpu_spinwait() /* nothing */ #define cpu_lock_delay() DELAY(1) /* * Core manufacturer IDs, as reported by the mvendorid CSR. */ #define MVENDORID_UNIMPL 0x0 #define MVENDORID_SIFIVE 0x489 #define MVENDORID_THEAD 0x5b7 /* * Micro-architecture ID register, marchid. * * IDs for open-source implementations are allocated globally. Commercial IDs * will have the most-significant bit set. */ #define MARCHID_UNIMPL 0x0 #define MARCHID_MSB (1ul << (XLEN - 1)) #define MARCHID_OPENSOURCE(v) (v) #define MARCHID_COMMERCIAL(v) (MARCHID_MSB | (v)) #define MARCHID_IS_OPENSOURCE(m) (((m) & MARCHID_MSB) == 0) /* * Open-source marchid values. * * https://github.com/riscv/riscv-isa-manual/blob/master/marchid.md */ #define MARCHID_UCB_ROCKET MARCHID_OPENSOURCE(1) #define MARCHID_UCB_BOOM MARCHID_OPENSOURCE(2) #define MARCHID_UCB_SPIKE MARCHID_OPENSOURCE(5) #define MARCHID_UCAM_RVBS MARCHID_OPENSOURCE(10) /* SiFive marchid values */ #define MARCHID_SIFIVE_U7 MARCHID_COMMERCIAL(7) /* * MMU virtual-addressing modes. Support for each level implies the previous, * so Sv48-enabled systems MUST support Sv39, etc. */ #define MMU_SV39 0x1 /* 3-level paging */ #define MMU_SV48 0x2 /* 4-level paging */ #define MMU_SV57 0x4 /* 5-level paging */ #ifdef _KERNEL +#ifndef LOCORE extern char btext[]; extern char etext[]; void cpu_halt(void) __dead2; void cpu_reset(void) __dead2; void fork_trampoline(void); void identify_cpu(u_int cpu); void printcpuinfo(u_int cpu); static __inline uint64_t get_cyclecount(void) { return (rdcycle()); } -#endif +#endif /* !LOCORE */ +#endif /* _KERNEL */ #endif /* !_MACHINE_CPU_H_ */ diff --git a/sys/riscv/include/sbi.h b/sys/riscv/include/sbi.h index c8093238e268..7b103b2e0dcf 100644 --- a/sys/riscv/include/sbi.h +++ b/sys/riscv/include/sbi.h @@ -1,245 +1,248 @@ /*- * Copyright (c) 2016-2017 Ruslan Bukin * All rights reserved. * Copyright (c) 2019 Mitchell Horne * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_SBI_H_ #define _MACHINE_SBI_H_ /* SBI Specification Version */ #define SBI_SPEC_VERS_MAJOR_OFFSET 24 #define SBI_SPEC_VERS_MAJOR_MASK (0x7F << SBI_SPEC_VERS_MAJOR_OFFSET) #define SBI_SPEC_VERS_MINOR_OFFSET 0 #define SBI_SPEC_VERS_MINOR_MASK (0xFFFFFF << SBI_SPEC_VERS_MINOR_OFFSET) /* SBI Implementation IDs */ #define SBI_IMPL_ID_BBL 0 #define SBI_IMPL_ID_OPENSBI 1 #define SBI_IMPL_ID_XVISOR 2 #define SBI_IMPL_ID_KVM 3 #define SBI_IMPL_ID_RUSTSBI 4 #define SBI_IMPL_ID_DIOSIX 5 #define SBI_IMPL_ID_COFFER 6 #define SBI_IMPL_ID_XEN_PROJECT 7 #define SBI_IMPL_ID_POLARFIRE_HSS 8 #define SBI_IMPL_ID_COREBOOT 9 #define SBI_IMPL_ID_OREBOOT 10 #define SBI_IMPL_ID_BHYVE 11 /* SBI Error Codes */ #define SBI_SUCCESS 0 #define SBI_ERR_FAILURE -1 #define SBI_ERR_NOT_SUPPORTED -2 #define SBI_ERR_INVALID_PARAM -3 #define SBI_ERR_DENIED -4 #define SBI_ERR_INVALID_ADDRESS -5 #define SBI_ERR_ALREADY_AVAILABLE -6 /* SBI Base Extension */ #define SBI_EXT_ID_BASE 0x10 #define SBI_BASE_GET_SPEC_VERSION 0 #define SBI_BASE_GET_IMPL_ID 1 #define SBI_BASE_GET_IMPL_VERSION 2 #define SBI_BASE_PROBE_EXTENSION 3 #define SBI_BASE_GET_MVENDORID 4 #define SBI_BASE_GET_MARCHID 5 #define SBI_BASE_GET_MIMPID 6 /* Timer (TIME) Extension */ #define SBI_EXT_ID_TIME 0x54494D45 #define SBI_TIME_SET_TIMER 0 /* IPI (IPI) Extension */ #define SBI_EXT_ID_IPI 0x735049 #define SBI_IPI_SEND_IPI 0 /* RFENCE (RFNC) Extension */ #define SBI_EXT_ID_RFNC 0x52464E43 #define SBI_RFNC_REMOTE_FENCE_I 0 #define SBI_RFNC_REMOTE_SFENCE_VMA 1 #define SBI_RFNC_REMOTE_SFENCE_VMA_ASID 2 #define SBI_RFNC_REMOTE_HFENCE_GVMA_VMID 3 #define SBI_RFNC_REMOTE_HFENCE_GVMA 4 #define SBI_RFNC_REMOTE_HFENCE_VVMA_ASID 5 #define SBI_RFNC_REMOTE_HFENCE_VVMA 6 /* Hart State Management (HSM) Extension */ #define SBI_EXT_ID_HSM 0x48534D #define SBI_HSM_HART_START 0 #define SBI_HSM_HART_STOP 1 #define SBI_HSM_HART_STATUS 2 #define SBI_HSM_STATUS_STARTED 0 #define SBI_HSM_STATUS_STOPPED 1 #define SBI_HSM_STATUS_START_PENDING 2 #define SBI_HSM_STATUS_STOP_PENDING 3 /* System Reset (SRST) Extension */ #define SBI_EXT_ID_SRST 0x53525354 #define SBI_SRST_SYSTEM_RESET 0 #define SBI_SRST_TYPE_SHUTDOWN 0 #define SBI_SRST_TYPE_COLD_REBOOT 1 #define SBI_SRST_TYPE_WARM_REBOOT 2 #define SBI_SRST_REASON_NONE 0 #define SBI_SRST_REASON_SYSTEM_FAILURE 1 /* Legacy Extensions */ #define SBI_SET_TIMER 0 #define SBI_CONSOLE_PUTCHAR 1 #define SBI_CONSOLE_GETCHAR 2 #define SBI_CLEAR_IPI 3 #define SBI_SEND_IPI 4 #define SBI_REMOTE_FENCE_I 5 #define SBI_REMOTE_SFENCE_VMA 6 #define SBI_REMOTE_SFENCE_VMA_ASID 7 #define SBI_SHUTDOWN 8 +#ifndef LOCORE + #define SBI_CALL0(e, f) SBI_CALL5(e, f, 0, 0, 0, 0, 0) #define SBI_CALL1(e, f, p1) SBI_CALL5(e, f, p1, 0, 0, 0, 0) #define SBI_CALL2(e, f, p1, p2) SBI_CALL5(e, f, p1, p2, 0, 0, 0) #define SBI_CALL3(e, f, p1, p2, p3) SBI_CALL5(e, f, p1, p2, p3, 0, 0) #define SBI_CALL4(e, f, p1, p2, p3, p4) SBI_CALL5(e, f, p1, p2, p3, p4, 0) #define SBI_CALL5(e, f, p1, p2, p3, p4, p5) sbi_call(e, f, p1, p2, p3, p4, p5) /* * Documentation available at * https://github.com/riscv/riscv-sbi-doc/blob/master/riscv-sbi.adoc */ struct sbi_ret { long error; long value; }; static __inline struct sbi_ret sbi_call(uint64_t arg7, uint64_t arg6, uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) { struct sbi_ret ret; register uintptr_t a0 __asm ("a0") = (uintptr_t)(arg0); register uintptr_t a1 __asm ("a1") = (uintptr_t)(arg1); register uintptr_t a2 __asm ("a2") = (uintptr_t)(arg2); register uintptr_t a3 __asm ("a3") = (uintptr_t)(arg3); register uintptr_t a4 __asm ("a4") = (uintptr_t)(arg4); register uintptr_t a6 __asm ("a6") = (uintptr_t)(arg6); register uintptr_t a7 __asm ("a7") = (uintptr_t)(arg7); __asm __volatile( \ "ecall" \ :"+r"(a0), "+r"(a1) \ :"r"(a2), "r"(a3), "r"(a4), "r"(a6), "r"(a7) \ :"memory"); ret.error = a0; ret.value = a1; return (ret); } /* Base extension functions. */ static __inline long sbi_probe_extension(long id) { return (SBI_CALL1(SBI_EXT_ID_BASE, SBI_BASE_PROBE_EXTENSION, id).value); } /* TIME extension functions. */ void sbi_set_timer(uint64_t val); /* IPI extension functions. */ void sbi_send_ipi(const u_long *hart_mask); /* RFENCE extension functions. */ void sbi_remote_fence_i(const u_long *hart_mask); void sbi_remote_sfence_vma(const u_long *hart_mask, u_long start, u_long size); void sbi_remote_sfence_vma_asid(const u_long *hart_mask, u_long start, u_long size, u_long asid); /* Hart State Management extension functions. */ /* * Start execution on the specified hart at physical address start_addr. The * register a0 will contain the hart's ID, and a1 will contain the value of * priv. */ int sbi_hsm_hart_start(u_long hart, u_long start_addr, u_long priv); /* * Stop execution on the current hart. Interrupts should be disabled, or this * function may return. */ void sbi_hsm_hart_stop(void); /* * Get the execution status of the specified hart. The status will be one of: * - SBI_HSM_STATUS_STARTED * - SBI_HSM_STATUS_STOPPED * - SBI_HSM_STATUS_START_PENDING * - SBI_HSM_STATUS_STOP_PENDING */ int sbi_hsm_hart_status(u_long hart); /* System Reset extension functions. */ /* * Reset the system based on the following 'type' and 'reason' chosen from: * - SBI_SRST_TYPE_SHUTDOWN * - SBI_SRST_TYPE_COLD_REBOOT * - SBI_SRST_TYPE_WARM_REBOOT * - SBI_SRST_REASON_NONE * - SBI_SRST_REASON_SYSTEM_FAILURE */ void sbi_system_reset(u_long reset_type, u_long reset_reason); /* Legacy extension functions. */ static __inline void sbi_console_putchar(int ch) { (void)SBI_CALL1(SBI_CONSOLE_PUTCHAR, 0, ch); } static __inline int sbi_console_getchar(void) { /* * XXX: The "error" is returned here because legacy SBI functions * continue to return their value in a0. */ return (SBI_CALL0(SBI_CONSOLE_GETCHAR, 0).error); } void sbi_print_version(void); void sbi_init(void); +#endif /* !LOCORE */ #endif /* !_MACHINE_SBI_H_ */ diff --git a/sys/riscv/riscv/identcpu.c b/sys/riscv/riscv/identcpu.c index e02907092b56..54e008122eab 100644 --- a/sys/riscv/riscv/identcpu.c +++ b/sys/riscv/riscv/identcpu.c @@ -1,557 +1,569 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015-2016 Ruslan Bukin * All rights reserved. * Copyright (c) 2022 Mitchell Horne * Copyright (c) 2023 The FreeBSD Foundation * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Portions of this software were developed by Mitchell Horne * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif const char machine[] = "riscv"; SYSCTL_CONST_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD | CTLFLAG_CAPRD, machine, "Machine class"); /* Hardware implementation info. These values may be empty. */ register_t mvendorid; /* The CPU's JEDEC vendor ID */ register_t marchid; /* The architecture ID */ register_t mimpid; /* The implementation ID */ u_int mmu_caps; /* Supervisor-mode extension support. */ bool has_hyp; bool __read_frequently has_sstc; bool __read_frequently has_sscofpmf; bool has_svpbmt; struct cpu_desc { const char *cpu_mvendor_name; const char *cpu_march_name; u_int isa_extensions; /* Single-letter extensions. */ u_int mmu_caps; u_int smode_extensions; #define SV_SSTC (1 << 0) #define SV_SVNAPOT (1 << 1) #define SV_SVPBMT (1 << 2) #define SV_SVINVAL (1 << 3) #define SV_SSCOFPMF (1 << 4) }; struct cpu_desc cpu_desc[MAXCPU]; /* * Micro-architecture tables. */ struct marchid_entry { register_t march_id; const char *march_name; }; #define MARCHID_END { -1ul, NULL } /* Open-source RISC-V architecture IDs; globally allocated. */ static const struct marchid_entry global_marchids[] = { { MARCHID_UCB_ROCKET, "UC Berkeley Rocket" }, { MARCHID_UCB_BOOM, "UC Berkeley Boom" }, { MARCHID_UCB_SPIKE, "UC Berkeley Spike" }, { MARCHID_UCAM_RVBS, "University of Cambridge RVBS" }, MARCHID_END }; static const struct marchid_entry sifive_marchids[] = { { MARCHID_SIFIVE_U7, "6/7/P200/X200-Series Processor" }, MARCHID_END }; /* * Known CPU vendor/manufacturer table. */ static const struct { register_t mvendor_id; const char *mvendor_name; const struct marchid_entry *marchid_table; } mvendor_ids[] = { { MVENDORID_UNIMPL, "Unspecified", NULL }, { MVENDORID_SIFIVE, "SiFive", sifive_marchids }, { MVENDORID_THEAD, "T-Head", NULL }, }; /* * The ISA string describes the complete set of instructions supported by a * RISC-V CPU. The string begins with a small prefix (e.g. rv64) indicating the * base ISA. It is followed first by single-letter ISA extensions, and then * multi-letter ISA extensions. * * Underscores are used mainly to separate consecutive multi-letter extensions, * but may optionally appear between any two extensions. An extension may be * followed by a version number, in the form of 'Mpm', where M is the * extension's major version number, and 'm' is the minor version number. * * The format is described in detail by the "ISA Extension Naming Conventions" * chapter of the unprivileged spec. */ #define ISA_PREFIX ("rv" __XSTRING(__riscv_xlen)) #define ISA_PREFIX_LEN (sizeof(ISA_PREFIX) - 1) static __inline int parse_ext_s(struct cpu_desc *desc, char *isa, int idx, int len) { #define CHECK_S_EXT(str, flag) \ do { \ if (strncmp(&isa[idx], (str), \ MIN(strlen(str), len - idx)) == 0) { \ desc->smode_extensions |= flag; \ return (idx + strlen(str)); \ } \ } while (0) /* Check for known/supported extensions. */ CHECK_S_EXT("sstc", SV_SSTC); CHECK_S_EXT("svnapot", SV_SVNAPOT); CHECK_S_EXT("svpbmt", SV_SVPBMT); CHECK_S_EXT("svinval", SV_SVINVAL); CHECK_S_EXT("sscofpmf", SV_SSCOFPMF); #undef CHECK_S_EXT /* * Proceed to the next multi-letter extension or the end of the * string. */ while (isa[idx] != '_' && idx < len) { idx++; } return (idx); } static __inline int parse_ext_x(struct cpu_desc *desc __unused, char *isa, int idx, int len) { /* * Proceed to the next multi-letter extension or the end of the * string. */ while (isa[idx] != '_' && idx < len) { idx++; } return (idx); } static __inline int parse_ext_z(struct cpu_desc *desc __unused, char *isa, int idx, int len) { /* * Proceed to the next multi-letter extension or the end of the * string. * * TODO: parse some of these. */ while (isa[idx] != '_' && idx < len) { idx++; } return (idx); } static __inline int parse_ext_version(char *isa, int idx, u_int *majorp __unused, u_int *minorp __unused) { /* Major version. */ while (isdigit(isa[idx])) idx++; if (isa[idx] != 'p') return (idx); else idx++; /* Minor version. */ while (isdigit(isa[idx])) idx++; return (idx); } /* * Parse the ISA string, building up the set of HWCAP bits as they are found. */ static int parse_riscv_isa(struct cpu_desc *desc, char *isa, int len) { int i; /* Check the string prefix. */ if (strncmp(isa, ISA_PREFIX, ISA_PREFIX_LEN) != 0) { printf("%s: Unrecognized ISA string: %s\n", __func__, isa); return (-1); } i = ISA_PREFIX_LEN; while (i < len) { switch(isa[i]) { case 'a': case 'b': case 'c': case 'd': case 'f': case 'h': case 'i': case 'm': desc->isa_extensions |= HWCAP_ISA_BIT(isa[i]); i++; break; case 'g': desc->isa_extensions |= HWCAP_ISA_G; i++; break; case 's': /* * XXX: older versions of this string erroneously * indicated supervisor and user mode support as * single-letter extensions. Detect and skip both 's' * and 'u'. */ if (isa[i - 1] != '_' && isa[i + 1] == 'u') { i += 2; continue; } /* * Supervisor-level extension namespace. */ i = parse_ext_s(desc, isa, i, len); break; case 'x': /* * Custom extension namespace. For now, we ignore * these. */ i = parse_ext_x(desc, isa, i, len); break; case 'z': /* * Multi-letter standard extension namespace. */ i = parse_ext_z(desc, isa, i, len); break; case '_': i++; continue; default: /* Unrecognized/unsupported. */ i++; break; } i = parse_ext_version(isa, i, NULL, NULL); } return (0); } #ifdef FDT static void parse_mmu_fdt(struct cpu_desc *desc, phandle_t node) { char mmu[16]; desc->mmu_caps |= MMU_SV39; if (OF_getprop(node, "mmu-type", mmu, sizeof(mmu)) > 0) { if (strcmp(mmu, "riscv,sv48") == 0) desc->mmu_caps |= MMU_SV48; else if (strcmp(mmu, "riscv,sv57") == 0) desc->mmu_caps |= MMU_SV48 | MMU_SV57; } } static void identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc) { char isa[1024]; phandle_t node; ssize_t len; pcell_t reg; u_int hart; node = OF_finddevice("/cpus"); if (node == -1) { printf("%s: could not find /cpus node in FDT\n", __func__); return; } hart = pcpu_find(cpu)->pc_hart; /* * Locate our current CPU's node in the device-tree, and parse its * contents to detect supported CPU/ISA features and extensions. */ for (node = OF_child(node); node > 0; node = OF_peer(node)) { /* Skip any non-CPU nodes, such as cpu-map. */ if (!ofw_bus_node_is_compatible(node, "riscv")) continue; /* Find this CPU */ if (OF_getencprop(node, "reg", ®, sizeof(reg)) <= 0 || reg != hart) continue; len = OF_getprop(node, "riscv,isa", isa, sizeof(isa)); KASSERT(len <= sizeof(isa), ("ISA string truncated")); if (len == -1) { printf("%s: could not find 'riscv,isa' property " "for CPU %d, hart %u\n", __func__, cpu, hart); return; } /* * The string is specified to be lowercase, but let's be * certain. */ for (int i = 0; i < len; i++) isa[i] = tolower(isa[i]); if (parse_riscv_isa(desc, isa, len) != 0) return; /* Check MMU features. */ parse_mmu_fdt(desc, node); /* We are done. */ break; } if (node <= 0) { printf("%s: could not find FDT node for CPU %u, hart %u\n", __func__, cpu, hart); } } #endif static void identify_cpu_features(u_int cpu, struct cpu_desc *desc) { #ifdef FDT identify_cpu_features_fdt(cpu, desc); #endif } /* * Update kernel/user global state based on the feature parsing results, stored * in desc. * * We keep only the subset of values common to all CPUs. */ static void update_global_capabilities(u_int cpu, struct cpu_desc *desc) { #define UPDATE_CAP(t, v) \ do { \ if (cpu == 0) { \ (t) = (v); \ } else { \ (t) &= (v); \ } \ } while (0) /* Update the capabilities exposed to userspace via AT_HWCAP. */ UPDATE_CAP(elf_hwcap, (u_long)desc->isa_extensions); /* * MMU capabilities, e.g. Sv48. */ UPDATE_CAP(mmu_caps, desc->mmu_caps); /* Supervisor-mode extension support. */ UPDATE_CAP(has_hyp, (desc->isa_extensions & HWCAP_ISA_H) != 0); UPDATE_CAP(has_sstc, (desc->smode_extensions & SV_SSTC) != 0); UPDATE_CAP(has_sscofpmf, (desc->smode_extensions & SV_SSCOFPMF) != 0); UPDATE_CAP(has_svpbmt, (desc->smode_extensions & SV_SVPBMT) != 0); #undef UPDATE_CAP } static void identify_cpu_ids(struct cpu_desc *desc) { const struct marchid_entry *table = NULL; int i; desc->cpu_mvendor_name = "Unknown"; desc->cpu_march_name = "Unknown"; /* * Search for a recognized vendor, and possibly obtain the secondary * table for marchid lookup. */ for (i = 0; i < nitems(mvendor_ids); i++) { if (mvendorid == mvendor_ids[i].mvendor_id) { desc->cpu_mvendor_name = mvendor_ids[i].mvendor_name; table = mvendor_ids[i].marchid_table; break; } } if (marchid == MARCHID_UNIMPL) { desc->cpu_march_name = "Unspecified"; return; } if (MARCHID_IS_OPENSOURCE(marchid)) { table = global_marchids; } else if (table == NULL) return; for (i = 0; table[i].march_name != NULL; i++) { if (marchid == table[i].march_id) { desc->cpu_march_name = table[i].march_name; break; } } } static void handle_thead_quirks(u_int cpu, struct cpu_desc *desc) { if (cpu != 0) return; + /* + * For now, it is assumed that T-HEAD CPUs have both marchid and mimpid + * values of zero (although we leave this unchecked). It is true in + * practice for the early generations of this hardware (C906, C910, + * C920). In the future, the identity checks may need to become more + * granular, but until then all known T-HEAD quirks are applied + * indiscriminantly. + * + * Note: any changes in this function relating to has_errata_thead_pbmt + * may need to be applied to get_pte_fixup_bits (in locore.S) as well. + */ + has_errata_thead_pbmt = true; thead_setup_cache(); } static void handle_cpu_quirks(u_int cpu, struct cpu_desc *desc) { switch (mvendorid) { case MVENDORID_THEAD: handle_thead_quirks(cpu, desc); break; } } void identify_cpu(u_int cpu) { struct cpu_desc *desc = &cpu_desc[cpu]; identify_cpu_ids(desc); identify_cpu_features(cpu, desc); update_global_capabilities(cpu, desc); handle_cpu_quirks(cpu, desc); } void printcpuinfo(u_int cpu) { struct cpu_desc *desc; u_int hart; desc = &cpu_desc[cpu]; hart = pcpu_find(cpu)->pc_hart; /* XXX: check this here so we are guaranteed to have console output. */ KASSERT(desc->isa_extensions != 0, ("Empty extension set for CPU %u, did parsing fail?", cpu)); /* * Suppress the output of some fields in the common case of identical * CPU features. */ #define SHOULD_PRINT(_field) \ (cpu == 0 || desc[0]._field != desc[-1]._field) /* Always print summary line. */ printf("CPU %-3u: Vendor=%s Core=%s (Hart %u)\n", cpu, desc->cpu_mvendor_name, desc->cpu_march_name, hart); /* These values are global. */ if (cpu == 0) printf(" marchid=%#lx, mimpid=%#lx\n", marchid, mimpid); if (SHOULD_PRINT(mmu_caps)) { printf(" MMU: %#b\n", desc->mmu_caps, "\020" "\01Sv39" "\02Sv48" "\03Sv57"); } if (SHOULD_PRINT(isa_extensions)) { printf(" ISA: %#b\n", desc->isa_extensions, "\020" "\01Atomic" "\03Compressed" "\04Double" "\06Float" "\10Hypervisor" "\15Mult/Div"); } if (SHOULD_PRINT(smode_extensions)) { printf(" S-mode Extensions: %#b\n", desc->smode_extensions, "\020" "\01Sstc" "\02Svnapot" "\03Svpbmt" "\04Svinval" "\05Sscofpmf"); } #undef SHOULD_PRINT } diff --git a/sys/riscv/riscv/locore.S b/sys/riscv/riscv/locore.S index 5c0ade6e66ca..305ed8d79f10 100644 --- a/sys/riscv/riscv/locore.S +++ b/sys/riscv/riscv/locore.S @@ -1,366 +1,395 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * Copyright (c) 2019-2021 Mitchell Horne * Copyright (c) 2022-2024 The FreeBSD Foundation * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Portions of this software were developed by Mitchell Horne * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "assym.inc" #include +#include #include -#include #include +#include +#include .globl kernbase .set kernbase, KERNBASE .text /* * Alternate entry point. Used when booting via SBI firmware. It must be placed * at the beginning of the .text section. Arguments are as follows: * - a0 = hart ID * - a1 = dtbp * * Multiple CPUs might enter from this point, so we perform a hart lottery and * send the losers to mpentry. */ .globl _alt_start _alt_start: /* Set the global pointer */ .option push .option norelax lla gp, __global_pointer$ .option pop /* Pick a hart to run the boot process. */ lla t0, hart_lottery li t1, 1 amoadd.w t0, t1, 0(t0) /* * We must jump to mpentry in the non-BSP case because the offset is * too large to fit in a 12-bit branch immediate. */ beqz t0, 1f j mpentry 1: /* Store the boot hart */ lla t0, boot_hart sw a0, 0(t0) /* * Stash the DTB pointer in the callee-saved register s4, and zero s3 * to indicate that we have no loader metadata. */ mv s4, a1 mv s3, zero j pagetables /* * Main entry point. This routine is marked as the ELF entry, and is where * loader(8) will enter the kernel. Arguments are as follows: * - a0 = modulep * - a1 = ??? * * It is expected that only a single CPU will enter here. */ .globl _start _start: /* Set the global pointer */ .option push .option norelax lla gp, __global_pointer$ .option pop /* * Stash modulep in the callee-saved register s3, and zero s4 to * indicate that we have no DTB pointer. It is already included in the * loader(8) metadata. */ mv s3, a0 mv s4, zero /* * Set up page tables: Our goal is to enable virtual memory, doing the * minimum amount of work in assembly; just what is required to * bootstrap. We will construct the real page tables in C code, in * pmap_bootstrap(). * * Here we map a 1GB region starting at KERNBASE using 2MB superpages, * starting from the first 2MB physical page into which the kernel was * loaded. * * We also use an L1 entry to create a 1GB identity map (1:1 PA->VA). * This is useful for two reasons: * - handling the DTB pointer passed from SBI firmware (physical addr) * - simpler construction of pagetables in pmap_bootstrap() * * Implementations are required to provide Sv39 mode, so we use that * here and will conditionally enable Sv48 (or higher) later. * * We arrive here with: * s3 - modulep or zero * s4 - zero or dtbp */ pagetables: /* Get the kernel's load address (kernstart) in s9 */ jal get_physmem + /* Get PTE attribute bits in s8 */ + jal get_pte_fixup_bits + /* Construct 1GB Identity Map (1:1 PA->VA) */ lla s1, bootstrap_pt_l1 srli s2, s9, L1_SHIFT /* kernstart >> L1_SHIFT */ andi a5, s2, Ln_ADDR_MASK /* & Ln_ADDR_MASK */ li t4, (PTE_KERN) + or t4, t4, s8 /* t4 |= pte bits */ slli s2, s2, PTE_PPN2_S /* (s2 << PTE_PPN2_S) */ or t6, t4, s2 /* Store L1 PTE entry to position */ li a6, PTE_SIZE mulw a5, a5, a6 /* calculate L1 slot */ add t0, s1, a5 sd t6, (t0) /* Store new PTE */ /* Construct the virtual address space at KERNBASE */ /* Add L1 entry for kernel */ lla s1, bootstrap_pt_l1 lla s2, bootstrap_pt_l2 /* Link to next level PN */ srli s2, s2, PAGE_SHIFT li a5, KERNBASE srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ andi a5, a5, Ln_ADDR_MASK /* & Ln_ADDR_MASK */ li t4, PTE_V slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ or t6, t4, t5 /* Store L1 PTE entry to position */ li a6, PTE_SIZE mulw a5, a5, a6 add t0, s1, a5 sd t6, (t0) /* Level 2 superpages (512 x 2MiB) */ lla s1, bootstrap_pt_l2 srli t4, s9, L2_SHIFT /* Div physmem base by 2 MiB */ li t2, Ln_ENTRIES /* Build 512 entries */ add t3, t4, t2 li t0, (PTE_KERN | PTE_X) + or t0, t0, s8 /* t0 |= pte bits */ 1: slli t2, t4, PTE_PPN1_S /* << PTE_PPN1_S */ or t5, t0, t2 sd t5, (s1) /* Store PTE entry to position */ addi s1, s1, PTE_SIZE addi t4, t4, 1 bltu t4, t3, 1b /* Page tables END */ /* * Set the supervisor trap vector temporarily. Enabling virtual memory * may generate a page fault. We simply wish to continue onwards, so * have the trap deliver us to 'va'. */ lla t0, va sub t0, t0, s9 li t1, KERNBASE add t0, t0, t1 csrw stvec, t0 /* Set page tables base register */ lla s2, bootstrap_pt_l1 srli s2, s2, PAGE_SHIFT li t0, SATP_MODE_SV39 or s2, s2, t0 sfence.vma csrw satp, s2 .align 2 va: /* Set the global pointer again, this time with the virtual address. */ .option push .option norelax lla gp, __global_pointer$ .option pop /* Set the trap vector to the real handler. */ la t0, cpu_exception_handler csrw stvec, t0 /* Ensure sscratch is zero */ li t0, 0 csrw sscratch, t0 /* Initialize stack pointer */ la sp, initstack_end /* Clear frame pointer */ mv s0, zero /* Allocate space for thread0 PCB and riscv_bootparams */ addi sp, sp, -(PCB_SIZE + RISCV_BOOTPARAMS_SIZE) & ~STACKALIGNBYTES /* Clear BSS */ la t0, _C_LABEL(__bss_start) la t1, _C_LABEL(_end) 1: sd zero, 0(t0) addi t0, t0, 8 bltu t0, t1, 1b /* Fill riscv_bootparams */ sd s9, RISCV_BOOTPARAMS_KERN_PHYS(sp) la t0, initstack sd t0, RISCV_BOOTPARAMS_KERN_STACK(sp) sd s4, RISCV_BOOTPARAMS_DTBP_PHYS(sp) sd s3, RISCV_BOOTPARAMS_MODULEP(sp) mv a0, sp call _C_LABEL(initriscv) /* Off we go */ call _C_LABEL(mi_startup) /* We should never reach here, but if so just hang. */ 2: wfi j 2b /* * Get the physical address the kernel is loaded to. Returned in s9. */ get_physmem: lla t0, virt_map /* physical address of virt_map */ ld t1, 0(t0) /* virtual address of virt_map */ sub t1, t1, t0 /* calculate phys->virt delta */ li t2, KERNBASE sub s9, t2, t1 /* s9 = physmem base */ ret +/* + * T-HEAD CPUs implement an alternate scheme for PTE attributes that is + * incompatible with the RISC-V PTE specification (see the definitions in + * pte.h). Worse, it defines a non-zero value for "main" memory, and this must + * be set in order to proceed with our new page tables. + * + * Therefore, we are forced to check the CPU identity here, which is both + * inconvenient and fragile. + * + * Return the required attribute bits in s8. For sane implementations this is + * zero. + */ +get_pte_fixup_bits: + mv s8, zero + SBI_CALL(SBI_EXT_ID_BASE, SBI_BASE_GET_MVENDORID) + li t0, MVENDORID_THEAD + xor t0, t0, a1 + bnez t0, 1f /* branch if a1 != t0 */ + li s8, PTE_THEAD_MA_NONE +1: + ret + .align 4 initstack: .space (PAGE_SIZE * KSTACK_PAGES) initstack_end: /* * Static space for the bootstrap page tables. Unused after pmap_bootstrap(). */ .balign PAGE_SIZE bootstrap_pt_l1: .space PAGE_SIZE bootstrap_pt_l2: .space PAGE_SIZE .align 3 virt_map: .quad virt_map hart_lottery: .space 4 #ifndef SMP ENTRY(mpentry) 1: wfi j 1b END(mpentry) #else /* * mpentry(unsigned long) * * Called by a core when it is being brought online. */ ENTRY(mpentry) /* * Calculate the offset to __riscv_boot_ap * for the current core, cpuid is in a0. */ li t1, 4 mulw t1, t1, a0 /* Get the pointer */ lla t0, __riscv_boot_ap add t0, t0, t1 1: /* Wait the kernel to be ready */ lw t1, 0(t0) beqz t1, 1b /* Setup stack pointer */ lla t0, bootstack ld sp, 0(t0) /* Get the kernel's load address */ jal get_physmem /* * Set the supervisor trap vector temporarily. Enabling virtual memory * may generate a page fault. We simply wish to continue onwards, so * have the trap deliver us to 'mpva'. */ lla t0, mpva sub t0, t0, s9 li t1, KERNBASE add t0, t0, t1 csrw stvec, t0 /* Set page tables base register */ lla t2, kernel_pmap_store ld s2, PM_SATP(t2) sfence.vma csrw satp, s2 .align 2 mpva: /* Set the global pointer again, this time with the virtual address. */ .option push .option norelax lla gp, __global_pointer$ .option pop /* Set the trap vector to the real handler. */ la t0, cpu_exception_handler csrw stvec, t0 /* Ensure sscratch is zero */ li t0, 0 csrw sscratch, t0 call init_secondary END(mpentry) #endif