diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv index 915bce34603d..6186ae9b3371 100644 --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -1,88 +1,90 @@ cddl/dev/dtrace/riscv/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/riscv/dtrace_isa.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/dtrace/riscv/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/dtrace/riscv/instr_size.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/riscv/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" crypto/des/des_enc.c optional netsmb dev/ofw/ofw_cpu.c optional fdt dev/ofw/ofw_pcib.c optional pci fdt dev/pci/pci_dw.c optional pci fdt dev/pci/pci_dw_if.m optional pci fdt dev/pci/pci_host_generic.c optional pci dev/pci/pci_host_generic_fdt.c optional pci fdt dev/uart/uart_cpu_fdt.c optional uart fdt dev/uart/uart_dev_lowrisc.c optional uart_lowrisc dev/vmm/vmm_dev.c optional vmm dev/vmm/vmm_stat.c optional vmm dev/xilinx/axi_quad_spi.c optional xilinx_spi dev/xilinx/axidma.c optional axidma xdma dev/xilinx/if_xae.c optional xae dev/xilinx/xlnx_pcib.c optional pci fdt xlnx_pcib kern/msi_if.m standard kern/pic_if.m standard kern/subr_devmap.c standard kern/subr_dummy_vdso_tc.c standard kern/subr_intr.c standard kern/subr_physmem.c standard libkern/bcopy.c standard libkern/memcmp.c standard libkern/memset.c standard libkern/strcmp.c standard libkern/strlen.c standard libkern/strncmp.c standard riscv/riscv/aplic.c standard riscv/riscv/autoconf.c standard riscv/riscv/bus_machdep.c standard riscv/riscv/bus_space_asm.S standard riscv/riscv/busdma_bounce.c standard riscv/riscv/busdma_machdep.c standard riscv/riscv/cache.c standard riscv/riscv/clock.c standard riscv/riscv/copyinout.S standard riscv/riscv/cpufunc_asm.S standard riscv/riscv/db_disasm.c optional ddb riscv/riscv/db_interface.c optional ddb riscv/riscv/db_trace.c optional ddb riscv/riscv/dump_machdep.c standard riscv/riscv/elf_machdep.c standard riscv/riscv/exception.S standard riscv/riscv/exec_machdep.c standard riscv/riscv/fpe.c standard riscv/riscv/gdb_machdep.c optional gdb riscv/riscv/intc.c standard riscv/riscv/identcpu.c standard riscv/riscv/locore.S standard no-obj riscv/riscv/machdep.c standard riscv/riscv/minidump_machdep.c standard riscv/riscv/mp_machdep.c optional smp riscv/riscv/mem.c standard riscv/riscv/nexus.c standard riscv/riscv/ofw_machdep.c optional fdt riscv/riscv/plic.c standard riscv/riscv/pmap.c standard riscv/riscv/ptrace_machdep.c standard riscv/riscv/riscv_console.c optional rcons riscv/riscv/riscv_syscon.c optional syscon riscv_syscon fdt riscv/riscv/sigtramp.S standard riscv/riscv/sbi.c standard riscv/riscv/sbi_ipi.c optional smp riscv/riscv/sdt_machdep.c optional kdtrace_hooks riscv/riscv/stack_machdep.c optional ddb | stack riscv/riscv/support.S standard riscv/riscv/swtch.S standard riscv/riscv/sys_machdep.c standard riscv/riscv/trap.c standard riscv/riscv/timer.c standard riscv/riscv/uio_machdep.c standard riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack riscv/riscv/vm_machdep.c standard riscv/vmm/vmm.c optional vmm riscv/vmm/vmm_aplic.c optional vmm riscv/vmm/vmm_dev_machdep.c optional vmm riscv/vmm/vmm_instruction_emul.c optional vmm riscv/vmm/vmm_riscv.c optional vmm riscv/vmm/vmm_sbi.c optional vmm riscv/vmm/vmm_switch.S optional vmm +riscv/thead/thead.c standard + # Zstd contrib/zstd/lib/freebsd/zstd_kfreebsd.c optional zstdio compile-with ${ZSTD_C} diff --git a/sys/riscv/include/thead.h b/sys/riscv/include/thead.h new file mode 100644 index 000000000000..e11d5c37374c --- /dev/null +++ b/sys/riscv/include/thead.h @@ -0,0 +1,35 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 The FreeBSD Foundation + * + * This software was developed by Mitchell Horne under + * sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#ifndef _RISCV_THEAD_H_ +#define _RISCV_THEAD_H_ + +void thead_setup_cache(void); + +#endif /* _RISCV_THEAD_H_ */ diff --git a/sys/riscv/riscv/identcpu.c b/sys/riscv/riscv/identcpu.c index 7823830c3136..f85aed88d3b9 100644 --- a/sys/riscv/riscv/identcpu.c +++ b/sys/riscv/riscv/identcpu.c @@ -1,535 +1,556 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015-2016 Ruslan Bukin * All rights reserved. * Copyright (c) 2022 Mitchell Horne * Copyright (c) 2023 The FreeBSD Foundation * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Portions of this software were developed by Mitchell Horne * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include +#include #ifdef FDT #include #include #include #endif const char machine[] = "riscv"; SYSCTL_CONST_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD | CTLFLAG_CAPRD, machine, "Machine class"); /* Hardware implementation info. These values may be empty. */ register_t mvendorid; /* The CPU's JEDEC vendor ID */ register_t marchid; /* The architecture ID */ register_t mimpid; /* The implementation ID */ u_int mmu_caps; /* Supervisor-mode extension support. */ bool has_hyp; bool __read_frequently has_sstc; bool __read_frequently has_sscofpmf; bool has_svpbmt; struct cpu_desc { const char *cpu_mvendor_name; const char *cpu_march_name; u_int isa_extensions; /* Single-letter extensions. */ u_int mmu_caps; u_int smode_extensions; #define SV_SSTC (1 << 0) #define SV_SVNAPOT (1 << 1) #define SV_SVPBMT (1 << 2) #define SV_SVINVAL (1 << 3) #define SV_SSCOFPMF (1 << 4) }; struct cpu_desc cpu_desc[MAXCPU]; /* * Micro-architecture tables. */ struct marchid_entry { register_t march_id; const char *march_name; }; #define MARCHID_END { -1ul, NULL } /* Open-source RISC-V architecture IDs; globally allocated. */ static const struct marchid_entry global_marchids[] = { { MARCHID_UCB_ROCKET, "UC Berkeley Rocket" }, { MARCHID_UCB_BOOM, "UC Berkeley Boom" }, { MARCHID_UCB_SPIKE, "UC Berkeley Spike" }, { MARCHID_UCAM_RVBS, "University of Cambridge RVBS" }, MARCHID_END }; static const struct marchid_entry sifive_marchids[] = { { MARCHID_SIFIVE_U7, "6/7/P200/X200-Series Processor" }, MARCHID_END }; /* * Known CPU vendor/manufacturer table. */ static const struct { register_t mvendor_id; const char *mvendor_name; const struct marchid_entry *marchid_table; } mvendor_ids[] = { { MVENDORID_UNIMPL, "Unspecified", NULL }, { MVENDORID_SIFIVE, "SiFive", sifive_marchids }, { MVENDORID_THEAD, "T-Head", NULL }, }; /* * The ISA string describes the complete set of instructions supported by a * RISC-V CPU. The string begins with a small prefix (e.g. rv64) indicating the * base ISA. It is followed first by single-letter ISA extensions, and then * multi-letter ISA extensions. * * Underscores are used mainly to separate consecutive multi-letter extensions, * but may optionally appear between any two extensions. An extension may be * followed by a version number, in the form of 'Mpm', where M is the * extension's major version number, and 'm' is the minor version number. * * The format is described in detail by the "ISA Extension Naming Conventions" * chapter of the unprivileged spec. */ #define ISA_PREFIX ("rv" __XSTRING(__riscv_xlen)) #define ISA_PREFIX_LEN (sizeof(ISA_PREFIX) - 1) static __inline int parse_ext_s(struct cpu_desc *desc, char *isa, int idx, int len) { #define CHECK_S_EXT(str, flag) \ do { \ if (strncmp(&isa[idx], (str), \ MIN(strlen(str), len - idx)) == 0) { \ desc->smode_extensions |= flag; \ return (idx + strlen(str)); \ } \ } while (0) /* Check for known/supported extensions. */ CHECK_S_EXT("sstc", SV_SSTC); CHECK_S_EXT("svnapot", SV_SVNAPOT); CHECK_S_EXT("svpbmt", SV_SVPBMT); CHECK_S_EXT("svinval", SV_SVINVAL); CHECK_S_EXT("sscofpmf", SV_SSCOFPMF); #undef CHECK_S_EXT /* * Proceed to the next multi-letter extension or the end of the * string. */ while (isa[idx] != '_' && idx < len) { idx++; } return (idx); } static __inline int parse_ext_x(struct cpu_desc *desc __unused, char *isa, int idx, int len) { /* * Proceed to the next multi-letter extension or the end of the * string. */ while (isa[idx] != '_' && idx < len) { idx++; } return (idx); } static __inline int parse_ext_z(struct cpu_desc *desc __unused, char *isa, int idx, int len) { /* * Proceed to the next multi-letter extension or the end of the * string. * * TODO: parse some of these. */ while (isa[idx] != '_' && idx < len) { idx++; } return (idx); } static __inline int parse_ext_version(char *isa, int idx, u_int *majorp __unused, u_int *minorp __unused) { /* Major version. */ while (isdigit(isa[idx])) idx++; if (isa[idx] != 'p') return (idx); else idx++; /* Minor version. */ while (isdigit(isa[idx])) idx++; return (idx); } /* * Parse the ISA string, building up the set of HWCAP bits as they are found. */ static int parse_riscv_isa(struct cpu_desc *desc, char *isa, int len) { int i; /* Check the string prefix. */ if (strncmp(isa, ISA_PREFIX, ISA_PREFIX_LEN) != 0) { printf("%s: Unrecognized ISA string: %s\n", __func__, isa); return (-1); } i = ISA_PREFIX_LEN; while (i < len) { switch(isa[i]) { case 'a': case 'b': case 'c': case 'd': case 'f': case 'h': case 'i': case 'm': desc->isa_extensions |= HWCAP_ISA_BIT(isa[i]); i++; break; case 'g': desc->isa_extensions |= HWCAP_ISA_G; i++; break; case 's': /* * XXX: older versions of this string erroneously * indicated supervisor and user mode support as * single-letter extensions. Detect and skip both 's' * and 'u'. */ if (isa[i - 1] != '_' && isa[i + 1] == 'u') { i += 2; continue; } /* * Supervisor-level extension namespace. */ i = parse_ext_s(desc, isa, i, len); break; case 'x': /* * Custom extension namespace. For now, we ignore * these. */ i = parse_ext_x(desc, isa, i, len); break; case 'z': /* * Multi-letter standard extension namespace. */ i = parse_ext_z(desc, isa, i, len); break; case '_': i++; continue; default: /* Unrecognized/unsupported. */ i++; break; } i = parse_ext_version(isa, i, NULL, NULL); } return (0); } #ifdef FDT static void parse_mmu_fdt(struct cpu_desc *desc, phandle_t node) { char mmu[16]; desc->mmu_caps |= MMU_SV39; if (OF_getprop(node, "mmu-type", mmu, sizeof(mmu)) > 0) { if (strcmp(mmu, "riscv,sv48") == 0) desc->mmu_caps |= MMU_SV48; else if (strcmp(mmu, "riscv,sv57") == 0) desc->mmu_caps |= MMU_SV48 | MMU_SV57; } } static void identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc) { char isa[1024]; phandle_t node; ssize_t len; pcell_t reg; u_int hart; node = OF_finddevice("/cpus"); if (node == -1) { printf("%s: could not find /cpus node in FDT\n", __func__); return; } hart = pcpu_find(cpu)->pc_hart; /* * Locate our current CPU's node in the device-tree, and parse its * contents to detect supported CPU/ISA features and extensions. */ for (node = OF_child(node); node > 0; node = OF_peer(node)) { /* Skip any non-CPU nodes, such as cpu-map. */ if (!ofw_bus_node_is_compatible(node, "riscv")) continue; /* Find this CPU */ if (OF_getencprop(node, "reg", ®, sizeof(reg)) <= 0 || reg != hart) continue; len = OF_getprop(node, "riscv,isa", isa, sizeof(isa)); KASSERT(len <= sizeof(isa), ("ISA string truncated")); if (len == -1) { printf("%s: could not find 'riscv,isa' property " "for CPU %d, hart %u\n", __func__, cpu, hart); return; } /* * The string is specified to be lowercase, but let's be * certain. */ for (int i = 0; i < len; i++) isa[i] = tolower(isa[i]); if (parse_riscv_isa(desc, isa, len) != 0) return; /* Check MMU features. */ parse_mmu_fdt(desc, node); /* We are done. */ break; } if (node <= 0) { printf("%s: could not find FDT node for CPU %u, hart %u\n", __func__, cpu, hart); } } #endif static void identify_cpu_features(u_int cpu, struct cpu_desc *desc) { #ifdef FDT identify_cpu_features_fdt(cpu, desc); #endif } /* * Update kernel/user global state based on the feature parsing results, stored * in desc. * * We keep only the subset of values common to all CPUs. */ static void update_global_capabilities(u_int cpu, struct cpu_desc *desc) { #define UPDATE_CAP(t, v) \ do { \ if (cpu == 0) { \ (t) = (v); \ } else { \ (t) &= (v); \ } \ } while (0) /* Update the capabilities exposed to userspace via AT_HWCAP. */ UPDATE_CAP(elf_hwcap, (u_long)desc->isa_extensions); /* * MMU capabilities, e.g. Sv48. */ UPDATE_CAP(mmu_caps, desc->mmu_caps); /* Supervisor-mode extension support. */ UPDATE_CAP(has_hyp, (desc->isa_extensions & HWCAP_ISA_H) != 0); UPDATE_CAP(has_sstc, (desc->smode_extensions & SV_SSTC) != 0); UPDATE_CAP(has_sscofpmf, (desc->smode_extensions & SV_SSCOFPMF) != 0); UPDATE_CAP(has_svpbmt, (desc->smode_extensions & SV_SVPBMT) != 0); #undef UPDATE_CAP } static void identify_cpu_ids(struct cpu_desc *desc) { const struct marchid_entry *table = NULL; int i; desc->cpu_mvendor_name = "Unknown"; desc->cpu_march_name = "Unknown"; /* * Search for a recognized vendor, and possibly obtain the secondary * table for marchid lookup. */ for (i = 0; i < nitems(mvendor_ids); i++) { if (mvendorid == mvendor_ids[i].mvendor_id) { desc->cpu_mvendor_name = mvendor_ids[i].mvendor_name; table = mvendor_ids[i].marchid_table; break; } } if (marchid == MARCHID_UNIMPL) { desc->cpu_march_name = "Unspecified"; return; } if (MARCHID_IS_OPENSOURCE(marchid)) { table = global_marchids; } else if (table == NULL) return; for (i = 0; table[i].march_name != NULL; i++) { if (marchid == table[i].march_id) { desc->cpu_march_name = table[i].march_name; break; } } } +static void +handle_thead_quirks(u_int cpu, struct cpu_desc *desc) +{ + if (cpu != 0) + return; + + thead_setup_cache(); +} + +static void +handle_cpu_quirks(u_int cpu, struct cpu_desc *desc) +{ + switch (mvendorid) { + case MVENDORID_THEAD: + handle_thead_quirks(cpu, desc); + break; + } +} + void identify_cpu(u_int cpu) { struct cpu_desc *desc = &cpu_desc[cpu]; identify_cpu_ids(desc); identify_cpu_features(cpu, desc); update_global_capabilities(cpu, desc); + handle_cpu_quirks(cpu, desc); } void printcpuinfo(u_int cpu) { struct cpu_desc *desc; u_int hart; desc = &cpu_desc[cpu]; hart = pcpu_find(cpu)->pc_hart; /* XXX: check this here so we are guaranteed to have console output. */ KASSERT(desc->isa_extensions != 0, ("Empty extension set for CPU %u, did parsing fail?", cpu)); /* * Suppress the output of some fields in the common case of identical * CPU features. */ #define SHOULD_PRINT(_field) \ (cpu == 0 || desc[0]._field != desc[-1]._field) /* Always print summary line. */ printf("CPU %-3u: Vendor=%s Core=%s (Hart %u)\n", cpu, desc->cpu_mvendor_name, desc->cpu_march_name, hart); /* These values are global. */ if (cpu == 0) printf(" marchid=%#lx, mimpid=%#lx\n", marchid, mimpid); if (SHOULD_PRINT(mmu_caps)) { printf(" MMU: %#b\n", desc->mmu_caps, "\020" "\01Sv39" "\02Sv48" "\03Sv57"); } if (SHOULD_PRINT(isa_extensions)) { printf(" ISA: %#b\n", desc->isa_extensions, "\020" "\01Atomic" "\03Compressed" "\04Double" "\06Float" "\10Hypervisor" "\15Mult/Div"); } if (SHOULD_PRINT(smode_extensions)) { printf(" S-mode Extensions: %#b\n", desc->smode_extensions, "\020" "\01Sstc" "\02Svnapot" "\03Svpbmt" "\04Svinval" "\05Sscofpmf"); } #undef SHOULD_PRINT } diff --git a/sys/riscv/thead/thead.c b/sys/riscv/thead/thead.c new file mode 100644 index 000000000000..f959d32cfed8 --- /dev/null +++ b/sys/riscv/thead/thead.c @@ -0,0 +1,99 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 The FreeBSD Foundation + * + * This software was developed by Mitchell Horne under + * sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include + +#include + +/* ----------------- dcache ops --------------------- */ + + +/* th.dcache.civa: clean & invalidate at VA stored in t0. */ +#define THEAD_DCACHE_CIVA ".long 0x0272800b\n" + +/* th.dcache.iva: invalidate at VA stored in t0. */ +#define THEAD_DCACHE_IVA ".long 0x0262800b\n" + +/* th.dcache.cva: clean at VA stored in t0. */ +#define THEAD_DCACHE_CVA ".long 0x0252800b\n" + +/* th.sync.s: two-way instruction barrier */ +#define THEAD_SYNC_S ".long 0x0190000b\n" + +/* MHTODO: we could parse this information from the device tree. */ +#define THEAD_DCACHE_SIZE 64 + +static void +thead_cpu_dcache_wbinv_range(vm_offset_t va, vm_size_t len) +{ + register vm_offset_t t0 __asm("t0") = rounddown(va, dcache_line_size); + + for (; t0 < va + len; t0 += dcache_line_size) { + __asm __volatile(THEAD_DCACHE_CIVA + :: "r" (t0) : "memory"); + } + __asm __volatile(THEAD_SYNC_S ::: "memory"); +} + +static void +thead_cpu_dcache_inv_range(vm_offset_t va, vm_size_t len) +{ + register vm_offset_t t0 __asm("t0") = rounddown(va, dcache_line_size); + + for (; t0 < va + len; t0 += dcache_line_size) { + __asm __volatile(THEAD_DCACHE_IVA + :: "r" (t0) : "memory"); + } + __asm __volatile(THEAD_SYNC_S ::: "memory"); +} + +static void +thead_cpu_dcache_wb_range(vm_offset_t va, vm_size_t len) +{ + register vm_offset_t t0 __asm("t0") = rounddown(va, dcache_line_size); + + for (; t0 < va + len; t0 += dcache_line_size) { + __asm __volatile(THEAD_DCACHE_CVA + :: "r" (t0) : "memory"); + } + __asm __volatile(THEAD_SYNC_S ::: "memory"); +} + +void +thead_setup_cache(void) +{ + struct riscv_cache_ops thead_ops; + + thead_ops.dcache_wbinv_range = thead_cpu_dcache_wbinv_range; + thead_ops.dcache_inv_range = thead_cpu_dcache_inv_range; + thead_ops.dcache_wb_range = thead_cpu_dcache_wb_range; + + riscv_cache_install_hooks(&thead_ops, THEAD_DCACHE_SIZE); +}