Index: sys/conf/files.riscv =================================================================== --- sys/conf/files.riscv +++ sys/conf/files.riscv @@ -46,6 +46,7 @@ riscv/riscv/busdma_bounce.c standard riscv/riscv/busdma_machdep.c standard riscv/riscv/cache.c standard +riscv/riscv/cbo.c standard riscv/riscv/clock.c standard riscv/riscv/copyinout.S standard riscv/riscv/cpufunc_asm.S standard Index: sys/riscv/include/cbo.h =================================================================== --- /dev/null +++ sys/riscv/include/cbo.h @@ -0,0 +1,33 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Ruslan Bukin + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _RISCV_CBO_H_ +#define _RISCV_CBO_H_ + +void cbo_zicbom_setup_cache(int cbom_block_size); + +#endif /* _RISCV_CBO_H_ */ Index: sys/riscv/riscv/cbo.c =================================================================== --- /dev/null +++ sys/riscv/riscv/cbo.c @@ -0,0 +1,104 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Ruslan Bukin + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* Cache Block Operations. */ + +#include +#include + +#include + +static void +cbo_zicbom_cpu_dcache_wbinv_range(vm_offset_t va, vm_size_t len) +{ + vm_offset_t addr; + + /* + * A flush operation atomically performs a clean operation followed by + * an invalidate operation. + */ + + va &= ~(dcache_line_size - 1); + for (addr = va; addr < va + len; addr += dcache_line_size) + __asm __volatile(".option push; .option arch, +zicbom\n" + "cbo.flush (%0); .option pop\n" :: "r"(addr)); +} + +static void +cbo_zicbom_cpu_dcache_inv_range(vm_offset_t va, vm_size_t len) +{ + vm_offset_t addr; + + /* + * An invalidate operation makes data from store operations performed by + * a set of non-coherent agents visible to the set of coherent agents at + * a point common to both sets by deallocating all copies of a cache + * block from the set of coherent caches up to that point. + */ + + va &= ~(dcache_line_size - 1); + for (addr = va; addr < va + len; addr += dcache_line_size) + __asm __volatile(".option push; .option arch, +zicbom\n" + "cbo.inval (%0); .option pop\n" :: "r"(addr)); +} + +static void +cbo_zicbom_cpu_dcache_wb_range(vm_offset_t va, vm_size_t len) +{ + vm_offset_t addr; + + /* + * A clean operation makes data from store operations performed by the + * set of coherent agents visible to a set of non-coherent agents at a + * point common to both sets by performing a write transfer of a copy of + * a cache block to that point provided a coherent agent performed a + * store operation that modified the data in the cache block since the + * previous invalidate, clean, or flush operation on the cache block. + */ + + va &= ~(dcache_line_size - 1); + for (addr = va; addr < va + len; addr += dcache_line_size) + __asm __volatile(".option push; .option arch, +zicbom\n" + "cbo.clean (%0); .option pop\n" :: "r"(addr)); +} + +void +cbo_zicbom_setup_cache(int cbom_block_size) +{ + struct riscv_cache_ops zicbom_ops; + + if (cbom_block_size <= 0 || !powerof2(cbom_block_size)) { + printf("Zicbom: could not initialise (invalid cache line %d)\n", + cbom_block_size); + return; + } + + zicbom_ops.dcache_wbinv_range = cbo_zicbom_cpu_dcache_wbinv_range; + zicbom_ops.dcache_inv_range = cbo_zicbom_cpu_dcache_inv_range; + zicbom_ops.dcache_wb_range = cbo_zicbom_cpu_dcache_wb_range; + riscv_cache_install_hooks(&zicbom_ops, cbom_block_size); +} Index: sys/riscv/riscv/identcpu.c =================================================================== --- sys/riscv/riscv/identcpu.c +++ sys/riscv/riscv/identcpu.c @@ -53,6 +53,7 @@ #include #include #include +#include #ifdef FDT #include @@ -78,6 +79,11 @@ bool __read_frequently has_sscofpmf; bool has_svpbmt; +/* Z-extensions support. */ +bool has_zicbom; +bool has_zicboz; +bool has_zicbop; + struct cpu_desc { const char *cpu_mvendor_name; const char *cpu_march_name; @@ -89,6 +95,12 @@ #define SV_SVPBMT (1 << 2) #define SV_SVINVAL (1 << 3) #define SV_SSCOFPMF (1 << 4) + u_int z_extensions; /* Multi-letter extensions. */ +#define Z_ZICBOM (1 << 0) +#define Z_ZICBOZ (1 << 1) +#define Z_ZICBOP (1 << 2) + int cbom_block_size; + int cboz_block_size; }; struct cpu_desc cpu_desc[MAXCPU]; @@ -196,11 +208,24 @@ static __inline int parse_ext_z(struct cpu_desc *desc __unused, char *isa, int idx, int len) { +#define CHECK_Z_EXT(str, flag) \ + do { \ + if (strncmp(&isa[idx], (str), \ + MIN(strlen(str), len - idx)) == 0) { \ + desc->z_extensions |= flag; \ + return (idx + strlen(str)); \ + } \ + } while (0) + + /* Check for known/supported extensions. */ + CHECK_Z_EXT("zicbom", Z_ZICBOM); + CHECK_Z_EXT("zicboz", Z_ZICBOZ); + CHECK_Z_EXT("zicbop", Z_ZICBOP); + +#undef CHECK_Z_EXT /* * Proceed to the next multi-letter extension or the end of the * string. - * - * TODO: parse some of these. */ while (isa[idx] != '_' && idx < len) { idx++; @@ -321,6 +346,22 @@ } } +static void +parse_cbo_fdt(struct cpu_desc *desc, phandle_t node) +{ + int error; + + error = OF_getencprop(node, "riscv,cbom-block-size", + &desc->cbom_block_size, sizeof(desc->cbom_block_size)); + if (error == -1) + desc->cbom_block_size = 0; + + error = OF_getencprop(node, "riscv,cboz-block-size", + &desc->cboz_block_size, sizeof(desc->cboz_block_size)); + if (error == -1) + desc->cboz_block_size = 0; +} + static void identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc) { @@ -372,6 +413,9 @@ /* Check MMU features. */ parse_mmu_fdt(desc, node); + /* Cache-block operations (CBO). */ + parse_cbo_fdt(desc, node); + /* We are done. */ break; } @@ -422,6 +466,11 @@ UPDATE_CAP(has_sscofpmf, (desc->smode_extensions & SV_SSCOFPMF) != 0); UPDATE_CAP(has_svpbmt, (desc->smode_extensions & SV_SVPBMT) != 0); + /* Z extension support. */ + UPDATE_CAP(has_zicbom, (desc->z_extensions & Z_ZICBOM) != 0); + UPDATE_CAP(has_zicboz, (desc->z_extensions & Z_ZICBOZ) != 0); + UPDATE_CAP(has_zicbop, (desc->z_extensions & Z_ZICBOP) != 0); + #undef UPDATE_CAP } @@ -506,6 +555,9 @@ update_global_capabilities(cpu, desc); handle_cpu_quirks(cpu, desc); + + if (has_zicbom && cpu == 0) + cbo_zicbom_setup_cache(desc->cbom_block_size); } void