Page MenuHomeFreeBSD
Files F144447629

D49852.id153734.diff
No OneTemporary
Actions

D49852.id153734.diff
View Options

Index: sys/conf/files.riscv
===================================================================
--- sys/conf/files.riscv
+++ sys/conf/files.riscv
@@ -46,6 +46,7 @@
riscv/riscv/busdma_bounce.c standard
riscv/riscv/busdma_machdep.c standard
riscv/riscv/cache.c standard
+riscv/riscv/cbo.c standard
riscv/riscv/clock.c standard
riscv/riscv/copyinout.S standard
riscv/riscv/cpufunc_asm.S standard
Index: sys/riscv/include/cbo.h
===================================================================
--- /dev/null
+++ sys/riscv/include/cbo.h
@@ -0,0 +1,33 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _RISCV_CBO_H_
+#define _RISCV_CBO_H_
+
+void cbo_zicbom_setup_cache(int cbom_block_size);
+
+#endif /* _RISCV_CBO_H_ */
Index: sys/riscv/riscv/cbo.c
===================================================================
--- /dev/null
+++ sys/riscv/riscv/cbo.c
@@ -0,0 +1,104 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* Cache Block Operations. */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <machine/cbo.h>
+
+static void
+cbo_zicbom_cpu_dcache_wbinv_range(vm_offset_t va, vm_size_t len)
+{
+ vm_offset_t addr;
+
+ /*
+ * A flush operation atomically performs a clean operation followed by
+ * an invalidate operation.
+ */
+
+ va &= ~(dcache_line_size - 1);
+ for (addr = va; addr < va + len; addr += dcache_line_size)
+ __asm __volatile(".option push; .option arch, +zicbom\n"
+ "cbo.flush 0(%0); .option pop\n" :: "r"(addr));
+}
+
+static void
+cbo_zicbom_cpu_dcache_inv_range(vm_offset_t va, vm_size_t len)
+{
+ vm_offset_t addr;
+
+ /*
+ * An invalidate operation makes data from store operations performed by
+ * a set of non-coherent agents visible to the set of coherent agents at
+ * a point common to both sets by deallocating all copies of a cache
+ * block from the set of coherent caches up to that point.
+ */
+
+ va &= ~(dcache_line_size - 1);
+ for (addr = va; addr < va + len; addr += dcache_line_size)
+ __asm __volatile(".option push; .option arch, +zicbom\n"
+ "cbo.inval 0(%0); .option pop\n" :: "r"(addr));
+}
+
+static void
+cbo_zicbom_cpu_dcache_wb_range(vm_offset_t va, vm_size_t len)
+{
+ vm_offset_t addr;
+
+ /*
+ * A clean operation makes data from store operations performed by the
+ * set of coherent agents visible to a set of non-coherent agents at a
+ * point common to both sets by performing a write transfer of a copy of
+ * a cache block to that point provided a coherent agent performed a
+ * store operation that modified the data in the cache block since the
+ * previous invalidate, clean, or flush operation on the cache block.
+ */
+
+ va &= ~(dcache_line_size - 1);
+ for (addr = va; addr < va + len; addr += dcache_line_size)
+ __asm __volatile(".option push; .option arch, +zicbom\n"
+ "cbo.clean 0(%0); .option pop\n" :: "r"(addr));
+}
+
+void
+cbo_zicbom_setup_cache(int cbom_block_size)
+{
+ struct riscv_cache_ops zicbom_ops;
+
+ if (cbom_block_size <= 0 || !powerof2(cbom_block_size)) {
+ printf("Zicbom: could not initialise (invalid cache line %d)\n",
+ cbom_block_size);
+ return;
+ }
+
+ zicbom_ops.dcache_wbinv_range = cbo_zicbom_cpu_dcache_wbinv_range;
+ zicbom_ops.dcache_inv_range = cbo_zicbom_cpu_dcache_inv_range;
+ zicbom_ops.dcache_wb_range = cbo_zicbom_cpu_dcache_wb_range;
+ riscv_cache_install_hooks(&zicbom_ops, cbom_block_size);
+}
Index: sys/riscv/riscv/identcpu.c
===================================================================
--- sys/riscv/riscv/identcpu.c
+++ sys/riscv/riscv/identcpu.c
@@ -53,6 +53,7 @@
#include <machine/elf.h>
#include <machine/md_var.h>
#include <machine/thead.h>
+#include <machine/cbo.h>
#ifdef FDT
#include <dev/fdt/fdt_common.h>
@@ -78,6 +79,11 @@
bool __read_frequently has_sscofpmf;
bool has_svpbmt;
+/* Z-extensions support. */
+bool has_zicbom;
+bool has_zicboz;
+bool has_zicbop;
+
struct cpu_desc {
const char *cpu_mvendor_name;
const char *cpu_march_name;
@@ -89,6 +95,12 @@
#define SV_SVPBMT (1 << 2)
#define SV_SVINVAL (1 << 3)
#define SV_SSCOFPMF (1 << 4)
+ u_int z_extensions; /* Multi-letter extensions. */
+#define Z_ZICBOM (1 << 0)
+#define Z_ZICBOZ (1 << 1)
+#define Z_ZICBOP (1 << 2)
+ int cbom_block_size;
+ int cboz_block_size;
};
struct cpu_desc cpu_desc[MAXCPU];
@@ -196,11 +208,24 @@
static __inline int
parse_ext_z(struct cpu_desc *desc __unused, char *isa, int idx, int len)
{
+#define CHECK_Z_EXT(str, flag) \
+ do { \
+ if (strncmp(&isa[idx], (str), \
+ MIN(strlen(str), len - idx)) == 0) { \
+ desc->z_extensions |= flag; \
+ return (idx + strlen(str)); \
+ } \
+ } while (0)
+
+ /* Check for known/supported extensions. */
+ CHECK_Z_EXT("zicbom", Z_ZICBOM);
+ CHECK_Z_EXT("zicboz", Z_ZICBOZ);
+ CHECK_Z_EXT("zicbop", Z_ZICBOP);
+
+#undef CHECK_Z_EXT
/*
* Proceed to the next multi-letter extension or the end of the
* string.
- *
- * TODO: parse some of these.
*/
while (isa[idx] != '_' && idx < len) {
idx++;
@@ -321,6 +346,22 @@
}
}
+static void
+parse_cbo_fdt(struct cpu_desc *desc, phandle_t node)
+{
+ int error;
+
+ error = OF_getencprop(node, "riscv,cbom-block-size",
+ &desc->cbom_block_size, sizeof(desc->cbom_block_size));
+ if (error)
+ desc->cbom_block_size = 0;
+
+ error = OF_getencprop(node, "riscv,cboz-block-size",
+ &desc->cboz_block_size, sizeof(desc->cboz_block_size));
+ if (error)
+ desc->cboz_block_size = 0;
+}
+
static void
identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc)
{
@@ -372,6 +413,9 @@
/* Check MMU features. */
parse_mmu_fdt(desc, node);
+ /* Cache-block operations (CBO). */
+ parse_cbo_fdt(desc, node);
+
/* We are done. */
break;
}
@@ -422,6 +466,11 @@
UPDATE_CAP(has_sscofpmf, (desc->smode_extensions & SV_SSCOFPMF) != 0);
UPDATE_CAP(has_svpbmt, (desc->smode_extensions & SV_SVPBMT) != 0);
+ /* Z extension support. */
+ UPDATE_CAP(has_zicbom, (desc->z_extensions & Z_ZICBOM) != 0);
+ UPDATE_CAP(has_zicboz, (desc->z_extensions & Z_ZICBOZ) != 0);
+ UPDATE_CAP(has_zicbop, (desc->z_extensions & Z_ZICBOP) != 0);
+
#undef UPDATE_CAP
}
@@ -506,6 +555,9 @@
update_global_capabilities(cpu, desc);
handle_cpu_quirks(cpu, desc);
+
+ if (has_zicbom && cpu == 0)
+ cbo_zicbom_setup_cache(desc->cbom_block_size);
}
void

File Metadata

Mime Type
text/plain
Expires
Mon, Feb 9, 12:08 PM (11 h, 38 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28576620
Default Alt Text
D49852.id153734.diff (8 KB)

Event Timeline