Page MenuHomeFreeBSD
Files F144909946

D54943.id170699.diff
No OneTemporary
Actions

D54943.id170699.diff
View Options

diff --git a/sys/arm64/arm64/identcpu.c b/sys/arm64/arm64/identcpu.c
--- a/sys/arm64/arm64/identcpu.c
+++ b/sys/arm64/arm64/identcpu.c
@@ -2675,14 +2675,15 @@
if (cpu == 0) {
/* Create a user visible cpu description with safe values */
- memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
+ memset_early(&user_cpu_desc, 0, sizeof(user_cpu_desc));
/* Safe values for these registers */
user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
ID_AA64PFR0_EL0_64;
user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
/* Create the Linux user visible cpu description */
- memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc));
+ memcpy_early(&l_user_cpu_desc, &user_cpu_desc,
+ sizeof(user_cpu_desc));
}
desc = get_cpu_desc(cpu);
@@ -2746,6 +2747,7 @@
/* HWCAP */
bool __read_frequently lse_supported = false;
+bool __read_frequently mops_supported = false;
bool __read_frequently icache_aliasing = false;
bool __read_frequently icache_vmid = false;
@@ -2868,6 +2870,12 @@
panic("CPU does not support LSE atomic instructions");
#endif
+ if ((elf_hwcap2 & HWCAP2_MOPS) != 0) {
+ mops_supported = true;
+ if (bootverbose)
+ printf("Enabling MOPS in the kernel\n");
+ }
+
install_sys_handler(user_ctr_handler);
install_sys_handler(user_idreg_handler);
}
diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c
--- a/sys/arm64/arm64/machdep.c
+++ b/sys/arm64/arm64/machdep.c
@@ -80,6 +80,7 @@
#include <machine/cpu_feat.h>
#include <machine/debug_monitor.h>
#include <machine/hypervisor.h>
+#include <machine/ifunc.h>
#include <machine/kdb.h>
#include <machine/machdep.h>
#include <machine/metadata.h>
@@ -809,6 +810,8 @@
update_special_regs(0);
+ link_elf_ireloc();
+
/* Set the pcpu data, this is needed by pmap_bootstrap */
pcpup = &pcpu0;
pcpu_init(pcpup, 0, sizeof(struct pcpu));
@@ -825,7 +828,6 @@
PCPU_SET(curthread, &thread0);
PCPU_SET(midr, get_midr());
- link_elf_ireloc();
#ifdef FDT
try_load_dtb();
#endif
@@ -1077,3 +1079,35 @@
db_printf("show vtop <virt_addr>\n");
}
#endif
+
+#undef memset
+#undef memmove
+#undef memcpy
+
+void *memset_std(void *buf, int c, size_t len);
+void *memset_mops(void *buf, int c, size_t len);
+void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memmove_mops(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memcpy_mops(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+
+DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
+{
+ return ((mops_supported) != 0 ? memset_mops : memset_std);
+}
+
+DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull,
+ size_t))
+{
+ return ((mops_supported) != 0 ? memmove_mops : memmove_std);
+}
+
+DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,
+ size_t))
+{
+ return ((mops_supported) != 0 ? memcpy_mops : memcpy_std);
+}
diff --git a/sys/arm64/arm64/machdep_boot.c b/sys/arm64/arm64/machdep_boot.c
--- a/sys/arm64/arm64/machdep_boot.c
+++ b/sys/arm64/arm64/machdep_boot.c
@@ -115,7 +115,7 @@
PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_DTBP);
PRELOAD_PUSH_VALUE(uint32_t, sizeof(uint64_t));
PRELOAD_PUSH_VALUE(uint64_t, (uint64_t)lastaddr);
- memmove((void *)lastaddr, dtb_ptr, dtb_size);
+ memmove_early((void *)lastaddr, dtb_ptr, dtb_size);
lastaddr += dtb_size;
lastaddr = roundup(lastaddr, sizeof(int));
}
diff --git a/sys/arm64/arm64/memcpy.S b/sys/arm64/arm64/memcpy.S
--- a/sys/arm64/arm64/memcpy.S
+++ b/sys/arm64/arm64/memcpy.S
@@ -57,8 +57,8 @@
The loop tail is handled by always copying 64 bytes from the end.
*/
-EENTRY(memmove)
-ENTRY(memcpy)
+EENTRY(memmove_std)
+ENTRY(memcpy_std)
add srcend, src, count
add dstend, dstin, count
cmp count, 128
@@ -239,7 +239,23 @@
stp B_l, B_h, [dstin, 16]
stp C_l, C_h, [dstin]
ret
-END(memcpy)
-EEND(memmove)
+END(memcpy_std)
+EEND(memmove_std)
+
+ENTRY(memcpy_mops)
+ mov x3, x0
+ .inst 0x19010443 /* cpyfp [x3]!, [x1]!, x2! */
+ .inst 0x19410443 /* cpyfm [x3]!, [x1]!, x2! */
+ .inst 0x19810443 /* cpyfe [x3]!, [x1]!, x2! */
+ ret
+END(memcpy_mops)
+
+ENTRY(memmove_mops)
+ mov x3, x0
+ .inst 0x1d010443 /* cpyp [x3]!, [x1]!, x2! */
+ .inst 0x1d410443 /* cpym [x3]!, [x1]!, x2! */
+ .inst 0x1d810443 /* cpye [x3]!, [x1]!, x2! */
+ ret
+END(memmove_mops)
GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL)
diff --git a/sys/arm64/arm64/memset.S b/sys/arm64/arm64/memset.S
--- a/sys/arm64/arm64/memset.S
+++ b/sys/arm64/arm64/memset.S
@@ -51,7 +51,7 @@
#define dst x8
#define tmp3w w9
-ENTRY(memset)
+ENTRY(memset_std)
mov dst, dstin /* Preserve return value. */
ands A_lw, val, #255
@@ -196,6 +196,14 @@
ands count, count, zva_bits_x
b.ne .Ltail_maybe_long
ret
-END(memset)
+END(memset_std)
+
+ENTRY(memset_mops)
+ mov x3, x0
+ .inst 0x19c10443 /* setp [x3]!, x2!, x1 */
+ .inst 0x19c14443 /* setm [x3]!, x2!, x1 */
+ .inst 0x19c18443 /* sete [x3]!, x2!, x1 */
+ ret
+END(memset_mops)
GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL)
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -327,8 +327,15 @@
ADDRESS_TRANSLATE_FUNC(s1e1r)
ADDRESS_TRANSLATE_FUNC(s1e1w)
+extern bool mops_supported;
+
#endif /* !__ASSEMBLER__ */
-#endif
+
+#define MEMSET_EARLY_FUNC memset_std
+#define MEMCPY_EARLY_FUNC memcpy_std
+#define MEMMOVE_EARLY_FUNC memmove_std
+
+#endif /* _KERNEL */
#endif /* !_MACHINE_CPU_H_ */

File Metadata

Mime Type
text/plain
Expires
Sun, Feb 15, 12:39 AM (6 h, 37 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28720699
Default Alt Text
D54943.id170699.diff (5 KB)

Event Timeline