diff --git a/sys/arm64/arm64/identcpu.c b/sys/arm64/arm64/identcpu.c --- a/sys/arm64/arm64/identcpu.c +++ b/sys/arm64/arm64/identcpu.c @@ -2675,14 +2675,15 @@ if (cpu == 0) { /* Create a user visible cpu description with safe values */ - memset(&user_cpu_desc, 0, sizeof(user_cpu_desc)); + memset_early(&user_cpu_desc, 0, sizeof(user_cpu_desc)); /* Safe values for these registers */ user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE | ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64; user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8; /* Create the Linux user visible cpu description */ - memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc)); + memcpy_early(&l_user_cpu_desc, &user_cpu_desc, + sizeof(user_cpu_desc)); } desc = get_cpu_desc(cpu); diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -80,6 +80,7 @@ #include #include #include +#include #include #include #include @@ -809,6 +810,9 @@ update_special_regs(0); + sched_instance_select(); + link_elf_ireloc(); + /* Set the pcpu data, this is needed by pmap_bootstrap */ pcpup = &pcpu0; pcpu_init(pcpup, 0, sizeof(struct pcpu)); @@ -825,8 +829,6 @@ PCPU_SET(curthread, &thread0); PCPU_SET(midr, get_midr()); - sched_instance_select(); - link_elf_ireloc(); #ifdef FDT try_load_dtb(); #endif @@ -1078,3 +1080,35 @@ db_printf("show vtop \n"); } #endif + +#undef memset +#undef memmove +#undef memcpy + +void *memset_std(void *buf, int c, size_t len); +void *memset_mops(void *buf, int c, size_t len); +void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memmove_mops(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_mops(void * _Nonnull dst, const void * _Nonnull src, + size_t len); + +DEFINE_IFUNC(, void *, memset, (void *, int, size_t)) +{ + return ((elf_hwcap2 & HWCAP2_MOPS) != 0 ? memset_mops : memset_std); +} + +DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull, + size_t)) +{ + return ((elf_hwcap2 & HWCAP2_MOPS) != 0 ? memmove_mops : memmove_std); +} + +DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull, + size_t)) +{ + return ((elf_hwcap2 & HWCAP2_MOPS) != 0 ? memcpy_mops : memcpy_std); +} diff --git a/sys/arm64/arm64/machdep_boot.c b/sys/arm64/arm64/machdep_boot.c --- a/sys/arm64/arm64/machdep_boot.c +++ b/sys/arm64/arm64/machdep_boot.c @@ -115,7 +115,7 @@ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_DTBP); PRELOAD_PUSH_VALUE(uint32_t, sizeof(uint64_t)); PRELOAD_PUSH_VALUE(uint64_t, (uint64_t)lastaddr); - memmove((void *)lastaddr, dtb_ptr, dtb_size); + memmove_early((void *)lastaddr, dtb_ptr, dtb_size); lastaddr += dtb_size; lastaddr = roundup(lastaddr, sizeof(int)); } diff --git a/sys/arm64/arm64/memcpy.S b/sys/arm64/arm64/memcpy.S --- a/sys/arm64/arm64/memcpy.S +++ b/sys/arm64/arm64/memcpy.S @@ -57,8 +57,8 @@ The loop tail is handled by always copying 64 bytes from the end. */ -EENTRY(memmove) -ENTRY(memcpy) +EENTRY(memmove_std) +ENTRY(memcpy_std) add srcend, src, count add dstend, dstin, count cmp count, 128 @@ -239,7 +239,23 @@ stp B_l, B_h, [dstin, 16] stp C_l, C_h, [dstin] ret -END(memcpy) -EEND(memmove) +END(memcpy_std) +EEND(memmove_std) + +ENTRY(memcpy_mops) + mov x3, x0 + .inst 0x19010443 /* cpyfp [x3]!, [x1]!, x2! */ + .inst 0x19410443 /* cpyfm [x3]!, [x1]!, x2! */ + .inst 0x19810443 /* cpyfe [x3]!, [x1]!, x2! */ + ret +END(memcpy_mops) + +ENTRY(memmove_mops) + mov x3, x0 + .inst 0x1d010443 /* cpyp [x3]!, [x1]!, x2! */ + .inst 0x1d410443 /* cpym [x3]!, [x1]!, x2! */ + .inst 0x1d810443 /* cpye [x3]!, [x1]!, x2! */ + ret +END(memmove_mops) GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/arm64/memset.S b/sys/arm64/arm64/memset.S --- a/sys/arm64/arm64/memset.S +++ b/sys/arm64/arm64/memset.S @@ -51,7 +51,7 @@ #define dst x8 #define tmp3w w9 -ENTRY(memset) +ENTRY(memset_std) mov dst, dstin /* Preserve return value. */ ands A_lw, val, #255 @@ -196,6 +196,14 @@ ands count, count, zva_bits_x b.ne .Ltail_maybe_long ret -END(memset) +END(memset_std) + +ENTRY(memset_mops) + mov x3, x0 + .inst 0x19c10443 /* setp [x3]!, x2!, x1 */ + .inst 0x19c14443 /* setm [x3]!, x2!, x1 */ + .inst 0x19c18443 /* sete [x3]!, x2!, x1 */ + ret +END(memset_mops) GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h --- a/sys/arm64/include/cpu.h +++ b/sys/arm64/include/cpu.h @@ -328,7 +328,12 @@ ADDRESS_TRANSLATE_FUNC(s1e1w) #endif /* !__ASSEMBLER__ */ -#endif + +#define MEMSET_EARLY_FUNC memset_std +#define MEMCPY_EARLY_FUNC memcpy_std +#define MEMMOVE_EARLY_FUNC memmove_std + +#endif /* _KERNEL */ #endif /* !_MACHINE_CPU_H_ */