diff --git a/sys/arm64/arm64/identcpu.c b/sys/arm64/arm64/identcpu.c --- a/sys/arm64/arm64/identcpu.c +++ b/sys/arm64/arm64/identcpu.c @@ -2675,14 +2675,15 @@ if (cpu == 0) { /* Create a user visible cpu description with safe values */ - memset(&user_cpu_desc, 0, sizeof(user_cpu_desc)); + memset_early(&user_cpu_desc, 0, sizeof(user_cpu_desc)); /* Safe values for these registers */ user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE | ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64; user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8; /* Create the Linux user visible cpu description */ - memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc)); + memcpy_early(&l_user_cpu_desc, &user_cpu_desc, + sizeof(user_cpu_desc)); } desc = get_cpu_desc(cpu); @@ -2746,6 +2747,7 @@ /* HWCAP */ bool __read_frequently lse_supported = false; +bool __read_frequently mops_supported = false; bool __read_frequently icache_aliasing = false; bool __read_frequently icache_vmid = false; @@ -2868,6 +2870,12 @@ panic("CPU does not support LSE atomic instructions"); #endif + if ((elf_hwcap2 & HWCAP2_MOPS) != 0) { + mops_supported = true; + if (bootverbose) + printf("Enabling MOPS in the kernel\n"); + } + install_sys_handler(user_ctr_handler); install_sys_handler(user_idreg_handler); } diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -80,6 +80,7 @@ #include #include #include +#include #include #include #include @@ -809,6 +810,8 @@ update_special_regs(0); + link_elf_ireloc(); + /* Set the pcpu data, this is needed by pmap_bootstrap */ pcpup = &pcpu0; pcpu_init(pcpup, 0, sizeof(struct pcpu)); @@ -825,7 +828,6 @@ PCPU_SET(curthread, &thread0); PCPU_SET(midr, get_midr()); - link_elf_ireloc(); #ifdef FDT try_load_dtb(); #endif @@ -1077,3 +1079,35 @@ db_printf("show vtop \n"); } #endif + +#undef memset +#undef memmove +#undef memcpy + +void *memset_std(void *buf, int c, size_t len); +void *memset_mops(void *buf, int c, size_t len); +void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memmove_mops(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_mops(void * _Nonnull dst, const void * _Nonnull src, + size_t len); + +DEFINE_IFUNC(, void *, memset, (void *, int, size_t)) +{ + return ((mops_supported) != 0 ? memset_mops : memset_std); +} + +DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull, + size_t)) +{ + return ((mops_supported) != 0 ? memmove_mops : memmove_std); +} + +DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull, + size_t)) +{ + return ((mops_supported) != 0 ? memcpy_mops : memcpy_std); +} diff --git a/sys/arm64/arm64/machdep_boot.c b/sys/arm64/arm64/machdep_boot.c --- a/sys/arm64/arm64/machdep_boot.c +++ b/sys/arm64/arm64/machdep_boot.c @@ -115,7 +115,7 @@ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_DTBP); PRELOAD_PUSH_VALUE(uint32_t, sizeof(uint64_t)); PRELOAD_PUSH_VALUE(uint64_t, (uint64_t)lastaddr); - memmove((void *)lastaddr, dtb_ptr, dtb_size); + memmove_early((void *)lastaddr, dtb_ptr, dtb_size); lastaddr += dtb_size; lastaddr = roundup(lastaddr, sizeof(int)); } diff --git a/sys/arm64/arm64/memcpy.S b/sys/arm64/arm64/memcpy.S --- a/sys/arm64/arm64/memcpy.S +++ b/sys/arm64/arm64/memcpy.S @@ -57,8 +57,8 @@ The loop tail is handled by always copying 64 bytes from the end. */ -EENTRY(memmove) -ENTRY(memcpy) +EENTRY(memmove_std) +ENTRY(memcpy_std) add srcend, src, count add dstend, dstin, count cmp count, 128 @@ -239,7 +239,23 @@ stp B_l, B_h, [dstin, 16] stp C_l, C_h, [dstin] ret -END(memcpy) -EEND(memmove) +END(memcpy_std) +EEND(memmove_std) + +ENTRY(memcpy_mops) + mov x3, x0 + .inst 0x19010443 /* cpyfp [x3]!, [x1]!, x2! */ + .inst 0x19410443 /* cpyfm [x3]!, [x1]!, x2! */ + .inst 0x19810443 /* cpyfe [x3]!, [x1]!, x2! */ + ret +END(memcpy_mops) + +ENTRY(memmove_mops) + mov x3, x0 + .inst 0x1d010443 /* cpyp [x3]!, [x1]!, x2! */ + .inst 0x1d410443 /* cpym [x3]!, [x1]!, x2! */ + .inst 0x1d810443 /* cpye [x3]!, [x1]!, x2! */ + ret +END(memmove_mops) GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/arm64/memset.S b/sys/arm64/arm64/memset.S --- a/sys/arm64/arm64/memset.S +++ b/sys/arm64/arm64/memset.S @@ -51,7 +51,7 @@ #define dst x8 #define tmp3w w9 -ENTRY(memset) +ENTRY(memset_std) mov dst, dstin /* Preserve return value. */ ands A_lw, val, #255 @@ -196,6 +196,14 @@ ands count, count, zva_bits_x b.ne .Ltail_maybe_long ret -END(memset) +END(memset_std) + +ENTRY(memset_mops) + mov x3, x0 + .inst 0x19c10443 /* setp [x3]!, x2!, x1 */ + .inst 0x19c14443 /* setm [x3]!, x2!, x1 */ + .inst 0x19c18443 /* sete [x3]!, x2!, x1 */ + ret +END(memset_mops) GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/arm64/trap.c b/sys/arm64/arm64/trap.c --- a/sys/arm64/arm64/trap.c +++ b/sys/arm64/arm64/trap.c @@ -480,6 +480,66 @@ } #endif +static void +handle_moe(struct thread *td, struct trapframe *frame, uint64_t esr) +{ + uint64_t src; + uint64_t dest; + uint64_t size; + int src_reg; + int dest_reg; + int size_reg; + int format_option; + + format_option = esr & ISS_MOE_FORMAT_OPTION_MASK; + dest_reg = (esr & ISS_MOE_DESTREG_MASK) >> ISS_MOE_DESTREG_SHIFT; + size_reg = (esr & ISS_MOE_SIZEREG_MASK) >> ISS_MOE_SIZEREG_SHIFT; + dest = frame->tf_x[dest_reg]; + size = frame->tf_x[size_reg]; + + /* + * Put the registers back in the original format suitable for a + * prologue instruction, using the generic return routine from the + * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH. + */ + if (esr & ISS_MOE_MEMINST) { + /* SET* instruction */ + if (format_option == ISS_MOE_FORMAT_OPTION_A || + format_option == ISS_MOE_FORMAT_OPTION_A2) { + /* Format is from Option A; forward set */ + frame->tf_x[dest_reg] = dest + size; + frame->tf_x[size_reg] = -size; + } + } else { + /* CPY* instruction */ + src_reg = (esr & ISS_MOE_SRCREG_MASK) >> ISS_MOE_SRCREG_SHIFT; + src = frame->tf_x[src_reg]; + + if (format_option == ISS_MOE_FORMAT_OPTION_B || + format_option == ISS_MOE_FORMAT_OPTION_B2) { + /* Format is from Option B */ + if (frame->tf_spsr & PSR_N) { + /* Backward copy */ + frame->tf_x[dest_reg] = dest - size; + frame->tf_x[src_reg] = src + size; + } + } else { + /* Format is from Option A */ + if (frame->tf_x[size_reg] & (1UL << 63)) { + /* Forward copy */ + frame->tf_x[dest_reg] = dest + size; + frame->tf_x[src_reg] = src + size; + frame->tf_x[size_reg] = -size; + } + } + } + + if (esr & ISS_MOE_FROM_EPILOGUE) + frame->tf_elr -= 8; + else + frame->tf_elr -= 4; +} + /* * See the comment above data_abort(). */ @@ -589,6 +649,9 @@ print_gp_register("far", far); panic("Branch Target exception"); break; + case EXCP_MOE: + handle_moe(td, frame, esr); + break; default: print_registers(frame); print_gp_register("far", far); @@ -597,66 +660,6 @@ } } -static void -handle_moe(struct thread *td, struct trapframe *frame, uint64_t esr) -{ - uint64_t src; - uint64_t dest; - uint64_t size; - int src_reg; - int dest_reg; - int size_reg; - int format_option; - - format_option = esr & ISS_MOE_FORMAT_OPTION_MASK; - dest_reg = (esr & ISS_MOE_DESTREG_MASK) >> ISS_MOE_DESTREG_SHIFT; - size_reg = (esr & ISS_MOE_SIZEREG_MASK) >> ISS_MOE_SIZEREG_SHIFT; - dest = frame->tf_x[dest_reg]; - size = frame->tf_x[size_reg]; - - /* - * Put the registers back in the original format suitable for a - * prologue instruction, using the generic return routine from the - * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH. - */ - if (esr & ISS_MOE_MEMINST) { - /* SET* instruction */ - if (format_option == ISS_MOE_FORMAT_OPTION_A || - format_option == ISS_MOE_FORMAT_OPTION_A2) { - /* Format is from Option A; forward set */ - frame->tf_x[dest_reg] = dest + size; - frame->tf_x[size_reg] = -size; - } - } else { - /* CPY* instruction */ - src_reg = (esr & ISS_MOE_SRCREG_MASK) >> ISS_MOE_SRCREG_SHIFT; - src = frame->tf_x[src_reg]; - - if (format_option == ISS_MOE_FORMAT_OPTION_B || - format_option == ISS_MOE_FORMAT_OPTION_B2) { - /* Format is from Option B */ - if (frame->tf_spsr & PSR_N) { - /* Backward copy */ - frame->tf_x[dest_reg] = dest - size; - frame->tf_x[src_reg] = src + size; - } - } else { - /* Format is from Option A */ - if (frame->tf_x[size_reg] & (1UL << 63)) { - /* Forward copy */ - frame->tf_x[dest_reg] = dest + size; - frame->tf_x[src_reg] = src + size; - frame->tf_x[size_reg] = -size; - } - } - } - - if (esr & ISS_MOE_FROM_EPILOGUE) - frame->tf_elr -= 8; - else - frame->tf_elr -= 4; -} - void do_el0_sync(struct thread *td, struct trapframe *frame) { diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h --- a/sys/arm64/include/cpu.h +++ b/sys/arm64/include/cpu.h @@ -327,8 +327,15 @@ ADDRESS_TRANSLATE_FUNC(s1e1r) ADDRESS_TRANSLATE_FUNC(s1e1w) +extern bool mops_supported; + #endif /* !__ASSEMBLER__ */ -#endif + +#define MEMSET_EARLY_FUNC memset_std +#define MEMCPY_EARLY_FUNC memcpy_std +#define MEMMOVE_EARLY_FUNC memmove_std + +#endif /* _KERNEL */ #endif /* !_MACHINE_CPU_H_ */