diff --git a/sys/arm64/arm64/identcpu.c b/sys/arm64/arm64/identcpu.c --- a/sys/arm64/arm64/identcpu.c +++ b/sys/arm64/arm64/identcpu.c @@ -1094,6 +1094,11 @@ MRS_FIELD_VALUE_END, }; +static const struct mrs_field_hwcap id_aa64isar2_mops_caps[] = { + MRS_HWCAP(2, HWCAP2_MOPS, ID_AA64ISAR2_MOPS_IMPL), + MRS_HWCAP_END +}; + static const struct mrs_field_value id_aa64isar2_apa3[] = { MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""), MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"), @@ -1149,7 +1154,8 @@ MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_LOWER, 0, id_aa64isar2_pac_frac), MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_LOWER, 0, id_aa64isar2_bc), - MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_LOWER, 0, id_aa64isar2_mops), + MRS_FIELD_HWCAP(ID_AA64ISAR2, MOPS, false, MRS_LOWER, MRS_USERSPACE, + id_aa64isar2_mops, id_aa64isar2_mops_caps), MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_LOWER, MRS_USERSPACE, id_aa64isar2_apa3, id_aa64isar2_apa3_caps), MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_LOWER, MRS_USERSPACE, diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -219,6 +219,41 @@ pan_check, NULL, pan_enable, pan_disabled, CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); +static cpu_feat_en +mops_check(const struct cpu_feat *feat __unused, u_int midr __unused) +{ + uint64_t id_aa64isar2; + + if (!get_kernel_reg(ID_AA64ISAR2_EL1, &id_aa64isar2)) + return (FEAT_ALWAYS_DISABLE); + if (ID_AA64ISAR2_MOPS_VAL(id_aa64isar2) == ID_AA64ISAR2_MOPS_NONE) + return (FEAT_ALWAYS_DISABLE); + + return (FEAT_DEFAULT_ENABLE); +} + +static bool +mops_enable(const struct cpu_feat *feat __unused, + cpu_feat_errata errata_status __unused, u_int *errata_list __unused, + u_int errata_count __unused) +{ + WRITE_SPECIALREG(sctlr_el1, READ_SPECIALREG(sctlr_el1) | SCTLR_MSCEn); + isb(); + + return (true); +} + +static void +mops_disabled(const struct cpu_feat *feat __unused) +{ + WRITE_SPECIALREG(sctlr_el1, READ_SPECIALREG(sctlr_el1) & ~SCTLR_MSCEn); + isb(); +} + +CPU_FEAT(feat_mops, "MOPS", + mops_check, NULL, mops_enable, mops_disabled, + CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); + bool has_hyp(void) { diff --git a/sys/arm64/arm64/trap.c b/sys/arm64/arm64/trap.c --- a/sys/arm64/arm64/trap.c +++ b/sys/arm64/arm64/trap.c @@ -597,6 +597,66 @@ } } +static void +handle_moe(struct thread *td, struct trapframe *frame, uint64_t esr) +{ + uint64_t src; + uint64_t dest; + uint64_t size; + int src_reg; + int dest_reg; + int size_reg; + int format_option; + + format_option = esr & ISS_MOE_FORMAT_OPTION_MASK; + dest_reg = (esr & ISS_MOE_DESTREG_MASK) >> ISS_MOE_DESTREG_SHIFT; + size_reg = (esr & ISS_MOE_SIZEREG_MASK) >> ISS_MOE_SIZEREG_SHIFT; + dest = frame->tf_x[dest_reg]; + size = frame->tf_x[size_reg]; + + /* + * Put the registers back in the original format suitable for a + * prologue instruction, using the generic return routine from the + * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH. + */ + if (esr & ISS_MOE_MEMINST) { + /* SET* instruction */ + if (format_option == ISS_MOE_FORMAT_OPTION_A || + format_option == ISS_MOE_FORMAT_OPTION_A2) { + /* Format is from Option A; forward set */ + frame->tf_x[dest_reg] = dest + size; + frame->tf_x[size_reg] = -size; + } + } else { + /* CPY* instruction */ + src_reg = (esr & ISS_MOE_SRCREG_MASK) >> ISS_MOE_SRCREG_SHIFT; + src = frame->tf_x[src_reg]; + + if (format_option == ISS_MOE_FORMAT_OPTION_B || + format_option == ISS_MOE_FORMAT_OPTION_B2) { + /* Format is from Option B */ + if (frame->tf_spsr & PSR_N) { + /* Backward copy */ + frame->tf_x[dest_reg] = dest - size; + frame->tf_x[src_reg] = src + size; + } + } else { + /* Format is from Option A */ + if (frame->tf_x[size_reg] & (1UL << 63)) { + /* Forward copy */ + frame->tf_x[dest_reg] = dest + size; + frame->tf_x[src_reg] = src + size; + frame->tf_x[size_reg] = -size; + } + } + } + + if (esr & ISS_MOE_FROM_EPILOGUE) + frame->tf_elr -= 8; + else + frame->tf_elr -= 4; +} + void do_el0_sync(struct thread *td, struct trapframe *frame) { @@ -738,6 +798,10 @@ exception); userret(td, frame); break; + case EXCP_MOE: + handle_moe(td, frame, esr); + userret(td, frame); + break; default: call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr, exception); diff --git a/sys/arm64/include/armreg.h b/sys/arm64/include/armreg.h --- a/sys/arm64/include/armreg.h +++ b/sys/arm64/include/armreg.h @@ -592,6 +592,27 @@ #define ISS_MSR_REG(reg) \ __ISS_MSR_REG(reg##_op0, reg##_op1, reg##_CRn, reg##_CRm, reg##_op2) +#define ISS_MOE_MEMINST_SHIFT 24 +#define ISS_MOE_MEMINST (0x01 << ISS_MOE_MEMINST_SHIFT) +#define ISS_MOE_isSETG_SHIFT 24 +#define ISS_MOE_isSETG (0x01 << ISS_MOE_isSETG_SHIFT) +#define ISS_MOE_OPTIONS_SHIFT 19 +#define ISS_MOE_OPTIONS_MASK (0x0f << ISS_MOE_OPTIONS_SHIFT) +#define ISS_MOE_FROM_EPILOGUE_SHIFT 18 +#define ISS_MOE_FROM_EPILOGUE (0x01 << ISS_MOE_FROM_EPILOGUE_SHIFT) +#define ISS_MOE_FORMAT_OPTION_SHIFT 16 +#define ISS_MOE_FORMAT_OPTION_MASK (0x03 << ISS_MOE_FORMAT_OPTION_SHIFT) +#define ISS_MOE_FORMAT_OPTION_B (0x00 << ISS_MOE_FORMAT_OPTION_SHIFT) +#define ISS_MOE_FORMAT_OPTION_A (0x01 << ISS_MOE_FORMAT_OPTION_SHIFT) +#define ISS_MOE_FORMAT_OPTION_A2 (0x02 << ISS_MOE_FORMAT_OPTION_SHIFT) +#define ISS_MOE_FORMAT_OPTION_B2 (0x03 << ISS_MOE_FORMAT_OPTION_SHIFT) +#define ISS_MOE_DESTREG_SHIFT 10 +#define ISS_MOE_DESTREG_MASK (0x1f << ISS_MOE_DESTREG_SHIFT) +#define ISS_MOE_SRCREG_SHIFT 5 +#define ISS_MOE_SRCREG_MASK (0x1f << ISS_MOE_SRCREG_SHIFT) +#define ISS_MOE_SIZEREG_SHIFT 0 +#define ISS_MOE_SIZEREG_MASK (0x1f << ISS_MOE_SIZEREG_SHIFT) + #define ISS_DATA_ISV_SHIFT 24 #define ISS_DATA_ISV (0x01 << ISS_DATA_ISV_SHIFT) #define ISS_DATA_SAS_SHIFT 22 @@ -656,6 +677,7 @@ #define EXCP_DATA_ABORT_L 0x24 /* Data abort, from lower EL */ #define EXCP_DATA_ABORT 0x25 /* Data abort, from same EL */ #define EXCP_SP_ALIGN 0x26 /* SP slignment fault */ +#define EXCP_MOE 0x27 /* Memory Operation Exception */ #define EXCP_TRAP_FP 0x2c /* Trapped FP exception */ #define EXCP_SERROR 0x2f /* SError interrupt */ #define EXCP_BRKPT_EL0 0x30 /* Hardware breakpoint, from same EL */ @@ -2627,7 +2649,9 @@ #define SCTLR_LSMAOE (UL(0x1) << 29) #define SCTLR_EnIB (UL(0x1) << 30) #define SCTLR_EnIA (UL(0x1) << 31) -/* Bits 34:32 are reserved */ +/* Bit 32 is reserved */ +#define SCTLR_MSCEn (UL(0x1) << 33) +/* Bit 34 is reserved */ #define SCTLR_BT0 (UL(0x1) << 35) #define SCTLR_BT1 (UL(0x1) << 36) #define SCTLR_ITFSB (UL(0x1) << 37)