Page MenuHomeFreeBSD
Files F172883392

D59815.diff
No OneTemporary
Actions

D59815.diff
View Options

diff --git a/sys/amd64/amd64/exec_machdep.c b/sys/amd64/amd64/exec_machdep.c
--- a/sys/amd64/amd64/exec_machdep.c
+++ b/sys/amd64/amd64/exec_machdep.c
@@ -406,6 +406,15 @@
clear_pcb_flags(pcb, PCB_DBREGS);
}
+void
+exec_splitlock(struct thread *td)
+{
+ if (ia32_splitlock_force)
+ enable_splitlock(td);
+ else if (ia32_splitlock)
+ disable_splitlock(td);
+}
+
/*
* Reset registers to default values on exec.
*/
@@ -443,6 +452,7 @@
regs->tf_flags = TF_HASSEGS;
x86_clear_dbregs(pcb);
+ exec_splitlock(td);
/*
* Drop the FP state if we hold it, so that the process gets a
diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c
--- a/sys/amd64/amd64/initcpu.c
+++ b/sys/amd64/amd64/initcpu.c
@@ -440,3 +440,32 @@
printf("FRED enabled\n");
}
+
+void
+amd64_init_splitlock(void)
+{
+ uint64_t msr;
+
+ if ((cpu_stdext_feature3 & CPUID_STDEXT3_CORE_CAP) != 0) {
+ if (rdmsr_safe(MSR_IA32_CORE_CAP, &msr) == 0)
+ ia32_splitlock = (msr & IA32_CORE_CAP_SPLITLOCK) != 0;
+ }
+ if (ia32_splitlock) {
+ TUNABLE_INT_FETCH("hw.splitlock_force", &ia32_splitlock_force);
+ if (ia32_splitlock_force != 0)
+ ia32_splitlock_force = 1;
+ } else {
+ ia32_splitlock_force = 0;
+ }
+}
+
+void
+amd64_cpu_init_msr_memctl(void)
+{
+ uint64_t msr;
+
+ if ((cpu_stdext_feature3 & CPUID_STDEXT3_CORE_CAP) != 0 &&
+ rdmsr_safe(MSR_IA32_CORE_CAP, &msr) == 0 &&
+ (msr & IA32_CORE_CAP_SPLITLOCK) != 0)
+ PCPU_SET(msr_memctl, rdmsr(MSR_MEMORY_CTL));
+}
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -206,6 +206,15 @@
int late_console = 1;
int lass_enabled = 0;
+int ia32_splitlock = 0;
+SYSCTL_INT(_hw, OID_AUTO, splitlock, CTLFLAG_RD,
+ &ia32_splitlock, 0,
+ "splitlock prevention supported");
+int ia32_splitlock_force = 1;
+SYSCTL_INT(_hw, OID_AUTO, splitlock_force, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+ &ia32_splitlock_force, 0,
+ "splitlock prevention enabled by default");
+
int __read_frequently fred = 0;
SYSCTL_INT(_hw, OID_AUTO, fred, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
&fred, 0,
@@ -1653,6 +1662,9 @@
/* setup proc 0's pcb */
thread0.td_pcb->pcb_flags = 0;
+ amd64_init_splitlock();
+ amd64_cpu_init_msr_memctl();
+
env = kern_getenv("kernelname");
if (env != NULL)
strlcpy(kernelname, env, sizeof(kernelname));
@@ -1923,6 +1935,70 @@
return (uiomove_mem(UIO_MEM_KMEM, &uio));
}
+static void
+enable_splitlock_ac_wrmsr(void)
+{
+ MPASS(ia32_splitlock);
+ wrmsr(MSR_MEMORY_CTL, PCPU_GET(msr_memctl) | MSR_MEMORY_CTL_SPLITLOCK);
+}
+
+static void
+enable_splitlock_ac_wrmsrimm(void)
+{
+ MPASS(ia32_splitlock);
+ wrmsr_imm(MSR_MEMORY_CTL, PCPU_GET(msr_memctl) |
+ MSR_MEMORY_CTL_SPLITLOCK);
+}
+
+DEFINE_IFUNC(, void, enable_splitlock_ac, (void))
+{
+ if ((cpu_stdext_feature5 & CPUID_STDEXT5_MSR_IMM) != 0)
+ return (enable_splitlock_ac_wrmsrimm);
+ return (enable_splitlock_ac_wrmsr);
+}
+
+void
+enable_splitlock(struct thread *td)
+{
+ MPASS(td == curthread);
+ td->td_md.md_td_flags |= TDF_MD_SPLITLOCK_AC;
+ critical_enter();
+ enable_splitlock_ac();
+ critical_exit();
+}
+
+static void
+disable_splitlock_ac_wrmsr(void)
+{
+ MPASS(ia32_splitlock);
+ wrmsr(MSR_MEMORY_CTL, PCPU_GET(msr_memctl) & ~MSR_MEMORY_CTL_SPLITLOCK);
+}
+
+static void
+disable_splitlock_ac_wrmsrimm(void)
+{
+ MPASS(ia32_splitlock);
+ wrmsr_imm(MSR_MEMORY_CTL, PCPU_GET(msr_memctl) &
+ ~MSR_MEMORY_CTL_SPLITLOCK);
+}
+
+DEFINE_IFUNC(, void, disable_splitlock_ac, (void))
+{
+ if ((cpu_stdext_feature5 & CPUID_STDEXT5_MSR_IMM) != 0)
+ return (disable_splitlock_ac_wrmsrimm);
+ return (disable_splitlock_ac_wrmsr);
+}
+
+void
+disable_splitlock(struct thread *td)
+{
+ MPASS(td == curthread);
+ td->td_md.md_td_flags &= ~TDF_MD_SPLITLOCK_AC;
+ critical_enter();
+ disable_splitlock_ac();
+ critical_exit();
+}
+
#ifdef KDB
/*
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -309,6 +309,8 @@
else
lidt(&r_idt);
+ amd64_cpu_init_msr_memctl();
+
gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
ltr(gsel_tss);
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -10236,9 +10236,12 @@
void
pmap_activate_sw(struct thread *td)
{
+ struct thread *oldtd;
pmap_t oldpmap, pmap;
u_int cpuid;
+ bool oldtd_sl, td_sl;
+ oldtd = curthread;
oldpmap = PCPU_GET(curpmap);
pmap = vmspace_pmap(td->td_proc->p_vmspace);
if (oldpmap == pmap) {
@@ -10250,6 +10253,16 @@
CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
pmap_activate_sw_mode(td, pmap, cpuid);
CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
+ if (ia32_splitlock) {
+ oldtd_sl = (atomic_load_int(&oldtd->td_md.md_td_flags) &
+ TDF_MD_SPLITLOCK_AC) != 0;
+ td_sl = (atomic_load_int(&td->td_md.md_td_flags) &
+ TDF_MD_SPLITLOCK_AC) != 0;
+ if (oldtd_sl && !td_sl)
+ disable_splitlock_ac();
+ else if (!oldtd_sl && td_sl)
+ enable_splitlock_ac();
+ }
}
void
diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c
--- a/sys/amd64/amd64/sys_machdep.c
+++ b/sys/amd64/amd64/sys_machdep.c
@@ -212,7 +212,7 @@
struct i386_set_pkru i386pkru;
struct amd64_get_xfpustate a64xfpu;
struct amd64_set_pkru a64pkru;
- int error;
+ int error, val;
#ifdef CAPABILITY_MODE
/*
@@ -231,6 +231,8 @@
case I386_GET_XFPUSTATE:
case I386_SET_PKRU:
case I386_CLEAR_PKRU:
+ case I386_SET_SPLITLOCK:
+ case I386_GET_SPLITLOCK:
case AMD64_GET_FSBASE:
case AMD64_SET_FSBASE:
case AMD64_GET_GSBASE:
@@ -299,6 +301,10 @@
sizeof(struct amd64_set_pkru))) != 0)
return (error);
break;
+ case I386_SET_SPLITLOCK:
+ if ((error = copyin(uap->parms, &val, sizeof(val))) != 0)
+ return (error);
+ break;
default:
break;
}
@@ -421,6 +427,27 @@
update_pcb_bases(pcb);
break;
+ case I386_SET_SPLITLOCK:
+ if (!ia32_splitlock) {
+ if (val != 0)
+ error = ENOTSUP;
+ break;
+ }
+ PROC_LOCK(td->td_proc);
+ if (val == 0)
+ disable_splitlock(td);
+ else if (val == 1)
+ enable_splitlock(td);
+ else
+ error = EINVAL;
+ PROC_UNLOCK(td->td_proc);
+ break;
+
+ case I386_GET_SPLITLOCK:
+ val = (td->td_md.md_td_flags & TDF_MD_SPLITLOCK_AC) != 0;
+ error = copyout(&val, uap->parms, sizeof(val));
+ break;
+
default:
error = EINVAL;
break;
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -532,6 +532,14 @@
return;
switch (type) {
+ case T_ALIGNFLT:
+ if (curpcb->pcb_onfault != NULL) {
+ frame->tf_rip = (long)curpcb->pcb_onfault;
+ return;
+ }
+ trap_fatal(frame, 0);
+ return;
+
case T_PAGEFLT: /* page fault */
(void)trap_pfault(frame, false, NULL, NULL);
return;
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -182,6 +182,8 @@
/* Setup to release spin count in fork_exit(). */
td2->td_md.md_spinlock_count = 1;
td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
+ td2->td_md.md_td_flags = td1->td_md.md_td_flags &
+ TDF_MD_SPLITLOCK_AC;
pmap_thread_init_invl_gen(td2);
/*
diff --git a/sys/amd64/ia32/ia32_signal.c b/sys/amd64/ia32/ia32_signal.c
--- a/sys/amd64/ia32/ia32_signal.c
+++ b/sys/amd64/ia32/ia32_signal.c
@@ -981,6 +981,7 @@
regs->tf_flags = TF_HASSEGS;
x86_clear_dbregs(pcb);
+ exec_splitlock(td);
fpstate_drop(td);
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -380,6 +380,19 @@
__asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr));
}
+static __inline void
+wrmsrns(u_int msr, uint64_t newval)
+{
+ uint32_t low, high;
+
+ low = newval;
+ high = newval >> 32;
+ __asm __volatile("wrmsrns" : : "a" (low), "d" (high), "c" (msr));
+}
+
+#define wrmsr_imm(msr, val) \
+ __asm __volatile("wrmsrns %0, %1" : : "r" (val), "i" (msr))
+
static __inline void
load_cr0(u_long data)
{
diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h
--- a/sys/amd64/include/md_var.h
+++ b/sys/amd64/include/md_var.h
@@ -46,6 +46,8 @@
extern int nmi_flush_l1d_sw;
extern int syscall_ret_l1d_flush_mode;
extern int lass_enabled;
+extern int ia32_splitlock;
+extern int ia32_splitlock_force;
extern vm_paddr_t intel_graphics_stolen_base;
extern vm_paddr_t intel_graphics_stolen_size;
@@ -66,7 +68,9 @@
void amd64_conf_fast_syscall(void);
void amd64_cpu_init_fred(void);
+void amd64_cpu_init_msr_memctl(void);
void amd64_db_resume_dbreg(void);
+void amd64_init_splitlock(void);
vm_paddr_t amd64_loadaddr(void);
void amd64_lower_shared_page(struct sysentvec *);
void amd64_bsp_pcpu_init1(struct pcpu *pc);
@@ -102,6 +106,11 @@
char **xfpusave, size_t *xfpusave_len);
int set_fpcontext(struct thread *td, struct __mcontext *mcp,
char *xfpustate, size_t xfpustate_len);
+void enable_splitlock_ac(void);
+void enable_splitlock(struct thread *td);
+void disable_splitlock_ac(void);
+void disable_splitlock(struct thread *td);
+void exec_splitlock(struct thread *td);
void wrmsr_early_safe_start(void);
void wrmsr_early_safe_end(void);
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -101,7 +101,8 @@
u_int pc_small_core; \
u_int pc_pcid_invlpg_workaround; \
struct pmap_pcid pc_kpmap_store; \
- char __pad[2900] /* pad to UMA_PCPU_ALLOC_SIZE */
+ uint64_t pc_msr_memctl; \
+ char __pad[2892] /* pad to UMA_PCPU_ALLOC_SIZE */
#define PC_DBREG_CMD_NONE 0
#define PC_DBREG_CMD_LOAD 1
diff --git a/sys/amd64/include/proc.h b/sys/amd64/include/proc.h
--- a/sys/amd64/include/proc.h
+++ b/sys/amd64/include/proc.h
@@ -69,6 +69,7 @@
*/
struct mdthread {
int md_spinlock_count; /* (k) */
+ int md_td_flags; /* (c) */
register_t md_saved_flags; /* (k) */
register_t md_spurflt_addr; /* (k) Spurious page fault address. */
struct pmap_invl_gen md_invl_gen;
@@ -79,6 +80,8 @@
void *md_usr_fpu_save;
};
+#define TDF_MD_SPLITLOCK_AC 0x00000001 /* Disable split cache lines */
+
struct mdproc {
struct proc_ldt *md_ldt; /* (t) per-process ldt */
struct system_segment_descriptor md_ldt_sd;
diff --git a/sys/amd64/linux/linux_sysvec.c b/sys/amd64/linux/linux_sysvec.c
--- a/sys/amd64/linux/linux_sysvec.c
+++ b/sys/amd64/linux/linux_sysvec.c
@@ -266,6 +266,7 @@
regs->tf_flags = TF_HASSEGS;
x86_clear_dbregs(pcb);
+ exec_splitlock(td);
/*
* Drop the FP state if we hold it, so that the process gets a
diff --git a/sys/amd64/linux32/linux32_sysvec.c b/sys/amd64/linux32/linux32_sysvec.c
--- a/sys/amd64/linux32/linux32_sysvec.c
+++ b/sys/amd64/linux32/linux32_sysvec.c
@@ -608,6 +608,7 @@
regs->tf_rbx = (register_t)imgp->ps_strings;
x86_clear_dbregs(pcb);
+ exec_splitlock(td);
fpstate_drop(td);
diff --git a/sys/x86/include/specialreg.h b/sys/x86/include/specialreg.h
--- a/sys/x86/include/specialreg.h
+++ b/sys/x86/include/specialreg.h
@@ -559,6 +559,11 @@
#define CPUID_STDEXT4_NMISRC 0x00100000
#define CPUID_STDEXT4_LAM 0x04000000
+/*
+ * CPUID instruction 7 Structured Extended Features, leaf 1 ecx info
+ */
+#define CPUID_STDEXT5_MSR_IMM 0x00000020
+
/* CPUID_HYBRID_ID leaf 0x1a */
#define CPUID_HYBRID_CORE_MASK 0xff000000
#define CPUID_HYBRID_SMALL_CORE 0x20000000
@@ -575,10 +580,16 @@
#define IA32_ARCH_CAP_TSX_CTRL 0x00000080
#define IA32_ARCH_CAP_TAA_NO 0x00000100
+/* MSR IA32_CORE_CAP bits (platform) */
+#define IA32_CORE_CAP_SPLITLOCK 0x00000020
+
/* MSR IA32_TSX_CTRL bits */
#define IA32_TSX_CTRL_RTM_DISABLE 0x00000001
#define IA32_TSX_CTRL_TSX_CPUID_CLEAR 0x00000002
+/* MSR MEMORY_CTL platform bits */
+#define MSR_MEMORY_CTL_SPLITLOCK 0x20000000
+
/*
* CPUID manufacturers identifiers
*/
@@ -607,6 +618,7 @@
#define MSR_APICBASE 0x01b
#define MSR_EBL_CR_POWERON 0x02a
#define MSR_TEST_CTL 0x033
+#define MSR_MEMORY_CTL 0x033 /* Platform, same index as TEST */
#define MSR_IA32_FEATURE_CONTROL 0x03a
#define MSR_IA32_SPEC_CTRL 0x048
#define MSR_IA32_PRED_CMD 0x049
@@ -618,6 +630,7 @@
#define MSR_PERFCTR0 0x0c1
#define MSR_PERFCTR1 0x0c2
#define MSR_PLATFORM_INFO 0x0ce
+#define MSR_IA32_CORE_CAP 0x0cf
#define MSR_MPERF 0x0e7
#define MSR_APERF 0x0e8
#define MSR_IA32_EXT_CONFIG 0x0ee /* Undocumented. Core Solo/Duo only */
diff --git a/sys/x86/include/sysarch.h b/sys/x86/include/sysarch.h
--- a/sys/x86/include/sysarch.h
+++ b/sys/x86/include/sysarch.h
@@ -52,6 +52,8 @@
#define I386_GET_XFPUSTATE 11
#define I386_SET_PKRU 12
#define I386_CLEAR_PKRU 13
+#define I386_SET_SPLITLOCK 14
+#define I386_GET_SPLITLOCK 15
/* Leave space for 0-127 for to avoid translating syscalls */
#define AMD64_GET_FSBASE 128
diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h
--- a/sys/x86/include/x86_var.h
+++ b/sys/x86/include/x86_var.h
@@ -51,6 +51,7 @@
extern u_int cpu_stdext_feature2;
extern u_int cpu_stdext_feature3;
extern u_int cpu_stdext_feature4;
+extern u_int cpu_stdext_feature5;
extern uint64_t cpu_ia32_arch_caps;
extern u_int cpu_high;
extern u_int cpu_id;
diff --git a/sys/x86/x86/identcpu.c b/sys/x86/x86/identcpu.c
--- a/sys/x86/x86/identcpu.c
+++ b/sys/x86/x86/identcpu.c
@@ -118,6 +118,7 @@
u_int cpu_stdext_feature3; /* %edx */
/* leaf 7 %ecx = 1 */
u_int cpu_stdext_feature4; /* %eax */
+u_int cpu_stdext_feature5; /* %ecx */
uint64_t cpu_ia32_arch_caps;
u_int cpu_max_ext_state_size;
u_int cpu_mon_mwait_flags; /* MONITOR/MWAIT flags (CPUID.05H.ECX) */
@@ -1064,6 +1065,14 @@
);
}
+ if (cpu_stdext_feature5 != 0) {
+ printf("\n Structured Extended Features5=0x%b",
+ cpu_stdext_feature5,
+ "\020"
+ "\006MSR_IMM"
+ );
+ }
+
if ((cpu_feature2 & CPUID2_XSAVE) != 0) {
cpuid_count(0xd, 0x1, regs);
if (regs[0] != 0) {
@@ -1616,6 +1625,7 @@
if (max_eax_l7 >= 1) {
cpuid_count(7, 1, regs);
cpu_stdext_feature4 = regs[0];
+ cpu_stdext_feature5 = regs[2];
}
}
}

File Metadata

Mime Type
text/plain
Expires
Tue, Sep 22, 8:09 PM (5 h, 14 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39433572
Default Alt Text
D59815.diff (13 KB)

Event Timeline