Page MenuHomeFreeBSD
Files F153379171

D17536.diff
No OneTemporary
Actions

D17536.diff
View Options

Index: head/sys/amd64/amd64/initcpu.c
===================================================================
--- head/sys/amd64/amd64/initcpu.c
+++ head/sys/amd64/amd64/initcpu.c
@@ -253,6 +253,7 @@
}
hw_ibrs_recalculate();
hw_ssb_recalculate(false);
+ amd64_syscall_ret_flush_l1d_recalc();
switch (cpu_vendor_id) {
case CPU_VENDOR_AMD:
init_amd();
Index: head/sys/amd64/amd64/machdep.c
===================================================================
--- head/sys/amd64/amd64/machdep.c
+++ head/sys/amd64/amd64/machdep.c
@@ -1722,6 +1722,11 @@
!= NULL)
vty_set_preferred(VTY_VT);
+ TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
+ TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
+ TUNABLE_INT_FETCH("machdep.syscall_ret_l1d_flush",
+ &syscall_ret_l1d_flush_mode);
+
finishidentcpu(); /* Final stage of CPU initialization */
initializecpu(); /* Initialize CPU registers */
@@ -1864,9 +1869,6 @@
x86_init_fdt();
#endif
thread0.td_critnest = 0;
-
- TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
- TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
TSEXIT();
Index: head/sys/amd64/amd64/support.S
===================================================================
--- head/sys/amd64/amd64/support.S
+++ head/sys/amd64/amd64/support.S
@@ -1556,3 +1556,10 @@
ret
#undef L1D_FLUSH_SIZE
END(flush_l1d_sw)
+
+ENTRY(flush_l1d_sw_abi)
+ pushq %rbx
+ call flush_l1d_sw
+ popq %rbx
+ ret
+END(flush_l1d_sw_abi)
Index: head/sys/amd64/amd64/trap.c
===================================================================
--- head/sys/amd64/amd64/trap.c
+++ head/sys/amd64/amd64/trap.c
@@ -1056,6 +1056,84 @@
#include "../../kern/subr_syscall.c"
+static void (*syscall_ret_l1d_flush)(void);
+int syscall_ret_l1d_flush_mode;
+
+static void
+flush_l1d_hw(void)
+{
+
+ wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
+}
+
+static void __inline
+amd64_syscall_ret_flush_l1d_inline(int error)
+{
+ void (*p)(void);
+
+ if (error != 0 && error != EEXIST && error != EAGAIN &&
+ error != EXDEV && error != ENOENT && error != ENOTCONN &&
+ error != EINPROGRESS) {
+ p = syscall_ret_l1d_flush;
+ if (p != NULL)
+ p();
+ }
+}
+
+void
+amd64_syscall_ret_flush_l1d(int error)
+{
+
+ amd64_syscall_ret_flush_l1d_inline(error);
+}
+
+void
+amd64_syscall_ret_flush_l1d_recalc(void)
+{
+ bool l1d_hw;
+
+ l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
+again:
+ switch (syscall_ret_l1d_flush_mode) {
+ case 0:
+ syscall_ret_l1d_flush = NULL;
+ break;
+ case 1:
+ syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
+ flush_l1d_sw_abi;
+ break;
+ case 2:
+ syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
+ break;
+ case 3:
+ syscall_ret_l1d_flush = flush_l1d_sw_abi;
+ break;
+ default:
+ syscall_ret_l1d_flush_mode = 1;
+ goto again;
+ }
+}
+
+static int
+machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
+{
+ int error, val;
+
+ val = syscall_ret_l1d_flush_mode;
+ error = sysctl_handle_int(oidp, &val, 0, req);
+ if (error != 0 || req->newptr == NULL)
+ return (error);
+ syscall_ret_l1d_flush_mode = val;
+ amd64_syscall_ret_flush_l1d_recalc();
+ return (0);
+}
+SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
+ CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
+ machdep_syscall_ret_flush_l1d, "I",
+ "Flush L1D on syscall return with error (0 - off, 1 - on, "
+ "2 - use hw only, 3 - use sw only");
+
+
/*
* System call handler for native binaries. The trap frame is already
* set up by the assembler trampoline and a pointer to it is saved in
@@ -1110,4 +1188,6 @@
*/
if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+
+ amd64_syscall_ret_flush_l1d_inline(error);
}
Index: head/sys/amd64/ia32/ia32_syscall.c
===================================================================
--- head/sys/amd64/ia32/ia32_syscall.c
+++ head/sys/amd64/ia32/ia32_syscall.c
@@ -231,6 +231,7 @@
}
syscallret(td, error);
+ amd64_syscall_ret_flush_l1d(error);
}
static void
Index: head/sys/amd64/include/md_var.h
===================================================================
--- head/sys/amd64/include/md_var.h
+++ head/sys/amd64/include/md_var.h
@@ -41,6 +41,7 @@
extern int hw_ibrs_disable;
extern int hw_ssb_disable;
extern int nmi_flush_l1d_sw;
+extern int syscall_ret_l1d_flush_mode;
/*
* The file "conf/ldscript.amd64" defines the symbol "kernphys". Its
@@ -55,8 +56,11 @@
void amd64_db_resume_dbreg(void);
void amd64_lower_shared_page(struct sysentvec *);
void amd64_syscall(struct thread *td, int traced);
+void amd64_syscall_ret_flush_l1d(int error);
+void amd64_syscall_ret_flush_l1d_recalc(void);
void doreti_iret(void) __asm(__STRING(doreti_iret));
void doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault));
+void flush_l1d_sw_abi(void);
void ld_ds(void) __asm(__STRING(ld_ds));
void ld_es(void) __asm(__STRING(ld_es));
void ld_fs(void) __asm(__STRING(ld_fs));
Index: head/sys/dev/cpuctl/cpuctl.c
===================================================================
--- head/sys/dev/cpuctl/cpuctl.c
+++ head/sys/dev/cpuctl/cpuctl.c
@@ -521,6 +521,9 @@
hw_ibrs_recalculate();
restore_cpu(oldcpu, is_bound, td);
hw_ssb_recalculate(true);
+#ifdef __amd64__
+ amd64_syscall_ret_flush_l1d_recalc();
+#endif
printcpuinfo();
return (0);
}

File Metadata

Mime Type
text/plain
Expires
Tue, Apr 21, 7:45 PM (4 h, 23 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
31929836
Default Alt Text
D17536.diff (5 KB)

Event Timeline