Index: sys/amd64/amd64/exception.S =================================================================== --- sys/amd64/amd64/exception.S +++ sys/amd64/amd64/exception.S @@ -198,6 +198,7 @@ movq %r15,TF_R15(%rsp) movl $TF_HASSEGS,TF_FLAGS(%rsp) cld + call handle_ibrs FAKE_MCOUNT(TF_RIP(%rsp)) #ifdef KDTRACE_HOOKS /* @@ -440,7 +441,9 @@ movq %r11,TF_RSP(%rsp) /* user stack pointer */ movq PCPU(SCRATCH_RAX),%rax movq %rax,TF_RAX(%rsp) /* syscall number */ + movq %rdx,TF_RDX(%rsp) /* arg 3 */ SAVE_SEGS + call handle_ibrs movq PCPU(CURPCB),%r11 andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) sti @@ -449,7 +452,6 @@ movq $2,TF_ERR(%rsp) movq %rdi,TF_RDI(%rsp) /* arg 1 */ movq %rsi,TF_RSI(%rsp) /* arg 2 */ - movq %rdx,TF_RDX(%rsp) /* arg 3 */ movq %r10,TF_RCX(%rsp) /* arg 4 */ movq %r8,TF_R8(%rsp) /* arg 5 */ movq %r9,TF_R9(%rsp) /* arg 6 */ Index: sys/amd64/amd64/initcpu.c =================================================================== --- sys/amd64/amd64/initcpu.c +++ sys/amd64/amd64/initcpu.c @@ -62,6 +62,11 @@ */ static int hw_clflush_disable = -1; +int hw_ibrs_disable; +SYSCTL_INT(_hw, OID_AUTO, ibrs_disable, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, + &hw_ibrs_disable, 0, + "IBRS"); + static void init_amd(void) { @@ -223,6 +228,8 @@ wrmsr(MSR_EFER, msr); pg_nx = PG_NX; } + if ((cpu_ia32_arch_caps & IA32_ARCH_CAP_IBRS_ALL) != 0) + wrmsr(MSR_IA32_SPEC_CTRL, IA32_SPEC_CTRL_IBRS); switch (cpu_vendor_id) { case CPU_VENDOR_AMD: init_amd(); Index: sys/amd64/amd64/machdep.c =================================================================== --- sys/amd64/amd64/machdep.c +++ sys/amd64/amd64/machdep.c @@ -1826,6 +1826,8 @@ #endif thread0.td_critnest = 0; + TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable); + TSEXIT(); /* Location of kernel stack for locore */ Index: sys/amd64/amd64/support.S =================================================================== --- sys/amd64/amd64/support.S +++ sys/amd64/amd64/support.S @@ -33,6 +33,7 @@ #include "opt_ddb.h" #include +#include #include #include "assym.s" @@ -802,3 +803,42 @@ movl $EFAULT,%eax POP_FRAME_POINTER ret + + .altmacro + .macro ibrs_seq_label l +handle_ibrs_\l: + .endm + .macro ibrs_call_label l + call handle_ibrs_\l + .endm + .macro ibrs_seq count + ll=1 + .rept \count + ibrs_call_label %(ll) + nop + ibrs_seq_label %(ll) + addq $8,%rsp + ll=ll+1 + .endr + .endm + +/* all callers already saved %rax, %rdx, and %rcx */ +ENTRY(handle_ibrs) + testl $IA32_ARCH_CAP_IBRS_ALL,cpu_ia32_arch_caps(%rip) + jne 2f + cmpb $0,hw_ibrs_disable(%rip) + jne 2f + testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) + je 1f + movl $MSR_IA32_SPEC_CTRL,%ecx + movl $IA32_SPEC_CTRL_IBRS,%eax + movl $IA32_SPEC_CTRL_IBRS>>32,%edx + wrmsr +1: testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip) + jne 2f + ibrs_seq 32 +2: ret +END(handle_ibrs) + + .noaltmacro + Index: sys/amd64/include/asmacros.h =================================================================== --- sys/amd64/include/asmacros.h +++ sys/amd64/include/asmacros.h @@ -257,6 +257,7 @@ jz 1f /* yes, leave PCB_FULL_IRET alone */ movq PCPU(CURPCB),%r8 andl $~PCB_FULL_IRET,PCB_FLAGS(%r8) + call handle_ibrs 1: .endm Index: sys/amd64/include/md_var.h =================================================================== --- sys/amd64/include/md_var.h +++ sys/amd64/include/md_var.h @@ -38,6 +38,7 @@ extern uint64_t *vm_page_dump; extern int hw_lower_amd64_sharedpage; +extern int hw_ibrs_disable; /* * The file "conf/ldscript.amd64" defines the symbol "kernphys". Its Index: sys/x86/x86/identcpu.c =================================================================== --- sys/x86/x86/identcpu.c +++ sys/x86/x86/identcpu.c @@ -1614,7 +1614,8 @@ if (strcmp(cpu_vendor, AMD_VENDOR_ID) == 0) return (0); - + if ((cpu_ia32_arch_caps & IA32_ARCH_CAP_RDCL_NO) != 0) + return (0); return (1); }