Page MenuHomeFreeBSD
Files F151273727

D55829.id174054.diff
No OneTemporary
Actions

D55829.id174054.diff
View Options

diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -187,13 +187,23 @@
cmpq %rax,%rdx
jne do_tss
done_tss:
+ movq %r8,PCPU(CURPCB)
+ cmpb $0,fred
+ jne 1f
movq TD_MD_STACK_BASE(%r12),%r9
movq %r9,PCPU(RSP0)
- movq %r8,PCPU(CURPCB)
movq PCPU(PTI_RSP0),%rax
cmpq $~0,PCPU(UCR3)
cmove %r9,%rax
movq %rax,TSS_RSP0(%rdx)
+ jmp 2f
+1:
+ movq TD_MD_STACK_BASE(%r12),%rdx
+ movl %edx,%eax
+ shrq $32,%rdx
+ movl $MSR_FRED_RSP0,%ecx
+ wrmsr
+2:
movq %r12,PCPU(CURTHREAD) /* into next thread */
/* Test if debug registers should be restored. */
@@ -365,8 +375,37 @@
movl %eax,PCB_SFMASK(%rdi)
movl %edx,PCB_SFMASK+4(%rdi)
+ cmpb $0, fred
+ je 1f
+ movl $MSR_FRED_RSP0,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_RSP0(%rdi)
+ movl %edx,PCB_FRED_RSP0+4(%rdi)
+ movl $MSR_FRED_RSP1,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_RSP1(%rdi)
+ movl %edx,PCB_FRED_RSP1+4(%rdi)
+ movl $MSR_FRED_RSP2,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_RSP2(%rdi)
+ movl %edx,PCB_FRED_RSP2+4(%rdi)
+ movl $MSR_FRED_RSP3,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_RSP3(%rdi)
+ movl %edx,PCB_FRED_RSP3+4(%rdi)
+ movl $MSR_FRED_STKLVLS,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_STKLVLS(%rdi)
+ movl %edx,PCB_FRED_STKLVLS+4(%rdi)
+ movl $MSR_FRED_CONFIG,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_CONFIG(%rdi)
+ movl %edx,PCB_FRED_CONFIG+4(%rdi)
+ jmp 2f
+
+1: sidt PCB_IDT(%rdi)
+2:
sgdt PCB_GDT(%rdi)
- sidt PCB_IDT(%rdi)
sldt PCB_LDT(%rdi)
str PCB_TR(%rdi)
@@ -442,8 +481,35 @@
movq %rax,%cr3
/* Restore descriptor tables. */
+ cmpb $0,fred
+ jne 1f
lidt PCB_IDT(%rdi)
- lldt PCB_LDT(%rdi)
+ jmp 2f
+1: movl $MSR_FRED_RSP0,%ecx
+ movl PCB_FRED_RSP0(%rdi),%eax
+ movl 4 + PCB_FRED_RSP0(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_RSP1,%ecx
+ movl PCB_FRED_RSP1(%rdi),%eax
+ movl 4 + PCB_FRED_RSP1(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_RSP2,%ecx
+ movl PCB_FRED_RSP2(%rdi),%eax
+ movl 4 + PCB_FRED_RSP2(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_RSP3,%ecx
+ movl PCB_FRED_RSP3(%rdi),%eax
+ movl 4 + PCB_FRED_RSP3(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_STKLVLS,%ecx
+ movl PCB_FRED_STKLVLS(%rdi),%eax
+ movl 4 + PCB_FRED_STKLVLS(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_CONFIG,%ecx
+ movl PCB_FRED_CONFIG(%rdi),%eax
+ movl 4 + PCB_FRED_CONFIG(%rdi),%edx
+ wrmsr
+2: lldt PCB_LDT(%rdi)
#define SDT_SYSTSS 9
#define SDT_SYSBSY 11
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -1063,6 +1063,8 @@
movq %rbx,%rsi /* arg1 */
movq %rsp,%rdx /* trapframe pointer */
call fork_exit
+ cmpb $0, fred
+ jne fred_checkast
jmp doreti /* Handle any ASTs */
/*
@@ -1370,6 +1372,233 @@
movq $0,PCB_GSBASE(%r8)
jmp doreti
+/*
+ * FRED guarantees that on entry, %rflags is set to '2', which in
+ * particular means that interrupts are disabled, and DF/AC are clear.
+ * Interrupts are re-enabled by the C handler, if appropriate.
+ */
+ ALIGN_TEXT
+user_fred:
+ movq %rdi,TF_RDI(%rsp)
+ movq %rdx,TF_RDX(%rsp)
+ movq %rcx,TF_RCX(%rsp)
+ movq %rsi,TF_RSI(%rsp)
+ movq %r8,TF_R8(%rsp)
+ movq %r9,TF_R9(%rsp)
+ movq %rbx,TF_RBX(%rsp)
+ movq %rbp,TF_RBP(%rsp)
+ movq %r10,TF_R10(%rsp)
+ movq %r11,TF_R11(%rsp)
+ movq %r12,TF_R12(%rsp)
+ movq %r13,TF_R13(%rsp)
+ movq %r14,TF_R14(%rsp)
+ movq %r15,TF_R15(%rsp)
+ SAVE_SEGS
+ movq PCPU(CURTHREAD),%r9
+ movq TD_PCB(%r9),%r8
+ rdfsbase %rax
+ movq %rax,PCB_FSBASE(%r8)
+ rdgsbase %rax
+ movq %rax,PCB_GSBASE(%r8)
+ movl $TF_HASSEGS,TF_FLAGS(%rsp)
+ KMSAN_ENTER
+ movq %rsp, %rdi
+ call trap_fred_u
+ KMSAN_LEAVE
+fred_checkast:
+ cli
+ movq PCPU(CURTHREAD),%rax
+ movq TD_PCB(%rax),%r8
+ cmpl $0,TD_AST(%rax)
+ je fred_gouser
+ sti
+ movq %rsp,%rdi /* pass a pointer to the trapframe */
+ call ast
+ jmp fred_checkast
+fred_gouser:
+ testl $PCB_FULL_IRET,PCB_FLAGS(%r8)
+ jz fred_u_regs
+ andl $~PCB_FULL_IRET,PCB_FLAGS(%r8)
+ testl $TF_HASSEGS,TF_FLAGS(%rsp)
+ jne fred_do_segs
+ movw $KUDSEL,%ax
+ movw %ax,TF_DS(%rsp)
+ movw %ax,TF_ES(%rsp)
+ movw $KUF32SEL,TF_FS(%rsp)
+ movw $KUG32SEL,TF_GS(%rsp)
+fred_do_segs:
+ /* Restore %fs and fsbase */
+ movw TF_FS(%rsp),%ax
+ .globl fred_ld_fs
+fred_ld_fs:
+ movw %ax,%fs
+ movq PCB_FSBASE(%r8),%rax
+ .globl fred_ld_fsbase
+fred_ld_fsbase:
+ wrfsbase %rax
+ /* Restore %gs and kgsbase */
+ movw TF_GS(%rsp),%ax
+ .globl fred_lkgs
+fred_lkgs:
+ lkgs %ax
+ movl PCB_GSBASE(%r8),%eax
+ movl PCB_GSBASE+4(%r8),%edx
+ movl $MSR_KGSBASE,%ecx
+ .globl fred_ld_kgsbase
+fred_ld_kgsbase:
+ wrmsr
+ .globl fred_ld_es
+fred_ld_es:
+ movw TF_ES(%rsp),%es
+ .globl fred_ld_ds
+fred_ld_ds:
+ movw TF_DS(%rsp),%ds
+fred_u_regs:
+ RESTORE_REGS
+ addq $TF_ERR,%rsp
+ .globl fred_eretu
+fred_eretu:
+ eretu
+
+ .globl fred_ld_fs_fault
+fred_ld_fs_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movw $KUF32SEL,TF_FS(%rsp)
+ jmp fred_checkast
+
+ .globl fred_ld_fsbase_fault
+fred_ld_fsbase_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movq PCPU(CURTHREAD),%r9
+ movq TD_PCB(%r9),%r8
+ movq $0,PCB_FSBASE(%r8)
+ jmp fred_checkast
+
+ .globl fred_lkgs_fault
+fred_lkgs_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movw $KUG32SEL,TF_GS(%rsp)
+ jmp fred_checkast
+
+ .globl fred_ld_kgsbase_fault
+fred_ld_kgsbase_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movq PCPU(CURTHREAD),%r9
+ movq TD_PCB(%r9),%r8
+ movq $0,PCB_GSBASE(%r8)
+ jmp fred_checkast
+
+ .globl fred_ld_es_fault
+fred_ld_es_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movw $KUDSEL,TF_ES(%rsp)
+ jmp fred_checkast
+
+ .globl fred_ld_ds_fault
+fred_ld_ds_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movw $KUDSEL,TF_DS(%rsp)
+ jmp fred_checkast
+
+ .globl fred_eretu_fault
+fred_eretu_fault:
+ subq $TF_ERR,%rsp
+ movq %rdi,TF_RDI(%rsp)
+ movq %rdx,TF_RDX(%rsp)
+ movq %rcx,TF_RCX(%rsp)
+ movq %rsi,TF_RSI(%rsp)
+ movq %r8,TF_R8(%rsp)
+ movq %r9,TF_R9(%rsp)
+ movq %rbx,TF_RBX(%rsp)
+ movq %rbp,TF_RBP(%rsp)
+ movq %r10,TF_R10(%rsp)
+ movq %r11,TF_R11(%rsp)
+ movq %r12,TF_R12(%rsp)
+ movq %r13,TF_R13(%rsp)
+ movq %r14,TF_R14(%rsp)
+ movq %r15,TF_R15(%rsp)
+ SAVE_SEGS
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_GP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp, %rdi
+ KMSAN_ENTER
+ call trap_fred_u
+ KMSAN_LEAVE
+ jmp fred_checkast
+
+ ALIGN_TEXT
+kernel_fred:
+ movq %rdi,TF_RDI(%rsp)
+ movq %rdx,TF_RDX(%rsp)
+ movq %rcx,TF_RCX(%rsp)
+ movq %rsi,TF_RSI(%rsp)
+ movq %r8,TF_R8(%rsp)
+ movq %r9,TF_R9(%rsp)
+ movq %rbx,TF_RBX(%rsp)
+ movq %rbp,TF_RBP(%rsp)
+ movq %r10,TF_R10(%rsp)
+ movq %r11,TF_R11(%rsp)
+ movq %r12,TF_R12(%rsp)
+ movq %r13,TF_R13(%rsp)
+ movq %r14,TF_R14(%rsp)
+ movq %r15,TF_R15(%rsp)
+ movq %rsp, %rdi
+ KMSAN_ENTER
+ call trap_fred_k
+ KMSAN_LEAVE
+ RESTORE_REGS
+ addq $TF_ERR,%rsp
+ erets
+
+/*
+ * Content for the FRED entries page. The code is copied to the
+ * allocated page, handlers must be PIC.
+ */
+ .globl fred_events_handlers, fred_events_handlers_end
+fred_events_handlers:
+ subq $TF_ERR,%rsp
+ movq %rax,TF_RAX(%rsp)
+ movq $user_fred, %rax
+ jmp *%rax
+ . = fred_events_handlers + 256
+ subq $TF_ERR,%rsp
+ movq %rax,TF_RAX(%rsp)
+ movq $kernel_fred, %rax
+ jmp *%rax
+fred_events_handlers_end:
+
+ .globl fred_out_of_nmi
+fred_out_of_nmi:
+ pushq %rbp
+ movq %rsp,%rbp
+ xorl %eax,%eax
+ pushq $0
+ pushq $0
+ movw %ss,%ax
+ orl $(1<<18),%eax /* NMI block release for ERETS */
+ pushq %rax
+ pushq %rbp
+ pushfq
+ xorl %eax,%eax
+ movw %cs,%ax
+ pushq %rax
+ pushq $1f
+ pushq $0
+ erets
+1: popq %rbp
+ retq
+
#ifdef HWPMC_HOOKS
ENTRY(end_exceptions)
#endif
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -149,6 +149,12 @@
ASSYM(PCB_LSTAR, offsetof(struct pcb, pcb_lstar));
ASSYM(PCB_CSTAR, offsetof(struct pcb, pcb_cstar));
ASSYM(PCB_SFMASK, offsetof(struct pcb, pcb_sfmask));
+ASSYM(PCB_FRED_RSP0, offsetof(struct pcb, pcb_fred_rsp0));
+ASSYM(PCB_FRED_RSP1, offsetof(struct pcb, pcb_fred_rsp1));
+ASSYM(PCB_FRED_RSP2, offsetof(struct pcb, pcb_fred_rsp2));
+ASSYM(PCB_FRED_RSP3, offsetof(struct pcb, pcb_fred_rsp3));
+ASSYM(PCB_FRED_STKLVLS, offsetof(struct pcb, pcb_fred_stklvls));
+ASSYM(PCB_FRED_CONFIG, offsetof(struct pcb, pcb_fred_config));
ASSYM(PCB_SIZE, sizeof(struct pcb));
ASSYM(PCB_FULL_IRET, PCB_FULL_IRET);
ASSYM(PCB_DBREGS, PCB_DBREGS);
@@ -183,9 +189,11 @@
ASSYM(TF_ES, offsetof(struct trapframe, tf_es));
ASSYM(TF_FS, offsetof(struct trapframe, tf_fs));
ASSYM(TF_GS, offsetof(struct trapframe, tf_gs));
+ASSYM(TF_FRED_EVINFO2, offsetof(struct trapframe, tf_fred_evinfo2));
ASSYM(TF_FLAGS, offsetof(struct trapframe, tf_flags));
ASSYM(TF_SIZE, sizeof(struct trapframe));
ASSYM(TF_HASSEGS, TF_HASSEGS);
+ASSYM(TF_FRED_EVINFO2_TYPE_EXC, TF_FRED_EVINFO2_TYPE_EXC);
ASSYM(PTI_RDX, offsetof(struct pti_frame, pti_rdx));
ASSYM(PTI_RAX, offsetof(struct pti_frame, pti_rax));
@@ -275,6 +283,9 @@
ASSYM(LDTSEL, GSEL(GUSERLDT_SEL, SEL_KPL));
ASSYM(SEL_RPL_MASK, SEL_RPL_MASK);
+ASSYM(IDT_NP, IDT_NP);
+ASSYM(IDT_GP, IDT_GP);
+
ASSYM(__FreeBSD_version, __FreeBSD_version);
#ifdef HWPMC_HOOKS
diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c
--- a/sys/amd64/amd64/initcpu.c
+++ b/sys/amd64/amd64/initcpu.c
@@ -397,3 +397,40 @@
cpu_stdext_feature &= ~CPUID_STDEXT_CLFLUSHOPT;
}
}
+
+void
+amd64_cpu_init_fred(void)
+{
+ /*
+ * 128b = 0x2 * 64b redzone
+ * SLC 0 for EXTINT
+ * SLC 0 current
+ */
+ wrmsr(MSR_FRED_CONFIG, (uintptr_t)fred_page |
+ (0x2 << IA32_FRED_CONFIG_REDZONESZ_SHIFT) |
+ (0 << IA32_FRED_CONFIG_EXTINT_SLC_SHIFT) | 0);
+
+ /*
+ * With FRED, there is no need for workarounds requiring
+ * dedicated stack for #DB. Also, there is no need to have
+ * separate stacks for #NM and #MC, because CSL rules ensure
+ * that interrupted frame is safe even if NMI was interrupted
+ * by machine check or vice versa.
+ *
+ * Still, we need to provide a stack for events that can occur
+ * with the interrupts disabled, to avoid caring about the
+ * ordering of the context update in cpu_switch.S. Use CSL 1
+ * for #NM and #MC.
+ *
+ * Since double fault handler is used to handle stack
+ * overflow, it gets the dedicated stack, of the highest used
+ * CSL 2.
+ *
+ * CSL 3 is unused, and we have some space to re-arrange this
+ * if a need in the future appears.
+ */
+ wrmsr(MSR_FRED_STKLVLS, (1ul << (IDT_NM * 2)) |
+ (1ul << (IDT_MC * 2)) | (2ul << (IDT_DF * 2)));
+ wrmsr(MSR_FRED_RSP3, 0);
+ load_cr4(rcr4() | CR4_FRED);
+}
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -160,6 +160,7 @@
offsetof(struct pti_frame, pti_rip));
extern u_int64_t hammer_time(u_int64_t, u_int64_t);
+extern const char fred_events_handlers[], fred_events_handlers_end[];
static void cpu_startup(void *);
SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
@@ -205,11 +206,12 @@
long realmem = 0;
int late_console = 1;
int lass_enabled = 0;
+int fred = 0;
struct kva_md_info kmi;
struct region_descriptor r_idt;
-
+char *fred_page;
struct pcpu *__pcpu;
struct pcpu temp_bsp_pcpu;
@@ -347,9 +349,9 @@
static struct gate_descriptor idt0[NIDT];
struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */
-static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(16);
+static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(64);
static char mce0_stack[MCE_STACK_SIZE] __aligned(16);
-static char nmi0_stack[NMI_STACK_SIZE] __aligned(16);
+static char nmi0_stack[NMI_STACK_SIZE] __aligned(64);
static char dbg0_stack[DBG_STACK_SIZE] __aligned(16);
CTASSERT(sizeof(struct nmi_pcpu) == 16);
@@ -1186,9 +1188,11 @@
msr = rdmsr(MSR_EFER) | EFER_SCE;
wrmsr(MSR_EFER, msr);
- wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
- (u_int64_t)IDTVEC(fast_syscall));
- wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
+ if (!fred) {
+ wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
+ (u_int64_t)IDTVEC(fast_syscall));
+ wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
+ }
msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
wrmsr(MSR_STAR, msr);
@@ -1230,33 +1234,49 @@
tssp = &pc->pc_common_tss;
- /* doublefault stack space, runs on ist1 */
- np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(dblfault_stack)]) - 1;
- np->np_pcpu = (register_t)pc;
- tssp->tss_ist1 = (long)np;
+ /* Doublefault stack space, runs on ist1 for IDT. */
+ if (fred) {
+ wrmsr(MSR_FRED_RSP2, (uint64_t)&dblfault_stack[
+ sizeof(dblfault_stack)]);
+ } else {
+ np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(
+ dblfault_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ tssp->tss_ist1 = (long)np;
+ }
/*
- * NMI stack, runs on ist2. The pcpu pointer is stored just
- * above the start of the ist2 stack.
+ * NMI stack.
*/
- np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
- np->np_pcpu = (register_t)pc;
- tssp->tss_ist2 = (long)np;
+ if (fred) {
+ wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi0_stack[
+ sizeof(nmi0_stack)]);
+ } else {
+ /*
+ * Runs on ist2 for IDT. The pcpu pointer is stored
+ * just above the start of the ist2 stack.
+ */
+ np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ tssp->tss_ist2 = (long)np;
+ }
- /*
- * MC# stack, runs on ist3. The pcpu pointer is stored just
- * above the start of the ist3 stack.
- */
- np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
- np->np_pcpu = (register_t)pc;
- tssp->tss_ist3 = (long)np;
+ if (!fred) {
+ /*
+ * MC# stack for IDT, runs on ist3. The pcpu pointer
+ * is stored just above the start of the ist3 stack.
+ */
+ np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ tssp->tss_ist3 = (long)np;
- /*
- * DB# stack, runs on ist4.
- */
- np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
- np->np_pcpu = (register_t)pc;
- tssp->tss_ist4 = (long)np;
+ /*
+ * DB# stack for IDT, runs on ist4.
+ */
+ np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ tssp->tss_ist4 = (long)np;
+ }
}
/*
@@ -1352,6 +1372,13 @@
TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
}
+ if ((cpu_stdext_feature4 & (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS)) ==
+ (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS) &&
+ (cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 && !pti) {
+ fred = 1;
+ TUNABLE_INT_FETCH("hw.fred", &fred);
+ }
+
sched_instance_select();
link_elf_ireloc();
@@ -1419,55 +1446,57 @@
mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS);
mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF);
- /* exceptions */
- for (x = 0; x < NIDT; x++)
- setidt(x, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_DE, pti ? &IDTVEC(div_pti) : &IDTVEC(div), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 4);
- setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2);
- setidt(IDT_BP, pti ? &IDTVEC(bpt_pti) : &IDTVEC(bpt), SDT_SYSIGT,
- SEL_UPL, 0);
- setidt(IDT_OF, pti ? &IDTVEC(ofl_pti) : &IDTVEC(ofl), SDT_SYSIGT,
- SEL_UPL, 0);
- setidt(IDT_BR, pti ? &IDTVEC(bnd_pti) : &IDTVEC(bnd), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_UD, pti ? &IDTVEC(ill_pti) : &IDTVEC(ill), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_NM, pti ? &IDTVEC(dna_pti) : &IDTVEC(dna), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
- setidt(IDT_FPUGP, pti ? &IDTVEC(fpusegm_pti) : &IDTVEC(fpusegm),
- SDT_SYSIGT, SEL_KPL, 0);
- setidt(IDT_TS, pti ? &IDTVEC(tss_pti) : &IDTVEC(tss), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_NP, pti ? &IDTVEC(missing_pti) : &IDTVEC(missing),
- SDT_SYSIGT, SEL_KPL, 0);
- setidt(IDT_SS, pti ? &IDTVEC(stk_pti) : &IDTVEC(stk), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_GP, pti ? &IDTVEC(prot_pti) : &IDTVEC(prot), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_PF, pti ? &IDTVEC(page_pti) : &IDTVEC(page), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_MF, pti ? &IDTVEC(fpu_pti) : &IDTVEC(fpu), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_AC, pti ? &IDTVEC(align_pti) : &IDTVEC(align), SDT_SYSIGT,
- SEL_KPL, 0);
- setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 3);
- setidt(IDT_XF, pti ? &IDTVEC(xmm_pti) : &IDTVEC(xmm), SDT_SYSIGT,
- SEL_KPL, 0);
+ if (!fred) {
+ /* exceptions */
+ for (x = 0; x < NIDT; x++)
+ setidt(x, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_DE, pti ? &IDTVEC(div_pti) : &IDTVEC(div),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 4);
+ setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2);
+ setidt(IDT_BP, pti ? &IDTVEC(bpt_pti) : &IDTVEC(bpt),
+ SDT_SYSIGT, SEL_UPL, 0);
+ setidt(IDT_OF, pti ? &IDTVEC(ofl_pti) : &IDTVEC(ofl),
+ SDT_SYSIGT, SEL_UPL, 0);
+ setidt(IDT_BR, pti ? &IDTVEC(bnd_pti) : &IDTVEC(bnd),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_UD, pti ? &IDTVEC(ill_pti) : &IDTVEC(ill),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_NM, pti ? &IDTVEC(dna_pti) : &IDTVEC(dna),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
+ setidt(IDT_FPUGP, pti ? &IDTVEC(fpusegm_pti) :
+ &IDTVEC(fpusegm), SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_TS, pti ? &IDTVEC(tss_pti) : &IDTVEC(tss),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_NP, pti ? &IDTVEC(missing_pti) : &IDTVEC(missing),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_SS, pti ? &IDTVEC(stk_pti) : &IDTVEC(stk),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_GP, pti ? &IDTVEC(prot_pti) : &IDTVEC(prot),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_PF, pti ? &IDTVEC(page_pti) : &IDTVEC(page),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_MF, pti ? &IDTVEC(fpu_pti) : &IDTVEC(fpu),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_AC, pti ? &IDTVEC(align_pti) : &IDTVEC(align),
+ SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 3);
+ setidt(IDT_XF, pti ? &IDTVEC(xmm_pti) : &IDTVEC(xmm),
+ SDT_SYSIGT, SEL_KPL, 0);
#ifdef KDTRACE_HOOKS
- setidt(IDT_DTRACE_RET, pti ? &IDTVEC(dtrace_ret_pti) :
- &IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0);
+ setidt(IDT_DTRACE_RET, pti ? &IDTVEC(dtrace_ret_pti) :
+ &IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0);
#endif
#ifdef XENHVM
- setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) :
- &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0);
+ setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) :
+ &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0);
#endif
- r_idt.rd_limit = sizeof(idt0) - 1;
- r_idt.rd_base = (long) idt;
- lidt(&r_idt);
+ r_idt.rd_limit = sizeof(idt0) - 1;
+ r_idt.rd_base = (long) idt;
+ lidt(&r_idt);
+ }
TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
TUNABLE_INT_FETCH("machdep.mitigations.ibrs.disable", &hw_ibrs_disable);
@@ -1558,6 +1587,14 @@
if (getenv_is_true("debug.dump_modinfo_at_boot"))
preload_dump();
+ if (fred) {
+ memset(fred_page, 0x90, PAGE_SIZE);
+ memcpy(fred_page, fred_events_handlers,
+ fred_events_handlers_end - fred_events_handlers);
+
+ amd64_cpu_init_fred();
+ }
+
#ifdef DEV_ISA
#ifdef DEV_ATPIC
elcr_probe();
@@ -1566,12 +1603,16 @@
/* Reset and mask the atpics and leave them shut down. */
atpic_reset();
- /*
- * Point the ICU spurious interrupt vectors at the APIC spurious
- * interrupt handler.
- */
- setidt(IDT_IO_INTS + 7, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
- setidt(IDT_IO_INTS + 15, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
+ if (!fred) {
+ /*
+ * Point the ICU spurious interrupt vectors at the
+ * APIC spurious interrupt handler.
+ */
+ setidt(IDT_IO_INTS + 7, IDTVEC(spuriousint), SDT_SYSIGT,
+ SEL_KPL, 0);
+ setidt(IDT_IO_INTS + 15, IDTVEC(spuriousint), SDT_SYSIGT,
+ SEL_KPL, 0);
+ }
#endif
#else
#error "have you forgotten the isa device?"
@@ -1823,6 +1864,11 @@
extern const char wrmsr_early_safe_gp_handler[];
static struct region_descriptor wrmsr_early_safe_orig_efi_idt;
+/*
+ * Note about FRED. wrmsr_early_safe_start() is used before we
+ * switched CPU to the FRED mode. We use IDT to catch #GP from MSR
+ * write even if BSP is switched to the FRED mode later.
+ */
void
wrmsr_early_safe_start(void)
{
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -125,33 +125,36 @@
cpu_apic_ids[i] = -1;
}
- /* Install an inter-CPU IPI for cache and TLB invalidations. */
- setidt(IPI_INVLOP, pti ? IDTVEC(invlop_pti) : IDTVEC(invlop),
- SDT_SYSIGT, SEL_KPL, 0);
+ if (!fred) {
+ /* Install an inter-CPU IPI for cache and TLB invalidations. */
+ setidt(IPI_INVLOP, pti ? IDTVEC(invlop_pti) : IDTVEC(invlop),
+ SDT_SYSIGT, SEL_KPL, 0);
- /* Install an inter-CPU IPI for all-CPU rendezvous */
- setidt(IPI_RENDEZVOUS, pti ? IDTVEC(rendezvous_pti) :
- IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0);
+ /* Install an inter-CPU IPI for all-CPU rendezvous */
+ setidt(IPI_RENDEZVOUS, pti ? IDTVEC(rendezvous_pti) :
+ IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0);
- /* Install generic inter-CPU IPI handler */
- setidt(IPI_BITMAP_VECTOR, pti ? IDTVEC(ipi_intr_bitmap_handler_pti) :
- IDTVEC(ipi_intr_bitmap_handler), SDT_SYSIGT, SEL_KPL, 0);
+ /* Install generic inter-CPU IPI handler */
+ setidt(IPI_BITMAP_VECTOR, pti ?
+ IDTVEC(ipi_intr_bitmap_handler_pti) :
+ IDTVEC(ipi_intr_bitmap_handler), SDT_SYSIGT, SEL_KPL, 0);
- /* Install an inter-CPU IPI for CPU stop/restart */
- setidt(IPI_STOP, pti ? IDTVEC(cpustop_pti) : IDTVEC(cpustop),
- SDT_SYSIGT, SEL_KPL, 0);
+ /* Install an inter-CPU IPI for CPU stop/restart */
+ setidt(IPI_STOP, pti ? IDTVEC(cpustop_pti) : IDTVEC(cpustop),
+ SDT_SYSIGT, SEL_KPL, 0);
- /* Install an inter-CPU IPI for CPU offline */
- setidt(IPI_OFF, pti ? IDTVEC(cpuoff_pti) : IDTVEC(cpuoff),
- SDT_SYSIGT, SEL_KPL, 0);
+ /* Install an inter-CPU IPI for CPU offline */
+ setidt(IPI_OFF, pti ? IDTVEC(cpuoff_pti) : IDTVEC(cpuoff),
+ SDT_SYSIGT, SEL_KPL, 0);
- /* Install an inter-CPU IPI for CPU suspend/resume */
- setidt(IPI_SUSPEND, pti ? IDTVEC(cpususpend_pti) : IDTVEC(cpususpend),
- SDT_SYSIGT, SEL_KPL, 0);
+ /* Install an inter-CPU IPI for CPU suspend/resume */
+ setidt(IPI_SUSPEND, pti ? IDTVEC(cpususpend_pti) :
+ IDTVEC(cpususpend), SDT_SYSIGT, SEL_KPL, 0);
- /* Install an IPI for calling delayed SWI */
- setidt(IPI_SWI, pti ? IDTVEC(ipi_swi_pti) : IDTVEC(ipi_swi),
- SDT_SYSIGT, SEL_KPL, 0);
+ /* Install an IPI for calling delayed SWI */
+ setidt(IPI_SWI, pti ? IDTVEC(ipi_swi_pti) : IDTVEC(ipi_swi),
+ SDT_SYSIGT, SEL_KPL, 0);
+ }
/* Set boot_cpu_id if needed. */
if (boot_cpu_id == -1) {
@@ -240,25 +243,51 @@
IOPERM_BITMAP_SIZE;
pc->pc_common_tss.tss_rsp0 = 0;
- /* The doublefault stack runs on IST1. */
- np = ((struct nmi_pcpu *)&doublefault_stack[DBLFAULT_STACK_SIZE]) - 1;
- np->np_pcpu = (register_t)pc;
- pc->pc_common_tss.tss_ist1 = (long)np;
+ /*
+ * The doublefault stack.
+ * Runs on IST1 for IDT.
+ * Uses CSL 2 for FRED.
+ */
+ if (fred) {
+ wrmsr(MSR_FRED_RSP2, (uint64_t)&doublefault_stack[
+ DBLFAULT_STACK_SIZE]);
+ } else {
+ np = ((struct nmi_pcpu *)&doublefault_stack[
+ DBLFAULT_STACK_SIZE]) - 1;
+ np->np_pcpu = (register_t)pc;
+ pc->pc_common_tss.tss_ist1 = (long)np;
+ }
- /* The NMI stack runs on IST2. */
- np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1;
- np->np_pcpu = (register_t)pc;
- pc->pc_common_tss.tss_ist2 = (long)np;
+ /*
+ * The NMI stack.
+ * Runs on IST2 for IDT.
+ * Uses CSL 1 for FRED.
+ */
+ if (fred) {
+ wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi_stack[NMI_STACK_SIZE]);
+ } else {
+ np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1;
+ np->np_pcpu = (register_t)pc;
+ pc->pc_common_tss.tss_ist2 = (long)np;
+ }
- /* The MC# stack runs on IST3. */
- np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1;
- np->np_pcpu = (register_t)pc;
- pc->pc_common_tss.tss_ist3 = (long)np;
+ /*
+ * The MC# stack.
+ * Runs on IST3 for IDT.
+ * Shares CSL 1 with NMI for FRED.
+ */
+ if (!fred) {
+ np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1;
+ np->np_pcpu = (register_t)pc;
+ pc->pc_common_tss.tss_ist3 = (long)np;
+ }
- /* The DB# stack runs on IST4. */
- np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1;
- np->np_pcpu = (register_t)pc;
- pc->pc_common_tss.tss_ist4 = (long)np;
+ if (!fred) {
+ /* The DB# stack, used for for IDT, runs on IST4. */
+ np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1;
+ np->np_pcpu = (register_t)pc;
+ pc->pc_common_tss.tss_ist4 = (long)np;
+ }
/* Prepare private GDT */
gdt_segs[GPROC0_SEL].ssd_base = (long)&pc->pc_common_tss;
@@ -278,7 +307,10 @@
wrmsr(MSR_KGSBASE, 0); /* User value */
fix_cpuid();
- lidt(&r_idt);
+ if (fred)
+ amd64_cpu_init_fred();
+ else
+ lidt(&r_idt);
gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
ltr(gsel_tss);
@@ -430,12 +462,17 @@
M_WAITOK | M_ZERO);
doublefault_stack = kmem_malloc(DBLFAULT_STACK_SIZE,
M_WAITOK | M_ZERO);
- mce_stack = kmem_malloc(MCE_STACK_SIZE,
- M_WAITOK | M_ZERO);
+ if (!fred) {
+ mce_stack = kmem_malloc(MCE_STACK_SIZE,
+ M_WAITOK | M_ZERO);
+ }
nmi_stack = kmem_malloc_domainset(
DOMAINSET_PREF(domain), NMI_STACK_SIZE, M_WAITOK | M_ZERO);
- dbg_stack = kmem_malloc_domainset(
- DOMAINSET_PREF(domain), DBG_STACK_SIZE, M_WAITOK | M_ZERO);
+ if (!fred) {
+ dbg_stack = kmem_malloc_domainset(
+ DOMAINSET_PREF(domain), DBG_STACK_SIZE,
+ M_WAITOK | M_ZERO);
+ }
dpcpu = kmem_malloc_domainset(DOMAINSET_PREF(domain),
DPCPU_SIZE, M_WAITOK | M_ZERO);
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -2066,9 +2066,9 @@
pmap_bootstrap(vm_paddr_t *firstaddr)
{
vm_offset_t va;
- pt_entry_t *pte, *pcpu_pte;
+ pt_entry_t *fred_page_pte, *pte, *pcpu_pte;
struct region_descriptor r_gdt;
- uint64_t cr4, pcpu0_phys;
+ uint64_t cr4, fred_phys, pcpu0_phys;
u_long res;
int i;
@@ -2086,6 +2086,8 @@
create_pagetables(firstaddr);
pcpu0_phys = allocpages(firstaddr, 1);
+ if (fred)
+ fred_phys = allocpages(firstaddr, 1);
/*
* Add a physical memory segment (vm_phys_seg) corresponding to the
@@ -2175,6 +2177,12 @@
va = virtual_avail;
pte = vtopte(va);
+ if (fred) {
+ SYSMAP(caddr_t, fred_page_pte, fred_page, 1);
+ *fred_page_pte = fred_phys | X86_PG_V | X86_PG_RW | pg_g |
+ X86_PG_M | X86_PG_A;
+ }
+
/*
* Crashdump maps. The first page is reused as CMAP1 for the
* memory test.
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -41,6 +41,7 @@
* AMD64 Trap and System call handling
*/
+#include "opt_atpic.h"
#include "opt_clock.h"
#include "opt_cpu.h"
#include "opt_hwpmc_hooks.h"
@@ -61,6 +62,7 @@
#include <sys/mutex.h>
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
+#include <sys/smp.h>
#include <sys/syscall.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
@@ -83,6 +85,8 @@
#include <machine/cpu.h>
#include <machine/intr_machdep.h>
+#include <x86/apicreg.h>
+#include <x86/apicvar.h>
#include <x86/mca.h>
#include <machine/md_var.h>
#include <machine/pcb.h>
@@ -90,11 +94,19 @@
#include <machine/stack.h>
#include <machine/trap.h>
#include <machine/tss.h>
+#include <x86/isa/icu.h>
#ifdef KDTRACE_HOOKS
#include <sys/dtrace_bsd.h>
#endif
+#ifdef XENHVM
+#include <sys/interrupt.h>
+#include <xen/xen-os.h>
+#include <xen/xen_intr.h>
+#include <machine/xen/arch-intr.h>
+#endif
+
extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),
IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),
IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);
@@ -1004,6 +1016,12 @@
printf("r13: %016lx r14: %016lx r15: %016lx\n", frame->tf_r13,
frame->tf_r14, frame->tf_r15);
+ if (fred) {
+ printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n",
+ frame->tf_fred_evdata, frame->tf_fred_evinfo1,
+ frame->tf_fred_evinfo2);
+ }
+
printf("trap number = %d\n", type);
}
@@ -1079,6 +1097,11 @@
frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
frame->tf_fs, frame->tf_gs,
rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
+ if (fred) {
+ printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n",
+ frame->tf_fred_evdata, frame->tf_fred_evinfo1,
+ frame->tf_fred_evinfo2);
+ }
/* Print these separately in case pcpu accesses trap. */
printf("cpuid = %d; apic id = %02x\n", PCPU_GET(cpuid),
PCPU_GET(apic_id));
@@ -1305,3 +1328,526 @@
amd64_syscall_ret_flush_l1d_check_inline(td->td_errno);
}
+
+static void
+trap_fred_extint(struct trapframe *frame, u_int vec)
+{
+ switch (vec) {
+ case IPI_RENDEZVOUS:
+#ifdef COUNT_IPIS
+ ipi_rendezvous_counts[PCPU_GET(cpuid)]++;
+#endif
+ smp_rendezvous_action();
+ lapic_eoi();
+ break;
+ case IPI_INVLOP:
+ invlop_handler();
+ lapic_eoi();
+ break;
+ case IPI_BITMAP_VECTOR:
+ lapic_eoi();
+ ipi_bitmap_handler(frame);
+ break;
+ case IPI_STOP:
+ lapic_eoi();
+ cpustop_handler();
+ break;
+ case IPI_SUSPEND:
+ cpususpend_handler();
+ lapic_eoi();
+ break;
+ case IPI_SWI:
+ lapic_eoi();
+ ipi_swi_handler(frame);
+ break;
+ case IPI_OFF:
+ cpuoff_handler();
+ lapic_eoi();
+ break;
+ case APIC_SPURIOUS_INT:
+ break;
+ case APIC_TIMER_INT:
+ lapic_handle_timer(frame);
+ break;
+ case APIC_CMC_INT:
+ lapic_handle_cmc();
+ break;
+ case APIC_ERROR_INT:
+ lapic_handle_error();
+ break;
+#ifdef XENHVM
+ case IDT_EVTCHN:
+ xen_arch_intr_handle_upcall(frame);
+ break;
+#endif
+ default:
+ if (vec >= IPI_DYN_FIRST && vec <= IPI_DYN_LAST) {
+ void (*fi)(struct trapframe *);
+
+ fi = fred_ipi_handlers[vec - IPI_DYN_FIRST];
+ if (fi != NULL) {
+ fi(frame);
+ lapic_eoi();
+ } else {
+ panic("DYN IPI %d without FRED handler", vec);
+ }
+ break;
+ }
+#ifdef DEV_ISA
+#ifdef DEV_ATPIC
+ if (vec >= IDT_IO_INTS && vec < IDT_IO_INTS + NUM_ISA_IRQS) {
+ atpic_handle_intr(vec - IDT_IO_INTS, frame);
+ break;
+ }
+#endif
+#endif
+ /* apic ioint */
+ lapic_handle_intr(vec, frame);
+ break;
+ }
+}
+
+void fred_out_of_nmi(void);
+
+static const int fred_ev_to_trapno_table[] = {
+ [IDT_DE] = T_DIVIDE,
+ [IDT_DB] = T_TRCTRAP,
+ [IDT_NMI] = T_RESERVED,
+ [IDT_BP] = T_BPTFLT,
+ [IDT_OF] = T_OFLOW,
+ [IDT_BR] = T_BOUND,
+ [IDT_UD] = T_PRIVINFLT,
+ [IDT_NM] = T_DNA,
+ [IDT_DF] = T_DOUBLEFLT,
+ [IDT_FPUGP] = T_FPOPFLT,
+ [IDT_TS] = T_TSSFLT,
+ [IDT_NP] = T_SEGNPFLT,
+ [IDT_SS] = T_STKFLT,
+ [IDT_GP] = T_PROTFLT,
+ [IDT_PF] = T_PAGEFLT,
+ [15] = T_RESERVED,
+ [IDT_MF] = T_ARITHTRAP,
+ [IDT_AC] = T_ALIGNFLT,
+ [IDT_MC] = T_MCHK,
+ [IDT_XF] = T_XMMFLT,
+ [20] = T_RESERVED,
+ [21] = T_RESERVED,
+ [22] = T_RESERVED,
+ [23] = T_RESERVED,
+ [24] = T_RESERVED,
+ [25] = T_RESERVED,
+ [26] = T_RESERVED,
+ [27] = T_RESERVED,
+ [28] = T_RESERVED,
+ [29] = T_RESERVED,
+ [30] = T_RESERVED,
+ [31] = T_RESERVED,
+};
+
+static void
+trap_fred_ev_to_trapno(struct trapframe *frame, u_int exc)
+{
+ frame->tf_trapno = fred_ev_to_trapno_table[exc];
+}
+
+static void
+trap_fred_handle_u_exc(struct thread *td, struct trapframe *frame, u_int exc,
+ ksiginfo_t *ksi, bool *do_trapsig)
+{
+ struct proc *p;
+ register_t addr;
+ int pf, signo, ucode;
+
+ addr = frame->tf_rip;
+ signo = 0;
+ ucode = 0;
+ frame->tf_addr = 0;
+ trap_check_intr_user(td, frame);
+
+ switch (exc) {
+ case IDT_DE:
+ ucode = FPE_INTDIV;
+ signo = SIGFPE;
+ break;
+ case IDT_DB:
+ signo = SIGTRAP;
+ ucode = TRAP_TRACE;
+ if ((frame->tf_fred_evdata & TF_FRED_EVDATA_BS) != 0)
+ trap_clear_step(td, frame);
+ break;
+ case IDT_BP:
+#ifdef KDTRACE_HOOKS
+ if (trap_user_dtrace(frame, &dtrace_pid_probe_ptr))
+ return;
+#endif
+ signo = SIGTRAP;
+ ucode = TRAP_BRKPT;
+ break;
+ case IDT_OF:
+ ucode = FPE_INTOVF;
+ signo = SIGFPE;
+ break;
+ case IDT_BR:
+ ucode = FPE_FLTSUB;
+ signo = SIGFPE;
+ break;
+ case IDT_UD:
+ signo = SIGILL;
+ ucode = ILL_PRVOPC;
+ break;
+ case IDT_NM:
+ KASSERT(PCB_USER_FPU(td->td_pcb),
+ ("kernel FPU ctx has leaked"));
+ fpudna();
+ return;
+ case IDT_DF:
+ signo = SIGBUS;
+ ucode = BUS_OBJERR;
+ break;
+ case IDT_FPUGP:
+ ucode = ILL_COPROC;
+ signo = SIGILL;
+ break;
+ case IDT_TS:
+ signo = SIGBUS;
+ ucode = BUS_OBJERR;
+ break;
+ case IDT_NP:
+ signo = SIGBUS;
+ ucode = BUS_ADRERR;
+ break;
+ case IDT_SS:
+ signo = SIGBUS;
+ ucode = BUS_ADRERR;
+ break;
+ case IDT_GP:
+ signo = SIGBUS;
+ ucode = BUS_OBJERR;
+ break;
+ case IDT_PF:
+ p = td->td_proc;
+ frame->tf_addr = addr = frame->tf_fred_evdata;
+ if (*p->p_sysent->sv_trap != NULL &&
+ (*p->p_sysent->sv_trap)(td) == 0)
+ return;
+ pf = trap_pfault(frame, true, &signo, &ucode);
+ if (pf == -1 || pf == 0)
+ return;
+ break;
+ case IDT_MF:
+ ucode = fputrap_x87();
+ if (ucode == -1)
+ return;
+ signo = SIGFPE;
+ break;
+ case IDT_AC:
+ signo = SIGBUS;
+ ucode = BUS_ADRALN;
+ break;
+ case IDT_MC:
+ mca_intr();
+ return;
+ case IDT_XF:
+ ucode = fputrap_sse();
+ if (ucode == -1)
+ return;
+ signo = SIGFPE;
+ break;
+ default:
+ panic("missed FRED handler for exception %d", exc);
+ }
+
+ ksiginfo_init_trap(ksi);
+ ksi->ksi_signo = signo;
+ ksi->ksi_code = ucode;
+ ksi->ksi_trapno = frame->tf_trapno;
+ ksi->ksi_addr = (void *)addr;
+ trap_uprintf_signal(td, frame, addr, signo, ucode);
+ KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
+ *do_trapsig = true;
+}
+
+static u_int
+fred_inst_len(const struct trapframe *frame)
+{
+ return ((frame->tf_fred_evinfo2 & TF_FRED_EVINFO2_INSTLENMASK) >>
+ TF_FRED_EVINFO2_INSTLENSHIFT);
+}
+
+int fred_verbose_events;
+
+void trap_fred_u(struct trapframe *frame);
+void
+trap_fred_u(struct trapframe *frame)
+{
+ struct thread *td;
+ ksiginfo_t ksi;
+ u_int type, vec;
+ bool do_trapsig;
+
+ td = curthread;
+ type = frame->tf_fred_evinfo2 & TF_FRED_EVINFO2_TYPEMASK;
+ vec = frame->tf_fred_evinfo2 & TF_FRED_EVINFO2_VECMASK;
+ do_trapsig = false;
+ if (fred_verbose_events) {
+ printf("pid %d tid %d type %d vec %d rax %#lx\n",
+ td->td_proc->p_pid, td->td_tid,
+ (frame->tf_fred_evinfo2 & TF_FRED_EVINFO2_TYPEMASK) >> 16,
+ vec, frame->tf_rax);
+ }
+ if (__predict_true((td->td_proc->p_sysent->sv_flags &
+ (SV_ABI_MASK | SV_LP64)) == (SV_ABI_FREEBSD | SV_LP64) &&
+ (type == TF_FRED_EVINFO2_TYPE_SYSCALL ||
+ type == TF_FRED_EVINFO2_TYPE_EXTINT ||
+ (type == TF_FRED_EVINFO2_TYPE_EXC && vec == IDT_PF))))
+ clear_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+ else
+ set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+ switch (type) {
+ case TF_FRED_EVINFO2_TYPE_EXTINT:
+ trap_fred_extint(frame, vec);
+ break;
+ case TF_FRED_EVINFO2_TYPE_NMI:
+ frame->tf_trapno = T_NMI;
+ nmi_handle_intr(frame);
+#ifdef HWPMC_HOOKS
+ if ((td->td_pflags & TDP_CALLCHAIN) != 0) {
+ int (*ph)(struct thread *, int, void *);
+
+ fred_out_of_nmi();
+ ph = atomic_load_ptr(&pmc_hook);
+ if (ph != NULL) {
+ enable_intr();
+ ph(td, PMC_FN_USER_CALLCHAIN, frame);
+ }
+ }
+#endif
+ break;
+ case TF_FRED_EVINFO2_TYPE_INT1:
+ case TF_FRED_EVINFO2_TYPE_INT3:
+ case TF_FRED_EVINFO2_TYPE_EXC:
+ enable_intr();
+ trap_fred_ev_to_trapno(frame, vec);
+#ifdef KDTRACE_HOOKS
+ if (dtrace_trap_func != NULL &&
+ (*dtrace_trap_func)(frame, frame->tf_trapno) != 0)
+ return;
+#endif
+ td->td_pticks = 0;
+ td->td_frame = frame;
+ if (td->td_cowgen != atomic_load_int(&td->td_proc->p_cowgen))
+ thread_cow_update(td);
+ trap_fred_handle_u_exc(td, frame, vec, &ksi, &do_trapsig);
+ break;
+ case TF_FRED_EVINFO2_TYPE_INTn:
+ enable_intr();
+#ifdef COMPAT_FREEBSD32
+ if (vec == IDT_SYSCALL) {
+ frame->tf_err = fred_inst_len(frame);
+ ia32_syscall(frame);
+ break;
+ }
+#endif
+ frame->tf_trapno = vec;
+ ksiginfo_init_trap(&ksi);
+ ksi.ksi_signo = SIGBUS;
+ ksi.ksi_code = BUS_OBJERR;
+ ksi.ksi_trapno = frame->tf_trapno;
+ ksi.ksi_addr = (void *)frame->tf_rip;
+ do_trapsig = true;
+ break;
+ case TF_FRED_EVINFO2_TYPE_SYSCALL:
+ enable_intr();
+ if (vec == TF_FRED_EVINFO2_VEC_SYSCALL) {
+ frame->tf_err = fred_inst_len(frame);
+ td->td_frame = frame;
+ amd64_syscall(td, (frame->tf_rflags & PSL_T) != 0);
+ } else {
+ /* SYSENTER returns without action */
+ }
+ break;
+ default:
+ __unreachable();
+ }
+ if (do_trapsig)
+ trapsignal(td, &ksi);
+ userret(td, frame);
+ KASSERT(PCB_USER_FPU(td->td_pcb),
+ ("Return from trap with kernel FPU ctx leaked"));
+}
+
+extern const char fred_ld_fs[];
+extern const char fred_ld_fs_fault[];
+extern const char fred_ld_fsbase[];
+extern const char fred_ld_fsbase_fault[];
+extern const char fred_lkgs[];
+extern const char fred_lkgs_fault[];
+extern const char fred_ld_kgsbase[];
+extern const char fred_ld_kgsbase_fault[];
+extern const char fred_ld_es[];
+extern const char fred_ld_es_fault[];
+extern const char fred_ld_ds[];
+extern const char fred_ld_ds_fault[];
+extern const char fred_eretu[];
+extern const char fred_eretu_fault[];
+
+static const struct sfhandler fred_sfhandlers[] = {
+ {
+ .faddr = (uintptr_t)fred_ld_fs,
+ .fhandler = (uintptr_t)fred_ld_fs_fault,
+ },
+ {
+ .faddr = (uintptr_t)fred_ld_fsbase,
+ .fhandler = (uintptr_t)fred_ld_fsbase_fault,
+ },
+ {
+ .faddr = (uintptr_t)fred_lkgs,
+ .fhandler = (uintptr_t)fred_lkgs_fault,
+ },
+ {
+ .faddr = (uintptr_t)fred_ld_kgsbase,
+ .fhandler = (uintptr_t)fred_ld_kgsbase_fault,
+ },
+ {
+ .faddr = (uintptr_t)fred_ld_es,
+ .fhandler = (uintptr_t)fred_ld_es_fault,
+ },
+ {
+ .faddr = (uintptr_t)fred_ld_ds,
+ .fhandler = (uintptr_t)fred_ld_ds_fault,
+ },
+ {
+ .faddr = (uintptr_t)fred_eretu,
+ .fhandler = (uintptr_t)fred_eretu_fault,
+ },
+};
+
+static void
+trap_fred_handle_k_exc(struct thread *td, struct trapframe *frame, u_int exc)
+{
+ register_t addr;
+ int i;
+
+ addr = frame->tf_rip;
+ frame->tf_addr = 0;
+ KASSERT(cold || td->td_ucred != NULL,
+ ("kernel trap doesn't have ucred"));
+
+ /*
+ * FRED exception handler runs with the interrupts disabled,
+ * unlike the IDT case. Only for page faults the interrupts
+ * are enabled, since this is the only legitimate exception
+ * from the kernel mode. Anything else results in panic, so
+ * there is no sense to enable interrupts.
+ */
+
+ if (exc != IDT_PF && trap_check_pcb_onfault(td, frame))
+ return;
+
+ switch (exc) {
+ case IDT_DE:
+ break;
+ case IDT_DB:
+ if (user_dbreg_trap(frame->tf_fred_evdata))
+ return;
+ /* FALLTHROUGH */
+ case IDT_BP:
+#ifdef KDB
+ if (kdb_trap(frame->tf_trapno, frame->tf_fred_evdata, frame))
+ return;
+#endif
+ break;
+ case IDT_OF:
+ break;
+ case IDT_BR:
+ break;
+ case IDT_UD:
+ break;
+ case IDT_NM:
+ if (PCB_USER_FPU(td->td_pcb))
+ panic("Unregistered use of FPU in kernel");
+ fpudna();
+ return;
+ case IDT_DF:
+ dblfault_handler(frame);
+ break;
+ case IDT_FPUGP:
+ break;
+ case IDT_TS:
+ /*
+ * PSL_NT is cleared automatically by FRED entry.
+ */
+ break;
+ case IDT_NP:
+ case IDT_SS:
+ case IDT_GP:
+ if (td->td_intr_nesting_level != 0)
+ break;
+ for (i = 0; i < nitems(fred_sfhandlers); i++) {
+ if (frame->tf_rip == fred_sfhandlers[i].faddr) {
+ KASSERT((read_rflags() & PSL_I) == 0,
+ ("interrupts enabled"));
+ frame->tf_rip = fred_sfhandlers[i].fhandler;
+ return;
+ }
+ }
+ if (curpcb->pcb_onfault != NULL) {
+ frame->tf_rip = (long)curpcb->pcb_onfault;
+ return;
+ }
+ break;
+ case IDT_PF:
+ frame->tf_addr = addr = frame->tf_fred_evdata;
+ if ((frame->tf_rflags & PSL_I) != 0)
+ enable_intr();
+ trap_pfault(frame, false, NULL, NULL);
+ return;
+ case IDT_MF:
+ break;
+ case IDT_AC:
+ break;
+ case IDT_MC:
+ mca_intr();
+ return;
+ case IDT_XF:
+ break;
+ default:
+ printf("Missing FRED handler for exception %d", exc);
+ trap_fatal(frame, 0);
+ }
+ trap_fatal(frame, 0);
+}
+
+void trap_fred_k(struct trapframe *frame);
+void
+trap_fred_k(struct trapframe *frame)
+{
+ struct thread *td;
+ u_int vec;
+
+ td = curthread;
+ vec = frame->tf_fred_evinfo2 & TF_FRED_EVINFO2_VECMASK;
+ switch (frame->tf_fred_evinfo2 & TF_FRED_EVINFO2_TYPEMASK) {
+ case TF_FRED_EVINFO2_TYPE_EXTINT:
+ trap_fred_extint(frame, vec);
+ break;
+ case TF_FRED_EVINFO2_TYPE_NMI:
+ frame->tf_trapno = T_NMI;
+ nmi_handle_intr(frame);
+ break;
+ case TF_FRED_EVINFO2_TYPE_INT1:
+ case TF_FRED_EVINFO2_TYPE_INT3:
+ case TF_FRED_EVINFO2_TYPE_EXC:
+ trap_fred_ev_to_trapno(frame, vec);
+ trap_fred_handle_k_exc(td, frame, vec);
+ break;
+ case TF_FRED_EVINFO2_TYPE_INTn:
+ panic("INT%d from kernel", vec);
+ break;
+ case TF_FRED_EVINFO2_TYPE_SYSCALL:
+ panic("syscall from kernel");
+ break;
+ default:
+ __unreachable();
+ }
+}
diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h
--- a/sys/amd64/include/md_var.h
+++ b/sys/amd64/include/md_var.h
@@ -46,6 +46,7 @@
extern int nmi_flush_l1d_sw;
extern int syscall_ret_l1d_flush_mode;
extern int lass_enabled;
+extern char *fred_page;
extern vm_paddr_t intel_graphics_stolen_base;
extern vm_paddr_t intel_graphics_stolen_size;
@@ -65,12 +66,14 @@
struct trapframe;
void amd64_conf_fast_syscall(void);
+void amd64_cpu_init_fred(void);
void amd64_db_resume_dbreg(void);
vm_paddr_t amd64_loadaddr(void);
void amd64_lower_shared_page(struct sysentvec *);
void amd64_bsp_pcpu_init1(struct pcpu *pc);
void amd64_bsp_pcpu_init2(uint64_t rsp0);
void amd64_bsp_ist_init(struct pcpu *pc);
+void amd64_bsp_fred_csl_init(struct pcpu *pc);
void amd64_syscall(struct thread *td, int traced);
void amd64_syscall_ret_flush_l1d(int error);
void amd64_syscall_ret_flush_l1d_recalc(void);
diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h
--- a/sys/amd64/include/pcb.h
+++ b/sys/amd64/include/pcb.h
@@ -102,11 +102,17 @@
register_t pcb_lstar;
register_t pcb_cstar;
register_t pcb_sfmask;
+ /* FRED MSRs */
+ register_t pcb_fred_rsp0;
+ register_t pcb_fred_rsp1;
+ register_t pcb_fred_rsp2;
+ register_t pcb_fred_rsp3;
+ register_t pcb_fred_stklvls;
+ register_t pcb_fred_config;
struct savefpu *pcb_save;
register_t pcb_tlsbase; /* not same as pcb_fsbase */
- uint64_t pcb_pad[4];
};
/* Per-CPU state saved during suspend and resume. */
diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c
--- a/sys/amd64/vmm/intel/vmx.c
+++ b/sys/amd64/vmm/intel/vmx.c
@@ -894,7 +894,7 @@
&tmp);
if (error == 0) {
pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
- &IDTVEC(justreturn));
+ &IDTVEC(justreturn), vmm_justreturn);
if (pirvec < 0) {
if (bootverbose) {
printf("vmx_modinit: unable to "
diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c
--- a/sys/amd64/vmm/vmm.c
+++ b/sys/amd64/vmm/vmm.c
@@ -287,7 +287,7 @@
vmm_host_state_init();
vmm_ipinum = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
- &IDTVEC(justreturn));
+ &IDTVEC(justreturn), vmm_justreturn);
if (vmm_ipinum < 0)
vmm_ipinum = IPI_AST;
diff --git a/sys/amd64/vmm/vmm_lapic.h b/sys/amd64/vmm/vmm_lapic.h
--- a/sys/amd64/vmm/vmm_lapic.h
+++ b/sys/amd64/vmm/vmm_lapic.h
@@ -71,4 +71,6 @@
int lapic_intr_msi(struct vm *vm, uint64_t addr, uint64_t msg);
+void vmm_justreturn(struct trapframe *);
+
#endif
diff --git a/sys/amd64/vmm/vmm_lapic.c b/sys/amd64/vmm/vmm_lapic.c
--- a/sys/amd64/vmm/vmm_lapic.c
+++ b/sys/amd64/vmm/vmm_lapic.c
@@ -237,3 +237,8 @@
error = vlapic_read(vlapic, 1, off, rval, arg);
return (error);
}
+
+void
+vmm_justreturn(struct trapframe *frame)
+{
+}
diff --git a/sys/dev/hyperv/vmbus/x86/vmbus_x86.c b/sys/dev/hyperv/vmbus/x86/vmbus_x86.c
--- a/sys/dev/hyperv/vmbus/x86/vmbus_x86.c
+++ b/sys/dev/hyperv/vmbus/x86/vmbus_x86.c
@@ -151,7 +151,8 @@
* free IDT vector for Hyper-V ISR and set it up.
*/
sc->vmbus_idtvec = lapic_ipi_alloc(
- pti ? IDTVEC(vmbus_isr_pti) : IDTVEC(vmbus_isr));
+ pti ? IDTVEC(vmbus_isr_pti) : IDTVEC(vmbus_isr),
+ vmbus_handle_intr);
if (sc->vmbus_idtvec < 0) {
#if defined(__amd64__) && defined(KLD_MODULE)
pmap_pti_remove_kva(VMBUS_ISR_ADDR, VMBUS_ISR_ADDR + PAGE_SIZE);
diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c
--- a/sys/i386/i386/machdep.c
+++ b/sys/i386/i386/machdep.c
@@ -191,6 +191,8 @@
struct mem_range_softc mem_range_softc;
+int fred;
+
extern char start_exceptions[], end_exceptions[];
extern struct sysentvec elf32_freebsd_sysvec;
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -92,7 +92,7 @@
"struct thread KBI td_pflags");
_Static_assert(offsetof(struct thread, td_frame) == 0x4e8,
"struct thread KBI td_frame");
-_Static_assert(offsetof(struct thread, td_emuldata) == 0x6f0,
+_Static_assert(offsetof(struct thread, td_emuldata) == 0x700,
"struct thread KBI td_emuldata");
_Static_assert(offsetof(struct proc, p_flag) == 0xb8,
"struct proc KBI p_flag");
diff --git a/sys/x86/include/apicvar.h b/sys/x86/include/apicvar.h
--- a/sys/x86/include/apicvar.h
+++ b/sys/x86/include/apicvar.h
@@ -247,7 +247,8 @@
}
int lapic_ipi_wait(int delay);
-int lapic_ipi_alloc(inthand_t *ipifunc);
+int lapic_ipi_alloc(inthand_t *ipifunc,
+ void (*fred_ipifunc)(struct trapframe *));
void lapic_ipi_free(int vector);
enum LAPIC_REGISTERS;
uint32_t lapic_read32_ext(enum LAPIC_REGISTERS reg);
diff --git a/sys/x86/include/frame.h b/sys/x86/include/frame.h
--- a/sys/x86/include/frame.h
+++ b/sys/x86/include/frame.h
@@ -196,7 +196,6 @@
#define TF_HASBASES 0x00000002
#define TF_HASFPXSTATE 0x00000004
#define TF_RESERV0 0x00000008 /* no tlsbase in the trapframe */
-#define TF_FRED 0x00000010
#endif /* __amd64__ */
#endif /* _MACHINE_FRAME_H_ */
diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h
--- a/sys/x86/include/x86_var.h
+++ b/sys/x86/include/x86_var.h
@@ -97,6 +97,7 @@
extern int zenbleed_enable;
extern int cpu_amdc1e_bug;
extern char bootmethod[16];
+extern int fred;
struct pcb;
struct thread;
@@ -114,6 +115,8 @@
*/
typedef void alias_for_inthand_t(void);
+extern void (*fred_ipi_handlers[])(struct trapframe *);
+
bool acpi_get_fadt_bootflags(uint16_t *flagsp);
void *alloc_fpusave(int flags);
u_int cpu_auxmsr(void);
diff --git a/sys/x86/isa/atpic.c b/sys/x86/isa/atpic.c
--- a/sys/x86/isa/atpic.c
+++ b/sys/x86/isa/atpic.c
@@ -466,9 +466,11 @@
continue;
ai->at_intsrc.is_count = &ai->at_count;
ai->at_intsrc.is_straycount = &ai->at_straycount;
- setidt(((struct atpic *)ai->at_intsrc.is_pic)->at_intbase +
- ai->at_irq, pti ? ai->at_intr_pti : ai->at_intr, SDT_ATPIC,
- SEL_KPL, GSEL_ATPIC);
+ if (!fred) {
+ setidt(((struct atpic *)ai->at_intsrc.is_pic)->
+ at_intbase + ai->at_irq, pti ? ai->at_intr_pti :
+ ai->at_intr, SDT_ATPIC, SEL_KPL, GSEL_ATPIC);
+ }
}
/*
diff --git a/sys/x86/x86/local_apic.c b/sys/x86/x86/local_apic.c
--- a/sys/x86/x86/local_apic.c
+++ b/sys/x86/x86/local_apic.c
@@ -580,9 +580,11 @@
lapic_map = NULL;
}
- /* Setup the spurious interrupt handler. */
- setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
- GSEL_APIC);
+ if (!fred) {
+ /* Setup the spurious interrupt handler. */
+ setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC,
+ SEL_KPL, GSEL_APIC);
+ }
/* Perform basic initialization of the BSP's local APIC. */
lapic_enable();
@@ -591,19 +593,22 @@
/* Set BSP's per-CPU local APIC ID. */
PCPU_SET(apic_id, lapic_id());
- /* Local APIC timer interrupt. */
- setidt(APIC_TIMER_INT, pti ? IDTVEC(timerint_pti) : IDTVEC(timerint),
- SDT_APIC, SEL_KPL, GSEL_APIC);
+ if (!fred) {
+ /* Local APIC timer interrupt. */
+ setidt(APIC_TIMER_INT, pti ? IDTVEC(timerint_pti) :
+ IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC);
- /* Local APIC error interrupt. */
- setidt(APIC_ERROR_INT, pti ? IDTVEC(errorint_pti) : IDTVEC(errorint),
- SDT_APIC, SEL_KPL, GSEL_APIC);
+ /* Local APIC error interrupt. */
+ setidt(APIC_ERROR_INT, pti ? IDTVEC(errorint_pti) :
+ IDTVEC(errorint),
+ SDT_APIC, SEL_KPL, GSEL_APIC);
- /* XXX: Thermal interrupt */
+ /* XXX: Thermal interrupt */
- /* Local APIC CMCI. */
- setidt(APIC_CMC_INT, pti ? IDTVEC(cmcint_pti) : IDTVEC(cmcint),
- SDT_APIC, SEL_KPL, GSEL_APIC);
+ /* Local APIC CMCI. */
+ setidt(APIC_CMC_INT, pti ? IDTVEC(cmcint_pti) : IDTVEC(cmcint),
+ SDT_APIC, SEL_KPL, GSEL_APIC);
+ }
if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
/* Set if APIC timer runs in C3. */
@@ -1762,8 +1767,11 @@
KASSERT(vector != IDT_DTRACE_RET,
("Attempt to overwrite DTrace entry"));
#endif
- setidt(vector, (pti ? ioint_pti_handlers : ioint_handlers)[vector / 32],
- SDT_APIC, SEL_KPL, GSEL_APIC);
+ if (!fred) {
+ setidt(vector, (pti ? ioint_pti_handlers :
+ ioint_handlers)[vector / 32], SDT_APIC, SEL_KPL,
+ GSEL_APIC);
+ }
}
void
@@ -2272,6 +2280,8 @@
void (*ipi_vectored)(u_int, int) = &native_lapic_ipi_vectored;
#endif /* SMP */
+void (*fred_ipi_handlers[IPI_DYN_LAST - IPI_DYN_FIRST])(struct trapframe *);
+
/*
* Since the IDT is shared by all CPUs the IPI slot update needs to be globally
* visible.
@@ -2286,10 +2296,9 @@
* the IDT slot update is globally visible before the IPI is delivered.
*/
int
-lapic_ipi_alloc(inthand_t *ipifunc)
+lapic_ipi_alloc(inthand_t *ipifunc, void (*fred_ipifunc)(struct trapframe *))
{
- struct gate_descriptor *ip;
- long func;
+ void (*fip)(struct trapframe *);
int idx, vector;
KASSERT(ipifunc != &IDTVEC(rsvd) && ipifunc != &IDTVEC(rsvd_pti),
@@ -2298,15 +2307,14 @@
vector = -1;
mtx_lock_spin(&icu_lock);
for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) {
- ip = &idt[idx];
- func = (ip->gd_hioffset << 16) | ip->gd_looffset;
-#ifdef __i386__
- func -= setidt_disp;
-#endif
- if ((!pti && func == (uintptr_t)&IDTVEC(rsvd)) ||
- (pti && func == (uintptr_t)&IDTVEC(rsvd_pti))) {
+ fip = fred_ipi_handlers[idx - IPI_DYN_FIRST];
+ if (fip == NULL) {
vector = idx;
- setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC);
+ fred_ipi_handlers[idx - IPI_DYN_FIRST] = fred_ipifunc;
+ if (!fred) {
+ setidt(vector, ipifunc, SDT_APIC, SEL_KPL,
+ GSEL_APIC);
+ }
break;
}
}
@@ -2324,15 +2332,20 @@
("%s: invalid vector %d", __func__, vector));
mtx_lock_spin(&icu_lock);
- ip = &idt[vector];
- func = (ip->gd_hioffset << 16) | ip->gd_looffset;
+ KASSERT(fred_ipi_handlers[vector - IPI_DYN_FIRST] != NULL,
+ ("NULL fred func %d", vector));
+ fred_ipi_handlers[vector - IPI_DYN_FIRST] = NULL;
+ if (!fred) {
+ ip = &idt[vector];
+ func = (ip->gd_hioffset << 16) | ip->gd_looffset;
#ifdef __i386__
- func -= setidt_disp;
+ func -= setidt_disp;
#endif
- KASSERT(func != (uintptr_t)&IDTVEC(rsvd) &&
- func != (uintptr_t)&IDTVEC(rsvd_pti),
- ("invalid idtfunc %#lx", func));
- setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC,
- SEL_KPL, GSEL_APIC);
+ KASSERT(func != (uintptr_t)&IDTVEC(rsvd) &&
+ func != (uintptr_t)&IDTVEC(rsvd_pti),
+ ("invalid idtfunc %d %#lx", vector, func));
+ setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd),
+ SDT_APIC, SEL_KPL, GSEL_APIC);
+ }
mtx_unlock_spin(&icu_lock);
}

File Metadata

Mime Type
text/plain
Expires
Wed, Apr 8, 6:22 AM (18 h, 19 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
31082303
Default Alt Text
D55829.id174054.diff (50 KB)

Event Timeline