Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/machdep.c
Show First 20 Lines • Show All 652 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Initialize segments & interrupt table | * Initialize segments & interrupt table | ||||
*/ | */ | ||||
struct user_segment_descriptor gdt[NGDT * MAXCPU];/* global descriptor tables */ | struct user_segment_descriptor gdt[NGDT * MAXCPU];/* global descriptor tables */ | ||||
static struct gate_descriptor idt0[NIDT]; | static struct gate_descriptor idt0[NIDT]; | ||||
struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */ | struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */ | ||||
static char dblfault_stack[PAGE_SIZE] __aligned(16); | char dblfault_stack[PAGE_SIZE] __aligned(16); | ||||
static char nmi0_stack[PAGE_SIZE] __aligned(16); | char nmi0_stack[PAGE_SIZE] __aligned(16); | ||||
CTASSERT(sizeof(struct nmi_pcpu) == 16); | CTASSERT(sizeof(struct nmi_pcpu) == 16); | ||||
struct amd64tss common_tss[MAXCPU]; | struct amd64tss common_tss[MAXCPU]; | ||||
/* | /* | ||||
* Software prototypes -- in more palatable form. | * Software prototypes -- in more palatable form. | ||||
* | * | ||||
* Keep GUFS32, GUGS32, GUCODE32 and GUDATA at the same | * Keep GUFS32, GUGS32, GUCODE32 and GUDATA at the same | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
extern inthand_t | extern inthand_t | ||||
IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl), | IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl), | ||||
IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm), | IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm), | ||||
IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot), | IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot), | ||||
IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align), | IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align), | ||||
IDTVEC(xmm), IDTVEC(dblfault), | IDTVEC(xmm), IDTVEC(dblfault), | ||||
IDTVEC(div_pti), IDTVEC(dbg_pti), IDTVEC(nmi_pti), IDTVEC(bpt_pti), | |||||
IDTVEC(ofl_pti), IDTVEC(bnd_pti), IDTVEC(ill_pti), IDTVEC(dna_pti), | |||||
IDTVEC(fpusegm_pti), IDTVEC(tss_pti), IDTVEC(missing_pti), | |||||
IDTVEC(stk_pti), IDTVEC(prot_pti), IDTVEC(page_pti), IDTVEC(mchk_pti), | |||||
IDTVEC(rsvd_pti), IDTVEC(fpu_pti), IDTVEC(align_pti), | |||||
IDTVEC(xmm_pti), IDTVEC(dblfault_pti), | |||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
IDTVEC(dtrace_ret), | IDTVEC(dtrace_ret), IDTVEC(dtrace_ret_pti), | ||||
#endif | #endif | ||||
#ifdef XENHVM | #ifdef XENHVM | ||||
IDTVEC(xen_intr_upcall), | IDTVEC(xen_intr_upcall), IDTVEC(xen_intr_upcall_pti), | ||||
#endif | #endif | ||||
IDTVEC(fast_syscall), IDTVEC(fast_syscall32); | IDTVEC(fast_syscall), IDTVEC(fast_syscall32), | ||||
IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32_pti); | |||||
#ifdef DDB | #ifdef DDB | ||||
/* | /* | ||||
* Display the index and function name of any IDT entries that don't use | * Display the index and function name of any IDT entries that don't use | ||||
* the default 'rsvd' entry point. | * the default 'rsvd' entry point. | ||||
*/ | */ | ||||
DB_SHOW_COMMAND(idt, db_show_idt) | DB_SHOW_COMMAND(idt, db_show_idt) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 684 Lines • ▼ Show 20 Lines | |||||
/* Set up the fast syscall stuff */ | /* Set up the fast syscall stuff */ | ||||
void | void | ||||
amd64_conf_fast_syscall(void) | amd64_conf_fast_syscall(void) | ||||
{ | { | ||||
uint64_t msr; | uint64_t msr; | ||||
msr = rdmsr(MSR_EFER) | EFER_SCE; | msr = rdmsr(MSR_EFER) | EFER_SCE; | ||||
wrmsr(MSR_EFER, msr); | wrmsr(MSR_EFER, msr); | ||||
wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall)); | wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) : | ||||
(u_int64_t)IDTVEC(fast_syscall)); | |||||
wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32)); | wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32)); | ||||
msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | | msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | | ||||
((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48); | ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48); | ||||
wrmsr(MSR_STAR, msr); | wrmsr(MSR_STAR, msr); | ||||
wrmsr(MSR_SF_MASK, PSL_NT | PSL_T | PSL_I | PSL_C | PSL_D); | wrmsr(MSR_SF_MASK, PSL_NT | PSL_T | PSL_I | PSL_C | PSL_D); | ||||
} | } | ||||
u_int64_t | u_int64_t | ||||
hammer_time(u_int64_t modulep, u_int64_t physfree) | hammer_time(u_int64_t modulep, u_int64_t physfree) | ||||
{ | { | ||||
caddr_t kmdp; | caddr_t kmdp; | ||||
int gsel_tss, x; | int gsel_tss, x; | ||||
struct pcpu *pc; | struct pcpu *pc; | ||||
struct nmi_pcpu *np; | struct nmi_pcpu *np; | ||||
struct xstate_hdr *xhdr; | struct xstate_hdr *xhdr; | ||||
u_int64_t rsp0; | |||||
char *env; | char *env; | ||||
size_t kstack0_sz; | size_t kstack0_sz; | ||||
int late_console; | int late_console; | ||||
TSRAW(&thread0, TS_ENTER, __func__, NULL); | TSRAW(&thread0, TS_ENTER, __func__, NULL); | ||||
/* | /* | ||||
* This may be done better later if it gets more high level | * This may be done better later if it gets more high level | ||||
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | hammer_time(u_int64_t modulep, u_int64_t physfree) | ||||
* must be able to get the icu lock, so it can't be | * must be able to get the icu lock, so it can't be | ||||
* under witness. | * under witness. | ||||
*/ | */ | ||||
mutex_init(); | mutex_init(); | ||||
mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS); | mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS); | ||||
mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF); | mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF); | ||||
/* exceptions */ | /* exceptions */ | ||||
TUNABLE_INT_FETCH("vm.pmap.pti", &pti); | |||||
for (x = 0; x < NIDT; x++) | for (x = 0; x < NIDT; x++) | ||||
setidt(x, &IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0); | setidt(x, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_SYSIGT, | ||||
setidt(IDT_DE, &IDTVEC(div), SDT_SYSIGT, SEL_KPL, 0); | SEL_KPL, 0); | ||||
setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 0); | setidt(IDT_DE, pti ? &IDTVEC(div_pti) : &IDTVEC(div), SDT_SYSIGT, | ||||
SEL_KPL, 0); | |||||
setidt(IDT_DB, pti ? &IDTVEC(dbg_pti) : &IDTVEC(dbg), SDT_SYSIGT, | |||||
SEL_KPL, 0); | |||||
setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2); | setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2); | ||||
setidt(IDT_BP, &IDTVEC(bpt), SDT_SYSIGT, SEL_UPL, 0); | setidt(IDT_BP, pti ? &IDTVEC(bpt_pti) : &IDTVEC(bpt), SDT_SYSIGT, | ||||
setidt(IDT_OF, &IDTVEC(ofl), SDT_SYSIGT, SEL_KPL, 0); | SEL_UPL, 0); | ||||
setidt(IDT_BR, &IDTVEC(bnd), SDT_SYSIGT, SEL_KPL, 0); | setidt(IDT_OF, pti ? &IDTVEC(ofl_pti) : &IDTVEC(ofl), SDT_SYSIGT, | ||||
setidt(IDT_UD, &IDTVEC(ill), SDT_SYSIGT, SEL_KPL, 0); | SEL_KPL, 0); | ||||
setidt(IDT_NM, &IDTVEC(dna), SDT_SYSIGT, SEL_KPL, 0); | setidt(IDT_BR, pti ? &IDTVEC(bnd_pti) : &IDTVEC(bnd), SDT_SYSIGT, | ||||
SEL_KPL, 0); | |||||
setidt(IDT_UD, pti ? &IDTVEC(ill_pti) : &IDTVEC(ill), SDT_SYSIGT, | |||||
SEL_KPL, 0); | |||||
setidt(IDT_NM, pti ? &IDTVEC(dna_pti) : &IDTVEC(dna), SDT_SYSIGT, | |||||
SEL_KPL, 0); | |||||
setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1); | setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1); | ||||
setidt(IDT_FPUGP, &IDTVEC(fpusegm), SDT_SYSIGT, SEL_KPL, 0); | setidt(IDT_FPUGP, pti ? &IDTVEC(fpusegm_pti) : &IDTVEC(fpusegm), | ||||
setidt(IDT_TS, &IDTVEC(tss), SDT_SYSIGT, SEL_KPL, 0); | SDT_SYSIGT, SEL_KPL, 0); | ||||
setidt(IDT_NP, &IDTVEC(missing), SDT_SYSIGT, SEL_KPL, 0); | setidt(IDT_TS, pti ? &IDTVEC(tss_pti) : &IDTVEC(tss), SDT_SYSIGT, | ||||
setidt(IDT_SS, &IDTVEC(stk), SDT_SYSIGT, SEL_KPL, 0); | SEL_KPL, 0); | ||||
setidt(IDT_GP, &IDTVEC(prot), SDT_SYSIGT, SEL_KPL, 0); | setidt(IDT_NP, pti ? &IDTVEC(missing_pti) : &IDTVEC(missing), | ||||
setidt(IDT_PF, &IDTVEC(page), SDT_SYSIGT, SEL_KPL, 0); | SDT_SYSIGT, SEL_KPL, 0); | ||||
setidt(IDT_MF, &IDTVEC(fpu), SDT_SYSIGT, SEL_KPL, 0); | setidt(IDT_SS, pti ? &IDTVEC(stk_pti) : &IDTVEC(stk), SDT_SYSIGT, | ||||
setidt(IDT_AC, &IDTVEC(align), SDT_SYSIGT, SEL_KPL, 0); | SEL_KPL, 0); | ||||
setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 0); | setidt(IDT_GP, pti ? &IDTVEC(prot_pti) : &IDTVEC(prot), SDT_SYSIGT, | ||||
setidt(IDT_XF, &IDTVEC(xmm), SDT_SYSIGT, SEL_KPL, 0); | SEL_KPL, 0); | ||||
setidt(IDT_PF, pti ? &IDTVEC(page_pti) : &IDTVEC(page), SDT_SYSIGT, | |||||
SEL_KPL, 0); | |||||
setidt(IDT_MF, pti ? &IDTVEC(fpu_pti) : &IDTVEC(fpu), SDT_SYSIGT, | |||||
SEL_KPL, 0); | |||||
setidt(IDT_AC, pti ? &IDTVEC(align_pti) : &IDTVEC(align), SDT_SYSIGT, | |||||
SEL_KPL, 0); | |||||
setidt(IDT_MC, pti ? &IDTVEC(mchk_pti) : &IDTVEC(mchk), SDT_SYSIGT, | |||||
SEL_KPL, 0); | |||||
setidt(IDT_XF, pti ? &IDTVEC(xmm_pti) : &IDTVEC(xmm), SDT_SYSIGT, | |||||
SEL_KPL, 0); | |||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
setidt(IDT_DTRACE_RET, &IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0); | setidt(IDT_DTRACE_RET, pti ? &IDTVEC(dtrace_ret_pti) : | ||||
&IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0); | |||||
#endif | #endif | ||||
#ifdef XENHVM | #ifdef XENHVM | ||||
setidt(IDT_EVTCHN, &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_UPL, 0); | setidt(IDT_EVTCHN, &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_UPL, 0); | ||||
#endif | #endif | ||||
r_idt.rd_limit = sizeof(idt0) - 1; | r_idt.rd_limit = sizeof(idt0) - 1; | ||||
r_idt.rd_base = (long) idt; | r_idt.rd_base = (long) idt; | ||||
lidt(&r_idt); | lidt(&r_idt); | ||||
/* | /* | ||||
* Initialize the clock before the console so that console | * Initialize the clock before the console so that console | ||||
* initialization can use DELAY(). | * initialization can use DELAY(). | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines | #endif | ||||
thread0.td_pcb->pcb_save = get_pcb_user_save_td(&thread0); | thread0.td_pcb->pcb_save = get_pcb_user_save_td(&thread0); | ||||
bzero(get_pcb_user_save_td(&thread0), cpu_max_ext_state_size); | bzero(get_pcb_user_save_td(&thread0), cpu_max_ext_state_size); | ||||
if (use_xsave) { | if (use_xsave) { | ||||
xhdr = (struct xstate_hdr *)(get_pcb_user_save_td(&thread0) + | xhdr = (struct xstate_hdr *)(get_pcb_user_save_td(&thread0) + | ||||
1); | 1); | ||||
xhdr->xstate_bv = xsave_mask; | xhdr->xstate_bv = xsave_mask; | ||||
} | } | ||||
/* make an initial tss so cpu can get interrupt stack on syscall! */ | /* make an initial tss so cpu can get interrupt stack on syscall! */ | ||||
common_tss[0].tss_rsp0 = (vm_offset_t)thread0.td_pcb; | rsp0 = (vm_offset_t)thread0.td_pcb; | ||||
/* Ensure the stack is aligned to 16 bytes */ | /* Ensure the stack is aligned to 16 bytes */ | ||||
common_tss[0].tss_rsp0 &= ~0xFul; | rsp0 &= ~0xFul; | ||||
PCPU_SET(rsp0, common_tss[0].tss_rsp0); | common_tss[0].tss_rsp0 = pti ? ((vm_offset_t)PCPU_PTR(pti_stack) + | ||||
PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : rsp0; | |||||
kithrup_mac.com: Would it make it clearer to set rsp0 in an if/else there, rather than relying on the ?: syntax? | |||||
Not Done Inline ActionsI prefer the ?:. kib: I prefer the ?:. | |||||
PCPU_SET(rsp0, rsp0); | |||||
PCPU_SET(curpcb, thread0.td_pcb); | PCPU_SET(curpcb, thread0.td_pcb); | ||||
/* transfer to user mode */ | /* transfer to user mode */ | ||||
_ucodesel = GSEL(GUCODE_SEL, SEL_UPL); | _ucodesel = GSEL(GUCODE_SEL, SEL_UPL); | ||||
_udatasel = GSEL(GUDATA_SEL, SEL_UPL); | _udatasel = GSEL(GUDATA_SEL, SEL_UPL); | ||||
_ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL); | _ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL); | ||||
_ufssel = GSEL(GUFS32_SEL, SEL_UPL); | _ufssel = GSEL(GUFS32_SEL, SEL_UPL); | ||||
▲ Show 20 Lines • Show All 814 Lines • Show Last 20 Lines |
Would it make it clearer to set rsp0 in an if/else there, rather than relying on the ?: syntax?