Differential D13797 Diff 37676 sys/amd64/amd64/machdep.c

Changeset View

Standalone View

sys/amd64/amd64/machdep.c

Show First 20 Lines • Show All 652 Lines • ▼ Show 20 Lines
/*		/*
* Initialize segments & interrupt table		* Initialize segments & interrupt table
*/		*/

struct user_segment_descriptor gdt[NGDT * MAXCPU];/* global descriptor tables */		struct user_segment_descriptor gdt[NGDT * MAXCPU];/* global descriptor tables */
static struct gate_descriptor idt0[NIDT];		static struct gate_descriptor idt0[NIDT];
struct gate_descriptor idt = &idt0[0]; / interrupt descriptor table */		struct gate_descriptor idt = &idt0[0]; / interrupt descriptor table */

static char dblfault_stack[PAGE_SIZE] __aligned(16);		char dblfault_stack[PAGE_SIZE] __aligned(16);

static char nmi0_stack[PAGE_SIZE] __aligned(16);		char nmi0_stack[PAGE_SIZE] __aligned(16);
CTASSERT(sizeof(struct nmi_pcpu) == 16);		CTASSERT(sizeof(struct nmi_pcpu) == 16);

struct amd64tss common_tss[MAXCPU];		struct amd64tss common_tss[MAXCPU];

/*		/*
* Software prototypes -- in more palatable form.		* Software prototypes -- in more palatable form.
*		*
* Keep GUFS32, GUGS32, GUCODE32 and GUDATA at the same		* Keep GUFS32, GUGS32, GUCODE32 and GUDATA at the same
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines
}		}

extern inthand_t		extern inthand_t
IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),		IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),		IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),		IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),		IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
IDTVEC(xmm), IDTVEC(dblfault),		IDTVEC(xmm), IDTVEC(dblfault),
		IDTVEC(div_pti), IDTVEC(dbg_pti), IDTVEC(nmi_pti), IDTVEC(bpt_pti),
		IDTVEC(ofl_pti), IDTVEC(bnd_pti), IDTVEC(ill_pti), IDTVEC(dna_pti),
		IDTVEC(fpusegm_pti), IDTVEC(tss_pti), IDTVEC(missing_pti),
		IDTVEC(stk_pti), IDTVEC(prot_pti), IDTVEC(page_pti), IDTVEC(mchk_pti),
		IDTVEC(rsvd_pti), IDTVEC(fpu_pti), IDTVEC(align_pti),
		IDTVEC(xmm_pti), IDTVEC(dblfault_pti),
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
IDTVEC(dtrace_ret),		IDTVEC(dtrace_ret), IDTVEC(dtrace_ret_pti),
#endif		#endif
#ifdef XENHVM		#ifdef XENHVM
IDTVEC(xen_intr_upcall),		IDTVEC(xen_intr_upcall), IDTVEC(xen_intr_upcall_pti),
#endif		#endif
IDTVEC(fast_syscall), IDTVEC(fast_syscall32);		IDTVEC(fast_syscall), IDTVEC(fast_syscall32),
		IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32_pti);

#ifdef DDB		#ifdef DDB
/*		/*
* Display the index and function name of any IDT entries that don't use		* Display the index and function name of any IDT entries that don't use
* the default 'rsvd' entry point.		* the default 'rsvd' entry point.
*/		*/
DB_SHOW_COMMAND(idt, db_show_idt)		DB_SHOW_COMMAND(idt, db_show_idt)
{		{
▲ Show 20 Lines • Show All 684 Lines • ▼ Show 20 Lines
u_int64_t		u_int64_t
hammer_time(u_int64_t modulep, u_int64_t physfree)		hammer_time(u_int64_t modulep, u_int64_t physfree)
{		{
caddr_t kmdp;		caddr_t kmdp;
int gsel_tss, x;		int gsel_tss, x;
struct pcpu *pc;		struct pcpu *pc;
struct nmi_pcpu *np;		struct nmi_pcpu *np;
struct xstate_hdr *xhdr;		struct xstate_hdr *xhdr;
u_int64_t msr;		u_int64_t msr, rsp0;
char *env;		char *env;
size_t kstack0_sz;		size_t kstack0_sz;
int late_console;		int late_console;

TSRAW(&thread0, TS_ENTER, __func__, NULL);		TSRAW(&thread0, TS_ENTER, __func__, NULL);

/*		/*
* This may be done better later if it gets more high level		* This may be done better later if it gets more high level
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines	hammer_time(u_int64_t modulep, u_int64_t physfree)
* must be able to get the icu lock, so it can't be		* must be able to get the icu lock, so it can't be
* under witness.		* under witness.
*/		*/
mutex_init();		mutex_init();
mtx_init(&icu_lock, "icu", NULL, MTX_SPIN \| MTX_NOWITNESS);		mtx_init(&icu_lock, "icu", NULL, MTX_SPIN \| MTX_NOWITNESS);
mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF);		mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF);

/* exceptions */		/* exceptions */
		TUNABLE_INT_FETCH("vm.pmap.pti", &pti);

for (x = 0; x < NIDT; x++)		for (x = 0; x < NIDT; x++)
setidt(x, &IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0);		setidt(x, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_SYSIGT,
setidt(IDT_DE, &IDTVEC(div), SDT_SYSIGT, SEL_KPL, 0);		SEL_KPL, 0);
setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 0);		setidt(IDT_DE, pti ? &IDTVEC(div_pti) : &IDTVEC(div), SDT_SYSIGT,
		SEL_KPL, 0);
		setidt(IDT_DB, pti ? &IDTVEC(dbg_pti) : &IDTVEC(dbg), SDT_SYSIGT,
		SEL_KPL, 0);
setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2);		setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2);
setidt(IDT_BP, &IDTVEC(bpt), SDT_SYSIGT, SEL_UPL, 0);		setidt(IDT_BP, pti ? &IDTVEC(bpt_pti) : &IDTVEC(bpt), SDT_SYSIGT,
setidt(IDT_OF, &IDTVEC(ofl), SDT_SYSIGT, SEL_KPL, 0);		SEL_UPL, 0);
setidt(IDT_BR, &IDTVEC(bnd), SDT_SYSIGT, SEL_KPL, 0);		setidt(IDT_OF, pti ? &IDTVEC(ofl_pti) : &IDTVEC(ofl), SDT_SYSIGT,
setidt(IDT_UD, &IDTVEC(ill), SDT_SYSIGT, SEL_KPL, 0);		SEL_KPL, 0);
setidt(IDT_NM, &IDTVEC(dna), SDT_SYSIGT, SEL_KPL, 0);		setidt(IDT_BR, pti ? &IDTVEC(bnd_pti) : &IDTVEC(bnd), SDT_SYSIGT,
		SEL_KPL, 0);
		setidt(IDT_UD, pti ? &IDTVEC(ill_pti) : &IDTVEC(ill), SDT_SYSIGT,
		SEL_KPL, 0);
		setidt(IDT_NM, pti ? &IDTVEC(dna_pti) : &IDTVEC(dna), SDT_SYSIGT,
		SEL_KPL, 0);
setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);		setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
setidt(IDT_FPUGP, &IDTVEC(fpusegm), SDT_SYSIGT, SEL_KPL, 0);		setidt(IDT_FPUGP, pti ? &IDTVEC(fpusegm_pti) : &IDTVEC(fpusegm),
setidt(IDT_TS, &IDTVEC(tss), SDT_SYSIGT, SEL_KPL, 0);		SDT_SYSIGT, SEL_KPL, 0);
setidt(IDT_NP, &IDTVEC(missing), SDT_SYSIGT, SEL_KPL, 0);		setidt(IDT_TS, pti ? &IDTVEC(tss_pti) : &IDTVEC(tss), SDT_SYSIGT,
setidt(IDT_SS, &IDTVEC(stk), SDT_SYSIGT, SEL_KPL, 0);		SEL_KPL, 0);
setidt(IDT_GP, &IDTVEC(prot), SDT_SYSIGT, SEL_KPL, 0);		setidt(IDT_NP, pti ? &IDTVEC(missing_pti) : &IDTVEC(missing),
setidt(IDT_PF, &IDTVEC(page), SDT_SYSIGT, SEL_KPL, 0);		SDT_SYSIGT, SEL_KPL, 0);
setidt(IDT_MF, &IDTVEC(fpu), SDT_SYSIGT, SEL_KPL, 0);		setidt(IDT_SS, pti ? &IDTVEC(stk_pti) : &IDTVEC(stk), SDT_SYSIGT,
setidt(IDT_AC, &IDTVEC(align), SDT_SYSIGT, SEL_KPL, 0);		SEL_KPL, 0);
setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 0);		setidt(IDT_GP, pti ? &IDTVEC(prot_pti) : &IDTVEC(prot), SDT_SYSIGT,
setidt(IDT_XF, &IDTVEC(xmm), SDT_SYSIGT, SEL_KPL, 0);		SEL_KPL, 0);
		setidt(IDT_PF, pti ? &IDTVEC(page_pti) : &IDTVEC(page), SDT_SYSIGT,
		SEL_KPL, 0);
		setidt(IDT_MF, pti ? &IDTVEC(fpu_pti) : &IDTVEC(fpu), SDT_SYSIGT,
		SEL_KPL, 0);
		setidt(IDT_AC, pti ? &IDTVEC(align_pti) : &IDTVEC(align), SDT_SYSIGT,
		SEL_KPL, 0);
		setidt(IDT_MC, pti ? &IDTVEC(mchk_pti) : &IDTVEC(mchk), SDT_SYSIGT,
		SEL_KPL, 0);
		setidt(IDT_XF, pti ? &IDTVEC(xmm_pti) : &IDTVEC(xmm), SDT_SYSIGT,
		SEL_KPL, 0);
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
setidt(IDT_DTRACE_RET, &IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0);		setidt(IDT_DTRACE_RET, pti ? &IDTVEC(dtrace_ret_pti) :
		&IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0);
#endif		#endif
#ifdef XENHVM		#ifdef XENHVM
setidt(IDT_EVTCHN, &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_UPL, 0);		setidt(IDT_EVTCHN, &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_UPL, 0);
#endif		#endif

r_idt.rd_limit = sizeof(idt0) - 1;		r_idt.rd_limit = sizeof(idt0) - 1;
r_idt.rd_base = (long) idt;		r_idt.rd_base = (long) idt;
lidt(&r_idt);		lidt(&r_idt);

/*		/*
* Initialize the clock before the console so that console		* Initialize the clock before the console so that console
* initialization can use DELAY().		* initialization can use DELAY().
*/		*/
Show All 28 Lines	#endif
common_tss[0].tss_iobase = sizeof(struct amd64tss) + IOPERM_BITMAP_SIZE;		common_tss[0].tss_iobase = sizeof(struct amd64tss) + IOPERM_BITMAP_SIZE;

gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);		gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
ltr(gsel_tss);		ltr(gsel_tss);

/* Set up the fast syscall stuff */		/* Set up the fast syscall stuff */
msr = rdmsr(MSR_EFER) \| EFER_SCE;		msr = rdmsr(MSR_EFER) \| EFER_SCE;
wrmsr(MSR_EFER, msr);		wrmsr(MSR_EFER, msr);
wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall));		wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
		(u_int64_t)IDTVEC(fast_syscall));
wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));		wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) \|		msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) \|
((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);		((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
wrmsr(MSR_STAR, msr);		wrmsr(MSR_STAR, msr);
wrmsr(MSR_SF_MASK, PSL_NT\|PSL_T\|PSL_I\|PSL_C\|PSL_D);		wrmsr(MSR_SF_MASK, PSL_NT\|PSL_T\|PSL_I\|PSL_C\|PSL_D);

/*		/*
* Temporary forge some valid pointer to PCB, for exception		* Temporary forge some valid pointer to PCB, for exception
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines	#endif
thread0.td_pcb->pcb_save = get_pcb_user_save_td(&thread0);		thread0.td_pcb->pcb_save = get_pcb_user_save_td(&thread0);
bzero(get_pcb_user_save_td(&thread0), cpu_max_ext_state_size);		bzero(get_pcb_user_save_td(&thread0), cpu_max_ext_state_size);
if (use_xsave) {		if (use_xsave) {
xhdr = (struct xstate_hdr *)(get_pcb_user_save_td(&thread0) +		xhdr = (struct xstate_hdr *)(get_pcb_user_save_td(&thread0) +
1);		1);
xhdr->xstate_bv = xsave_mask;		xhdr->xstate_bv = xsave_mask;
}		}
/* make an initial tss so cpu can get interrupt stack on syscall! */		/* make an initial tss so cpu can get interrupt stack on syscall! */
common_tss[0].tss_rsp0 = (vm_offset_t)thread0.td_pcb;		rsp0 = (vm_offset_t)thread0.td_pcb;
/* Ensure the stack is aligned to 16 bytes */		/* Ensure the stack is aligned to 16 bytes */
common_tss[0].tss_rsp0 &= ~0xFul;		rsp0 &= ~0xFul;
PCPU_SET(rsp0, common_tss[0].tss_rsp0);		common_tss[0].tss_rsp0 = pti ? ((vm_offset_t)PCPU_PTR(pti_stack) +
		PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : rsp0;
		kithrup_mac.comUnsubmitted Not Done Inline Actions Would it make it clearer to set rsp0 in an if/else there, rather than relying on the ?: syntax? kithrup_mac.com: Would it make it clearer to set rsp0 in an if/else there, rather than relying on the ?: syntax?
		kibAuthorUnsubmitted Not Done Inline Actions I prefer the ?:. kib: I prefer the ?:.
		PCPU_SET(rsp0, rsp0);
PCPU_SET(curpcb, thread0.td_pcb);		PCPU_SET(curpcb, thread0.td_pcb);

/* transfer to user mode */		/* transfer to user mode */

_ucodesel = GSEL(GUCODE_SEL, SEL_UPL);		_ucodesel = GSEL(GUCODE_SEL, SEL_UPL);
_udatasel = GSEL(GUDATA_SEL, SEL_UPL);		_udatasel = GSEL(GUDATA_SEL, SEL_UPL);
_ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);		_ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);
_ufssel = GSEL(GUFS32_SEL, SEL_UPL);		_ufssel = GSEL(GUFS32_SEL, SEL_UPL);
▲ Show 20 Lines • Show All 814 Lines • Show Last 20 Lines