Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/mp_machdep.c
Show First 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | |||||
#define BIOS_WARM (0x0a) | #define BIOS_WARM (0x0a) | ||||
extern struct pcpu __pcpu[]; | extern struct pcpu __pcpu[]; | ||||
/* Temporary variables for init_secondary() */ | /* Temporary variables for init_secondary() */ | ||||
char *doublefault_stack; | char *doublefault_stack; | ||||
char *nmi_stack; | char *nmi_stack; | ||||
extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); | extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32), | ||||
IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32_pti); | |||||
/* | /* | ||||
* Local data and functions. | * Local data and functions. | ||||
*/ | */ | ||||
static int start_ap(int apic_id); | static int start_ap(int apic_id); | ||||
static u_int bootMP_size; | static u_int bootMP_size; | ||||
Show All 28 Lines | cpu_mp_start(void) | ||||
for (i = 0; i < MAXCPU; i++) { | for (i = 0; i < MAXCPU; i++) { | ||||
cpu_apic_ids[i] = -1; | cpu_apic_ids[i] = -1; | ||||
cpu_ipi_pending[i] = 0; | cpu_ipi_pending[i] = 0; | ||||
} | } | ||||
/* Install an inter-CPU IPI for TLB invalidation */ | /* Install an inter-CPU IPI for TLB invalidation */ | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
if (invpcid_works) { | if (invpcid_works) { | ||||
setidt(IPI_INVLTLB, IDTVEC(invltlb_invpcid), | setidt(IPI_INVLTLB, pti ? IDTVEC(invltlb_invpcid_pti) : | ||||
SDT_SYSIGT, SEL_KPL, 0); | IDTVEC(invltlb_invpcid), SDT_SYSIGT, SEL_KPL, 0); | ||||
} else { | } else { | ||||
setidt(IPI_INVLTLB, IDTVEC(invltlb_pcid), SDT_SYSIGT, | setidt(IPI_INVLTLB, pti ? IDTVEC(invltlb_pcid_pti) : | ||||
SEL_KPL, 0); | IDTVEC(invltlb_pcid), SDT_SYSIGT, SEL_KPL, 0); | ||||
} | } | ||||
} else { | } else { | ||||
setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYSIGT, SEL_KPL, 0); | setidt(IPI_INVLTLB, pti ? IDTVEC(invltlb_pti) : IDTVEC(invltlb), | ||||
SDT_SYSIGT, SEL_KPL, 0); | |||||
} | } | ||||
setidt(IPI_INVLPG, IDTVEC(invlpg), SDT_SYSIGT, SEL_KPL, 0); | setidt(IPI_INVLPG, pti ? IDTVEC(invlpg_pti) : IDTVEC(invlpg), | ||||
setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0); | SDT_SYSIGT, SEL_KPL, 0); | ||||
setidt(IPI_INVLRNG, pti ? IDTVEC(invlrng_pti) : IDTVEC(invlrng), | |||||
SDT_SYSIGT, SEL_KPL, 0); | |||||
/* Install an inter-CPU IPI for cache invalidation. */ | /* Install an inter-CPU IPI for cache invalidation. */ | ||||
setidt(IPI_INVLCACHE, IDTVEC(invlcache), SDT_SYSIGT, SEL_KPL, 0); | setidt(IPI_INVLCACHE, pti ? IDTVEC(invlcache_pti) : IDTVEC(invlcache), | ||||
SDT_SYSIGT, SEL_KPL, 0); | |||||
/* Install an inter-CPU IPI for all-CPU rendezvous */ | /* Install an inter-CPU IPI for all-CPU rendezvous */ | ||||
setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0); | setidt(IPI_RENDEZVOUS, pti ? IDTVEC(rendezvous_pti) : | ||||
IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0); | |||||
/* Install generic inter-CPU IPI handler */ | /* Install generic inter-CPU IPI handler */ | ||||
setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler), | setidt(IPI_BITMAP_VECTOR, pti ? IDTVEC(ipi_intr_bitmap_handler_pti) : | ||||
SDT_SYSIGT, SEL_KPL, 0); | IDTVEC(ipi_intr_bitmap_handler), SDT_SYSIGT, SEL_KPL, 0); | ||||
/* Install an inter-CPU IPI for CPU stop/restart */ | /* Install an inter-CPU IPI for CPU stop/restart */ | ||||
setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0); | setidt(IPI_STOP, pti ? IDTVEC(cpustop_pti) : IDTVEC(cpustop), | ||||
SDT_SYSIGT, SEL_KPL, 0); | |||||
/* Install an inter-CPU IPI for CPU suspend/resume */ | /* Install an inter-CPU IPI for CPU suspend/resume */ | ||||
setidt(IPI_SUSPEND, IDTVEC(cpususpend), SDT_SYSIGT, SEL_KPL, 0); | setidt(IPI_SUSPEND, pti ? IDTVEC(cpususpend_pti) : IDTVEC(cpususpend), | ||||
SDT_SYSIGT, SEL_KPL, 0); | |||||
/* Set boot_cpu_id if needed. */ | /* Set boot_cpu_id if needed. */ | ||||
if (boot_cpu_id == -1) { | if (boot_cpu_id == -1) { | ||||
boot_cpu_id = PCPU_GET(apic_id); | boot_cpu_id = PCPU_GET(apic_id); | ||||
cpu_info[boot_cpu_id].cpu_bsp = 1; | cpu_info[boot_cpu_id].cpu_bsp = 1; | ||||
} else | } else | ||||
KASSERT(boot_cpu_id == PCPU_GET(apic_id), | KASSERT(boot_cpu_id == PCPU_GET(apic_id), | ||||
("BSP's APIC ID doesn't match boot_cpu_id")); | ("BSP's APIC ID doesn't match boot_cpu_id")); | ||||
Show All 22 Lines | init_secondary(void) | ||||
int cpu, gsel_tss, x; | int cpu, gsel_tss, x; | ||||
struct region_descriptor ap_gdt; | struct region_descriptor ap_gdt; | ||||
/* Set by the startup code for us to use */ | /* Set by the startup code for us to use */ | ||||
cpu = bootAP; | cpu = bootAP; | ||||
/* Init tss */ | /* Init tss */ | ||||
common_tss[cpu] = common_tss[0]; | common_tss[cpu] = common_tss[0]; | ||||
common_tss[cpu].tss_rsp0 = 0; /* not used until after switch */ | |||||
common_tss[cpu].tss_iobase = sizeof(struct amd64tss) + | common_tss[cpu].tss_iobase = sizeof(struct amd64tss) + | ||||
IOPERM_BITMAP_SIZE; | IOPERM_BITMAP_SIZE; | ||||
common_tss[cpu].tss_ist1 = (long)&doublefault_stack[PAGE_SIZE]; | common_tss[cpu].tss_ist1 = (long)&doublefault_stack[PAGE_SIZE]; | ||||
/* The NMI stack runs on IST2. */ | /* The NMI stack runs on IST2. */ | ||||
np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; | np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; | ||||
common_tss[cpu].tss_ist2 = (long) np; | common_tss[cpu].tss_ist2 = (long) np; | ||||
Show All 26 Lines | pc->pc_tss = (struct system_segment_descriptor *)&gdt[NGDT * cpu + | ||||
GPROC0_SEL]; | GPROC0_SEL]; | ||||
pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL]; | pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL]; | ||||
pc->pc_gs32p = &gdt[NGDT * cpu + GUGS32_SEL]; | pc->pc_gs32p = &gdt[NGDT * cpu + GUGS32_SEL]; | ||||
pc->pc_ldt = (struct system_segment_descriptor *)&gdt[NGDT * cpu + | pc->pc_ldt = (struct system_segment_descriptor *)&gdt[NGDT * cpu + | ||||
GUSERLDT_SEL]; | GUSERLDT_SEL]; | ||||
pc->pc_curpmap = kernel_pmap; | pc->pc_curpmap = kernel_pmap; | ||||
pc->pc_pcid_gen = 1; | pc->pc_pcid_gen = 1; | ||||
pc->pc_pcid_next = PMAP_PCID_KERN + 1; | pc->pc_pcid_next = PMAP_PCID_KERN + 1; | ||||
common_tss[cpu].tss_rsp0 = pti ? ((vm_offset_t)&pc->pc_pti_stack + | |||||
PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : 0; | |||||
/* Save the per-cpu pointer for use by the NMI handler. */ | /* Save the per-cpu pointer for use by the NMI handler. */ | ||||
np->np_pcpu = (register_t) pc; | np->np_pcpu = (register_t) pc; | ||||
wrmsr(MSR_FSBASE, 0); /* User value */ | wrmsr(MSR_FSBASE, 0); /* User value */ | ||||
wrmsr(MSR_GSBASE, (u_int64_t)pc); | wrmsr(MSR_GSBASE, (u_int64_t)pc); | ||||
wrmsr(MSR_KGSBASE, (u_int64_t)pc); /* XXX User value while we're in the kernel */ | wrmsr(MSR_KGSBASE, (u_int64_t)pc); /* XXX User value while we're in the kernel */ | ||||
fix_cpuid(); | fix_cpuid(); | ||||
Show All 10 Lines | init_secondary(void) | ||||
*/ | */ | ||||
cr0 = rcr0(); | cr0 = rcr0(); | ||||
cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); | cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); | ||||
load_cr0(cr0); | load_cr0(cr0); | ||||
/* Set up the fast syscall stuff */ | /* Set up the fast syscall stuff */ | ||||
msr = rdmsr(MSR_EFER) | EFER_SCE; | msr = rdmsr(MSR_EFER) | EFER_SCE; | ||||
wrmsr(MSR_EFER, msr); | wrmsr(MSR_EFER, msr); | ||||
wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall)); | wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) : | ||||
(u_int64_t)IDTVEC(fast_syscall)); | |||||
wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32)); | wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32)); | ||||
msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | | msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | | ||||
((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48); | ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48); | ||||
wrmsr(MSR_STAR, msr); | wrmsr(MSR_STAR, msr); | ||||
wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D); | wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D); | ||||
/* signal our startup to the BSP. */ | /* signal our startup to the BSP. */ | ||||
mp_naps++; | mp_naps++; | ||||
▲ Show 20 Lines • Show All 182 Lines • Show Last 20 Lines |