Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/mp_machdep.c
Show First 20 Lines • Show All 277 Lines • ▼ Show 20 Lines | init_secondary(void) | ||||
pc->pc_rsp0 = 0; | pc->pc_rsp0 = 0; | ||||
pc->pc_pti_rsp0 = (((vm_offset_t)&pc->pc_pti_stack + | pc->pc_pti_rsp0 = (((vm_offset_t)&pc->pc_pti_stack + | ||||
PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful); | PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful); | ||||
gdt = pc->pc_gdt; | gdt = pc->pc_gdt; | ||||
pc->pc_tss = (struct system_segment_descriptor *)&gdt[GPROC0_SEL]; | pc->pc_tss = (struct system_segment_descriptor *)&gdt[GPROC0_SEL]; | ||||
pc->pc_fs32p = &gdt[GUFS32_SEL]; | pc->pc_fs32p = &gdt[GUFS32_SEL]; | ||||
pc->pc_gs32p = &gdt[GUGS32_SEL]; | pc->pc_gs32p = &gdt[GUGS32_SEL]; | ||||
pc->pc_ldt = (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]; | pc->pc_ldt = (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]; | ||||
pc->pc_ucr3_load_mask = PMAP_UCR3_NOMASK; | |||||
/* See comment in pmap_bootstrap(). */ | /* See comment in pmap_bootstrap(). */ | ||||
pc->pc_pcid_next = PMAP_PCID_KERN + 2; | pc->pc_pcid_next = PMAP_PCID_KERN + 2; | ||||
pc->pc_pcid_gen = 1; | pc->pc_pcid_gen = 1; | ||||
pc->pc_smp_tlb_gen = 1; | pc->pc_smp_tlb_gen = 1; | ||||
/* Init tss */ | /* Init tss */ | ||||
pc->pc_common_tss = __pcpu[0].pc_common_tss; | pc->pc_common_tss = __pcpu[0].pc_common_tss; | ||||
▲ Show 20 Lines • Show All 522 Lines • ▼ Show 20 Lines | if (smp_tlb_pmap == kernel_pmap) { | ||||
* This invalidation actually needs to clear kernel | * This invalidation actually needs to clear kernel | ||||
* mappings from the TLB in the current pmap, but | * mappings from the TLB in the current pmap, but | ||||
* since we were asked for the flush in the kernel | * since we were asked for the flush in the kernel | ||||
* pmap, achieve it by performing global flush. | * pmap, achieve it by performing global flush. | ||||
*/ | */ | ||||
invpcid(&d, INVPCID_CTXGLOB); | invpcid(&d, INVPCID_CTXGLOB); | ||||
} else { | } else { | ||||
invpcid(&d, INVPCID_CTX); | invpcid(&d, INVPCID_CTX); | ||||
d.pcid |= PMAP_PCID_USER_PT; | if (smp_tlb_pmap == PCPU_GET(curpmap)) | ||||
invpcid(&d, INVPCID_CTX); | PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
invltlb_pcid_handler(pmap_t smp_tlb_pmap) | invltlb_pcid_handler(pmap_t smp_tlb_pmap) | ||||
{ | { | ||||
uint64_t kcr3, ucr3; | |||||
uint32_t pcid; | uint32_t pcid; | ||||
#ifdef COUNT_XINVLTLB_HITS | #ifdef COUNT_XINVLTLB_HITS | ||||
xhits_gbl[PCPU_GET(cpuid)]++; | xhits_gbl[PCPU_GET(cpuid)]++; | ||||
#endif /* COUNT_XINVLTLB_HITS */ | #endif /* COUNT_XINVLTLB_HITS */ | ||||
#ifdef COUNT_IPIS | #ifdef COUNT_IPIS | ||||
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++; | (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; | ||||
#endif /* COUNT_IPIS */ | #endif /* COUNT_IPIS */ | ||||
if (smp_tlb_pmap == kernel_pmap) { | if (smp_tlb_pmap == kernel_pmap) { | ||||
invltlb_glob(); | invltlb_glob(); | ||||
} else { | } else { | ||||
/* | /* | ||||
* The current pmap might not be equal to | * The current pmap might not be equal to | ||||
* smp_tlb_pmap. The clearing of the pm_gen in | * smp_tlb_pmap. The clearing of the pm_gen in | ||||
* pmap_invalidate_all() takes care of TLB | * pmap_invalidate_all() takes care of TLB | ||||
* invalidation when switching to the pmap on this | * invalidation when switching to the pmap on this | ||||
* CPU. | * CPU. | ||||
*/ | */ | ||||
if (PCPU_GET(curpmap) == smp_tlb_pmap) { | if (smp_tlb_pmap == PCPU_GET(curpmap)) { | ||||
alc: Everywhere else you write this as `smp_tlb_pmap == PCPU_GET(curpmap)`. | |||||
pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; | pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; | ||||
kcr3 = smp_tlb_pmap->pm_cr3 | pcid; | load_cr3(smp_tlb_pmap->pm_cr3 | pcid); | ||||
Done Inline ActionsThe variable "ucr3" no longer serves to simplify this code, as it is only used once. (The below function, invlpg_invpcid_handler(), similarly does without a "ucr3" variable.) alc: The variable "ucr3" no longer serves to simplify this code, as it is only used once. (The… | |||||
Done Inline Actionskcr3 is also not too much needed. I kept pcid since otherwith load_cr3() line becomes too unreadable. kib: kcr3 is also not too much needed. I kept pcid since otherwith load_cr3() line becomes too… | |||||
ucr3 = smp_tlb_pmap->pm_ucr3; | if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3) | ||||
if (ucr3 != PMAP_NO_CR3) { | PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE); | ||||
ucr3 |= PMAP_PCID_USER_PT | pcid; | |||||
pmap_pti_pcid_invalidate(ucr3, kcr3); | |||||
} else | |||||
load_cr3(kcr3); | |||||
} | } | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
invlpg_handler(vm_offset_t smp_tlb_addr1) | invlpg_handler(vm_offset_t smp_tlb_addr1) | ||||
{ | { | ||||
#ifdef COUNT_XINVLTLB_HITS | #ifdef COUNT_XINVLTLB_HITS | ||||
Show All 14 Lines | |||||
#ifdef COUNT_XINVLTLB_HITS | #ifdef COUNT_XINVLTLB_HITS | ||||
xhits_pg[PCPU_GET(cpuid)]++; | xhits_pg[PCPU_GET(cpuid)]++; | ||||
#endif /* COUNT_XINVLTLB_HITS */ | #endif /* COUNT_XINVLTLB_HITS */ | ||||
#ifdef COUNT_IPIS | #ifdef COUNT_IPIS | ||||
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++; | (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; | ||||
#endif /* COUNT_IPIS */ | #endif /* COUNT_IPIS */ | ||||
invlpg(smp_tlb_addr1); | invlpg(smp_tlb_addr1); | ||||
if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3) { | if (smp_tlb_pmap == PCPU_GET(curpmap) && | ||||
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 && | |||||
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { | |||||
Done Inline ActionsI think we can have ucr3_load_mask != NOMASK only if the thread was interrupted while in kernel mode. Is that correct? markj: I think we can have ucr3_load_mask != NOMASK only if the thread was interrupted while in kernel… | |||||
Done Inline ActionsYes, this should be true. kib: Yes, this should be true. | |||||
d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid | | d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid | | ||||
PMAP_PCID_USER_PT; | PMAP_PCID_USER_PT; | ||||
d.pad = 0; | d.pad = 0; | ||||
d.addr = smp_tlb_addr1; | d.addr = smp_tlb_addr1; | ||||
invpcid(&d, INVPCID_ADDR); | invpcid(&d, INVPCID_ADDR); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
invlpg_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1) | invlpg_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1) | ||||
{ | { | ||||
uint64_t kcr3, ucr3; | uint64_t kcr3, ucr3; | ||||
uint32_t pcid; | uint32_t pcid; | ||||
#ifdef COUNT_XINVLTLB_HITS | #ifdef COUNT_XINVLTLB_HITS | ||||
xhits_pg[PCPU_GET(cpuid)]++; | xhits_pg[PCPU_GET(cpuid)]++; | ||||
#endif /* COUNT_XINVLTLB_HITS */ | #endif /* COUNT_XINVLTLB_HITS */ | ||||
#ifdef COUNT_IPIS | #ifdef COUNT_IPIS | ||||
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++; | (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; | ||||
#endif /* COUNT_IPIS */ | #endif /* COUNT_IPIS */ | ||||
invlpg(smp_tlb_addr1); | invlpg(smp_tlb_addr1); | ||||
if (smp_tlb_pmap == PCPU_GET(curpmap) && | if (smp_tlb_pmap == PCPU_GET(curpmap) && | ||||
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3) { | (ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 && | ||||
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { | |||||
pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; | pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; | ||||
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE; | kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE; | ||||
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; | ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; | ||||
pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1); | pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
Show All 31 Lines | |||||
#endif /* COUNT_IPIS */ | #endif /* COUNT_IPIS */ | ||||
addr = smp_tlb_addr1; | addr = smp_tlb_addr1; | ||||
addr2 = smp_tlb_addr2; | addr2 = smp_tlb_addr2; | ||||
do { | do { | ||||
invlpg(addr); | invlpg(addr); | ||||
addr += PAGE_SIZE; | addr += PAGE_SIZE; | ||||
} while (addr < addr2); | } while (addr < addr2); | ||||
if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3) { | if (smp_tlb_pmap == PCPU_GET(curpmap) && | ||||
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 && | |||||
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { | |||||
d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid | | d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid | | ||||
PMAP_PCID_USER_PT; | PMAP_PCID_USER_PT; | ||||
d.pad = 0; | d.pad = 0; | ||||
d.addr = smp_tlb_addr1; | d.addr = smp_tlb_addr1; | ||||
do { | do { | ||||
invpcid(&d, INVPCID_ADDR); | invpcid(&d, INVPCID_ADDR); | ||||
d.addr += PAGE_SIZE; | d.addr += PAGE_SIZE; | ||||
} while (d.addr < addr2); | } while (d.addr < addr2); | ||||
Show All 17 Lines | #endif /* COUNT_IPIS */ | ||||
addr = smp_tlb_addr1; | addr = smp_tlb_addr1; | ||||
addr2 = smp_tlb_addr2; | addr2 = smp_tlb_addr2; | ||||
do { | do { | ||||
invlpg(addr); | invlpg(addr); | ||||
addr += PAGE_SIZE; | addr += PAGE_SIZE; | ||||
} while (addr < addr2); | } while (addr < addr2); | ||||
if (smp_tlb_pmap == PCPU_GET(curpmap) && | if (smp_tlb_pmap == PCPU_GET(curpmap) && | ||||
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3) { | (ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 && | ||||
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { | |||||
pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; | pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; | ||||
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE; | kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE; | ||||
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; | ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; | ||||
pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, addr2); | pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, addr2); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
▲ Show 20 Lines • Show All 110 Lines • Show Last 20 Lines |
Everywhere else you write this as smp_tlb_pmap == PCPU_GET(curpmap).