Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
Show First 20 Lines • Show All 1,054 Lines • ▼ Show 20 Lines | if (cpu_stdext_feature & CPUID_STDEXT_SMEP) | ||||
load_cr4(rcr4() | CR4_SMEP); | load_cr4(rcr4() | CR4_SMEP); | ||||
/* | /* | ||||
* Initialize the kernel pmap (which is statically allocated). | * Initialize the kernel pmap (which is statically allocated). | ||||
*/ | */ | ||||
PMAP_LOCK_INIT(kernel_pmap); | PMAP_LOCK_INIT(kernel_pmap); | ||||
kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); | kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); | ||||
kernel_pmap->pm_cr3 = KPML4phys; | kernel_pmap->pm_cr3 = KPML4phys; | ||||
kernel_pmap->pm_ucr3 = PMAP_NO_CR3; | |||||
CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ | CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ | ||||
TAILQ_INIT(&kernel_pmap->pm_pvchunk); | TAILQ_INIT(&kernel_pmap->pm_pvchunk); | ||||
kernel_pmap->pm_flags = pmap_flags; | kernel_pmap->pm_flags = pmap_flags; | ||||
/* | /* | ||||
* Initialize the TLB invalidations generation number lock. | * Initialize the TLB invalidations generation number lock. | ||||
*/ | */ | ||||
mtx_init(&invl_gen_mtx, "invlgn", NULL, MTX_DEF); | mtx_init(&invl_gen_mtx, "invlgn", NULL, MTX_DEF); | ||||
Show All 21 Lines | #define SYSMAP(c, p, v, n) \ | ||||
* Initialize the PAT MSR. | * Initialize the PAT MSR. | ||||
* pmap_init_pat() clears and sets CR4_PGE, which, as a | * pmap_init_pat() clears and sets CR4_PGE, which, as a | ||||
* side-effect, invalidates stale PG_G TLB entries that might | * side-effect, invalidates stale PG_G TLB entries that might | ||||
* have been created in our pre-boot environment. | * have been created in our pre-boot environment. | ||||
*/ | */ | ||||
pmap_init_pat(); | pmap_init_pat(); | ||||
/* Initialize TLB Context Id. */ | /* Initialize TLB Context Id. */ | ||||
if (pti) | |||||
pmap_pcid_enabled = 0; | |||||
TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled); | TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled); | ||||
if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) { | if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) { | ||||
/* Check for INVPCID support */ | /* Check for INVPCID support */ | ||||
invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) | invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) | ||||
!= 0; | != 0; | ||||
for (i = 0; i < MAXCPU; i++) { | for (i = 0; i < MAXCPU; i++) { | ||||
kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN; | kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN; | ||||
kernel_pmap->pm_pcids[i].pm_gen = 1; | kernel_pmap->pm_pcids[i].pm_gen = 1; | ||||
▲ Show 20 Lines • Show All 461 Lines • ▼ Show 20 Lines | pmap_invalidate_ept(pmap_t pmap) | ||||
ipi_selected(pmap->pm_active, ipinum); | ipi_selected(pmap->pm_active, ipinum); | ||||
sched_unpin(); | sched_unpin(); | ||||
} | } | ||||
void | void | ||||
pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
cpuset_t *mask; | cpuset_t *mask; | ||||
struct invpcid_descr d; | |||||
uint64_t kcr3, ucr3; | |||||
uint32_t pcid; | |||||
u_int cpuid, i; | u_int cpuid, i; | ||||
if (pmap_type_guest(pmap)) { | if (pmap_type_guest(pmap)) { | ||||
pmap_invalidate_ept(pmap); | pmap_invalidate_ept(pmap); | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_page: invalid type %d", pmap->pm_type)); | ("pmap_invalidate_page: invalid type %d", pmap->pm_type)); | ||||
sched_pin(); | sched_pin(); | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
invlpg(va); | invlpg(va); | ||||
mask = &all_cpus; | mask = &all_cpus; | ||||
} else { | } else { | ||||
cpuid = PCPU_GET(cpuid); | cpuid = PCPU_GET(cpuid); | ||||
if (pmap == PCPU_GET(curpmap)) | if (pmap == PCPU_GET(curpmap)) { | ||||
invlpg(va); | invlpg(va); | ||||
else if (pmap_pcid_enabled) | if (pmap_pcid_enabled && pmap->pm_ucr3 != PMAP_NO_CR3) { | ||||
/* | |||||
markj: Why does this need to be a critical section? | |||||
Not Done Inline Actionspm_pcid are recalculated on the context switch. If we are removed from CPU and then rescheduled (pin still allows that) we would end up using incorrect pcid for the operation. kib: pm_pcid are recalculated on the context switch. If we are removed from CPU and then… | |||||
* Disable context switching. pm_pcid | |||||
* is recalculated on switch, which | |||||
* might make us use wrong pcid below. | |||||
*/ | |||||
critical_enter(); | |||||
pcid = pmap->pm_pcids[cpuid].pm_pcid; | |||||
if (invpcid_works) { | |||||
d.pcid = pcid | PMAP_PCID_USER_PT; | |||||
d.pad = 0; | |||||
d.addr = va; | |||||
invpcid(&d, INVPCID_ADDR); | |||||
} else { | |||||
kcr3 = pmap->pm_cr3 | pcid | | |||||
CR3_PCID_SAVE; | |||||
ucr3 = pmap->pm_ucr3 | pcid | | |||||
PMAP_PCID_USER_PT | CR3_PCID_SAVE; | |||||
pmap_pti_pcid_invlpg(ucr3, kcr3, va); | |||||
} | |||||
critical_exit(); | |||||
} | |||||
} else if (pmap_pcid_enabled) | |||||
pmap->pm_pcids[cpuid].pm_gen = 0; | pmap->pm_pcids[cpuid].pm_gen = 0; | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
if (cpuid != i) | if (cpuid != i) | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
mask = &pmap->pm_active; | mask = &pmap->pm_active; | ||||
} | } | ||||
smp_masked_invlpg(*mask, va); | smp_masked_invlpg(*mask, va, pmap); | ||||
sched_unpin(); | sched_unpin(); | ||||
} | } | ||||
/* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */ | /* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */ | ||||
#define PMAP_INVLPG_THRESHOLD (4 * 1024 * PAGE_SIZE) | #define PMAP_INVLPG_THRESHOLD (4 * 1024 * PAGE_SIZE) | ||||
void | void | ||||
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
{ | { | ||||
cpuset_t *mask; | cpuset_t *mask; | ||||
struct invpcid_descr d; | |||||
vm_offset_t addr; | vm_offset_t addr; | ||||
uint64_t kcr3, ucr3; | |||||
uint32_t pcid; | |||||
u_int cpuid, i; | u_int cpuid, i; | ||||
if (eva - sva >= PMAP_INVLPG_THRESHOLD) { | if (eva - sva >= PMAP_INVLPG_THRESHOLD) { | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
return; | return; | ||||
} | } | ||||
if (pmap_type_guest(pmap)) { | if (pmap_type_guest(pmap)) { | ||||
Show All 9 Lines | pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
for (addr = sva; addr < eva; addr += PAGE_SIZE) | for (addr = sva; addr < eva; addr += PAGE_SIZE) | ||||
invlpg(addr); | invlpg(addr); | ||||
mask = &all_cpus; | mask = &all_cpus; | ||||
} else { | } else { | ||||
if (pmap == PCPU_GET(curpmap)) { | if (pmap == PCPU_GET(curpmap)) { | ||||
for (addr = sva; addr < eva; addr += PAGE_SIZE) | for (addr = sva; addr < eva; addr += PAGE_SIZE) | ||||
invlpg(addr); | invlpg(addr); | ||||
if (pmap_pcid_enabled && pmap->pm_ucr3 != PMAP_NO_CR3) { | |||||
critical_enter(); | |||||
pcid = pmap->pm_pcids[cpuid].pm_pcid; | |||||
if (invpcid_works) { | |||||
d.pcid = pcid | PMAP_PCID_USER_PT; | |||||
d.pad = 0; | |||||
d.addr = sva; | |||||
for (; d.addr < eva; d.addr += | |||||
PAGE_SIZE) | |||||
invpcid(&d, INVPCID_ADDR); | |||||
} else { | |||||
kcr3 = pmap->pm_cr3 | pcid | | |||||
CR3_PCID_SAVE; | |||||
ucr3 = pmap->pm_ucr3 | pcid | | |||||
PMAP_PCID_USER_PT | CR3_PCID_SAVE; | |||||
pmap_pti_pcid_invlrng(ucr3, kcr3, sva, | |||||
eva); | |||||
} | |||||
critical_exit(); | |||||
} | |||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[cpuid].pm_gen = 0; | pmap->pm_pcids[cpuid].pm_gen = 0; | ||||
Not Done Inline ActionsStray blank line. alc: Stray blank line. | |||||
} | } | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
if (cpuid != i) | if (cpuid != i) | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
mask = &pmap->pm_active; | mask = &pmap->pm_active; | ||||
} | } | ||||
smp_masked_invlpg_range(*mask, sva, eva); | smp_masked_invlpg_range(*mask, sva, eva, pmap); | ||||
sched_unpin(); | sched_unpin(); | ||||
} | } | ||||
void | void | ||||
pmap_invalidate_all(pmap_t pmap) | pmap_invalidate_all(pmap_t pmap) | ||||
{ | { | ||||
cpuset_t *mask; | cpuset_t *mask; | ||||
struct invpcid_descr d; | struct invpcid_descr d; | ||||
uint64_t kcr3, ucr3; | |||||
uint32_t pcid; | |||||
u_int cpuid, i; | u_int cpuid, i; | ||||
if (pmap_type_guest(pmap)) { | if (pmap_type_guest(pmap)) { | ||||
pmap_invalidate_ept(pmap); | pmap_invalidate_ept(pmap); | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_all: invalid type %d", pmap->pm_type)); | ("pmap_invalidate_all: invalid type %d", pmap->pm_type)); | ||||
sched_pin(); | sched_pin(); | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
if (pmap_pcid_enabled && invpcid_works) { | if (pmap_pcid_enabled && invpcid_works) { | ||||
bzero(&d, sizeof(d)); | bzero(&d, sizeof(d)); | ||||
invpcid(&d, INVPCID_CTXGLOB); | invpcid(&d, INVPCID_CTXGLOB); | ||||
} else { | } else { | ||||
invltlb_glob(); | invltlb_glob(); | ||||
} | } | ||||
mask = &all_cpus; | mask = &all_cpus; | ||||
} else { | } else { | ||||
cpuid = PCPU_GET(cpuid); | cpuid = PCPU_GET(cpuid); | ||||
if (pmap == PCPU_GET(curpmap)) { | if (pmap == PCPU_GET(curpmap)) { | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
critical_enter(); | |||||
pcid = pmap->pm_pcids[cpuid].pm_pcid; | |||||
if (invpcid_works) { | if (invpcid_works) { | ||||
d.pcid = pmap->pm_pcids[cpuid].pm_pcid; | d.pcid = pcid; | ||||
d.pad = 0; | d.pad = 0; | ||||
d.addr = 0; | d.addr = 0; | ||||
invpcid(&d, INVPCID_CTX); | invpcid(&d, INVPCID_CTX); | ||||
if (pmap->pm_ucr3 != PMAP_NO_CR3) { | |||||
d.pcid |= PMAP_PCID_USER_PT; | |||||
invpcid(&d, INVPCID_CTX); | |||||
} | |||||
} else { | } else { | ||||
load_cr3(pmap->pm_cr3 | pmap->pm_pcids | kcr3 = pmap->pm_cr3 | pcid; | ||||
[PCPU_GET(cpuid)].pm_pcid); | ucr3 = pmap->pm_ucr3; | ||||
if (ucr3 != PMAP_NO_CR3) { | |||||
ucr3 |= pcid | PMAP_PCID_USER_PT; | |||||
pmap_pti_pcid_invalidate(ucr3, | |||||
kcr3); | |||||
} else { | |||||
load_cr3(kcr3); | |||||
} | } | ||||
} | |||||
critical_exit(); | |||||
Not Done Inline ActionsStray blank line. alc: Stray blank line. | |||||
} else { | } else { | ||||
invltlb(); | invltlb(); | ||||
} | } | ||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[cpuid].pm_gen = 0; | pmap->pm_pcids[cpuid].pm_gen = 0; | ||||
} | } | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
#else /* !SMP */ | #else /* !SMP */ | ||||
/* | /* | ||||
* Normal, non-SMP, invalidation functions. | * Normal, non-SMP, invalidation functions. | ||||
*/ | */ | ||||
void | void | ||||
pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
struct invpcid_descr d; | |||||
uint64_t kcr3, ucr3; | |||||
uint32_t pcid; | |||||
if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | ||||
pmap->pm_eptgen++; | pmap->pm_eptgen++; | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_range: unknown type %d", pmap->pm_type)); | ("pmap_invalidate_range: unknown type %d", pmap->pm_type)); | ||||
if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) | if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) { | ||||
invlpg(va); | invlpg(va); | ||||
else if (pmap_pcid_enabled) | if (pmap == PCPU_GET(curpmap) && pmap_pcid_enabled && | ||||
pmap->pm_ucr3 != PMAP_NO_CR3) { | |||||
critical_enter(); | |||||
pcid = pmap->pm_pcids[0].pm_pcid; | |||||
if (invpcid_works) { | |||||
d.pcid = pcid | PMAP_PCID_USER_PT; | |||||
d.pad = 0; | |||||
d.addr = va; | |||||
invpcid(&d, INVPCID_ADDR); | |||||
} else { | |||||
kcr3 = pmap->pm_cr3 | pcid | CR3_PCID_SAVE; | |||||
ucr3 = pmap->pm_ucr3 | pcid | | |||||
PMAP_PCID_USER_PT | CR3_PCID_SAVE; | |||||
pmap_pti_pcid_invlpg(ucr3, kcr3, va); | |||||
} | |||||
critical_exit(); | |||||
} | |||||
} else if (pmap_pcid_enabled) | |||||
pmap->pm_pcids[0].pm_gen = 0; | pmap->pm_pcids[0].pm_gen = 0; | ||||
} | } | ||||
void | void | ||||
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
{ | { | ||||
struct invpcid_descr d; | |||||
vm_offset_t addr; | vm_offset_t addr; | ||||
uint64_t kcr3, ucr3; | |||||
if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | ||||
pmap->pm_eptgen++; | pmap->pm_eptgen++; | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_range: unknown type %d", pmap->pm_type)); | ("pmap_invalidate_range: unknown type %d", pmap->pm_type)); | ||||
if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) { | if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) { | ||||
for (addr = sva; addr < eva; addr += PAGE_SIZE) | for (addr = sva; addr < eva; addr += PAGE_SIZE) | ||||
invlpg(addr); | invlpg(addr); | ||||
if (pmap == PCPU_GET(curpmap) && pmap_pcid_enabled && | |||||
pmap->pm_ucr3 != PMAP_NO_CR3) { | |||||
critical_enter(); | |||||
if (invpcid_works) { | |||||
d.pcid = pmap->pm_pcids[0].pm_pcid | | |||||
PMAP_PCID_USER_PT; | |||||
d.pad = 0; | |||||
d.addr = sva; | |||||
for (; d.addr < eva; d.addr += PAGE_SIZE) | |||||
invpcid(&d, INVPCID_ADDR); | |||||
} else { | |||||
kcr3 = pmap->pm_cr3 | pmap->pm_pcids[0]. | |||||
pm_pcid | CR3_PCID_SAVE; | |||||
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[0]. | |||||
pm_pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; | |||||
pmap_pti_pcid_invlrng(ucr3, kcr3, sva, eva); | |||||
} | |||||
critical_exit(); | |||||
} | |||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[0].pm_gen = 0; | pmap->pm_pcids[0].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
void | void | ||||
pmap_invalidate_all(pmap_t pmap) | pmap_invalidate_all(pmap_t pmap) | ||||
{ | { | ||||
struct invpcid_descr d; | struct invpcid_descr d; | ||||
uint64_t kcr3, ucr3; | |||||
if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | ||||
pmap->pm_eptgen++; | pmap->pm_eptgen++; | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_all: unknown type %d", pmap->pm_type)); | ("pmap_invalidate_all: unknown type %d", pmap->pm_type)); | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
if (pmap_pcid_enabled && invpcid_works) { | if (pmap_pcid_enabled && invpcid_works) { | ||||
bzero(&d, sizeof(d)); | bzero(&d, sizeof(d)); | ||||
invpcid(&d, INVPCID_CTXGLOB); | invpcid(&d, INVPCID_CTXGLOB); | ||||
} else { | } else { | ||||
invltlb_glob(); | invltlb_glob(); | ||||
} | } | ||||
} else if (pmap == PCPU_GET(curpmap)) { | } else if (pmap == PCPU_GET(curpmap)) { | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
critical_enter(); | |||||
if (invpcid_works) { | if (invpcid_works) { | ||||
d.pcid = pmap->pm_pcids[0].pm_pcid; | d.pcid = pmap->pm_pcids[0].pm_pcid; | ||||
d.pad = 0; | d.pad = 0; | ||||
d.addr = 0; | d.addr = 0; | ||||
invpcid(&d, INVPCID_CTX); | invpcid(&d, INVPCID_CTX); | ||||
if (pmap->pm_ucr3 != PMAP_NO_CR3) { | |||||
d.pcid |= PMAP_PCID_USER_PT; | |||||
invpcid(&d, INVPCID_CTX); | |||||
} | |||||
} else { | } else { | ||||
load_cr3(pmap->pm_cr3 | pmap->pm_pcids[0]. | kcr3 = pmap->pm_cr3 | pmap->pm_pcids[0].pm_pcid; | ||||
pm_pcid); | if (pmap->pm_ucr3 != PMAP_NO_CR3) { | ||||
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[ | |||||
0].pm_pcid | PMAP_PCID_USER_PT; | |||||
pmap_pti_pcid_invalidate(ucr3, kcr3); | |||||
} else | |||||
load_cr3(kcr3); | |||||
} | } | ||||
critical_exit(); | |||||
} else { | } else { | ||||
invltlb(); | invltlb(); | ||||
} | } | ||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[0].pm_gen = 0; | pmap->pm_pcids[0].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 521 Lines • ▼ Show 20 Lines | |||||
pmap_pinit0(pmap_t pmap) | pmap_pinit0(pmap_t pmap) | ||||
{ | { | ||||
int i; | int i; | ||||
PMAP_LOCK_INIT(pmap); | PMAP_LOCK_INIT(pmap); | ||||
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); | pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); | ||||
pmap->pm_pml4u = NULL; | pmap->pm_pml4u = NULL; | ||||
pmap->pm_cr3 = KPML4phys; | pmap->pm_cr3 = KPML4phys; | ||||
pmap->pm_ucr3 = ~0UL; | /* hack to keep pmap_pti_pcid_invalidate() alive */ | ||||
pmap->pm_ucr3 = PMAP_NO_CR3; | |||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
TAILQ_INIT(&pmap->pm_pvchunk); | TAILQ_INIT(&pmap->pm_pvchunk); | ||||
bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | ||||
pmap->pm_flags = pmap_flags; | pmap->pm_flags = pmap_flags; | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
if (!pti) | if (!pti) | ||||
__pcpu[i].pc_kcr3 = ~0ul; | __pcpu[i].pc_kcr3 = PMAP_NO_CR3; | ||||
} | } | ||||
PCPU_SET(curpmap, kernel_pmap); | PCPU_SET(curpmap, kernel_pmap); | ||||
pmap_activate(curthread); | pmap_activate(curthread); | ||||
CPU_FILL(&kernel_pmap->pm_active); | CPU_FILL(&kernel_pmap->pm_active); | ||||
} | } | ||||
void | void | ||||
pmap_pinit_pml4(vm_page_t pml4pg) | pmap_pinit_pml4(vm_page_t pml4pg) | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); | VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); | ||||
pml4phys = VM_PAGE_TO_PHYS(pml4pg); | pml4phys = VM_PAGE_TO_PHYS(pml4pg); | ||||
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(pml4phys); | pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(pml4phys); | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
} | } | ||||
pmap->pm_cr3 = ~0l; /* initialize to an invalid value */ | pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */ | ||||
pmap->pm_ucr3 = PMAP_NO_CR3; | |||||
pmap->pm_pml4u = NULL; | pmap->pm_pml4u = NULL; | ||||
pmap->pm_type = pm_type; | pmap->pm_type = pm_type; | ||||
if ((pml4pg->flags & PG_ZERO) == 0) | if ((pml4pg->flags & PG_ZERO) == 0) | ||||
pagezero(pmap->pm_pml4); | pagezero(pmap->pm_pml4); | ||||
/* | /* | ||||
* Do not install the host kernel mappings in the nested page | * Do not install the host kernel mappings in the nested page | ||||
▲ Show 20 Lines • Show All 4,645 Lines • ▼ Show 20 Lines | |||||
static uint64_t | static uint64_t | ||||
pmap_pcid_alloc(pmap_t pmap, u_int cpuid) | pmap_pcid_alloc(pmap_t pmap, u_int cpuid) | ||||
{ | { | ||||
uint32_t gen, new_gen, pcid_next; | uint32_t gen, new_gen, pcid_next; | ||||
CRITICAL_ASSERT(curthread); | CRITICAL_ASSERT(curthread); | ||||
gen = PCPU_GET(pcid_gen); | gen = PCPU_GET(pcid_gen); | ||||
if (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN || | if (!pti && (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN || | ||||
pmap->pm_pcids[cpuid].pm_gen == gen) | pmap->pm_pcids[cpuid].pm_gen == gen)) | ||||
return (CR3_PCID_SAVE); | return (CR3_PCID_SAVE); | ||||
pcid_next = PCPU_GET(pcid_next); | pcid_next = PCPU_GET(pcid_next); | ||||
KASSERT(pcid_next <= PMAP_PCID_OVERMAX, ("cpu %d pcid_next %#x", | KASSERT((!pti && pcid_next <= PMAP_PCID_OVERMAX) || | ||||
cpuid, pcid_next)); | (pti && pcid_next <= PMAP_PCID_OVERMAX_KERN), | ||||
if (pcid_next == PMAP_PCID_OVERMAX) { | ("cpu %d pcid_next %#x", cpuid, pcid_next)); | ||||
if ((!pti && pcid_next == PMAP_PCID_OVERMAX) || | |||||
(pti && pcid_next == PMAP_PCID_OVERMAX_KERN)) { | |||||
new_gen = gen + 1; | new_gen = gen + 1; | ||||
if (new_gen == 0) | if (new_gen == 0) | ||||
new_gen = 1; | new_gen = 1; | ||||
PCPU_SET(pcid_gen, new_gen); | PCPU_SET(pcid_gen, new_gen); | ||||
pcid_next = PMAP_PCID_KERN + 1; | pcid_next = PMAP_PCID_KERN + 1; | ||||
} else { | } else { | ||||
new_gen = gen; | new_gen = gen; | ||||
} | } | ||||
pmap->pm_pcids[cpuid].pm_pcid = pcid_next; | pmap->pm_pcids[cpuid].pm_pcid = pcid_next; | ||||
pmap->pm_pcids[cpuid].pm_gen = new_gen; | pmap->pm_pcids[cpuid].pm_gen = new_gen; | ||||
PCPU_SET(pcid_next, pcid_next + 1); | PCPU_SET(pcid_next, pcid_next + 1); | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
pmap_activate_sw(struct thread *td) | pmap_activate_sw(struct thread *td) | ||||
{ | { | ||||
pmap_t oldpmap, pmap; | pmap_t oldpmap, pmap; | ||||
uint64_t cached, cr3; | struct invpcid_descr d; | ||||
uint64_t cached, cr3, kcr3, ucr3; | |||||
register_t rflags; | register_t rflags; | ||||
u_int cpuid; | u_int cpuid; | ||||
oldpmap = PCPU_GET(curpmap); | oldpmap = PCPU_GET(curpmap); | ||||
pmap = vmspace_pmap(td->td_proc->p_vmspace); | pmap = vmspace_pmap(td->td_proc->p_vmspace); | ||||
if (oldpmap == pmap) | if (oldpmap == pmap) | ||||
return; | return; | ||||
cpuid = PCPU_GET(cpuid); | cpuid = PCPU_GET(cpuid); | ||||
Show All 39 Lines | if (pmap_pcid_enabled) { | ||||
if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) { | if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) { | ||||
load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid | | load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid | | ||||
cached); | cached); | ||||
if (cached) | if (cached) | ||||
PCPU_INC(pm_save_cnt); | PCPU_INC(pm_save_cnt); | ||||
} | } | ||||
PCPU_SET(curpmap, pmap); | PCPU_SET(curpmap, pmap); | ||||
if (pti) { | |||||
kcr3 = pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid; | |||||
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[cpuid].pm_pcid | | |||||
PMAP_PCID_USER_PT; | |||||
/* | |||||
* Manually invalidate translations cached | |||||
* from the user page table, which are not | |||||
* flushed by reload of cr3 with the kernel | |||||
* page table pointer above. | |||||
*/ | |||||
if (pmap ->pm_ucr3 != PMAP_NO_CR3) { | |||||
markjUnsubmitted Done Inline ActionsThere's a stray space here. markj: There's a stray space here. | |||||
if (invpcid_works) { | |||||
d.pcid = PMAP_PCID_USER_PT | | |||||
pmap->pm_pcids[cpuid].pm_pcid; | |||||
d.pad = 0; | |||||
d.addr = 0; | |||||
invpcid(&d, INVPCID_CTX); | |||||
} else { | |||||
pmap_pti_pcid_invalidate(ucr3, kcr3); | |||||
} | |||||
} | |||||
PCPU_SET(kcr3, kcr3 | CR3_PCID_SAVE); | |||||
PCPU_SET(ucr3, ucr3 | CR3_PCID_SAVE); | |||||
} | |||||
if (!invpcid_works) | if (!invpcid_works) | ||||
intr_restore(rflags); | intr_restore(rflags); | ||||
} else if (cr3 != pmap->pm_cr3) { | } else if (cr3 != pmap->pm_cr3) { | ||||
load_cr3(pmap->pm_cr3); | load_cr3(pmap->pm_cr3); | ||||
PCPU_SET(curpmap, pmap); | PCPU_SET(curpmap, pmap); | ||||
if (pti) { | if (pti) { | ||||
PCPU_SET(kcr3, pmap->pm_cr3); | PCPU_SET(kcr3, pmap->pm_cr3); | ||||
PCPU_SET(ucr3, pmap->pm_ucr3); | PCPU_SET(ucr3, pmap->pm_ucr3); | ||||
▲ Show 20 Lines • Show All 669 Lines • Show Last 20 Lines |
Why does this need to be a critical section?