Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
Show First 20 Lines • Show All 1,091 Lines • ▼ Show 20 Lines | #define SYSMAP(c, p, v, n) \ | ||||
* Initialize the PAT MSR. | * Initialize the PAT MSR. | ||||
* pmap_init_pat() clears and sets CR4_PGE, which, as a | * pmap_init_pat() clears and sets CR4_PGE, which, as a | ||||
* side-effect, invalidates stale PG_G TLB entries that might | * side-effect, invalidates stale PG_G TLB entries that might | ||||
* have been created in our pre-boot environment. | * have been created in our pre-boot environment. | ||||
*/ | */ | ||||
pmap_init_pat(); | pmap_init_pat(); | ||||
/* Initialize TLB Context Id. */ | /* Initialize TLB Context Id. */ | ||||
if (pti) | |||||
pmap_pcid_enabled = 0; | |||||
TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled); | TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled); | ||||
if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) { | if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) { | ||||
/* Check for INVPCID support */ | /* Check for INVPCID support */ | ||||
invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) | invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) | ||||
!= 0; | != 0; | ||||
for (i = 0; i < MAXCPU; i++) { | for (i = 0; i < MAXCPU; i++) { | ||||
kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN; | kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN; | ||||
kernel_pmap->pm_pcids[i].pm_gen = 1; | kernel_pmap->pm_pcids[i].pm_gen = 1; | ||||
▲ Show 20 Lines • Show All 461 Lines • ▼ Show 20 Lines | pmap_invalidate_ept(pmap_t pmap) | ||||
ipi_selected(pmap->pm_active, ipinum); | ipi_selected(pmap->pm_active, ipinum); | ||||
sched_unpin(); | sched_unpin(); | ||||
} | } | ||||
void | void | ||||
pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
cpuset_t *mask; | cpuset_t *mask; | ||||
struct invpcid_descr d; | |||||
uint64_t kcr3, ucr3; | |||||
u_int cpuid, i; | u_int cpuid, i; | ||||
if (pmap_type_guest(pmap)) { | if (pmap_type_guest(pmap)) { | ||||
pmap_invalidate_ept(pmap); | pmap_invalidate_ept(pmap); | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_page: invalid type %d", pmap->pm_type)); | ("pmap_invalidate_page: invalid type %d", pmap->pm_type)); | ||||
sched_pin(); | sched_pin(); | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
invlpg(va); | invlpg(va); | ||||
mask = &all_cpus; | mask = &all_cpus; | ||||
} else { | } else { | ||||
cpuid = PCPU_GET(cpuid); | cpuid = PCPU_GET(cpuid); | ||||
if (pmap == PCPU_GET(curpmap)) | if (pmap == PCPU_GET(curpmap)) { | ||||
invlpg(va); | invlpg(va); | ||||
else if (pmap_pcid_enabled) | if (pmap_pcid_enabled && pti) { | ||||
critical_enter(); | |||||
markj: Why does this need to be a critical section? | |||||
Not Done Inline Actionspm_pcid are recalculated on the context switch. If we are removed from CPU and then rescheduled (pin still allows that) we would end up using incorrect pcid for the operation. kib: pm_pcid are recalculated on the context switch. If we are removed from CPU and then… | |||||
if (invpcid_works) { | |||||
d.pcid = pmap->pm_pcids[cpuid].pm_pcid | | |||||
PMAP_PCID_USER_PT; | |||||
d.pad = 0; | |||||
d.addr = va; | |||||
invpcid(&d, INVPCID_ADDR); | |||||
} else { | |||||
kcr3 = pmap->pm_cr3 | pmap->pm_pcids[ | |||||
cpuid].pm_pcid | CR3_PCID_SAVE; | |||||
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[ | |||||
cpuid].pm_pcid | PMAP_PCID_USER_PT | | |||||
CR3_PCID_SAVE; | |||||
pmap_pti_pcid_invlpg(ucr3, kcr3, va); | |||||
} | |||||
critical_exit(); | |||||
} | |||||
} else if (pmap_pcid_enabled) | |||||
pmap->pm_pcids[cpuid].pm_gen = 0; | pmap->pm_pcids[cpuid].pm_gen = 0; | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
if (cpuid != i) | if (cpuid != i) | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
mask = &pmap->pm_active; | mask = &pmap->pm_active; | ||||
} | } | ||||
smp_masked_invlpg(*mask, va); | smp_masked_invlpg(*mask, va, pmap); | ||||
sched_unpin(); | sched_unpin(); | ||||
} | } | ||||
/* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */ | /* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */ | ||||
#define PMAP_INVLPG_THRESHOLD (4 * 1024 * PAGE_SIZE) | #define PMAP_INVLPG_THRESHOLD (4 * 1024 * PAGE_SIZE) | ||||
void | void | ||||
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
{ | { | ||||
cpuset_t *mask; | cpuset_t *mask; | ||||
struct invpcid_descr d; | |||||
vm_offset_t addr; | vm_offset_t addr; | ||||
uint64_t kcr3, ucr3; | |||||
u_int cpuid, i; | u_int cpuid, i; | ||||
if (eva - sva >= PMAP_INVLPG_THRESHOLD) { | if (eva - sva >= PMAP_INVLPG_THRESHOLD) { | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
return; | return; | ||||
} | } | ||||
if (pmap_type_guest(pmap)) { | if (pmap_type_guest(pmap)) { | ||||
Show All 9 Lines | pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
for (addr = sva; addr < eva; addr += PAGE_SIZE) | for (addr = sva; addr < eva; addr += PAGE_SIZE) | ||||
invlpg(addr); | invlpg(addr); | ||||
mask = &all_cpus; | mask = &all_cpus; | ||||
} else { | } else { | ||||
if (pmap == PCPU_GET(curpmap)) { | if (pmap == PCPU_GET(curpmap)) { | ||||
for (addr = sva; addr < eva; addr += PAGE_SIZE) | for (addr = sva; addr < eva; addr += PAGE_SIZE) | ||||
invlpg(addr); | invlpg(addr); | ||||
if (pmap_pcid_enabled && pti) { | |||||
critical_enter(); | |||||
if (invpcid_works) { | |||||
d.pcid = pmap->pm_pcids[cpuid]. | |||||
pm_pcid | PMAP_PCID_USER_PT; | |||||
d.pad = 0; | |||||
d.addr = sva; | |||||
for (; d.addr < eva; d.addr += | |||||
PAGE_SIZE) | |||||
invpcid(&d, INVPCID_ADDR); | |||||
} else { | |||||
kcr3 = pmap->pm_cr3 | pmap->pm_pcids[ | |||||
cpuid].pm_pcid | CR3_PCID_SAVE; | |||||
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[ | |||||
cpuid].pm_pcid | PMAP_PCID_USER_PT | | |||||
CR3_PCID_SAVE; | |||||
pmap_pti_pcid_invlrng(ucr3, kcr3, sva, | |||||
eva); | |||||
} | |||||
critical_exit(); | |||||
} | |||||
Not Done Inline ActionsStray blank line. alc: Stray blank line. | |||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[cpuid].pm_gen = 0; | pmap->pm_pcids[cpuid].pm_gen = 0; | ||||
} | } | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
if (cpuid != i) | if (cpuid != i) | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
mask = &pmap->pm_active; | mask = &pmap->pm_active; | ||||
} | } | ||||
smp_masked_invlpg_range(*mask, sva, eva); | smp_masked_invlpg_range(*mask, sva, eva, pmap); | ||||
sched_unpin(); | sched_unpin(); | ||||
} | } | ||||
void | void | ||||
pmap_invalidate_all(pmap_t pmap) | pmap_invalidate_all(pmap_t pmap) | ||||
{ | { | ||||
cpuset_t *mask; | cpuset_t *mask; | ||||
struct invpcid_descr d; | struct invpcid_descr d; | ||||
uint64_t kcr3, ucr3; | |||||
u_int cpuid, i; | u_int cpuid, i; | ||||
if (pmap_type_guest(pmap)) { | if (pmap_type_guest(pmap)) { | ||||
pmap_invalidate_ept(pmap); | pmap_invalidate_ept(pmap); | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_all: invalid type %d", pmap->pm_type)); | ("pmap_invalidate_all: invalid type %d", pmap->pm_type)); | ||||
sched_pin(); | sched_pin(); | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
if (pmap_pcid_enabled && invpcid_works) { | if (pmap_pcid_enabled && invpcid_works) { | ||||
bzero(&d, sizeof(d)); | bzero(&d, sizeof(d)); | ||||
invpcid(&d, INVPCID_CTXGLOB); | invpcid(&d, INVPCID_CTXGLOB); | ||||
} else { | } else { | ||||
invltlb_glob(); | invltlb_glob(); | ||||
} | } | ||||
mask = &all_cpus; | mask = &all_cpus; | ||||
} else { | } else { | ||||
cpuid = PCPU_GET(cpuid); | cpuid = PCPU_GET(cpuid); | ||||
if (pmap == PCPU_GET(curpmap)) { | if (pmap == PCPU_GET(curpmap)) { | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
critical_enter(); | |||||
if (invpcid_works) { | if (invpcid_works) { | ||||
d.pcid = pmap->pm_pcids[cpuid].pm_pcid; | d.pcid = pmap->pm_pcids[cpuid].pm_pcid; | ||||
d.pad = 0; | d.pad = 0; | ||||
d.addr = 0; | d.addr = 0; | ||||
invpcid(&d, INVPCID_CTX); | invpcid(&d, INVPCID_CTX); | ||||
if (pti) { | |||||
d.pcid |= PMAP_PCID_USER_PT; | |||||
invpcid(&d, INVPCID_CTX); | |||||
} | |||||
} else { | } else { | ||||
load_cr3(pmap->pm_cr3 | pmap->pm_pcids | kcr3 = pmap->pm_cr3 | pmap->pm_pcids[ | ||||
[PCPU_GET(cpuid)].pm_pcid); | cpuid].pm_pcid; | ||||
if (pti) { | |||||
ucr3 = pmap->pm_ucr3 | | |||||
pmap->pm_pcids[cpuid]. | |||||
pm_pcid | PMAP_PCID_USER_PT; | |||||
pmap_pti_pcid_invalidate(ucr3, | |||||
kcr3); | |||||
} else | |||||
load_cr3(kcr3); | |||||
Not Done Inline ActionsStray blank line. alc: Stray blank line. | |||||
} | } | ||||
critical_exit(); | |||||
} else { | } else { | ||||
invltlb(); | invltlb(); | ||||
} | } | ||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[cpuid].pm_gen = 0; | pmap->pm_pcids[cpuid].pm_gen = 0; | ||||
} | } | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
#else /* !SMP */ | #else /* !SMP */ | ||||
/* | /* | ||||
* Normal, non-SMP, invalidation functions. | * Normal, non-SMP, invalidation functions. | ||||
*/ | */ | ||||
void | void | ||||
pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
struct invpcid_descr d; | |||||
uint64_t kcr3, ucr3; | |||||
if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | ||||
pmap->pm_eptgen++; | pmap->pm_eptgen++; | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_range: unknown type %d", pmap->pm_type)); | ("pmap_invalidate_range: unknown type %d", pmap->pm_type)); | ||||
if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) | if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) { | ||||
invlpg(va); | invlpg(va); | ||||
else if (pmap_pcid_enabled) | if (pmap == PCPU_GET(curpmap) && pmap_pcid_enabled && pti) { | ||||
critical_enter(); | |||||
if (invpcid_works) { | |||||
d.pcid = pmap->pm_pcids[0].pm_pcid | | |||||
PMAP_PCID_USER_PT; | |||||
d.pad = 0; | |||||
d.addr = va; | |||||
invpcid(&d, INVPCID_ADDR); | |||||
} else { | |||||
kcr3 = pmap->pm_cr3 | pmap->pm_pcids[0]. | |||||
pm_pcid | CR3_PCID_SAVE; | |||||
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[0]. | |||||
pm_pcid | PMAP_PCID_USER_PT | | |||||
CR3_PCID_SAVE; | |||||
pmap_pti_pcid_invlpg(ucr3, kcr3, va); | |||||
} | |||||
critical_exit(); | |||||
} | |||||
} else if (pmap_pcid_enabled) | |||||
pmap->pm_pcids[0].pm_gen = 0; | pmap->pm_pcids[0].pm_gen = 0; | ||||
} | } | ||||
void | void | ||||
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
{ | { | ||||
struct invpcid_descr d; | |||||
vm_offset_t addr; | vm_offset_t addr; | ||||
uint64_t kcr3, ucr3; | |||||
if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | ||||
pmap->pm_eptgen++; | pmap->pm_eptgen++; | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_range: unknown type %d", pmap->pm_type)); | ("pmap_invalidate_range: unknown type %d", pmap->pm_type)); | ||||
if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) { | if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) { | ||||
for (addr = sva; addr < eva; addr += PAGE_SIZE) | for (addr = sva; addr < eva; addr += PAGE_SIZE) | ||||
invlpg(addr); | invlpg(addr); | ||||
if (pmap == PCPU_GET(curpmap) && pmap_pcid_enabled && pti) { | |||||
critical_enter(); | |||||
if (invpcid_works) { | |||||
d.pcid = pmap->pm_pcids[0].pm_pcid | | |||||
PMAP_PCID_USER_PT; | |||||
d.pad = 0; | |||||
d.addr = sva; | |||||
for (; d.addr < eva; d.addr += PAGE_SIZE) | |||||
invpcid(&d, INVPCID_ADDR); | |||||
} else { | |||||
kcr3 = pmap->pm_cr3 | pmap->pm_pcids[0]. | |||||
pm_pcid | CR3_PCID_SAVE; | |||||
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[0]. | |||||
pm_pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; | |||||
pmap_pti_pcid_invlrng(ucr3, kcr3, sva, eva); | |||||
} | |||||
critical_exit(); | |||||
} | |||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[0].pm_gen = 0; | pmap->pm_pcids[0].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
void | void | ||||
pmap_invalidate_all(pmap_t pmap) | pmap_invalidate_all(pmap_t pmap) | ||||
{ | { | ||||
struct invpcid_descr d; | struct invpcid_descr d; | ||||
uint64_t kcr3, ucr3; | |||||
if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { | ||||
pmap->pm_eptgen++; | pmap->pm_eptgen++; | ||||
return; | return; | ||||
} | } | ||||
KASSERT(pmap->pm_type == PT_X86, | KASSERT(pmap->pm_type == PT_X86, | ||||
("pmap_invalidate_all: unknown type %d", pmap->pm_type)); | ("pmap_invalidate_all: unknown type %d", pmap->pm_type)); | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
if (pmap_pcid_enabled && invpcid_works) { | if (pmap_pcid_enabled && invpcid_works) { | ||||
bzero(&d, sizeof(d)); | bzero(&d, sizeof(d)); | ||||
invpcid(&d, INVPCID_CTXGLOB); | invpcid(&d, INVPCID_CTXGLOB); | ||||
} else { | } else { | ||||
invltlb_glob(); | invltlb_glob(); | ||||
} | } | ||||
} else if (pmap == PCPU_GET(curpmap)) { | } else if (pmap == PCPU_GET(curpmap)) { | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
critical_enter(); | |||||
if (invpcid_works) { | if (invpcid_works) { | ||||
d.pcid = pmap->pm_pcids[0].pm_pcid; | d.pcid = pmap->pm_pcids[0].pm_pcid; | ||||
d.pad = 0; | d.pad = 0; | ||||
d.addr = 0; | d.addr = 0; | ||||
invpcid(&d, INVPCID_CTX); | invpcid(&d, INVPCID_CTX); | ||||
if (pti) { | |||||
d.pcid |= PMAP_PCID_USER_PT; | |||||
invpcid(&d, INVPCID_CTX); | |||||
} | |||||
} else { | } else { | ||||
load_cr3(pmap->pm_cr3 | pmap->pm_pcids[0]. | kcr3 = pmap->pm_cr3 | pmap->pm_pcids[0].pm_pcid; | ||||
pm_pcid); | if (pti) { | ||||
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[ | |||||
0].pm_pcid | PMAP_PCID_USER_PT; | |||||
pmap_pti_pcid_invalidate(ucr3, kcr3); | |||||
} else | |||||
load_cr3(kcr3); | |||||
} | } | ||||
critical_exit(); | |||||
} else { | } else { | ||||
invltlb(); | invltlb(); | ||||
} | } | ||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[0].pm_gen = 0; | pmap->pm_pcids[0].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 5,257 Lines • ▼ Show 20 Lines | |||||
static uint64_t | static uint64_t | ||||
pmap_pcid_alloc(pmap_t pmap, u_int cpuid) | pmap_pcid_alloc(pmap_t pmap, u_int cpuid) | ||||
{ | { | ||||
uint32_t gen, new_gen, pcid_next; | uint32_t gen, new_gen, pcid_next; | ||||
CRITICAL_ASSERT(curthread); | CRITICAL_ASSERT(curthread); | ||||
gen = PCPU_GET(pcid_gen); | gen = PCPU_GET(pcid_gen); | ||||
if (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN || | if (!pti && (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN || | ||||
pmap->pm_pcids[cpuid].pm_gen == gen) | pmap->pm_pcids[cpuid].pm_gen == gen)) | ||||
return (CR3_PCID_SAVE); | return (CR3_PCID_SAVE); | ||||
pcid_next = PCPU_GET(pcid_next); | pcid_next = PCPU_GET(pcid_next); | ||||
KASSERT(pcid_next <= PMAP_PCID_OVERMAX, ("cpu %d pcid_next %#x", | KASSERT((!pti && pcid_next <= PMAP_PCID_OVERMAX) || | ||||
cpuid, pcid_next)); | (pti && pcid_next <= PMAP_PCID_OVERMAX_KERN), | ||||
if (pcid_next == PMAP_PCID_OVERMAX) { | ("cpu %d pcid_next %#x", cpuid, pcid_next)); | ||||
if ((!pti && pcid_next == PMAP_PCID_OVERMAX) || | |||||
(pti && pcid_next == PMAP_PCID_OVERMAX_KERN)) { | |||||
new_gen = gen + 1; | new_gen = gen + 1; | ||||
if (new_gen == 0) | if (new_gen == 0) | ||||
new_gen = 1; | new_gen = 1; | ||||
PCPU_SET(pcid_gen, new_gen); | PCPU_SET(pcid_gen, new_gen); | ||||
pcid_next = PMAP_PCID_KERN + 1; | pcid_next = PMAP_PCID_KERN + 1; | ||||
} else { | } else { | ||||
new_gen = gen; | new_gen = gen; | ||||
} | } | ||||
pmap->pm_pcids[cpuid].pm_pcid = pcid_next; | pmap->pm_pcids[cpuid].pm_pcid = pcid_next; | ||||
pmap->pm_pcids[cpuid].pm_gen = new_gen; | pmap->pm_pcids[cpuid].pm_gen = new_gen; | ||||
PCPU_SET(pcid_next, pcid_next + 1); | PCPU_SET(pcid_next, pcid_next + 1); | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
pmap_activate_sw(struct thread *td) | pmap_activate_sw(struct thread *td) | ||||
{ | { | ||||
pmap_t oldpmap, pmap; | pmap_t oldpmap, pmap; | ||||
struct invpcid_descr d; | |||||
uint64_t cached, cr3; | uint64_t cached, cr3; | ||||
register_t rflags; | register_t rflags; | ||||
u_int cpuid; | u_int cpuid; | ||||
oldpmap = PCPU_GET(curpmap); | oldpmap = PCPU_GET(curpmap); | ||||
pmap = vmspace_pmap(td->td_proc->p_vmspace); | pmap = vmspace_pmap(td->td_proc->p_vmspace); | ||||
if (oldpmap == pmap) | if (oldpmap == pmap) | ||||
return; | return; | ||||
Show All 40 Lines | if (pmap_pcid_enabled) { | ||||
if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) { | if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) { | ||||
load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid | | load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid | | ||||
cached); | cached); | ||||
if (cached) | if (cached) | ||||
PCPU_INC(pm_save_cnt); | PCPU_INC(pm_save_cnt); | ||||
} | } | ||||
PCPU_SET(curpmap, pmap); | PCPU_SET(curpmap, pmap); | ||||
if (pti) { | |||||
PCPU_SET(kcr3, pmap->pm_cr3 | | |||||
pmap->pm_pcids[cpuid].pm_pcid | CR3_PCID_SAVE); | |||||
PCPU_SET(ucr3, pmap->pm_ucr3 | PMAP_PCID_USER_PT | | |||||
pmap->pm_pcids[cpuid].pm_pcid | CR3_PCID_SAVE); | |||||
if (pmap != kernel_pmap) { | |||||
if (invpcid_works) { | |||||
d.pcid = PMAP_PCID_USER_PT | | |||||
pmap->pm_pcids[cpuid].pm_pcid; | |||||
d.pad = 0; | |||||
d.addr = 0; | |||||
invpcid(&d, INVPCID_CTX); | |||||
Done Inline ActionsThere's a stray space here. markj: There's a stray space here. | |||||
} else { | |||||
pmap_pti_pcid_invalidate( | |||||
PCPU_GET(ucr3) & ~CR3_PCID_SAVE, | |||||
PCPU_GET(kcr3) & ~CR3_PCID_SAVE); | |||||
} | |||||
} | |||||
} | |||||
if (!invpcid_works) | if (!invpcid_works) | ||||
intr_restore(rflags); | intr_restore(rflags); | ||||
} else if (cr3 != pmap->pm_cr3) { | } else if (cr3 != pmap->pm_cr3) { | ||||
load_cr3(pmap->pm_cr3); | load_cr3(pmap->pm_cr3); | ||||
PCPU_SET(curpmap, pmap); | PCPU_SET(curpmap, pmap); | ||||
if (pti) { | if (pti) { | ||||
PCPU_SET(kcr3, pmap->pm_cr3); | PCPU_SET(kcr3, pmap->pm_cr3); | ||||
PCPU_SET(ucr3, pmap->pm_ucr3); | PCPU_SET(ucr3, pmap->pm_ucr3); | ||||
▲ Show 20 Lines • Show All 669 Lines • Show Last 20 Lines |
Why does this need to be a critical section?