Page MenuHomeFreeBSD

D39890.id121283.diff
No OneTemporary

D39890.id121283.diff

diff --git a/sys/amd64/amd64/efirt_machdep.c b/sys/amd64/amd64/efirt_machdep.c
--- a/sys/amd64/amd64/efirt_machdep.c
+++ b/sys/amd64/amd64/efirt_machdep.c
@@ -298,6 +298,7 @@
efi_arch_enter(void)
{
pmap_t curpmap;
+ uint64_t cr3;
curpmap = PCPU_GET(curpmap);
PMAP_LOCK_ASSERT(curpmap, MA_OWNED);
@@ -313,8 +314,10 @@
if (pmap_pcid_enabled && !invpcid_works)
PCPU_SET(curpmap, NULL);
- load_cr3(VM_PAGE_TO_PHYS(efi_pmltop_page) | (pmap_pcid_enabled ?
- curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0));
+ cr3 = VM_PAGE_TO_PHYS(efi_pmltop_page);
+ if (pmap_pcid_enabled)
+ cr3 |= pmap_get_pcid(curpmap);
+ load_cr3(cr3);
/*
* If PCID is enabled, the clear CR3_PCID_SAVE bit in the loaded %cr3
* causes TLB invalidation.
@@ -328,12 +331,16 @@
efi_arch_leave(void)
{
pmap_t curpmap;
+ uint64_t cr3;
curpmap = &curproc->p_vmspace->vm_pmap;
- if (pmap_pcid_enabled && !invpcid_works)
- PCPU_SET(curpmap, curpmap);
- load_cr3(curpmap->pm_cr3 | (pmap_pcid_enabled ?
- curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0));
+ cr3 = curpmap->pm_cr3;
+ if (pmap_pcid_enabled) {
+ cr3 |= pmap_get_pcid(curpmap);
+ if (!invpcid_works)
+ PCPU_SET(curpmap, curpmap);
+ }
+ load_cr3(cr3);
if (!pmap_pcid_enabled)
invltlb();
vm_fault_enable_pagefaults(curthread->td_md.md_efirt_dis_pf);
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -767,7 +767,7 @@
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap);
d.pad = 0;
d.addr = 0;
invpcid(&d, smp_tlb_pmap == kernel_pmap ? INVPCID_CTXGLOB :
@@ -786,7 +786,7 @@
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap);
d.pad = 0;
d.addr = 0;
if (smp_tlb_pmap == kernel_pmap) {
@@ -808,8 +808,6 @@
static void
invltlb_pcid_handler(pmap_t smp_tlb_pmap)
{
- uint32_t pcid;
-
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@@ -828,8 +826,8 @@
* CPU.
*/
if (smp_tlb_pmap == PCPU_GET(curpmap)) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
- load_cr3(smp_tlb_pmap->pm_cr3 | pcid);
+ load_cr3(smp_tlb_pmap->pm_cr3 |
+ pmap_get_pcid(smp_tlb_pmap));
if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3)
PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
}
@@ -865,8 +863,7 @@
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
- PMAP_PCID_USER_PT;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
d.pad = 0;
d.addr = smp_tlb_addr1;
invpcid(&d, INVPCID_ADDR);
@@ -890,7 +887,7 @@
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ pcid = pmap_get_pcid(smp_tlb_pmap);
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1);
@@ -944,8 +941,7 @@
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
- PMAP_PCID_USER_PT;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
d.pad = 0;
d.addr = smp_tlb_addr1;
do {
@@ -978,7 +974,7 @@
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ pcid = pmap_get_pcid(smp_tlb_pmap);
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, smp_tlb_addr2);
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -384,7 +384,14 @@
#define VM_PAGE_TO_PV_LIST_LOCK(m) \
PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
-struct pmap kernel_pmap_store;
+/*
+ * Kernel pmap memory. pm_pcids are instantiated after the memory
+ * allocations are operational, initialize it with a non-canonical
+ * pointer to catch early accesses regardless of the active mapping.
+ */
+struct pmap kernel_pmap_store = {
+ .pm_pcidp = (void *)0xdeadbeefdeadbeef,
+};
vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
@@ -2026,9 +2033,15 @@
/* Initialize TLB Context Id. */
if (pmap_pcid_enabled) {
+ kernel_pmap->pm_pcidp = (void *)(
+ (uintptr_t)&__pcpu[0].pc_kpmap_store -
+ (uintptr_t)&__pcpu[0]);
for (i = 0; i < MAXCPU; i++) {
- kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN;
- kernel_pmap->pm_pcids[i].pm_gen = 1;
+ struct pmap_pcids *pcidp;
+
+ pcidp = zpcpu_get_cpu(kernel_pmap->pm_pcidp, i);
+ pcidp->pm_pcid = PMAP_PCID_KERN;
+ pcidp->pm_gen = 1;
}
/*
@@ -3031,6 +3044,7 @@
static inline void
pmap_invalidate_preipi_pcid(pmap_t pmap)
{
+ struct pmap_pcids *pcidp;
u_int cpuid, i;
sched_pin();
@@ -3040,8 +3054,10 @@
cpuid = 0xffffffff; /* An impossible value */
CPU_FOREACH(i) {
- if (cpuid != i)
- pmap->pm_pcids[i].pm_gen = 0;
+ if (cpuid != i) {
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);
+ pcidp->pm_gen = 0;
+ }
}
/*
@@ -3076,7 +3092,6 @@
struct invpcid_descr d;
uint64_t kcr3, ucr3;
uint32_t pcid;
- u_int cpuid;
/*
* Because pm_pcid is recalculated on a context switch, we
@@ -3095,9 +3110,7 @@
PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK)
return;
- cpuid = PCPU_GET(cpuid);
-
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid | PMAP_PCID_USER_PT;
d.pad = 0;
@@ -3172,7 +3185,6 @@
struct invpcid_descr d;
uint64_t kcr3, ucr3;
uint32_t pcid;
- u_int cpuid;
CRITICAL_ASSERT(curthread);
@@ -3181,9 +3193,7 @@
PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK)
return;
- cpuid = PCPU_GET(cpuid);
-
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid | PMAP_PCID_USER_PT;
d.pad = 0;
@@ -3273,7 +3283,6 @@
struct invpcid_descr d;
uint64_t kcr3;
uint32_t pcid;
- u_int cpuid;
if (pmap == kernel_pmap) {
if (invpcid_works1) {
@@ -3284,9 +3293,8 @@
}
} else if (pmap == PCPU_GET(curpmap)) {
CRITICAL_ASSERT(curthread);
- cpuid = PCPU_GET(cpuid);
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid;
d.pad = 0;
@@ -4193,12 +4201,24 @@
}
}
+static void
+pmap_pinit_pcids(pmap_t pmap, uint32_t pcid, int gen)
+{
+ struct pmap_pcids *pcidp;
+ int i;
+
+ CPU_FOREACH(i) {
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);
+ pcidp->pm_pcid = pcid;
+ pcidp->pm_gen = gen;
+ }
+}
+
void
pmap_pinit0(pmap_t pmap)
{
struct proc *p;
struct thread *td;
- int i;
PMAP_LOCK_INIT(pmap);
pmap->pm_pmltop = kernel_pmap->pm_pmltop;
@@ -4211,10 +4231,8 @@
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
pmap->pm_flags = pmap_flags;
- CPU_FOREACH(i) {
- pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN + 1;
- pmap->pm_pcids[i].pm_gen = 1;
- }
+ pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8, M_WAITOK);
+ pmap_pinit_pcids(pmap, PMAP_PCID_KERN + 1, 1);
pmap_activate_boot(pmap);
td = curthread;
if (pti) {
@@ -4362,6 +4380,8 @@
pmap_pt_page_count_adj(pmap, -1);
}
+_Static_assert(sizeof(struct pmap_pcids) == 8, "Fix pcpu zone for pm_pcidp");
+
/*
* Initialize a preallocated and zeroed pmap structure,
* such as one in a vmspace structure.
@@ -4371,7 +4391,6 @@
{
vm_page_t pmltop_pg, pmltop_pgu;
vm_paddr_t pmltop_phys;
- int i;
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -4395,9 +4414,11 @@
pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg);
pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys);
- CPU_FOREACH(i) {
- pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;
- pmap->pm_pcids[i].pm_gen = 0;
+ if (pmap_pcid_enabled) {
+ if (pmap->pm_pcidp == NULL)
+ pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8,
+ M_WAITOK);
+ pmap_pinit_pcids(pmap, PMAP_PCID_NONE, 0);
}
pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */
pmap->pm_ucr3 = PMAP_NO_CR3;
@@ -9925,20 +9946,20 @@
}
static uint64_t
-pmap_pcid_alloc(pmap_t pmap, u_int cpuid)
+pmap_pcid_alloc(pmap_t pmap, struct pmap_pcids *pcidp)
{
uint32_t gen, new_gen, pcid_next;
CRITICAL_ASSERT(curthread);
gen = PCPU_GET(pcid_gen);
- if (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN)
+ if (pcidp->pm_pcid == PMAP_PCID_KERN)
return (pti ? 0 : CR3_PCID_SAVE);
- if (pmap->pm_pcids[cpuid].pm_gen == gen)
+ if (pcidp->pm_gen == gen)
return (CR3_PCID_SAVE);
pcid_next = PCPU_GET(pcid_next);
KASSERT((!pti && pcid_next <= PMAP_PCID_OVERMAX) ||
(pti && pcid_next <= PMAP_PCID_OVERMAX_KERN),
- ("cpu %d pcid_next %#x", cpuid, pcid_next));
+ ("cpu %d pcid_next %#x", PCPU_GET(cpuid), pcid_next));
if ((!pti && pcid_next == PMAP_PCID_OVERMAX) ||
(pti && pcid_next == PMAP_PCID_OVERMAX_KERN)) {
new_gen = gen + 1;
@@ -9949,25 +9970,23 @@
} else {
new_gen = gen;
}
- pmap->pm_pcids[cpuid].pm_pcid = pcid_next;
- pmap->pm_pcids[cpuid].pm_gen = new_gen;
+ pcidp->pm_pcid = pcid_next;
+ pcidp->pm_gen = new_gen;
PCPU_SET(pcid_next, pcid_next + 1);
return (0);
}
static uint64_t
-pmap_pcid_alloc_checked(pmap_t pmap, u_int cpuid)
+pmap_pcid_alloc_checked(pmap_t pmap, struct pmap_pcids *pcidp)
{
uint64_t cached;
- cached = pmap_pcid_alloc(pmap, cpuid);
- KASSERT(pmap->pm_pcids[cpuid].pm_pcid < PMAP_PCID_OVERMAX,
- ("pmap %p cpu %d pcid %#x", pmap, cpuid,
- pmap->pm_pcids[cpuid].pm_pcid));
- KASSERT(pmap->pm_pcids[cpuid].pm_pcid != PMAP_PCID_KERN ||
- pmap == kernel_pmap,
+ cached = pmap_pcid_alloc(pmap, pcidp);
+ KASSERT(pcidp->pm_pcid < PMAP_PCID_OVERMAX,
+ ("pmap %p cpu %d pcid %#x", pmap, PCPU_GET(cpuid), pcidp->pm_pcid));
+ KASSERT(pcidp->pm_pcid != PMAP_PCID_KERN || pmap == kernel_pmap,
("non-kernel pmap pmap %p cpu %d pcid %#x",
- pmap, cpuid, pmap->pm_pcids[cpuid].pm_pcid));
+ pmap, PCPU_GET(cpuid), pcidp->pm_pcid));
return (cached);
}
@@ -9983,6 +10002,7 @@
pmap_activate_sw_pcid_pti(struct thread *td, pmap_t pmap, u_int cpuid)
{
pmap_t old_pmap;
+ struct pmap_pcids *pcidp, *old_pcidp;
uint64_t cached, cr3, kcr3, ucr3;
KASSERT((read_rflags() & PSL_I) == 0,
@@ -9993,17 +10013,18 @@
PCPU_SET(ucr3_load_mask, PMAP_UCR3_NOMASK);
old_pmap = PCPU_GET(curpmap);
MPASS(old_pmap->pm_ucr3 != PMAP_NO_CR3);
- old_pmap->pm_pcids[cpuid].pm_gen = 0;
+ old_pcidp = zpcpu_get_cpu(old_pmap->pm_pcidp, cpuid);
+ old_pcidp->pm_gen = 0;
}
- cached = pmap_pcid_alloc_checked(pmap, cpuid);
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);
+ cached = pmap_pcid_alloc_checked(pmap, pcidp);
cr3 = rcr3();
if ((cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)
- load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid);
+ load_cr3(pmap->pm_cr3 | pcidp->pm_pcid);
PCPU_SET(curpmap, pmap);
- kcr3 = pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid;
- ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[cpuid].pm_pcid |
- PMAP_PCID_USER_PT;
+ kcr3 = pmap->pm_cr3 | pcidp->pm_pcid;
+ ucr3 = pmap->pm_ucr3 | pcidp->pm_pcid | PMAP_PCID_USER_PT;
if (!cached && pmap->pm_ucr3 != PMAP_NO_CR3)
PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
@@ -10020,16 +10041,17 @@
pmap_activate_sw_pcid_nopti(struct thread *td __unused, pmap_t pmap,
u_int cpuid)
{
+ struct pmap_pcids *pcidp;
uint64_t cached, cr3;
KASSERT((read_rflags() & PSL_I) == 0,
("PCID needs interrupts disabled in pmap_activate_sw()"));
- cached = pmap_pcid_alloc_checked(pmap, cpuid);
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);
+ cached = pmap_pcid_alloc_checked(pmap, pcidp);
cr3 = rcr3();
if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)
- load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid |
- cached);
+ load_cr3(pmap->pm_cr3 | pcidp->pm_pcid | cached);
PCPU_SET(curpmap, pmap);
if (cached)
counter_u64_add(pcid_save_cnt, 1);
@@ -10143,7 +10165,7 @@
if (pti) {
kcr3 = pmap->pm_cr3;
if (pmap_pcid_enabled)
- kcr3 |= pmap->pm_pcids[cpuid].pm_pcid | CR3_PCID_SAVE;
+ kcr3 |= pmap_get_pcid(pmap) | CR3_PCID_SAVE;
} else {
kcr3 = PMAP_NO_CR3;
}
diff --git a/sys/amd64/include/_pmap.h b/sys/amd64/include/_pmap.h
new file mode 100644
--- /dev/null
+++ b/sys/amd64/include/_pmap.h
@@ -0,0 +1,9 @@
+#ifndef _MACHINE__PMAP_H_
+#define _MACHINE__PMAP_H_
+
+struct pmap_pcids {
+ uint32_t pm_pcid;
+ uint32_t pm_gen;
+};
+
+#endif
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -35,6 +35,7 @@
#ifndef _MACHINE_PCPU_H_
#define _MACHINE_PCPU_H_
+#include <machine/_pmap.h>
#include <machine/segments.h>
#include <machine/tss.h>
@@ -101,7 +102,8 @@
uint64_t pc_ucr3_load_mask; \
u_int pc_small_core; \
u_int pc_pcid_invlpg_workaround; \
- char __pad[2908] /* pad to UMA_PCPU_ALLOC_SIZE */
+ struct pmap_pcids pc_kpmap_store; \
+ char __pad[2900] /* pad to UMA_PCPU_ALLOC_SIZE */
#define PC_DBREG_CMD_NONE 0
#define PC_DBREG_CMD_LOAD 1
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -291,6 +291,7 @@
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_pctrie.h>
+#include <machine/_pmap.h>
#include <sys/_pv_entry.h>
#include <sys/_rangeset.h>
#include <sys/_smr.h>
@@ -371,11 +372,6 @@
PT_RVI, /* AMD's nested page tables */
};
-struct pmap_pcids {
- uint32_t pm_pcid;
- uint32_t pm_gen;
-};
-
/*
* The kernel virtual address (KVA) of the level 4 page table page is always
* within the direct map (DMAP) region.
@@ -394,7 +390,7 @@
long pm_eptgen; /* EPT pmap generation id */
smr_t pm_eptsmr;
int pm_flags;
- struct pmap_pcids pm_pcids[MAXCPU];
+ struct pmap_pcids *pm_pcidp;
struct rangeset pm_pkru;
};
@@ -537,6 +533,17 @@
}
#endif /* sys/pcpu.h && machine/cpufunc.h */
+/* Return pcid for the pmap pmap on current cpu */
+static __inline uint32_t
+pmap_get_pcid(pmap_t pmap)
+{
+ struct pmap_pcids *pcidp;
+
+ MPASS(pmap_pcid_enabled);
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, PCPU_GET(cpuid));
+ return (pcidp->pm_pcid);
+}
+
#endif /* _KERNEL */
/* Return various clipped indexes for a given VA */

File Metadata

Mime Type
text/plain
Expires
Mon, May 18, 1:08 PM (13 h, 55 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
33249655
Default Alt Text
D39890.id121283.diff (14 KB)

Event Timeline