Page Menu
Home
FreeBSD
Search
Configure Global Search
Log In
Files
F156943442
D39890.id121283.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Size
14 KB
Referenced Files
None
Subscribers
None
D39890.id121283.diff
View Options
diff --git a/sys/amd64/amd64/efirt_machdep.c b/sys/amd64/amd64/efirt_machdep.c
--- a/sys/amd64/amd64/efirt_machdep.c
+++ b/sys/amd64/amd64/efirt_machdep.c
@@ -298,6 +298,7 @@
efi_arch_enter(void)
{
pmap_t curpmap;
+ uint64_t cr3;
curpmap = PCPU_GET(curpmap);
PMAP_LOCK_ASSERT(curpmap, MA_OWNED);
@@ -313,8 +314,10 @@
if (pmap_pcid_enabled && !invpcid_works)
PCPU_SET(curpmap, NULL);
- load_cr3(VM_PAGE_TO_PHYS(efi_pmltop_page) | (pmap_pcid_enabled ?
- curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0));
+ cr3 = VM_PAGE_TO_PHYS(efi_pmltop_page);
+ if (pmap_pcid_enabled)
+ cr3 |= pmap_get_pcid(curpmap);
+ load_cr3(cr3);
/*
* If PCID is enabled, the clear CR3_PCID_SAVE bit in the loaded %cr3
* causes TLB invalidation.
@@ -328,12 +331,16 @@
efi_arch_leave(void)
{
pmap_t curpmap;
+ uint64_t cr3;
curpmap = &curproc->p_vmspace->vm_pmap;
- if (pmap_pcid_enabled && !invpcid_works)
- PCPU_SET(curpmap, curpmap);
- load_cr3(curpmap->pm_cr3 | (pmap_pcid_enabled ?
- curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0));
+ cr3 = curpmap->pm_cr3;
+ if (pmap_pcid_enabled) {
+ cr3 |= pmap_get_pcid(curpmap);
+ if (!invpcid_works)
+ PCPU_SET(curpmap, curpmap);
+ }
+ load_cr3(cr3);
if (!pmap_pcid_enabled)
invltlb();
vm_fault_enable_pagefaults(curthread->td_md.md_efirt_dis_pf);
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -767,7 +767,7 @@
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap);
d.pad = 0;
d.addr = 0;
invpcid(&d, smp_tlb_pmap == kernel_pmap ? INVPCID_CTXGLOB :
@@ -786,7 +786,7 @@
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap);
d.pad = 0;
d.addr = 0;
if (smp_tlb_pmap == kernel_pmap) {
@@ -808,8 +808,6 @@
static void
invltlb_pcid_handler(pmap_t smp_tlb_pmap)
{
- uint32_t pcid;
-
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@@ -828,8 +826,8 @@
* CPU.
*/
if (smp_tlb_pmap == PCPU_GET(curpmap)) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
- load_cr3(smp_tlb_pmap->pm_cr3 | pcid);
+ load_cr3(smp_tlb_pmap->pm_cr3 |
+ pmap_get_pcid(smp_tlb_pmap));
if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3)
PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
}
@@ -865,8 +863,7 @@
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
- PMAP_PCID_USER_PT;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
d.pad = 0;
d.addr = smp_tlb_addr1;
invpcid(&d, INVPCID_ADDR);
@@ -890,7 +887,7 @@
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ pcid = pmap_get_pcid(smp_tlb_pmap);
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1);
@@ -944,8 +941,7 @@
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
- PMAP_PCID_USER_PT;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
d.pad = 0;
d.addr = smp_tlb_addr1;
do {
@@ -978,7 +974,7 @@
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ pcid = pmap_get_pcid(smp_tlb_pmap);
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, smp_tlb_addr2);
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -384,7 +384,14 @@
#define VM_PAGE_TO_PV_LIST_LOCK(m) \
PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
-struct pmap kernel_pmap_store;
+/*
+ * Kernel pmap memory. pm_pcids are instantiated after the memory
+ * allocations are operational, initialize it with a non-canonical
+ * pointer to catch early accesses regardless of the active mapping.
+ */
+struct pmap kernel_pmap_store = {
+ .pm_pcidp = (void *)0xdeadbeefdeadbeef,
+};
vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
@@ -2026,9 +2033,15 @@
/* Initialize TLB Context Id. */
if (pmap_pcid_enabled) {
+ kernel_pmap->pm_pcidp = (void *)(
+ (uintptr_t)&__pcpu[0].pc_kpmap_store -
+ (uintptr_t)&__pcpu[0]);
for (i = 0; i < MAXCPU; i++) {
- kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN;
- kernel_pmap->pm_pcids[i].pm_gen = 1;
+ struct pmap_pcids *pcidp;
+
+ pcidp = zpcpu_get_cpu(kernel_pmap->pm_pcidp, i);
+ pcidp->pm_pcid = PMAP_PCID_KERN;
+ pcidp->pm_gen = 1;
}
/*
@@ -3031,6 +3044,7 @@
static inline void
pmap_invalidate_preipi_pcid(pmap_t pmap)
{
+ struct pmap_pcids *pcidp;
u_int cpuid, i;
sched_pin();
@@ -3040,8 +3054,10 @@
cpuid = 0xffffffff; /* An impossible value */
CPU_FOREACH(i) {
- if (cpuid != i)
- pmap->pm_pcids[i].pm_gen = 0;
+ if (cpuid != i) {
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);
+ pcidp->pm_gen = 0;
+ }
}
/*
@@ -3076,7 +3092,6 @@
struct invpcid_descr d;
uint64_t kcr3, ucr3;
uint32_t pcid;
- u_int cpuid;
/*
* Because pm_pcid is recalculated on a context switch, we
@@ -3095,9 +3110,7 @@
PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK)
return;
- cpuid = PCPU_GET(cpuid);
-
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid | PMAP_PCID_USER_PT;
d.pad = 0;
@@ -3172,7 +3185,6 @@
struct invpcid_descr d;
uint64_t kcr3, ucr3;
uint32_t pcid;
- u_int cpuid;
CRITICAL_ASSERT(curthread);
@@ -3181,9 +3193,7 @@
PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK)
return;
- cpuid = PCPU_GET(cpuid);
-
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid | PMAP_PCID_USER_PT;
d.pad = 0;
@@ -3273,7 +3283,6 @@
struct invpcid_descr d;
uint64_t kcr3;
uint32_t pcid;
- u_int cpuid;
if (pmap == kernel_pmap) {
if (invpcid_works1) {
@@ -3284,9 +3293,8 @@
}
} else if (pmap == PCPU_GET(curpmap)) {
CRITICAL_ASSERT(curthread);
- cpuid = PCPU_GET(cpuid);
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid;
d.pad = 0;
@@ -4193,12 +4201,24 @@
}
}
+static void
+pmap_pinit_pcids(pmap_t pmap, uint32_t pcid, int gen)
+{
+ struct pmap_pcids *pcidp;
+ int i;
+
+ CPU_FOREACH(i) {
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);
+ pcidp->pm_pcid = pcid;
+ pcidp->pm_gen = gen;
+ }
+}
+
void
pmap_pinit0(pmap_t pmap)
{
struct proc *p;
struct thread *td;
- int i;
PMAP_LOCK_INIT(pmap);
pmap->pm_pmltop = kernel_pmap->pm_pmltop;
@@ -4211,10 +4231,8 @@
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
pmap->pm_flags = pmap_flags;
- CPU_FOREACH(i) {
- pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN + 1;
- pmap->pm_pcids[i].pm_gen = 1;
- }
+ pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8, M_WAITOK);
+ pmap_pinit_pcids(pmap, PMAP_PCID_KERN + 1, 1);
pmap_activate_boot(pmap);
td = curthread;
if (pti) {
@@ -4362,6 +4380,8 @@
pmap_pt_page_count_adj(pmap, -1);
}
+_Static_assert(sizeof(struct pmap_pcids) == 8, "Fix pcpu zone for pm_pcidp");
+
/*
* Initialize a preallocated and zeroed pmap structure,
* such as one in a vmspace structure.
@@ -4371,7 +4391,6 @@
{
vm_page_t pmltop_pg, pmltop_pgu;
vm_paddr_t pmltop_phys;
- int i;
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -4395,9 +4414,11 @@
pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg);
pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys);
- CPU_FOREACH(i) {
- pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;
- pmap->pm_pcids[i].pm_gen = 0;
+ if (pmap_pcid_enabled) {
+ if (pmap->pm_pcidp == NULL)
+ pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8,
+ M_WAITOK);
+ pmap_pinit_pcids(pmap, PMAP_PCID_NONE, 0);
}
pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */
pmap->pm_ucr3 = PMAP_NO_CR3;
@@ -9925,20 +9946,20 @@
}
static uint64_t
-pmap_pcid_alloc(pmap_t pmap, u_int cpuid)
+pmap_pcid_alloc(pmap_t pmap, struct pmap_pcids *pcidp)
{
uint32_t gen, new_gen, pcid_next;
CRITICAL_ASSERT(curthread);
gen = PCPU_GET(pcid_gen);
- if (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN)
+ if (pcidp->pm_pcid == PMAP_PCID_KERN)
return (pti ? 0 : CR3_PCID_SAVE);
- if (pmap->pm_pcids[cpuid].pm_gen == gen)
+ if (pcidp->pm_gen == gen)
return (CR3_PCID_SAVE);
pcid_next = PCPU_GET(pcid_next);
KASSERT((!pti && pcid_next <= PMAP_PCID_OVERMAX) ||
(pti && pcid_next <= PMAP_PCID_OVERMAX_KERN),
- ("cpu %d pcid_next %#x", cpuid, pcid_next));
+ ("cpu %d pcid_next %#x", PCPU_GET(cpuid), pcid_next));
if ((!pti && pcid_next == PMAP_PCID_OVERMAX) ||
(pti && pcid_next == PMAP_PCID_OVERMAX_KERN)) {
new_gen = gen + 1;
@@ -9949,25 +9970,23 @@
} else {
new_gen = gen;
}
- pmap->pm_pcids[cpuid].pm_pcid = pcid_next;
- pmap->pm_pcids[cpuid].pm_gen = new_gen;
+ pcidp->pm_pcid = pcid_next;
+ pcidp->pm_gen = new_gen;
PCPU_SET(pcid_next, pcid_next + 1);
return (0);
}
static uint64_t
-pmap_pcid_alloc_checked(pmap_t pmap, u_int cpuid)
+pmap_pcid_alloc_checked(pmap_t pmap, struct pmap_pcids *pcidp)
{
uint64_t cached;
- cached = pmap_pcid_alloc(pmap, cpuid);
- KASSERT(pmap->pm_pcids[cpuid].pm_pcid < PMAP_PCID_OVERMAX,
- ("pmap %p cpu %d pcid %#x", pmap, cpuid,
- pmap->pm_pcids[cpuid].pm_pcid));
- KASSERT(pmap->pm_pcids[cpuid].pm_pcid != PMAP_PCID_KERN ||
- pmap == kernel_pmap,
+ cached = pmap_pcid_alloc(pmap, pcidp);
+ KASSERT(pcidp->pm_pcid < PMAP_PCID_OVERMAX,
+ ("pmap %p cpu %d pcid %#x", pmap, PCPU_GET(cpuid), pcidp->pm_pcid));
+ KASSERT(pcidp->pm_pcid != PMAP_PCID_KERN || pmap == kernel_pmap,
("non-kernel pmap pmap %p cpu %d pcid %#x",
- pmap, cpuid, pmap->pm_pcids[cpuid].pm_pcid));
+ pmap, PCPU_GET(cpuid), pcidp->pm_pcid));
return (cached);
}
@@ -9983,6 +10002,7 @@
pmap_activate_sw_pcid_pti(struct thread *td, pmap_t pmap, u_int cpuid)
{
pmap_t old_pmap;
+ struct pmap_pcids *pcidp, *old_pcidp;
uint64_t cached, cr3, kcr3, ucr3;
KASSERT((read_rflags() & PSL_I) == 0,
@@ -9993,17 +10013,18 @@
PCPU_SET(ucr3_load_mask, PMAP_UCR3_NOMASK);
old_pmap = PCPU_GET(curpmap);
MPASS(old_pmap->pm_ucr3 != PMAP_NO_CR3);
- old_pmap->pm_pcids[cpuid].pm_gen = 0;
+ old_pcidp = zpcpu_get_cpu(old_pmap->pm_pcidp, cpuid);
+ old_pcidp->pm_gen = 0;
}
- cached = pmap_pcid_alloc_checked(pmap, cpuid);
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);
+ cached = pmap_pcid_alloc_checked(pmap, pcidp);
cr3 = rcr3();
if ((cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)
- load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid);
+ load_cr3(pmap->pm_cr3 | pcidp->pm_pcid);
PCPU_SET(curpmap, pmap);
- kcr3 = pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid;
- ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[cpuid].pm_pcid |
- PMAP_PCID_USER_PT;
+ kcr3 = pmap->pm_cr3 | pcidp->pm_pcid;
+ ucr3 = pmap->pm_ucr3 | pcidp->pm_pcid | PMAP_PCID_USER_PT;
if (!cached && pmap->pm_ucr3 != PMAP_NO_CR3)
PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
@@ -10020,16 +10041,17 @@
pmap_activate_sw_pcid_nopti(struct thread *td __unused, pmap_t pmap,
u_int cpuid)
{
+ struct pmap_pcids *pcidp;
uint64_t cached, cr3;
KASSERT((read_rflags() & PSL_I) == 0,
("PCID needs interrupts disabled in pmap_activate_sw()"));
- cached = pmap_pcid_alloc_checked(pmap, cpuid);
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);
+ cached = pmap_pcid_alloc_checked(pmap, pcidp);
cr3 = rcr3();
if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)
- load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid |
- cached);
+ load_cr3(pmap->pm_cr3 | pcidp->pm_pcid | cached);
PCPU_SET(curpmap, pmap);
if (cached)
counter_u64_add(pcid_save_cnt, 1);
@@ -10143,7 +10165,7 @@
if (pti) {
kcr3 = pmap->pm_cr3;
if (pmap_pcid_enabled)
- kcr3 |= pmap->pm_pcids[cpuid].pm_pcid | CR3_PCID_SAVE;
+ kcr3 |= pmap_get_pcid(pmap) | CR3_PCID_SAVE;
} else {
kcr3 = PMAP_NO_CR3;
}
diff --git a/sys/amd64/include/_pmap.h b/sys/amd64/include/_pmap.h
new file mode 100644
--- /dev/null
+++ b/sys/amd64/include/_pmap.h
@@ -0,0 +1,9 @@
+#ifndef _MACHINE__PMAP_H_
+#define _MACHINE__PMAP_H_
+
+struct pmap_pcids {
+ uint32_t pm_pcid;
+ uint32_t pm_gen;
+};
+
+#endif
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -35,6 +35,7 @@
#ifndef _MACHINE_PCPU_H_
#define _MACHINE_PCPU_H_
+#include <machine/_pmap.h>
#include <machine/segments.h>
#include <machine/tss.h>
@@ -101,7 +102,8 @@
uint64_t pc_ucr3_load_mask; \
u_int pc_small_core; \
u_int pc_pcid_invlpg_workaround; \
- char __pad[2908] /* pad to UMA_PCPU_ALLOC_SIZE */
+ struct pmap_pcids pc_kpmap_store; \
+ char __pad[2900] /* pad to UMA_PCPU_ALLOC_SIZE */
#define PC_DBREG_CMD_NONE 0
#define PC_DBREG_CMD_LOAD 1
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -291,6 +291,7 @@
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_pctrie.h>
+#include <machine/_pmap.h>
#include <sys/_pv_entry.h>
#include <sys/_rangeset.h>
#include <sys/_smr.h>
@@ -371,11 +372,6 @@
PT_RVI, /* AMD's nested page tables */
};
-struct pmap_pcids {
- uint32_t pm_pcid;
- uint32_t pm_gen;
-};
-
/*
* The kernel virtual address (KVA) of the level 4 page table page is always
* within the direct map (DMAP) region.
@@ -394,7 +390,7 @@
long pm_eptgen; /* EPT pmap generation id */
smr_t pm_eptsmr;
int pm_flags;
- struct pmap_pcids pm_pcids[MAXCPU];
+ struct pmap_pcids *pm_pcidp;
struct rangeset pm_pkru;
};
@@ -537,6 +533,17 @@
}
#endif /* sys/pcpu.h && machine/cpufunc.h */
+/* Return pcid for the pmap pmap on current cpu */
+static __inline uint32_t
+pmap_get_pcid(pmap_t pmap)
+{
+ struct pmap_pcids *pcidp;
+
+ MPASS(pmap_pcid_enabled);
+ pcidp = zpcpu_get_cpu(pmap->pm_pcidp, PCPU_GET(cpuid));
+ return (pcidp->pm_pcid);
+}
+
#endif /* _KERNEL */
/* Return various clipped indexes for a given VA */
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Mon, May 18, 1:08 PM (13 h, 55 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
33249655
Default Alt Text
D39890.id121283.diff (14 KB)
Attached To
Mode
D39890: amd64: store pcids pmap data in pcpu zone
Attached
Detach File
Event Timeline
Log In to Comment