Page MenuHomeFreeBSD
Files F136916521

D51565.id159637.diff
No OneTemporary
Actions

D51565.id159637.diff
View Options

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -1336,12 +1336,9 @@
pti = pti_get_default();
TUNABLE_INT_FETCH("vm.pmap.pti", &pti);
TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled);
- if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) {
- invpcid_works = (cpu_stdext_feature &
- CPUID_STDEXT_INVPCID) != 0;
- } else {
+ if ((cpu_feature2 & CPUID2_PCID) == 0)
pmap_pcid_enabled = 0;
- }
+ invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) != 0;
/*
* Now we can do small core initialization, after the PCID
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -2904,6 +2904,9 @@
/*
* Promotion: flush every 4KB page mapping from the TLB,
* including any global (PG_G) mappings.
+ *
+ * This function is only used on older processors that
+ * do not support the invpcid instruction.
*/
invltlb_glob();
}
@@ -3050,13 +3053,13 @@
* local user page: INVLPG
* local kernel page: INVLPG
* local user total: reload %cr3
- * local kernel total: invltlb_glob()
+ * local kernel total: INVPCID(CTXGLOB) or invltlb_glob()
* remote user page, inactive pmap: -
* remote user page, active pmap: IPI:INVLPG
* remote kernel page: IPI:INVLPG
* remote user total, inactive pmap: -
* remote user total, active pmap: IPI:(reload %cr3)
- * remote kernel total: IPI:invltlb_glob()
+ * remote kernel total: IPI:INVPCID(CTXGLOB) or invltlb_glob()
* Since on return to user mode, the reload of %cr3 with ucr3 causes
* TLB invalidation, no specific action is required for user page table.
*
@@ -3356,7 +3359,8 @@
}
static inline void
-pmap_invalidate_all_pcid_cb(pmap_t pmap, bool invpcid_works1)
+pmap_invalidate_all_cb_template(pmap_t pmap, bool pmap_pcid_enabled1,
+ bool invpcid_works1)
{
struct invpcid_descr d;
uint64_t kcr3;
@@ -3370,57 +3374,63 @@
invltlb_glob();
}
} else if (pmap == PCPU_GET(curpmap)) {
- CRITICAL_ASSERT(curthread);
-
- pcid = pmap_get_pcid(pmap);
- if (invpcid_works1) {
- d.pcid = pcid;
- d.pad = 0;
- d.addr = 0;
- invpcid(&d, INVPCID_CTX);
+ if (pmap_pcid_enabled1) {
+ CRITICAL_ASSERT(curthread);
+
+ pcid = pmap_get_pcid(pmap);
+ if (invpcid_works1) {
+ d.pcid = pcid;
+ d.pad = 0;
+ d.addr = 0;
+ invpcid(&d, INVPCID_CTX);
+ } else {
+ kcr3 = pmap->pm_cr3 | pcid;
+ load_cr3(kcr3);
+ }
+ if (pmap->pm_ucr3 != PMAP_NO_CR3)
+ PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
} else {
- kcr3 = pmap->pm_cr3 | pcid;
- load_cr3(kcr3);
+ invltlb();
}
- if (pmap->pm_ucr3 != PMAP_NO_CR3)
- PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
}
}
static void
-pmap_invalidate_all_pcid_invpcid_cb(pmap_t pmap)
+pmap_invalidate_all_pcid_invpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,
+ vm_offset_t addr2 __unused)
{
- pmap_invalidate_all_pcid_cb(pmap, true);
+ pmap_invalidate_all_cb_template(pmap, true, true);
}
static void
-pmap_invalidate_all_pcid_noinvpcid_cb(pmap_t pmap)
+pmap_invalidate_all_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,
+ vm_offset_t addr2 __unused)
{
- pmap_invalidate_all_pcid_cb(pmap, false);
+ pmap_invalidate_all_cb_template(pmap, true, false);
}
static void
-pmap_invalidate_all_nopcid_cb(pmap_t pmap)
+pmap_invalidate_all_nopcid_invpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,
+ vm_offset_t addr2 __unused)
{
- if (pmap == kernel_pmap)
- invltlb_glob();
- else if (pmap == PCPU_GET(curpmap))
- invltlb();
+ pmap_invalidate_all_cb_template(pmap, false, true);
}
-DEFINE_IFUNC(static, void, pmap_invalidate_all_cb, (pmap_t))
+static void
+pmap_invalidate_all_nopcid_noinvpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,
+ vm_offset_t addr2 __unused)
{
- if (pmap_pcid_enabled)
- return (invpcid_works ? pmap_invalidate_all_pcid_invpcid_cb :
- pmap_invalidate_all_pcid_noinvpcid_cb);
- return (pmap_invalidate_all_nopcid_cb);
+ pmap_invalidate_all_cb_template(pmap, false, false);
}
-static void
-pmap_invalidate_all_curcpu_cb(pmap_t pmap, vm_offset_t addr1 __unused,
- vm_offset_t addr2 __unused)
+DEFINE_IFUNC(static, void, pmap_invalidate_all_curcpu_cb, (pmap_t, vm_offset_t,
+ vm_offset_t))
{
- pmap_invalidate_all_cb(pmap);
+ if (pmap_pcid_enabled)
+ return (invpcid_works ? pmap_invalidate_all_pcid_invpcid_cb :
+ pmap_invalidate_all_pcid_noinvpcid_cb);
+ return (invpcid_works ? pmap_invalidate_all_nopcid_invpcid_cb :
+ pmap_invalidate_all_nopcid_noinvpcid_cb);
}
void

File Metadata

Mime Type
text/plain
Expires
Fri, Nov 21, 3:44 PM (1 h, 56 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
25773880
Default Alt Text
D51565.id159637.diff (4 KB)

Event Timeline