Page MenuHomeFreeBSD
Files F109391352

D32792.id97839.diff
No OneTemporary
Actions

D32792.id97839.diff
View Options

Index: sys/amd64/amd64/mp_machdep.c
===================================================================
--- sys/amd64/amd64/mp_machdep.c
+++ sys/amd64/amd64/mp_machdep.c
@@ -613,7 +613,7 @@
* completion.
*/
static void
-smp_targeted_tlb_shootdown(cpuset_t mask, pmap_t pmap, vm_offset_t addr1,
+smp_targeted_tlb_shootdown(const cpuset_t *mask, pmap_t pmap, vm_offset_t addr1,
vm_offset_t addr2, smp_invl_cb_t curcpu_cb, enum invl_op_codes op)
{
cpuset_t other_cpus;
@@ -625,7 +625,7 @@
* It is not necessary to signal other CPUs while booting or
* when in the debugger.
*/
- if (kdb_active || KERNEL_PANICKED() || !smp_started)
+ if (__predict_false(kdb_active || KERNEL_PANICKED() || !smp_started))
goto local_cb;
KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
@@ -633,9 +633,10 @@
/*
* Check for other cpus. Return if none.
*/
- is_all = !CPU_CMP(&mask, &all_cpus);
- CPU_CLR(PCPU_GET(cpuid), &mask);
- if (CPU_EMPTY(&mask))
+ CPU_COPY(mask, &other_cpus);
+ is_all = !CPU_CMP(mask, &all_cpus);
+ CPU_CLR(curcpu, &other_cpus);
+ if (CPU_EMPTY(&other_cpus))
goto local_cb;
/*
@@ -660,10 +661,10 @@
/* Fence between filling smp_tlb fields and clearing scoreboard. */
atomic_thread_fence_rel();
- CPU_FOREACH_ISSET(cpu, &mask) {
+ CPU_FOREACH_ISSET(cpu, &other_cpus) {
KASSERT(*invl_scoreboard_slot(cpu) != 0,
("IPI scoreboard is zero, initiator %d target %d",
- PCPU_GET(cpuid), cpu));
+ curcpu, cpu));
*invl_scoreboard_slot(cpu) = 0;
}
@@ -674,11 +675,8 @@
*/
if (is_all) {
ipi_all_but_self(IPI_INVLOP);
- other_cpus = all_cpus;
- CPU_CLR(PCPU_GET(cpuid), &other_cpus);
} else {
- other_cpus = mask;
- ipi_selected(mask, IPI_INVLOP);
+ ipi_selected(other_cpus, IPI_INVLOP);
}
curcpu_cb(pmap, addr1, addr2);
CPU_FOREACH_ISSET(cpu, &other_cpus) {
@@ -705,7 +703,7 @@
}
void
-smp_masked_invltlb(cpuset_t mask, pmap_t pmap, smp_invl_cb_t curcpu_cb)
+smp_masked_invltlb(const cpuset_t *mask, pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
smp_targeted_tlb_shootdown(mask, pmap, 0, 0, curcpu_cb, invl_op_tlb);
#ifdef COUNT_XINVLTLB_HITS
@@ -714,7 +712,7 @@
}
void
-smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, pmap_t pmap,
+smp_masked_invlpg(const cpuset_t *mask, vm_offset_t addr, pmap_t pmap,
smp_invl_cb_t curcpu_cb)
{
smp_targeted_tlb_shootdown(mask, pmap, addr, 0, curcpu_cb, invl_op_pg);
@@ -724,8 +722,8 @@
}
void
-smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2,
- pmap_t pmap, smp_invl_cb_t curcpu_cb)
+smp_masked_invlpg_range(const cpuset_t *mask, vm_offset_t addr1,
+ vm_offset_t addr2, pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
smp_targeted_tlb_shootdown(mask, pmap, addr1, addr2, curcpu_cb,
invl_op_pgrng);
@@ -738,7 +736,7 @@
void
smp_cache_flush(smp_invl_cb_t curcpu_cb)
{
- smp_targeted_tlb_shootdown(all_cpus, NULL, 0, 0, curcpu_cb,
+ smp_targeted_tlb_shootdown(&all_cpus, NULL, 0, 0, curcpu_cb,
INVL_OP_CACHE);
}
Index: sys/amd64/amd64/pmap.c
===================================================================
--- sys/amd64/amd64/pmap.c
+++ sys/amd64/amd64/pmap.c
@@ -2960,7 +2960,7 @@
* This will force the vcpu to exit and the cached EPT mappings
* will be invalidated by the host before the next vmresume.
*/
-static __inline void
+static __noinline void
pmap_invalidate_ept(pmap_t pmap)
{
smr_seq_t goal;
@@ -3010,10 +3010,15 @@
smr_wait(pmap->pm_eptsmr, goal);
}
-static cpuset_t
+/*
+ * Returns a reference to a set of CPUs on which the pmap is currently active.
+ * Note that the set can be modified without any mutual exclusion, so a copy
+ * must be made if a stable snapshot is required.
+ */
+static volatile cpuset_t *
pmap_invalidate_cpu_mask(pmap_t pmap)
{
- return (pmap == kernel_pmap ? all_cpus : pmap->pm_active);
+ return (pmap == kernel_pmap ? &all_cpus : &pmap->pm_active);
}
static inline void
@@ -3138,6 +3143,8 @@
void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
+ cpuset_t mask;
+
if (pmap_type_guest(pmap)) {
pmap_invalidate_ept(pmap);
return;
@@ -3147,8 +3154,8 @@
("pmap_invalidate_page: invalid type %d", pmap->pm_type));
pmap_invalidate_preipi(pmap);
- smp_masked_invlpg(pmap_invalidate_cpu_mask(pmap), va, pmap,
- pmap_invalidate_page_curcpu_cb);
+ mask = *pmap_invalidate_cpu_mask(pmap);
+ smp_masked_invlpg(&mask, va, pmap, pmap_invalidate_page_curcpu_cb);
}
/* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */
@@ -3232,6 +3239,8 @@
void
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
+ cpuset_t mask;
+
if (eva - sva >= PMAP_INVLPG_THRESHOLD) {
pmap_invalidate_all(pmap);
return;
@@ -3246,7 +3255,8 @@
("pmap_invalidate_range: invalid type %d", pmap->pm_type));
pmap_invalidate_preipi(pmap);
- smp_masked_invlpg_range(pmap_invalidate_cpu_mask(pmap), sva, eva, pmap,
+ mask = *pmap_invalidate_cpu_mask(pmap);
+ smp_masked_invlpg_range(&mask, sva, eva, pmap,
pmap_invalidate_range_curcpu_cb);
}
@@ -3323,6 +3333,8 @@
void
pmap_invalidate_all(pmap_t pmap)
{
+ cpuset_t mask;
+
if (pmap_type_guest(pmap)) {
pmap_invalidate_ept(pmap);
return;
@@ -3332,8 +3344,8 @@
("pmap_invalidate_all: invalid type %d", pmap->pm_type));
pmap_invalidate_preipi(pmap);
- smp_masked_invltlb(pmap_invalidate_cpu_mask(pmap), pmap,
- pmap_invalidate_all_curcpu_cb);
+ mask = *pmap_invalidate_cpu_mask(pmap);
+ smp_masked_invltlb(&mask, pmap, pmap_invalidate_all_curcpu_cb);
}
static void
Index: sys/x86/include/x86_smp.h
===================================================================
--- sys/x86/include/x86_smp.h
+++ sys/x86/include/x86_smp.h
@@ -107,14 +107,24 @@
void ipi_selected(cpuset_t cpus, u_int ipi);
void ipi_self_from_nmi(u_int vector);
void set_interrupt_apic_ids(void);
+void mem_range_AP_init(void);
+void topo_probe(void);
+
+/* functions in mp_machdep.c */
void smp_cache_flush(smp_invl_cb_t curcpu_cb);
+#ifdef __i386__
void smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, struct pmap *pmap,
smp_invl_cb_t curcpu_cb);
void smp_masked_invlpg_range(cpuset_t mask, vm_offset_t startva,
vm_offset_t endva, struct pmap *pmap, smp_invl_cb_t curcpu_cb);
void smp_masked_invltlb(cpuset_t mask, struct pmap *pmap,
smp_invl_cb_t curcpu_cb);
-void mem_range_AP_init(void);
-void topo_probe(void);
-
+#else
+void smp_masked_invlpg(const cpuset_t *mask, vm_offset_t addr,
+ struct pmap *pmap, smp_invl_cb_t curcpu_cb);
+void smp_masked_invlpg_range(const cpuset_t *mask, vm_offset_t startva,
+ vm_offset_t endva, struct pmap *pmap, smp_invl_cb_t curcpu_cb);
+void smp_masked_invltlb(const cpuset_t *mask, struct pmap *pmap,
+ smp_invl_cb_t curcpu_cb);
+#endif
#endif

File Metadata

Mime Type
text/plain
Expires
Wed, Feb 5, 11:17 AM (3 h, 27 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16473194
Default Alt Text
D32792.id97839.diff (6 KB)

Event Timeline