Page MenuHomeFreeBSD
Files F167314805

D59048.id184555.diff
No OneTemporary
Actions

D59048.id184555.diff
View Options

diff --git a/sys/riscv/include/cpufunc.h b/sys/riscv/include/cpufunc.h
--- a/sys/riscv/include/cpufunc.h
+++ b/sys/riscv/include/cpufunc.h
@@ -90,6 +90,9 @@
__asm __volatile("fence.i" ::: "memory");
}
+/*
+ * Invalidate the entire local TLB. This includes global mappings.
+ */
static __inline void
sfence_vma(void)
{
@@ -97,6 +100,10 @@
__asm __volatile("sfence.vma" ::: "memory");
}
+/*
+ * Invalidate any leaf PTEs corresponding to addr, across all address spaces.
+ * This includes global mappings.
+ */
static __inline void
sfence_vma_page(uintptr_t addr)
{
@@ -104,6 +111,10 @@
__asm __volatile("sfence.vma %0" :: "r" (addr) : "memory");
}
+/*
+ * Invalidate all mappings for the address space denoted by asid. An asid value
+ * of zero (not register x0) will invalidate all non-global mappings.
+ */
static __inline void
sfence_vma_asid(uint64_t asid)
{
@@ -111,8 +122,12 @@
__asm __volatile("sfence.vma x0, %0" :: "r" (asid) : "memory");
}
+/*
+ * Invalidate any leaf PTEs corresponding to addr, in the address space denoted
+ * by asid. Global mappings are not affected.
+ */
static __inline void
-sfence_vma_asid_page(uint64_t asid, uintptr_t addr)
+sfence_vma_page_asid(uint64_t addr, uintptr_t asid)
{
__asm __volatile("sfence.vma %0, %1" :: "r" (addr), "r" (asid)
diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c
--- a/sys/riscv/riscv/pmap.c
+++ b/sys/riscv/riscv/pmap.c
@@ -1070,46 +1070,108 @@
pagesizes[1] = L2_SIZE;
}
+static __inline void
+pmap_local_invalidate_all(pmap_t pmap)
+{
+ if (pmap == kernel_pmap)
+ sfence_vma();
+ else
+ sfence_vma_asid(0);
+}
+
+
#ifdef SMP
/*
* For SMP, these functions have to use IPIs for coherence.
*
* In general, the calling thread uses a plain fence to order the
* writes to the page tables before invoking an SBI callback to invoke
- * sfence_vma() on remote CPUs.
+ * sfence.vma on remote CPUs.
*/
+
static void
-pmap_invalidate_page_sbi(pmap_t pmap, vm_offset_t va)
+pmap_invalidate_page_sbi_global(pmap_t pmap, vm_offset_t va)
{
cpuset_t mask;
sched_pin();
mask = pmap->pm_active;
CPU_CLR(PCPU_GET(hart), &mask);
+
fence();
if (!CPU_EMPTY(&mask) && smp_started)
sbi_remote_sfence_vma(mask.__bits, va, 1);
sfence_vma_page(va);
+
sched_unpin();
}
static void
-pmap_invalidate_range_sbi(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+pmap_invalidate_page_sbi(pmap_t pmap, vm_offset_t va)
{
cpuset_t mask;
+ if (pmap == kernel_pmap) {
+ pmap_invalidate_page_sbi_global(pmap, va);
+ return;
+ }
+
sched_pin();
mask = pmap->pm_active;
CPU_CLR(PCPU_GET(hart), &mask);
+
fence();
if (!CPU_EMPTY(&mask) && smp_started)
- sbi_remote_sfence_vma(mask.__bits, sva, eva - sva + 1);
+ sbi_remote_sfence_vma_asid(mask.__bits, va, 1, 0);
+ sfence_vma_page_asid(va, 0);
+
+ sched_unpin();
+}
+
+static void
+pmap_invalidate_range_sbi_global(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+{
+ cpuset_t mask;
+
+ sched_pin();
+ mask = pmap->pm_active;
+ CPU_CLR(PCPU_GET(hart), &mask);
+ fence();
+ if (!CPU_EMPTY(&mask) && smp_started)
+ sbi_remote_sfence_vma(mask.__bits, sva, eva - sva + 1);
/*
* Might consider a loop of sfence_vma_page() for a small
* number of pages in the future.
*/
sfence_vma();
+
+ sched_unpin();
+}
+
+static void
+pmap_invalidate_range_sbi(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+{
+ cpuset_t mask;
+
+ if (pmap == kernel_pmap) {
+ pmap_invalidate_range_sbi_global(pmap, sva, eva);
+ return;
+ }
+
+ sched_pin();
+ mask = pmap->pm_active;
+ CPU_CLR(PCPU_GET(hart), &mask);
+
+ fence();
+ if (!CPU_EMPTY(&mask) && smp_started)
+ sbi_remote_sfence_vma_asid(mask.__bits, sva, eva - sva + 1, 0);
+ /*
+ * Might consider a loop of sfence_vma_page_asid() for a small
+ * number of pages in the future.
+ */
+ sfence_vma_asid(0);
+
sched_unpin();
}
@@ -1118,6 +1180,7 @@
struct svinval_args {
vm_offset_t sva;
vm_offset_t eva;
+ bool do_global;
};
static void
@@ -1127,8 +1190,12 @@
vm_offset_t va;
sfence_w_inval();
- for (va = args->sva; va < args->eva; va += PAGE_SIZE)
- sinval_vma_page(va);
+ for (va = args->sva; va < args->eva; va += PAGE_SIZE) {
+ if (args->do_global)
+ sinval_vma_page(va);
+ else
+ sinval_vma_page_asid(va, 0);
+ }
sfence_inval_ir();
}
@@ -1148,6 +1215,7 @@
sched_pin();
args.sva = sva;
args.eva = eva;
+ args.do_global = pmap == kernel_pmap;
fence();
if (smp_started)
smp_rendezvous_cpus(pmap->pm_active, smp_no_rendezvous_barrier,
@@ -1181,7 +1249,7 @@
}
static void
-pmap_invalidate_all(pmap_t pmap)
+pmap_invalidate_all_global(pmap_t pmap)
{
cpuset_t mask;
@@ -1189,18 +1257,38 @@
mask = pmap->pm_active;
CPU_CLR(PCPU_GET(hart), &mask);
- /*
- * XXX: The SBI doc doesn't detail how to specify x0 as the
- * address to perform a global fence. BBL currently treats
- * all sfence_vma requests as global however.
- */
fence();
if (!CPU_EMPTY(&mask) && smp_started)
sbi_remote_sfence_vma(mask.__bits, 0, 0);
sfence_vma();
+
sched_unpin();
}
-#else
+
+static void
+pmap_invalidate_all(pmap_t pmap)
+{
+ cpuset_t mask;
+
+ if (pmap == kernel_pmap) {
+ pmap_invalidate_all_global(pmap);
+ return;
+ }
+
+ sched_pin();
+ mask = pmap->pm_active;
+ CPU_CLR(PCPU_GET(hart), &mask);
+
+ fence();
+ if (!CPU_EMPTY(&mask) && smp_started)
+ sbi_remote_sfence_vma_asid(mask.__bits, 0, 0, 0);
+ sfence_vma_asid(0);
+
+ sched_unpin();
+}
+
+#else /* !SMP */
+
/*
* Normal, non-SMP, invalidation functions.
* We inline these within pmap.c for speed.
@@ -1209,7 +1297,10 @@
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
- sfence_vma_page(va);
+ if (pmap == kernel_pmap)
+ sfence_vma_page(va);
+ else
+ sfence_vma_page_asid(va, 0);
}
static __inline void
@@ -1220,16 +1311,16 @@
* Might consider a loop of sfence_vma_page() for a small
* number of pages in the future.
*/
- sfence_vma();
+ pmap_local_invalidate_all(pmap);
}
static __inline void
pmap_invalidate_all(pmap_t pmap)
{
- sfence_vma();
+ pmap_local_invalidate_all(pmap);
}
-#endif
+#endif /* SMP */
/*
* Routine: pmap_extract
@@ -1403,7 +1494,7 @@
KASSERT(l3 != NULL, ("pmap_kremove: Invalid address"));
pmap_clear(l3);
- sfence_vma();
+ pmap_local_invalidate_all(kernel_pmap);
}
void
@@ -3044,7 +3135,8 @@
*/
if ((oldpte & bits) != bits)
pmap_store_bits(pte, bits);
- sfence_vma();
+ pmap_local_invalidate_all(pmap);
+
rv = 1;
done:
PMAP_UNLOCK(pmap);
@@ -5294,7 +5386,7 @@
#endif
PCPU_SET(curpmap, pmap);
- sfence_vma();
+ pmap_local_invalidate_all(pmap);
}
void
diff --git a/sys/riscv/vmm/vmm_fence.c b/sys/riscv/vmm/vmm_fence.c
--- a/sys/riscv/vmm/vmm_fence.c
+++ b/sys/riscv/vmm/vmm_fence.c
@@ -113,7 +113,7 @@
else
for (va = fence->start; va < fence->start + fence->size;
va += PAGE_SIZE)
- sfence_vma_asid_page(fence->asid, va);
+ sfence_vma_page_asid(va, fence->asid);
break;
default:
break;

File Metadata

Mime Type
text/plain
Expires
Fri, Aug 21, 8:47 PM (16 h, 4 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
37053928
Default Alt Text
D59048.id184555.diff (6 KB)

Event Timeline