diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -372,6 +372,8 @@ void (*pmap_clean_stage2_tlbi)(void); void (*pmap_invalidate_vpipt_icache)(void); +void (*pmap_stage2_invalidate_range)(uint64_t, vm_offset_t, vm_offset_t, bool); +void (*pmap_stage2_invalidate_all)(uint64_t); /* * A pmap's cookie encodes an ASID and epoch number. Cookies for reserved @@ -1485,6 +1487,24 @@ isb(); } +static __inline void +pmap_s2_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) +{ + PMAP_ASSERT_STAGE2(pmap); + MPASS(pmap_stage2_invalidate_range != NULL); + pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), va, va + PAGE_SIZE, + final_only); +} + +static __inline void +pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) +{ + if (pmap->pm_stage == PM_STAGE1) + pmap_s1_invalidate_page(pmap, va, final_only); + else + pmap_s2_invalidate_page(pmap, va, final_only); +} + /* * Invalidates any cached final- and optionally intermediate-level TLB entries * for the specified virtual address range in the given virtual address space. @@ -1514,6 +1534,25 @@ isb(); } +static __inline void +pmap_s2_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, + bool final_only) +{ + PMAP_ASSERT_STAGE2(pmap); + MPASS(pmap_stage2_invalidate_range != NULL); + pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), sva, eva, final_only); +} + +static __inline void +pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, + bool final_only) +{ + if (pmap->pm_stage == PM_STAGE1) + pmap_s1_invalidate_range(pmap, sva, eva, final_only); + else + pmap_s2_invalidate_range(pmap, sva, eva, final_only); +} + /* * Invalidates all cached intermediate- and final-level TLB entries for the * given virtual address space. @@ -1536,6 +1575,23 @@ isb(); } +static __inline void +pmap_s2_invalidate_all(pmap_t pmap) +{ + PMAP_ASSERT_STAGE2(pmap); + MPASS(pmap_stage2_invalidate_all != NULL); + pmap_stage2_invalidate_all(pmap_to_ttbr0(pmap)); +} + +static __inline void +pmap_invalidate_all(pmap_t pmap) +{ + if (pmap->pm_stage == PM_STAGE1) + pmap_s1_invalidate_all(pmap); + else + pmap_s2_invalidate_all(pmap); +} + /* * Routine: pmap_extract * Function: @@ -1979,7 +2035,7 @@ l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); pmap_unwire_l3(pmap, va, l1pg, free); } - pmap_s1_invalidate_page(pmap, va, false); + pmap_invalidate_page(pmap, va, false); /* * Put page on a list so that it is released after @@ -3280,7 +3336,7 @@ for (l3 = pmap_l2_to_l3(&l2e, sva); sva != eva; l3++, sva += L3_SIZE) { if (!pmap_l3_valid(pmap_load(l3))) { if (va != eva) { - pmap_s1_invalidate_range(pmap, va, sva, true); + pmap_invalidate_range(pmap, va, sva, true); va = eva; } continue; @@ -3307,7 +3363,7 @@ * still provides access to that page. */ if (va != eva) { - pmap_s1_invalidate_range(pmap, va, + pmap_invalidate_range(pmap, va, sva, true); va = eva; } @@ -3338,7 +3394,7 @@ va = sva; } if (va != eva) - pmap_s1_invalidate_range(pmap, va, sva, true); + pmap_invalidate_range(pmap, va, sva, true); } /* @@ -5482,7 +5538,7 @@ } if (lock != NULL) rw_wunlock(lock); - pmap_s1_invalidate_all(pmap); + pmap_invalidate_all(pmap); free_pv_chunk_batch(free_chunks); PMAP_UNLOCK(pmap); vm_page_free_pages_toq(&free, true); @@ -5807,7 +5863,7 @@ (uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 && (tpte & ATTR_SW_WIRED) == 0) { pmap_clear_bits(pte, ATTR_AF); - pmap_s1_invalidate_page(pmap, va, true); + pmap_invalidate_page(pmap, va, true); cleared++; } else not_cleared++; @@ -5848,7 +5904,7 @@ if ((tpte & ATTR_AF) != 0) { if ((tpte & ATTR_SW_WIRED) == 0) { pmap_clear_bits(pte, ATTR_AF); - pmap_s1_invalidate_page(pmap, pv->pv_va, true); + pmap_invalidate_page(pmap, pv->pv_va, true); cleared++; } else not_cleared++; diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h --- a/sys/arm64/include/pmap.h +++ b/sys/arm64/include/pmap.h @@ -172,6 +172,9 @@ extern void (*pmap_clean_stage2_tlbi)(void); extern void (*pmap_invalidate_vpipt_icache)(void); +extern void (*pmap_stage2_invalidate_range)(uint64_t, vm_offset_t, vm_offset_t, + bool); +extern void (*pmap_stage2_invalidate_all)(uint64_t); static inline int pmap_vmspace_copy(pmap_t dst_pmap __unused, pmap_t src_pmap __unused)