diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -360,6 +360,10 @@ #define COOKIE_TO_ASID(cookie) ((int)(cookie)) #define COOKIE_TO_EPOCH(cookie) ((int)((u_long)(cookie) >> 32)) +#define TLBI_VA_SHIFT 12 +#define TLBI_VA(addr) ((addr) >> TLBI_VA_SHIFT) +#define TLBI_VA_L3_INCR (L3_SIZE >> TLBI_VA_SHIFT) + static int superpages_enabled = 1; SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0, @@ -1248,11 +1252,11 @@ PMAP_ASSERT_STAGE1(pmap); dsb(ishst); + r = TLBI_VA(va); if (pmap == kernel_pmap) { - r = atop(va); pmap_invalidate_kernel(r, final_only); } else { - r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)) | atop(va); + r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); pmap_invalidate_user(r, final_only); } dsb(ish); @@ -1273,15 +1277,15 @@ dsb(ishst); if (pmap == kernel_pmap) { - start = atop(sva); - end = atop(eva); - for (r = start; r < end; r++) + start = TLBI_VA(sva); + end = TLBI_VA(eva); + for (r = start; r < end; r += TLBI_VA_L3_INCR) pmap_invalidate_kernel(r, final_only); } else { start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); - start |= atop(sva); - end |= atop(eva); - for (r = start; r < end; r++) + start |= TLBI_VA(sva); + end |= TLBI_VA(eva); + for (r = start; r < end; r += TLBI_VA_L3_INCR) pmap_invalidate_user(r, final_only); } dsb(ish);