Index: sys/arm64/arm64/pmap.c =================================================================== --- sys/arm64/arm64/pmap.c +++ sys/arm64/arm64/pmap.c @@ -359,6 +359,8 @@ ((u_long)(epoch) << 32))) #define COOKIE_TO_ASID(cookie) ((int)(cookie)) #define COOKIE_TO_EPOCH(cookie) ((int)((u_long)(cookie) >> 32)) +#define COOKIE_TO_EPOCH_ATOMIC(cookie) ((int)(atomic_load_long( \ + &(cookie)) >> 32)) static int superpages_enabled = 1; SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, @@ -1175,6 +1177,32 @@ SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, promotions, CTLFLAG_RD, &pmap_l2_promotions, 0, "2MB page promotions"); +static bool +pmap_is_current_epoch(pmap_t pmap) +{ + struct asid_set *set; + int asid_epoch, epoch; + + /* The kernel pmap is always current, and may not be locked */ + if (pmap == kernel_pmap) + return (true); + + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + set = pmap->pm_asid_set; + + /* Check if the epoch is marked as invalid */ + epoch = COOKIE_TO_EPOCH_ATOMIC(pmap->pm_cookie); + if (epoch == INT_MAX) + return (false); + + /* Assume current while pmap_reset_asid_set is being called */ + asid_epoch = atomic_load_int(&set->asid_epoch); + if (asid_epoch == -1) + return (true); + + return (epoch == asid_epoch); +} + /* * Invalidate a single TLB entry. */ @@ -1228,6 +1256,15 @@ PMAP_ASSERT_STAGE1(pmap); + /* + * Invalidating all entries can be expensive. Only do so if the pmap + * is on the current epoch. If not the pmap will have not been + * activated since the last time the entire tlb was invalidated so + * won't have been used. + */ + if (!pmap_is_current_epoch(pmap)) + return; + dsb(ishst); if (pmap == kernel_pmap) { __asm __volatile("tlbi vmalle1is"); @@ -6451,7 +6488,7 @@ epoch = set->asid_epoch + 1; if (epoch == INT_MAX) epoch = 0; - set->asid_epoch = epoch; + atomic_store_int(&set->asid_epoch, -1); dsb(ishst); if (stage == PM_STAGE1) { __asm __volatile("tlbi vmalle1is"); @@ -6481,8 +6518,10 @@ if (asid == -1) continue; bit_set(set->asid_set, asid); - curpmap->pm_cookie = COOKIE_FROM(asid, epoch); + atomic_store_long(&curpmap->pm_cookie, + COOKIE_FROM(asid, epoch)); } + atomic_store_rel_int(&set->asid_epoch, epoch); } /*