Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 353 Lines • ▼ Show 20 Lines | |||||
* which indicates that an ASID should never be allocated to the pmap, and | * which indicates that an ASID should never be allocated to the pmap, and | ||||
* (2) COOKIE_FROM(-1, INT_MAX), which indicates that an ASID should be | * (2) COOKIE_FROM(-1, INT_MAX), which indicates that an ASID should be | ||||
* allocated when the pmap is next activated. | * allocated when the pmap is next activated. | ||||
*/ | */ | ||||
#define COOKIE_FROM(asid, epoch) ((long)((u_int)(asid) | \ | #define COOKIE_FROM(asid, epoch) ((long)((u_int)(asid) | \ | ||||
((u_long)(epoch) << 32))) | ((u_long)(epoch) << 32))) | ||||
#define COOKIE_TO_ASID(cookie) ((int)(cookie)) | #define COOKIE_TO_ASID(cookie) ((int)(cookie)) | ||||
#define COOKIE_TO_EPOCH(cookie) ((int)((u_long)(cookie) >> 32)) | #define COOKIE_TO_EPOCH(cookie) ((int)((u_long)(cookie) >> 32)) | ||||
#define COOKIE_TO_EPOCH_ATOMIC(cookie) ((int)(atomic_load_long( \ | |||||
&(cookie)) >> 32)) | |||||
static int superpages_enabled = 1; | static int superpages_enabled = 1; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, | SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, | ||||
CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0, | CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0, | ||||
"Are large page mappings enabled?"); | "Are large page mappings enabled?"); | ||||
/* | /* | ||||
* Internal flags for pmap_enter()'s helper functions. | * Internal flags for pmap_enter()'s helper functions. | ||||
▲ Show 20 Lines • Show All 800 Lines • ▼ Show 20 Lines | |||||
static u_long pmap_l2_p_failures; | static u_long pmap_l2_p_failures; | ||||
SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, p_failures, CTLFLAG_RD, | SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, p_failures, CTLFLAG_RD, | ||||
&pmap_l2_p_failures, 0, "2MB page promotion failures"); | &pmap_l2_p_failures, 0, "2MB page promotion failures"); | ||||
static u_long pmap_l2_promotions; | static u_long pmap_l2_promotions; | ||||
SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, promotions, CTLFLAG_RD, | SYSCTL_ULONG(_vm_pmap_l2, OID_AUTO, promotions, CTLFLAG_RD, | ||||
&pmap_l2_promotions, 0, "2MB page promotions"); | &pmap_l2_promotions, 0, "2MB page promotions"); | ||||
static bool | |||||
pmap_is_current_epoch(pmap_t pmap) | |||||
{ | |||||
struct asid_set *set; | |||||
int asid_epoch, epoch; | |||||
/* The kernel pmap is always current, and may not be locked */ | |||||
if (pmap == kernel_pmap) | |||||
return (true); | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
set = pmap->pm_asid_set; | |||||
/* Check if the epoch is marked as invalid */ | |||||
epoch = COOKIE_TO_EPOCH_ATOMIC(pmap->pm_cookie); | |||||
if (epoch == INT_MAX) | |||||
return (false); | |||||
/* Assume current while pmap_reset_asid_set is being called */ | |||||
asid_epoch = atomic_load_int(&set->asid_epoch); | |||||
if (asid_epoch == -1) | |||||
return (true); | |||||
return (epoch == asid_epoch); | |||||
} | |||||
/* | /* | ||||
* Invalidate a single TLB entry. | * Invalidate a single TLB entry. | ||||
*/ | */ | ||||
static __inline void | static __inline void | ||||
pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
uint64_t r; | uint64_t r; | ||||
Show All 37 Lines | |||||
static __inline void | static __inline void | ||||
pmap_invalidate_all(pmap_t pmap) | pmap_invalidate_all(pmap_t pmap) | ||||
{ | { | ||||
uint64_t r; | uint64_t r; | ||||
PMAP_ASSERT_STAGE1(pmap); | PMAP_ASSERT_STAGE1(pmap); | ||||
/* | |||||
* Invalidating all entries can be expensive. Only do so if the pmap | |||||
* is on the current epoch. If not the pmap will have not been | |||||
* activated since the last time the entire tlb was invalidated so | |||||
* won't have been used. | |||||
*/ | |||||
if (!pmap_is_current_epoch(pmap)) | |||||
return; | |||||
dsb(ishst); | dsb(ishst); | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
__asm __volatile("tlbi vmalle1is"); | __asm __volatile("tlbi vmalle1is"); | ||||
} else { | } else { | ||||
r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); | r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); | ||||
__asm __volatile("tlbi aside1is, %0" : : "r" (r)); | __asm __volatile("tlbi aside1is, %0" : : "r" (r)); | ||||
} | } | ||||
dsb(ish); | dsb(ish); | ||||
▲ Show 20 Lines • Show All 5,207 Lines • ▼ Show 20 Lines | pmap_reset_asid_set(pmap_t pmap) | ||||
/* | /* | ||||
* Ensure that the store to asid_epoch is globally visible before the | * Ensure that the store to asid_epoch is globally visible before the | ||||
* loads from pc_curpmap are performed. | * loads from pc_curpmap are performed. | ||||
*/ | */ | ||||
epoch = set->asid_epoch + 1; | epoch = set->asid_epoch + 1; | ||||
if (epoch == INT_MAX) | if (epoch == INT_MAX) | ||||
epoch = 0; | epoch = 0; | ||||
set->asid_epoch = epoch; | atomic_store_int(&set->asid_epoch, -1); | ||||
dsb(ishst); | dsb(ishst); | ||||
if (stage == PM_STAGE1) { | if (stage == PM_STAGE1) { | ||||
__asm __volatile("tlbi vmalle1is"); | __asm __volatile("tlbi vmalle1is"); | ||||
} else { | } else { | ||||
KASSERT(pmap_clean_stage2_tlbi != NULL, | KASSERT(pmap_clean_stage2_tlbi != NULL, | ||||
("%s: Unset stage 2 tlb invalidation callback\n", | ("%s: Unset stage 2 tlb invalidation callback\n", | ||||
__func__)); | __func__)); | ||||
pmap_clean_stage2_tlbi(); | pmap_clean_stage2_tlbi(); | ||||
Show All 13 Lines | if (stage == PM_STAGE1) { | ||||
continue; | continue; | ||||
PMAP_ASSERT_STAGE2(pmap); | PMAP_ASSERT_STAGE2(pmap); | ||||
} | } | ||||
KASSERT(curpmap->pm_asid_set == set, ("Incorrect set")); | KASSERT(curpmap->pm_asid_set == set, ("Incorrect set")); | ||||
asid = COOKIE_TO_ASID(curpmap->pm_cookie); | asid = COOKIE_TO_ASID(curpmap->pm_cookie); | ||||
if (asid == -1) | if (asid == -1) | ||||
continue; | continue; | ||||
bit_set(set->asid_set, asid); | bit_set(set->asid_set, asid); | ||||
curpmap->pm_cookie = COOKIE_FROM(asid, epoch); | atomic_store_long(&curpmap->pm_cookie, | ||||
COOKIE_FROM(asid, epoch)); | |||||
} | } | ||||
atomic_store_rel_int(&set->asid_epoch, epoch); | |||||
} | } | ||||
/* | /* | ||||
* Allocate a new ASID for the specified pmap. | * Allocate a new ASID for the specified pmap. | ||||
*/ | */ | ||||
static void | static void | ||||
pmap_alloc_asid(pmap_t pmap) | pmap_alloc_asid(pmap_t pmap) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 672 Lines • Show Last 20 Lines |