Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | |||||
pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t mpte, | pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t mpte, | ||||
struct rwlock **lockp) | struct rwlock **lockp) | ||||
{ | { | ||||
pt_entry_t *firstl3, *l3, newl2, oldl3, pa; | pt_entry_t *firstl3, *l3, newl2, oldl3, pa; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PMAP_ASSERT_STAGE1(pmap); | PMAP_ASSERT_STAGE1(pmap); | ||||
/* | |||||
* Examine the first L3E in the specified PTP. Abort if this L3E is | |||||
* ineligible for promotion, invalid, or does not map the first 4KB | |||||
* physical page within a 2MB page. | |||||
*/ | |||||
firstl3 = (pt_entry_t *)PHYS_TO_DMAP(pmap_load(l2) & ~ATTR_MASK); | firstl3 = (pt_entry_t *)PHYS_TO_DMAP(pmap_load(l2) & ~ATTR_MASK); | ||||
newl2 = pmap_load(firstl3); | newl2 = pmap_load(firstl3); | ||||
if ((newl2 & ATTR_SW_NO_PROMOTE) != 0) | |||||
if (((newl2 & (~ATTR_MASK | ATTR_AF)) & L2_OFFSET) != ATTR_AF || | return; | ||||
(newl2 & ATTR_SW_NO_PROMOTE) != 0) { | if ((newl2 & ((~ATTR_MASK & L2_OFFSET) | ATTR_DESCR_MASK)) != L3_PAGE) { | ||||
atomic_add_long(&pmap_l2_p_failures, 1); | atomic_add_long(&pmap_l2_p_failures, 1); | ||||
CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return; | return; | ||||
} | } | ||||
/* | |||||
* Both here and in the below "for" loop, to allow for repromotion | |||||
* after MADV_FREE, conditionally write protect a clean L3E before | |||||
* possibly aborting the promotion due to other L3E attributes. Why? | |||||
* Suppose that MADV_FREE is applied to a part of a superpage, the | |||||
* address range [S, E). pmap_advise() will demote the superpage | |||||
* mapping, destroy the 4KB page mapping at the end of [S, E), and | |||||
* set AP_RO and clear AF in the L3Es for the rest of [S, E). Later, | |||||
* imagine that the memory in [S, E) is recycled, but the last 4KB | |||||
* page in [S, E) is not the last to be rewritten, or simply accessed. | |||||
* In other words, there is still a 4KB page in [S, E), call it P, | |||||
* that is writeable but AP_RO is set and AF is clear in P's L3E. | |||||
* Unless we write protect P before aborting the promotion, if and | |||||
* when P is finally rewritten, there won't be a page fault to trigger | |||||
* repromotion. | |||||
*/ | |||||
setl2: | setl2: | ||||
if ((newl2 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == | if ((newl2 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == | ||||
(ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { | (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { | ||||
/* | /* | ||||
* When the mapping is clean, i.e., ATTR_S1_AP_RO is set, | * When the mapping is clean, i.e., ATTR_S1_AP_RO is set, | ||||
* ATTR_SW_DBM can be cleared without a TLB invalidation. | * ATTR_SW_DBM can be cleared without a TLB invalidation. | ||||
*/ | */ | ||||
if (!atomic_fcmpset_64(firstl3, &newl2, newl2 & ~ATTR_SW_DBM)) | if (!atomic_fcmpset_64(firstl3, &newl2, newl2 & ~ATTR_SW_DBM)) | ||||
goto setl2; | goto setl2; | ||||
newl2 &= ~ATTR_SW_DBM; | newl2 &= ~ATTR_SW_DBM; | ||||
} | } | ||||
if ((newl2 & ATTR_AF) == 0) { | |||||
atomic_add_long(&pmap_l2_p_failures, 1); | |||||
CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" | |||||
" in pmap %p", va, pmap); | |||||
return; | |||||
} | |||||
pa = newl2 + L2_SIZE - PAGE_SIZE; | /* | ||||
* Examine each of the other L3Es in the specified PTP. Abort if this | |||||
* L3E maps an unexpected 4KB physical page or does not have identical | |||||
* characteristics to the first L3E. | |||||
*/ | |||||
pa = (newl2 & (~ATTR_MASK | ATTR_DESCR_MASK)) + L2_SIZE - PAGE_SIZE; | |||||
for (l3 = firstl3 + NL3PG - 1; l3 > firstl3; l3--) { | for (l3 = firstl3 + NL3PG - 1; l3 > firstl3; l3--) { | ||||
oldl3 = pmap_load(l3); | oldl3 = pmap_load(l3); | ||||
if ((oldl3 & (~ATTR_MASK | ATTR_DESCR_MASK)) != pa) { | |||||
atomic_add_long(&pmap_l2_p_failures, 1); | |||||
CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" | |||||
" in pmap %p", va, pmap); | |||||
return; | |||||
} | |||||
setl3: | setl3: | ||||
if ((oldl3 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == | if ((oldl3 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == | ||||
(ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { | (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { | ||||
/* | /* | ||||
* When the mapping is clean, i.e., ATTR_S1_AP_RO is | * When the mapping is clean, i.e., ATTR_S1_AP_RO is | ||||
* set, ATTR_SW_DBM can be cleared without a TLB | * set, ATTR_SW_DBM can be cleared without a TLB | ||||
* invalidation. | * invalidation. | ||||
*/ | */ | ||||
if (!atomic_fcmpset_64(l3, &oldl3, oldl3 & | if (!atomic_fcmpset_64(l3, &oldl3, oldl3 & | ||||
~ATTR_SW_DBM)) | ~ATTR_SW_DBM)) | ||||
goto setl3; | goto setl3; | ||||
oldl3 &= ~ATTR_SW_DBM; | oldl3 &= ~ATTR_SW_DBM; | ||||
} | } | ||||
if (oldl3 != pa) { | if ((oldl3 & ATTR_MASK) != (newl2 & ATTR_MASK)) { | ||||
atomic_add_long(&pmap_l2_p_failures, 1); | atomic_add_long(&pmap_l2_p_failures, 1); | ||||
CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return; | return; | ||||
} | } | ||||
pa -= PAGE_SIZE; | pa -= PAGE_SIZE; | ||||
} | } | ||||
Show All 22 Lines | |||||
newl2 &= ~ATTR_DESCR_MASK; | newl2 &= ~ATTR_DESCR_MASK; | ||||
newl2 |= L2_BLOCK; | newl2 |= L2_BLOCK; | ||||
pmap_update_entry(pmap, l2, newl2, va & ~L2_OFFSET, L2_SIZE); | pmap_update_entry(pmap, l2, newl2, va & ~L2_OFFSET, L2_SIZE); | ||||
atomic_add_long(&pmap_l2_promotions, 1); | atomic_add_long(&pmap_l2_promotions, 1); | ||||
CTR2(KTR_PMAP, "pmap_promote_l2: success for va %#lx in pmap %p", va, | CTR2(KTR_PMAP, "pmap_promote_l2: success for va %#lx in pmap %p", va, | ||||
pmap); | pmap); | ||||
} | } | ||||
#endif /* VM_NRESERVLEVEL > 0 */ | #endif /* VM_NRESERVLEVEL > 0 */ | ||||
static int | static int | ||||
pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags, | pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags, | ||||
int psind) | int psind) | ||||
{ | { | ||||
pd_entry_t *l0p, *l1p, *l2p, origpte; | pd_entry_t *l0p, *l1p, *l2p, origpte; | ||||
▲ Show 20 Lines • Show All 91 Lines • Show Last 20 Lines |