Index: sys/arm64/arm64/pmap.c =================================================================== --- sys/arm64/arm64/pmap.c +++ sys/arm64/arm64/pmap.c @@ -596,6 +596,40 @@ return (l3); } +/* + * XXX + */ +static __inline pt_entry_t * +pmap_pte_exists(pmap_t pmap, vm_offset_t va, int level) +{ + pd_entry_t *l0p, *l1p, *l2p; + pt_entry_t desc, *l3p; + + l0p = pmap_l0(pmap, va); + desc = pmap_load(l0p) & ATTR_DESCR_MASK; + if (desc == L0_TABLE && level > 0) { + l1p = pmap_l0_to_l1(l0p, va); + desc = pmap_load(l1p) & ATTR_DESCR_MASK; + if (desc == L1_BLOCK && level == 1) + return (l1p); + else if (desc == L1_TABLE && level > 1) { + l2p = pmap_l1_to_l2(l1p, va); + desc = pmap_load(l2p) & ATTR_DESCR_MASK; + if (desc == L2_BLOCK && level == 2) + return (l2p); + else if (desc == L2_TABLE && level > 2) { + l3p = pmap_l2_to_l3(l2p, va); + desc = pmap_load(l3p) & ATTR_DESCR_MASK; + if (desc == L3_PAGE && level == 3) + return (l3p); + } + } + } + KASSERT(false, + ("no BLOCK or PAGE mapping at the specified address and level")); + return (NULL); +} + bool pmap_ps_enabled(pmap_t pmap __unused) { @@ -5117,7 +5151,7 @@ struct md_page *pvh; pt_entry_t *pte, mask, value; pmap_t pmap; - int lvl, md_gen, pvh_gen; + int md_gen, pvh_gen; boolean_t rv; rv = FALSE; @@ -5137,9 +5171,7 @@ goto restart; } } - pte = pmap_pte(pmap, pv->pv_va, &lvl); - KASSERT(lvl == 3, - ("pmap_page_test_mappings: Invalid level %d", lvl)); + pte = pmap_pte_exists(pmap, pv->pv_va, 3); mask = 0; value = 0; if (modified) { @@ -5172,9 +5204,7 @@ goto restart; } } - pte = pmap_pte(pmap, pv->pv_va, &lvl); - KASSERT(lvl == 2, - ("pmap_page_test_mappings: Invalid level %d", lvl)); + pte = pmap_pte_exists(pmap, pv->pv_va, 2); mask = 0; value = 0; if (modified) { @@ -5267,7 +5297,7 @@ pv_entry_t next_pv, pv; pt_entry_t oldpte, *pte; vm_offset_t va; - int lvl, md_gen, pvh_gen; + int md_gen, pvh_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_write: page %p is not managed", m)); @@ -5293,7 +5323,7 @@ } } va = pv->pv_va; - pte = pmap_pte(pmap, va, &lvl); + pte = pmap_pte_exists(pmap, va, 2); if ((pmap_load(pte) & ATTR_SW_DBM) != 0) (void)pmap_demote_l2_locked(pmap, pte, va, &lock); KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), @@ -5316,7 +5346,7 @@ goto retry; } } - pte = pmap_pte(pmap, pv->pv_va, &lvl); + pte = pmap_pte_exists(pmap, pv->pv_va, 3); oldpte = pmap_load(pte); if ((oldpte & ATTR_SW_DBM) != 0) { while (!atomic_fcmpset_64(pte, &oldpte,