Index: head/sys/arm64/arm64/pmap.c =================================================================== --- head/sys/arm64/arm64/pmap.c +++ head/sys/arm64/arm64/pmap.c @@ -1128,40 +1128,35 @@ pmap_kextract(vm_offset_t va) { pt_entry_t *pte, tpte; - vm_paddr_t pa; - int lvl; - if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { - pa = DMAP_TO_PHYS(va); - } else { - pa = 0; - pte = pmap_pte(kernel_pmap, va, &lvl); - if (pte != NULL) { - tpte = pmap_load(pte); - pa = tpte & ~ATTR_MASK; - switch(lvl) { - case 1: - KASSERT((tpte & ATTR_DESCR_MASK) == L1_BLOCK, - ("pmap_kextract: Invalid L1 pte found: %lx", - tpte & ATTR_DESCR_MASK)); - pa |= (va & L1_OFFSET); - break; - case 2: - KASSERT((tpte & ATTR_DESCR_MASK) == L2_BLOCK, - ("pmap_kextract: Invalid L2 pte found: %lx", - tpte & ATTR_DESCR_MASK)); - pa |= (va & L2_OFFSET); - break; - case 3: - KASSERT((tpte & ATTR_DESCR_MASK) == L3_PAGE, - ("pmap_kextract: Invalid L3 pte found: %lx", - tpte & ATTR_DESCR_MASK)); - pa |= (va & L3_OFFSET); - break; - } - } - } - return (pa); + if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) + return (DMAP_TO_PHYS(va)); + pte = pmap_l1(kernel_pmap, va); + if (pte == NULL) + return (0); + + /* + * A concurrent pmap_update_entry() will clear the entry's valid bit + * but leave the rest of the entry unchanged. Therefore, we treat a + * non-zero entry as being valid, and we ignore the valid bit when + * determining whether the entry maps a block, page, or table. + */ + tpte = pmap_load(pte); + if (tpte == 0) + return (0); + if ((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK) + return ((tpte & ~ATTR_MASK) | (va & L1_OFFSET)); + pte = pmap_l1_to_l2(&tpte, va); + tpte = pmap_load(pte); + if (tpte == 0) + return (0); + if ((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK) + return ((tpte & ~ATTR_MASK) | (va & L2_OFFSET)); + pte = pmap_l2_to_l3(&tpte, va); + tpte = pmap_load(pte); + if (tpte == 0) + return (0); + return ((tpte & ~ATTR_MASK) | (va & L3_OFFSET)); } /*************************************************** @@ -3021,8 +3016,12 @@ intr = intr_disable(); critical_enter(); - /* Clear the old mapping */ - pmap_clear(pte); + /* + * Clear the old mapping's valid bit, but leave the rest of the entry + * unchanged, so that a lockless, concurrent pmap_kextract() can still + * lookup the physical address. + */ + pmap_clear_bits(pte, ATTR_DESCR_VALID); pmap_invalidate_range_nopin(pmap, va, va + size); /* Create the new mapping */ Index: head/sys/arm64/include/pte.h =================================================================== --- head/sys/arm64/include/pte.h +++ head/sys/arm64/include/pte.h @@ -71,7 +71,12 @@ #define ATTR_DEFAULT (ATTR_AF | ATTR_SH(ATTR_SH_IS)) -#define ATTR_DESCR_MASK 3 +#define ATTR_DESCR_MASK 3 +#define ATTR_DESCR_VALID 1 +#define ATTR_DESCR_TYPE_MASK 2 +#define ATTR_DESCR_TYPE_TABLE 2 +#define ATTR_DESCR_TYPE_PAGE 2 +#define ATTR_DESCR_TYPE_BLOCK 0 /* Level 0 table, 512GiB per entry */ #define L0_SHIFT 39