diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c --- a/sys/riscv/riscv/pmap.c +++ b/sys/riscv/riscv/pmap.c @@ -2913,28 +2913,62 @@ PMAP_UNLOCK(pmap); } +static bool +pmap_fault_lookup_unlocked(pmap_t pmap, vm_offset_t va, pt_entry_t **ptep, + pt_entry_t *oldptep) +{ + pd_entry_t *l2, l2e; + + l2 = pmap_l2(pmap, va); + if (l2 == NULL || ((l2e = pmap_load(l2)) & PTE_V) == 0) + return (false); + if ((l2e & PTE_RWX) == 0) { + *ptep = pmap_l2_to_l3(l2, va); + if (((*oldptep = pmap_load(*ptep)) & PTE_V) == 0) + return (false); + } else { + *ptep = l2; + *oldptep = l2e; + } + return (true); +} + int pmap_fault(pmap_t pmap, vm_offset_t va, vm_prot_t ftype) { - pd_entry_t *l2, l2e; pt_entry_t bits, *pte, oldpte; int rv; KASSERT(VIRT_IS_VALID(va), ("pmap_fault: invalid va %#lx", va)); + if (pmap == kernel_pmap) { + /* + * Perform a lockless lookup using the kernel pmap + * since we might be running in a non-sleepable context. + */ + if (!pmap_fault_lookup_unlocked(pmap, va, &pte, &oldpte)) + return (0); + if ((oldpte & PTE_KERN) == PTE_KERN) { + sfence_vma_page(va); + return (1); + } + /* Only PTE_A is expected to be missing at this point. */ + KASSERT((oldpte & PTE_KERN) == (PTE_KERN & ~PTE_A), + ("pmap_fault: unexpected bits missing for %#lx: pte: %lx", + va, oldpte)); + /* + * We've encountered an entry without PTE_A. + * This can happen with pageable memory allocated + * using 'kmap_alloc' (e.g., 'exec_alloc_args'). + * Fall back to the locked lookup below since + * we need to update the missing PTE_A bit. + */ + } + rv = 0; PMAP_LOCK(pmap); - l2 = pmap_l2(pmap, va); - if (l2 == NULL || ((l2e = pmap_load(l2)) & PTE_V) == 0) + if (!pmap_fault_lookup_unlocked(pmap, va, &pte, &oldpte)) goto done; - if ((l2e & PTE_RWX) == 0) { - pte = pmap_l2_to_l3(l2, va); - if (((oldpte = pmap_load(pte)) & PTE_V) == 0) - goto done; - } else { - pte = l2; - oldpte = l2e; - } if ((pmap != kernel_pmap && (oldpte & PTE_U) == 0) || (ftype == VM_PROT_WRITE && (oldpte & PTE_W) == 0) ||