diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -1302,7 +1302,7 @@ static bool pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, struct rwlock **lockp); static bool pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, - vm_offset_t va); + vm_offset_t va, vm_page_t *mp); static int pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, struct rwlock **lockp); static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, @@ -1311,7 +1311,7 @@ vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted, - bool allpte_PG_A_set); + bool allpte_PG_A_set, void *node); static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva); static void pmap_invalidate_cache_range_all(vm_offset_t sva, @@ -2541,7 +2541,7 @@ */ if ((i == 0 || kernphys + ((vm_paddr_t)(i - 1) << PDRSHIFT) < KERNend) && - pmap_insert_pt_page(kernel_pmap, mpte, false, false)) + pmap_insert_pt_page(kernel_pmap, mpte, false, false, NULL)) panic("pmap_init: pmap_insert_pt_page failed"); } PMAP_UNLOCK(kernel_pmap); @@ -4153,14 +4153,21 @@ */ static __inline int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted, - bool allpte_PG_A_set) + bool allpte_PG_A_set, void *node) { + int res; PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT(promoted || !allpte_PG_A_set, ("a zero-filled PTP can't have PG_A set in every PTE")); mpte->valid = promoted ? (allpte_PG_A_set ? VM_PAGE_BITS_ALL : 1) : 0; - return (vm_radix_insert(&pmap->pm_root, mpte)); + if (node == NULL) { + res = vm_radix_insert(&pmap->pm_root, mpte); + } else { + vm_radix_insert_prealloc(&pmap->pm_root, mpte, node); + res = 0; + } + return (res); } /* @@ -7070,7 +7077,7 @@ ("pmap_promote_pde: page table page's pindex is wrong " "mpte %p pidx %#lx va %#lx va pde pidx %#lx", mpte, mpte->pindex, va, pmap_pde_pindex(va))); - if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0)) { + if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0, NULL)) { counter_u64_add(pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx in pmap %p", va, @@ -7554,9 +7561,11 @@ { struct spglist free; pd_entry_t oldpde, *pde; + void *node; pt_entry_t PG_G, PG_RW, PG_V; vm_page_t mt, pdpg; vm_page_t uwptpg; + int rv; PG_G = pmap_global_bit(pmap); PG_RW = pmap_rw_bit(pmap); @@ -7588,6 +7597,13 @@ return (KERN_PROTECTION_FAILURE); } + node = vm_radix_node_alloc(&pmap->pm_root.rt_trie); + if (node == NULL) { + pmap_abort_ptp(pmap, va, pdpg); + return (KERN_RESOURCE_SHORTAGE); + } + rv = KERN_SUCCESS; + /* * If there are existing mappings, either abort or remove them. */ @@ -7602,7 +7618,8 @@ CTR2(KTR_PMAP, "pmap_enter_pde: no space for va %#lx" " in pmap %p", va, pmap); - return (KERN_NO_SPACE); + rv = KERN_NO_SPACE; + goto out; } else if (va < VM_MAXUSER_ADDRESS || !pmap_every_pte_zero(oldpde & PG_FRAME)) { if (pdpg != NULL) @@ -7610,7 +7627,8 @@ CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" " in pmap %p", va, pmap); - return (KERN_FAILURE); + rv = KERN_FAILURE; + goto out; } } /* Break the existing mapping(s). */ @@ -7645,8 +7663,9 @@ * leave the kernel page table page zero filled. */ mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME); - if (pmap_insert_pt_page(pmap, mt, false, false)) + if (pmap_insert_pt_page(pmap, mt, false, false, node)) panic("pmap_enter_pde: trie insert failed"); + node = NULL; } } @@ -7659,12 +7678,14 @@ VM_ALLOC_WIRED); if (uwptpg == NULL) { pmap_abort_ptp(pmap, va, pdpg); - return (KERN_RESOURCE_SHORTAGE); + rv = KERN_RESOURCE_SHORTAGE; + goto out; } - if (pmap_insert_pt_page(pmap, uwptpg, true, false)) { + if (pmap_insert_pt_page(pmap, uwptpg, true, false, node)) { pmap_free_pt_page(pmap, uwptpg, false); pmap_abort_ptp(pmap, va, pdpg); - return (KERN_RESOURCE_SHORTAGE); + rv = KERN_RESOURCE_SHORTAGE; + goto out; } uwptpg->ref_count = NPTEPG; @@ -7686,7 +7707,8 @@ } CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" " in pmap %p", va, pmap); - return (KERN_RESOURCE_SHORTAGE); + rv = KERN_RESOURCE_SHORTAGE; + goto out; } if ((newpde & PG_RW) != 0) { for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) @@ -7710,7 +7732,9 @@ counter_u64_add(pmap_pde_mappings, 1); CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p", va, pmap); - return (KERN_SUCCESS); +out: + vm_radix_node_free(&pmap->pm_root.rt_trie, node); + return (rv); } /* @@ -9614,7 +9638,7 @@ * Tries to demote a 1GB page mapping. */ static bool -pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va) +pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va, vm_page_t *mp) { pdp_entry_t newpdpe, oldpdpe; pd_entry_t *firstpde, newpde, *pde; @@ -9631,8 +9655,15 @@ oldpdpe = *pdpe; KASSERT((oldpdpe & (PG_PS | PG_V)) == (PG_PS | PG_V), ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); - pdpg = pmap_alloc_pt_page(pmap, va >> PDPSHIFT, - VM_ALLOC_WIRED | VM_ALLOC_INTERRUPT); + if (mp == NULL) { + pdpg = pmap_alloc_pt_page(pmap, va >> PDPSHIFT, + VM_ALLOC_WIRED); + } else { + pdpg = *mp; + *mp = NULL; + pdpg->pindex = va >> PDPSHIFT; + pmap_pt_page_count_adj(pmap, 1); + } if (pdpg == NULL) { CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" " in pmap %p", va, pmap); @@ -9846,7 +9877,7 @@ tmpva += NBPDP; continue; } - if (!pmap_demote_pdpe(kernel_pmap, pdpe, tmpva)) + if (!pmap_demote_pdpe(kernel_pmap, pdpe, tmpva, NULL)) return (ENOMEM); } pde = pmap_pdpe_to_pde(pdpe, tmpva); @@ -10015,6 +10046,7 @@ { pdp_entry_t *pdpe; pd_entry_t *pde; + vm_page_t m; vm_offset_t va; bool changed; @@ -10023,15 +10055,24 @@ KASSERT(powerof2(len), ("pmap_demote_DMAP: len is not a power of 2")); KASSERT((base & (len - 1)) == 0, ("pmap_demote_DMAP: base is not a multiple of len")); + m = NULL; if (len < NBPDP && base < dmaplimit) { va = PHYS_TO_DMAP(base); changed = false; + + /* + * Assume that it is fine to sleep there. Existing + * only caller of pmap_demote_DMAP() is the + * x86_mr_split_dmap() function. + */ + m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_WAITOK); + PMAP_LOCK(kernel_pmap); pdpe = pmap_pdpe(kernel_pmap, va); if ((*pdpe & X86_PG_V) == 0) panic("pmap_demote_DMAP: invalid PDPE"); if ((*pdpe & PG_PS) != 0) { - if (!pmap_demote_pdpe(kernel_pmap, pdpe, va)) + if (!pmap_demote_pdpe(kernel_pmap, pdpe, va, &m)) panic("pmap_demote_DMAP: PDPE failed"); changed = true; } @@ -10049,6 +10090,10 @@ pmap_invalidate_page(kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } + if (m != NULL) { + vm_page_unwire_noq(m); + vm_page_free(m); + } } /* diff --git a/sys/vm/vm_radix.h b/sys/vm/vm_radix.h --- a/sys/vm/vm_radix.h +++ b/sys/vm/vm_radix.h @@ -357,5 +357,8 @@ return (VM_RADIX_PCTRIE_REPLACE(&rtree->rt_trie, newpage)); } +void vm_radix_insert_prealloc(struct vm_radix *rtree, vm_page_t m, + void *node); + #endif /* _KERNEL */ #endif /* !_VM_RADIX_H_ */ diff --git a/sys/vm/vm_radix.c b/sys/vm/vm_radix.c --- a/sys/vm/vm_radix.c +++ b/sys/vm/vm_radix.c @@ -124,3 +124,19 @@ { uma_zwait(vm_radix_node_zone); } + +void +vm_radix_insert_prealloc(struct vm_radix *rtree, vm_page_t m, void *node) +{ + struct pctrie *ptree; + void *parentp; + struct pctrie_node *child, *parent; + uint64_t *val; + + ptree = &rtree->rt_trie; + val = VM_RADIX_PCTRIE_PTR2VAL(m); + child = node; + parentp = pctrie_insert_lookup_strict(ptree, val, &parent); + if (parentp != NULL) + pctrie_insert_node(val, parent, parentp, child); +}