Page MenuHomeFreeBSD
Files F144444877

D50970.id157419.diff
No OneTemporary
Actions

D50970.id157419.diff
View Options

diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -1302,7 +1302,7 @@
static bool pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde,
vm_offset_t va, struct rwlock **lockp);
static bool pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,
- vm_offset_t va);
+ vm_offset_t va, vm_page_t *mp);
static int pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, struct rwlock **lockp);
static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,
@@ -1311,7 +1311,7 @@
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,
- bool allpte_PG_A_set);
+ bool allpte_PG_A_set, void *node);
static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva,
vm_offset_t eva);
static void pmap_invalidate_cache_range_all(vm_offset_t sva,
@@ -2541,7 +2541,7 @@
*/
if ((i == 0 ||
kernphys + ((vm_paddr_t)(i - 1) << PDRSHIFT) < KERNend) &&
- pmap_insert_pt_page(kernel_pmap, mpte, false, false))
+ pmap_insert_pt_page(kernel_pmap, mpte, false, false, NULL))
panic("pmap_init: pmap_insert_pt_page failed");
}
PMAP_UNLOCK(kernel_pmap);
@@ -4153,14 +4153,21 @@
*/
static __inline int
pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,
- bool allpte_PG_A_set)
+ bool allpte_PG_A_set, void *node)
{
+ int res;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT(promoted || !allpte_PG_A_set,
("a zero-filled PTP can't have PG_A set in every PTE"));
mpte->valid = promoted ? (allpte_PG_A_set ? VM_PAGE_BITS_ALL : 1) : 0;
- return (vm_radix_insert(&pmap->pm_root, mpte));
+ if (node == NULL) {
+ res = vm_radix_insert(&pmap->pm_root, mpte);
+ } else {
+ vm_radix_insert_prealloc(&pmap->pm_root, mpte, node);
+ res = 0;
+ }
+ return (res);
}
/*
@@ -7070,7 +7077,7 @@
("pmap_promote_pde: page table page's pindex is wrong "
"mpte %p pidx %#lx va %#lx va pde pidx %#lx",
mpte, mpte->pindex, va, pmap_pde_pindex(va)));
- if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0)) {
+ if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0, NULL)) {
counter_u64_add(pmap_pde_p_failures, 1);
CTR2(KTR_PMAP,
"pmap_promote_pde: failure for va %#lx in pmap %p", va,
@@ -7554,9 +7561,11 @@
{
struct spglist free;
pd_entry_t oldpde, *pde;
+ void *node;
pt_entry_t PG_G, PG_RW, PG_V;
vm_page_t mt, pdpg;
vm_page_t uwptpg;
+ int rv;
PG_G = pmap_global_bit(pmap);
PG_RW = pmap_rw_bit(pmap);
@@ -7588,6 +7597,13 @@
return (KERN_PROTECTION_FAILURE);
}
+ node = vm_radix_node_alloc(&pmap->pm_root.rt_trie);
+ if (node == NULL) {
+ pmap_abort_ptp(pmap, va, pdpg);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
+ rv = KERN_SUCCESS;
+
/*
* If there are existing mappings, either abort or remove them.
*/
@@ -7602,7 +7618,8 @@
CTR2(KTR_PMAP,
"pmap_enter_pde: no space for va %#lx"
" in pmap %p", va, pmap);
- return (KERN_NO_SPACE);
+ rv = KERN_NO_SPACE;
+ goto out;
} else if (va < VM_MAXUSER_ADDRESS ||
!pmap_every_pte_zero(oldpde & PG_FRAME)) {
if (pdpg != NULL)
@@ -7610,7 +7627,8 @@
CTR2(KTR_PMAP,
"pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);
- return (KERN_FAILURE);
+ rv = KERN_FAILURE;
+ goto out;
}
}
/* Break the existing mapping(s). */
@@ -7645,8 +7663,9 @@
* leave the kernel page table page zero filled.
*/
mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
- if (pmap_insert_pt_page(pmap, mt, false, false))
+ if (pmap_insert_pt_page(pmap, mt, false, false, node))
panic("pmap_enter_pde: trie insert failed");
+ node = NULL;
}
}
@@ -7659,12 +7678,14 @@
VM_ALLOC_WIRED);
if (uwptpg == NULL) {
pmap_abort_ptp(pmap, va, pdpg);
- return (KERN_RESOURCE_SHORTAGE);
+ rv = KERN_RESOURCE_SHORTAGE;
+ goto out;
}
- if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {
+ if (pmap_insert_pt_page(pmap, uwptpg, true, false, node)) {
pmap_free_pt_page(pmap, uwptpg, false);
pmap_abort_ptp(pmap, va, pdpg);
- return (KERN_RESOURCE_SHORTAGE);
+ rv = KERN_RESOURCE_SHORTAGE;
+ goto out;
}
uwptpg->ref_count = NPTEPG;
@@ -7686,7 +7707,8 @@
}
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);
- return (KERN_RESOURCE_SHORTAGE);
+ rv = KERN_RESOURCE_SHORTAGE;
+ goto out;
}
if ((newpde & PG_RW) != 0) {
for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
@@ -7710,7 +7732,9 @@
counter_u64_add(pmap_pde_mappings, 1);
CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p",
va, pmap);
- return (KERN_SUCCESS);
+out:
+ vm_radix_node_free(&pmap->pm_root.rt_trie, node);
+ return (rv);
}
/*
@@ -9614,7 +9638,7 @@
* Tries to demote a 1GB page mapping.
*/
static bool
-pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va)
+pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va, vm_page_t *mp)
{
pdp_entry_t newpdpe, oldpdpe;
pd_entry_t *firstpde, newpde, *pde;
@@ -9631,8 +9655,15 @@
oldpdpe = *pdpe;
KASSERT((oldpdpe & (PG_PS | PG_V)) == (PG_PS | PG_V),
("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
- pdpg = pmap_alloc_pt_page(pmap, va >> PDPSHIFT,
- VM_ALLOC_WIRED | VM_ALLOC_INTERRUPT);
+ if (mp == NULL) {
+ pdpg = pmap_alloc_pt_page(pmap, va >> PDPSHIFT,
+ VM_ALLOC_WIRED);
+ } else {
+ pdpg = *mp;
+ *mp = NULL;
+ pdpg->pindex = va >> PDPSHIFT;
+ pmap_pt_page_count_adj(pmap, 1);
+ }
if (pdpg == NULL) {
CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
" in pmap %p", va, pmap);
@@ -9846,7 +9877,7 @@
tmpva += NBPDP;
continue;
}
- if (!pmap_demote_pdpe(kernel_pmap, pdpe, tmpva))
+ if (!pmap_demote_pdpe(kernel_pmap, pdpe, tmpva, NULL))
return (ENOMEM);
}
pde = pmap_pdpe_to_pde(pdpe, tmpva);
@@ -10015,6 +10046,7 @@
{
pdp_entry_t *pdpe;
pd_entry_t *pde;
+ vm_page_t m;
vm_offset_t va;
bool changed;
@@ -10023,15 +10055,24 @@
KASSERT(powerof2(len), ("pmap_demote_DMAP: len is not a power of 2"));
KASSERT((base & (len - 1)) == 0,
("pmap_demote_DMAP: base is not a multiple of len"));
+ m = NULL;
if (len < NBPDP && base < dmaplimit) {
va = PHYS_TO_DMAP(base);
changed = false;
+
+ /*
+ * Assume that it is fine to sleep there. Existing
+ * only caller of pmap_demote_DMAP() is the
+ * x86_mr_split_dmap() function.
+ */
+ m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_WAITOK);
+
PMAP_LOCK(kernel_pmap);
pdpe = pmap_pdpe(kernel_pmap, va);
if ((*pdpe & X86_PG_V) == 0)
panic("pmap_demote_DMAP: invalid PDPE");
if ((*pdpe & PG_PS) != 0) {
- if (!pmap_demote_pdpe(kernel_pmap, pdpe, va))
+ if (!pmap_demote_pdpe(kernel_pmap, pdpe, va, &m))
panic("pmap_demote_DMAP: PDPE failed");
changed = true;
}
@@ -10049,6 +10090,10 @@
pmap_invalidate_page(kernel_pmap, va);
PMAP_UNLOCK(kernel_pmap);
}
+ if (m != NULL) {
+ vm_page_unwire_noq(m);
+ vm_page_free(m);
+ }
}
/*
diff --git a/sys/vm/vm_radix.h b/sys/vm/vm_radix.h
--- a/sys/vm/vm_radix.h
+++ b/sys/vm/vm_radix.h
@@ -357,5 +357,8 @@
return (VM_RADIX_PCTRIE_REPLACE(&rtree->rt_trie, newpage));
}
+void vm_radix_insert_prealloc(struct vm_radix *rtree, vm_page_t m,
+ void *node);
+
#endif /* _KERNEL */
#endif /* !_VM_RADIX_H_ */
diff --git a/sys/vm/vm_radix.c b/sys/vm/vm_radix.c
--- a/sys/vm/vm_radix.c
+++ b/sys/vm/vm_radix.c
@@ -124,3 +124,19 @@
{
uma_zwait(vm_radix_node_zone);
}
+
+void
+vm_radix_insert_prealloc(struct vm_radix *rtree, vm_page_t m, void *node)
+{
+ struct pctrie *ptree;
+ void *parentp;
+ struct pctrie_node *child, *parent;
+ uint64_t *val;
+
+ ptree = &rtree->rt_trie;
+ val = VM_RADIX_PCTRIE_PTR2VAL(m);
+ child = node;
+ parentp = pctrie_insert_lookup_strict(ptree, val, &parent);
+ if (parentp != NULL)
+ pctrie_insert_node(val, parent, parentp, child);
+}

File Metadata

Mime Type
text/plain
Expires
Mon, Feb 9, 11:41 AM (12 h, 32 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28575055
Default Alt Text
D50970.id157419.diff (7 KB)

Event Timeline