Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 1,342 Lines • ▼ Show 20 Lines | pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, | ||||
if (set_PG_ZERO) | if (set_PG_ZERO) | ||||
m->flags |= PG_ZERO; | m->flags |= PG_ZERO; | ||||
else | else | ||||
m->flags &= ~PG_ZERO; | m->flags &= ~PG_ZERO; | ||||
SLIST_INSERT_HEAD(free, m, plinks.s.ss); | SLIST_INSERT_HEAD(free, m, plinks.s.ss); | ||||
} | } | ||||
/* | /* | ||||
* Decrements a page table page's wire count, which is used to record the | * Decrements a page table page's reference count, which is used to record the | ||||
* number of valid page table entries within the page. If the wire count | * number of valid page table entries within the page. If the reference count | ||||
* drops to zero, then the page table page is unmapped. Returns TRUE if the | * drops to zero, then the page table page is unmapped. Returns TRUE if the | ||||
* page table page was unmapped and FALSE otherwise. | * page table page was unmapped and FALSE otherwise. | ||||
*/ | */ | ||||
static inline boolean_t | static inline boolean_t | ||||
pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
{ | { | ||||
--m->wire_count; | --m->ref_count; | ||||
if (m->wire_count == 0) { | if (m->ref_count == 0) { | ||||
_pmap_unwire_l3(pmap, va, m, free); | _pmap_unwire_l3(pmap, va, m, free); | ||||
return (TRUE); | return (TRUE); | ||||
} else | } else | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
static void | static void | ||||
_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
* Put page on a list so that it is released after | * Put page on a list so that it is released after | ||||
* *ALL* TLB shootdown is done | * *ALL* TLB shootdown is done | ||||
*/ | */ | ||||
pmap_add_delayed_free_list(m, free, TRUE); | pmap_add_delayed_free_list(m, free, TRUE); | ||||
} | } | ||||
/* | /* | ||||
* After removing a page table entry, this routine is used to | * After removing a page table entry, this routine is used to | ||||
* conditionally free the page, and manage the hold/wire counts. | * conditionally free the page, and manage the reference count. | ||||
*/ | */ | ||||
static int | static int | ||||
pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | ||||
struct spglist *free) | struct spglist *free) | ||||
{ | { | ||||
vm_page_t mpte; | vm_page_t mpte; | ||||
if (va >= VM_MAXUSER_ADDRESS) | if (va >= VM_MAXUSER_ADDRESS) | ||||
▲ Show 20 Lines • Show All 114 Lines • ▼ Show 20 Lines | if (tl0 == 0) { | ||||
if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index, | if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index, | ||||
lockp) == NULL) { | lockp) == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} else { | } else { | ||||
l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); | l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); | ||||
l1pg->wire_count++; | l1pg->ref_count++; | ||||
} | } | ||||
l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); | l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); | ||||
l1 = &l1[ptepindex & Ln_ADDR_MASK]; | l1 = &l1[ptepindex & Ln_ADDR_MASK]; | ||||
pmap_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE); | pmap_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE); | ||||
} else { | } else { | ||||
vm_pindex_t l0index, l1index; | vm_pindex_t l0index, l1index; | ||||
pd_entry_t *l0, *l1, *l2; | pd_entry_t *l0, *l1, *l2; | ||||
Show All 24 Lines | if (tl0 == 0) { | ||||
if (_pmap_alloc_l3(pmap, NUL2E + l1index, | if (_pmap_alloc_l3(pmap, NUL2E + l1index, | ||||
lockp) == NULL) { | lockp) == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} else { | } else { | ||||
l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); | l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); | ||||
l2pg->wire_count++; | l2pg->ref_count++; | ||||
} | } | ||||
} | } | ||||
l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); | l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); | ||||
l2 = &l2[ptepindex & Ln_ADDR_MASK]; | l2 = &l2[ptepindex & Ln_ADDR_MASK]; | ||||
pmap_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE); | pmap_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE); | ||||
} | } | ||||
Show All 9 Lines | pmap_alloc_l2(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) | ||||
vm_page_t l2pg; | vm_page_t l2pg; | ||||
vm_pindex_t l2pindex; | vm_pindex_t l2pindex; | ||||
retry: | retry: | ||||
l1 = pmap_l1(pmap, va); | l1 = pmap_l1(pmap, va); | ||||
if (l1 != NULL && (pmap_load(l1) & ATTR_DESCR_MASK) == L1_TABLE) { | if (l1 != NULL && (pmap_load(l1) & ATTR_DESCR_MASK) == L1_TABLE) { | ||||
/* Add a reference to the L2 page. */ | /* Add a reference to the L2 page. */ | ||||
l2pg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK); | l2pg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK); | ||||
l2pg->wire_count++; | l2pg->ref_count++; | ||||
} else { | } else { | ||||
/* Allocate a L2 page. */ | /* Allocate a L2 page. */ | ||||
l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT; | l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT; | ||||
l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp); | l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp); | ||||
if (l2pg == NULL && lockp != NULL) | if (l2pg == NULL && lockp != NULL) | ||||
goto retry; | goto retry; | ||||
} | } | ||||
return (l2pg); | return (l2pg); | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | #ifdef INVARIANTS | ||||
KASSERT(pmap_load(pte) == 0, | KASSERT(pmap_load(pte) == 0, | ||||
("pmap_alloc_l3: TODO: l1 superpages")); | ("pmap_alloc_l3: TODO: l1 superpages")); | ||||
#endif | #endif | ||||
break; | break; | ||||
case 2: | case 2: | ||||
tpde = pmap_load(pde); | tpde = pmap_load(pde); | ||||
if (tpde != 0) { | if (tpde != 0) { | ||||
m = PHYS_TO_VM_PAGE(tpde & ~ATTR_MASK); | m = PHYS_TO_VM_PAGE(tpde & ~ATTR_MASK); | ||||
m->wire_count++; | m->ref_count++; | ||||
return (m); | return (m); | ||||
} | } | ||||
break; | break; | ||||
default: | default: | ||||
panic("pmap_alloc_l3: Invalid level %d", lvl); | panic("pmap_alloc_l3: Invalid level %d", lvl); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 348 Lines • ▼ Show 20 Lines | next_chunk: | ||||
active_reclaims--; | active_reclaims--; | ||||
mtx_unlock(&pv_chunks_mutex); | mtx_unlock(&pv_chunks_mutex); | ||||
if (pmap != NULL && pmap != locked_pmap) | if (pmap != NULL && pmap != locked_pmap) | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
if (m_pc == NULL && !SLIST_EMPTY(&free)) { | if (m_pc == NULL && !SLIST_EMPTY(&free)) { | ||||
m_pc = SLIST_FIRST(&free); | m_pc = SLIST_FIRST(&free); | ||||
SLIST_REMOVE_HEAD(&free, plinks.s.ss); | SLIST_REMOVE_HEAD(&free, plinks.s.ss); | ||||
/* Recycle a freed page table page. */ | /* Recycle a freed page table page. */ | ||||
m_pc->wire_count = 1; | m_pc->ref_count = 1; | ||||
} | } | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
return (m_pc); | return (m_pc); | ||||
} | } | ||||
/* | /* | ||||
* free the pv_entry back to the free list | * free the pv_entry back to the free list | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 421 Lines • ▼ Show 20 Lines | pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
pmap_remove_kernel_l2(pmap, l2, sva); | pmap_remove_kernel_l2(pmap, l2, sva); | ||||
} else { | } else { | ||||
ml3 = pmap_remove_pt_page(pmap, sva); | ml3 = pmap_remove_pt_page(pmap, sva); | ||||
if (ml3 != NULL) { | if (ml3 != NULL) { | ||||
KASSERT(ml3->valid == VM_PAGE_BITS_ALL, | KASSERT(ml3->valid == VM_PAGE_BITS_ALL, | ||||
("pmap_remove_l2: l3 page not promoted")); | ("pmap_remove_l2: l3 page not promoted")); | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
KASSERT(ml3->wire_count == NL3PG, | KASSERT(ml3->ref_count == NL3PG, | ||||
("pmap_remove_l2: l3 page wire count error")); | ("pmap_remove_l2: l3 page ref count error")); | ||||
ml3->wire_count = 0; | ml3->ref_count = 0; | ||||
pmap_add_delayed_free_list(ml3, free, FALSE); | pmap_add_delayed_free_list(ml3, free, FALSE); | ||||
} | } | ||||
} | } | ||||
return (pmap_unuse_pt(pmap, sva, l1e, free)); | return (pmap_unuse_pt(pmap, sva, l1e, free)); | ||||
} | } | ||||
/* | /* | ||||
* pmap_remove_l3: do the things to unmap a page in a process | * pmap_remove_l3: do the things to unmap a page in a process | ||||
▲ Show 20 Lines • Show All 728 Lines • ▼ Show 20 Lines | pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
* resident, we are creating it here. | * resident, we are creating it here. | ||||
*/ | */ | ||||
retry: | retry: | ||||
pde = pmap_pde(pmap, va, &lvl); | pde = pmap_pde(pmap, va, &lvl); | ||||
if (pde != NULL && lvl == 2) { | if (pde != NULL && lvl == 2) { | ||||
l3 = pmap_l2_to_l3(pde, va); | l3 = pmap_l2_to_l3(pde, va); | ||||
if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { | if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { | ||||
mpte = PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK); | mpte = PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK); | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} | } | ||||
goto havel3; | goto havel3; | ||||
} else if (pde != NULL && lvl == 1) { | } else if (pde != NULL && lvl == 1) { | ||||
l2 = pmap_l1_to_l2(pde, va); | l2 = pmap_l1_to_l2(pde, va); | ||||
if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK && | if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK && | ||||
(l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) { | (l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) { | ||||
l3 = &l3[pmap_l3_index(va)]; | l3 = &l3[pmap_l3_index(va)]; | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
mpte = PHYS_TO_VM_PAGE( | mpte = PHYS_TO_VM_PAGE( | ||||
pmap_load(l2) & ~ATTR_MASK); | pmap_load(l2) & ~ATTR_MASK); | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} | } | ||||
goto havel3; | goto havel3; | ||||
} | } | ||||
/* We need to allocate an L3 table. */ | /* We need to allocate an L3 table. */ | ||||
} | } | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | ||||
Show All 34 Lines | if (pmap_l3_valid(orig_l3)) { | ||||
else if ((flags & PMAP_ENTER_WIRED) == 0 && | else if ((flags & PMAP_ENTER_WIRED) == 0 && | ||||
(orig_l3 & ATTR_SW_WIRED) != 0) | (orig_l3 & ATTR_SW_WIRED) != 0) | ||||
pmap->pm_stats.wired_count--; | pmap->pm_stats.wired_count--; | ||||
/* | /* | ||||
* Remove the extra PT page reference. | * Remove the extra PT page reference. | ||||
*/ | */ | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
mpte->wire_count--; | mpte->ref_count--; | ||||
KASSERT(mpte->wire_count > 0, | KASSERT(mpte->ref_count > 0, | ||||
("pmap_enter: missing reference to page table page," | ("pmap_enter: missing reference to page table page," | ||||
" va: 0x%lx", va)); | " va: 0x%lx", va)); | ||||
} | } | ||||
/* | /* | ||||
* Has the physical page changed? | * Has the physical page changed? | ||||
*/ | */ | ||||
if (opa == pa) { | if (opa == pa) { | ||||
▲ Show 20 Lines • Show All 108 Lines • ▼ Show 20 Lines | if (pmap_l3_valid(orig_l3)) { | ||||
} | } | ||||
} else { | } else { | ||||
/* New mapping */ | /* New mapping */ | ||||
pmap_store(l3, new_l3); | pmap_store(l3, new_l3); | ||||
dsb(ishst); | dsb(ishst); | ||||
} | } | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if ((mpte == NULL || mpte->wire_count == NL3PG) && | if ((mpte == NULL || mpte->ref_count == NL3PG) && | ||||
pmap_ps_enabled(pmap) && | pmap_ps_enabled(pmap) && | ||||
(m->flags & PG_FICTITIOUS) == 0 && | (m->flags & PG_FICTITIOUS) == 0 && | ||||
vm_reserv_level_iffullpop(m) == 0) { | vm_reserv_level_iffullpop(m) == 0) { | ||||
pmap_promote_l2(pmap, pde, va, &lock); | pmap_promote_l2(pmap, pde, va, &lock); | ||||
} | } | ||||
#endif | #endif | ||||
rv = KERN_SUCCESS; | rv = KERN_SUCCESS; | ||||
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | if ((l2pg = pmap_alloc_l2(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? | ||||
CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p", | CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p", | ||||
va, pmap); | va, pmap); | ||||
return (KERN_RESOURCE_SHORTAGE); | return (KERN_RESOURCE_SHORTAGE); | ||||
} | } | ||||
l2 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(l2pg)); | l2 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(l2pg)); | ||||
l2 = &l2[pmap_l2_index(va)]; | l2 = &l2[pmap_l2_index(va)]; | ||||
if ((old_l2 = pmap_load(l2)) != 0) { | if ((old_l2 = pmap_load(l2)) != 0) { | ||||
KASSERT(l2pg->wire_count > 1, | KASSERT(l2pg->ref_count > 1, | ||||
("pmap_enter_l2: l2pg's wire count is too low")); | ("pmap_enter_l2: l2pg's ref count is too low")); | ||||
if ((flags & PMAP_ENTER_NOREPLACE) != 0) { | if ((flags & PMAP_ENTER_NOREPLACE) != 0) { | ||||
l2pg->wire_count--; | l2pg->ref_count--; | ||||
CTR2(KTR_PMAP, | CTR2(KTR_PMAP, | ||||
"pmap_enter_l2: failure for va %#lx in pmap %p", | "pmap_enter_l2: failure for va %#lx in pmap %p", | ||||
va, pmap); | va, pmap); | ||||
return (KERN_FAILURE); | return (KERN_FAILURE); | ||||
} | } | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) | if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) | ||||
(void)pmap_remove_l2(pmap, l2, va, | (void)pmap_remove_l2(pmap, l2, va, | ||||
▲ Show 20 Lines • Show All 157 Lines • ▼ Show 20 Lines | pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
vm_pindex_t l2pindex; | vm_pindex_t l2pindex; | ||||
/* | /* | ||||
* Calculate pagetable page index | * Calculate pagetable page index | ||||
*/ | */ | ||||
l2pindex = pmap_l2_pindex(va); | l2pindex = pmap_l2_pindex(va); | ||||
if (mpte && (mpte->pindex == l2pindex)) { | if (mpte && (mpte->pindex == l2pindex)) { | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Get the l2 entry | * Get the l2 entry | ||||
*/ | */ | ||||
pde = pmap_pde(pmap, va, &lvl); | pde = pmap_pde(pmap, va, &lvl); | ||||
/* | /* | ||||
* If the page table page is mapped, we just increment | * If the page table page is mapped, we just increment | ||||
* the hold count, and activate it. Otherwise, we | * the hold count, and activate it. Otherwise, we | ||||
* attempt to allocate a page table page. If this | * attempt to allocate a page table page. If this | ||||
* attempt fails, we don't retry. Instead, we give up. | * attempt fails, we don't retry. Instead, we give up. | ||||
*/ | */ | ||||
if (lvl == 1) { | if (lvl == 1) { | ||||
l2 = pmap_l1_to_l2(pde, va); | l2 = pmap_l1_to_l2(pde, va); | ||||
if ((pmap_load(l2) & ATTR_DESCR_MASK) == | if ((pmap_load(l2) & ATTR_DESCR_MASK) == | ||||
L2_BLOCK) | L2_BLOCK) | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
if (lvl == 2 && pmap_load(pde) != 0) { | if (lvl == 2 && pmap_load(pde) != 0) { | ||||
mpte = | mpte = | ||||
PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK); | PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK); | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Pass NULL instead of the PV list lock | * Pass NULL instead of the PV list lock | ||||
* pointer, because we don't intend to sleep. | * pointer, because we don't intend to sleep. | ||||
*/ | */ | ||||
mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); | mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); | ||||
if (mpte == NULL) | if (mpte == NULL) | ||||
return (mpte); | return (mpte); | ||||
Show All 12 Lines | if (va < VM_MAXUSER_ADDRESS) { | ||||
l3 = pmap_l2_to_l3(pde, va); | l3 = pmap_l2_to_l3(pde, va); | ||||
} | } | ||||
/* | /* | ||||
* Abort if a mapping already exists. | * Abort if a mapping already exists. | ||||
*/ | */ | ||||
if (pmap_load(l3) != 0) { | if (pmap_load(l3) != 0) { | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
mpte->wire_count--; | mpte->ref_count--; | ||||
mpte = NULL; | mpte = NULL; | ||||
} | } | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
/* | /* | ||||
* Enter on the PV list if part of our managed memory. | * Enter on the PV list if part of our managed memory. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 213 Lines • ▼ Show 20 Lines | if ((srcptepaddr & ATTR_DESCR_MASK) == L2_BLOCK) { | ||||
nbits = 0; | nbits = 0; | ||||
if ((srcptepaddr & ATTR_SW_DBM) != 0) | if ((srcptepaddr & ATTR_SW_DBM) != 0) | ||||
nbits |= ATTR_AP_RW_BIT; | nbits |= ATTR_AP_RW_BIT; | ||||
pmap_store(l2, (srcptepaddr & ~mask) | nbits); | pmap_store(l2, (srcptepaddr & ~mask) | nbits); | ||||
pmap_resident_count_inc(dst_pmap, L2_SIZE / | pmap_resident_count_inc(dst_pmap, L2_SIZE / | ||||
PAGE_SIZE); | PAGE_SIZE); | ||||
atomic_add_long(&pmap_l2_mappings, 1); | atomic_add_long(&pmap_l2_mappings, 1); | ||||
} else | } else | ||||
dst_l2pg->wire_count--; | dst_l2pg->ref_count--; | ||||
continue; | continue; | ||||
} | } | ||||
KASSERT((srcptepaddr & ATTR_DESCR_MASK) == L2_TABLE, | KASSERT((srcptepaddr & ATTR_DESCR_MASK) == L2_TABLE, | ||||
("pmap_copy: invalid L2 entry")); | ("pmap_copy: invalid L2 entry")); | ||||
srcptepaddr &= ~ATTR_MASK; | srcptepaddr &= ~ATTR_MASK; | ||||
srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); | srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); | ||||
KASSERT(srcmpte->wire_count > 0, | KASSERT(srcmpte->ref_count > 0, | ||||
("pmap_copy: source page table page is unused")); | ("pmap_copy: source page table page is unused")); | ||||
if (va_next > end_addr) | if (va_next > end_addr) | ||||
va_next = end_addr; | va_next = end_addr; | ||||
src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); | src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); | ||||
src_pte = &src_pte[pmap_l3_index(addr)]; | src_pte = &src_pte[pmap_l3_index(addr)]; | ||||
dstmpte = NULL; | dstmpte = NULL; | ||||
for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { | for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { | ||||
ptetemp = pmap_load(src_pte); | ptetemp = pmap_load(src_pte); | ||||
/* | /* | ||||
* We only virtual copy managed pages. | * We only virtual copy managed pages. | ||||
*/ | */ | ||||
if ((ptetemp & ATTR_SW_MANAGED) == 0) | if ((ptetemp & ATTR_SW_MANAGED) == 0) | ||||
continue; | continue; | ||||
if (dstmpte != NULL) { | if (dstmpte != NULL) { | ||||
KASSERT(dstmpte->pindex == pmap_l2_pindex(addr), | KASSERT(dstmpte->pindex == pmap_l2_pindex(addr), | ||||
("dstmpte pindex/addr mismatch")); | ("dstmpte pindex/addr mismatch")); | ||||
dstmpte->wire_count++; | dstmpte->ref_count++; | ||||
} else if ((dstmpte = pmap_alloc_l3(dst_pmap, addr, | } else if ((dstmpte = pmap_alloc_l3(dst_pmap, addr, | ||||
NULL)) == NULL) | NULL)) == NULL) | ||||
goto out; | goto out; | ||||
dst_pte = (pt_entry_t *) | dst_pte = (pt_entry_t *) | ||||
PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); | PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); | ||||
dst_pte = &dst_pte[pmap_l3_index(addr)]; | dst_pte = &dst_pte[pmap_l3_index(addr)]; | ||||
if (pmap_load(dst_pte) == 0 && | if (pmap_load(dst_pte) == 0 && | ||||
pmap_try_insert_pv_entry(dst_pmap, addr, | pmap_try_insert_pv_entry(dst_pmap, addr, | ||||
Show All 22 Lines | for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { | ||||
* XXX redundant invalidation | * XXX redundant invalidation | ||||
*/ | */ | ||||
pmap_invalidate_page(dst_pmap, addr); | pmap_invalidate_page(dst_pmap, addr); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
goto out; | goto out; | ||||
} | } | ||||
/* Have we copied all of the valid mappings? */ | /* Have we copied all of the valid mappings? */ | ||||
if (dstmpte->wire_count >= srcmpte->wire_count) | if (dstmpte->ref_count >= srcmpte->ref_count) | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
out: | out: | ||||
/* | /* | ||||
* XXX This barrier may not be needed because the destination pmap is | * XXX This barrier may not be needed because the destination pmap is | ||||
* not active. | * not active. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 348 Lines • ▼ Show 20 Lines | */ | ||||
vm_page_aflag_clear(mt, PGA_WRITEABLE); | vm_page_aflag_clear(mt, PGA_WRITEABLE); | ||||
} | } | ||||
ml3 = pmap_remove_pt_page(pmap, | ml3 = pmap_remove_pt_page(pmap, | ||||
pv->pv_va); | pv->pv_va); | ||||
if (ml3 != NULL) { | if (ml3 != NULL) { | ||||
KASSERT(ml3->valid == VM_PAGE_BITS_ALL, | KASSERT(ml3->valid == VM_PAGE_BITS_ALL, | ||||
("pmap_remove_pages: l3 page not promoted")); | ("pmap_remove_pages: l3 page not promoted")); | ||||
pmap_resident_count_dec(pmap,1); | pmap_resident_count_dec(pmap,1); | ||||
KASSERT(ml3->wire_count == NL3PG, | KASSERT(ml3->ref_count == NL3PG, | ||||
("pmap_remove_pages: l3 page wire count error")); | ("pmap_remove_pages: l3 page ref count error")); | ||||
ml3->wire_count = 0; | ml3->ref_count = 0; | ||||
pmap_add_delayed_free_list(ml3, | pmap_add_delayed_free_list(ml3, | ||||
&free, FALSE); | &free, FALSE); | ||||
} | } | ||||
break; | break; | ||||
case 2: | case 2: | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
TAILQ_REMOVE(&m->md.pv_list, pv, | TAILQ_REMOVE(&m->md.pv_list, pv, | ||||
pv_next); | pv_next); | ||||
▲ Show 20 Lines • Show All 1,120 Lines • ▼ Show 20 Lines | if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) { | ||||
if (ml3 == NULL) { | if (ml3 == NULL) { | ||||
pmap_demote_l2_abort(pmap, va, l2, lockp); | pmap_demote_l2_abort(pmap, va, l2, lockp); | ||||
CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
ml3->wire_count = NL3PG; | ml3->ref_count = NL3PG; | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | } | ||||
} | } | ||||
l3phys = VM_PAGE_TO_PHYS(ml3); | l3phys = VM_PAGE_TO_PHYS(ml3); | ||||
l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); | l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); | ||||
newl3 = (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE; | newl3 = (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE; | ||||
KASSERT((oldl2 & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) != | KASSERT((oldl2 & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) != | ||||
(ATTR_AP(ATTR_AP_RO) | ATTR_SW_DBM), | (ATTR_AP(ATTR_AP_RO) | ATTR_SW_DBM), | ||||
▲ Show 20 Lines • Show All 419 Lines • Show Last 20 Lines |