Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 1,854 Lines • ▼ Show 20 Lines | pmap_init(void) | ||||
PMAP_LOCK(kernel_pmap); | PMAP_LOCK(kernel_pmap); | ||||
for (i = 0; i < nkpt; i++) { | for (i = 0; i < nkpt; i++) { | ||||
mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); | mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); | ||||
KASSERT(mpte >= vm_page_array && | KASSERT(mpte >= vm_page_array && | ||||
mpte < &vm_page_array[vm_page_array_size], | mpte < &vm_page_array[vm_page_array_size], | ||||
("pmap_init: page table page is out of range")); | ("pmap_init: page table page is out of range")); | ||||
mpte->pindex = pmap_pde_pindex(KERNBASE) + i; | mpte->pindex = pmap_pde_pindex(KERNBASE) + i; | ||||
mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); | mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); | ||||
mpte->wire_count = 1; | mpte->ref_count = 1; | ||||
/* | /* | ||||
* Collect the page table pages that were replaced by a 2MB | * Collect the page table pages that were replaced by a 2MB | ||||
* page in create_pagetables(). They are zero filled. | * page in create_pagetables(). They are zero filled. | ||||
*/ | */ | ||||
if (i << PDRSHIFT < KERNend && | if (i << PDRSHIFT < KERNend && | ||||
pmap_insert_pt_page(kernel_pmap, mpte, false)) | pmap_insert_pt_page(kernel_pmap, mpte, false)) | ||||
panic("pmap_init: pmap_insert_pt_page failed"); | panic("pmap_init: pmap_insert_pt_page failed"); | ||||
▲ Show 20 Lines • Show All 1,412 Lines • ▼ Show 20 Lines | |||||
pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) | pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
return (vm_radix_remove(&pmap->pm_root, pmap_pde_pindex(va))); | return (vm_radix_remove(&pmap->pm_root, pmap_pde_pindex(va))); | ||||
} | } | ||||
/* | /* | ||||
* Decrements a page table page's wire count, which is used to record the | * Decrements a page table page's reference count, which is used to record the | ||||
* number of valid page table entries within the page. If the wire count | * number of valid page table entries within the page. If the reference count | ||||
* drops to zero, then the page table page is unmapped. Returns TRUE if the | * drops to zero, then the page table page is unmapped. Returns TRUE if the | ||||
* page table page was unmapped and FALSE otherwise. | * page table page was unmapped and FALSE otherwise. | ||||
*/ | */ | ||||
static inline boolean_t | static inline boolean_t | ||||
pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
{ | { | ||||
--m->wire_count; | --m->ref_count; | ||||
if (m->wire_count == 0) { | if (m->ref_count == 0) { | ||||
_pmap_unwire_ptp(pmap, va, m, free); | _pmap_unwire_ptp(pmap, va, m, free); | ||||
return (TRUE); | return (TRUE); | ||||
} else | } else | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
static void | static void | ||||
_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
* Put page on a list so that it is released after | * Put page on a list so that it is released after | ||||
* *ALL* TLB shootdown is done | * *ALL* TLB shootdown is done | ||||
*/ | */ | ||||
pmap_add_delayed_free_list(m, free, TRUE); | pmap_add_delayed_free_list(m, free, TRUE); | ||||
} | } | ||||
/* | /* | ||||
* After removing a page table entry, this routine is used to | * After removing a page table entry, this routine is used to | ||||
* conditionally free the page, and manage the hold/wire counts. | * conditionally free the page, and manage the reference count. | ||||
*/ | */ | ||||
static int | static int | ||||
pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | ||||
struct spglist *free) | struct spglist *free) | ||||
{ | { | ||||
vm_page_t mpte; | vm_page_t mpte; | ||||
if (va >= VM_MAXUSER_ADDRESS) | if (va >= VM_MAXUSER_ADDRESS) | ||||
▲ Show 20 Lines • Show All 243 Lines • ▼ Show 20 Lines | if ((*pml4 & PG_V) == 0) { | ||||
lockp) == NULL) { | lockp) == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} else { | } else { | ||||
/* Add reference to pdp page */ | /* Add reference to pdp page */ | ||||
pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME); | pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME); | ||||
pdppg->wire_count++; | pdppg->ref_count++; | ||||
} | } | ||||
pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); | pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); | ||||
/* Now find the pdp page */ | /* Now find the pdp page */ | ||||
pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)]; | pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)]; | ||||
*pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
} else { | } else { | ||||
Show All 28 Lines | if ((*pml4 & PG_V) == 0) { | ||||
lockp) == NULL) { | lockp) == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} else { | } else { | ||||
/* Add reference to the pd page */ | /* Add reference to the pd page */ | ||||
pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME); | pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME); | ||||
pdpg->wire_count++; | pdpg->ref_count++; | ||||
} | } | ||||
} | } | ||||
pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME); | pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME); | ||||
/* Now we know where the page directory page is */ | /* Now we know where the page directory page is */ | ||||
pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)]; | pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)]; | ||||
*pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
} | } | ||||
Show All 12 Lines | pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
retry: | retry: | ||||
pdpe = pmap_pdpe(pmap, va); | pdpe = pmap_pdpe(pmap, va); | ||||
if (pdpe != NULL && (*pdpe & PG_V) != 0) { | if (pdpe != NULL && (*pdpe & PG_V) != 0) { | ||||
/* Add a reference to the pd page. */ | /* Add a reference to the pd page. */ | ||||
pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); | pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); | ||||
pdpg->wire_count++; | pdpg->ref_count++; | ||||
} else { | } else { | ||||
/* Allocate a pd page. */ | /* Allocate a pd page. */ | ||||
ptepindex = pmap_pde_pindex(va); | ptepindex = pmap_pde_pindex(va); | ||||
pdpindex = ptepindex >> NPDPEPGSHIFT; | pdpindex = ptepindex >> NPDPEPGSHIFT; | ||||
pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); | pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); | ||||
if (pdpg == NULL && lockp != NULL) | if (pdpg == NULL && lockp != NULL) | ||||
goto retry; | goto retry; | ||||
} | } | ||||
Show All 34 Lines | retry: | ||||
} | } | ||||
/* | /* | ||||
* If the page table page is mapped, we just increment the | * If the page table page is mapped, we just increment the | ||||
* hold count, and activate it. | * hold count, and activate it. | ||||
*/ | */ | ||||
if (pd != NULL && (*pd & PG_V) != 0) { | if (pd != NULL && (*pd & PG_V) != 0) { | ||||
m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); | m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); | ||||
m->wire_count++; | m->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Here if the pte page isn't mapped, or if it has been | * Here if the pte page isn't mapped, or if it has been | ||||
* deallocated. | * deallocated. | ||||
*/ | */ | ||||
m = _pmap_allocpte(pmap, ptepindex, lockp); | m = _pmap_allocpte(pmap, ptepindex, lockp); | ||||
if (m == NULL && lockp != NULL) | if (m == NULL && lockp != NULL) | ||||
goto retry; | goto retry; | ||||
▲ Show 20 Lines • Show All 448 Lines • ▼ Show 20 Lines | next_chunk: | ||||
TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); | TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); | ||||
active_reclaims--; | active_reclaims--; | ||||
mtx_unlock(&pv_chunks_mutex); | mtx_unlock(&pv_chunks_mutex); | ||||
reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di); | reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di); | ||||
if (m_pc == NULL && !SLIST_EMPTY(&free)) { | if (m_pc == NULL && !SLIST_EMPTY(&free)) { | ||||
m_pc = SLIST_FIRST(&free); | m_pc = SLIST_FIRST(&free); | ||||
SLIST_REMOVE_HEAD(&free, plinks.s.ss); | SLIST_REMOVE_HEAD(&free, plinks.s.ss); | ||||
/* Recycle a freed page table page. */ | /* Recycle a freed page table page. */ | ||||
m_pc->wire_count = 1; | m_pc->ref_count = 1; | ||||
} | } | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
return (m_pc); | return (m_pc); | ||||
} | } | ||||
/* | /* | ||||
* free the pv_entry back to the free list | * free the pv_entry back to the free list | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 563 Lines • ▼ Show 20 Lines | if (mpte == NULL) { | ||||
* invalidate the 2MB page mapping and return "failure". | * invalidate the 2MB page mapping and return "failure". | ||||
*/ | */ | ||||
if (mpte == NULL) { | if (mpte == NULL) { | ||||
pmap_demote_pde_abort(pmap, va, pde, oldpde, lockp); | pmap_demote_pde_abort(pmap, va, pde, oldpde, lockp); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
if (!in_kernel) { | if (!in_kernel) { | ||||
mpte->wire_count = NPTEPG; | mpte->ref_count = NPTEPG; | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | } | ||||
} | } | ||||
mptepa = VM_PAGE_TO_PHYS(mpte); | mptepa = VM_PAGE_TO_PHYS(mpte); | ||||
firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | ||||
newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V; | newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V; | ||||
KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, | KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, | ||||
("pmap_demote_pde: oldpde is missing PG_M")); | ("pmap_demote_pde: oldpde is missing PG_M")); | ||||
▲ Show 20 Lines • Show All 144 Lines • ▼ Show 20 Lines | pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
pmap_remove_kernel_pde(pmap, pdq, sva); | pmap_remove_kernel_pde(pmap, pdq, sva); | ||||
} else { | } else { | ||||
mpte = pmap_remove_pt_page(pmap, sva); | mpte = pmap_remove_pt_page(pmap, sva); | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | ||||
("pmap_remove_pde: pte page not promoted")); | ("pmap_remove_pde: pte page not promoted")); | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
KASSERT(mpte->wire_count == NPTEPG, | KASSERT(mpte->ref_count == NPTEPG, | ||||
("pmap_remove_pde: pte page wire count error")); | ("pmap_remove_pde: pte page ref count error")); | ||||
mpte->wire_count = 0; | mpte->ref_count = 0; | ||||
pmap_add_delayed_free_list(mpte, free, FALSE); | pmap_add_delayed_free_list(mpte, free, FALSE); | ||||
} | } | ||||
} | } | ||||
return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free)); | return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free)); | ||||
} | } | ||||
/* | /* | ||||
* pmap_remove_pte: do the things to unmap a page in a process | * pmap_remove_pte: do the things to unmap a page in a process | ||||
▲ Show 20 Lines • Show All 744 Lines • ▼ Show 20 Lines | pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
*/ | */ | ||||
retry: | retry: | ||||
pde = pmap_pde(pmap, va); | pde = pmap_pde(pmap, va); | ||||
if (pde != NULL && (*pde & PG_V) != 0 && ((*pde & PG_PS) == 0 || | if (pde != NULL && (*pde & PG_V) != 0 && ((*pde & PG_PS) == 0 || | ||||
pmap_demote_pde_locked(pmap, pde, va, &lock))) { | pmap_demote_pde_locked(pmap, pde, va, &lock))) { | ||||
pte = pmap_pde_to_pte(pde, va); | pte = pmap_pde_to_pte(pde, va); | ||||
if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { | if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { | ||||
mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} | } | ||||
} else if (va < VM_MAXUSER_ADDRESS) { | } else if (va < VM_MAXUSER_ADDRESS) { | ||||
/* | /* | ||||
* Here if the pte page isn't mapped, or if it has been | * Here if the pte page isn't mapped, or if it has been | ||||
* deallocated. | * deallocated. | ||||
*/ | */ | ||||
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | ||||
mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), | mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), | ||||
Show All 25 Lines | if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) | ||||
pmap->pm_stats.wired_count++; | pmap->pm_stats.wired_count++; | ||||
else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) | else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) | ||||
pmap->pm_stats.wired_count--; | pmap->pm_stats.wired_count--; | ||||
/* | /* | ||||
* Remove the extra PT page reference. | * Remove the extra PT page reference. | ||||
*/ | */ | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
mpte->wire_count--; | mpte->ref_count--; | ||||
KASSERT(mpte->wire_count > 0, | KASSERT(mpte->ref_count > 0, | ||||
("pmap_enter: missing reference to page table page," | ("pmap_enter: missing reference to page table page," | ||||
" va: 0x%lx", va)); | " va: 0x%lx", va)); | ||||
} | } | ||||
/* | /* | ||||
* Has the physical page changed? | * Has the physical page changed? | ||||
*/ | */ | ||||
opa = origpte & PG_FRAME; | opa = origpte & PG_FRAME; | ||||
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | |||||
unchanged: | unchanged: | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
/* | /* | ||||
* If both the page table page and the reservation are fully | * If both the page table page and the reservation are fully | ||||
* populated, then attempt promotion. | * populated, then attempt promotion. | ||||
*/ | */ | ||||
if ((mpte == NULL || mpte->wire_count == NPTEPG) && | if ((mpte == NULL || mpte->ref_count == NPTEPG) && | ||||
pmap_ps_enabled(pmap) && | pmap_ps_enabled(pmap) && | ||||
(m->flags & PG_FICTITIOUS) == 0 && | (m->flags & PG_FICTITIOUS) == 0 && | ||||
vm_reserv_level_iffullpop(m) == 0) | vm_reserv_level_iffullpop(m) == 0) | ||||
pmap_promote_pde(pmap, pde, va, &lock); | pmap_promote_pde(pmap, pde, va, &lock); | ||||
#endif | #endif | ||||
rv = KERN_SUCCESS; | rv = KERN_SUCCESS; | ||||
out: | out: | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) { | ||||
newpde &= ~X86_PG_PKU_MASK; | newpde &= ~X86_PG_PKU_MASK; | ||||
newpde |= pmap_pkru_get(pmap, va); | newpde |= pmap_pkru_get(pmap, va); | ||||
} | } | ||||
pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); | pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); | ||||
pde = &pde[pmap_pde_index(va)]; | pde = &pde[pmap_pde_index(va)]; | ||||
oldpde = *pde; | oldpde = *pde; | ||||
if ((oldpde & PG_V) != 0) { | if ((oldpde & PG_V) != 0) { | ||||
KASSERT(pdpg->wire_count > 1, | KASSERT(pdpg->ref_count > 1, | ||||
("pmap_enter_pde: pdpg's wire count is too low")); | ("pmap_enter_pde: pdpg's reference count is too low")); | ||||
if ((flags & PMAP_ENTER_NOREPLACE) != 0) { | if ((flags & PMAP_ENTER_NOREPLACE) != 0) { | ||||
pdpg->wire_count--; | pdpg->ref_count--; | ||||
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (KERN_FAILURE); | return (KERN_FAILURE); | ||||
} | } | ||||
/* Break the existing mapping(s). */ | /* Break the existing mapping(s). */ | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
if ((oldpde & PG_PS) != 0) { | if ((oldpde & PG_PS) != 0) { | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 157 Lines • ▼ Show 20 Lines | if (va < VM_MAXUSER_ADDRESS) { | ||||
vm_pindex_t ptepindex; | vm_pindex_t ptepindex; | ||||
pd_entry_t *ptepa; | pd_entry_t *ptepa; | ||||
/* | /* | ||||
* Calculate pagetable page index | * Calculate pagetable page index | ||||
*/ | */ | ||||
ptepindex = pmap_pde_pindex(va); | ptepindex = pmap_pde_pindex(va); | ||||
if (mpte && (mpte->pindex == ptepindex)) { | if (mpte && (mpte->pindex == ptepindex)) { | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Get the page directory entry | * Get the page directory entry | ||||
*/ | */ | ||||
ptepa = pmap_pde(pmap, va); | ptepa = pmap_pde(pmap, va); | ||||
/* | /* | ||||
* If the page table page is mapped, we just increment | * If the page table page is mapped, we just increment | ||||
* the hold count, and activate it. Otherwise, we | * the hold count, and activate it. Otherwise, we | ||||
* attempt to allocate a page table page. If this | * attempt to allocate a page table page. If this | ||||
* attempt fails, we don't retry. Instead, we give up. | * attempt fails, we don't retry. Instead, we give up. | ||||
*/ | */ | ||||
if (ptepa && (*ptepa & PG_V) != 0) { | if (ptepa && (*ptepa & PG_V) != 0) { | ||||
if (*ptepa & PG_PS) | if (*ptepa & PG_PS) | ||||
return (NULL); | return (NULL); | ||||
mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME); | mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME); | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Pass NULL instead of the PV list lock | * Pass NULL instead of the PV list lock | ||||
* pointer, because we don't intend to sleep. | * pointer, because we don't intend to sleep. | ||||
*/ | */ | ||||
mpte = _pmap_allocpte(pmap, ptepindex, NULL); | mpte = _pmap_allocpte(pmap, ptepindex, NULL); | ||||
if (mpte == NULL) | if (mpte == NULL) | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
} | } | ||||
pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | ||||
pte = &pte[pmap_pte_index(va)]; | pte = &pte[pmap_pte_index(va)]; | ||||
} else { | } else { | ||||
mpte = NULL; | mpte = NULL; | ||||
pte = vtopte(va); | pte = vtopte(va); | ||||
} | } | ||||
if (*pte) { | if (*pte) { | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
mpte->wire_count--; | mpte->ref_count--; | ||||
mpte = NULL; | mpte = NULL; | ||||
} | } | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
/* | /* | ||||
* Enter on the PV list if part of our managed memory. | * Enter on the PV list if part of our managed memory. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 129 Lines • ▼ Show 20 Lines | for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, 1); | ||||
pde = &pde[pmap_pde_index(addr)]; | pde = &pde[pmap_pde_index(addr)]; | ||||
if ((*pde & PG_V) == 0) { | if ((*pde & PG_V) == 0) { | ||||
pde_store(pde, pa | PG_PS | PG_M | PG_A | | pde_store(pde, pa | PG_PS | PG_M | PG_A | | ||||
PG_U | PG_RW | PG_V); | PG_U | PG_RW | PG_V); | ||||
pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); | pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); | ||||
atomic_add_long(&pmap_pde_mappings, 1); | atomic_add_long(&pmap_pde_mappings, 1); | ||||
} else { | } else { | ||||
/* Continue on if the PDE is already valid. */ | /* Continue on if the PDE is already valid. */ | ||||
pdpg->wire_count--; | pdpg->ref_count--; | ||||
KASSERT(pdpg->wire_count > 0, | KASSERT(pdpg->ref_count > 0, | ||||
("pmap_object_init_pt: missing reference " | ("pmap_object_init_pt: missing reference " | ||||
"to page directory page, va: 0x%lx", addr)); | "to page directory page, va: 0x%lx", addr)); | ||||
} | } | ||||
addr += NBPDR; | addr += NBPDR; | ||||
} | } | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 173 Lines • ▼ Show 20 Lines | if (srcptepaddr & PG_PS) { | ||||
if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 || | if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 || | ||||
pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr, | pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr, | ||||
PMAP_ENTER_NORECLAIM, &lock))) { | PMAP_ENTER_NORECLAIM, &lock))) { | ||||
*pde = srcptepaddr & ~PG_W; | *pde = srcptepaddr & ~PG_W; | ||||
pmap_resident_count_inc(dst_pmap, NBPDR / | pmap_resident_count_inc(dst_pmap, NBPDR / | ||||
PAGE_SIZE); | PAGE_SIZE); | ||||
atomic_add_long(&pmap_pde_mappings, 1); | atomic_add_long(&pmap_pde_mappings, 1); | ||||
} else | } else | ||||
dst_pdpg->wire_count--; | dst_pdpg->ref_count--; | ||||
continue; | continue; | ||||
} | } | ||||
srcptepaddr &= PG_FRAME; | srcptepaddr &= PG_FRAME; | ||||
srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); | srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); | ||||
KASSERT(srcmpte->wire_count > 0, | KASSERT(srcmpte->ref_count > 0, | ||||
("pmap_copy: source page table page is unused")); | ("pmap_copy: source page table page is unused")); | ||||
if (va_next > end_addr) | if (va_next > end_addr) | ||||
va_next = end_addr; | va_next = end_addr; | ||||
src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); | src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); | ||||
src_pte = &src_pte[pmap_pte_index(addr)]; | src_pte = &src_pte[pmap_pte_index(addr)]; | ||||
dstmpte = NULL; | dstmpte = NULL; | ||||
for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { | for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { | ||||
ptetemp = *src_pte; | ptetemp = *src_pte; | ||||
/* | /* | ||||
* We only virtual copy managed pages. | * We only virtual copy managed pages. | ||||
*/ | */ | ||||
if ((ptetemp & PG_MANAGED) == 0) | if ((ptetemp & PG_MANAGED) == 0) | ||||
continue; | continue; | ||||
if (dstmpte != NULL) { | if (dstmpte != NULL) { | ||||
KASSERT(dstmpte->pindex == | KASSERT(dstmpte->pindex == | ||||
pmap_pde_pindex(addr), | pmap_pde_pindex(addr), | ||||
("dstmpte pindex/addr mismatch")); | ("dstmpte pindex/addr mismatch")); | ||||
dstmpte->wire_count++; | dstmpte->ref_count++; | ||||
} else if ((dstmpte = pmap_allocpte(dst_pmap, addr, | } else if ((dstmpte = pmap_allocpte(dst_pmap, addr, | ||||
NULL)) == NULL) | NULL)) == NULL) | ||||
goto out; | goto out; | ||||
dst_pte = (pt_entry_t *) | dst_pte = (pt_entry_t *) | ||||
PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); | PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); | ||||
dst_pte = &dst_pte[pmap_pte_index(addr)]; | dst_pte = &dst_pte[pmap_pte_index(addr)]; | ||||
if (*dst_pte == 0 && | if (*dst_pte == 0 && | ||||
pmap_try_insert_pv_entry(dst_pmap, addr, | pmap_try_insert_pv_entry(dst_pmap, addr, | ||||
Show All 16 Lines | for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { | ||||
* Invalidate those entries. | * Invalidate those entries. | ||||
*/ | */ | ||||
pmap_invalidate_page(dst_pmap, addr); | pmap_invalidate_page(dst_pmap, addr); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
goto out; | goto out; | ||||
} | } | ||||
/* Have we copied all of the valid mappings? */ | /* Have we copied all of the valid mappings? */ | ||||
if (dstmpte->wire_count >= srcmpte->wire_count) | if (dstmpte->ref_count >= srcmpte->ref_count) | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
out: | out: | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
PMAP_UNLOCK(src_pmap); | PMAP_UNLOCK(src_pmap); | ||||
PMAP_UNLOCK(dst_pmap); | PMAP_UNLOCK(dst_pmap); | ||||
▲ Show 20 Lines • Show All 393 Lines • ▼ Show 20 Lines | */ | ||||
TAILQ_EMPTY(&mt->md.pv_list)) | TAILQ_EMPTY(&mt->md.pv_list)) | ||||
vm_page_aflag_clear(mt, PGA_WRITEABLE); | vm_page_aflag_clear(mt, PGA_WRITEABLE); | ||||
} | } | ||||
mpte = pmap_remove_pt_page(pmap, pv->pv_va); | mpte = pmap_remove_pt_page(pmap, pv->pv_va); | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | ||||
("pmap_remove_pages: pte page not promoted")); | ("pmap_remove_pages: pte page not promoted")); | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
KASSERT(mpte->wire_count == NPTEPG, | KASSERT(mpte->ref_count == NPTEPG, | ||||
("pmap_remove_pages: pte page wire count error")); | ("pmap_remove_pages: pte page reference count error")); | ||||
mpte->wire_count = 0; | mpte->ref_count = 0; | ||||
pmap_add_delayed_free_list(mpte, &free, FALSE); | pmap_add_delayed_free_list(mpte, &free, FALSE); | ||||
} | } | ||||
} else { | } else { | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
if ((m->aflags & PGA_WRITEABLE) != 0 && | if ((m->aflags & PGA_WRITEABLE) != 0 && | ||||
TAILQ_EMPTY(&m->md.pv_list) && | TAILQ_EMPTY(&m->md.pv_list) && | ||||
▲ Show 20 Lines • Show All 1,767 Lines • ▼ Show 20 Lines | #if VM_NRESERVLEVEL > 0 | ||||
/* try to promote the mapping */ | /* try to promote the mapping */ | ||||
if (va < VM_MAXUSER_ADDRESS) | if (va < VM_MAXUSER_ADDRESS) | ||||
mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | ||||
else | else | ||||
mpte = NULL; | mpte = NULL; | ||||
m = PHYS_TO_VM_PAGE(*pte & PG_FRAME); | m = PHYS_TO_VM_PAGE(*pte & PG_FRAME); | ||||
if ((mpte == NULL || mpte->wire_count == NPTEPG) && | if ((mpte == NULL || mpte->ref_count == NPTEPG) && | ||||
pmap_ps_enabled(pmap) && | pmap_ps_enabled(pmap) && | ||||
(m->flags & PG_FICTITIOUS) == 0 && | (m->flags & PG_FICTITIOUS) == 0 && | ||||
vm_reserv_level_iffullpop(m) == 0) { | vm_reserv_level_iffullpop(m) == 0) { | ||||
pmap_promote_pde(pmap, pde, va, &lock); | pmap_promote_pde(pmap, pde, va, &lock); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
atomic_add_long(&ad_emulation_superpage_promotions, 1); | atomic_add_long(&ad_emulation_superpage_promotions, 1); | ||||
#endif | #endif | ||||
} | } | ||||
▲ Show 20 Lines • Show All 174 Lines • ▼ Show 20 Lines | pmap_quick_remove_page(vm_offset_t addr) | ||||
pte_store(vtopte(qframe), 0); | pte_store(vtopte(qframe), 0); | ||||
invlpg(qframe); | invlpg(qframe); | ||||
mtx_unlock_spin(&qframe_mtx); | mtx_unlock_spin(&qframe_mtx); | ||||
} | } | ||||
/* | /* | ||||
* Pdp pages from the large map are managed differently from either | * Pdp pages from the large map are managed differently from either | ||||
* kernel or user page table pages. They are permanently allocated at | * kernel or user page table pages. They are permanently allocated at | ||||
* initialization time, and their wire count is permanently set to | * initialization time, and their reference count is permanently set to | ||||
* zero. The pml4 entries pointing to those pages are copied into | * zero. The pml4 entries pointing to those pages are copied into | ||||
* each allocated pmap. | * each allocated pmap. | ||||
* | * | ||||
* In contrast, pd and pt pages are managed like user page table | * In contrast, pd and pt pages are managed like user page table | ||||
* pages. They are dynamically allocated, and their wire count | * pages. They are dynamically allocated, and their reference count | ||||
* represents the number of valid entries within the page. | * represents the number of valid entries within the page. | ||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
pmap_large_map_getptp_unlocked(void) | pmap_large_map_getptp_unlocked(void) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | ||||
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | |||||
retry: | retry: | ||||
pde = pmap_large_map_pde(va); | pde = pmap_large_map_pde(va); | ||||
if (*pde == 0) { | if (*pde == 0) { | ||||
m = pmap_large_map_getptp(); | m = pmap_large_map_getptp(); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto retry; | goto retry; | ||||
mphys = VM_PAGE_TO_PHYS(m); | mphys = VM_PAGE_TO_PHYS(m); | ||||
*pde = mphys | X86_PG_A | X86_PG_RW | X86_PG_V | pg_nx; | *pde = mphys | X86_PG_A | X86_PG_RW | X86_PG_V | pg_nx; | ||||
PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pde))->wire_count++; | PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pde))->ref_count++; | ||||
} else { | } else { | ||||
MPASS((*pde & X86_PG_PS) == 0); | MPASS((*pde & X86_PG_PS) == 0); | ||||
mphys = *pde & PG_FRAME; | mphys = *pde & PG_FRAME; | ||||
} | } | ||||
return ((pt_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pte_index(va)); | return ((pt_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pte_index(va)); | ||||
} | } | ||||
static vm_paddr_t | static vm_paddr_t | ||||
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | for (pa = spa, va = vmem_res; len > 0; pa += inc, va += inc, | ||||
} else if (len >= NBPDR && (pa & PDRMASK) == 0 && | } else if (len >= NBPDR && (pa & PDRMASK) == 0 && | ||||
(va & PDRMASK) == 0) { | (va & PDRMASK) == 0) { | ||||
pde = pmap_large_map_pde(va); | pde = pmap_large_map_pde(va); | ||||
MPASS(*pde == 0); | MPASS(*pde == 0); | ||||
*pde = pa | pg_g | X86_PG_PS | X86_PG_RW | | *pde = pa | pg_g | X86_PG_PS | X86_PG_RW | | ||||
X86_PG_V | X86_PG_A | pg_nx | | X86_PG_V | X86_PG_A | pg_nx | | ||||
pmap_cache_bits(kernel_pmap, mattr, TRUE); | pmap_cache_bits(kernel_pmap, mattr, TRUE); | ||||
PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pde))-> | PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pde))-> | ||||
wire_count++; | ref_count++; | ||||
inc = NBPDR; | inc = NBPDR; | ||||
} else { | } else { | ||||
pte = pmap_large_map_pte(va); | pte = pmap_large_map_pte(va); | ||||
MPASS(*pte == 0); | MPASS(*pte == 0); | ||||
*pte = pa | pg_g | X86_PG_RW | X86_PG_V | | *pte = pa | pg_g | X86_PG_RW | X86_PG_V | | ||||
X86_PG_A | pg_nx | pmap_cache_bits(kernel_pmap, | X86_PG_A | pg_nx | pmap_cache_bits(kernel_pmap, | ||||
mattr, FALSE); | mattr, FALSE); | ||||
PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pte))-> | PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pte))-> | ||||
wire_count++; | ref_count++; | ||||
inc = PAGE_SIZE; | inc = PAGE_SIZE; | ||||
} | } | ||||
} | } | ||||
PMAP_UNLOCK(kernel_pmap); | PMAP_UNLOCK(kernel_pmap); | ||||
MPASS(len == 0); | MPASS(len == 0); | ||||
*addr = (void *)vmem_res; | *addr = (void *)vmem_res; | ||||
return (0); | return (0); | ||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | if ((pd & X86_PG_PS) != 0) { | ||||
(u_long)pde, pd)); | (u_long)pde, pd)); | ||||
KASSERT(va + NBPDR <= sva + len, | KASSERT(va + NBPDR <= sva + len, | ||||
("unmap covers partial 2MB page, sva %#lx va %#lx " | ("unmap covers partial 2MB page, sva %#lx va %#lx " | ||||
"pde %#lx pd %#lx len %#lx", sva, va, (u_long)pde, | "pde %#lx pd %#lx len %#lx", sva, va, (u_long)pde, | ||||
pd, len)); | pd, len)); | ||||
pde_store(pde, 0); | pde_store(pde, 0); | ||||
inc = NBPDR; | inc = NBPDR; | ||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pde)); | m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pde)); | ||||
m->wire_count--; | m->ref_count--; | ||||
if (m->wire_count == 0) { | if (m->ref_count == 0) { | ||||
*pdpe = 0; | *pdpe = 0; | ||||
SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); | SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
pte = pmap_pde_to_pte(pde, va); | pte = pmap_pde_to_pte(pde, va); | ||||
KASSERT((*pte & X86_PG_V) != 0, | KASSERT((*pte & X86_PG_V) != 0, | ||||
("invalid pte va %#lx pte %#lx pt %#lx", va, | ("invalid pte va %#lx pte %#lx pt %#lx", va, | ||||
(u_long)pte, *pte)); | (u_long)pte, *pte)); | ||||
pte_clear(pte); | pte_clear(pte); | ||||
inc = PAGE_SIZE; | inc = PAGE_SIZE; | ||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pte)); | m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pte)); | ||||
m->wire_count--; | m->ref_count--; | ||||
if (m->wire_count == 0) { | if (m->ref_count == 0) { | ||||
*pde = 0; | *pde = 0; | ||||
SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); | SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); | ||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pde)); | m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pde)); | ||||
m->wire_count--; | m->ref_count--; | ||||
if (m->wire_count == 0) { | if (m->ref_count == 0) { | ||||
*pdpe = 0; | *pdpe = 0; | ||||
SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); | SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
pmap_invalidate_range(kernel_pmap, sva, sva + len); | pmap_invalidate_range(kernel_pmap, sva, sva + len); | ||||
PMAP_UNLOCK(kernel_pmap); | PMAP_UNLOCK(kernel_pmap); | ||||
vm_page_free_pages_toq(&spgf, false); | vm_page_free_pages_toq(&spgf, false); | ||||
▲ Show 20 Lines • Show All 190 Lines • ▼ Show 20 Lines | m = vm_page_grab(pti_obj, pti_pg_idx++, VM_ALLOC_NOBUSY | | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO); | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | ||||
return (m); | return (m); | ||||
} | } | ||||
static bool | static bool | ||||
pmap_pti_free_page(vm_page_t m) | pmap_pti_free_page(vm_page_t m) | ||||
{ | { | ||||
KASSERT(m->wire_count > 0, ("page %p not wired", m)); | KASSERT(m->ref_count > 0, ("page %p not referenced", m)); | ||||
if (!vm_page_unwire_noq(m)) | if (!vm_page_unwire_noq(m)) | ||||
return (false); | return (false); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (true); | return (true); | ||||
} | } | ||||
static void | static void | ||||
pmap_pti_init(void) | pmap_pti_init(void) | ||||
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
pmap_pti_wire_pte(void *pte) | pmap_pti_wire_pte(void *pte) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | VM_OBJECT_ASSERT_WLOCKED(pti_obj); | ||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pte)); | m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pte)); | ||||
m->wire_count++; | m->ref_count++; | ||||
} | } | ||||
static void | static void | ||||
pmap_pti_unwire_pde(void *pde, bool only_ref) | pmap_pti_unwire_pde(void *pde, bool only_ref) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | VM_OBJECT_ASSERT_WLOCKED(pti_obj); | ||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pde)); | m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pde)); | ||||
MPASS(m->wire_count > 0); | MPASS(m->ref_count > 0); | ||||
MPASS(only_ref || m->wire_count > 1); | MPASS(only_ref || m->ref_count > 1); | ||||
pmap_pti_free_page(m); | pmap_pti_free_page(m); | ||||
} | } | ||||
static void | static void | ||||
pmap_pti_unwire_pte(void *pte, vm_offset_t va) | pmap_pti_unwire_pte(void *pte, vm_offset_t va) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | VM_OBJECT_ASSERT_WLOCKED(pti_obj); | ||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pte)); | m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pte)); | ||||
MPASS(m->wire_count > 0); | MPASS(m->ref_count > 0); | ||||
if (pmap_pti_free_page(m)) { | if (pmap_pti_free_page(m)) { | ||||
pde = pmap_pti_pde(va); | pde = pmap_pti_pde(va); | ||||
MPASS((*pde & (X86_PG_PS | X86_PG_V)) == X86_PG_V); | MPASS((*pde & (X86_PG_PS | X86_PG_V)) == X86_PG_V); | ||||
*pde = 0; | *pde = 0; | ||||
pmap_pti_unwire_pde(pde, false); | pmap_pti_unwire_pde(pde, false); | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 745 Lines • Show Last 20 Lines |