Changeset View
Changeset View
Standalone View
Standalone View
sys/i386/i386/pmap.c
Show First 20 Lines • Show All 986 Lines • ▼ Show 20 Lines | __CONCAT(PMTYPE, init)(void) | ||||
PMAP_LOCK(kernel_pmap); | PMAP_LOCK(kernel_pmap); | ||||
for (i = 0; i < NKPT; i++) { | for (i = 0; i < NKPT; i++) { | ||||
mpte = PHYS_TO_VM_PAGE(KPTphys + ptoa(i)); | mpte = PHYS_TO_VM_PAGE(KPTphys + ptoa(i)); | ||||
KASSERT(mpte >= vm_page_array && | KASSERT(mpte >= vm_page_array && | ||||
mpte < &vm_page_array[vm_page_array_size], | mpte < &vm_page_array[vm_page_array_size], | ||||
("pmap_init: page table page is out of range")); | ("pmap_init: page table page is out of range")); | ||||
mpte->pindex = i + KPTDI; | mpte->pindex = i + KPTDI; | ||||
mpte->phys_addr = KPTphys + ptoa(i); | mpte->phys_addr = KPTphys + ptoa(i); | ||||
mpte->wire_count = 1; | mpte->ref_count = 1; | ||||
/* | /* | ||||
* Collect the page table pages that were replaced by a 2/4MB | * Collect the page table pages that were replaced by a 2/4MB | ||||
* page. They are filled with equivalent 4KB page mappings. | * page. They are filled with equivalent 4KB page mappings. | ||||
*/ | */ | ||||
if (pseflag != 0 && | if (pseflag != 0 && | ||||
KERNBASE <= i << PDRSHIFT && i << PDRSHIFT < KERNend && | KERNBASE <= i << PDRSHIFT && i << PDRSHIFT < KERNend && | ||||
pmap_insert_pt_page(kernel_pmap, mpte, true)) | pmap_insert_pt_page(kernel_pmap, mpte, true)) | ||||
▲ Show 20 Lines • Show All 943 Lines • ▼ Show 20 Lines | |||||
pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) | pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT)); | return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT)); | ||||
} | } | ||||
/* | /* | ||||
* Decrements a page table page's wire count, which is used to record the | * Decrements a page table page's reference count, which is used to record the | ||||
* number of valid page table entries within the page. If the wire count | * number of valid page table entries within the page. If the reference count | ||||
* drops to zero, then the page table page is unmapped. Returns TRUE if the | * drops to zero, then the page table page is unmapped. Returns TRUE if the | ||||
* page table page was unmapped and FALSE otherwise. | * page table page was unmapped and FALSE otherwise. | ||||
*/ | */ | ||||
static inline boolean_t | static inline boolean_t | ||||
pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free) | pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free) | ||||
{ | { | ||||
--m->wire_count; | --m->ref_count; | ||||
if (m->wire_count == 0) { | if (m->ref_count == 0) { | ||||
_pmap_unwire_ptp(pmap, m, free); | _pmap_unwire_ptp(pmap, m, free); | ||||
return (TRUE); | return (TRUE); | ||||
} else | } else | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
static void | static void | ||||
_pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free) | _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free) | ||||
Show All 13 Lines | _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free) | ||||
* shootdown is done. | * shootdown is done. | ||||
*/ | */ | ||||
MPASS(pmap != kernel_pmap); | MPASS(pmap != kernel_pmap); | ||||
pmap_add_delayed_free_list(m, free, TRUE); | pmap_add_delayed_free_list(m, free, TRUE); | ||||
} | } | ||||
/* | /* | ||||
* After removing a page table entry, this routine is used to | * After removing a page table entry, this routine is used to | ||||
* conditionally free the page, and manage the hold/wire counts. | * conditionally free the page, and manage the reference count. | ||||
*/ | */ | ||||
static int | static int | ||||
pmap_unuse_pt(pmap_t pmap, vm_offset_t va, struct spglist *free) | pmap_unuse_pt(pmap_t pmap, vm_offset_t va, struct spglist *free) | ||||
{ | { | ||||
pd_entry_t ptepde; | pd_entry_t ptepde; | ||||
vm_page_t mpte; | vm_page_t mpte; | ||||
if (pmap == kernel_pmap) | if (pmap == kernel_pmap) | ||||
▲ Show 20 Lines • Show All 162 Lines • ▼ Show 20 Lines | retry: | ||||
} | } | ||||
/* | /* | ||||
* If the page table page is mapped, we just increment the | * If the page table page is mapped, we just increment the | ||||
* hold count, and activate it. | * hold count, and activate it. | ||||
*/ | */ | ||||
if (ptepa) { | if (ptepa) { | ||||
m = PHYS_TO_VM_PAGE(ptepa & PG_FRAME); | m = PHYS_TO_VM_PAGE(ptepa & PG_FRAME); | ||||
m->wire_count++; | m->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Here if the pte page isn't mapped, or if it has | * Here if the pte page isn't mapped, or if it has | ||||
* been deallocated. | * been deallocated. | ||||
*/ | */ | ||||
m = _pmap_allocpte(pmap, ptepindex, flags); | m = _pmap_allocpte(pmap, ptepindex, flags); | ||||
if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0) | if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0) | ||||
goto retry; | goto retry; | ||||
▲ Show 20 Lines • Show All 250 Lines • ▼ Show 20 Lines | if (pmap != NULL) { | ||||
pmap_invalidate_all_int(pmap); | pmap_invalidate_all_int(pmap); | ||||
if (pmap != locked_pmap) | if (pmap != locked_pmap) | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) { | if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) { | ||||
m_pc = SLIST_FIRST(&free); | m_pc = SLIST_FIRST(&free); | ||||
SLIST_REMOVE_HEAD(&free, plinks.s.ss); | SLIST_REMOVE_HEAD(&free, plinks.s.ss); | ||||
/* Recycle a freed page table page. */ | /* Recycle a freed page table page. */ | ||||
m_pc->wire_count = 1; | m_pc->ref_count = 1; | ||||
} | } | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
return (m_pc); | return (m_pc); | ||||
} | } | ||||
/* | /* | ||||
* free the pv_entry back to the free list | * free the pv_entry back to the free list | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 342 Lines • ▼ Show 20 Lines | if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, | ||||
if ((oldpde & PG_G) == 0) | if ((oldpde & PG_G) == 0) | ||||
pmap_invalidate_pde_page(pmap, sva, oldpde); | pmap_invalidate_pde_page(pmap, sva, oldpde); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x" | CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
if (pmap != kernel_pmap) { | if (pmap != kernel_pmap) { | ||||
mpte->wire_count = NPTEPG; | mpte->ref_count = NPTEPG; | ||||
pmap->pm_stats.resident_count++; | pmap->pm_stats.resident_count++; | ||||
} | } | ||||
} | } | ||||
mptepa = VM_PAGE_TO_PHYS(mpte); | mptepa = VM_PAGE_TO_PHYS(mpte); | ||||
/* | /* | ||||
* If the page mapping is in the kernel's address space, then the | * If the page mapping is in the kernel's address space, then the | ||||
* KPTmap can provide access to the page table page. Otherwise, | * KPTmap can provide access to the page table page. Otherwise, | ||||
▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
pmap_remove_kernel_pde(pmap, pdq, sva); | pmap_remove_kernel_pde(pmap, pdq, sva); | ||||
} else { | } else { | ||||
mpte = pmap_remove_pt_page(pmap, sva); | mpte = pmap_remove_pt_page(pmap, sva); | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | ||||
("pmap_remove_pde: pte page not promoted")); | ("pmap_remove_pde: pte page not promoted")); | ||||
pmap->pm_stats.resident_count--; | pmap->pm_stats.resident_count--; | ||||
KASSERT(mpte->wire_count == NPTEPG, | KASSERT(mpte->ref_count == NPTEPG, | ||||
("pmap_remove_pde: pte page wire count error")); | ("pmap_remove_pde: pte page ref count error")); | ||||
mpte->wire_count = 0; | mpte->ref_count = 0; | ||||
pmap_add_delayed_free_list(mpte, free, FALSE); | pmap_add_delayed_free_list(mpte, free, FALSE); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* pmap_remove_pte: do the things to unmap a page in a process | * pmap_remove_pte: do the things to unmap a page in a process | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 719 Lines • ▼ Show 20 Lines | if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) | ||||
pmap->pm_stats.wired_count++; | pmap->pm_stats.wired_count++; | ||||
else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) | else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) | ||||
pmap->pm_stats.wired_count--; | pmap->pm_stats.wired_count--; | ||||
/* | /* | ||||
* Remove the extra PT page reference. | * Remove the extra PT page reference. | ||||
*/ | */ | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
mpte->wire_count--; | mpte->ref_count--; | ||||
KASSERT(mpte->wire_count > 0, | KASSERT(mpte->ref_count > 0, | ||||
("pmap_enter: missing reference to page table page," | ("pmap_enter: missing reference to page table page," | ||||
" va: 0x%x", va)); | " va: 0x%x", va)); | ||||
} | } | ||||
/* | /* | ||||
* Has the physical page changed? | * Has the physical page changed? | ||||
*/ | */ | ||||
opa = origpte & PG_FRAME; | opa = origpte & PG_FRAME; | ||||
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | |||||
unchanged: | unchanged: | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
/* | /* | ||||
* If both the page table page and the reservation are fully | * If both the page table page and the reservation are fully | ||||
* populated, then attempt promotion. | * populated, then attempt promotion. | ||||
*/ | */ | ||||
if ((mpte == NULL || mpte->wire_count == NPTEPG) && | if ((mpte == NULL || mpte->ref_count == NPTEPG) && | ||||
pg_ps_enabled && (m->flags & PG_FICTITIOUS) == 0 && | pg_ps_enabled && (m->flags & PG_FICTITIOUS) == 0 && | ||||
vm_reserv_level_iffullpop(m) == 0) | vm_reserv_level_iffullpop(m) == 0) | ||||
pmap_promote_pde(pmap, pde, va); | pmap_promote_pde(pmap, pde, va); | ||||
#endif | #endif | ||||
rv = KERN_SUCCESS; | rv = KERN_SUCCESS; | ||||
out: | out: | ||||
sched_unpin(); | sched_unpin(); | ||||
▲ Show 20 Lines • Show All 206 Lines • ▼ Show 20 Lines | if (pmap != kernel_pmap) { | ||||
u_int ptepindex; | u_int ptepindex; | ||||
pd_entry_t ptepa; | pd_entry_t ptepa; | ||||
/* | /* | ||||
* Calculate pagetable page index | * Calculate pagetable page index | ||||
*/ | */ | ||||
ptepindex = va >> PDRSHIFT; | ptepindex = va >> PDRSHIFT; | ||||
if (mpte && (mpte->pindex == ptepindex)) { | if (mpte && (mpte->pindex == ptepindex)) { | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Get the page directory entry | * Get the page directory entry | ||||
*/ | */ | ||||
ptepa = pmap->pm_pdir[ptepindex]; | ptepa = pmap->pm_pdir[ptepindex]; | ||||
/* | /* | ||||
* If the page table page is mapped, we just increment | * If the page table page is mapped, we just increment | ||||
* the hold count, and activate it. | * the hold count, and activate it. | ||||
*/ | */ | ||||
if (ptepa) { | if (ptepa) { | ||||
if (ptepa & PG_PS) | if (ptepa & PG_PS) | ||||
return (NULL); | return (NULL); | ||||
mpte = PHYS_TO_VM_PAGE(ptepa & PG_FRAME); | mpte = PHYS_TO_VM_PAGE(ptepa & PG_FRAME); | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
mpte = _pmap_allocpte(pmap, ptepindex, | mpte = _pmap_allocpte(pmap, ptepindex, | ||||
PMAP_ENTER_NOSLEEP); | PMAP_ENTER_NOSLEEP); | ||||
if (mpte == NULL) | if (mpte == NULL) | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
} | } | ||||
} else { | } else { | ||||
mpte = NULL; | mpte = NULL; | ||||
} | } | ||||
sched_pin(); | sched_pin(); | ||||
pte = pmap_pte_quick(pmap, va); | pte = pmap_pte_quick(pmap, va); | ||||
if (*pte) { | if (*pte) { | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
mpte->wire_count--; | mpte->ref_count--; | ||||
mpte = NULL; | mpte = NULL; | ||||
} | } | ||||
sched_unpin(); | sched_unpin(); | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
/* | /* | ||||
* Enter on the PV list if part of our managed memory. | * Enter on the PV list if part of our managed memory. | ||||
▲ Show 20 Lines • Show All 278 Lines • ▼ Show 20 Lines | if (srcptepaddr & PG_PS) { | ||||
dst_pmap->pm_stats.resident_count += | dst_pmap->pm_stats.resident_count += | ||||
NBPDR / PAGE_SIZE; | NBPDR / PAGE_SIZE; | ||||
pmap_pde_mappings++; | pmap_pde_mappings++; | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME); | srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME); | ||||
KASSERT(srcmpte->wire_count > 0, | KASSERT(srcmpte->ref_count > 0, | ||||
("pmap_copy: source page table page is unused")); | ("pmap_copy: source page table page is unused")); | ||||
if (pdnxt > end_addr) | if (pdnxt > end_addr) | ||||
pdnxt = end_addr; | pdnxt = end_addr; | ||||
src_pte = pmap_pte_quick3(src_pmap, addr); | src_pte = pmap_pte_quick3(src_pmap, addr); | ||||
while (addr < pdnxt) { | while (addr < pdnxt) { | ||||
ptetemp = *src_pte; | ptetemp = *src_pte; | ||||
Show All 23 Lines | while (addr < pdnxt) { | ||||
&free)) { | &free)) { | ||||
pmap_invalidate_page_int( | pmap_invalidate_page_int( | ||||
dst_pmap, addr); | dst_pmap, addr); | ||||
vm_page_free_pages_toq(&free, | vm_page_free_pages_toq(&free, | ||||
true); | true); | ||||
} | } | ||||
goto out; | goto out; | ||||
} | } | ||||
if (dstmpte->wire_count >= srcmpte->wire_count) | if (dstmpte->ref_count >= srcmpte->ref_count) | ||||
break; | break; | ||||
} | } | ||||
addr += PAGE_SIZE; | addr += PAGE_SIZE; | ||||
src_pte++; | src_pte++; | ||||
} | } | ||||
} | } | ||||
out: | out: | ||||
sched_unpin(); | sched_unpin(); | ||||
▲ Show 20 Lines • Show All 370 Lines • ▼ Show 20 Lines | */ | ||||
if (TAILQ_EMPTY(&mt->md.pv_list)) | if (TAILQ_EMPTY(&mt->md.pv_list)) | ||||
vm_page_aflag_clear(mt, PGA_WRITEABLE); | vm_page_aflag_clear(mt, PGA_WRITEABLE); | ||||
} | } | ||||
mpte = pmap_remove_pt_page(pmap, pv->pv_va); | mpte = pmap_remove_pt_page(pmap, pv->pv_va); | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | ||||
("pmap_remove_pages: pte page not promoted")); | ("pmap_remove_pages: pte page not promoted")); | ||||
pmap->pm_stats.resident_count--; | pmap->pm_stats.resident_count--; | ||||
KASSERT(mpte->wire_count == NPTEPG, | KASSERT(mpte->ref_count == NPTEPG, | ||||
("pmap_remove_pages: pte page wire count error")); | ("pmap_remove_pages: pte page ref count error")); | ||||
mpte->wire_count = 0; | mpte->ref_count = 0; | ||||
pmap_add_delayed_free_list(mpte, &free, FALSE); | pmap_add_delayed_free_list(mpte, &free, FALSE); | ||||
} | } | ||||
} else { | } else { | ||||
pmap->pm_stats.resident_count--; | pmap->pm_stats.resident_count--; | ||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | ||||
if (TAILQ_EMPTY(&m->md.pv_list) && | if (TAILQ_EMPTY(&m->md.pv_list) && | ||||
(m->flags & PG_FICTITIOUS) == 0) { | (m->flags & PG_FICTITIOUS) == 0) { | ||||
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
▲ Show 20 Lines • Show All 1,548 Lines • Show Last 20 Lines |