Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
Show First 20 Lines • Show All 2,210 Lines • ▼ Show 20 Lines | while (count-- > 0) { | ||||
va += PAGE_SIZE; | va += PAGE_SIZE; | ||||
} | } | ||||
pmap_invalidate_range(kernel_pmap, sva, va); | pmap_invalidate_range(kernel_pmap, sva, va); | ||||
} | } | ||||
/*************************************************** | /*************************************************** | ||||
* Page table page management routines..... | * Page table page management routines..... | ||||
***************************************************/ | ***************************************************/ | ||||
static __inline void | |||||
pmap_free_zero_pages(struct spglist *free) | |||||
{ | |||||
vm_page_t m; | |||||
int count; | |||||
for (count = 0; (m = SLIST_FIRST(free)) != NULL; count++) { | |||||
SLIST_REMOVE_HEAD(free, plinks.s.ss); | |||||
/* Preserve the page's PG_ZERO setting. */ | |||||
vm_page_free_toq(m); | |||||
} | |||||
atomic_subtract_int(&vm_cnt.v_wire_count, count); | |||||
} | |||||
/* | /* | ||||
* Schedule the specified unused page table page to be freed. Specifically, | * Schedule the specified unused page table page to be freed. Specifically, | ||||
* add the page to the specified list of pages that will be released to the | * add the page to the specified list of pages that will be released to the | ||||
* physical memory manager after the TLB has been updated. | * physical memory manager after the TLB has been updated. | ||||
*/ | */ | ||||
static __inline void | static __inline void | ||||
pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, | pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, | ||||
boolean_t set_PG_ZERO) | boolean_t set_PG_ZERO) | ||||
▲ Show 20 Lines • Show All 836 Lines • ▼ Show 20 Lines | next_chunk: | ||||
mtx_unlock(&pv_chunks_mutex); | mtx_unlock(&pv_chunks_mutex); | ||||
reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di); | reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di); | ||||
if (m_pc == NULL && !SLIST_EMPTY(&free)) { | if (m_pc == NULL && !SLIST_EMPTY(&free)) { | ||||
m_pc = SLIST_FIRST(&free); | m_pc = SLIST_FIRST(&free); | ||||
SLIST_REMOVE_HEAD(&free, plinks.s.ss); | SLIST_REMOVE_HEAD(&free, plinks.s.ss); | ||||
/* Recycle a freed page table page. */ | /* Recycle a freed page table page. */ | ||||
m_pc->wire_count = 1; | m_pc->wire_count = 1; | ||||
} | } | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
return (m_pc); | return (m_pc); | ||||
} | } | ||||
/* | /* | ||||
* free the pv_entry back to the free list | * free the pv_entry back to the free list | ||||
*/ | */ | ||||
static void | static void | ||||
free_pv_entry(pmap_t pmap, pv_entry_t pv) | free_pv_entry(pmap_t pmap, pv_entry_t pv) | ||||
▲ Show 20 Lines • Show All 482 Lines • ▼ Show 20 Lines | if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, | ||||
pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va < | pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va < | ||||
DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | | DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
sva = trunc_2mpage(va); | sva = trunc_2mpage(va); | ||||
pmap_remove_pde(pmap, pde, sva, &free, lockp); | pmap_remove_pde(pmap, pde, sva, &free, lockp); | ||||
if ((oldpde & PG_G) == 0) | if ((oldpde & PG_G) == 0) | ||||
pmap_invalidate_pde_page(pmap, sva, oldpde); | pmap_invalidate_pde_page(pmap, sva, oldpde); | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
if (va < VM_MAXUSER_ADDRESS) | if (va < VM_MAXUSER_ADDRESS) | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | } | ||||
mptepa = VM_PAGE_TO_PHYS(mpte); | mptepa = VM_PAGE_TO_PHYS(mpte); | ||||
▲ Show 20 Lines • Show All 385 Lines • ▼ Show 20 Lines | pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
} | } | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
out: | out: | ||||
if (anyvalid) | if (anyvalid) | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
pmap_delayed_invl_finished(); | pmap_delayed_invl_finished(); | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
/* | /* | ||||
* Routine: pmap_remove_all | * Routine: pmap_remove_all | ||||
* Function: | * Function: | ||||
* Removes this physical page from | * Removes this physical page from | ||||
* all physical maps in which it resides. | * all physical maps in which it resides. | ||||
* Reflects back modify bits to the pager. | * Reflects back modify bits to the pager. | ||||
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { | ||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
free_pv_entry(pmap, pv); | free_pv_entry(pmap, pv); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
vm_page_aflag_clear(m, PGA_WRITEABLE); | vm_page_aflag_clear(m, PGA_WRITEABLE); | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
pmap_delayed_invl_wait(m); | pmap_delayed_invl_wait(m); | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
/* | /* | ||||
* pmap_protect_pde: do the things to protect a 2mpage in a process | * pmap_protect_pde: do the things to protect a 2mpage in a process | ||||
*/ | */ | ||||
static boolean_t | static boolean_t | ||||
pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot) | pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 651 Lines • ▼ Show 20 Lines | if ((oldpde & PG_PS) != 0) { | ||||
pmap_invalidate_pde_page(pmap, va, oldpde); | pmap_invalidate_pde_page(pmap, va, oldpde); | ||||
} else { | } else { | ||||
pmap_delayed_invl_started(); | pmap_delayed_invl_started(); | ||||
if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free, | if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free, | ||||
lockp)) | lockp)) | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
pmap_delayed_invl_finished(); | pmap_delayed_invl_finished(); | ||||
} | } | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
if (va >= VM_MAXUSER_ADDRESS) { | if (va >= VM_MAXUSER_ADDRESS) { | ||||
mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | ||||
if (pmap_insert_pt_page(pmap, mt)) { | if (pmap_insert_pt_page(pmap, mt)) { | ||||
/* | /* | ||||
* XXX Currently, this can't happen because | * XXX Currently, this can't happen because | ||||
* we do not perform pmap_enter(psind == 1) | * we do not perform pmap_enter(psind == 1) | ||||
* on the kernel pmap. | * on the kernel pmap. | ||||
*/ | */ | ||||
Show All 12 Lines | if (!pmap_pv_insert_pde(pmap, va, newpde, flags, lockp)) { | ||||
if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { | if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { | ||||
/* | /* | ||||
* Although "va" is not mapped, paging- | * Although "va" is not mapped, paging- | ||||
* structure caches could nonetheless have | * structure caches could nonetheless have | ||||
* entries that refer to the freed page table | * entries that refer to the freed page table | ||||
* pages. Invalidate those entries. | * pages. Invalidate those entries. | ||||
*/ | */ | ||||
pmap_invalidate_page(pmap, va); | pmap_invalidate_page(pmap, va); | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (KERN_RESOURCE_SHORTAGE); | return (KERN_RESOURCE_SHORTAGE); | ||||
} | } | ||||
if ((newpde & PG_RW) != 0) { | if ((newpde & PG_RW) != 0) { | ||||
for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) | for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) | ||||
vm_page_aflag_set(mt, PGA_WRITEABLE); | vm_page_aflag_set(mt, PGA_WRITEABLE); | ||||
▲ Show 20 Lines • Show All 164 Lines • ▼ Show 20 Lines | if (mpte != NULL) { | ||||
if (pmap_unwire_ptp(pmap, va, mpte, &free)) { | if (pmap_unwire_ptp(pmap, va, mpte, &free)) { | ||||
/* | /* | ||||
* Although "va" is not mapped, paging- | * Although "va" is not mapped, paging- | ||||
* structure caches could nonetheless have | * structure caches could nonetheless have | ||||
* entries that refer to the freed page table | * entries that refer to the freed page table | ||||
* pages. Invalidate those entries. | * pages. Invalidate those entries. | ||||
*/ | */ | ||||
pmap_invalidate_page(pmap, va); | pmap_invalidate_page(pmap, va); | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
mpte = NULL; | mpte = NULL; | ||||
} | } | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
/* | /* | ||||
* Increment counters | * Increment counters | ||||
▲ Show 20 Lines • Show All 361 Lines • ▼ Show 20 Lines | while (addr < va_next) { | ||||
* mapped, paging-structure | * mapped, paging-structure | ||||
* caches could nonetheless | * caches could nonetheless | ||||
* have entries that refer to | * have entries that refer to | ||||
* the freed page table pages. | * the freed page table pages. | ||||
* Invalidate those entries. | * Invalidate those entries. | ||||
*/ | */ | ||||
pmap_invalidate_page(dst_pmap, | pmap_invalidate_page(dst_pmap, | ||||
addr); | addr); | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, | ||||
true); | |||||
} | } | ||||
goto out; | goto out; | ||||
} | } | ||||
if (dstmpte->wire_count >= srcmpte->wire_count) | if (dstmpte->wire_count >= srcmpte->wire_count) | ||||
break; | break; | ||||
} | } | ||||
addr += PAGE_SIZE; | addr += PAGE_SIZE; | ||||
src_pte++; | src_pte++; | ||||
▲ Show 20 Lines • Show All 401 Lines • ▼ Show 20 Lines | if (allfree) { | ||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | ||||
free_pv_chunk(pc); | free_pv_chunk(pc); | ||||
} | } | ||||
} | } | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
static boolean_t | static boolean_t | ||||
pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) | pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) | ||||
{ | { | ||||
struct rwlock *lock; | struct rwlock *lock; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
▲ Show 20 Lines • Show All 468 Lines • ▼ Show 20 Lines | if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { | ||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | ||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
} | } | ||||
} while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + | } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + | ||||
not_cleared < PMAP_TS_REFERENCED_MAX); | not_cleared < PMAP_TS_REFERENCED_MAX); | ||||
out: | out: | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
pmap_free_zero_pages(&free); | vm_page_free_pages_toq(&free, true); | ||||
return (cleared + not_cleared); | return (cleared + not_cleared); | ||||
} | } | ||||
/* | /* | ||||
* Apply the given advice to the specified range of addresses within the | * Apply the given advice to the specified range of addresses within the | ||||
* given pmap. Depending on the advice, clear the referenced and/or | * given pmap. Depending on the advice, clear the referenced and/or | ||||
* modified flags in each mapping and set the mapped page's dirty field. | * modified flags in each mapping and set the mapped page's dirty field. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 1,275 Lines • Show Last 20 Lines |