Differential D13485 Diff 39946 head/sys/amd64/amd64/pmap.c

Changeset View

Standalone View

head/sys/amd64/amd64/pmap.c

Show First 20 Lines • Show All 2,363 Lines • ▼ Show 20 Lines	while (count-- > 0) {
va += PAGE_SIZE;		va += PAGE_SIZE;
}		}
pmap_invalidate_range(kernel_pmap, sva, va);		pmap_invalidate_range(kernel_pmap, sva, va);
}		}

/***************************************************		/***************************************************
* Page table page management routines.....		* Page table page management routines.....
***************************************************/		***************************************************/
static __inline void
pmap_free_zero_pages(struct spglist *free)
{
vm_page_t m;
int count;

for (count = 0; (m = SLIST_FIRST(free)) != NULL; count++) {
SLIST_REMOVE_HEAD(free, plinks.s.ss);
/* Preserve the page's PG_ZERO setting. */
vm_page_free_toq(m);
}
vm_wire_sub(count);
}

/*		/*
* Schedule the specified unused page table page to be freed. Specifically,		* Schedule the specified unused page table page to be freed. Specifically,
* add the page to the specified list of pages that will be released to the		* add the page to the specified list of pages that will be released to the
* physical memory manager after the TLB has been updated.		* physical memory manager after the TLB has been updated.
*/		*/
static __inline void		static __inline void
pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,		pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
boolean_t set_PG_ZERO)		boolean_t set_PG_ZERO)
▲ Show 20 Lines • Show All 883 Lines • ▼ Show 20 Lines	next_chunk:
mtx_unlock(&pv_chunks_mutex);		mtx_unlock(&pv_chunks_mutex);
reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di);		reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di);
if (m_pc == NULL && !SLIST_EMPTY(&free)) {		if (m_pc == NULL && !SLIST_EMPTY(&free)) {
m_pc = SLIST_FIRST(&free);		m_pc = SLIST_FIRST(&free);
SLIST_REMOVE_HEAD(&free, plinks.s.ss);		SLIST_REMOVE_HEAD(&free, plinks.s.ss);
/* Recycle a freed page table page. */		/* Recycle a freed page table page. */
m_pc->wire_count = 1;		m_pc->wire_count = 1;
}		}
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
return (m_pc);		return (m_pc);
}		}

/*		/*
* free the pv_entry back to the free list		* free the pv_entry back to the free list
*/		*/
static void		static void
free_pv_entry(pmap_t pmap, pv_entry_t pv)		free_pv_entry(pmap_t pmap, pv_entry_t pv)
▲ Show 20 Lines • Show All 482 Lines • ▼ Show 20 Lines	if ((oldpde & PG_A) == 0 \|\| (mpte = vm_page_alloc(NULL,
pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va <		pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va <
DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) \|		DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) \|
VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED)) == NULL) {		VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED)) == NULL) {
SLIST_INIT(&free);		SLIST_INIT(&free);
sva = trunc_2mpage(va);		sva = trunc_2mpage(va);
pmap_remove_pde(pmap, pde, sva, &free, lockp);		pmap_remove_pde(pmap, pde, sva, &free, lockp);
if ((oldpde & PG_G) == 0)		if ((oldpde & PG_G) == 0)
pmap_invalidate_pde_page(pmap, sva, oldpde);		pmap_invalidate_pde_page(pmap, sva, oldpde);
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"		CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"
" in pmap %p", va, pmap);		" in pmap %p", va, pmap);
return (FALSE);		return (FALSE);
}		}
if (va < VM_MAXUSER_ADDRESS)		if (va < VM_MAXUSER_ADDRESS)
pmap_resident_count_inc(pmap, 1);		pmap_resident_count_inc(pmap, 1);
}		}
mptepa = VM_PAGE_TO_PHYS(mpte);		mptepa = VM_PAGE_TO_PHYS(mpte);
▲ Show 20 Lines • Show All 385 Lines • ▼ Show 20 Lines	pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
}		}
if (lock != NULL)		if (lock != NULL)
rw_wunlock(lock);		rw_wunlock(lock);
out:		out:
if (anyvalid)		if (anyvalid)
pmap_invalidate_all(pmap);		pmap_invalidate_all(pmap);
PMAP_UNLOCK(pmap);		PMAP_UNLOCK(pmap);
pmap_delayed_invl_finished();		pmap_delayed_invl_finished();
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
}		}

/*		/*
* Routine: pmap_remove_all		* Routine: pmap_remove_all
* Function:		* Function:
* Removes this physical page from		* Removes this physical page from
* all physical maps in which it resides.		* all physical maps in which it resides.
* Reflects back modify bits to the pager.		* Reflects back modify bits to the pager.
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines	while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);		TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
m->md.pv_gen++;		m->md.pv_gen++;
free_pv_entry(pmap, pv);		free_pv_entry(pmap, pv);
PMAP_UNLOCK(pmap);		PMAP_UNLOCK(pmap);
}		}
vm_page_aflag_clear(m, PGA_WRITEABLE);		vm_page_aflag_clear(m, PGA_WRITEABLE);
rw_wunlock(lock);		rw_wunlock(lock);
pmap_delayed_invl_wait(m);		pmap_delayed_invl_wait(m);
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
}		}

/*		/*
* pmap_protect_pde: do the things to protect a 2mpage in a process		* pmap_protect_pde: do the things to protect a 2mpage in a process
*/		*/
static boolean_t		static boolean_t
pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)		pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
{		{
▲ Show 20 Lines • Show All 651 Lines • ▼ Show 20 Lines	if ((oldpde & PG_PS) != 0) {
pmap_invalidate_pde_page(pmap, va, oldpde);		pmap_invalidate_pde_page(pmap, va, oldpde);
} else {		} else {
pmap_delayed_invl_started();		pmap_delayed_invl_started();
if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free,		if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free,
lockp))		lockp))
pmap_invalidate_all(pmap);		pmap_invalidate_all(pmap);
pmap_delayed_invl_finished();		pmap_delayed_invl_finished();
}		}
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
if (va >= VM_MAXUSER_ADDRESS) {		if (va >= VM_MAXUSER_ADDRESS) {
mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);		mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
if (pmap_insert_pt_page(pmap, mt)) {		if (pmap_insert_pt_page(pmap, mt)) {
/*		/*
* XXX Currently, this can't happen because		* XXX Currently, this can't happen because
* we do not perform pmap_enter(psind == 1)		* we do not perform pmap_enter(psind == 1)
* on the kernel pmap.		* on the kernel pmap.
*/		*/
Show All 12 Lines	if (!pmap_pv_insert_pde(pmap, va, newpde, flags, lockp)) {
if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {		if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
/*		/*
* Although "va" is not mapped, paging-		* Although "va" is not mapped, paging-
* structure caches could nonetheless have		* structure caches could nonetheless have
* entries that refer to the freed page table		* entries that refer to the freed page table
* pages. Invalidate those entries.		* pages. Invalidate those entries.
*/		*/
pmap_invalidate_page(pmap, va);		pmap_invalidate_page(pmap, va);
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
}		}
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"		CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);		" in pmap %p", va, pmap);
return (KERN_RESOURCE_SHORTAGE);		return (KERN_RESOURCE_SHORTAGE);
}		}
if ((newpde & PG_RW) != 0) {		if ((newpde & PG_RW) != 0) {
for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)		for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
vm_page_aflag_set(mt, PGA_WRITEABLE);		vm_page_aflag_set(mt, PGA_WRITEABLE);
▲ Show 20 Lines • Show All 164 Lines • ▼ Show 20 Lines	if (mpte != NULL) {
if (pmap_unwire_ptp(pmap, va, mpte, &free)) {		if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
/*		/*
* Although "va" is not mapped, paging-		* Although "va" is not mapped, paging-
* structure caches could nonetheless have		* structure caches could nonetheless have
* entries that refer to the freed page table		* entries that refer to the freed page table
* pages. Invalidate those entries.		* pages. Invalidate those entries.
*/		*/
pmap_invalidate_page(pmap, va);		pmap_invalidate_page(pmap, va);
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
}		}
mpte = NULL;		mpte = NULL;
}		}
return (mpte);		return (mpte);
}		}

/*		/*
* Increment counters		* Increment counters
▲ Show 20 Lines • Show All 361 Lines • ▼ Show 20 Lines	while (addr < va_next) {
* mapped, paging-structure		* mapped, paging-structure
* caches could nonetheless		* caches could nonetheless
* have entries that refer to		* have entries that refer to
* the freed page table pages.		* the freed page table pages.
* Invalidate those entries.		* Invalidate those entries.
*/		*/
pmap_invalidate_page(dst_pmap,		pmap_invalidate_page(dst_pmap,
addr);		addr);
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free,
		true);
}		}
goto out;		goto out;
}		}
if (dstmpte->wire_count >= srcmpte->wire_count)		if (dstmpte->wire_count >= srcmpte->wire_count)
break;		break;
}		}
addr += PAGE_SIZE;		addr += PAGE_SIZE;
src_pte++;		src_pte++;
▲ Show 20 Lines • Show All 401 Lines • ▼ Show 20 Lines	if (allfree) {
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);		TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
free_pv_chunk(pc);		free_pv_chunk(pc);
}		}
}		}
if (lock != NULL)		if (lock != NULL)
rw_wunlock(lock);		rw_wunlock(lock);
pmap_invalidate_all(pmap);		pmap_invalidate_all(pmap);
PMAP_UNLOCK(pmap);		PMAP_UNLOCK(pmap);
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
}		}

static boolean_t		static boolean_t
pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified)		pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified)
{		{
struct rwlock *lock;		struct rwlock *lock;
pv_entry_t pv;		pv_entry_t pv;
struct md_page *pvh;		struct md_page *pvh;
▲ Show 20 Lines • Show All 468 Lines • ▼ Show 20 Lines	if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) {
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);		TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);		TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
m->md.pv_gen++;		m->md.pv_gen++;
}		}
} while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +		} while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
not_cleared < PMAP_TS_REFERENCED_MAX);		not_cleared < PMAP_TS_REFERENCED_MAX);
out:		out:
rw_wunlock(lock);		rw_wunlock(lock);
pmap_free_zero_pages(&free);		vm_page_free_pages_toq(&free, true);
return (cleared + not_cleared);		return (cleared + not_cleared);
}		}

/*		/*
* Apply the given advice to the specified range of addresses within the		* Apply the given advice to the specified range of addresses within the
* given pmap. Depending on the advice, clear the referenced and/or		* given pmap. Depending on the advice, clear the referenced and/or
* modified flags in each mapping and set the mapped page's dirty field.		* modified flags in each mapping and set the mapped page's dirty field.
*/		*/
▲ Show 20 Lines • Show All 1,591 Lines • Show Last 20 Lines