Differential D20538 Diff 58423 head/sys/amd64/amd64/pmap.c

Changeset View

Standalone View

head/sys/amd64/amd64/pmap.c

This file is larger than 256 KB, so syntax highlighting is disabled by default.

Show First 20 Lines • Show All 1,047 Lines • ▼ Show 20 Lines	static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,
vm_offset_t va);		vm_offset_t va);
static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,		static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, struct rwlock **lockp);		vm_prot_t prot, struct rwlock **lockp);
static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,		static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,
u_int flags, vm_page_t m, struct rwlock **lockp);		u_int flags, vm_page_t m, struct rwlock **lockp);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,		static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);		vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);		static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);		static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted);
static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva,		static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva,
vm_offset_t eva);		vm_offset_t eva);
static void pmap_invalidate_cache_range_all(vm_offset_t sva,		static void pmap_invalidate_cache_range_all(vm_offset_t sva,
vm_offset_t eva);		vm_offset_t eva);
static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,		static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
pd_entry_t pde);		pd_entry_t pde);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);		static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
static vm_page_t pmap_large_map_getptp_unlocked(void);		static vm_page_t pmap_large_map_getptp_unlocked(void);
▲ Show 20 Lines • Show All 687 Lines • ▼ Show 20 Lines	pmap_init(void)
for (i = 0; i < nkpt; i++) {		for (i = 0; i < nkpt; i++) {
mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));		mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
KASSERT(mpte >= vm_page_array &&		KASSERT(mpte >= vm_page_array &&
mpte < &vm_page_array[vm_page_array_size],		mpte < &vm_page_array[vm_page_array_size],
("pmap_init: page table page is out of range"));		("pmap_init: page table page is out of range"));
mpte->pindex = pmap_pde_pindex(KERNBASE) + i;		mpte->pindex = pmap_pde_pindex(KERNBASE) + i;
mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);		mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
mpte->wire_count = 1;		mpte->wire_count = 1;

		/*
		* Collect the page table pages that were replaced by a 2MB
		* page in create_pagetables(). They are zero filled.
		*/
if (i << PDRSHIFT < KERNend &&		if (i << PDRSHIFT < KERNend &&
pmap_insert_pt_page(kernel_pmap, mpte))		pmap_insert_pt_page(kernel_pmap, mpte, false))
panic("pmap_init: pmap_insert_pt_page failed");		panic("pmap_init: pmap_insert_pt_page failed");
}		}
PMAP_UNLOCK(kernel_pmap);		PMAP_UNLOCK(kernel_pmap);
vm_wire_add(nkpt);		vm_wire_add(nkpt);

/*		/*
* If the kernel is running on a virtual machine, then it must assume		* If the kernel is running on a virtual machine, then it must assume
* that MCA is enabled by the hypervisor. Moreover, the kernel must		* that MCA is enabled by the hypervisor. Moreover, the kernel must
▲ Show 20 Lines • Show All 1,354 Lines • ▼ Show 20 Lines	pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
SLIST_INSERT_HEAD(free, m, plinks.s.ss);		SLIST_INSERT_HEAD(free, m, plinks.s.ss);
}		}

/*		/*
* Inserts the specified page table page into the specified pmap's collection		* Inserts the specified page table page into the specified pmap's collection
* of idle page table pages. Each of a pmap's page table pages is responsible		* of idle page table pages. Each of a pmap's page table pages is responsible
* for mapping a distinct range of virtual addresses. The pmap's collection is		* for mapping a distinct range of virtual addresses. The pmap's collection is
* ordered by this virtual address range.		* ordered by this virtual address range.
		*
		* If "promoted" is false, then the page table page "mpte" must be zero filled.
*/		*/
static __inline int		static __inline int
pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)		pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted)
{		{

PMAP_LOCK_ASSERT(pmap, MA_OWNED);		PMAP_LOCK_ASSERT(pmap, MA_OWNED);
		mpte->valid = promoted ? VM_PAGE_BITS_ALL : 0;
return (vm_radix_insert(&pmap->pm_root, mpte));		return (vm_radix_insert(&pmap->pm_root, mpte));
}		}

/*		/*
* Removes the page table page mapping the specified virtual address from the		* Removes the page table page mapping the specified virtual address from the
* specified pmap's collection of idle page table pages, and returns it.		* specified pmap's collection of idle page table pages, and returns it.
* Otherwise, returns NULL if there is no page table page corresponding to the		* Otherwise, returns NULL if there is no page table page corresponding to the
* specified virtual address.		* specified virtual address.
▲ Show 20 Lines • Show All 1,475 Lines • ▼ Show 20 Lines	KASSERT((oldpde & (PG_M \| PG_RW)) != PG_RW,
("pmap_demote_pde: oldpde is missing PG_M"));		("pmap_demote_pde: oldpde is missing PG_M"));
newpte = oldpde & ~PG_PS;		newpte = oldpde & ~PG_PS;
newpte = pmap_swap_pat(pmap, newpte);		newpte = pmap_swap_pat(pmap, newpte);

/*		/*
* If the page table page is not leftover from an earlier promotion,		* If the page table page is not leftover from an earlier promotion,
* initialize it.		* initialize it.
*/		*/
if ((oldpde & PG_PROMOTED) == 0)		if (mpte->valid == 0)
pmap_fill_ptp(firstpte, newpte);		pmap_fill_ptp(firstpte, newpte);

pmap_demote_pde_check(firstpte, newpte);		pmap_demote_pde_check(firstpte, newpte);

/*		/*
* If the mapping has changed attributes, update the page table		* If the mapping has changed attributes, update the page table
* entries.		* entries.
*/		*/
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines	pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
mpte = pmap_remove_pt_page(pmap, va);		mpte = pmap_remove_pt_page(pmap, va);
if (mpte == NULL)		if (mpte == NULL)
panic("pmap_remove_kernel_pde: Missing pt page.");		panic("pmap_remove_kernel_pde: Missing pt page.");

mptepa = VM_PAGE_TO_PHYS(mpte);		mptepa = VM_PAGE_TO_PHYS(mpte);
newpde = mptepa \| X86_PG_M \| X86_PG_A \| X86_PG_RW \| X86_PG_V;		newpde = mptepa \| X86_PG_M \| X86_PG_A \| X86_PG_RW \| X86_PG_V;

/*		/*
* Initialize the page table page.		* If this page table page was unmapped by a promotion, then it
		* contains valid mappings. Zero it to invalidate those mappings.
*/		*/
		if (mpte->valid != 0)
pagezero((void *)PHYS_TO_DMAP(mptepa));		pagezero((void *)PHYS_TO_DMAP(mptepa));

/*		/*
* Demote the mapping.		* Demote the mapping.
*/		*/
if (workaround_erratum383)		if (workaround_erratum383)
pmap_update_pde(pmap, va, pde, newpde);		pmap_update_pde(pmap, va, pde, newpde);
else		else
pde_store(pde, newpde);		pde_store(pde, newpde);
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines	for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
pmap_delayed_invl_page(m);		pmap_delayed_invl_page(m);
}		}
}		}
if (pmap == kernel_pmap) {		if (pmap == kernel_pmap) {
pmap_remove_kernel_pde(pmap, pdq, sva);		pmap_remove_kernel_pde(pmap, pdq, sva);
} else {		} else {
mpte = pmap_remove_pt_page(pmap, sva);		mpte = pmap_remove_pt_page(pmap, sva);
if (mpte != NULL) {		if (mpte != NULL) {
		KASSERT(mpte->valid == VM_PAGE_BITS_ALL,
		("pmap_remove_pde: pte page not promoted"));
pmap_resident_count_dec(pmap, 1);		pmap_resident_count_dec(pmap, 1);
KASSERT(mpte->wire_count == NPTEPG,		KASSERT(mpte->wire_count == NPTEPG,
("pmap_remove_pde: pte page wire count error"));		("pmap_remove_pde: pte page wire count error"));
mpte->wire_count = 0;		mpte->wire_count = 0;
pmap_add_delayed_free_list(mpte, free, FALSE);		pmap_add_delayed_free_list(mpte, free, FALSE);
}		}
}		}
return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free));		return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free));
▲ Show 20 Lines • Show All 617 Lines • ▼ Show 20 Lines	setpte:
* destroyed by pmap_remove_pde().		* destroyed by pmap_remove_pde().
*/		*/
mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);		mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
KASSERT(mpte >= vm_page_array &&		KASSERT(mpte >= vm_page_array &&
mpte < &vm_page_array[vm_page_array_size],		mpte < &vm_page_array[vm_page_array_size],
("pmap_promote_pde: page table page is out of range"));		("pmap_promote_pde: page table page is out of range"));
KASSERT(mpte->pindex == pmap_pde_pindex(va),		KASSERT(mpte->pindex == pmap_pde_pindex(va),
("pmap_promote_pde: page table page's pindex is wrong"));		("pmap_promote_pde: page table page's pindex is wrong"));
if (pmap_insert_pt_page(pmap, mpte)) {		if (pmap_insert_pt_page(pmap, mpte, true)) {
atomic_add_long(&pmap_pde_p_failures, 1);		atomic_add_long(&pmap_pde_p_failures, 1);
CTR2(KTR_PMAP,		CTR2(KTR_PMAP,
"pmap_promote_pde: failure for va %#lx in pmap %p", va,		"pmap_promote_pde: failure for va %#lx in pmap %p", va,
pmap);		pmap);
return;		return;
}		}

/*		/*
▲ Show 20 Lines • Show All 410 Lines • ▼ Show 20 Lines	if ((oldpde & PG_PS) != 0) {
pmap_delayed_invl_start();		pmap_delayed_invl_start();
if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free,		if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free,
lockp))		lockp))
pmap_invalidate_all(pmap);		pmap_invalidate_all(pmap);
pmap_delayed_invl_finish();		pmap_delayed_invl_finish();
}		}
vm_page_free_pages_toq(&free, true);		vm_page_free_pages_toq(&free, true);
if (va >= VM_MAXUSER_ADDRESS) {		if (va >= VM_MAXUSER_ADDRESS) {
mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
if (pmap_insert_pt_page(pmap, mt)) {
/*		/*
* XXX Currently, this can't happen because		* Both pmap_remove_pde() and pmap_remove_ptes() will
* we do not perform pmap_enter(psind == 1)		* leave the kernel page table page zero filled.
* on the kernel pmap.
*/		*/
		mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
		if (pmap_insert_pt_page(pmap, mt, false))
panic("pmap_enter_pde: trie insert failed");		panic("pmap_enter_pde: trie insert failed");
}
} else		} else
KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",		KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",
pde));		pde));
}		}
if ((newpde & PG_MANAGED) != 0) {		if ((newpde & PG_MANAGED) != 0) {
/*		/*
* Abort this mapping if its PV entry could not be created.		* Abort this mapping if its PV entry could not be created.
*/		*/
▲ Show 20 Lines • Show All 973 Lines • ▼ Show 20 Lines	*/
if (TAILQ_EMPTY(&pvh->pv_list)) {		if (TAILQ_EMPTY(&pvh->pv_list)) {
for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)		for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
if ((mt->aflags & PGA_WRITEABLE) != 0 &&		if ((mt->aflags & PGA_WRITEABLE) != 0 &&
TAILQ_EMPTY(&mt->md.pv_list))		TAILQ_EMPTY(&mt->md.pv_list))
vm_page_aflag_clear(mt, PGA_WRITEABLE);		vm_page_aflag_clear(mt, PGA_WRITEABLE);
}		}
mpte = pmap_remove_pt_page(pmap, pv->pv_va);		mpte = pmap_remove_pt_page(pmap, pv->pv_va);
if (mpte != NULL) {		if (mpte != NULL) {
		KASSERT(mpte->valid == VM_PAGE_BITS_ALL,
		("pmap_remove_pages: pte page not promoted"));
pmap_resident_count_dec(pmap, 1);		pmap_resident_count_dec(pmap, 1);
KASSERT(mpte->wire_count == NPTEPG,		KASSERT(mpte->wire_count == NPTEPG,
("pmap_remove_pages: pte page wire count error"));		("pmap_remove_pages: pte page wire count error"));
mpte->wire_count = 0;		mpte->wire_count = 0;
pmap_add_delayed_free_list(mpte, &free, FALSE);		pmap_add_delayed_free_list(mpte, &free, FALSE);
}		}
} else {		} else {
pmap_resident_count_dec(pmap, 1);		pmap_resident_count_dec(pmap, 1);
▲ Show 20 Lines • Show All 3,034 Lines • Show Last 20 Lines