Differential D20538 Diff 58423 head/sys/i386/i386/pmap.c

Changeset View

Standalone View

head/sys/i386/i386/pmap.c

Show First 20 Lines • Show All 312 Lines • ▼ Show 20 Lines

static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);		static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
static bool pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,		static bool pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot);		vm_prot_t prot);
static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,		static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,
u_int flags, vm_page_t m);		u_int flags, vm_page_t m);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,		static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte);		vm_page_t m, vm_prot_t prot, vm_page_t mpte);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);		static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted);
static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,		static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
pd_entry_t pde);		pd_entry_t pde);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);		static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static boolean_t pmap_is_modified_pvh(struct md_page *pvh);		static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);		static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);		static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);		static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);		static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
▲ Show 20 Lines • Show All 655 Lines • ▼ Show 20 Lines	__CONCAT(PMTYPE, init)(void)
for (i = 0; i < NKPT; i++) {		for (i = 0; i < NKPT; i++) {
mpte = PHYS_TO_VM_PAGE(KPTphys + ptoa(i));		mpte = PHYS_TO_VM_PAGE(KPTphys + ptoa(i));
KASSERT(mpte >= vm_page_array &&		KASSERT(mpte >= vm_page_array &&
mpte < &vm_page_array[vm_page_array_size],		mpte < &vm_page_array[vm_page_array_size],
("pmap_init: page table page is out of range"));		("pmap_init: page table page is out of range"));
mpte->pindex = i + KPTDI;		mpte->pindex = i + KPTDI;
mpte->phys_addr = KPTphys + ptoa(i);		mpte->phys_addr = KPTphys + ptoa(i);
mpte->wire_count = 1;		mpte->wire_count = 1;

		/*
		* Collect the page table pages that were replaced by a 2/4MB
		* page. They are filled with equivalent 4KB page mappings.
		*/
if (pseflag != 0 &&		if (pseflag != 0 &&
KERNBASE <= i << PDRSHIFT && i << PDRSHIFT < KERNend &&		KERNBASE <= i << PDRSHIFT && i << PDRSHIFT < KERNend &&
pmap_insert_pt_page(kernel_pmap, mpte))		pmap_insert_pt_page(kernel_pmap, mpte, true))
panic("pmap_init: pmap_insert_pt_page failed");		panic("pmap_init: pmap_insert_pt_page failed");
}		}
PMAP_UNLOCK(kernel_pmap);		PMAP_UNLOCK(kernel_pmap);
vm_wire_add(NKPT);		vm_wire_add(NKPT);

/*		/*
* Initialize the address space (zone) for the pv entries. Set a		* Initialize the address space (zone) for the pv entries. Set a
* high water mark so that the system can recover from excessive		* high water mark so that the system can recover from excessive
▲ Show 20 Lines • Show All 891 Lines • ▼ Show 20 Lines	pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
SLIST_INSERT_HEAD(free, m, plinks.s.ss);		SLIST_INSERT_HEAD(free, m, plinks.s.ss);
}		}

/*		/*
* Inserts the specified page table page into the specified pmap's collection		* Inserts the specified page table page into the specified pmap's collection
* of idle page table pages. Each of a pmap's page table pages is responsible		* of idle page table pages. Each of a pmap's page table pages is responsible
* for mapping a distinct range of virtual addresses. The pmap's collection is		* for mapping a distinct range of virtual addresses. The pmap's collection is
* ordered by this virtual address range.		* ordered by this virtual address range.
		*
		* If "promoted" is false, then the page table page "mpte" must be zero filled.
*/		*/
static __inline int		static __inline int
pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)		pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted)
{		{

PMAP_LOCK_ASSERT(pmap, MA_OWNED);		PMAP_LOCK_ASSERT(pmap, MA_OWNED);
		mpte->valid = promoted ? VM_PAGE_BITS_ALL : 0;
return (vm_radix_insert(&pmap->pm_root, mpte));		return (vm_radix_insert(&pmap->pm_root, mpte));
}		}

/*		/*
* Removes the page table page mapping the specified virtual address from the		* Removes the page table page mapping the specified virtual address from the
* specified pmap's collection of idle page table pages, and returns it.		* specified pmap's collection of idle page table pages, and returns it.
* Otherwise, returns NULL if there is no page table page corresponding to the		* Otherwise, returns NULL if there is no page table page corresponding to the
* specified virtual address.		* specified virtual address.
▲ Show 20 Lines • Show All 901 Lines • ▼ Show 20 Lines	#endif
newpte = oldpde & ~PG_PS;		newpte = oldpde & ~PG_PS;
if ((newpte & PG_PDE_PAT) != 0)		if ((newpte & PG_PDE_PAT) != 0)
newpte ^= PG_PDE_PAT \| PG_PTE_PAT;		newpte ^= PG_PDE_PAT \| PG_PTE_PAT;

/*		/*
* If the page table page is not leftover from an earlier promotion,		* If the page table page is not leftover from an earlier promotion,
* initialize it.		* initialize it.
*/		*/
if ((oldpde & PG_PROMOTED) == 0)		if (mpte->valid == 0)
pmap_fill_ptp(firstpte, newpte);		pmap_fill_ptp(firstpte, newpte);

KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),		KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
("pmap_demote_pde: firstpte and newpte map different physical"		("pmap_demote_pde: firstpte and newpte map different physical"
" addresses"));		" addresses"));

/*		/*
* If the mapping has changed attributes, update the page table		* If the mapping has changed attributes, update the page table
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines	pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
mpte = pmap_remove_pt_page(pmap, va);		mpte = pmap_remove_pt_page(pmap, va);
if (mpte == NULL)		if (mpte == NULL)
panic("pmap_remove_kernel_pde: Missing pt page.");		panic("pmap_remove_kernel_pde: Missing pt page.");

mptepa = VM_PAGE_TO_PHYS(mpte);		mptepa = VM_PAGE_TO_PHYS(mpte);
newpde = mptepa \| PG_M \| PG_A \| PG_RW \| PG_V;		newpde = mptepa \| PG_M \| PG_A \| PG_RW \| PG_V;

/*		/*
* Initialize the page table page.		* If this page table page was unmapped by a promotion, then it
		* contains valid mappings. Zero it to invalidate those mappings.
*/		*/
		if (mpte->valid != 0)
pagezero((void *)&KPTmap[i386_btop(trunc_4mpage(va))]);		pagezero((void *)&KPTmap[i386_btop(trunc_4mpage(va))]);

/*		/*
* Remove the mapping.		* Remove the mapping.
*/		*/
if (workaround_erratum383)		if (workaround_erratum383)
pmap_update_pde(pmap, va, pde, newpde);		pmap_update_pde(pmap, va, pde, newpde);
else		else
pmap_kenter_pde(va, newpde);		pmap_kenter_pde(va, newpde);
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines	for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
vm_page_aflag_clear(m, PGA_WRITEABLE);		vm_page_aflag_clear(m, PGA_WRITEABLE);
}		}
}		}
if (pmap == kernel_pmap) {		if (pmap == kernel_pmap) {
pmap_remove_kernel_pde(pmap, pdq, sva);		pmap_remove_kernel_pde(pmap, pdq, sva);
} else {		} else {
mpte = pmap_remove_pt_page(pmap, sva);		mpte = pmap_remove_pt_page(pmap, sva);
if (mpte != NULL) {		if (mpte != NULL) {
		KASSERT(mpte->valid == VM_PAGE_BITS_ALL,
		("pmap_remove_pde: pte page not promoted"));
pmap->pm_stats.resident_count--;		pmap->pm_stats.resident_count--;
KASSERT(mpte->wire_count == NPTEPG,		KASSERT(mpte->wire_count == NPTEPG,
("pmap_remove_pde: pte page wire count error"));		("pmap_remove_pde: pte page wire count error"));
mpte->wire_count = 0;		mpte->wire_count = 0;
pmap_add_delayed_free_list(mpte, free, FALSE);		pmap_add_delayed_free_list(mpte, free, FALSE);
}		}
}		}
}		}
▲ Show 20 Lines • Show All 557 Lines • ▼ Show 20 Lines	setpte:
* destroyed by pmap_remove_pde().		* destroyed by pmap_remove_pde().
*/		*/
mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);		mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
KASSERT(mpte >= vm_page_array &&		KASSERT(mpte >= vm_page_array &&
mpte < &vm_page_array[vm_page_array_size],		mpte < &vm_page_array[vm_page_array_size],
("pmap_promote_pde: page table page is out of range"));		("pmap_promote_pde: page table page is out of range"));
KASSERT(mpte->pindex == va >> PDRSHIFT,		KASSERT(mpte->pindex == va >> PDRSHIFT,
("pmap_promote_pde: page table page's pindex is wrong"));		("pmap_promote_pde: page table page's pindex is wrong"));
if (pmap_insert_pt_page(pmap, mpte)) {		if (pmap_insert_pt_page(pmap, mpte, true)) {
pmap_pde_p_failures++;		pmap_pde_p_failures++;
CTR2(KTR_PMAP,		CTR2(KTR_PMAP,
"pmap_promote_pde: failure for va %#x in pmap %p", va,		"pmap_promote_pde: failure for va %#x in pmap %p", va,
pmap);		pmap);
return;		return;
}		}

/*		/*
▲ Show 20 Lines • Show All 361 Lines • ▼ Show 20 Lines	if ((oldpde & PG_PS) != 0) {
if ((oldpde & PG_G) == 0)		if ((oldpde & PG_G) == 0)
pmap_invalidate_pde_page(pmap, va, oldpde);		pmap_invalidate_pde_page(pmap, va, oldpde);
} else {		} else {
if (pmap_remove_ptes(pmap, va, va + NBPDR, &free))		if (pmap_remove_ptes(pmap, va, va + NBPDR, &free))
pmap_invalidate_all_int(pmap);		pmap_invalidate_all_int(pmap);
}		}
vm_page_free_pages_toq(&free, true);		vm_page_free_pages_toq(&free, true);
if (pmap == kernel_pmap) {		if (pmap == kernel_pmap) {
mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
if (pmap_insert_pt_page(pmap, mt)) {
/*		/*
* XXX Currently, this can't happen because		* Both pmap_remove_pde() and pmap_remove_ptes() will
* we do not perform pmap_enter(psind == 1)		* leave the kernel page table page zero filled.
* on the kernel pmap.
*/		*/
		mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
		if (pmap_insert_pt_page(pmap, mt, false))
panic("pmap_enter_pde: trie insert failed");		panic("pmap_enter_pde: trie insert failed");
}
} else		} else
KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",		KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",
pde));		pde));
}		}
if ((newpde & PG_MANAGED) != 0) {		if ((newpde & PG_MANAGED) != 0) {
/*		/*
* Abort this mapping if its PV entry could not be created.		* Abort this mapping if its PV entry could not be created.
*/		*/
▲ Show 20 Lines • Show All 861 Lines • ▼ Show 20 Lines	*/
TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);		TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
if (TAILQ_EMPTY(&pvh->pv_list)) {		if (TAILQ_EMPTY(&pvh->pv_list)) {
for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)		for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
if (TAILQ_EMPTY(&mt->md.pv_list))		if (TAILQ_EMPTY(&mt->md.pv_list))
vm_page_aflag_clear(mt, PGA_WRITEABLE);		vm_page_aflag_clear(mt, PGA_WRITEABLE);
}		}
mpte = pmap_remove_pt_page(pmap, pv->pv_va);		mpte = pmap_remove_pt_page(pmap, pv->pv_va);
if (mpte != NULL) {		if (mpte != NULL) {
		KASSERT(mpte->valid == VM_PAGE_BITS_ALL,
		("pmap_remove_pages: pte page not promoted"));
pmap->pm_stats.resident_count--;		pmap->pm_stats.resident_count--;
KASSERT(mpte->wire_count == NPTEPG,		KASSERT(mpte->wire_count == NPTEPG,
("pmap_remove_pages: pte page wire count error"));		("pmap_remove_pages: pte page wire count error"));
mpte->wire_count = 0;		mpte->wire_count = 0;
pmap_add_delayed_free_list(mpte, &free, FALSE);		pmap_add_delayed_free_list(mpte, &free, FALSE);
}		}
} else {		} else {
pmap->pm_stats.resident_count--;		pmap->pm_stats.resident_count--;
▲ Show 20 Lines • Show All 1,346 Lines • Show Last 20 Lines