Differential D9665 Diff 25723 head/sys/amd64/amd64/pmap.c

Changeset View

Standalone View

head/sys/amd64/amd64/pmap.c

Show First 20 Lines • Show All 607 Lines • ▼ Show 20 Lines
static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,		static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,
vm_offset_t va);		vm_offset_t va);
static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,		static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, struct rwlock **lockp);		vm_prot_t prot, struct rwlock **lockp);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,		static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);		vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);		static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);		static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
		static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
		pd_entry_t pde);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);		static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask);		static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask);
static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,		static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
struct rwlock **lockp);		struct rwlock **lockp);
static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,		static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
vm_prot_t prot);		vm_prot_t prot);
static void pmap_pte_attr(pt_entry_t *pte, int cache_bits, int mask);		static void pmap_pte_attr(pt_entry_t *pte, int cache_bits, int mask);
static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,		static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
▲ Show 20 Lines • Show All 1,209 Lines • ▼ Show 20 Lines	pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
pmap_update_pde_store(pmap, pde, newpde);		pmap_update_pde_store(pmap, pde, newpde);
if (pmap == kernel_pmap \|\| pmap == PCPU_GET(curpmap))		if (pmap == kernel_pmap \|\| pmap == PCPU_GET(curpmap))
pmap_update_pde_invalidate(pmap, va, newpde);		pmap_update_pde_invalidate(pmap, va, newpde);
else		else
pmap->pm_pcids[0].pm_gen = 0;		pmap->pm_pcids[0].pm_gen = 0;
}		}
#endif /* !SMP */		#endif /* !SMP */

		static void
		pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
		{

		/*
		* When the PDE has PG_PROMOTED set, the 2MB page mapping was created
		* by a promotion that did not invalidate the 512 4KB page mappings
		* that might exist in the TLB. Consequently, at this point, the TLB
		* may hold both 4KB and 2MB page mappings for the address range [va,
		* va + NBPDR). Therefore, the entire range must be invalidated here.
		* In contrast, when PG_PROMOTED is clear, the TLB will not hold any
		* 4KB page mappings for the address range [va, va + NBPDR), and so a
		* single INVLPG suffices to invalidate the 2MB page mapping from the
		* TLB.
		*/
		if ((pde & PG_PROMOTED) != 0)
		pmap_invalidate_range(pmap, va, va + NBPDR - 1);
		else
		pmap_invalidate_page(pmap, va);
		}

#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)		#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)

void		void
pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force)		pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force)
{		{

if (force) {		if (force) {
sva &= ~(vm_offset_t)cpu_clflush_line_size;		sva &= ~(vm_offset_t)cpu_clflush_line_size;
▲ Show 20 Lines • Show All 1,618 Lines • ▼ Show 20 Lines	if ((oldpde & PG_A) == 0 \|\| (mpte = pmap_remove_pt_page(pmap, va)) ==
*/		*/
if ((oldpde & PG_A) == 0 \|\| (mpte = vm_page_alloc(NULL,		if ((oldpde & PG_A) == 0 \|\| (mpte = vm_page_alloc(NULL,
pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va <		pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va <
DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) \|		DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) \|
VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED)) == NULL) {		VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED)) == NULL) {
SLIST_INIT(&free);		SLIST_INIT(&free);
sva = trunc_2mpage(va);		sva = trunc_2mpage(va);
pmap_remove_pde(pmap, pde, sva, &free, lockp);		pmap_remove_pde(pmap, pde, sva, &free, lockp);
pmap_invalidate_range(pmap, sva, sva + NBPDR - 1);		if ((oldpde & PG_G) == 0)
		pmap_invalidate_pde_page(pmap, sva, oldpde);
pmap_free_zero_pages(&free);		pmap_free_zero_pages(&free);
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"		CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"
" in pmap %p", va, pmap);		" in pmap %p", va, pmap);
return (FALSE);		return (FALSE);
}		}
if (va < VM_MAXUSER_ADDRESS)		if (va < VM_MAXUSER_ADDRESS)
pmap_resident_count_inc(pmap, 1);		pmap_resident_count_inc(pmap, 1);
}		}
▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Lines	pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
PG_RW = pmap_rw_bit(pmap);		PG_RW = pmap_rw_bit(pmap);

PMAP_LOCK_ASSERT(pmap, MA_OWNED);		PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT((sva & PDRMASK) == 0,		KASSERT((sva & PDRMASK) == 0,
("pmap_remove_pde: sva is not 2mpage aligned"));		("pmap_remove_pde: sva is not 2mpage aligned"));
oldpde = pte_load_clear(pdq);		oldpde = pte_load_clear(pdq);
if (oldpde & PG_W)		if (oldpde & PG_W)
pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;		pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
		if ((oldpde & PG_G) != 0)
/*		pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
* When workaround_erratum383 is false, a promotion to a 2M
* page mapping does not invalidate the 512 4K page mappings
* from the TLB. Consequently, at this point, the TLB may
* hold both 4K and 2M page mappings. Therefore, the entire
* range of addresses must be invalidated here. In contrast,
* when workaround_erratum383 is true, a promotion does
* invalidate the 512 4K page mappings, and so a single INVLPG
* suffices to invalidate the 2M page mapping.
*/
if ((oldpde & PG_G) != 0) {
if (workaround_erratum383)
pmap_invalidate_page(kernel_pmap, sva);
else
pmap_invalidate_range(kernel_pmap, sva,
sva + NBPDR - 1);
}

pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);		pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
if (oldpde & PG_MANAGED) {		if (oldpde & PG_MANAGED) {
CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);		CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);
pvh = pa_to_pvh(oldpde & PG_PS_FRAME);		pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
pmap_pvh_free(pvh, pmap, sva);		pmap_pvh_free(pvh, pmap, sva);
eva = sva + NBPDR;		eva = sva + NBPDR;
for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);		for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
va < eva; va += PAGE_SIZE, m++) {		va < eva; va += PAGE_SIZE, m++) {
▲ Show 20 Lines • Show All 363 Lines • ▼ Show 20 Lines	for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
va < eva; va += PAGE_SIZE, m++)		va < eva; va += PAGE_SIZE, m++)
vm_page_dirty(m);		vm_page_dirty(m);
}		}
if ((prot & VM_PROT_WRITE) == 0)		if ((prot & VM_PROT_WRITE) == 0)
newpde &= ~(PG_RW \| PG_M);		newpde &= ~(PG_RW \| PG_M);
if ((prot & VM_PROT_EXECUTE) == 0)		if ((prot & VM_PROT_EXECUTE) == 0)
newpde \|= pg_nx;		newpde \|= pg_nx;
if (newpde != oldpde) {		if (newpde != oldpde) {
if (!atomic_cmpset_long(pde, oldpde, newpde))		/*
		* As an optimization to future operations on this PDE, clear
		* PG_PROMOTED. The impending invalidation will remove any
		* lingering 4KB page mappings from the TLB.
		*/
		if (!atomic_cmpset_long(pde, oldpde, newpde & ~PG_PROMOTED))
goto retry;		goto retry;
if (oldpde & PG_G) {		if ((oldpde & PG_G) != 0)
/* See pmap_remove_pde() for explanation. */		pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
if (workaround_erratum383)
pmap_invalidate_page(kernel_pmap, sva);
else		else
pmap_invalidate_range(kernel_pmap, sva,
sva + NBPDR - 1);
} else
anychanged = TRUE;		anychanged = TRUE;
}		}
return (anychanged);		return (anychanged);
}		}

/*		/*
* Set the physical protection on the		* Set the physical protection on the
* specified range of this map as requested.		* specified range of this map as requested.
▲ Show 20 Lines • Show All 236 Lines • ▼ Show 20 Lines	setpte:
newpde = pmap_swap_pat(pmap, newpde);		newpde = pmap_swap_pat(pmap, newpde);

/*		/*
* Map the superpage.		* Map the superpage.
*/		*/
if (workaround_erratum383)		if (workaround_erratum383)
pmap_update_pde(pmap, va, pde, PG_PS \| newpde);		pmap_update_pde(pmap, va, pde, PG_PS \| newpde);
else		else
pde_store(pde, PG_PS \| newpde);		pde_store(pde, PG_PROMOTED \| PG_PS \| newpde);

atomic_add_long(&pmap_pde_promotions, 1);		atomic_add_long(&pmap_pde_promotions, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"		CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
" in pmap %p", va, pmap);		" in pmap %p", va, pmap);
}		}

/*		/*
* Insert the given physical page (p) at		* Insert the given physical page (p) at
▲ Show 20 Lines • Show All 296 Lines • ▼ Show 20 Lines	if (va < VM_MAXUSER_ADDRESS)
newpde \|= PG_U;		newpde \|= PG_U;

/*		/*
* Increment counters.		* Increment counters.
*/		*/
pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);		pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);

/*		/*
* Map the superpage.		* Map the superpage. (This is not a promoted mapping; there will not
		* be any lingering 4KB page mappings in the TLB.)
*/		*/
pde_store(pde, newpde);		pde_store(pde, newpde);

atomic_add_long(&pmap_pde_mappings, 1);		atomic_add_long(&pmap_pde_mappings, 1);
CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"		CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
" in pmap %p", va, pmap);		" in pmap %p", va, pmap);
return (TRUE);		return (TRUE);
}		}
▲ Show 20 Lines • Show All 2,703 Lines • Show Last 20 Lines