Differential D16736 Diff 48237 head/sys/i386/i386/pmap.c

Changeset View

Standalone View

head/sys/i386/i386/pmap.c

Show First 20 Lines • Show All 142 Lines • ▼ Show 20 Lines
#include <vm/vm_reserv.h>		#include <vm/vm_reserv.h>
#include <vm/uma.h>		#include <vm/uma.h>

#ifdef DEV_APIC		#ifdef DEV_APIC
#include <sys/bus.h>		#include <sys/bus.h>
#include <machine/intr_machdep.h>		#include <machine/intr_machdep.h>
#include <x86/apicvar.h>		#include <x86/apicvar.h>
#endif		#endif
		#include <x86/ifunc.h>
#include <machine/bootinfo.h>		#include <machine/bootinfo.h>
#include <machine/cpu.h>		#include <machine/cpu.h>
#include <machine/cputypes.h>		#include <machine/cputypes.h>
#include <machine/md_var.h>		#include <machine/md_var.h>
#include <machine/pcb.h>		#include <machine/pcb.h>
#include <machine/specialreg.h>		#include <machine/specialreg.h>
#ifdef SMP		#ifdef SMP
#include <machine/smp.h>		#include <machine/smp.h>
▲ Show 20 Lines • Show All 150 Lines • ▼ Show 20 Lines
static bool pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,		static bool pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot);		vm_prot_t prot);
static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,		static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,
u_int flags, vm_page_t m);		u_int flags, vm_page_t m);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,		static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte);		vm_page_t m, vm_prot_t prot, vm_page_t mpte);
static void pmap_flush_page(vm_page_t m);		static void pmap_flush_page(vm_page_t m);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);		static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
		static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva,
		vm_offset_t eva);
		static void pmap_invalidate_cache_range_all(vm_offset_t sva,
		vm_offset_t eva);
static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,		static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
pd_entry_t pde);		pd_entry_t pde);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);		static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static boolean_t pmap_is_modified_pvh(struct md_page *pvh);		static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);		static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);		static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);		static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);		static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
▲ Show 20 Lines • Show All 1,077 Lines • ▼ Show 20 Lines	pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
* 2- or 4MB page mapping from the TLB.		* 2- or 4MB page mapping from the TLB.
*/		*/
if ((pde & PG_PROMOTED) != 0)		if ((pde & PG_PROMOTED) != 0)
pmap_invalidate_range(pmap, va, va + NBPDR - 1);		pmap_invalidate_range(pmap, va, va + NBPDR - 1);
else		else
pmap_invalidate_page(pmap, va);		pmap_invalidate_page(pmap, va);
}		}

		DEFINE_IFUNC(, void, pmap_invalidate_cache_range, (vm_offset_t, vm_offset_t),
		static)
		{

		if ((cpu_feature & CPUID_SS) != 0)
		return (pmap_invalidate_cache_range_selfsnoop);
		if ((cpu_feature & CPUID_CLFSH) != 0)
		return (pmap_force_invalidate_cache_range);
		return (pmap_invalidate_cache_range_all);
		}

#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)		#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)

void		static void
pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force)		pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva)
{		{

if (force) {
sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
} else {
KASSERT((sva & PAGE_MASK) == 0,		KASSERT((sva & PAGE_MASK) == 0,
("pmap_invalidate_cache_range: sva not page-aligned"));		("pmap_invalidate_cache_range: sva not page-aligned"));
KASSERT((eva & PAGE_MASK) == 0,		KASSERT((eva & PAGE_MASK) == 0,
("pmap_invalidate_cache_range: eva not page-aligned"));		("pmap_invalidate_cache_range: eva not page-aligned"));
}		}

if ((cpu_feature & CPUID_SS) != 0 && !force)		static void
; /* If "Self Snoop" is supported and allowed, do nothing. */		pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva)
else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0 &&		{
eva - sva < PMAP_CLFLUSH_THRESHOLD) {
#ifdef DEV_APIC		pmap_invalidate_cache_range_check_align(sva, eva);
		}

		void
		pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
		{

		sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
		if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) {
/*		/*
		* The supplied range is bigger than 2MB.
		* Globally invalidate cache.
		*/
		pmap_invalidate_cache();
		return;
		}

		/*
* XXX: Some CPUs fault, hang, or trash the local APIC		* XXX: Some CPUs fault, hang, or trash the local APIC
* registers if we use CLFLUSH on the local APIC		* registers if we use CLFLUSH on the local APIC
* range. The local APIC is always uncached, so we		* range. The local APIC is always uncached, so we
* don't need to flush for that range anyway.		* don't need to flush for that range anyway.
*/		*/
if (pmap_kextract(sva) == lapic_paddr)		if (pmap_kextract(sva) == lapic_paddr)
return;		return;
#endif
		if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) {
/*		/*
* Otherwise, do per-cache line flush. Use the sfence		* Do per-cache line flush. Use the sfence
* instruction to insure that previous stores are		* instruction to insure that previous stores are
* included in the write-back. The processor		* included in the write-back. The processor
* propagates flush to other processors in the cache		* propagates flush to other processors in the cache
* coherence domain.		* coherence domain.
*/		*/
sfence();		sfence();
for (; sva < eva; sva += cpu_clflush_line_size)		for (; sva < eva; sva += cpu_clflush_line_size)
clflushopt(sva);		clflushopt(sva);
sfence();		sfence();
} else if ((cpu_feature & CPUID_CLFSH) != 0 &&		} else {
eva - sva < PMAP_CLFLUSH_THRESHOLD) {
#ifdef DEV_APIC
if (pmap_kextract(sva) == lapic_paddr)
return;
#endif
/*		/*
* Writes are ordered by CLFLUSH on Intel CPUs.		* Writes are ordered by CLFLUSH on Intel CPUs.
*/		*/
if (cpu_vendor_id != CPU_VENDOR_INTEL)		if (cpu_vendor_id != CPU_VENDOR_INTEL)
mfence();		mfence();
for (; sva < eva; sva += cpu_clflush_line_size)		for (; sva < eva; sva += cpu_clflush_line_size)
clflush(sva);		clflush(sva);
if (cpu_vendor_id != CPU_VENDOR_INTEL)		if (cpu_vendor_id != CPU_VENDOR_INTEL)
mfence();		mfence();
} else {		}
		}

/*		static void
* No targeted cache flush methods are supported by CPU,		pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva)
* or the supplied range is bigger than 2MB.		{
* Globally invalidate cache.
*/		pmap_invalidate_cache_range_check_align(sva, eva);
pmap_invalidate_cache();		pmap_invalidate_cache();
}		}
}

void		void
pmap_invalidate_cache_pages(vm_page_t *pages, int count)		pmap_invalidate_cache_pages(vm_page_t *pages, int count)
{		{
int i;		int i;

if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE \|\|		if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE \|\|
(cpu_feature & CPUID_CLFSH) == 0) {		(cpu_feature & CPUID_CLFSH) == 0) {
▲ Show 20 Lines • Show All 3,991 Lines • ▼ Show 20 Lines	else if (!pmap_initialized) {
}		}
va = kva_alloc(size);		va = kva_alloc(size);
if (va == 0)		if (va == 0)
panic("%s: Couldn't allocate KVA", __func__);		panic("%s: Couldn't allocate KVA", __func__);
}		}
for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)		for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);		pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
pmap_invalidate_range(kernel_pmap, va, va + tmpsize);		pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
pmap_invalidate_cache_range(va, va + size, FALSE);		pmap_invalidate_cache_range(va, va + size);
return ((void *)(va + offset));		return ((void *)(va + offset));
}		}

void *		void *
pmap_mapdev(vm_paddr_t pa, vm_size_t size)		pmap_mapdev(vm_paddr_t pa, vm_size_t size)
{		{

return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));		return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
▲ Show 20 Lines • Show All 222 Lines • ▼ Show 20 Lines	pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
}		}

/*		/*
* Flush CPU caches to make sure any data isn't cached that		* Flush CPU caches to make sure any data isn't cached that
* shouldn't be, etc.		* shouldn't be, etc.
*/		*/
if (changed) {		if (changed) {
pmap_invalidate_range(kernel_pmap, base, tmpva);		pmap_invalidate_range(kernel_pmap, base, tmpva);
pmap_invalidate_cache_range(base, tmpva, FALSE);		pmap_invalidate_cache_range(base, tmpva);
}		}
return (0);		return (0);
}		}

/*		/*
* perform the pmap work for mincore		* perform the pmap work for mincore
*/		*/
int		int
▲ Show 20 Lines • Show All 290 Lines • Show Last 20 Lines