Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 392 Lines • ▼ Show 20 Lines | |||||
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | ||||
"VM/pmap parameters"); | "VM/pmap parameters"); | ||||
static int pg_ps_enabled = 1; | static int pg_ps_enabled = 1; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | ||||
&pg_ps_enabled, 0, "Are large page mappings enabled?"); | &pg_ps_enabled, 0, "Are large page mappings enabled?"); | ||||
int __read_frequently la57 = 0; | |||||
SYSCTL_INT(_vm_pmap, OID_AUTO, la57, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | |||||
&la57, 0, | |||||
"5-level paging for host is enabled"); | |||||
static bool | |||||
pmap_is_la57(pmap_t pmap) | |||||
{ | |||||
if (pmap->pm_type == PT_X86) | |||||
return (la57); | |||||
return (false); /* XXXKIB handle EPT */ | |||||
} | |||||
#define PAT_INDEX_SIZE 8 | #define PAT_INDEX_SIZE 8 | ||||
static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */ | static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */ | ||||
static u_int64_t KPTphys; /* phys addr of kernel level 1 */ | static u_int64_t KPTphys; /* phys addr of kernel level 1 */ | ||||
static u_int64_t KPDphys; /* phys addr of kernel level 2 */ | static u_int64_t KPDphys; /* phys addr of kernel level 2 */ | ||||
u_int64_t KPDPphys; /* phys addr of kernel level 3 */ | u_int64_t KPDPphys; /* phys addr of kernel level 3 */ | ||||
u_int64_t KPML4phys; /* phys addr of kernel level 4 */ | u_int64_t KPML4phys; /* phys addr of kernel level 4 */ | ||||
u_int64_t KPML5phys; /* phys addr of kernel level 5, | |||||
if supported */ | |||||
static pml4_entry_t *kernel_pml4; | |||||
static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ | static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ | ||||
static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ | static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ | ||||
static int ndmpdpphys; /* number of DMPDPphys pages */ | static int ndmpdpphys; /* number of DMPDPphys pages */ | ||||
static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ | static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ | ||||
/* | /* | ||||
* pmap_mapdev support pre initialization (i.e. console) | * pmap_mapdev support pre initialization (i.e. console) | ||||
▲ Show 20 Lines • Show All 835 Lines • ▼ Show 20 Lines | static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | ||||
vm_page_t m, struct rwlock **lockp); | vm_page_t m, struct rwlock **lockp); | ||||
static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | ||||
pd_entry_t newpde); | pd_entry_t newpde); | ||||
static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); | static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); | ||||
static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, | static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, | ||||
struct rwlock **lockp); | struct rwlock **lockp, vm_offset_t va); | ||||
static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, | static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, | static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
struct spglist *free); | struct spglist *free); | ||||
static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); | static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); | ||||
/********************/ | /********************/ | ||||
/* Inline functions */ | /* Inline functions */ | ||||
/********************/ | /********************/ | ||||
/* Return a non-clipped PD index for a given VA */ | /* | ||||
* Return a non-clipped indexes for a given VA, which are page table | |||||
* pages indexes at the corresponding level. | |||||
*/ | |||||
static __inline vm_pindex_t | static __inline vm_pindex_t | ||||
pmap_pde_pindex(vm_offset_t va) | pmap_pde_pindex(vm_offset_t va) | ||||
{ | { | ||||
return (va >> PDRSHIFT); | return (va >> PDRSHIFT); | ||||
} | } | ||||
static __inline vm_pindex_t | |||||
pmap_pdpe_pindex(vm_offset_t va) | |||||
{ | |||||
return (NUPDE + (va >> PDPSHIFT)); | |||||
} | |||||
static __inline vm_pindex_t | |||||
pmap_pml4e_pindex(vm_offset_t va) | |||||
{ | |||||
return (NUPDE + NUPDPE + (va >> PML4SHIFT)); | |||||
} | |||||
static __inline vm_pindex_t | |||||
pmap_pml5e_pindex(vm_offset_t va) | |||||
{ | |||||
return (NUPDE + NUPDPE + NUPML4E + (va >> PML5SHIFT)); | |||||
} | |||||
static __inline pml4_entry_t * | |||||
pmap_pml5e(pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
MPASS(pmap_is_la57(pmap)); | |||||
return (&pmap->pm_pmltop[pmap_pml5e_index(va)]); | |||||
} | |||||
static __inline pml4_entry_t * | |||||
pmap_pml5e_u(pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
MPASS(pmap_is_la57(pmap)); | |||||
return (&pmap->pm_pmltopu[pmap_pml5e_index(va)]); | |||||
} | |||||
static __inline pml4_entry_t * | |||||
pmap_pml5e_to_pml4e(pml5_entry_t *pml5e, vm_offset_t va) | |||||
{ | |||||
pml4_entry_t *pml4e; | |||||
/* XXX MPASS(pmap_is_la57(pmap); */ | |||||
pml4e = (pml4_entry_t *)PHYS_TO_DMAP(*pml5e & PG_FRAME); | |||||
return (&pml4e[pmap_pml4e_index(va)]); | |||||
} | |||||
/* Return a pointer to the PML4 slot that corresponds to a VA */ | /* Return a pointer to the PML4 slot that corresponds to a VA */ | ||||
static __inline pml4_entry_t * | static __inline pml4_entry_t * | ||||
pmap_pml4e(pmap_t pmap, vm_offset_t va) | pmap_pml4e(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
pml5_entry_t *pml5e; | |||||
pml4_entry_t *pml4e; | |||||
pt_entry_t PG_V; | |||||
return (&pmap->pm_pml4[pmap_pml4e_index(va)]); | if (pmap_is_la57(pmap)) { | ||||
pml5e = pmap_pml5e(pmap, va); | |||||
PG_V = pmap_valid_bit(pmap); | |||||
if ((*pml5e & PG_V) == 0) | |||||
return (NULL); | |||||
pml4e = (pml4_entry_t *)PHYS_TO_DMAP(*pml5e & PG_FRAME); | |||||
} else { | |||||
pml4e = pmap->pm_pmltop; | |||||
} | } | ||||
return (&pml4e[pmap_pml4e_index(va)]); | |||||
} | |||||
static __inline pml4_entry_t * | |||||
pmap_pml4e_u(pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
MPASS(!pmap_is_la57(pmap)); | |||||
return (&pmap->pm_pmltopu[pmap_pml4e_index(va)]); | |||||
} | |||||
/* Return a pointer to the PDP slot that corresponds to a VA */ | /* Return a pointer to the PDP slot that corresponds to a VA */ | ||||
static __inline pdp_entry_t * | static __inline pdp_entry_t * | ||||
pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va) | pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va) | ||||
{ | { | ||||
pdp_entry_t *pdpe; | pdp_entry_t *pdpe; | ||||
pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME); | pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME); | ||||
return (&pdpe[pmap_pdpe_index(va)]); | return (&pdpe[pmap_pdpe_index(va)]); | ||||
} | } | ||||
/* Return a pointer to the PDP slot that corresponds to a VA */ | /* Return a pointer to the PDP slot that corresponds to a VA */ | ||||
static __inline pdp_entry_t * | static __inline pdp_entry_t * | ||||
pmap_pdpe(pmap_t pmap, vm_offset_t va) | pmap_pdpe(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
pml4_entry_t *pml4e; | pml4_entry_t *pml4e; | ||||
pt_entry_t PG_V; | pt_entry_t PG_V; | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
pml4e = pmap_pml4e(pmap, va); | pml4e = pmap_pml4e(pmap, va); | ||||
if ((*pml4e & PG_V) == 0) | if (pml4e == NULL || (*pml4e & PG_V) == 0) | ||||
return (NULL); | return (NULL); | ||||
return (pmap_pml4e_to_pdpe(pml4e, va)); | return (pmap_pml4e_to_pdpe(pml4e, va)); | ||||
} | } | ||||
/* Return a pointer to the PD slot that corresponds to a VA */ | /* Return a pointer to the PD slot that corresponds to a VA */ | ||||
static __inline pd_entry_t * | static __inline pd_entry_t * | ||||
pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va) | pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | KASSERT(pmap->pm_stats.resident_count >= count, | ||||
("pmap %p resident count underflow %ld %d", pmap, | ("pmap %p resident count underflow %ld %d", pmap, | ||||
pmap->pm_stats.resident_count, count)); | pmap->pm_stats.resident_count, count)); | ||||
pmap->pm_stats.resident_count -= count; | pmap->pm_stats.resident_count -= count; | ||||
} | } | ||||
PMAP_INLINE pt_entry_t * | PMAP_INLINE pt_entry_t * | ||||
vtopte(vm_offset_t va) | vtopte(vm_offset_t va) | ||||
{ | { | ||||
u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1); | u_int64_t mask; | ||||
KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopte on a uva/gpa 0x%0lx", va)); | KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopte on a uva/gpa 0x%0lx", va)); | ||||
return (PTmap + ((va >> PAGE_SHIFT) & mask)); | if (la57) { | ||||
mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + | |||||
NPML4EPGSHIFT + NPML5EPGSHIFT)) - 1); | |||||
return (P5Tmap + ((va >> PAGE_SHIFT) & mask)); | |||||
} else { | |||||
mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + | |||||
NPML4EPGSHIFT)) - 1); | |||||
return (P4Tmap + ((va >> PAGE_SHIFT) & mask)); | |||||
} | } | ||||
} | |||||
static __inline pd_entry_t * | static __inline pd_entry_t * | ||||
vtopde(vm_offset_t va) | vtopde(vm_offset_t va) | ||||
{ | { | ||||
u_int64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1); | u_int64_t mask; | ||||
KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopde on a uva/gpa 0x%0lx", va)); | KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopde on a uva/gpa 0x%0lx", va)); | ||||
return (PDmap + ((va >> PDRSHIFT) & mask)); | if (la57) { | ||||
mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + | |||||
NPML4EPGSHIFT + NPML5EPGSHIFT)) - 1); | |||||
return (P5Dmap + ((va >> PDRSHIFT) & mask)); | |||||
} else { | |||||
mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + | |||||
NPML4EPGSHIFT)) - 1); | |||||
return (P4Dmap + ((va >> PDRSHIFT) & mask)); | |||||
} | } | ||||
} | |||||
static u_int64_t | static u_int64_t | ||||
allocpages(vm_paddr_t *firstaddr, int n) | allocpages(vm_paddr_t *firstaddr, int n) | ||||
{ | { | ||||
u_int64_t ret; | u_int64_t ret; | ||||
ret = *firstaddr; | ret = *firstaddr; | ||||
bzero((void *)ret, n * PAGE_SIZE); | bzero((void *)ret, n * PAGE_SIZE); | ||||
▲ Show 20 Lines • Show All 239 Lines • ▼ Show 20 Lines | for (i = 0; i < ndmpdpphys; i++) { | ||||
p4_p[DMPML4I + i] |= X86_PG_RW | X86_PG_V | pg_nx; | p4_p[DMPML4I + i] |= X86_PG_RW | X86_PG_V | pg_nx; | ||||
} | } | ||||
/* Connect the KVA slots up to the PML4 */ | /* Connect the KVA slots up to the PML4 */ | ||||
for (i = 0; i < NKPML4E; i++) { | for (i = 0; i < NKPML4E; i++) { | ||||
p4_p[KPML4BASE + i] = KPDPphys + ptoa(i); | p4_p[KPML4BASE + i] = KPDPphys + ptoa(i); | ||||
p4_p[KPML4BASE + i] |= X86_PG_RW | X86_PG_V; | p4_p[KPML4BASE + i] |= X86_PG_RW | X86_PG_V; | ||||
} | } | ||||
kernel_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); | |||||
} | } | ||||
/* | /* | ||||
* Bootstrap the system enough to run with virtual memory. | * Bootstrap the system enough to run with virtual memory. | ||||
* | * | ||||
* On amd64 this is called after mapping has already been enabled | * On amd64 this is called after mapping has already been enabled | ||||
* and just syncs the pmap module with what has already been done. | * and just syncs the pmap module with what has already been done. | ||||
* [We can't call it easily with mapping off since the kernel is not | * [We can't call it easily with mapping off since the kernel is not | ||||
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines | pmap_bootstrap(vm_paddr_t *firstaddr) | ||||
load_cr4(cr4); | load_cr4(cr4); | ||||
/* | /* | ||||
* Initialize the kernel pmap (which is statically allocated). | * Initialize the kernel pmap (which is statically allocated). | ||||
* Count bootstrap data as being resident in case any of this data is | * Count bootstrap data as being resident in case any of this data is | ||||
* later unmapped (using pmap_remove()) and freed. | * later unmapped (using pmap_remove()) and freed. | ||||
*/ | */ | ||||
PMAP_LOCK_INIT(kernel_pmap); | PMAP_LOCK_INIT(kernel_pmap); | ||||
kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); | kernel_pmap->pm_pmltop = kernel_pml4; | ||||
kernel_pmap->pm_cr3 = KPML4phys; | kernel_pmap->pm_cr3 = KPML4phys; | ||||
kernel_pmap->pm_ucr3 = PMAP_NO_CR3; | kernel_pmap->pm_ucr3 = PMAP_NO_CR3; | ||||
CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ | CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ | ||||
TAILQ_INIT(&kernel_pmap->pm_pvchunk); | TAILQ_INIT(&kernel_pmap->pm_pvchunk); | ||||
kernel_pmap->pm_stats.resident_count = res; | kernel_pmap->pm_stats.resident_count = res; | ||||
kernel_pmap->pm_flags = pmap_flags; | kernel_pmap->pm_flags = pmap_flags; | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 144 Lines • ▼ Show 20 Lines | pmap_init_pat(void) | ||||
wbinvd(); | wbinvd(); | ||||
invltlb(); | invltlb(); | ||||
/* Restore caches and PGE. */ | /* Restore caches and PGE. */ | ||||
load_cr0(cr0); | load_cr0(cr0); | ||||
load_cr4(cr4); | load_cr4(cr4); | ||||
} | } | ||||
extern const char la57_trampoline[], la57_trampoline_gdt_desc[], | |||||
la57_trampoline_gdt[], la57_trampoline_end[]; | |||||
static void | |||||
pmap_bootstrap_la57(void *arg __unused) | |||||
{ | |||||
char *v_code; | |||||
pml5_entry_t *v_pml5; | |||||
pml4_entry_t *v_pml4; | |||||
pdp_entry_t *v_pdp; | |||||
pd_entry_t *v_pd; | |||||
pt_entry_t *v_pt; | |||||
vm_page_t m_code, m_pml4, m_pdp, m_pd, m_pt, m_pml5; | |||||
void (*la57_tramp)(uint64_t pml5); | |||||
struct region_descriptor r_gdt; | |||||
if ((cpu_stdext_feature2 & CPUID_STDEXT2_LA57) == 0) | |||||
return; | |||||
if (!TUNABLE_INT_FETCH("vm.pmap.la57", &la57)) | |||||
la57 = 1; | |||||
if (!la57) | |||||
return; | |||||
r_gdt.rd_limit = NGDT * sizeof(struct user_segment_descriptor) - 1; | |||||
r_gdt.rd_base = (long)__pcpu[0].pc_gdt; | |||||
m_code = vm_page_alloc_contig(NULL, 0, | |||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | |||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | |||||
if ((m_code->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_code); | |||||
v_code = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_code)); | |||||
m_pml5 = vm_page_alloc_contig(NULL, 0, | |||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | |||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | |||||
if ((m_pml5->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pml5); | |||||
KPML5phys = VM_PAGE_TO_PHYS(m_pml5); | |||||
v_pml5 = (pml5_entry_t *)PHYS_TO_DMAP(KPML5phys); | |||||
m_pml4 = vm_page_alloc_contig(NULL, 0, | |||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | |||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | |||||
if ((m_pml4->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pml4); | |||||
v_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pml4)); | |||||
m_pdp = vm_page_alloc_contig(NULL, 0, | |||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | |||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | |||||
if ((m_pdp->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pdp); | |||||
v_pdp = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pdp)); | |||||
m_pd = vm_page_alloc_contig(NULL, 0, | |||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | |||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | |||||
if ((m_pd->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pd); | |||||
v_pd = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pd)); | |||||
m_pt = vm_page_alloc_contig(NULL, 0, | |||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | |||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | |||||
if ((m_pt->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pt); | |||||
v_pt = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pt)); | |||||
/* | /* | ||||
* Map m_code 1:1, it appears below 4G in KVA due to physical | |||||
* address being below 4G. Since kernel KVA is in upper half, | |||||
* the pml4e should be zero and free for temporal use. | |||||
alc: "temporal" -> "temporary" | |||||
*/ | |||||
kernel_pmap->pm_pmltop[pmap_pml4e_index(VM_PAGE_TO_PHYS(m_code))] = | |||||
VM_PAGE_TO_PHYS(m_pdp) | X86_PG_V | X86_PG_RW | X86_PG_A | | |||||
X86_PG_M; | |||||
v_pdp[pmap_pdpe_index(VM_PAGE_TO_PHYS(m_code))] = | |||||
VM_PAGE_TO_PHYS(m_pd) | X86_PG_V | X86_PG_RW | X86_PG_A | | |||||
X86_PG_M; | |||||
v_pd[pmap_pde_index(VM_PAGE_TO_PHYS(m_code))] = | |||||
VM_PAGE_TO_PHYS(m_pt) | X86_PG_V | X86_PG_RW | X86_PG_A | | |||||
X86_PG_M; | |||||
v_pt[pmap_pte_index(VM_PAGE_TO_PHYS(m_code))] = | |||||
VM_PAGE_TO_PHYS(m_code) | X86_PG_V | X86_PG_RW | X86_PG_A | | |||||
X86_PG_M; | |||||
/* | |||||
* Add pml5 entry at top of KVA pointing to existing pml4 table, | |||||
* entering all existing kernel mapping into level 5 table. | |||||
alcUnsubmitted Done Inline Actions"mapping" -> "mappings" alc: "mapping" -> "mappings" | |||||
*/ | |||||
v_pml5[pmap_pml5e_index(UPT_MAX_ADDRESS)] = KPML4phys | X86_PG_V | | |||||
X86_PG_RW | X86_PG_A | X86_PG_M | pg_g; | |||||
/* | |||||
* Add pml5 entry for 1:1 trampoline mapping after LA57 is turned on. | |||||
*/ | |||||
v_pml5[pmap_pml5e_index(VM_PAGE_TO_PHYS(m_code))] = | |||||
VM_PAGE_TO_PHYS(m_pml4) | X86_PG_V | X86_PG_RW | X86_PG_A | | |||||
X86_PG_M; | |||||
v_pml4[pmap_pml4e_index(VM_PAGE_TO_PHYS(m_code))] = | |||||
VM_PAGE_TO_PHYS(m_pdp) | X86_PG_V | X86_PG_RW | X86_PG_A | | |||||
X86_PG_M; | |||||
/* | |||||
* Copy and call the 48->57 trampoline, hope we return there, alive. | |||||
*/ | |||||
bcopy(la57_trampoline, v_code, la57_trampoline_end - la57_trampoline); | |||||
*(u_long *)(v_code + 2 + (la57_trampoline_gdt_desc - la57_trampoline)) = | |||||
la57_trampoline_gdt - la57_trampoline + VM_PAGE_TO_PHYS(m_code); | |||||
la57_tramp = (void (*)(uint64_t))VM_PAGE_TO_PHYS(m_code); | |||||
la57_tramp(KPML5phys); | |||||
/* | |||||
* gdt was necessary reset, switch back to our gdt. | |||||
*/ | |||||
lgdt(&r_gdt); | |||||
wrmsr(MSR_GSBASE, (uint64_t)&__pcpu[0]); | |||||
load_ds(_udatasel); | |||||
load_es(_udatasel); | |||||
load_fs(_ufssel); | |||||
ssdtosyssd(&gdt_segs[GPROC0_SEL], | |||||
(struct system_segment_descriptor *)&__pcpu[0].pc_gdt[GPROC0_SEL]); | |||||
ltr(GSEL(GPROC0_SEL, SEL_KPL)); | |||||
/* | |||||
* Now unmap the trampoline, and free the pages. | |||||
* Clear pml5 entry used for 1:1 trampoline mapping. | |||||
*/ | |||||
pte_clear(&v_pml5[pmap_pml5e_index(VM_PAGE_TO_PHYS(m_code))]); | |||||
invlpg((vm_offset_t)v_code); | |||||
vm_page_free(m_code); | |||||
vm_page_free(m_pdp); | |||||
vm_page_free(m_pd); | |||||
vm_page_free(m_pt); | |||||
/* | |||||
* Recursively map PML5 to itself in order to get PTmap and | |||||
* PDmap. | |||||
*/ | |||||
v_pml5[PML5PML5I] = KPML5phys | X86_PG_RW | X86_PG_V | pg_nx; | |||||
kernel_pmap->pm_cr3 = KPML5phys; | |||||
kernel_pmap->pm_pmltop = v_pml5; | |||||
} | |||||
SYSINIT(la57, SI_SUB_KMEM, SI_ORDER_ANY, pmap_bootstrap_la57, NULL); | |||||
/* | |||||
* Initialize a vm_page's machine-dependent fields. | * Initialize a vm_page's machine-dependent fields. | ||||
*/ | */ | ||||
void | void | ||||
pmap_page_init(vm_page_t m) | pmap_page_init(vm_page_t m) | ||||
{ | { | ||||
TAILQ_INIT(&m->md.pv_list); | TAILQ_INIT(&m->md.pv_list); | ||||
m->md.pat_mode = PAT_WRITE_BACK; | m->md.pat_mode = PAT_WRITE_BACK; | ||||
▲ Show 20 Lines • Show All 282 Lines • ▼ Show 20 Lines | if (lm_ents != 0) { | ||||
large_vmem = vmem_create("large", LARGEMAP_MIN_ADDRESS, | large_vmem = vmem_create("large", LARGEMAP_MIN_ADDRESS, | ||||
(vmem_size_t)lm_ents * NBPML4, PAGE_SIZE, 0, M_WAITOK); | (vmem_size_t)lm_ents * NBPML4, PAGE_SIZE, 0, M_WAITOK); | ||||
if (large_vmem == NULL) { | if (large_vmem == NULL) { | ||||
printf("pmap: cannot create large map\n"); | printf("pmap: cannot create large map\n"); | ||||
lm_ents = 0; | lm_ents = 0; | ||||
} | } | ||||
for (i = 0; i < lm_ents; i++) { | for (i = 0; i < lm_ents; i++) { | ||||
m = pmap_large_map_getptp_unlocked(); | m = pmap_large_map_getptp_unlocked(); | ||||
kernel_pmap->pm_pml4[LMSPML4I + i] = X86_PG_V | | /* XXXKIB la57 */ | ||||
kernel_pml4[LMSPML4I + i] = X86_PG_V | | |||||
X86_PG_RW | X86_PG_A | X86_PG_M | pg_nx | | X86_PG_RW | X86_PG_A | X86_PG_M | pg_nx | | ||||
VM_PAGE_TO_PHYS(m); | VM_PAGE_TO_PHYS(m); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
SYSCTL_UINT(_vm_pmap, OID_AUTO, large_map_pml4_entries, | SYSCTL_UINT(_vm_pmap, OID_AUTO, large_map_pml4_entries, | ||||
CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lm_ents, 0, | CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lm_ents, 0, | ||||
▲ Show 20 Lines • Show All 1,359 Lines • ▼ Show 20 Lines | if (m->ref_count == 0) { | ||||
return (TRUE); | return (TRUE); | ||||
} else | } else | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
static void | static void | ||||
_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
{ | { | ||||
pml5_entry_t *pml5; | |||||
pml4_entry_t *pml4; | |||||
pdp_entry_t *pdp; | |||||
pd_entry_t *pd; | |||||
vm_page_t pdpg, pdppg, pml4pg; | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
/* | /* | ||||
* unmap the page table page | * unmap the page table page | ||||
*/ | */ | ||||
if (m->pindex >= NUPDE + NUPDPE) { | if (m->pindex >= NUPDE + NUPDPE + NUPML4E) { | ||||
/* PML4 page */ | |||||
MPASS(pmap_is_la57(pmap)); | |||||
pml5 = pmap_pml5e(pmap, va); | |||||
*pml5 = 0; | |||||
if (pmap->pm_pmltopu != NULL && va <= VM_MAXUSER_ADDRESS) { | |||||
pml5 = pmap_pml5e_u(pmap, va); | |||||
*pml5 = 0; | |||||
} | |||||
} else if (m->pindex >= NUPDE + NUPDPE) { | |||||
/* PDP page */ | /* PDP page */ | ||||
pml4_entry_t *pml4; | |||||
pml4 = pmap_pml4e(pmap, va); | pml4 = pmap_pml4e(pmap, va); | ||||
*pml4 = 0; | *pml4 = 0; | ||||
if (pmap->pm_pml4u != NULL && va <= VM_MAXUSER_ADDRESS) { | if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL && | ||||
pml4 = &pmap->pm_pml4u[pmap_pml4e_index(va)]; | va <= VM_MAXUSER_ADDRESS) { | ||||
pml4 = pmap_pml4e_u(pmap, va); | |||||
*pml4 = 0; | *pml4 = 0; | ||||
} | } | ||||
} else if (m->pindex >= NUPDE) { | } else if (m->pindex >= NUPDE) { | ||||
/* PD page */ | /* PD page */ | ||||
pdp_entry_t *pdp; | |||||
pdp = pmap_pdpe(pmap, va); | pdp = pmap_pdpe(pmap, va); | ||||
*pdp = 0; | *pdp = 0; | ||||
} else { | } else { | ||||
/* PTE page */ | /* PTE page */ | ||||
pd_entry_t *pd; | |||||
pd = pmap_pde(pmap, va); | pd = pmap_pde(pmap, va); | ||||
*pd = 0; | *pd = 0; | ||||
} | } | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
if (m->pindex < NUPDE) { | if (m->pindex < NUPDE) { | ||||
/* We just released a PT, unhold the matching PD */ | /* We just released a PT, unhold the matching PD */ | ||||
vm_page_t pdpg; | |||||
pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME); | pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME); | ||||
pmap_unwire_ptp(pmap, va, pdpg, free); | pmap_unwire_ptp(pmap, va, pdpg, free); | ||||
} else if (m->pindex < NUPDE + NUPDPE) { | } else if (m->pindex < NUPDE + NUPDPE) { | ||||
/* We just released a PD, unhold the matching PDP */ | /* We just released a PD, unhold the matching PDP */ | ||||
vm_page_t pdppg; | |||||
pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME); | pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME); | ||||
pmap_unwire_ptp(pmap, va, pdppg, free); | pmap_unwire_ptp(pmap, va, pdppg, free); | ||||
} else if (m->pindex < NUPDE + NUPDPE + NUPML4E && pmap_is_la57(pmap)) { | |||||
/* We just released a PDP, unhold the matching PML4 */ | |||||
pml4pg = PHYS_TO_VM_PAGE(*pmap_pml5e(pmap, va) & PG_FRAME); | |||||
pmap_unwire_ptp(pmap, va, pml4pg, free); | |||||
} | } | ||||
/* | /* | ||||
* Put page on a list so that it is released after | * Put page on a list so that it is released after | ||||
* *ALL* TLB shootdown is done | * *ALL* TLB shootdown is done | ||||
*/ | */ | ||||
pmap_add_delayed_free_list(m, free, TRUE); | pmap_add_delayed_free_list(m, free, TRUE); | ||||
} | } | ||||
Show All 39 Lines | |||||
void | void | ||||
pmap_pinit0(pmap_t pmap) | pmap_pinit0(pmap_t pmap) | ||||
{ | { | ||||
struct proc *p; | struct proc *p; | ||||
struct thread *td; | struct thread *td; | ||||
int i; | int i; | ||||
PMAP_LOCK_INIT(pmap); | PMAP_LOCK_INIT(pmap); | ||||
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); | pmap->pm_pmltop = kernel_pmap->pm_pmltop; | ||||
pmap->pm_pml4u = NULL; | pmap->pm_pmltopu = NULL; | ||||
pmap->pm_cr3 = KPML4phys; | pmap->pm_cr3 = kernel_pmap->pm_cr3; | ||||
/* hack to keep pmap_pti_pcid_invalidate() alive */ | /* hack to keep pmap_pti_pcid_invalidate() alive */ | ||||
pmap->pm_ucr3 = PMAP_NO_CR3; | pmap->pm_ucr3 = PMAP_NO_CR3; | ||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
TAILQ_INIT(&pmap->pm_pvchunk); | TAILQ_INIT(&pmap->pm_pvchunk); | ||||
bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | ||||
pmap->pm_flags = pmap_flags; | pmap->pm_flags = pmap_flags; | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
Show All 36 Lines | pmap_pinit_pml4(vm_page_t pml4pg) | ||||
} | } | ||||
/* install self-referential address mapping entry(s) */ | /* install self-referential address mapping entry(s) */ | ||||
pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | X86_PG_V | X86_PG_RW | | pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | X86_PG_V | X86_PG_RW | | ||||
X86_PG_A | X86_PG_M; | X86_PG_A | X86_PG_M; | ||||
/* install large map entries if configured */ | /* install large map entries if configured */ | ||||
for (i = 0; i < lm_ents; i++) | for (i = 0; i < lm_ents; i++) | ||||
pm_pml4[LMSPML4I + i] = kernel_pmap->pm_pml4[LMSPML4I + i]; | pm_pml4[LMSPML4I + i] = kernel_pmap->pm_pmltop[LMSPML4I + i]; | ||||
} | } | ||||
void | |||||
pmap_pinit_pml5(vm_page_t pml5pg) | |||||
{ | |||||
pml5_entry_t *pm_pml5; | |||||
pm_pml5 = (pml5_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml5pg)); | |||||
/* | |||||
* Add pml5 entry at top of KVA pointing to existing pml4 table, | |||||
* entering all existing kernel mapping into level 5 table. | |||||
alcUnsubmitted Done Inline Actions"mapping" -> "mappings" alc: "mapping" -> "mappings" | |||||
*/ | |||||
pm_pml5[pmap_pml5e_index(UPT_MAX_ADDRESS)] = KPML4phys | X86_PG_V | | |||||
X86_PG_RW | X86_PG_A | X86_PG_M | pg_g | | |||||
pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, FALSE); | |||||
/* | |||||
* Install self-referential address mapping entry. | |||||
*/ | |||||
pm_pml5[PML5PML5I] = VM_PAGE_TO_PHYS(pml5pg) | | |||||
X86_PG_RW | X86_PG_V | X86_PG_M | X86_PG_A | | |||||
pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, FALSE); | |||||
} | |||||
static void | static void | ||||
pmap_pinit_pml4_pti(vm_page_t pml4pg) | pmap_pinit_pml4_pti(vm_page_t pml4pgu) | ||||
{ | { | ||||
pml4_entry_t *pm_pml4; | pml4_entry_t *pm_pml4u; | ||||
int i; | int i; | ||||
pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg)); | pm_pml4u = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pgu)); | ||||
for (i = 0; i < NPML4EPG; i++) | for (i = 0; i < NPML4EPG; i++) | ||||
pm_pml4[i] = pti_pml4[i]; | pm_pml4u[i] = pti_pml4[i]; | ||||
} | } | ||||
static void | |||||
pmap_pinit_pml5_pti(vm_page_t pml5pgu) | |||||
{ | |||||
pml5_entry_t *pm_pml5u; | |||||
pm_pml5u = (pml5_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml5pgu)); | |||||
/* | /* | ||||
* Add pml5 entry at top of KVA pointing to existing pml4 pti | |||||
* table, entering all kernel mapping needed for usermode, | |||||
alcUnsubmitted Done Inline Actions"mapping" -> "mappings" No comma at the end alc: "mapping" -> "mappings"
No comma at the end | |||||
* into level 5 table. | |||||
*/ | |||||
pm_pml5u[pmap_pml5e_index(UPT_MAX_ADDRESS)] = | |||||
pmap_kextract((vm_offset_t)pti_pml4) | | |||||
X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M | pg_g | | |||||
pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, FALSE); | |||||
} | |||||
/* | |||||
* Initialize a preallocated and zeroed pmap structure, | * Initialize a preallocated and zeroed pmap structure, | ||||
* such as one in a vmspace structure. | * such as one in a vmspace structure. | ||||
*/ | */ | ||||
int | int | ||||
pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) | pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) | ||||
{ | { | ||||
vm_page_t pml4pg, pml4pgu; | vm_page_t pmltop_pg, pmltop_pgu; | ||||
vm_paddr_t pml4phys; | vm_paddr_t pmltop_phys; | ||||
int i; | int i; | ||||
/* | /* | ||||
* allocate the page directory page | * allocate the page directory page | ||||
*/ | */ | ||||
pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | pmltop_pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); | VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); | ||||
pml4phys = VM_PAGE_TO_PHYS(pml4pg); | pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg); | ||||
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(pml4phys); | pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys); | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
} | } | ||||
pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */ | pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */ | ||||
pmap->pm_ucr3 = PMAP_NO_CR3; | pmap->pm_ucr3 = PMAP_NO_CR3; | ||||
pmap->pm_pml4u = NULL; | pmap->pm_pmltopu = NULL; | ||||
pmap->pm_type = pm_type; | pmap->pm_type = pm_type; | ||||
if ((pml4pg->flags & PG_ZERO) == 0) | if ((pmltop_pg->flags & PG_ZERO) == 0) | ||||
pagezero(pmap->pm_pml4); | pagezero(pmap->pm_pmltop); | ||||
/* | /* | ||||
* Do not install the host kernel mappings in the nested page | * Do not install the host kernel mappings in the nested page | ||||
* tables. These mappings are meaningless in the guest physical | * tables. These mappings are meaningless in the guest physical | ||||
* address space. | * address space. | ||||
* Install minimal kernel mappings in PTI case. | * Install minimal kernel mappings in PTI case. | ||||
*/ | */ | ||||
if (pm_type == PT_X86) { | if (pm_type == PT_X86) { | ||||
pmap->pm_cr3 = pml4phys; | pmap->pm_cr3 = pmltop_phys; | ||||
pmap_pinit_pml4(pml4pg); | if (pmap_is_la57(pmap)) | ||||
pmap_pinit_pml5(pmltop_pg); | |||||
else | |||||
pmap_pinit_pml4(pmltop_pg); | |||||
if ((curproc->p_md.md_flags & P_MD_KPTI) != 0) { | if ((curproc->p_md.md_flags & P_MD_KPTI) != 0) { | ||||
pml4pgu = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | pmltop_pgu = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_WAITOK); | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_WAITOK); | ||||
pmap->pm_pml4u = (pml4_entry_t *)PHYS_TO_DMAP( | pmap->pm_pmltopu = (pml4_entry_t *)PHYS_TO_DMAP( | ||||
VM_PAGE_TO_PHYS(pml4pgu)); | VM_PAGE_TO_PHYS(pmltop_pgu)); | ||||
pmap_pinit_pml4_pti(pml4pgu); | if (pmap_is_la57(pmap)) | ||||
pmap->pm_ucr3 = VM_PAGE_TO_PHYS(pml4pgu); | pmap_pinit_pml5_pti(pmltop_pgu); | ||||
else | |||||
pmap_pinit_pml4_pti(pmltop_pgu); | |||||
pmap->pm_ucr3 = VM_PAGE_TO_PHYS(pmltop_pgu); | |||||
} | } | ||||
if ((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) { | if ((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) { | ||||
rangeset_init(&pmap->pm_pkru, pkru_dup_range, | rangeset_init(&pmap->pm_pkru, pkru_dup_range, | ||||
pkru_free_range, pmap, M_NOWAIT); | pkru_free_range, pmap, M_NOWAIT); | ||||
} | } | ||||
} | } | ||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
TAILQ_INIT(&pmap->pm_pvchunk); | TAILQ_INIT(&pmap->pm_pvchunk); | ||||
bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | ||||
pmap->pm_flags = flags; | pmap->pm_flags = flags; | ||||
pmap->pm_eptgen = 0; | pmap->pm_eptgen = 0; | ||||
return (1); | return (1); | ||||
} | } | ||||
int | int | ||||
pmap_pinit(pmap_t pmap) | pmap_pinit(pmap_t pmap) | ||||
{ | { | ||||
return (pmap_pinit_type(pmap, PT_X86, pmap_flags)); | return (pmap_pinit_type(pmap, PT_X86, pmap_flags)); | ||||
} | } | ||||
static pml4_entry_t * | |||||
pmap_allocpte_getpml4(pmap_t pmap, struct rwlock **lockp, vm_offset_t va, | |||||
bool addref) | |||||
{ | |||||
vm_pindex_t pml5index; | |||||
pml5_entry_t *pml5; | |||||
pml4_entry_t *pml4; | |||||
vm_page_t pml4pg; | |||||
pt_entry_t PG_V; | |||||
bool allocated; | |||||
if (!pmap_is_la57(pmap)) | |||||
return (&pmap->pm_pmltop[pmap_pml4e_index(va)]); | |||||
PG_V = pmap_valid_bit(pmap); | |||||
pml5index = pmap_pml5e_index(va); | |||||
pml5 = &pmap->pm_pmltop[pml5index]; | |||||
if ((*pml5 & PG_V) == 0) { | |||||
if (_pmap_allocpte(pmap, pmap_pml5e_pindex(va), lockp, va) == | |||||
NULL) | |||||
return (NULL); | |||||
allocated = true; | |||||
} else { | |||||
allocated = false; | |||||
} | |||||
pml4 = (pml4_entry_t *)PHYS_TO_DMAP(*pml5 & PG_FRAME); | |||||
pml4 = &pml4[pmap_pml4e_index(va)]; | |||||
if ((*pml4 & PG_V) == 0) { | |||||
pml4pg = PHYS_TO_VM_PAGE(*pml5 & PG_FRAME); | |||||
if (allocated && !addref) | |||||
pml4pg->ref_count--; | |||||
else if (!allocated && addref) | |||||
pml4pg->ref_count++; | |||||
} | |||||
return (pml4); | |||||
} | |||||
static pdp_entry_t * | |||||
pmap_allocpte_getpdp(pmap_t pmap, struct rwlock **lockp, vm_offset_t va, | |||||
bool addref) | |||||
{ | |||||
vm_page_t pdppg; | |||||
pml4_entry_t *pml4; | |||||
pdp_entry_t *pdp; | |||||
pt_entry_t PG_V; | |||||
bool allocated; | |||||
PG_V = pmap_valid_bit(pmap); | |||||
pml4 = pmap_allocpte_getpml4(pmap, lockp, va, false); | |||||
if (pml4 == NULL) | |||||
return (NULL); | |||||
if ((*pml4 & PG_V) == 0) { | |||||
/* Have to allocate a new pdp, recurse */ | |||||
if (_pmap_allocpte(pmap, pmap_pml4e_pindex(va), lockp, va) == | |||||
NULL) | |||||
return (NULL); | |||||
allocated = true; | |||||
} else { | |||||
allocated = false; | |||||
} | |||||
pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); | |||||
pdp = &pdp[pmap_pdpe_index(va)]; | |||||
if ((*pdp & PG_V) == 0) { | |||||
pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME); | |||||
if (allocated && !addref) | |||||
pdppg->ref_count--; | |||||
else if (!allocated && addref) | |||||
pdppg->ref_count++; | |||||
} | |||||
return (pdp); | |||||
} | |||||
/* | /* | ||||
* This routine is called if the desired page table page does not exist. | * This routine is called if the desired page table page does not exist. | ||||
* | * | ||||
* If page table page allocation fails, this routine may sleep before | * If page table page allocation fails, this routine may sleep before | ||||
* returning NULL. It sleeps only if a lock pointer was given. | * returning NULL. It sleeps only if a lock pointer was given. | ||||
* | * | ||||
* Note: If a page allocation fails at page table level two or three, | * Note: If a page allocation fails at page table level two, three, or four, | ||||
* one or two pages may be held during the wait, only to be released | * one or two pages may be held during the wait, only to be released | ||||
alcUnsubmitted Done Inline ActionsCouldn't this be up to 3 now? alc: Couldn't this be up to 3 now? | |||||
* afterwards. This conservative approach is easily argued to avoid | * afterwards. This conservative approach is easily argued to avoid | ||||
* race conditions. | * race conditions. | ||||
* | * | ||||
* The ptepindexes, i.e. page indices, of the page table pages encountered | * The ptepindexes, i.e. page indices, of the page table pages encountered | ||||
* while translating virtual address va are defined as follows: | * while translating virtual address va are defined as follows: | ||||
* - for the page table page (last level), | * - for the page table page (last level), | ||||
* ptepindex = pmap_pde_pindex(va) = va >> PDRSHIFT, | * ptepindex = pmap_pde_pindex(va) = va >> PDRSHIFT, | ||||
* in other words, it is just the index of the PDE that maps the page | * in other words, it is just the index of the PDE that maps the page | ||||
* table page. | * table page. | ||||
* - for the page directory page, | * - for the page directory page, | ||||
* ptepindex = NUPDE (number of userland PD entries) + | * ptepindex = NUPDE (number of userland PD entries) + | ||||
* (pmap_pde_index(va) >> NPDEPGSHIFT) | * (pmap_pde_index(va) >> NPDEPGSHIFT) | ||||
* i.e. index of PDPE is put after the last index of PDE, | * i.e. index of PDPE is put after the last index of PDE, | ||||
* - for the page directory pointer page, | * - for the page directory pointer page, | ||||
* ptepindex = NUPDE + NUPDPE + (pmap_pde_index(va) >> (NPDEPGSHIFT + | * ptepindex = NUPDE + NUPDPE + (pmap_pde_index(va) >> (NPDEPGSHIFT + | ||||
* NPML4EPGSHIFT), | * NPML4EPGSHIFT), | ||||
* i.e. index of pml4e is put after the last index of PDPE. | * i.e. index of pml4e is put after the last index of PDPE, | ||||
* - for the PML4 page (if LA57 mode is enabled), | |||||
* ptepindex = NUPDE + NUPDPE + NUPML4E + (pmap_pde_index(va) >> | |||||
* (NPDEPGSHIFT + NPML4EPGSHIFT + NPML5EPGSHIFT), | |||||
* i.e. index of pml5e is put after the last index of PML4E. | |||||
* | * | ||||
* Define an order on the paging entries, where all entries of the | * Define an order on the paging entries, where all entries of the | ||||
* same height are put together, then heights are put from deepest to | * same height are put together, then heights are put from deepest to | ||||
* root. Then ptexpindex is the sequential number of the | * root. Then ptexpindex is the sequential number of the | ||||
* corresponding paging entry in this order. | * corresponding paging entry in this order. | ||||
* | * | ||||
* The root page at PML4 does not participate in this indexing scheme, since | * The values of NUPDE, NUPDPE, and NUPML4E are fixed by the size of | ||||
alcUnsubmitted Done Inline Actions"fixed" -> "determined" alc: "fixed" -> "determined" | |||||
* it is statically allocated by pmap_pinit() and not by _pmap_allocpte(). | * LA57 paging structures even in LA48 paging mode, as well as the | ||||
alcUnsubmitted Done Inline Actions"mode. Moreover, the ..." alc: "mode. Moreover, the ..." | |||||
* ptepindexes are calculated as if the paging structures were 5-level | |||||
* regardless of the actual mode of operation. | |||||
* | |||||
* The root page at PML4/PML5 does not participate in this indexing scheme, | |||||
* since it is statically allocated by pmap_pinit() and not by _pmap_allocpte(). | |||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) | _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp, | ||||
vm_offset_t va __unused) | |||||
{ | { | ||||
vm_page_t m, pdppg, pdpg; | vm_pindex_t pml5index, pml4index; | ||||
pml5_entry_t *pml5, *pml5u; | |||||
pml4_entry_t *pml4, *pml4u; | |||||
pdp_entry_t *pdp; | |||||
pd_entry_t *pd; | |||||
vm_page_t m, pdpg; | |||||
pt_entry_t PG_A, PG_M, PG_RW, PG_V; | pt_entry_t PG_A, PG_M, PG_RW, PG_V; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PG_A = pmap_accessed_bit(pmap); | PG_A = pmap_accessed_bit(pmap); | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
Show All 19 Lines | _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp, | ||||
} | } | ||||
if ((m->flags & PG_ZERO) == 0) | if ((m->flags & PG_ZERO) == 0) | ||||
pmap_zero_page(m); | pmap_zero_page(m); | ||||
/* | /* | ||||
* Map the pagetable page into the process address space, if | * Map the pagetable page into the process address space, if | ||||
* it isn't already there. | * it isn't already there. | ||||
*/ | */ | ||||
if (ptepindex >= NUPDE + NUPDPE + NUPML4E) { | |||||
MPASS(pmap_is_la57(pmap)); | |||||
if (ptepindex >= (NUPDE + NUPDPE)) { | pml5index = pmap_pml5e_index(va); | ||||
pml4_entry_t *pml4, *pml4u; | pml5 = &pmap->pm_pmltop[pml5index]; | ||||
vm_pindex_t pml4index; | KASSERT((*pml5 & PG_V) == 0, | ||||
("pmap %p va %#lx pml5 %#lx", pmap, va, *pml5)); | |||||
*pml5 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | |||||
if (pmap->pm_pmltopu != NULL && pml5index < NUPML5E) { | |||||
if (pmap->pm_ucr3 != PMAP_NO_CR3) | |||||
*pml5 |= pg_nx; | |||||
pml5u = &pmap->pm_pmltopu[pml5index]; | |||||
*pml5u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | | |||||
PG_A | PG_M; | |||||
} | |||||
} else if (ptepindex >= NUPDE + NUPDPE) { | |||||
pml4index = pmap_pml4e_index(va); | |||||
/* Wire up a new PDPE page */ | /* Wire up a new PDPE page */ | ||||
pml4index = ptepindex - (NUPDE + NUPDPE); | pml4 = pmap_allocpte_getpml4(pmap, lockp, va, true); | ||||
pml4 = &pmap->pm_pml4[pml4index]; | if (pml4 == NULL) { | ||||
vm_page_unwire_noq(m); | |||||
vm_page_free_zero(m); | |||||
return (NULL); | |||||
} | |||||
KASSERT((*pml4 & PG_V) == 0, | |||||
("pmap %p va %#lx pml4 %#lx", pmap, va, *pml4)); | |||||
*pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
if (pmap->pm_pml4u != NULL && pml4index < NUPML4E) { | |||||
if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL && | |||||
pml4index < NUPML4E) { | |||||
/* | /* | ||||
* PTI: Make all user-space mappings in the | * PTI: Make all user-space mappings in the | ||||
* kernel-mode page table no-execute so that | * kernel-mode page table no-execute so that | ||||
* we detect any programming errors that leave | * we detect any programming errors that leave | ||||
* the kernel-mode page table active on return | * the kernel-mode page table active on return | ||||
* to user space. | * to user space. | ||||
*/ | */ | ||||
if (pmap->pm_ucr3 != PMAP_NO_CR3) | if (pmap->pm_ucr3 != PMAP_NO_CR3) | ||||
*pml4 |= pg_nx; | *pml4 |= pg_nx; | ||||
pml4u = &pmap->pm_pml4u[pml4index]; | pml4u = &pmap->pm_pmltopu[pml4index]; | ||||
*pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | | *pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | | ||||
PG_A | PG_M; | PG_A | PG_M; | ||||
} | } | ||||
} else if (ptepindex >= NUPDE) { | } else if (ptepindex >= NUPDE) { | ||||
vm_pindex_t pml4index; | |||||
vm_pindex_t pdpindex; | |||||
pml4_entry_t *pml4; | |||||
pdp_entry_t *pdp; | |||||
/* Wire up a new PDE page */ | /* Wire up a new PDE page */ | ||||
pdpindex = ptepindex - NUPDE; | pdp = pmap_allocpte_getpdp(pmap, lockp, va, true); | ||||
pml4index = pdpindex >> NPML4EPGSHIFT; | if (pdp == NULL) { | ||||
pml4 = &pmap->pm_pml4[pml4index]; | |||||
if ((*pml4 & PG_V) == 0) { | |||||
/* Have to allocate a new pdp, recurse */ | |||||
if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index, | |||||
lockp) == NULL) { | |||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} else { | KASSERT((*pdp & PG_V) == 0, | ||||
/* Add reference to pdp page */ | ("pmap %p va %#lx pdp %#lx", pmap, va, *pdp)); | ||||
pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME); | |||||
pdppg->ref_count++; | |||||
} | |||||
pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); | |||||
/* Now find the pdp page */ | |||||
pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)]; | |||||
*pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
} else { | } else { | ||||
vm_pindex_t pml4index; | |||||
vm_pindex_t pdpindex; | |||||
pml4_entry_t *pml4; | |||||
pdp_entry_t *pdp; | |||||
pd_entry_t *pd; | |||||
/* Wire up a new PTE page */ | /* Wire up a new PTE page */ | ||||
pdpindex = ptepindex >> NPDPEPGSHIFT; | pdp = pmap_allocpte_getpdp(pmap, lockp, va, false); | ||||
pml4index = pdpindex >> NPML4EPGSHIFT; | if (pdp == NULL) { | ||||
/* First, find the pdp and check that its valid. */ | |||||
pml4 = &pmap->pm_pml4[pml4index]; | |||||
if ((*pml4 & PG_V) == 0) { | |||||
/* Have to allocate a new pd, recurse */ | |||||
if (_pmap_allocpte(pmap, NUPDE + pdpindex, | |||||
lockp) == NULL) { | |||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); | |||||
pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)]; | |||||
} else { | |||||
pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); | |||||
pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)]; | |||||
if ((*pdp & PG_V) == 0) { | if ((*pdp & PG_V) == 0) { | ||||
/* Have to allocate a new pd, recurse */ | /* Have to allocate a new pd, recurse */ | ||||
if (_pmap_allocpte(pmap, NUPDE + pdpindex, | if (_pmap_allocpte(pmap, pmap_pdpe_pindex(va), | ||||
lockp) == NULL) { | lockp, va) == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} else { | } else { | ||||
/* Add reference to the pd page */ | /* Add reference to the pd page */ | ||||
pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME); | pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME); | ||||
pdpg->ref_count++; | pdpg->ref_count++; | ||||
} | } | ||||
} | |||||
pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME); | pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME); | ||||
/* Now we know where the page directory page is */ | /* Now we know where the page directory page is */ | ||||
pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)]; | pd = &pd[pmap_pde_index(va)]; | ||||
KASSERT((*pd & PG_V) == 0, | |||||
("pmap %p va %#lx pd %#lx", pmap, va, *pd)); | |||||
*pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
} | } | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
return (m); | return (m); | ||||
} | } | ||||
Show All 16 Lines | if (va < VM_MAXUSER_ADDRESS) { | ||||
/* Add a reference to the pd page. */ | /* Add a reference to the pd page. */ | ||||
pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); | pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); | ||||
pdpg->ref_count++; | pdpg->ref_count++; | ||||
} else | } else | ||||
pdpg = NULL; | pdpg = NULL; | ||||
} else if (va < VM_MAXUSER_ADDRESS) { | } else if (va < VM_MAXUSER_ADDRESS) { | ||||
/* Allocate a pd page. */ | /* Allocate a pd page. */ | ||||
pdpindex = pmap_pde_pindex(va) >> NPDPEPGSHIFT; | pdpindex = pmap_pde_pindex(va) >> NPDPEPGSHIFT; | ||||
pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); | pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp, va); | ||||
if (pdpg == NULL) { | if (pdpg == NULL) { | ||||
if (lockp != NULL) | if (lockp != NULL) | ||||
goto retry; | goto retry; | ||||
else | else | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); | pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); | ||||
pde = &pde[pmap_pde_index(va)]; | pde = &pde[pmap_pde_index(va)]; | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | retry: | ||||
if (pd != NULL && (*pd & PG_V) != 0) { | if (pd != NULL && (*pd & PG_V) != 0) { | ||||
m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); | m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); | ||||
m->ref_count++; | m->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Here if the pte page isn't mapped, or if it has been | * Here if the pte page isn't mapped, or if it has been | ||||
* deallocated. | * deallocated. | ||||
*/ | */ | ||||
m = _pmap_allocpte(pmap, ptepindex, lockp); | m = _pmap_allocpte(pmap, ptepindex, lockp, va); | ||||
if (m == NULL && lockp != NULL) | if (m == NULL && lockp != NULL) | ||||
goto retry; | goto retry; | ||||
} | } | ||||
return (m); | return (m); | ||||
} | } | ||||
/*************************************************** | /*************************************************** | ||||
* Pmap allocation/deallocation routines. | * Pmap allocation/deallocation routines. | ||||
***************************************************/ | ***************************************************/ | ||||
/* | /* | ||||
* Release any resources held by the given physical map. | * Release any resources held by the given physical map. | ||||
* Called when a pmap initialized by pmap_pinit is being released. | * Called when a pmap initialized by pmap_pinit is being released. | ||||
* Should only be called if the map contains no valid mappings. | * Should only be called if the map contains no valid mappings. | ||||
*/ | */ | ||||
void | void | ||||
pmap_release(pmap_t pmap) | pmap_release(pmap_t pmap) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
int i; | int i; | ||||
KASSERT(pmap->pm_stats.resident_count == 0, | KASSERT(pmap->pm_stats.resident_count == 0, | ||||
("pmap_release: pmap resident count %ld != 0", | ("pmap_release: pmap %p resident count %ld != 0", | ||||
pmap->pm_stats.resident_count)); | pmap, pmap->pm_stats.resident_count)); | ||||
KASSERT(vm_radix_is_empty(&pmap->pm_root), | KASSERT(vm_radix_is_empty(&pmap->pm_root), | ||||
("pmap_release: pmap has reserved page table page(s)")); | ("pmap_release: pmap %p has reserved page table page(s)", | ||||
pmap)); | |||||
KASSERT(CPU_EMPTY(&pmap->pm_active), | KASSERT(CPU_EMPTY(&pmap->pm_active), | ||||
("releasing active pmap %p", pmap)); | ("releasing active pmap %p", pmap)); | ||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4)); | m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_pmltop)); | ||||
if (pmap_is_la57(pmap)) { | |||||
pmap->pm_pmltop[pmap_pml5e_index(UPT_MAX_ADDRESS)] = 0; | |||||
pmap->pm_pmltop[PML5PML5I] = 0; | |||||
} else { | |||||
for (i = 0; i < NKPML4E; i++) /* KVA */ | for (i = 0; i < NKPML4E; i++) /* KVA */ | ||||
pmap->pm_pml4[KPML4BASE + i] = 0; | pmap->pm_pmltop[KPML4BASE + i] = 0; | ||||
for (i = 0; i < ndmpdpphys; i++)/* Direct Map */ | for (i = 0; i < ndmpdpphys; i++)/* Direct Map */ | ||||
pmap->pm_pml4[DMPML4I + i] = 0; | pmap->pm_pmltop[DMPML4I + i] = 0; | ||||
pmap->pm_pml4[PML4PML4I] = 0; /* Recursive Mapping */ | pmap->pm_pmltop[PML4PML4I] = 0; /* Recursive Mapping */ | ||||
for (i = 0; i < lm_ents; i++) /* Large Map */ | for (i = 0; i < lm_ents; i++) /* Large Map */ | ||||
pmap->pm_pml4[LMSPML4I + i] = 0; | pmap->pm_pmltop[LMSPML4I + i] = 0; | ||||
} | |||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
if (pmap->pm_pml4u != NULL) { | if (pmap->pm_pmltopu != NULL) { | ||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4u)); | m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap-> | ||||
pm_pmltopu)); | |||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free(m); | vm_page_free(m); | ||||
} | } | ||||
if (pmap->pm_type == PT_X86 && | if (pmap->pm_type == PT_X86 && | ||||
(cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) | (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) | ||||
rangeset_fini(&pmap->pm_pkru); | rangeset_fini(&pmap->pm_pkru); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 1,322 Lines • ▼ Show 20 Lines | |||||
* It is assumed that the start and end are properly | * It is assumed that the start and end are properly | ||||
* rounded to the page size. | * rounded to the page size. | ||||
*/ | */ | ||||
void | void | ||||
pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
{ | { | ||||
struct rwlock *lock; | struct rwlock *lock; | ||||
vm_offset_t va_next; | vm_offset_t va_next; | ||||
pml5_entry_t *pml5e; | |||||
pml4_entry_t *pml4e; | pml4_entry_t *pml4e; | ||||
pdp_entry_t *pdpe; | pdp_entry_t *pdpe; | ||||
pd_entry_t ptpaddr, *pde; | pd_entry_t ptpaddr, *pde; | ||||
pt_entry_t PG_G, PG_V; | pt_entry_t PG_G, PG_V; | ||||
struct spglist free; | struct spglist free; | ||||
int anyvalid; | int anyvalid; | ||||
PG_G = pmap_global_bit(pmap); | PG_G = pmap_global_bit(pmap); | ||||
Show All 26 Lines | pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
} | } | ||||
lock = NULL; | lock = NULL; | ||||
for (; sva < eva; sva = va_next) { | for (; sva < eva; sva = va_next) { | ||||
if (pmap->pm_stats.resident_count == 0) | if (pmap->pm_stats.resident_count == 0) | ||||
break; | break; | ||||
if (pmap_is_la57(pmap)) { | |||||
pml5e = pmap_pml5e(pmap, sva); | |||||
if ((*pml5e & PG_V) == 0) { | |||||
va_next = (sva + NBPML5) & ~PML5MASK; | |||||
if (va_next < sva) | |||||
va_next = eva; | |||||
continue; | |||||
} | |||||
pml4e = pmap_pml5e_to_pml4e(pml5e, sva); | |||||
} else { | |||||
pml4e = pmap_pml4e(pmap, sva); | pml4e = pmap_pml4e(pmap, sva); | ||||
} | |||||
if ((*pml4e & PG_V) == 0) { | if ((*pml4e & PG_V) == 0) { | ||||
va_next = (sva + NBPML4) & ~PML4MASK; | va_next = (sva + NBPML4) & ~PML4MASK; | ||||
if (va_next < sva) | if (va_next < sva) | ||||
va_next = eva; | va_next = eva; | ||||
continue; | continue; | ||||
} | } | ||||
pdpe = pmap_pml4e_to_pdpe(pml4e, sva); | pdpe = pmap_pml4e_to_pdpe(pml4e, sva); | ||||
▲ Show 20 Lines • Show All 603 Lines • ▼ Show 20 Lines | if (pde != NULL && (*pde & PG_V) != 0 && ((*pde & PG_PS) == 0 || | ||||
} | } | ||||
} else if (va < VM_MAXUSER_ADDRESS) { | } else if (va < VM_MAXUSER_ADDRESS) { | ||||
/* | /* | ||||
* Here if the pte page isn't mapped, or if it has been | * Here if the pte page isn't mapped, or if it has been | ||||
* deallocated. | * deallocated. | ||||
*/ | */ | ||||
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | ||||
mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), | mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), | ||||
nosleep ? NULL : &lock); | nosleep ? NULL : &lock, va); | ||||
if (mpte == NULL && nosleep) { | if (mpte == NULL && nosleep) { | ||||
rv = KERN_RESOURCE_SHORTAGE; | rv = KERN_RESOURCE_SHORTAGE; | ||||
goto out; | goto out; | ||||
} | } | ||||
goto retry; | goto retry; | ||||
} else | } else | ||||
panic("pmap_enter: invalid page directory va=%#lx", va); | panic("pmap_enter: invalid page directory va=%#lx", va); | ||||
▲ Show 20 Lines • Show All 466 Lines • ▼ Show 20 Lines | if (mpte && (mpte->pindex == ptepindex)) { | ||||
return (NULL); | return (NULL); | ||||
mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME); | mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME); | ||||
mpte->ref_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Pass NULL instead of the PV list lock | * Pass NULL instead of the PV list lock | ||||
* pointer, because we don't intend to sleep. | * pointer, because we don't intend to sleep. | ||||
*/ | */ | ||||
mpte = _pmap_allocpte(pmap, ptepindex, NULL); | mpte = _pmap_allocpte(pmap, ptepindex, NULL, | ||||
va); | |||||
if (mpte == NULL) | if (mpte == NULL) | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
} | } | ||||
pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | ||||
pte = &pte[pmap_pte_index(va)]; | pte = &pte[pmap_pte_index(va)]; | ||||
} else { | } else { | ||||
mpte = NULL; | mpte = NULL; | ||||
▲ Show 20 Lines • Show All 2,736 Lines • ▼ Show 20 Lines | pmap_large_map_pdpe(vm_offset_t va) | ||||
vm_pindex_t pml4_idx; | vm_pindex_t pml4_idx; | ||||
vm_paddr_t mphys; | vm_paddr_t mphys; | ||||
pml4_idx = pmap_pml4e_index(va); | pml4_idx = pmap_pml4e_index(va); | ||||
KASSERT(LMSPML4I <= pml4_idx && pml4_idx < LMSPML4I + lm_ents, | KASSERT(LMSPML4I <= pml4_idx && pml4_idx < LMSPML4I + lm_ents, | ||||
("pmap_large_map_pdpe: va %#jx out of range idx %#jx LMSPML4I " | ("pmap_large_map_pdpe: va %#jx out of range idx %#jx LMSPML4I " | ||||
"%#jx lm_ents %d", | "%#jx lm_ents %d", | ||||
(uintmax_t)va, (uintmax_t)pml4_idx, LMSPML4I, lm_ents)); | (uintmax_t)va, (uintmax_t)pml4_idx, LMSPML4I, lm_ents)); | ||||
KASSERT((kernel_pmap->pm_pml4[pml4_idx] & X86_PG_V) != 0, | KASSERT((kernel_pml4[pml4_idx] & X86_PG_V) != 0, | ||||
("pmap_large_map_pdpe: invalid pml4 for va %#jx idx %#jx " | ("pmap_large_map_pdpe: invalid pml4 for va %#jx idx %#jx " | ||||
"LMSPML4I %#jx lm_ents %d", | "LMSPML4I %#jx lm_ents %d", | ||||
(uintmax_t)va, (uintmax_t)pml4_idx, LMSPML4I, lm_ents)); | (uintmax_t)va, (uintmax_t)pml4_idx, LMSPML4I, lm_ents)); | ||||
mphys = kernel_pmap->pm_pml4[pml4_idx] & PG_FRAME; | mphys = kernel_pml4[pml4_idx] & PG_FRAME; | ||||
return ((pdp_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pdpe_index(va)); | return ((pdp_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pdpe_index(va)); | ||||
} | } | ||||
static pd_entry_t * | static pd_entry_t * | ||||
pmap_large_map_pde(vm_offset_t va) | pmap_large_map_pde(vm_offset_t va) | ||||
{ | { | ||||
pdp_entry_t *pdpe; | pdp_entry_t *pdpe; | ||||
vm_page_t m; | vm_page_t m; | ||||
▲ Show 20 Lines • Show All 1,058 Lines • ▼ Show 20 Lines | sbuf_printf(sb, "0x%016lx-0x%016lx r%c%c%c%c %s %d %d %d\n", | ||||
range->sva, eva, | range->sva, eva, | ||||
(range->attrs & X86_PG_RW) != 0 ? 'w' : '-', | (range->attrs & X86_PG_RW) != 0 ? 'w' : '-', | ||||
(range->attrs & pg_nx) != 0 ? '-' : 'x', | (range->attrs & pg_nx) != 0 ? '-' : 'x', | ||||
(range->attrs & X86_PG_U) != 0 ? 'u' : 's', | (range->attrs & X86_PG_U) != 0 ? 'u' : 's', | ||||
(range->attrs & X86_PG_G) != 0 ? 'g' : '-', | (range->attrs & X86_PG_G) != 0 ? 'g' : '-', | ||||
mode, range->pdpes, range->pdes, range->ptes); | mode, range->pdpes, range->pdes, range->ptes); | ||||
/* Reset to sentinel value. */ | /* Reset to sentinel value. */ | ||||
range->sva = KVADDR(NPML4EPG - 1, NPDPEPG - 1, NPDEPG - 1, NPTEPG - 1); | range->sva = la57 ? KV5ADDR(NPML5EPG - 1, NPML4EPG - 1, NPDPEPG - 1, | ||||
NPDEPG - 1, NPTEPG - 1) : KV4ADDR(NPML4EPG - 1, NPDPEPG - 1, | |||||
NPDEPG - 1, NPTEPG - 1); | |||||
} | } | ||||
/* | /* | ||||
* Determine whether the attributes specified by a page table entry match those | * Determine whether the attributes specified by a page table entry match those | ||||
* being tracked by the current range. This is not quite as simple as a direct | * being tracked by the current range. This is not quite as simple as a direct | ||||
* flag comparison since some PAT modes have multiple representations. | * flag comparison since some PAT modes have multiple representations. | ||||
*/ | */ | ||||
static bool | static bool | ||||
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines | sysctl_kmaps(SYSCTL_HANDLER_ARGS) | ||||
error = sysctl_wire_old_buffer(req, 0); | error = sysctl_wire_old_buffer(req, 0); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
sb = &sbuf; | sb = &sbuf; | ||||
sbuf_new_for_sysctl(sb, NULL, PAGE_SIZE, req); | sbuf_new_for_sysctl(sb, NULL, PAGE_SIZE, req); | ||||
/* Sentinel value. */ | /* Sentinel value. */ | ||||
range.sva = KVADDR(NPML4EPG - 1, NPDPEPG - 1, NPDEPG - 1, NPTEPG - 1); | range.sva = la57 ? KV5ADDR(NPML5EPG - 1, NPML4EPG - 1, NPDPEPG - 1, | ||||
NPDEPG - 1, NPTEPG - 1) : KV4ADDR(NPML4EPG - 1, NPDPEPG - 1, | |||||
NPDEPG - 1, NPTEPG - 1); | |||||
/* | /* | ||||
* Iterate over the kernel page tables without holding the kernel pmap | * Iterate over the kernel page tables without holding the kernel pmap | ||||
* lock. Outside of the large map, kernel page table pages are never | * lock. Outside of the large map, kernel page table pages are never | ||||
* freed, so at worst we will observe inconsistencies in the output. | * freed, so at worst we will observe inconsistencies in the output. | ||||
* Within the large map, ensure that PDP and PD page addresses are | * Within the large map, ensure that PDP and PD page addresses are | ||||
* valid before descending. | * valid before descending. | ||||
*/ | */ | ||||
Show All 13 Lines | case LMSPML4I: | ||||
break; | break; | ||||
} | } | ||||
/* Convert to canonical form. */ | /* Convert to canonical form. */ | ||||
if (sva == 1ul << 47) | if (sva == 1ul << 47) | ||||
sva |= -1ul << 48; | sva |= -1ul << 48; | ||||
restart: | restart: | ||||
pml4e = kernel_pmap->pm_pml4[i]; | pml4e = kernel_pml4[i]; | ||||
if ((pml4e & X86_PG_V) == 0) { | if ((pml4e & X86_PG_V) == 0) { | ||||
sva = rounddown2(sva, NBPML4); | sva = rounddown2(sva, NBPML4); | ||||
sysctl_kmaps_dump(sb, &range, sva); | sysctl_kmaps_dump(sb, &range, sva); | ||||
sva += NBPML4; | sva += NBPML4; | ||||
continue; | continue; | ||||
} | } | ||||
pa = pml4e & PG_FRAME; | pa = pml4e & PG_FRAME; | ||||
pdp = (pdp_entry_t *)PHYS_TO_DMAP(pa); | pdp = (pdp_entry_t *)PHYS_TO_DMAP(pa); | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | SYSCTL_OID(_vm_pmap, OID_AUTO, kernel_maps, | ||||
CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, | ||||
NULL, 0, sysctl_kmaps, "A", | NULL, 0, sysctl_kmaps, "A", | ||||
"Dump kernel address layout"); | "Dump kernel address layout"); | ||||
#ifdef DDB | #ifdef DDB | ||||
DB_SHOW_COMMAND(pte, pmap_print_pte) | DB_SHOW_COMMAND(pte, pmap_print_pte) | ||||
{ | { | ||||
pmap_t pmap; | pmap_t pmap; | ||||
pml5_entry_t *pml5; | |||||
pml4_entry_t *pml4; | pml4_entry_t *pml4; | ||||
pdp_entry_t *pdp; | pdp_entry_t *pdp; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
pt_entry_t *pte, PG_V; | pt_entry_t *pte, PG_V; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
if (!have_addr) { | if (!have_addr) { | ||||
db_printf("show pte addr\n"); | db_printf("show pte addr\n"); | ||||
return; | return; | ||||
} | } | ||||
va = (vm_offset_t)addr; | va = (vm_offset_t)addr; | ||||
if (kdb_thread != NULL) | if (kdb_thread != NULL) | ||||
pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); | pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); | ||||
else | else | ||||
pmap = PCPU_GET(curpmap); | pmap = PCPU_GET(curpmap); | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
db_printf("VA 0x%016lx", va); | |||||
if (pmap_is_la57(pmap)) { | |||||
pml5 = pmap_pml5e(pmap, va); | |||||
db_printf(" pml5e 0x%016lx", *pml5); | |||||
if ((*pml5 & PG_V) == 0) { | |||||
db_printf("\n"); | |||||
return; | |||||
} | |||||
pml4 = pmap_pml5e_to_pml4e(pml5, va); | |||||
} else { | |||||
pml4 = pmap_pml4e(pmap, va); | pml4 = pmap_pml4e(pmap, va); | ||||
db_printf("VA 0x%016lx pml4e 0x%016lx", va, *pml4); | } | ||||
db_printf(" pml4e 0x%016lx", *pml4); | |||||
if ((*pml4 & PG_V) == 0) { | if ((*pml4 & PG_V) == 0) { | ||||
db_printf("\n"); | db_printf("\n"); | ||||
return; | return; | ||||
} | } | ||||
pdp = pmap_pml4e_to_pdpe(pml4, va); | pdp = pmap_pml4e_to_pdpe(pml4, va); | ||||
db_printf(" pdpe 0x%016lx", *pdp); | db_printf(" pdpe 0x%016lx", *pdp); | ||||
if ((*pdp & PG_V) == 0 || (*pdp & PG_PS) != 0) { | if ((*pdp & PG_V) == 0 || (*pdp & PG_PS) != 0) { | ||||
db_printf("\n"); | db_printf("\n"); | ||||
Show All 13 Lines | |||||
{ | { | ||||
vm_paddr_t a; | vm_paddr_t a; | ||||
if (have_addr) { | if (have_addr) { | ||||
a = (vm_paddr_t)addr; | a = (vm_paddr_t)addr; | ||||
db_printf("0x%jx\n", (uintmax_t)PHYS_TO_DMAP(a)); | db_printf("0x%jx\n", (uintmax_t)PHYS_TO_DMAP(a)); | ||||
} else { | } else { | ||||
db_printf("show phys2dmap addr\n"); | db_printf("show phys2dmap addr\n"); | ||||
} | |||||
} | |||||
static void | |||||
ptpages_show_page(int level, int idx, vm_page_t pg) | |||||
{ | |||||
db_printf("l %d i %d pg %p phys %#lx ref %x\n", | |||||
level, idx, pg, VM_PAGE_TO_PHYS(pg), pg->ref_count); | |||||
} | |||||
static void | |||||
ptpages_show_complain(int level, int idx, uint64_t pte) | |||||
{ | |||||
db_printf("l %d i %d pte %#lx\n", level, idx, pte); | |||||
} | |||||
static void | |||||
ptpages_show_pml4(vm_page_t pg4, int num_entries, uint64_t PG_V) | |||||
{ | |||||
vm_page_t pg3, pg2, pg1; | |||||
pml4_entry_t *pml4; | |||||
pdp_entry_t *pdp; | |||||
pd_entry_t *pd; | |||||
int i4, i3, i2; | |||||
pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pg4)); | |||||
for (i4 = 0; i4 < num_entries; i4++) { | |||||
if ((pml4[i4] & PG_V) == 0) | |||||
continue; | |||||
pg3 = PHYS_TO_VM_PAGE(pml4[i4] & PG_FRAME); | |||||
if (pg3 == NULL) { | |||||
ptpages_show_complain(3, i4, pml4[i4]); | |||||
continue; | |||||
} | |||||
ptpages_show_page(3, i4, pg3); | |||||
pdp = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pg3)); | |||||
for (i3 = 0; i3 < NPDPEPG; i3++) { | |||||
if ((pdp[i3] & PG_V) == 0) | |||||
continue; | |||||
pg2 = PHYS_TO_VM_PAGE(pdp[i3] & PG_FRAME); | |||||
if (pg3 == NULL) { | |||||
ptpages_show_complain(2, i3, pdp[i3]); | |||||
continue; | |||||
} | |||||
ptpages_show_page(2, i3, pg2); | |||||
pd = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pg2)); | |||||
for (i2 = 0; i2 < NPDEPG; i2++) { | |||||
if ((pd[i2] & PG_V) == 0) | |||||
continue; | |||||
pg1 = PHYS_TO_VM_PAGE(pd[i2] & PG_FRAME); | |||||
if (pg1 == NULL) { | |||||
ptpages_show_complain(1, i2, pd[i2]); | |||||
continue; | |||||
} | |||||
ptpages_show_page(1, i2, pg1); | |||||
} | |||||
} | |||||
} | |||||
} | |||||
DB_SHOW_COMMAND(ptpages, pmap_ptpages) | |||||
{ | |||||
pmap_t pmap; | |||||
vm_page_t pg; | |||||
pml5_entry_t *pml5; | |||||
uint64_t PG_V; | |||||
int i5; | |||||
if (have_addr) | |||||
pmap = (pmap_t)addr; | |||||
else | |||||
pmap = PCPU_GET(curpmap); | |||||
PG_V = pmap_valid_bit(pmap); | |||||
if (pmap_is_la57(pmap)) { | |||||
pml5 = pmap->pm_pmltop; | |||||
for (i5 = 0; i5 < NUPML5E; i5++) { | |||||
if ((pml5[i5] & PG_V) == 0) | |||||
continue; | |||||
pg = PHYS_TO_VM_PAGE(pml5[i5] & PG_FRAME); | |||||
if (pg == NULL) { | |||||
ptpages_show_complain(4, i5, pml5[i5]); | |||||
continue; | |||||
} | |||||
ptpages_show_page(4, i5, pg); | |||||
ptpages_show_pml4(pg, NPML4EPG, PG_V); | |||||
} | |||||
} else { | |||||
ptpages_show_pml4(PHYS_TO_VM_PAGE(DMAP_TO_PHYS( | |||||
(vm_offset_t)pmap->pm_pmltop)), NUP4ML4E, PG_V); | |||||
} | } | ||||
} | } | ||||
#endif | #endif |
"temporal" -> "temporary"