Changeset View
Standalone View
sys/amd64/amd64/pmap.c
Show First 20 Lines • Show All 143 Lines • ▼ Show 20 Lines | |||||
#include <machine/cpu.h> | #include <machine/cpu.h> | ||||
#include <machine/cputypes.h> | #include <machine/cputypes.h> | ||||
#include <machine/md_var.h> | #include <machine/md_var.h> | ||||
#include <machine/pcb.h> | #include <machine/pcb.h> | ||||
#include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
#ifdef SMP | #ifdef SMP | ||||
#include <machine/smp.h> | #include <machine/smp.h> | ||||
#endif | #endif | ||||
#include <machine/tss.h> | |||||
static __inline boolean_t | static __inline boolean_t | ||||
pmap_type_guest(pmap_t pmap) | pmap_type_guest(pmap_t pmap) | ||||
{ | { | ||||
return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI)); | return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | |||||
static __inline pt_entry_t | static __inline pt_entry_t | ||||
pmap_global_bit(pmap_t pmap) | pmap_global_bit(pmap_t pmap) | ||||
{ | { | ||||
pt_entry_t mask; | pt_entry_t mask; | ||||
switch (pmap->pm_type) { | switch (pmap->pm_type) { | ||||
case PT_X86: | case PT_X86: | ||||
mask = X86_PG_G; | mask = pg_g; | ||||
break; | break; | ||||
case PT_RVI: | case PT_RVI: | ||||
case PT_EPT: | case PT_EPT: | ||||
mask = 0; | mask = 0; | ||||
break; | break; | ||||
default: | default: | ||||
panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type); | panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 106 Lines • ▼ Show 20 Lines | |||||
int nkpt; | int nkpt; | ||||
SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, | SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, | ||||
"Number of kernel page table pages allocated on bootup"); | "Number of kernel page table pages allocated on bootup"); | ||||
static int ndmpdp; | static int ndmpdp; | ||||
vm_paddr_t dmaplimit; | vm_paddr_t dmaplimit; | ||||
vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; | vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; | ||||
pt_entry_t pg_nx; | pt_entry_t pg_nx, pg_g; | ||||
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); | static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); | ||||
static int pat_works = 1; | static int pat_works = 1; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1, | SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1, | ||||
"Is page attribute table fully functional?"); | "Is page attribute table fully functional?"); | ||||
static int pg_ps_enabled = 1; | static int pg_ps_enabled = 1; | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | |||||
int pmap_pcid_enabled = 1; | int pmap_pcid_enabled = 1; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pcid_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | SYSCTL_INT(_vm_pmap, OID_AUTO, pcid_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | ||||
&pmap_pcid_enabled, 0, "Is TLB Context ID enabled ?"); | &pmap_pcid_enabled, 0, "Is TLB Context ID enabled ?"); | ||||
int invpcid_works = 0; | int invpcid_works = 0; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, invpcid_works, CTLFLAG_RD, &invpcid_works, 0, | SYSCTL_INT(_vm_pmap, OID_AUTO, invpcid_works, CTLFLAG_RD, &invpcid_works, 0, | ||||
"Is the invpcid instruction available ?"); | "Is the invpcid instruction available ?"); | ||||
int pti = 1; | |||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pti, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | |||||
&pti, 0, | |||||
"Page Table Isolation enabled"); | |||||
static vm_object_t pti_obj; | |||||
static pml4_entry_t *pti_pml4; | |||||
static vm_pindex_t pti_pg_idx; | |||||
static bool pti_finalized; | |||||
static int | static int | ||||
pmap_pcid_save_cnt_proc(SYSCTL_HANDLER_ARGS) | pmap_pcid_save_cnt_proc(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
int i; | int i; | ||||
uint64_t res; | uint64_t res; | ||||
res = 0; | res = 0; | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
▲ Show 20 Lines • Show All 218 Lines • ▼ Show 20 Lines | |||||
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask); | static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, | static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
#endif | #endif | ||||
static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, | static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, | ||||
vm_prot_t prot); | vm_prot_t prot); | ||||
static void pmap_pte_attr(pt_entry_t *pte, int cache_bits, int mask); | static void pmap_pte_attr(pt_entry_t *pte, int cache_bits, int mask); | ||||
static void pmap_pti_add_kva_locked(vm_offset_t sva, vm_offset_t eva, | |||||
bool exec); | |||||
static void pmap_pti_init(void); | |||||
static pdp_entry_t *pmap_pti_pdpe(vm_offset_t va); | |||||
static pd_entry_t *pmap_pti_pde(vm_offset_t va); | |||||
static void pmap_pti_wire_pte(void *pte); | |||||
static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | ||||
struct spglist *free, struct rwlock **lockp); | struct spglist *free, struct rwlock **lockp); | ||||
static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, | static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, | ||||
pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); | pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); | ||||
static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); | static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); | ||||
static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | ||||
struct spglist *free); | struct spglist *free); | ||||
static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, | static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, | ||||
▲ Show 20 Lines • Show All 266 Lines • ▼ Show 20 Lines | create_pagetables(vm_paddr_t *firstaddr) | ||||
KPTphys = allocpages(firstaddr, nkpt); | KPTphys = allocpages(firstaddr, nkpt); | ||||
KPDphys = allocpages(firstaddr, nkpdpe); | KPDphys = allocpages(firstaddr, nkpdpe); | ||||
/* Fill in the underlying page table pages */ | /* Fill in the underlying page table pages */ | ||||
/* Nominally read-only (but really R/W) from zero to physfree */ | /* Nominally read-only (but really R/W) from zero to physfree */ | ||||
/* XXX not fully used, underneath 2M pages */ | /* XXX not fully used, underneath 2M pages */ | ||||
pt_p = (pt_entry_t *)KPTphys; | pt_p = (pt_entry_t *)KPTphys; | ||||
for (i = 0; ptoa(i) < *firstaddr; i++) | for (i = 0; ptoa(i) < *firstaddr; i++) | ||||
pt_p[i] = ptoa(i) | X86_PG_RW | X86_PG_V | X86_PG_G; | pt_p[i] = ptoa(i) | X86_PG_RW | X86_PG_V | pg_g; | ||||
jeff: I would find this whole section more readable with a single 'gflag' variable and a single test… | |||||
/* Now map the page tables at their location within PTmap */ | /* Now map the page tables at their location within PTmap */ | ||||
pd_p = (pd_entry_t *)KPDphys; | pd_p = (pd_entry_t *)KPDphys; | ||||
for (i = 0; i < nkpt; i++) | for (i = 0; i < nkpt; i++) | ||||
pd_p[i] = (KPTphys + ptoa(i)) | X86_PG_RW | X86_PG_V; | pd_p[i] = (KPTphys + ptoa(i)) | X86_PG_RW | X86_PG_V; | ||||
/* Map from zero to end of allocations under 2M pages */ | /* Map from zero to end of allocations under 2M pages */ | ||||
/* This replaces some of the KPTphys entries above */ | /* This replaces some of the KPTphys entries above */ | ||||
for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) | for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) | ||||
pd_p[i] = (i << PDRSHIFT) | X86_PG_RW | X86_PG_V | PG_PS | | pd_p[i] = (i << PDRSHIFT) | X86_PG_RW | X86_PG_V | PG_PS | | ||||
X86_PG_G; | pg_g; | ||||
/* And connect up the PD to the PDP (leaving room for L4 pages) */ | /* And connect up the PD to the PDP (leaving room for L4 pages) */ | ||||
pdp_p = (pdp_entry_t *)(KPDPphys + ptoa(KPML4I - KPML4BASE)); | pdp_p = (pdp_entry_t *)(KPDPphys + ptoa(KPML4I - KPML4BASE)); | ||||
for (i = 0; i < nkpdpe; i++) | for (i = 0; i < nkpdpe; i++) | ||||
pdp_p[i + KPDPI] = (KPDphys + ptoa(i)) | X86_PG_RW | X86_PG_V | | pdp_p[i + KPDPI] = (KPDphys + ptoa(i)) | X86_PG_RW | X86_PG_V | | ||||
PG_U; | PG_U; | ||||
/* | /* | ||||
* Now, set up the direct map region using 2MB and/or 1GB pages. If | * Now, set up the direct map region using 2MB and/or 1GB pages. If | ||||
* the end of physical memory is not aligned to a 1GB page boundary, | * the end of physical memory is not aligned to a 1GB page boundary, | ||||
* then the residual physical memory is mapped with 2MB pages. Later, | * then the residual physical memory is mapped with 2MB pages. Later, | ||||
* if pmap_mapdev{_attr}() uses the direct map for non-write-back | * if pmap_mapdev{_attr}() uses the direct map for non-write-back | ||||
* memory, pmap_change_attr() will demote any 2MB or 1GB page mappings | * memory, pmap_change_attr() will demote any 2MB or 1GB page mappings | ||||
* that are partially used. | * that are partially used. | ||||
*/ | */ | ||||
pd_p = (pd_entry_t *)DMPDphys; | pd_p = (pd_entry_t *)DMPDphys; | ||||
for (i = NPDEPG * ndm1g, j = 0; i < NPDEPG * ndmpdp; i++, j++) { | for (i = NPDEPG * ndm1g, j = 0; i < NPDEPG * ndmpdp; i++, j++) { | ||||
pd_p[j] = (vm_paddr_t)i << PDRSHIFT; | pd_p[j] = (vm_paddr_t)i << PDRSHIFT; | ||||
/* Preset PG_M and PG_A because demotion expects it. */ | /* Preset PG_M and PG_A because demotion expects it. */ | ||||
pd_p[j] |= X86_PG_RW | X86_PG_V | PG_PS | X86_PG_G | | pd_p[j] |= X86_PG_RW | X86_PG_V | PG_PS | pg_g | | ||||
X86_PG_M | X86_PG_A | pg_nx; | X86_PG_M | X86_PG_A | pg_nx; | ||||
} | } | ||||
pdp_p = (pdp_entry_t *)DMPDPphys; | pdp_p = (pdp_entry_t *)DMPDPphys; | ||||
for (i = 0; i < ndm1g; i++) { | for (i = 0; i < ndm1g; i++) { | ||||
pdp_p[i] = (vm_paddr_t)i << PDPSHIFT; | pdp_p[i] = (vm_paddr_t)i << PDPSHIFT; | ||||
/* Preset PG_M and PG_A because demotion expects it. */ | /* Preset PG_M and PG_A because demotion expects it. */ | ||||
pdp_p[i] |= X86_PG_RW | X86_PG_V | PG_PS | X86_PG_G | | pdp_p[i] |= X86_PG_RW | X86_PG_V | PG_PS | pg_g | | ||||
X86_PG_M | X86_PG_A | pg_nx; | X86_PG_M | X86_PG_A | pg_nx; | ||||
} | } | ||||
for (j = 0; i < ndmpdp; i++, j++) { | for (j = 0; i < ndmpdp; i++, j++) { | ||||
pdp_p[i] = DMPDphys + ptoa(j); | pdp_p[i] = DMPDphys + ptoa(j); | ||||
pdp_p[i] |= X86_PG_RW | X86_PG_V | PG_U; | pdp_p[i] |= X86_PG_RW | X86_PG_V | PG_U; | ||||
} | } | ||||
/* And recursively map PML4 to itself in order to get PTmap */ | /* And recursively map PML4 to itself in order to get PTmap */ | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | pmap_bootstrap(vm_paddr_t *firstaddr) | ||||
virtual_end = VM_MAX_KERNEL_ADDRESS; | virtual_end = VM_MAX_KERNEL_ADDRESS; | ||||
/* XXX do %cr0 as well */ | /* XXX do %cr0 as well */ | ||||
load_cr4(rcr4() | CR4_PGE); | load_cr4(rcr4() | CR4_PGE); | ||||
load_cr3(KPML4phys); | load_cr3(KPML4phys); | ||||
if (cpu_stdext_feature & CPUID_STDEXT_SMEP) | if (cpu_stdext_feature & CPUID_STDEXT_SMEP) | ||||
load_cr4(rcr4() | CR4_SMEP); | load_cr4(rcr4() | CR4_SMEP); | ||||
if (!pti) | |||||
pg_g = X86_PG_G; | |||||
Done Inline Actionscreate_pagetables() references pg_g before we set it here. markj: create_pagetables() references pg_g before we set it here. | |||||
/* | /* | ||||
* Initialize the kernel pmap (which is statically allocated). | * Initialize the kernel pmap (which is statically allocated). | ||||
*/ | */ | ||||
PMAP_LOCK_INIT(kernel_pmap); | PMAP_LOCK_INIT(kernel_pmap); | ||||
kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); | kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); | ||||
kernel_pmap->pm_cr3 = KPML4phys; | kernel_pmap->pm_cr3 = KPML4phys; | ||||
CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ | CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ | ||||
Show All 28 Lines | #define SYSMAP(c, p, v, n) \ | ||||
* Initialize the PAT MSR. | * Initialize the PAT MSR. | ||||
* pmap_init_pat() clears and sets CR4_PGE, which, as a | * pmap_init_pat() clears and sets CR4_PGE, which, as a | ||||
* side-effect, invalidates stale PG_G TLB entries that might | * side-effect, invalidates stale PG_G TLB entries that might | ||||
* have been created in our pre-boot environment. | * have been created in our pre-boot environment. | ||||
*/ | */ | ||||
pmap_init_pat(); | pmap_init_pat(); | ||||
/* Initialize TLB Context Id. */ | /* Initialize TLB Context Id. */ | ||||
if (pti) | |||||
pmap_pcid_enabled = 0; | |||||
Not Done Inline ActionsAny particular reason for disabling PCID by default if PTI is enabled? xistence_0x58.com: Any particular reason for disabling PCID by default if PTI is enabled? | |||||
Not Done Inline ActionsBecause they do not work together. I plan to work on this after the first patch is finalized. kib: Because they do not work together. I plan to work on this after the first patch is finalized. | |||||
TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled); | TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled); | ||||
if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) { | if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) { | ||||
/* Check for INVPCID support */ | /* Check for INVPCID support */ | ||||
invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) | invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) | ||||
!= 0; | != 0; | ||||
for (i = 0; i < MAXCPU; i++) { | for (i = 0; i < MAXCPU; i++) { | ||||
kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN; | kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN; | ||||
kernel_pmap->pm_pcids[i].pm_gen = 1; | kernel_pmap->pm_pcids[i].pm_gen = 1; | ||||
} | } | ||||
PCPU_SET(pcid_next, PMAP_PCID_KERN + 1); | PCPU_SET(pcid_next, PMAP_PCID_KERN + 1); | ||||
PCPU_SET(pcid_gen, 1); | PCPU_SET(pcid_gen, 1); | ||||
/* | /* | ||||
* pcpu area for APs is zeroed during AP startup. | * pcpu area for APs is zeroed during AP startup. | ||||
* pc_pcid_next and pc_pcid_gen are initialized by AP | * pc_pcid_next and pc_pcid_gen are initialized by AP | ||||
* during pcpu setup. | * during pcpu setup. | ||||
*/ | */ | ||||
load_cr4(rcr4() | CR4_PCIDE); | load_cr4(rcr4() | CR4_PCIDE); | ||||
} else { | |||||
pmap_pcid_enabled = 0; | |||||
Done Inline ActionsDon't we still need this block? What if PTI is disabled and the PCID feature flag is not set? markj: Don't we still need this block? What if PTI is disabled and the PCID feature flag is not set? | |||||
Not Done Inline ActionsYou are right. I do not have non-pcid machines for development. kib: You are right. I do not have non-pcid machines for development. | |||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Setup the PAT MSR. | * Setup the PAT MSR. | ||||
*/ | */ | ||||
void | void | ||||
pmap_init_pat(void) | pmap_init_pat(void) | ||||
▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | printf("PPIM %u: PA=%#lx, VA=%#lx, size=%#lx, mode=%#x\n", i, | ||||
ppim->pa, ppim->va, ppim->sz, ppim->mode); | ppim->pa, ppim->va, ppim->sz, ppim->mode); | ||||
} | } | ||||
mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); | mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); | ||||
error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, | error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, | ||||
(vmem_addr_t *)&qframe); | (vmem_addr_t *)&qframe); | ||||
if (error != 0) | if (error != 0) | ||||
panic("qframe allocation failed"); | panic("qframe allocation failed"); | ||||
pmap_pti_init(); | |||||
} | } | ||||
static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0, | static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0, | ||||
"2MB page mapping counters"); | "2MB page mapping counters"); | ||||
static u_long pmap_pde_demotions; | static u_long pmap_pde_demotions; | ||||
SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD, | SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD, | ||||
&pmap_pde_demotions, 0, "2MB page demotions"); | &pmap_pde_demotions, 0, "2MB page demotions"); | ||||
▲ Show 20 Lines • Show All 528 Lines • ▼ Show 20 Lines | if (pmap == kernel_pmap) { | ||||
} | } | ||||
} else if (pmap == PCPU_GET(curpmap)) { | } else if (pmap == PCPU_GET(curpmap)) { | ||||
if (pmap_pcid_enabled) { | if (pmap_pcid_enabled) { | ||||
if (invpcid_works) { | if (invpcid_works) { | ||||
d.pcid = pmap->pm_pcids[0].pm_pcid; | d.pcid = pmap->pm_pcids[0].pm_pcid; | ||||
d.pad = 0; | d.pad = 0; | ||||
d.addr = 0; | d.addr = 0; | ||||
invpcid(&d, INVPCID_CTX); | invpcid(&d, INVPCID_CTX); | ||||
if (pti) { | |||||
d.pcid |= PMAP_PCID_USER; | |||||
invpcid(&d, INVPCID_CTX); | |||||
} | |||||
} else { | } else { | ||||
load_cr3(pmap->pm_cr3 | pmap->pm_pcids[0]. | load_cr3(pmap->pm_cr3 | pmap->pm_pcids[0]. | ||||
pm_pcid); | pm_pcid); | ||||
if (pti) | |||||
/* XXXKIB */; | |||||
} | } | ||||
} else { | } else { | ||||
invltlb(); | invltlb(); | ||||
} | } | ||||
} else if (pmap_pcid_enabled) { | } else if (pmap_pcid_enabled) { | ||||
pmap->pm_pcids[0].pm_gen = 0; | pmap->pm_pcids[0].pm_gen = 0; | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 266 Lines • ▼ Show 20 Lines | |||||
* Note: not SMP coherent. | * Note: not SMP coherent. | ||||
*/ | */ | ||||
PMAP_INLINE void | PMAP_INLINE void | ||||
pmap_kenter(vm_offset_t va, vm_paddr_t pa) | pmap_kenter(vm_offset_t va, vm_paddr_t pa) | ||||
{ | { | ||||
pt_entry_t *pte; | pt_entry_t *pte; | ||||
pte = vtopte(va); | pte = vtopte(va); | ||||
pte_store(pte, pa | X86_PG_RW | X86_PG_V | X86_PG_G); | pte_store(pte, pa | X86_PG_RW | X86_PG_V | pg_g); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode) | pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode) | ||||
{ | { | ||||
pt_entry_t *pte; | pt_entry_t *pte; | ||||
int cache_bits; | int cache_bits; | ||||
pte = vtopte(va); | pte = vtopte(va); | ||||
cache_bits = pmap_cache_bits(kernel_pmap, mode, 0); | cache_bits = pmap_cache_bits(kernel_pmap, mode, 0); | ||||
pte_store(pte, pa | X86_PG_RW | X86_PG_V | X86_PG_G | cache_bits); | pte_store(pte, pa | X86_PG_RW | X86_PG_V | pg_g | cache_bits); | ||||
} | } | ||||
/* | /* | ||||
* Remove a page from the kernel pagetables. | * Remove a page from the kernel pagetables. | ||||
* Note: not SMP coherent. | * Note: not SMP coherent. | ||||
*/ | */ | ||||
PMAP_INLINE void | PMAP_INLINE void | ||||
pmap_kremove(vm_offset_t va) | pmap_kremove(vm_offset_t va) | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) | ||||
pte = vtopte(sva); | pte = vtopte(sva); | ||||
endpte = pte + count; | endpte = pte + count; | ||||
while (pte < endpte) { | while (pte < endpte) { | ||||
m = *ma++; | m = *ma++; | ||||
cache_bits = pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0); | cache_bits = pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0); | ||||
pa = VM_PAGE_TO_PHYS(m) | cache_bits; | pa = VM_PAGE_TO_PHYS(m) | cache_bits; | ||||
if ((*pte & (PG_FRAME | X86_PG_PTE_CACHE)) != pa) { | if ((*pte & (PG_FRAME | X86_PG_PTE_CACHE)) != pa) { | ||||
oldpte |= *pte; | oldpte |= *pte; | ||||
pte_store(pte, pa | X86_PG_G | X86_PG_RW | X86_PG_V); | pte_store(pte, pa | pg_g | X86_PG_RW | X86_PG_V); | ||||
} | } | ||||
pte++; | pte++; | ||||
} | } | ||||
if (__predict_false((oldpte & X86_PG_V) != 0)) | if (__predict_false((oldpte & X86_PG_V) != 0)) | ||||
pmap_invalidate_range(kernel_pmap, sva, sva + count * | pmap_invalidate_range(kernel_pmap, sva, sva + count * | ||||
PAGE_SIZE); | PAGE_SIZE); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
/* | /* | ||||
* unmap the page table page | * unmap the page table page | ||||
*/ | */ | ||||
if (m->pindex >= (NUPDE + NUPDPE)) { | if (m->pindex >= (NUPDE + NUPDPE)) { | ||||
/* PDP page */ | /* PDP page */ | ||||
pml4_entry_t *pml4; | pml4_entry_t *pml4; | ||||
pml4 = pmap_pml4e(pmap, va); | pml4 = pmap_pml4e(pmap, va); | ||||
*pml4 = 0; | *pml4 = 0; | ||||
if (pti && va <= VM_MAXUSER_ADDRESS) { | |||||
pml4 = &pmap->pm_pml4u[pmap_pml4e_index(va)]; | |||||
*pml4 = 0; | |||||
} | |||||
} else if (m->pindex >= NUPDE) { | } else if (m->pindex >= NUPDE) { | ||||
/* PD page */ | /* PD page */ | ||||
pdp_entry_t *pdp; | pdp_entry_t *pdp; | ||||
pdp = pmap_pdpe(pmap, va); | pdp = pmap_pdpe(pmap, va); | ||||
*pdp = 0; | *pdp = 0; | ||||
} else { | } else { | ||||
/* PTE page */ | /* PTE page */ | ||||
pd_entry_t *pd; | pd_entry_t *pd; | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
pmap_pinit0(pmap_t pmap) | pmap_pinit0(pmap_t pmap) | ||||
{ | { | ||||
int i; | int i; | ||||
PMAP_LOCK_INIT(pmap); | PMAP_LOCK_INIT(pmap); | ||||
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); | pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); | ||||
pmap->pm_pml4u = NULL; | |||||
pmap->pm_cr3 = KPML4phys; | pmap->pm_cr3 = KPML4phys; | ||||
pmap->pm_ucr3 = ~0ULL; | |||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
TAILQ_INIT(&pmap->pm_pvchunk); | TAILQ_INIT(&pmap->pm_pvchunk); | ||||
bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | ||||
pmap->pm_flags = pmap_flags; | pmap->pm_flags = pmap_flags; | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
if (!pti) | |||||
__pcpu[i].pc_kcr3 = ~0ul; | |||||
} | } | ||||
PCPU_SET(curpmap, kernel_pmap); | PCPU_SET(curpmap, kernel_pmap); | ||||
pmap_activate(curthread); | pmap_activate(curthread); | ||||
CPU_FILL(&kernel_pmap->pm_active); | CPU_FILL(&kernel_pmap->pm_active); | ||||
} | } | ||||
void | void | ||||
pmap_pinit_pml4(vm_page_t pml4pg) | pmap_pinit_pml4(vm_page_t pml4pg) | ||||
Show All 13 Lines | pm_pml4[DMPML4I + i] = (DMPDPphys + ptoa(i)) | X86_PG_RW | | ||||
X86_PG_V | PG_U; | X86_PG_V | PG_U; | ||||
} | } | ||||
/* install self-referential address mapping entry(s) */ | /* install self-referential address mapping entry(s) */ | ||||
pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | X86_PG_V | X86_PG_RW | | pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | X86_PG_V | X86_PG_RW | | ||||
X86_PG_A | X86_PG_M; | X86_PG_A | X86_PG_M; | ||||
} | } | ||||
static void | |||||
pmap_pinit_pml4_pti(vm_page_t pml4pg) | |||||
{ | |||||
pml4_entry_t *pm_pml4; | |||||
int i; | |||||
pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg)); | |||||
for (i = 0; i < NPML4EPG; i++) | |||||
pm_pml4[i] = pti_pml4[i]; | |||||
} | |||||
/* | /* | ||||
* Initialize a preallocated and zeroed pmap structure, | * Initialize a preallocated and zeroed pmap structure, | ||||
* such as one in a vmspace structure. | * such as one in a vmspace structure. | ||||
*/ | */ | ||||
int | int | ||||
pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) | pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) | ||||
{ | { | ||||
vm_page_t pml4pg; | vm_page_t pml4pg, pml4pgu; | ||||
vm_paddr_t pml4phys; | vm_paddr_t pml4phys; | ||||
int i; | int i; | ||||
/* | /* | ||||
* allocate the page directory page | * allocate the page directory page | ||||
*/ | */ | ||||
pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); | VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); | ||||
pml4phys = VM_PAGE_TO_PHYS(pml4pg); | pml4phys = VM_PAGE_TO_PHYS(pml4pg); | ||||
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(pml4phys); | pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(pml4phys); | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
} | } | ||||
pmap->pm_cr3 = ~0; /* initialize to an invalid value */ | pmap->pm_cr3 = ~0; /* initialize to an invalid value */ | ||||
pmap->pm_type = pm_type; | |||||
if ((pml4pg->flags & PG_ZERO) == 0) | if ((pml4pg->flags & PG_ZERO) == 0) | ||||
pagezero(pmap->pm_pml4); | pagezero(pmap->pm_pml4); | ||||
/* | /* | ||||
* Do not install the host kernel mappings in the nested page | * Do not install the host kernel mappings in the nested page | ||||
* tables. These mappings are meaningless in the guest physical | * tables. These mappings are meaningless in the guest physical | ||||
* address space. | * address space. | ||||
* Install minimal kernel mappings in PTI case. | |||||
*/ | */ | ||||
if ((pmap->pm_type = pm_type) == PT_X86) { | if (pm_type == PT_X86) { | ||||
pmap->pm_cr3 = pml4phys; | pmap->pm_cr3 = pml4phys; | ||||
pmap_pinit_pml4(pml4pg); | pmap_pinit_pml4(pml4pg); | ||||
if (pti) { | |||||
pml4pgu = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | |||||
VM_ALLOC_NOOBJ | VM_ALLOC_WAITOK); | |||||
Not Done Inline ActionsI can't see why we don't use VM_ALLOC_{ZERO,WIRED} here as we do for the "global" PML4 page. markj: I can't see why we don't use VM_ALLOC_{ZERO,WIRED} here as we do for the "global" PML4 page. | |||||
Not Done Inline ActionsZERO would be a waste since I copy complete pti_pml4 page of pml4e's into the usermode pml4 page. For WIRED, I tihink it is only important to account for the page use in v_wire_count. There it seems that the decrements were missed at all for all pti pagetable pages. kib: ZERO would be a waste since I copy complete pti_pml4 page of pml4e's into the usermode pml4… | |||||
pmap->pm_pml4u = (pml4_entry_t *)PHYS_TO_DMAP( | |||||
VM_PAGE_TO_PHYS(pml4pgu)); | |||||
pmap_pinit_pml4_pti(pml4pgu); | |||||
pmap->pm_ucr3 = VM_PAGE_TO_PHYS(pml4pgu); | |||||
} | } | ||||
} | |||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
TAILQ_INIT(&pmap->pm_pvchunk); | TAILQ_INIT(&pmap->pm_pvchunk); | ||||
bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | ||||
pmap->pm_flags = flags; | pmap->pm_flags = flags; | ||||
pmap->pm_eptgen = 0; | pmap->pm_eptgen = 0; | ||||
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | if ((m->flags & PG_ZERO) == 0) | ||||
pmap_zero_page(m); | pmap_zero_page(m); | ||||
/* | /* | ||||
* Map the pagetable page into the process address space, if | * Map the pagetable page into the process address space, if | ||||
* it isn't already there. | * it isn't already there. | ||||
*/ | */ | ||||
if (ptepindex >= (NUPDE + NUPDPE)) { | if (ptepindex >= (NUPDE + NUPDPE)) { | ||||
pml4_entry_t *pml4; | pml4_entry_t *pml4, *pml4u; | ||||
vm_pindex_t pml4index; | vm_pindex_t pml4index; | ||||
/* Wire up a new PDPE page */ | /* Wire up a new PDPE page */ | ||||
pml4index = ptepindex - (NUPDE + NUPDPE); | pml4index = ptepindex - (NUPDE + NUPDPE); | ||||
pml4 = &pmap->pm_pml4[pml4index]; | pml4 = &pmap->pm_pml4[pml4index]; | ||||
*pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
if (pti && pml4index < NUPML4E) { | |||||
pml4u = &pmap->pm_pml4u[pml4index]; | |||||
*pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | | |||||
PG_A | PG_M; | |||||
} | |||||
} else if (ptepindex >= NUPDE) { | } else if (ptepindex >= NUPDE) { | ||||
vm_pindex_t pml4index; | vm_pindex_t pml4index; | ||||
vm_pindex_t pdpindex; | vm_pindex_t pdpindex; | ||||
pml4_entry_t *pml4; | pml4_entry_t *pml4; | ||||
pdp_entry_t *pdp; | pdp_entry_t *pdp; | ||||
/* Wire up a new PDE page */ | /* Wire up a new PDE page */ | ||||
▲ Show 20 Lines • Show All 184 Lines • ▼ Show 20 Lines | for (i = 0; i < NKPML4E; i++) /* KVA */ | ||||
pmap->pm_pml4[KPML4BASE + i] = 0; | pmap->pm_pml4[KPML4BASE + i] = 0; | ||||
for (i = 0; i < ndmpdpphys; i++)/* Direct Map */ | for (i = 0; i < ndmpdpphys; i++)/* Direct Map */ | ||||
pmap->pm_pml4[DMPML4I + i] = 0; | pmap->pm_pml4[DMPML4I + i] = 0; | ||||
pmap->pm_pml4[PML4PML4I] = 0; /* Recursive Mapping */ | pmap->pm_pml4[PML4PML4I] = 0; /* Recursive Mapping */ | ||||
m->wire_count--; | m->wire_count--; | ||||
atomic_subtract_int(&vm_cnt.v_wire_count, 1); | atomic_subtract_int(&vm_cnt.v_wire_count, 1); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
if (pti && pmap->pm_pml4u != NULL) { | |||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4u)); | |||||
vm_page_free(m); | |||||
} | } | ||||
} | |||||
␌ | ␌ | ||||
static int | static int | ||||
kvm_size(SYSCTL_HANDLER_ARGS) | kvm_size(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; | unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; | ||||
return sysctl_handle_long(oidp, &ksize, 0, req); | return sysctl_handle_long(oidp, &ksize, 0, req); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 4,408 Lines • ▼ Show 20 Lines | if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) { | ||||
PCPU_INC(pm_save_cnt); | PCPU_INC(pm_save_cnt); | ||||
} | } | ||||
PCPU_SET(curpmap, pmap); | PCPU_SET(curpmap, pmap); | ||||
if (!invpcid_works) | if (!invpcid_works) | ||||
intr_restore(rflags); | intr_restore(rflags); | ||||
} else if (cr3 != pmap->pm_cr3) { | } else if (cr3 != pmap->pm_cr3) { | ||||
load_cr3(pmap->pm_cr3); | load_cr3(pmap->pm_cr3); | ||||
PCPU_SET(curpmap, pmap); | PCPU_SET(curpmap, pmap); | ||||
if (pti) { | |||||
PCPU_SET(kcr3, pmap->pm_cr3); | |||||
PCPU_SET(ucr3, pmap->pm_ucr3); | |||||
} | } | ||||
} | |||||
#ifdef SMP | #ifdef SMP | ||||
CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); | CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); | ||||
#else | #else | ||||
CPU_CLR(cpuid, &oldpmap->pm_active); | CPU_CLR(cpuid, &oldpmap->pm_active); | ||||
#endif | #endif | ||||
} | } | ||||
void | void | ||||
▲ Show 20 Lines • Show All 304 Lines • ▼ Show 20 Lines | |||||
pmap_quick_remove_page(vm_offset_t addr) | pmap_quick_remove_page(vm_offset_t addr) | ||||
{ | { | ||||
if (addr != qframe) | if (addr != qframe) | ||||
return; | return; | ||||
pte_store(vtopte(qframe), 0); | pte_store(vtopte(qframe), 0); | ||||
invlpg(qframe); | invlpg(qframe); | ||||
mtx_unlock_spin(&qframe_mtx); | mtx_unlock_spin(&qframe_mtx); | ||||
} | |||||
static vm_page_t | |||||
pmap_pti_alloc_page(void) | |||||
{ | |||||
vm_page_t m; | |||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | |||||
m = vm_page_grab(pti_obj, pti_pg_idx++, VM_ALLOC_NOBUSY | | |||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO); | |||||
return (m); | |||||
} | |||||
extern char kernphys[], etext[], dblfault_stack[], nmi0_stack[]; | |||||
Done Inline ActionsIt seems the latest rev no longer needs the CPU 0 nmi and dblfault stack references. In particular, the symbols can remain local to machdep.c. markj: It seems the latest rev no longer needs the CPU 0 nmi and dblfault stack references. In… | |||||
static void | |||||
pmap_pti_init(void) | |||||
{ | |||||
vm_page_t pml4_pg; | |||||
pdp_entry_t *pdpe; | |||||
vm_offset_t va; | |||||
int i; | |||||
if (!pti) | |||||
return; | |||||
pti_obj = vm_pager_allocate(OBJT_PHYS, NULL, 0, VM_PROT_ALL, 0, NULL); | |||||
VM_OBJECT_WLOCK(pti_obj); | |||||
pml4_pg = pmap_pti_alloc_page(); | |||||
pti_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4_pg)); | |||||
for (va = VM_MIN_KERNEL_ADDRESS; va <= VM_MAX_KERNEL_ADDRESS && | |||||
va >= VM_MIN_KERNEL_ADDRESS && va > NBPML4; va += NBPML4) { | |||||
pdpe = pmap_pti_pdpe(va); | |||||
pmap_pti_wire_pte(pdpe); | |||||
} | |||||
pmap_pti_add_kva_locked((vm_offset_t)&__pcpu[0], | |||||
(vm_offset_t)&__pcpu[0] + sizeof(__pcpu[0]) * MAXCPU, false); | |||||
pmap_pti_add_kva_locked((vm_offset_t)gdt, (vm_offset_t)gdt + NGDT * | |||||
MAXCPU, false); | |||||
pmap_pti_add_kva_locked((vm_offset_t)idt, (vm_offset_t)idt + | |||||
sizeof(struct gate_descriptor) * NIDT, false); | |||||
pmap_pti_add_kva_locked((vm_offset_t)common_tss, | |||||
(vm_offset_t)common_tss + sizeof(struct amd64tss) * MAXCPU, false); | |||||
pmap_pti_add_kva_locked((vm_offset_t)dblfault_stack, | |||||
(vm_offset_t)dblfault_stack + PAGE_SIZE, false); | |||||
pmap_pti_add_kva_locked((vm_offset_t)nmi0_stack, | |||||
(vm_offset_t)nmi0_stack + PAGE_SIZE, false); | |||||
CPU_FOREACH(i) { | |||||
if (i == 0) | |||||
continue; | |||||
va = common_tss[i].tss_ist2 + sizeof(struct nmi_pcpu) - | |||||
PAGE_SIZE; | |||||
pmap_pti_add_kva_locked(va, va + PAGE_SIZE, false); | |||||
} | |||||
pmap_pti_add_kva_locked((vm_offset_t)kernphys + KERNBASE, | |||||
(vm_offset_t)etext, true); | |||||
pti_finalized = true; | |||||
VM_OBJECT_WUNLOCK(pti_obj); | |||||
} | |||||
static pdp_entry_t * | |||||
pmap_pti_pdpe(vm_offset_t va) | |||||
{ | |||||
pml4_entry_t *pml4e; | |||||
pdp_entry_t *pdpe; | |||||
vm_page_t m; | |||||
vm_pindex_t pml4_idx; | |||||
vm_paddr_t mphys; | |||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | |||||
pml4_idx = pmap_pml4e_index(va); | |||||
pml4e = &pti_pml4[pml4_idx]; | |||||
m = NULL; | |||||
if (*pml4e == 0) { | |||||
if (pti_finalized) | |||||
panic("pml4 alloc after finalization\n"); | |||||
m = pmap_pti_alloc_page(); | |||||
if (*pml4e != 0) { | |||||
vm_page_free_zero(m); | |||||
mphys = *pml4e & ~PAGE_MASK; | |||||
} else { | |||||
mphys = VM_PAGE_TO_PHYS(m); | |||||
*pml4e = mphys | X86_PG_RW | X86_PG_V; | |||||
} | |||||
} else { | |||||
mphys = *pml4e & ~PAGE_MASK; | |||||
} | |||||
pdpe = (pdp_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pdpe_index(va); | |||||
return (pdpe); | |||||
} | |||||
static void | |||||
pmap_pti_wire_pte(void *pte) | |||||
{ | |||||
vm_page_t m; | |||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | |||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pte)); | |||||
m->wire_count++; | |||||
} | |||||
static void | |||||
pmap_pti_unwire_pde(void *pde, bool only_ref) | |||||
{ | |||||
vm_page_t m; | |||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | |||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pde)); | |||||
MPASS(m->wire_count > 0); | |||||
MPASS(only_ref || m->wire_count > 1); | |||||
m->wire_count--; | |||||
if (m->wire_count == 0) | |||||
vm_page_free_zero(m); | |||||
} | |||||
static void | |||||
pmap_pti_unwire_pte(void *pte, vm_offset_t va) | |||||
{ | |||||
vm_page_t m; | |||||
pd_entry_t *pde; | |||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | |||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((uintptr_t)pte)); | |||||
MPASS(m->wire_count > 0); | |||||
m->wire_count--; | |||||
if (m->wire_count == 0) { | |||||
vm_page_free_zero(m); | |||||
pde = pmap_pti_pde(va); | |||||
MPASS((*pde & (X86_PG_PS | X86_PG_V)) == X86_PG_V); | |||||
*pde = 0; | |||||
pmap_pti_unwire_pde(pde, false); | |||||
} | |||||
} | |||||
static pd_entry_t * | |||||
pmap_pti_pde(vm_offset_t va) | |||||
{ | |||||
pdp_entry_t *pdpe; | |||||
pd_entry_t *pde; | |||||
vm_page_t m; | |||||
vm_pindex_t pd_idx; | |||||
vm_paddr_t mphys; | |||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | |||||
pdpe = pmap_pti_pdpe(va); | |||||
if (*pdpe == 0) { | |||||
m = pmap_pti_alloc_page(); | |||||
Done Inline ActionsI believe you need to unwire the page before freeing here and at the other levels. markj: I believe you need to unwire the page before freeing here and at the other levels. | |||||
if (*pdpe != 0) { | |||||
vm_page_free_zero(m); | |||||
MPASS((*pdpe & X86_PG_PS) == 0); | |||||
mphys = *pdpe & ~PAGE_MASK; | |||||
} else { | |||||
mphys = VM_PAGE_TO_PHYS(m); | |||||
*pdpe = mphys | X86_PG_RW | X86_PG_V; | |||||
} | |||||
} else { | |||||
MPASS((*pdpe & X86_PG_PS) == 0); | |||||
mphys = *pdpe & ~PAGE_MASK; | |||||
} | |||||
pde = (pd_entry_t *)PHYS_TO_DMAP(mphys); | |||||
pd_idx = pmap_pde_index(va); | |||||
pde += pd_idx; | |||||
return (pde); | |||||
} | |||||
static pt_entry_t * | |||||
pmap_pti_pte(vm_offset_t va, bool *unwire_pde) | |||||
{ | |||||
pd_entry_t *pde; | |||||
pt_entry_t *pte; | |||||
vm_page_t m; | |||||
vm_paddr_t mphys; | |||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | |||||
pde = pmap_pti_pde(va); | |||||
if (unwire_pde != NULL) { | |||||
*unwire_pde = true; | |||||
pmap_pti_wire_pte(pde); | |||||
} | |||||
if (*pde == 0) { | |||||
m = pmap_pti_alloc_page(); | |||||
if (*pde != 0) { | |||||
vm_page_free_zero(m); | |||||
MPASS((*pde & X86_PG_PS) == 0); | |||||
mphys = *pde & ~(PAGE_MASK | pg_nx); | |||||
} else { | |||||
mphys = VM_PAGE_TO_PHYS(m); | |||||
*pde = mphys | X86_PG_RW | X86_PG_V; | |||||
if (unwire_pde != NULL) | |||||
*unwire_pde = false; | |||||
} | |||||
} else { | |||||
MPASS((*pde & X86_PG_PS) == 0); | |||||
mphys = *pde & ~(PAGE_MASK | pg_nx); | |||||
} | |||||
pte = (pt_entry_t *)PHYS_TO_DMAP(mphys); | |||||
pte += pmap_pte_index(va); | |||||
return (pte); | |||||
} | |||||
static void | |||||
pmap_pti_add_kva_locked(vm_offset_t sva, vm_offset_t eva, bool exec) | |||||
{ | |||||
vm_paddr_t pa; | |||||
pd_entry_t *pde; | |||||
pt_entry_t *pte, ptev; | |||||
bool unwire_pde; | |||||
VM_OBJECT_ASSERT_WLOCKED(pti_obj); | |||||
sva = trunc_page(sva); | |||||
MPASS(sva > VM_MAXUSER_ADDRESS); | |||||
eva = round_page(eva); | |||||
MPASS(sva <= eva); | |||||
for (; sva < eva; sva += PAGE_SIZE) { | |||||
pte = pmap_pti_pte(sva, &unwire_pde); | |||||
pa = pmap_kextract(sva); | |||||
ptev = pa | X86_PG_RW | X86_PG_V | X86_PG_A | | |||||
(exec ? 0 : pg_nx) | pmap_cache_bits(kernel_pmap, | |||||
VM_MEMATTR_DEFAULT, FALSE); | |||||
if (*pte == 0) { | |||||
pte_store(pte, ptev); | |||||
pmap_pti_wire_pte(pte); | |||||
} else { | |||||
KASSERT(!pti_finalized, | |||||
("pti overlap after fin %#lx %#lx %#lx", | |||||
sva, *pte, ptev)); | |||||
KASSERT(*pte == ptev, | |||||
("pti non-identical pte after fin %#lx %#lx %#lx", | |||||
sva, *pte, ptev)); | |||||
} | |||||
if (unwire_pde) { | |||||
pde = pmap_pti_pde(sva); | |||||
pmap_pti_unwire_pde(pde, true); | |||||
} | |||||
} | |||||
} | |||||
void | |||||
pmap_pti_add_kva(vm_offset_t sva, vm_offset_t eva, bool exec) | |||||
{ | |||||
if (!pti) | |||||
return; | |||||
VM_OBJECT_WLOCK(pti_obj); | |||||
pmap_pti_add_kva_locked(sva, eva, exec); | |||||
VM_OBJECT_WUNLOCK(pti_obj); | |||||
} | |||||
void | |||||
pmap_pti_remove_kva(vm_offset_t sva, vm_offset_t eva) | |||||
{ | |||||
pt_entry_t *pte; | |||||
vm_size_t len; | |||||
if (!pti) | |||||
return; | |||||
sva = rounddown2(sva, PAGE_SIZE); | |||||
MPASS(sva > VM_MAXUSER_ADDRESS); | |||||
eva = roundup2(eva, PAGE_SIZE); | |||||
len = eva - sva; | |||||
VM_OBJECT_WLOCK(pti_obj); | |||||
for (; sva < eva; sva += PAGE_SIZE) { | |||||
pte = pmap_pti_pte(sva, NULL); | |||||
KASSERT((*pte & X86_PG_V) != 0, | |||||
("invalid pte va %#lx pte %#lx pt %#lx", sva, | |||||
(u_long)pte, *pte)); | |||||
pte_clear(pte); | |||||
pmap_pti_unwire_pte(pte, sva); | |||||
} | |||||
pmap_invalidate_range(kernel_pmap, sva, len); | |||||
VM_OBJECT_WUNLOCK(pti_obj); | |||||
} | } | ||||
#include "opt_ddb.h" | #include "opt_ddb.h" | ||||
#ifdef DDB | #ifdef DDB | ||||
#include <sys/kdb.h> | #include <sys/kdb.h> | ||||
#include <ddb/ddb.h> | #include <ddb/ddb.h> | ||||
DB_SHOW_COMMAND(pte, pmap_print_pte) | DB_SHOW_COMMAND(pte, pmap_print_pte) | ||||
▲ Show 20 Lines • Show All 54 Lines • Show Last 20 Lines |
I would find this whole section more readable with a single 'gflag' variable and a single test for pti;
ie; gflag = pti ? 0 : X86_PG_G;
pt_p[i] = .... | gflag;