Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 430 Lines • ▼ Show 20 Lines | ||||||||||
static uint64_t KASANPDPphys; | static uint64_t KASANPDPphys; | |||||||||
#endif | #endif | |||||||||
static pml4_entry_t *kernel_pml4; | static pml4_entry_t *kernel_pml4; | |||||||||
static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ | static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ | |||||||||
static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ | static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ | |||||||||
static int ndmpdpphys; /* number of DMPDPphys pages */ | static int ndmpdpphys; /* number of DMPDPphys pages */ | |||||||||
static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ | vm_paddr_t kernphys; /* phys addr of start of bootstrap data */ | |||||||||
vm_paddr_t KERNend; /* and the end */ | ||||||||||
/* | /* | |||||||||
* pmap_mapdev support pre initialization (i.e. console) | * pmap_mapdev support pre initialization (i.e. console) | |||||||||
*/ | */ | |||||||||
#define PMAP_PREINIT_MAPPING_COUNT 8 | #define PMAP_PREINIT_MAPPING_COUNT 8 | |||||||||
static struct pmap_preinit_mapping { | static struct pmap_preinit_mapping { | |||||||||
vm_paddr_t pa; | vm_paddr_t pa; | |||||||||
vm_offset_t va; | vm_offset_t va; | |||||||||
▲ Show 20 Lines • Show All 1,101 Lines • ▼ Show 20 Lines | ||||||||||
static void | static void | |||||||||
nkpt_init(vm_paddr_t addr) | nkpt_init(vm_paddr_t addr) | |||||||||
{ | { | |||||||||
int pt_pages; | int pt_pages; | |||||||||
#ifdef NKPT | #ifdef NKPT | |||||||||
pt_pages = NKPT; | pt_pages = NKPT; | |||||||||
#else | #else | |||||||||
pt_pages = howmany(addr, NBPDR); | pt_pages = howmany(addr - kernphys, NBPDR) + 1; /* +1 for 2M hole @0 */ | |||||||||
pt_pages += NKPDPE(pt_pages); | pt_pages += NKPDPE(pt_pages); | |||||||||
/* | /* | |||||||||
* Add some slop beyond the bare minimum required for bootstrapping | * Add some slop beyond the bare minimum required for bootstrapping | |||||||||
* the kernel. | * the kernel. | |||||||||
* | * | |||||||||
* This is quite important when allocating KVA for kernel modules. | * This is quite important when allocating KVA for kernel modules. | |||||||||
* The modules are required to be linked in the negative 2GB of | * The modules are required to be linked in the negative 2GB of | |||||||||
Show All 23 Lines | ||||||||||
* If the page is below/above the kernel range, it is marked as read-write. | * If the page is below/above the kernel range, it is marked as read-write. | |||||||||
* | * | |||||||||
* This function operates on 2M pages, since we map the kernel space that | * This function operates on 2M pages, since we map the kernel space that | |||||||||
* way. | * way. | |||||||||
*/ | */ | |||||||||
static inline pt_entry_t | static inline pt_entry_t | |||||||||
bootaddr_rwx(vm_paddr_t pa) | bootaddr_rwx(vm_paddr_t pa) | |||||||||
{ | { | |||||||||
vm_paddr_t fixed_kernphys; | ||||||||||
fixed_kernphys = kernphys - NBPDR; /* KERNBASE is off by NBPDR */ | ||||||||||
/* | /* | |||||||||
* The kernel is loaded at a 2MB-aligned address, and memory below that | * The kernel is loaded at a 2MB-aligned address, and memory below that | |||||||||
* need not be executable. The .bss section is padded to a 2MB | * need not be executable. The .bss section is padded to a 2MB | |||||||||
* boundary, so memory following the kernel need not be executable | * boundary, so memory following the kernel need not be executable | |||||||||
* either. Preloaded kernel modules have their mapping permissions | * either. Preloaded kernel modules have their mapping permissions | |||||||||
* fixed up by the linker. | * fixed up by the linker. | |||||||||
*/ | */ | |||||||||
if (pa < trunc_2mpage(btext - KERNBASE) || | if (pa < trunc_2mpage(fixed_kernphys + btext - KERNBASE) || | |||||||||
pa >= trunc_2mpage(_end - KERNBASE)) | pa >= trunc_2mpage(fixed_kernphys + _end - KERNBASE)) | |||||||||
return (X86_PG_RW | pg_nx); | return (X86_PG_RW | pg_nx); | |||||||||
/* | /* | |||||||||
* The linker should ensure that the read-only and read-write | * The linker should ensure that the read-only and read-write | |||||||||
* portions don't share the same 2M page, so this shouldn't | * portions don't share the same 2M page, so this shouldn't | |||||||||
* impact read-only data. However, in any case, any page with | * impact read-only data. However, in any case, any page with | |||||||||
* read-write data needs to be read-write. | * read-write data needs to be read-write. | |||||||||
*/ | */ | |||||||||
if (pa >= trunc_2mpage(brwsection - KERNBASE)) | if (pa >= trunc_2mpage(fixed_kernphys + brwsection - KERNBASE)) | |||||||||
return (X86_PG_RW | pg_nx); | return (X86_PG_RW | pg_nx); | |||||||||
/* | /* | |||||||||
* Mark any 2M page containing kernel text as read-only. Mark | * Mark any 2M page containing kernel text as read-only. Mark | |||||||||
* other pages with read-only data as read-only and not executable. | * other pages with read-only data as read-only and not executable. | |||||||||
* (It is likely a small portion of the read-only data section will | * (It is likely a small portion of the read-only data section will | |||||||||
* be marked as read-only, but executable. This should be acceptable | * be marked as read-only, but executable. This should be acceptable | |||||||||
* since the read-only protection will keep the data from changing.) | * since the read-only protection will keep the data from changing.) | |||||||||
* Note that fixups to the .text section will still work until we | * Note that fixups to the .text section will still work until we | |||||||||
* set CR0.WP. | * set CR0.WP. | |||||||||
*/ | */ | |||||||||
if (pa < round_2mpage(etext - KERNBASE)) | if (pa < round_2mpage(fixed_kernphys + etext - KERNBASE)) | |||||||||
return (0); | return (0); | |||||||||
return (pg_nx); | return (pg_nx); | |||||||||
} | } | |||||||||
static void | static void | |||||||||
create_pagetables(vm_paddr_t *firstaddr) | create_pagetables(vm_paddr_t *firstaddr) | |||||||||
{ | { | |||||||||
pd_entry_t *pd_p; | pd_entry_t *pd_p; | |||||||||
pdp_entry_t *pdp_p; | pdp_entry_t *pdp_p; | |||||||||
pml4_entry_t *p4_p; | pml4_entry_t *p4_p; | |||||||||
uint64_t DMPDkernphys; | uint64_t DMPDkernphys; | |||||||||
vm_paddr_t pax; | ||||||||||
#ifdef KASAN | #ifdef KASAN | |||||||||
pt_entry_t *pt_p; | pt_entry_t *pt_p; | |||||||||
uint64_t KASANPDphys, KASANPTphys, KASANphys; | uint64_t KASANPDphys, KASANPTphys, KASANphys; | |||||||||
vm_offset_t kasankernbase; | vm_offset_t kasankernbase; | |||||||||
int kasankpdpi, kasankpdi, nkasanpte; | int kasankpdpi, kasankpdi, nkasanpte; | |||||||||
#endif | #endif | |||||||||
int i, j, ndm1g, nkpdpe, nkdmpde; | int i, j, ndm1g, nkpdpe, nkdmpde; | |||||||||
Show All 18 Lines | if ((amd_feature & AMDID_PAGE1GB) != 0) { | |||||||||
/* | /* | |||||||||
* Calculate the number of 1G pages that will fully fit in | * Calculate the number of 1G pages that will fully fit in | |||||||||
* Maxmem. | * Maxmem. | |||||||||
*/ | */ | |||||||||
ndm1g = ptoa(Maxmem) >> PDPSHIFT; | ndm1g = ptoa(Maxmem) >> PDPSHIFT; | |||||||||
/* | /* | |||||||||
* Allocate 2M pages for the kernel. These will be used in | * Allocate 2M pages for the kernel. These will be used in | |||||||||
* place of the first one or more 1G pages from ndm1g. | * place of the one or more 1G pages from ndm1g that maps | |||||||||
* kernel memory into DMAP. | ||||||||||
*/ | */ | |||||||||
nkdmpde = howmany((vm_offset_t)(brwsection - KERNBASE), NBPDP); | nkdmpde = howmany((vm_offset_t)brwsection - KERNBASE - NBPDR + | |||||||||
kernphys - rounddown2(kernphys, NBPDP), NBPDP); | ||||||||||
DMPDkernphys = allocpages(firstaddr, nkdmpde); | DMPDkernphys = allocpages(firstaddr, nkdmpde); | |||||||||
} | } | |||||||||
if (ndm1g < ndmpdp) | if (ndm1g < ndmpdp) | |||||||||
DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g); | DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g); | |||||||||
dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT; | dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT; | |||||||||
/* Allocate pages */ | /* Allocate pages */ | |||||||||
KPML4phys = allocpages(firstaddr, 1); | KPML4phys = allocpages(firstaddr, 1); | |||||||||
Show All 30 Lines | #endif | |||||||||
* implicitly maps the PT pages at their correct locations within | * implicitly maps the PT pages at their correct locations within | |||||||||
* the PTmap. | * the PTmap. | |||||||||
*/ | */ | |||||||||
pd_p = (pd_entry_t *)KPDphys; | pd_p = (pd_entry_t *)KPDphys; | |||||||||
for (i = 0; i < nkpt; i++) | for (i = 0; i < nkpt; i++) | |||||||||
pd_p[i] = (KPTphys + ptoa(i)) | X86_PG_RW | X86_PG_V; | pd_p[i] = (KPTphys + ptoa(i)) | X86_PG_RW | X86_PG_V; | |||||||||
/* | /* | |||||||||
* Map from physical address zero to the end of loader preallocated | * Map from start of the kernel in physical memory (staging | |||||||||
* memory using 2MB pages. This replaces some of the PD entries | * area) to the end of loader preallocated memory using 2MB | |||||||||
* created above. | * pages. This replaces some of the PD entries created above. | |||||||||
* For compatibility, identity map 2M at the start. | ||||||||||
*/ | */ | |||||||||
for (i = 0; (i << PDRSHIFT) < KERNend; i++) | pd_p[0] = X86_PG_V | PG_PS | pg_g | X86_PG_M | X86_PG_A | | |||||||||
X86_PG_RW | pg_nx; | ||||||||||
for (i = 1, pax = kernphys; pax < KERNend; i++, pax += NBPDR) { | ||||||||||
/* Preset PG_M and PG_A because demotion expects it. */ | /* Preset PG_M and PG_A because demotion expects it. */ | |||||||||
pd_p[i] = (i << PDRSHIFT) | X86_PG_V | PG_PS | pg_g | | pd_p[i] = pax | X86_PG_V | PG_PS | pg_g | X86_PG_M | | |||||||||
X86_PG_M | X86_PG_A | bootaddr_rwx(i << PDRSHIFT); | X86_PG_A | bootaddr_rwx(pax); | |||||||||
} | ||||||||||
/* | /* | |||||||||
* Because we map the physical blocks in 2M pages, adjust firstaddr | * Because we map the physical blocks in 2M pages, adjust firstaddr | |||||||||
* to record the physical blocks we've actually mapped into kernel | * to record the physical blocks we've actually mapped into kernel | |||||||||
* virtual address space. | * virtual address space. | |||||||||
*/ | */ | |||||||||
if (*firstaddr < round_2mpage(KERNend)) | if (*firstaddr < round_2mpage(KERNend)) | |||||||||
*firstaddr = round_2mpage(KERNend); | *firstaddr = round_2mpage(KERNend); | |||||||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | for (j = 0; i < ndmpdp; i++, j++) { | |||||||||
pdp_p[i] |= X86_PG_RW | X86_PG_V | pg_nx; | pdp_p[i] |= X86_PG_RW | X86_PG_V | pg_nx; | |||||||||
} | } | |||||||||
/* | /* | |||||||||
* Instead of using a 1G page for the memory containing the kernel, | * Instead of using a 1G page for the memory containing the kernel, | |||||||||
* use 2M pages with read-only and no-execute permissions. (If using 1G | * use 2M pages with read-only and no-execute permissions. (If using 1G | |||||||||
* pages, this will partially overwrite the PDPEs above.) | * pages, this will partially overwrite the PDPEs above.) | |||||||||
*/ | */ | |||||||||
if (ndm1g) { | if (ndm1g > 0) { | |||||||||
pd_p = (pd_entry_t *)DMPDkernphys; | pd_p = (pd_entry_t *)DMPDkernphys; | |||||||||
for (i = 0; i < (NPDEPG * nkdmpde); i++) | for (i = 0, pax = rounddown2(kernphys, NBPDP); | |||||||||
pd_p[i] = (i << PDRSHIFT) | X86_PG_V | PG_PS | pg_g | | i < NPDEPG * nkdmpde; i++, pax += NBPDR) { | |||||||||
X86_PG_M | X86_PG_A | pg_nx | | pd_p[i] = pax | X86_PG_V | PG_PS | pg_g | X86_PG_M | | |||||||||
bootaddr_rwx(i << PDRSHIFT); | X86_PG_A | pg_nx | bootaddr_rwx(pax); | |||||||||
for (i = 0; i < nkdmpde; i++) | ||||||||||
pdp_p[i] = (DMPDkernphys + ptoa(i)) | X86_PG_RW | | ||||||||||
X86_PG_V | pg_nx; | ||||||||||
} | } | |||||||||
j = rounddown2(kernphys, NBPDP) >> PDPSHIFT; | ||||||||||
markjUnsubmitted Done Inline Actions
markj: | ||||||||||
for (i = 0; i < nkdmpde; i++) { | ||||||||||
pdp_p[i + j] = (DMPDkernphys + ptoa(i)) | | ||||||||||
X86_PG_RW | X86_PG_V | pg_nx; | ||||||||||
} | ||||||||||
} | ||||||||||
/* And recursively map PML4 to itself in order to get PTmap */ | /* And recursively map PML4 to itself in order to get PTmap */ | |||||||||
p4_p = (pml4_entry_t *)KPML4phys; | p4_p = (pml4_entry_t *)KPML4phys; | |||||||||
p4_p[PML4PML4I] = KPML4phys; | p4_p[PML4PML4I] = KPML4phys; | |||||||||
p4_p[PML4PML4I] |= X86_PG_RW | X86_PG_V | pg_nx; | p4_p[PML4PML4I] |= X86_PG_RW | X86_PG_V | pg_nx; | |||||||||
#ifdef KASAN | #ifdef KASAN | |||||||||
/* Connect the KASAN shadow map slots up to the PML4. */ | /* Connect the KASAN shadow map slots up to the PML4. */ | |||||||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | pmap_bootstrap(vm_paddr_t *firstaddr) | |||||||||
* are required for promotion of the corresponding kernel virtual | * are required for promotion of the corresponding kernel virtual | |||||||||
* addresses to superpage mappings. | * addresses to superpage mappings. | |||||||||
*/ | */ | |||||||||
vm_phys_early_add_seg(KPTphys, KPTphys + ptoa(nkpt)); | vm_phys_early_add_seg(KPTphys, KPTphys + ptoa(nkpt)); | |||||||||
/* | /* | |||||||||
* Account for the virtual addresses mapped by create_pagetables(). | * Account for the virtual addresses mapped by create_pagetables(). | |||||||||
*/ | */ | |||||||||
virtual_avail = (vm_offset_t)KERNBASE + round_2mpage(KERNend); | virtual_avail = (vm_offset_t)KERNBASE + round_2mpage(KERNend - | |||||||||
(vm_paddr_t)kernphys + NBPDR); | ||||||||||
virtual_end = VM_MAX_KERNEL_ADDRESS; | virtual_end = VM_MAX_KERNEL_ADDRESS; | |||||||||
/* | /* | |||||||||
* Enable PG_G global pages, then switch to the kernel page | * Enable PG_G global pages, then switch to the kernel page | |||||||||
* table from the bootstrap page table. After the switch, it | * table from the bootstrap page table. After the switch, it | |||||||||
* is possible to enable SMEP and SMAP since PG_U bits are | * is possible to enable SMEP and SMAP since PG_U bits are | |||||||||
* correct now. | * correct now. | |||||||||
*/ | */ | |||||||||
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines | pmap_init_pat(void) | |||||||||
wbinvd(); | wbinvd(); | |||||||||
invltlb(); | invltlb(); | |||||||||
/* Restore caches and PGE. */ | /* Restore caches and PGE. */ | |||||||||
load_cr0(cr0); | load_cr0(cr0); | |||||||||
load_cr4(cr4); | load_cr4(cr4); | |||||||||
} | } | |||||||||
vm_page_t | ||||||||||
pmap_page_alloc_below_4g(bool zeroed) | ||||||||||
{ | ||||||||||
vm_page_t m; | ||||||||||
m = vm_page_alloc_contig(NULL, 0, (zeroed ? VM_ALLOC_ZERO : 0) | | ||||||||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ, | ||||||||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||||||||
if (m != NULL && zeroed && (m->flags & PG_ZERO) == 0) | ||||||||||
Done Inline ActionsShould be conditional on zeroed too, otherwise we might as well not specify zeroed at all. markj: Should be conditional on `zeroed` too, otherwise we might as well not specify `zeroed` at all. | ||||||||||
pmap_zero_page(m); | ||||||||||
return (m); | ||||||||||
} | ||||||||||
extern const char la57_trampoline[], la57_trampoline_gdt_desc[], | extern const char la57_trampoline[], la57_trampoline_gdt_desc[], | |||||||||
la57_trampoline_gdt[], la57_trampoline_end[]; | la57_trampoline_gdt[], la57_trampoline_end[]; | |||||||||
static void | static void | |||||||||
pmap_bootstrap_la57(void *arg __unused) | pmap_bootstrap_la57(void *arg __unused) | |||||||||
{ | { | |||||||||
char *v_code; | char *v_code; | |||||||||
pml5_entry_t *v_pml5; | pml5_entry_t *v_pml5; | |||||||||
Show All 9 Lines | if ((cpu_stdext_feature2 & CPUID_STDEXT2_LA57) == 0) | |||||||||
return; | return; | |||||||||
TUNABLE_INT_FETCH("vm.pmap.la57", &la57); | TUNABLE_INT_FETCH("vm.pmap.la57", &la57); | |||||||||
if (!la57) | if (!la57) | |||||||||
return; | return; | |||||||||
r_gdt.rd_limit = NGDT * sizeof(struct user_segment_descriptor) - 1; | r_gdt.rd_limit = NGDT * sizeof(struct user_segment_descriptor) - 1; | |||||||||
r_gdt.rd_base = (long)__pcpu[0].pc_gdt; | r_gdt.rd_base = (long)__pcpu[0].pc_gdt; | |||||||||
m_code = vm_page_alloc_contig(NULL, 0, | m_code = pmap_page_alloc_below_4g(true); | |||||||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||||||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||||||||
if ((m_code->flags & PG_ZERO) == 0) | ||||||||||
pmap_zero_page(m_code); | ||||||||||
v_code = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_code)); | v_code = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_code)); | |||||||||
m_pml5 = vm_page_alloc_contig(NULL, 0, | m_pml5 = pmap_page_alloc_below_4g(true); | |||||||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||||||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||||||||
if ((m_pml5->flags & PG_ZERO) == 0) | ||||||||||
pmap_zero_page(m_pml5); | ||||||||||
KPML5phys = VM_PAGE_TO_PHYS(m_pml5); | KPML5phys = VM_PAGE_TO_PHYS(m_pml5); | |||||||||
v_pml5 = (pml5_entry_t *)PHYS_TO_DMAP(KPML5phys); | v_pml5 = (pml5_entry_t *)PHYS_TO_DMAP(KPML5phys); | |||||||||
m_pml4 = vm_page_alloc_contig(NULL, 0, | m_pml4 = pmap_page_alloc_below_4g(true); | |||||||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||||||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||||||||
if ((m_pml4->flags & PG_ZERO) == 0) | ||||||||||
pmap_zero_page(m_pml4); | ||||||||||
v_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pml4)); | v_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pml4)); | |||||||||
m_pdp = vm_page_alloc_contig(NULL, 0, | m_pdp = pmap_page_alloc_below_4g(true); | |||||||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||||||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||||||||
if ((m_pdp->flags & PG_ZERO) == 0) | ||||||||||
pmap_zero_page(m_pdp); | ||||||||||
v_pdp = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pdp)); | v_pdp = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pdp)); | |||||||||
m_pd = vm_page_alloc_contig(NULL, 0, | m_pd = pmap_page_alloc_below_4g(true); | |||||||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||||||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||||||||
if ((m_pd->flags & PG_ZERO) == 0) | ||||||||||
pmap_zero_page(m_pd); | ||||||||||
v_pd = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pd)); | v_pd = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pd)); | |||||||||
m_pt = vm_page_alloc_contig(NULL, 0, | m_pt = pmap_page_alloc_below_4g(true); | |||||||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||||||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||||||||
if ((m_pt->flags & PG_ZERO) == 0) | ||||||||||
pmap_zero_page(m_pt); | ||||||||||
v_pt = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pt)); | v_pt = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pt)); | |||||||||
/* | /* | |||||||||
* Map m_code 1:1, it appears below 4G in KVA due to physical | * Map m_code 1:1, it appears below 4G in KVA due to physical | |||||||||
* address being below 4G. Since kernel KVA is in upper half, | * address being below 4G. Since kernel KVA is in upper half, | |||||||||
* the pml4e should be zero and free for temporary use. | * the pml4e should be zero and free for temporary use. | |||||||||
*/ | */ | |||||||||
kernel_pmap->pm_pmltop[pmap_pml4e_index(VM_PAGE_TO_PHYS(m_code))] = | kernel_pmap->pm_pmltop[pmap_pml4e_index(VM_PAGE_TO_PHYS(m_code))] = | |||||||||
▲ Show 20 Lines • Show All 286 Lines • ▼ Show 20 Lines | for (i = 0; i < nkpt; i++) { | |||||||||
mpte->pindex = pmap_pde_pindex(KERNBASE) + i; | mpte->pindex = pmap_pde_pindex(KERNBASE) + i; | |||||||||
mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); | mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); | |||||||||
mpte->ref_count = 1; | mpte->ref_count = 1; | |||||||||
/* | /* | |||||||||
* Collect the page table pages that were replaced by a 2MB | * Collect the page table pages that were replaced by a 2MB | |||||||||
* page in create_pagetables(). They are zero filled. | * page in create_pagetables(). They are zero filled. | |||||||||
*/ | */ | |||||||||
if ((vm_paddr_t)i << PDRSHIFT < KERNend && | if ((i == 0 || | |||||||||
kernphys + ((vm_paddr_t)(i - 1) << PDRSHIFT) < KERNend) && | ||||||||||
pmap_insert_pt_page(kernel_pmap, mpte, false)) | pmap_insert_pt_page(kernel_pmap, mpte, false)) | |||||||||
panic("pmap_init: pmap_insert_pt_page failed"); | panic("pmap_init: pmap_insert_pt_page failed"); | |||||||||
} | } | |||||||||
PMAP_UNLOCK(kernel_pmap); | PMAP_UNLOCK(kernel_pmap); | |||||||||
vm_wire_add(nkpt); | vm_wire_add(nkpt); | |||||||||
/* | /* | |||||||||
* If the kernel is running on a virtual machine, then it must assume | * If the kernel is running on a virtual machine, then it must assume | |||||||||
▲ Show 20 Lines • Show All 4,250 Lines • ▼ Show 20 Lines | setpte: | |||||||||
* mapping the superpage is demoted by pmap_demote_pde() or | * mapping the superpage is demoted by pmap_demote_pde() or | |||||||||
* destroyed by pmap_remove_pde(). | * destroyed by pmap_remove_pde(). | |||||||||
*/ | */ | |||||||||
mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | |||||||||
KASSERT(mpte >= vm_page_array && | KASSERT(mpte >= vm_page_array && | |||||||||
mpte < &vm_page_array[vm_page_array_size], | mpte < &vm_page_array[vm_page_array_size], | |||||||||
("pmap_promote_pde: page table page is out of range")); | ("pmap_promote_pde: page table page is out of range")); | |||||||||
KASSERT(mpte->pindex == pmap_pde_pindex(va), | KASSERT(mpte->pindex == pmap_pde_pindex(va), | |||||||||
("pmap_promote_pde: page table page's pindex is wrong")); | ("pmap_promote_pde: page table page's pindex is wrong " | |||||||||
"mpte %p pidx %#lx va %#lx va pde pidx %#lx", | ||||||||||
mpte, mpte->pindex, va, pmap_pde_pindex(va))); | ||||||||||
if (pmap_insert_pt_page(pmap, mpte, true)) { | if (pmap_insert_pt_page(pmap, mpte, true)) { | |||||||||
counter_u64_add(pmap_pde_p_failures, 1); | counter_u64_add(pmap_pde_p_failures, 1); | |||||||||
CTR2(KTR_PMAP, | CTR2(KTR_PMAP, | |||||||||
"pmap_promote_pde: failure for va %#lx in pmap %p", va, | "pmap_promote_pde: failure for va %#lx in pmap %p", va, | |||||||||
pmap); | pmap); | |||||||||
return; | return; | |||||||||
} | } | |||||||||
▲ Show 20 Lines • Show All 4,054 Lines • ▼ Show 20 Lines | CPU_FOREACH(i) { | |||||||||
/* MC# stack IST 3 */ | /* MC# stack IST 3 */ | |||||||||
va = __pcpu[i].pc_common_tss.tss_ist3 + | va = __pcpu[i].pc_common_tss.tss_ist3 + | |||||||||
sizeof(struct nmi_pcpu); | sizeof(struct nmi_pcpu); | |||||||||
pmap_pti_add_kva_locked(va - MCE_STACK_SIZE, va, false); | pmap_pti_add_kva_locked(va - MCE_STACK_SIZE, va, false); | |||||||||
/* DB# stack IST 4 */ | /* DB# stack IST 4 */ | |||||||||
va = __pcpu[i].pc_common_tss.tss_ist4 + sizeof(struct nmi_pcpu); | va = __pcpu[i].pc_common_tss.tss_ist4 + sizeof(struct nmi_pcpu); | |||||||||
pmap_pti_add_kva_locked(va - DBG_STACK_SIZE, va, false); | pmap_pti_add_kva_locked(va - DBG_STACK_SIZE, va, false); | |||||||||
} | } | |||||||||
pmap_pti_add_kva_locked((vm_offset_t)kernphys + KERNBASE, | pmap_pti_add_kva_locked((vm_offset_t)KERNBASE + NBPDR, | |||||||||
(vm_offset_t)etext, true); | (vm_offset_t)etext, true); | |||||||||
pti_finalized = true; | pti_finalized = true; | |||||||||
VM_OBJECT_WUNLOCK(pti_obj); | VM_OBJECT_WUNLOCK(pti_obj); | |||||||||
} | } | |||||||||
SYSINIT(pmap_pti, SI_SUB_CPU + 1, SI_ORDER_ANY, pmap_pti_init, NULL); | SYSINIT(pmap_pti, SI_SUB_CPU + 1, SI_ORDER_ANY, pmap_pti_init, NULL); | |||||||||
static pdp_entry_t * | static pdp_entry_t * | |||||||||
pmap_pti_pdpe(vm_offset_t va) | pmap_pti_pdpe(vm_offset_t va) | |||||||||
▲ Show 20 Lines • Show All 1,013 Lines • Show Last 20 Lines |