Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 2,020 Lines • ▼ Show 20 Lines | if (!la57) | ||||
return; | return; | ||||
r_gdt.rd_limit = NGDT * sizeof(struct user_segment_descriptor) - 1; | r_gdt.rd_limit = NGDT * sizeof(struct user_segment_descriptor) - 1; | ||||
r_gdt.rd_base = (long)__pcpu[0].pc_gdt; | r_gdt.rd_base = (long)__pcpu[0].pc_gdt; | ||||
m_code = vm_page_alloc_contig(NULL, 0, | m_code = vm_page_alloc_contig(NULL, 0, | ||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | 1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||
if ((m_code->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_code); | |||||
v_code = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_code)); | v_code = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_code)); | ||||
m_pml5 = vm_page_alloc_contig(NULL, 0, | m_pml5 = vm_page_alloc_contig(NULL, 0, | ||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | 1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||
if ((m_pml5->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pml5); | |||||
KPML5phys = VM_PAGE_TO_PHYS(m_pml5); | KPML5phys = VM_PAGE_TO_PHYS(m_pml5); | ||||
v_pml5 = (pml5_entry_t *)PHYS_TO_DMAP(KPML5phys); | v_pml5 = (pml5_entry_t *)PHYS_TO_DMAP(KPML5phys); | ||||
m_pml4 = vm_page_alloc_contig(NULL, 0, | m_pml4 = vm_page_alloc_contig(NULL, 0, | ||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | 1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||
if ((m_pml4->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pml4); | |||||
v_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pml4)); | v_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pml4)); | ||||
m_pdp = vm_page_alloc_contig(NULL, 0, | m_pdp = vm_page_alloc_contig(NULL, 0, | ||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | 1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||
if ((m_pdp->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pdp); | |||||
v_pdp = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pdp)); | v_pdp = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pdp)); | ||||
m_pd = vm_page_alloc_contig(NULL, 0, | m_pd = vm_page_alloc_contig(NULL, 0, | ||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | 1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||
if ((m_pd->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pd); | |||||
v_pd = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pd)); | v_pd = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pd)); | ||||
m_pt = vm_page_alloc_contig(NULL, 0, | m_pt = vm_page_alloc_contig(NULL, 0, | ||||
VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ, | ||||
1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | 1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); | ||||
if ((m_pt->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m_pt); | |||||
v_pt = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pt)); | v_pt = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m_pt)); | ||||
/* | /* | ||||
* Map m_code 1:1, it appears below 4G in KVA due to physical | * Map m_code 1:1, it appears below 4G in KVA due to physical | ||||
* address being below 4G. Since kernel KVA is in upper half, | * address being below 4G. Since kernel KVA is in upper half, | ||||
* the pml4e should be zero and free for temporary use. | * the pml4e should be zero and free for temporary use. | ||||
*/ | */ | ||||
kernel_pmap->pm_pmltop[pmap_pml4e_index(VM_PAGE_TO_PHYS(m_code))] = | kernel_pmap->pm_pmltop[pmap_pml4e_index(VM_PAGE_TO_PHYS(m_code))] = | ||||
▲ Show 20 Lines • Show All 2,133 Lines • ▼ Show 20 Lines | CPU_FOREACH(i) { | ||||
pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; | ||||
pmap->pm_pcids[i].pm_gen = 0; | pmap->pm_pcids[i].pm_gen = 0; | ||||
} | } | ||||
pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */ | pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */ | ||||
pmap->pm_ucr3 = PMAP_NO_CR3; | pmap->pm_ucr3 = PMAP_NO_CR3; | ||||
pmap->pm_pmltopu = NULL; | pmap->pm_pmltopu = NULL; | ||||
pmap->pm_type = pm_type; | pmap->pm_type = pm_type; | ||||
if ((pmltop_pg->flags & PG_ZERO) == 0) | |||||
pagezero(pmap->pm_pmltop); | |||||
/* | /* | ||||
* Do not install the host kernel mappings in the nested page | * Do not install the host kernel mappings in the nested page | ||||
* tables. These mappings are meaningless in the guest physical | * tables. These mappings are meaningless in the guest physical | ||||
* address space. | * address space. | ||||
* Install minimal kernel mappings in PTI case. | * Install minimal kernel mappings in PTI case. | ||||
*/ | */ | ||||
switch (pm_type) { | switch (pm_type) { | ||||
▲ Show 20 Lines • Show All 188 Lines • ▼ Show 20 Lines | pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp, | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
/* | /* | ||||
* Allocate a page table page. | * Allocate a page table page. | ||||
*/ | */ | ||||
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) | ||||
return (NULL); | return (NULL); | ||||
if ((m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m); | |||||
/* | /* | ||||
* Map the pagetable page into the process address space, if | * Map the pagetable page into the process address space, if | ||||
* it isn't already there. | * it isn't already there. | ||||
*/ | */ | ||||
if (ptepindex >= NUPDE + NUPDPE + NUPML4E) { | if (ptepindex >= NUPDE + NUPDPE + NUPML4E) { | ||||
MPASS(pmap_is_la57(pmap)); | MPASS(pmap_is_la57(pmap)); | ||||
▲ Show 20 Lines • Show All 360 Lines • ▼ Show 20 Lines | while (kernel_vm_end < addr) { | ||||
pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end); | pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end); | ||||
if ((*pdpe & X86_PG_V) == 0) { | if ((*pdpe & X86_PG_V) == 0) { | ||||
/* We need a new PDP entry */ | /* We need a new PDP entry */ | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDPSHIFT, | nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDPSHIFT, | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | | VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO); | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(nkpg); | |||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
*pdpe = (pdp_entry_t)(paddr | X86_PG_V | X86_PG_RW | | *pdpe = (pdp_entry_t)(paddr | X86_PG_V | X86_PG_RW | | ||||
X86_PG_A | X86_PG_M); | X86_PG_A | X86_PG_M); | ||||
continue; /* try again */ | continue; /* try again */ | ||||
} | } | ||||
pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end); | pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end); | ||||
if ((*pde & X86_PG_V) != 0) { | if ((*pde & X86_PG_V) != 0) { | ||||
kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; | kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
break; | break; | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end), | nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end), | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | ||||
VM_ALLOC_ZERO); | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(nkpg); | |||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
newpdir = paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M; | newpdir = paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M; | ||||
pde_store(pde, newpdir); | pde_store(pde, newpdir); | ||||
kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; | kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
break; | break; | ||||
▲ Show 20 Lines • Show All 5,252 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* In contrast, pd and pt pages are managed like user page table | * In contrast, pd and pt pages are managed like user page table | ||||
* pages. They are dynamically allocated, and their reference count | * pages. They are dynamically allocated, and their reference count | ||||
* represents the number of valid entries within the page. | * represents the number of valid entries within the page. | ||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
pmap_large_map_getptp_unlocked(void) | pmap_large_map_getptp_unlocked(void) | ||||
{ | { | ||||
vm_page_t m; | return (vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | ||||
VM_ALLOC_ZERO)); | |||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | |||||
VM_ALLOC_ZERO); | |||||
if (m != NULL && (m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m); | |||||
return (m); | |||||
} | } | ||||
static vm_page_t | static vm_page_t | ||||
pmap_large_map_getptp(void) | pmap_large_map_getptp(void) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); | PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); | ||||
▲ Show 20 Lines • Show All 1,469 Lines • Show Last 20 Lines |