Changeset View
Changeset View
Standalone View
Standalone View
amd64/amd64/pmap.c
Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | |||||
vm_offset_t va); | vm_offset_t va); | ||||
static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, | static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
vm_prot_t prot, struct rwlock **lockp); | vm_prot_t prot, struct rwlock **lockp); | ||||
static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, | static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, | ||||
u_int flags, vm_page_t m, struct rwlock **lockp); | u_int flags, vm_page_t m, struct rwlock **lockp); | ||||
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | ||||
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); | vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); | ||||
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); | static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); | ||||
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); | static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted); | ||||
static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, | static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, | ||||
vm_offset_t eva); | vm_offset_t eva); | ||||
static void pmap_invalidate_cache_range_all(vm_offset_t sva, | static void pmap_invalidate_cache_range_all(vm_offset_t sva, | ||||
vm_offset_t eva); | vm_offset_t eva); | ||||
static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, | static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, | ||||
pd_entry_t pde); | pd_entry_t pde); | ||||
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); | static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); | ||||
static vm_page_t pmap_large_map_getptp_unlocked(void); | static vm_page_t pmap_large_map_getptp_unlocked(void); | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); | mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); | ||||
KASSERT(mpte >= vm_page_array && | KASSERT(mpte >= vm_page_array && | ||||
mpte < &vm_page_array[vm_page_array_size], | mpte < &vm_page_array[vm_page_array_size], | ||||
("pmap_init: page table page is out of range")); | ("pmap_init: page table page is out of range")); | ||||
mpte->pindex = pmap_pde_pindex(KERNBASE) + i; | mpte->pindex = pmap_pde_pindex(KERNBASE) + i; | ||||
mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); | mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); | ||||
mpte->wire_count = 1; | mpte->wire_count = 1; | ||||
if (i << PDRSHIFT < KERNend && | if (i << PDRSHIFT < KERNend && | ||||
pmap_insert_pt_page(kernel_pmap, mpte)) | pmap_insert_pt_page(kernel_pmap, mpte, false)) | ||||
panic("pmap_init: pmap_insert_pt_page failed"); | panic("pmap_init: pmap_insert_pt_page failed"); | ||||
} | } | ||||
PMAP_UNLOCK(kernel_pmap); | PMAP_UNLOCK(kernel_pmap); | ||||
vm_wire_add(nkpt); | vm_wire_add(nkpt); | ||||
/* | /* | ||||
* If the kernel is running on a virtual machine, then it must assume | * If the kernel is running on a virtual machine, then it must assume | ||||
* that MCA is enabled by the hypervisor. Moreover, the kernel must | * that MCA is enabled by the hypervisor. Moreover, the kernel must | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Inserts the specified page table page into the specified pmap's collection | * Inserts the specified page table page into the specified pmap's collection | ||||
* of idle page table pages. Each of a pmap's page table pages is responsible | * of idle page table pages. Each of a pmap's page table pages is responsible | ||||
* for mapping a distinct range of virtual addresses. The pmap's collection is | * for mapping a distinct range of virtual addresses. The pmap's collection is | ||||
* ordered by this virtual address range. | * ordered by this virtual address range. | ||||
*/ | */ | ||||
static __inline int | static __inline int | ||||
pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) | pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted) | ||||
{ | { | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
mpte->valid = promoted ? VM_PAGE_BITS_ALL : 0; | |||||
return (vm_radix_insert(&pmap->pm_root, mpte)); | return (vm_radix_insert(&pmap->pm_root, mpte)); | ||||
} | } | ||||
/* | /* | ||||
* Removes the page table page mapping the specified virtual address from the | * Removes the page table page mapping the specified virtual address from the | ||||
* specified pmap's collection of idle page table pages, and returns it. | * specified pmap's collection of idle page table pages, and returns it. | ||||
* Otherwise, returns NULL if there is no page table page corresponding to the | * Otherwise, returns NULL if there is no page table page corresponding to the | ||||
* specified virtual address. | * specified virtual address. | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
("pmap_demote_pde: oldpde is missing PG_M")); | ("pmap_demote_pde: oldpde is missing PG_M")); | ||||
newpte = oldpde & ~PG_PS; | newpte = oldpde & ~PG_PS; | ||||
newpte = pmap_swap_pat(pmap, newpte); | newpte = pmap_swap_pat(pmap, newpte); | ||||
/* | /* | ||||
* If the page table page is not leftover from an earlier promotion, | * If the page table page is not leftover from an earlier promotion, | ||||
* initialize it. | * initialize it. | ||||
*/ | */ | ||||
if ((oldpde & PG_PROMOTED) == 0) | if (mpte->valid == 0) | ||||
pmap_fill_ptp(firstpte, newpte); | pmap_fill_ptp(firstpte, newpte); | ||||
pmap_demote_pde_check(firstpte, newpte); | pmap_demote_pde_check(firstpte, newpte); | ||||
/* | /* | ||||
* If the mapping has changed attributes, update the page table | * If the mapping has changed attributes, update the page table | ||||
* entries. | * entries. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
* destroyed by pmap_remove_pde(). | * destroyed by pmap_remove_pde(). | ||||
*/ | */ | ||||
mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | ||||
KASSERT(mpte >= vm_page_array && | KASSERT(mpte >= vm_page_array && | ||||
mpte < &vm_page_array[vm_page_array_size], | mpte < &vm_page_array[vm_page_array_size], | ||||
("pmap_promote_pde: page table page is out of range")); | ("pmap_promote_pde: page table page is out of range")); | ||||
KASSERT(mpte->pindex == pmap_pde_pindex(va), | KASSERT(mpte->pindex == pmap_pde_pindex(va), | ||||
("pmap_promote_pde: page table page's pindex is wrong")); | ("pmap_promote_pde: page table page's pindex is wrong")); | ||||
if (pmap_insert_pt_page(pmap, mpte)) { | if (pmap_insert_pt_page(pmap, mpte, true)) { | ||||
atomic_add_long(&pmap_pde_p_failures, 1); | atomic_add_long(&pmap_pde_p_failures, 1); | ||||
CTR2(KTR_PMAP, | CTR2(KTR_PMAP, | ||||
"pmap_promote_pde: failure for va %#lx in pmap %p", va, | "pmap_promote_pde: failure for va %#lx in pmap %p", va, | ||||
pmap); | pmap); | ||||
return; | return; | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
pmap_delayed_invl_start(); | pmap_delayed_invl_start(); | ||||
if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free, | if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free, | ||||
lockp)) | lockp)) | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
pmap_delayed_invl_finish(); | pmap_delayed_invl_finish(); | ||||
} | } | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
if (va >= VM_MAXUSER_ADDRESS) { | if (va >= VM_MAXUSER_ADDRESS) { | ||||
/* | |||||
* Both pmap_remove_pde() and pmap_remove_ptes() will | |||||
* leave the kernel page table page zero filled. | |||||
*/ | |||||
mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME); | ||||
if (pmap_insert_pt_page(pmap, mt)) { | if (pmap_insert_pt_page(pmap, mt, false)) | ||||
/* | |||||
* XXX Currently, this can't happen because | |||||
* we do not perform pmap_enter(psind == 1) | |||||
* on the kernel pmap. | |||||
*/ | |||||
panic("pmap_enter_pde: trie insert failed"); | panic("pmap_enter_pde: trie insert failed"); | ||||
} | |||||
} else | } else | ||||
KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p", | KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p", | ||||
pde)); | pde)); | ||||
} | } | ||||
if ((newpde & PG_MANAGED) != 0) { | if ((newpde & PG_MANAGED) != 0) { | ||||
/* | /* | ||||
* Abort this mapping if its PV entry could not be created. | * Abort this mapping if its PV entry could not be created. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 91 Lines • Show Last 20 Lines |