Changeset View
Changeset View
Standalone View
Standalone View
riscv/riscv/pmap.c
Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Inserts the specified page table page into the specified pmap's collection | * Inserts the specified page table page into the specified pmap's collection | ||||
* of idle page table pages. Each of a pmap's page table pages is responsible | * of idle page table pages. Each of a pmap's page table pages is responsible | ||||
* for mapping a distinct range of virtual addresses. The pmap's collection is | * for mapping a distinct range of virtual addresses. The pmap's collection is | ||||
* ordered by this virtual address range. | * ordered by this virtual address range. | ||||
*/ | */ | ||||
static __inline int | static __inline int | ||||
pmap_insert_pt_page(pmap_t pmap, vm_page_t ml3) | pmap_insert_pt_page(pmap_t pmap, vm_page_t ml3, bool promoted) | ||||
{ | { | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
ml3->valid = promoted ? VM_PAGE_BITS_ALL : 0; | |||||
return (vm_radix_insert(&pmap->pm_root, ml3)); | return (vm_radix_insert(&pmap->pm_root, ml3)); | ||||
} | } | ||||
/* | /* | ||||
* Removes the page table page mapping the specified virtual address from the | * Removes the page table page mapping the specified virtual address from the | ||||
* specified pmap's collection of idle page table pages, and returns it. | * specified pmap's collection of idle page table pages, and returns it. | ||||
* Otherwise, returns NULL if there is no page table page corresponding to the | * Otherwise, returns NULL if there is no page table page corresponding to the | ||||
* specified virtual address. | * specified virtual address. | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
(void)pmap_remove_l2(pmap, l2, va & ~L2_OFFSET, | (void)pmap_remove_l2(pmap, l2, va & ~L2_OFFSET, | ||||
pmap_load(pmap_l1(pmap, va)), &free, lockp); | pmap_load(pmap_l1(pmap, va)), &free, lockp); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
CTR2(KTR_PMAP, "pmap_demote_l2_locked: " | CTR2(KTR_PMAP, "pmap_demote_l2_locked: " | ||||
"failure for va %#lx in pmap %p", va, pmap); | "failure for va %#lx in pmap %p", va, pmap); | ||||
return (false); | return (false); | ||||
} | } | ||||
if (va < VM_MAXUSER_ADDRESS) | if (va < VM_MAXUSER_ADDRESS) { | ||||
mpte->wire_count = Ln_ENTRIES; | |||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | |||||
} | } | ||||
mptepa = VM_PAGE_TO_PHYS(mpte); | mptepa = VM_PAGE_TO_PHYS(mpte); | ||||
firstl3 = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | firstl3 = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | ||||
newl2 = ((mptepa / PAGE_SIZE) << PTE_PPN0_S) | PTE_V; | newl2 = ((mptepa / PAGE_SIZE) << PTE_PPN0_S) | PTE_V; | ||||
KASSERT((oldl2 & PTE_A) != 0, | KASSERT((oldl2 & PTE_A) != 0, | ||||
("pmap_demote_l2_locked: oldl2 is missing PTE_A")); | ("pmap_demote_l2_locked: oldl2 is missing PTE_A")); | ||||
KASSERT((oldl2 & (PTE_D | PTE_W)) != PTE_W, | KASSERT((oldl2 & (PTE_D | PTE_W)) != PTE_W, | ||||
("pmap_demote_l2_locked: oldl2 is missing PTE_D")); | ("pmap_demote_l2_locked: oldl2 is missing PTE_D")); | ||||
newl3 = oldl2; | newl3 = oldl2; | ||||
/* | /* | ||||
* If the page table page is new, initialize it. | * If the page table page is not leftover from an earlier promotion, | ||||
* initialize it. | |||||
*/ | */ | ||||
if (mpte->wire_count == 1) { | if (mpte->valid == 0) { | ||||
mpte->wire_count = Ln_ENTRIES; | |||||
for (i = 0; i < Ln_ENTRIES; i++) | for (i = 0; i < Ln_ENTRIES; i++) | ||||
pmap_store(firstl3 + i, newl3 + (i << PTE_PPN0_S)); | pmap_store(firstl3 + i, newl3 + (i << PTE_PPN0_S)); | ||||
} | } | ||||
KASSERT(PTE_TO_PHYS(pmap_load(firstl3)) == PTE_TO_PHYS(newl3), | KASSERT(PTE_TO_PHYS(pmap_load(firstl3)) == PTE_TO_PHYS(newl3), | ||||
("pmap_demote_l2_locked: firstl3 and newl3 map different physical " | ("pmap_demote_l2_locked: firstl3 and newl3 map different physical " | ||||
"addresses")); | "addresses")); | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | |||||
return; | return; | ||||
} | } | ||||
pa += PAGE_SIZE; | pa += PAGE_SIZE; | ||||
} | } | ||||
ml3 = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l2))); | ml3 = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l2))); | ||||
KASSERT(ml3->pindex == pmap_l2_pindex(va), | KASSERT(ml3->pindex == pmap_l2_pindex(va), | ||||
("pmap_promote_l2: page table page's pindex is wrong")); | ("pmap_promote_l2: page table page's pindex is wrong")); | ||||
if (pmap_insert_pt_page(pmap, ml3)) { | if (pmap_insert_pt_page(pmap, ml3, true)) { | ||||
CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx pmap %p", | CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx pmap %p", | ||||
va, pmap); | va, pmap); | ||||
atomic_add_long(&pmap_l2_p_failures, 1); | atomic_add_long(&pmap_l2_p_failures, 1); | ||||
return; | return; | ||||
} | } | ||||
if ((pmap_load(firstl3) & PTE_SW_MANAGED) != 0) | if ((pmap_load(firstl3) & PTE_SW_MANAGED) != 0) | ||||
pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(pmap_load(firstl3)), | pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(pmap_load(firstl3)), | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
l3 = pmap_l2_to_l3(l2, sva); | l3 = pmap_l2_to_l3(l2, sva); | ||||
if ((pmap_load(l3) & PTE_V) != 0 && | if ((pmap_load(l3) & PTE_V) != 0 && | ||||
pmap_remove_l3(pmap, l3, sva, oldl2, &free, | pmap_remove_l3(pmap, l3, sva, oldl2, &free, | ||||
lockp) != 0) | lockp) != 0) | ||||
break; | break; | ||||
} | } | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
if (va >= VM_MAXUSER_ADDRESS) { | if (va >= VM_MAXUSER_ADDRESS) { | ||||
/* | |||||
* Both pmap_remove_l2() and pmap_remove_l3() will | |||||
* leave the kernel page table page zero filled. | |||||
*/ | |||||
mt = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l2))); | mt = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l2))); | ||||
if (pmap_insert_pt_page(pmap, mt)) { | if (pmap_insert_pt_page(pmap, mt, false)) | ||||
/* | |||||
* XXX Currently, this can't happen bacuse | |||||
* we do not perform pmap_enter(psind == 1) | |||||
* on the kernel pmap. | |||||
*/ | |||||
panic("pmap_enter_l2: trie insert failed"); | panic("pmap_enter_l2: trie insert failed"); | ||||
} | |||||
} else | } else | ||||
KASSERT(pmap_load(l2) == 0, | KASSERT(pmap_load(l2) == 0, | ||||
("pmap_enter_l2: non-zero L2 entry %p", l2)); | ("pmap_enter_l2: non-zero L2 entry %p", l2)); | ||||
} | } | ||||
if ((new_l2 & PTE_SW_MANAGED) != 0) { | if ((new_l2 & PTE_SW_MANAGED) != 0) { | ||||
/* | /* | ||||
* Abort this mapping if its PV entry could not be created. | * Abort this mapping if its PV entry could not be created. | ||||
▲ Show 20 Lines • Show All 91 Lines • Show Last 20 Lines |