Changeset View
Changeset View
Standalone View
Standalone View
sys/riscv/riscv/pmap.c
Show First 20 Lines • Show All 1,121 Lines • ▼ Show 20 Lines | |||||
pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) | pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) | ||||
{ | { | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va))); | return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va))); | ||||
} | } | ||||
/* | /* | ||||
* Decrements a page table page's wire count, which is used to record the | * Decrements a page table page's reference count, which is used to record the | ||||
* number of valid page table entries within the page. If the wire count | * number of valid page table entries within the page. If the reference count | ||||
* drops to zero, then the page table page is unmapped. Returns TRUE if the | * drops to zero, then the page table page is unmapped. Returns TRUE if the | ||||
* page table page was unmapped and FALSE otherwise. | * page table page was unmapped and FALSE otherwise. | ||||
*/ | */ | ||||
static inline boolean_t | static inline boolean_t | ||||
pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
{ | { | ||||
--m->wire_count; | --m->ref_count; | ||||
if (m->wire_count == 0) { | if (m->ref_count == 0) { | ||||
_pmap_unwire_ptp(pmap, va, m, free); | _pmap_unwire_ptp(pmap, va, m, free); | ||||
return (TRUE); | return (TRUE); | ||||
} else { | } else { | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
Show All 30 Lines | _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | ||||
* Put page on a list so that it is released after | * Put page on a list so that it is released after | ||||
* *ALL* TLB shootdown is done | * *ALL* TLB shootdown is done | ||||
*/ | */ | ||||
pmap_add_delayed_free_list(m, free, TRUE); | pmap_add_delayed_free_list(m, free, TRUE); | ||||
} | } | ||||
/* | /* | ||||
* After removing a page table entry, this routine is used to | * After removing a page table entry, this routine is used to | ||||
* conditionally free the page, and manage the hold/wire counts. | * conditionally free the page, and manage the reference count. | ||||
*/ | */ | ||||
static int | static int | ||||
pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | ||||
struct spglist *free) | struct spglist *free) | ||||
{ | { | ||||
vm_page_t mpte; | vm_page_t mpte; | ||||
if (va >= VM_MAXUSER_ADDRESS) | if (va >= VM_MAXUSER_ADDRESS) | ||||
▲ Show 20 Lines • Show All 126 Lines • ▼ Show 20 Lines | if (pmap_load(l1) == 0) { | ||||
lockp) == NULL) { | lockp) == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} else { | } else { | ||||
phys = PTE_TO_PHYS(pmap_load(l1)); | phys = PTE_TO_PHYS(pmap_load(l1)); | ||||
pdpg = PHYS_TO_VM_PAGE(phys); | pdpg = PHYS_TO_VM_PAGE(phys); | ||||
pdpg->wire_count++; | pdpg->ref_count++; | ||||
} | } | ||||
phys = PTE_TO_PHYS(pmap_load(l1)); | phys = PTE_TO_PHYS(pmap_load(l1)); | ||||
l2 = (pd_entry_t *)PHYS_TO_DMAP(phys); | l2 = (pd_entry_t *)PHYS_TO_DMAP(phys); | ||||
l2 = &l2[ptepindex & Ln_ADDR_MASK]; | l2 = &l2[ptepindex & Ln_ADDR_MASK]; | ||||
pn = (VM_PAGE_TO_PHYS(m) / PAGE_SIZE); | pn = (VM_PAGE_TO_PHYS(m) / PAGE_SIZE); | ||||
entry = (PTE_V); | entry = (PTE_V); | ||||
Show All 13 Lines | pmap_alloc_l2(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) | ||||
vm_page_t l2pg; | vm_page_t l2pg; | ||||
vm_pindex_t l2pindex; | vm_pindex_t l2pindex; | ||||
retry: | retry: | ||||
l1 = pmap_l1(pmap, va); | l1 = pmap_l1(pmap, va); | ||||
if (l1 != NULL && (pmap_load(l1) & PTE_RWX) == 0) { | if (l1 != NULL && (pmap_load(l1) & PTE_RWX) == 0) { | ||||
/* Add a reference to the L2 page. */ | /* Add a reference to the L2 page. */ | ||||
l2pg = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l1))); | l2pg = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l1))); | ||||
l2pg->wire_count++; | l2pg->ref_count++; | ||||
} else { | } else { | ||||
/* Allocate a L2 page. */ | /* Allocate a L2 page. */ | ||||
l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT; | l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT; | ||||
l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp); | l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp); | ||||
if (l2pg == NULL && lockp != NULL) | if (l2pg == NULL && lockp != NULL) | ||||
goto retry; | goto retry; | ||||
} | } | ||||
return (l2pg); | return (l2pg); | ||||
Show All 19 Lines | retry: | ||||
/* | /* | ||||
* If the page table page is mapped, we just increment the | * If the page table page is mapped, we just increment the | ||||
* hold count, and activate it. | * hold count, and activate it. | ||||
*/ | */ | ||||
if (l2 != NULL && pmap_load(l2) != 0) { | if (l2 != NULL && pmap_load(l2) != 0) { | ||||
phys = PTE_TO_PHYS(pmap_load(l2)); | phys = PTE_TO_PHYS(pmap_load(l2)); | ||||
m = PHYS_TO_VM_PAGE(phys); | m = PHYS_TO_VM_PAGE(phys); | ||||
m->wire_count++; | m->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Here if the pte page isn't mapped, or if it has been | * Here if the pte page isn't mapped, or if it has been | ||||
* deallocated. | * deallocated. | ||||
*/ | */ | ||||
m = _pmap_alloc_l3(pmap, ptepindex, lockp); | m = _pmap_alloc_l3(pmap, ptepindex, lockp); | ||||
if (m == NULL && lockp != NULL) | if (m == NULL && lockp != NULL) | ||||
goto retry; | goto retry; | ||||
▲ Show 20 Lines • Show All 658 Lines • ▼ Show 20 Lines | pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, | ||||
if (pmap == kernel_pmap) { | if (pmap == kernel_pmap) { | ||||
pmap_remove_kernel_l2(pmap, l2, sva); | pmap_remove_kernel_l2(pmap, l2, sva); | ||||
} else { | } else { | ||||
ml3 = pmap_remove_pt_page(pmap, sva); | ml3 = pmap_remove_pt_page(pmap, sva); | ||||
if (ml3 != NULL) { | if (ml3 != NULL) { | ||||
KASSERT(ml3->valid == VM_PAGE_BITS_ALL, | KASSERT(ml3->valid == VM_PAGE_BITS_ALL, | ||||
("pmap_remove_l2: l3 page not promoted")); | ("pmap_remove_l2: l3 page not promoted")); | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
KASSERT(ml3->wire_count == Ln_ENTRIES, | KASSERT(ml3->ref_count == Ln_ENTRIES, | ||||
("pmap_remove_l2: l3 page wire count error")); | ("pmap_remove_l2: l3 page ref count error")); | ||||
ml3->wire_count = 1; | ml3->ref_count = 1; | ||||
vm_page_unwire_noq(ml3); | vm_page_unwire_noq(ml3); | ||||
pmap_add_delayed_free_list(ml3, free, FALSE); | pmap_add_delayed_free_list(ml3, free, FALSE); | ||||
} | } | ||||
} | } | ||||
return (pmap_unuse_pt(pmap, sva, l1e, free)); | return (pmap_unuse_pt(pmap, sva, l1e, free)); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 400 Lines • ▼ Show 20 Lines | if ((oldl2 & PTE_A) == 0 || (mpte = vm_page_alloc(NULL, | ||||
(void)pmap_remove_l2(pmap, l2, va & ~L2_OFFSET, | (void)pmap_remove_l2(pmap, l2, va & ~L2_OFFSET, | ||||
pmap_load(pmap_l1(pmap, va)), &free, lockp); | pmap_load(pmap_l1(pmap, va)), &free, lockp); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
CTR2(KTR_PMAP, "pmap_demote_l2_locked: " | CTR2(KTR_PMAP, "pmap_demote_l2_locked: " | ||||
"failure for va %#lx in pmap %p", va, pmap); | "failure for va %#lx in pmap %p", va, pmap); | ||||
return (false); | return (false); | ||||
} | } | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
mpte->wire_count = Ln_ENTRIES; | mpte->ref_count = Ln_ENTRIES; | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | } | ||||
} | } | ||||
mptepa = VM_PAGE_TO_PHYS(mpte); | mptepa = VM_PAGE_TO_PHYS(mpte); | ||||
firstl3 = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | firstl3 = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | ||||
newl2 = ((mptepa / PAGE_SIZE) << PTE_PPN0_S) | PTE_V; | newl2 = ((mptepa / PAGE_SIZE) << PTE_PPN0_S) | PTE_V; | ||||
KASSERT((oldl2 & PTE_A) != 0, | KASSERT((oldl2 & PTE_A) != 0, | ||||
("pmap_demote_l2_locked: oldl2 is missing PTE_A")); | ("pmap_demote_l2_locked: oldl2 is missing PTE_A")); | ||||
▲ Show 20 Lines • Show All 191 Lines • ▼ Show 20 Lines | pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
l2 = pmap_l2(pmap, va); | l2 = pmap_l2(pmap, va); | ||||
if (l2 != NULL && ((l2e = pmap_load(l2)) & PTE_V) != 0 && | if (l2 != NULL && ((l2e = pmap_load(l2)) & PTE_V) != 0 && | ||||
((l2e & PTE_RWX) == 0 || pmap_demote_l2_locked(pmap, l2, | ((l2e & PTE_RWX) == 0 || pmap_demote_l2_locked(pmap, l2, | ||||
va, &lock))) { | va, &lock))) { | ||||
l3 = pmap_l2_to_l3(l2, va); | l3 = pmap_l2_to_l3(l2, va); | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
mpte = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l2))); | mpte = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l2))); | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} | } | ||||
} else if (va < VM_MAXUSER_ADDRESS) { | } else if (va < VM_MAXUSER_ADDRESS) { | ||||
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | ||||
mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock); | mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock); | ||||
if (mpte == NULL && nosleep) { | if (mpte == NULL && nosleep) { | ||||
CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); | CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | if ((orig_l3 & PTE_V) != 0) { | ||||
else if ((flags & PMAP_ENTER_WIRED) == 0 && | else if ((flags & PMAP_ENTER_WIRED) == 0 && | ||||
(orig_l3 & PTE_SW_WIRED) != 0) | (orig_l3 & PTE_SW_WIRED) != 0) | ||||
pmap->pm_stats.wired_count--; | pmap->pm_stats.wired_count--; | ||||
/* | /* | ||||
* Remove the extra PT page reference. | * Remove the extra PT page reference. | ||||
*/ | */ | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
mpte->wire_count--; | mpte->ref_count--; | ||||
KASSERT(mpte->wire_count > 0, | KASSERT(mpte->ref_count > 0, | ||||
("pmap_enter: missing reference to page table page," | ("pmap_enter: missing reference to page table page," | ||||
" va: 0x%lx", va)); | " va: 0x%lx", va)); | ||||
} | } | ||||
/* | /* | ||||
* Has the physical page changed? | * Has the physical page changed? | ||||
*/ | */ | ||||
if (opa == pa) { | if (opa == pa) { | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | if (orig_l3 != 0) { | ||||
if ((orig_l3 & (PTE_D | PTE_SW_MANAGED)) == | if ((orig_l3 & (PTE_D | PTE_SW_MANAGED)) == | ||||
(PTE_D | PTE_SW_MANAGED)) | (PTE_D | PTE_SW_MANAGED)) | ||||
vm_page_dirty(m); | vm_page_dirty(m); | ||||
} else { | } else { | ||||
pmap_store(l3, new_l3); | pmap_store(l3, new_l3); | ||||
} | } | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (mpte != NULL && mpte->wire_count == Ln_ENTRIES && | if (mpte != NULL && mpte->ref_count == Ln_ENTRIES && | ||||
pmap_ps_enabled(pmap) && | pmap_ps_enabled(pmap) && | ||||
(m->flags & PG_FICTITIOUS) == 0 && | (m->flags & PG_FICTITIOUS) == 0 && | ||||
vm_reserv_level_iffullpop(m) == 0) | vm_reserv_level_iffullpop(m) == 0) | ||||
pmap_promote_l2(pmap, l2, va, &lock); | pmap_promote_l2(pmap, l2, va, &lock); | ||||
#endif | #endif | ||||
rv = KERN_SUCCESS; | rv = KERN_SUCCESS; | ||||
out: | out: | ||||
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | if ((l2pg = pmap_alloc_l2(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? | ||||
CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p", | CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p", | ||||
va, pmap); | va, pmap); | ||||
return (KERN_RESOURCE_SHORTAGE); | return (KERN_RESOURCE_SHORTAGE); | ||||
} | } | ||||
l2 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(l2pg)); | l2 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(l2pg)); | ||||
l2 = &l2[pmap_l2_index(va)]; | l2 = &l2[pmap_l2_index(va)]; | ||||
if ((oldl2 = pmap_load(l2)) != 0) { | if ((oldl2 = pmap_load(l2)) != 0) { | ||||
KASSERT(l2pg->wire_count > 1, | KASSERT(l2pg->ref_count > 1, | ||||
("pmap_enter_l2: l2pg's wire count is too low")); | ("pmap_enter_l2: l2pg's ref count is too low")); | ||||
if ((flags & PMAP_ENTER_NOREPLACE) != 0) { | if ((flags & PMAP_ENTER_NOREPLACE) != 0) { | ||||
l2pg->wire_count--; | l2pg->ref_count--; | ||||
CTR2(KTR_PMAP, | CTR2(KTR_PMAP, | ||||
"pmap_enter_l2: failure for va %#lx in pmap %p", | "pmap_enter_l2: failure for va %#lx in pmap %p", | ||||
va, pmap); | va, pmap); | ||||
return (KERN_FAILURE); | return (KERN_FAILURE); | ||||
} | } | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
if ((oldl2 & PTE_RWX) != 0) | if ((oldl2 & PTE_RWX) != 0) | ||||
(void)pmap_remove_l2(pmap, l2, va, | (void)pmap_remove_l2(pmap, l2, va, | ||||
▲ Show 20 Lines • Show All 158 Lines • ▼ Show 20 Lines | pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
vm_pindex_t l2pindex; | vm_pindex_t l2pindex; | ||||
/* | /* | ||||
* Calculate pagetable page index | * Calculate pagetable page index | ||||
*/ | */ | ||||
l2pindex = pmap_l2_pindex(va); | l2pindex = pmap_l2_pindex(va); | ||||
if (mpte && (mpte->pindex == l2pindex)) { | if (mpte && (mpte->pindex == l2pindex)) { | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Get the l2 entry | * Get the l2 entry | ||||
*/ | */ | ||||
l2 = pmap_l2(pmap, va); | l2 = pmap_l2(pmap, va); | ||||
/* | /* | ||||
* If the page table page is mapped, we just increment | * If the page table page is mapped, we just increment | ||||
* the hold count, and activate it. Otherwise, we | * the hold count, and activate it. Otherwise, we | ||||
* attempt to allocate a page table page. If this | * attempt to allocate a page table page. If this | ||||
* attempt fails, we don't retry. Instead, we give up. | * attempt fails, we don't retry. Instead, we give up. | ||||
*/ | */ | ||||
if (l2 != NULL && pmap_load(l2) != 0) { | if (l2 != NULL && pmap_load(l2) != 0) { | ||||
phys = PTE_TO_PHYS(pmap_load(l2)); | phys = PTE_TO_PHYS(pmap_load(l2)); | ||||
mpte = PHYS_TO_VM_PAGE(phys); | mpte = PHYS_TO_VM_PAGE(phys); | ||||
mpte->wire_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Pass NULL instead of the PV list lock | * Pass NULL instead of the PV list lock | ||||
* pointer, because we don't intend to sleep. | * pointer, because we don't intend to sleep. | ||||
*/ | */ | ||||
mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); | mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); | ||||
if (mpte == NULL) | if (mpte == NULL) | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
} | } | ||||
l3 = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | l3 = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | ||||
l3 = &l3[pmap_l3_index(va)]; | l3 = &l3[pmap_l3_index(va)]; | ||||
} else { | } else { | ||||
mpte = NULL; | mpte = NULL; | ||||
l3 = pmap_l3(kernel_pmap, va); | l3 = pmap_l3(kernel_pmap, va); | ||||
} | } | ||||
if (l3 == NULL) | if (l3 == NULL) | ||||
panic("pmap_enter_quick_locked: No l3"); | panic("pmap_enter_quick_locked: No l3"); | ||||
if (pmap_load(l3) != 0) { | if (pmap_load(l3) != 0) { | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
mpte->wire_count--; | mpte->ref_count--; | ||||
mpte = NULL; | mpte = NULL; | ||||
} | } | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
/* | /* | ||||
* Enter on the PV list if part of our managed memory. | * Enter on the PV list if part of our managed memory. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 377 Lines • ▼ Show 20 Lines | if (TAILQ_EMPTY(&pvh->pv_list)) { | ||||
(mt->aflags & PGA_WRITEABLE) != 0) | (mt->aflags & PGA_WRITEABLE) != 0) | ||||
vm_page_aflag_clear(mt, PGA_WRITEABLE); | vm_page_aflag_clear(mt, PGA_WRITEABLE); | ||||
} | } | ||||
mpte = pmap_remove_pt_page(pmap, pv->pv_va); | mpte = pmap_remove_pt_page(pmap, pv->pv_va); | ||||
if (mpte != NULL) { | if (mpte != NULL) { | ||||
KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | KASSERT(mpte->valid == VM_PAGE_BITS_ALL, | ||||
("pmap_remove_pages: pte page not promoted")); | ("pmap_remove_pages: pte page not promoted")); | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
KASSERT(mpte->wire_count == Ln_ENTRIES, | KASSERT(mpte->ref_count == Ln_ENTRIES, | ||||
("pmap_remove_pages: pte page wire count error")); | ("pmap_remove_pages: pte page ref count error")); | ||||
mpte->wire_count = 0; | mpte->ref_count = 0; | ||||
pmap_add_delayed_free_list(mpte, free, FALSE); | pmap_add_delayed_free_list(mpte, free, FALSE); | ||||
} | } | ||||
} else { | } else { | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
if (TAILQ_EMPTY(&m->md.pv_list) && | if (TAILQ_EMPTY(&m->md.pv_list) && | ||||
(m->aflags & PGA_WRITEABLE) != 0) { | (m->aflags & PGA_WRITEABLE) != 0) { | ||||
▲ Show 20 Lines • Show All 891 Lines • Show Last 20 Lines |