Changeset View
Changeset View
Standalone View
Standalone View
sys/riscv/riscv/pmap.c
Show First 20 Lines • Show All 2,099 Lines • ▼ Show 20 Lines | pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | ||||
struct rwlock **lockp) | struct rwlock **lockp) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
vm_page_t m; | vm_page_t m; | ||||
vm_offset_t va_last; | vm_offset_t va_last; | ||||
rw_assert(&pvh_global_lock, RA_LOCKED); | rw_assert(&pvh_global_lock, RA_LOCKED); | ||||
KASSERT((va & L2_OFFSET) == 0, | KASSERT((pa & L2_OFFSET) == 0, | ||||
("pmap_pv_promote_l2: misaligned va %#lx", va)); | ("pmap_pv_promote_l2: misaligned pa %#lx", pa)); | ||||
CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); | CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); | ||||
m = PHYS_TO_VM_PAGE(pa); | m = PHYS_TO_VM_PAGE(pa); | ||||
va = va & ~L2_OFFSET; | |||||
pv = pmap_pvh_remove(&m->md, pmap, va); | pv = pmap_pvh_remove(&m->md, pmap, va); | ||||
KASSERT(pv != NULL, ("pmap_pv_promote_l2: pv for %#lx not found", va)); | KASSERT(pv != NULL, ("pmap_pv_promote_l2: pv for %#lx not found", va)); | ||||
pvh = pa_to_pvh(pa); | pvh = pa_to_pvh(pa); | ||||
TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); | TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); | ||||
pvh->pv_gen++; | pvh->pv_gen++; | ||||
va_last = va + L2_SIZE - PAGE_SIZE; | va_last = va + L2_SIZE - PAGE_SIZE; | ||||
do { | do { | ||||
▲ Show 20 Lines • Show All 628 Lines • ▼ Show 20 Lines | pmap_demote_l2_locked(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, | ||||
atomic_add_long(&pmap_l2_demotions, 1); | atomic_add_long(&pmap_l2_demotions, 1); | ||||
CTR2(KTR_PMAP, "pmap_demote_l2_locked: success for va %#lx in pmap %p", | CTR2(KTR_PMAP, "pmap_demote_l2_locked: success for va %#lx in pmap %p", | ||||
va, pmap); | va, pmap); | ||||
return (true); | return (true); | ||||
} | } | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
static void | static void | ||||
pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, | pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t ml3, | ||||
struct rwlock **lockp) | struct rwlock **lockp) | ||||
{ | { | ||||
pt_entry_t *firstl3, firstl3e, *l3, l3e; | pt_entry_t *firstl3, firstl3e, *l3, l3e; | ||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||
vm_page_t ml3; | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
va &= ~L2_OFFSET; | |||||
KASSERT((pmap_load(l2) & PTE_RWX) == 0, | KASSERT((pmap_load(l2) & PTE_RWX) == 0, | ||||
("pmap_promote_l2: invalid l2 entry %p", l2)); | ("pmap_promote_l2: invalid l2 entry %p", l2)); | ||||
firstl3 = (pt_entry_t *)PHYS_TO_DMAP(PTE_TO_PHYS(pmap_load(l2))); | firstl3 = (pt_entry_t *)PHYS_TO_DMAP(PTE_TO_PHYS(pmap_load(l2))); | ||||
firstl3e = pmap_load(firstl3); | firstl3e = pmap_load(firstl3); | ||||
pa = PTE_TO_PHYS(firstl3e); | pa = PTE_TO_PHYS(firstl3e); | ||||
if ((pa & L2_OFFSET) != 0) { | if ((pa & L2_OFFSET) != 0) { | ||||
CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx pmap %p", | CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx pmap %p", | ||||
Show All 15 Lines | pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t ml3, | ||||
while ((firstl3e & (PTE_W | PTE_D)) == PTE_W) { | while ((firstl3e & (PTE_W | PTE_D)) == PTE_W) { | ||||
if (atomic_fcmpset_64(firstl3, &firstl3e, firstl3e & ~PTE_W)) { | if (atomic_fcmpset_64(firstl3, &firstl3e, firstl3e & ~PTE_W)) { | ||||
firstl3e &= ~PTE_W; | firstl3e &= ~PTE_W; | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
pa += PAGE_SIZE; | pa += PAGE_SIZE; | ||||
for (l3 = firstl3 + 1; l3 < firstl3 + Ln_ENTRIES; l3++) { | for (l3 = firstl3 + 1; l3 < firstl3 + Ln_ENTRIES; l3++) { | ||||
alc: As an aside, long ago, I found empirically that it was more efficient for this loop to iterate… | |||||
l3e = pmap_load(l3); | l3e = pmap_load(l3); | ||||
if (PTE_TO_PHYS(l3e) != pa) { | if (PTE_TO_PHYS(l3e) != pa) { | ||||
CTR2(KTR_PMAP, | CTR2(KTR_PMAP, | ||||
"pmap_promote_l2: failure for va %#lx pmap %p", | "pmap_promote_l2: failure for va %#lx pmap %p", | ||||
va, pmap); | va, pmap); | ||||
atomic_add_long(&pmap_l2_p_failures, 1); | atomic_add_long(&pmap_l2_p_failures, 1); | ||||
return; | return; | ||||
} | } | ||||
while ((l3e & (PTE_W | PTE_D)) == PTE_W) { | while ((l3e & (PTE_W | PTE_D)) == PTE_W) { | ||||
if (atomic_fcmpset_64(l3, &l3e, l3e & ~PTE_W)) { | if (atomic_fcmpset_64(l3, &l3e, l3e & ~PTE_W)) { | ||||
l3e &= ~PTE_W; | l3e &= ~PTE_W; | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
if ((l3e & PTE_PROMOTE) != (firstl3e & PTE_PROMOTE)) { | if ((l3e & PTE_PROMOTE) != (firstl3e & PTE_PROMOTE)) { | ||||
CTR2(KTR_PMAP, | CTR2(KTR_PMAP, | ||||
"pmap_promote_l2: failure for va %#lx pmap %p", | "pmap_promote_l2: failure for va %#lx pmap %p", | ||||
va, pmap); | va, pmap); | ||||
atomic_add_long(&pmap_l2_p_failures, 1); | atomic_add_long(&pmap_l2_p_failures, 1); | ||||
return; | return; | ||||
} | } | ||||
pa += PAGE_SIZE; | pa += PAGE_SIZE; | ||||
} | } | ||||
if (ml3 == NULL) | |||||
ml3 = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l2))); | ml3 = PHYS_TO_VM_PAGE(PTE_TO_PHYS(pmap_load(l2))); | ||||
KASSERT(ml3->pindex == pmap_l2_pindex(va), | KASSERT(ml3->pindex == pmap_l2_pindex(va), | ||||
("pmap_promote_l2: page table page's pindex is wrong")); | ("pmap_promote_l2: page table page's pindex is wrong")); | ||||
if (pmap_insert_pt_page(pmap, ml3, true)) { | if (pmap_insert_pt_page(pmap, ml3, true)) { | ||||
CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx pmap %p", | CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx pmap %p", | ||||
va, pmap); | va, pmap); | ||||
atomic_add_long(&pmap_l2_p_failures, 1); | atomic_add_long(&pmap_l2_p_failures, 1); | ||||
return; | return; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 269 Lines • ▼ Show 20 Lines | if (orig_l3 != 0) { | ||||
pmap_store(l3, new_l3); | pmap_store(l3, new_l3); | ||||
} | } | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (mpte != NULL && mpte->ref_count == Ln_ENTRIES && | if (mpte != NULL && mpte->ref_count == Ln_ENTRIES && | ||||
pmap_ps_enabled(pmap) && | pmap_ps_enabled(pmap) && | ||||
(m->flags & PG_FICTITIOUS) == 0 && | (m->flags & PG_FICTITIOUS) == 0 && | ||||
vm_reserv_level_iffullpop(m) == 0) | vm_reserv_level_iffullpop(m) == 0) | ||||
pmap_promote_l2(pmap, l2, va, &lock); | pmap_promote_l2(pmap, l2, va, mpte, &lock); | ||||
#endif | #endif | ||||
rv = KERN_SUCCESS; | rv = KERN_SUCCESS; | ||||
out: | out: | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
rw_runlock(&pvh_global_lock); | rw_runlock(&pvh_global_lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
▲ Show 20 Lines • Show All 1,877 Lines • Show Last 20 Lines |
As an aside, long ago, I found empirically that it was more efficient for this loop to iterate from the end of the page to the beginning, meaning that failures took fewer iterations.