Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 8,453 Lines • ▼ Show 20 Lines | pmap_remove_write(vm_page_t m) | ||||
vm_page_assert_busied(m); | vm_page_assert_busied(m); | ||||
if (!pmap_page_is_write_mapped(m)) | if (!pmap_page_is_write_mapped(m)) | ||||
return; | return; | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : | pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : | ||||
pa_to_pvh(VM_PAGE_TO_PHYS(m)); | pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
rw_wlock(lock); | rw_wlock(lock); | ||||
retry_pv_loop: | retry: | ||||
TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { | TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | rw_wlock(lock); | ||||
if (pvh_gen != pvh->pv_gen) { | if (pvh_gen != pvh->pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto retry_pv_loop; | goto retry; | ||||
} | } | ||||
} | } | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
va = pv->pv_va; | va = pv->pv_va; | ||||
pde = pmap_pde(pmap, va); | pde = pmap_pde(pmap, va); | ||||
if ((*pde & PG_RW) != 0) | if ((*pde & PG_RW) != 0) | ||||
(void)pmap_demote_pde_locked(pmap, pde, va, &lock); | (void)pmap_demote_pde_locked(pmap, pde, va, &lock); | ||||
KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), | KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), | ||||
("inconsistent pv lock %p %p for page %p", | ("inconsistent pv lock %p %p for page %p", | ||||
lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); | lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | rw_wlock(lock); | ||||
if (pvh_gen != pvh->pv_gen || | if (pvh_gen != pvh->pv_gen || | ||||
md_gen != m->md.pv_gen) { | md_gen != m->md.pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto retry_pv_loop; | goto retry; | ||||
} | } | ||||
} | } | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
pde = pmap_pde(pmap, pv->pv_va); | pde = pmap_pde(pmap, pv->pv_va); | ||||
KASSERT((*pde & PG_PS) == 0, | KASSERT((*pde & PG_PS) == 0, | ||||
("pmap_remove_write: found a 2mpage in page %p's pv list", | ("pmap_remove_write: found a 2mpage in page %p's pv list", | ||||
m)); | m)); | ||||
pte = pmap_pde_to_pte(pde, pv->pv_va); | pte = pmap_pde_to_pte(pde, pv->pv_va); | ||||
retry: | |||||
oldpte = *pte; | oldpte = *pte; | ||||
if (oldpte & PG_RW) { | if (oldpte & PG_RW) { | ||||
if (!atomic_cmpset_long(pte, oldpte, oldpte & | while (!atomic_fcmpset_long(pte, &oldpte, oldpte & | ||||
~(PG_RW | PG_M))) | ~(PG_RW | PG_M))) | ||||
goto retry; | cpu_spinwait(); | ||||
if ((oldpte & PG_M) != 0) | if ((oldpte & PG_M) != 0) | ||||
vm_page_dirty(m); | vm_page_dirty(m); | ||||
pmap_invalidate_page(pmap, pv->pv_va); | pmap_invalidate_page(pmap, pv->pv_va); | ||||
} | } | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
vm_page_aflag_clear(m, PGA_WRITEABLE); | vm_page_aflag_clear(m, PGA_WRITEABLE); | ||||
▲ Show 20 Lines • Show All 3,248 Lines • Show Last 20 Lines |