Changeset View
Standalone View
sys/riscv/riscv/pmap.c
Show First 20 Lines • Show All 2,255 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Set the physical protection on the | * Set the physical protection on the | ||||
* specified range of this map as requested. | * specified range of this map as requested. | ||||
*/ | */ | ||||
void | void | ||||
pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) | pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) | ||||
{ | { | ||||
pd_entry_t *l1, *l2; | pd_entry_t *l1, *l2, l2e; | ||||
pt_entry_t *l3, l3e, mask; | pt_entry_t *l3, l3e, mask; | ||||
vm_page_t m; | vm_page_t m; | ||||
vm_offset_t va_next; | vm_paddr_t pa; | ||||
vm_offset_t va, va_next; | |||||
bool anychanged, pv_lists_locked; | |||||
if ((prot & VM_PROT_READ) == VM_PROT_NONE) { | if ((prot & VM_PROT_READ) == VM_PROT_NONE) { | ||||
pmap_remove(pmap, sva, eva); | pmap_remove(pmap, sva, eva); | ||||
return; | return; | ||||
} | } | ||||
if ((prot & (VM_PROT_WRITE | VM_PROT_EXECUTE)) == | if ((prot & (VM_PROT_WRITE | VM_PROT_EXECUTE)) == | ||||
(VM_PROT_WRITE | VM_PROT_EXECUTE)) | (VM_PROT_WRITE | VM_PROT_EXECUTE)) | ||||
return; | return; | ||||
anychanged = false; | |||||
pv_lists_locked = false; | |||||
mask = 0; | mask = 0; | ||||
if ((prot & VM_PROT_WRITE) == 0) | if ((prot & VM_PROT_WRITE) == 0) | ||||
mask |= PTE_W | PTE_D; | mask |= PTE_W | PTE_D; | ||||
if ((prot & VM_PROT_EXECUTE) == 0) | if ((prot & VM_PROT_EXECUTE) == 0) | ||||
mask |= PTE_X; | mask |= PTE_X; | ||||
resume: | |||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
for (; sva < eva; sva = va_next) { | for (; sva < eva; sva = va_next) { | ||||
l1 = pmap_l1(pmap, sva); | l1 = pmap_l1(pmap, sva); | ||||
if (pmap_load(l1) == 0) { | if (pmap_load(l1) == 0) { | ||||
va_next = (sva + L1_SIZE) & ~L1_OFFSET; | va_next = (sva + L1_SIZE) & ~L1_OFFSET; | ||||
if (va_next < sva) | if (va_next < sva) | ||||
va_next = eva; | va_next = eva; | ||||
continue; | continue; | ||||
} | } | ||||
va_next = (sva + L2_SIZE) & ~L2_OFFSET; | va_next = (sva + L2_SIZE) & ~L2_OFFSET; | ||||
if (va_next < sva) | if (va_next < sva) | ||||
va_next = eva; | va_next = eva; | ||||
l2 = pmap_l1_to_l2(l1, sva); | l2 = pmap_l1_to_l2(l1, sva); | ||||
if (l2 == NULL || pmap_load(l2) == 0) | if (l2 == NULL || (l2e = pmap_load(l2)) == 0) | ||||
continue; | continue; | ||||
if ((pmap_load(l2) & PTE_RX) != 0) | if ((l2e & PTE_RWX) != 0) { | ||||
if (sva + L2_SIZE == va_next && eva >= va_next) { | |||||
retryl2: | |||||
if ((l2e & (PTE_SW_MANAGED | PTE_D)) == | |||||
(PTE_SW_MANAGED | PTE_D)) { | |||||
pa = PTE_TO_PHYS(l2e); | |||||
for (va = sva, m = PHYS_TO_VM_PAGE(pa); | |||||
va < va_next; m++, va += PAGE_SIZE) | |||||
vm_page_dirty(m); | |||||
} | |||||
if (!atomic_fcmpset_long(l2, &l2e, l2e & ~mask)) | |||||
goto retryl2; | |||||
anychanged = true; | |||||
} else { | |||||
if (!pv_lists_locked) { | |||||
pv_lists_locked = true; | |||||
if (!rw_try_rlock(&pvh_global_lock)) { | |||||
if (anychanged) | |||||
pmap_invalidate_all( | |||||
pmap); | |||||
PMAP_UNLOCK(pmap); | |||||
rw_rlock(&pvh_global_lock); | |||||
goto resume; | |||||
} | |||||
} | |||||
if (!pmap_demote_l2(pmap, l2, sva)) { | |||||
/* | |||||
* The large page mapping was destroyed. | |||||
*/ | |||||
continue; | continue; | ||||
} | |||||
} | |||||
} | |||||
if (va_next > eva) | if (va_next > eva) | ||||
va_next = eva; | va_next = eva; | ||||
for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, | for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, | ||||
sva += L3_SIZE) { | sva += L3_SIZE) { | ||||
l3e = pmap_load(l3); | l3e = pmap_load(l3); | ||||
retry: | retryl3: | ||||
if ((l3e & PTE_V) == 0) | if ((l3e & PTE_V) == 0) | ||||
continue; | continue; | ||||
if ((prot & VM_PROT_WRITE) == 0 && | if ((prot & VM_PROT_WRITE) == 0 && | ||||
(l3e & (PTE_SW_MANAGED | PTE_D)) == | (l3e & (PTE_SW_MANAGED | PTE_D)) == | ||||
(PTE_SW_MANAGED | PTE_D)) { | (PTE_SW_MANAGED | PTE_D)) { | ||||
m = PHYS_TO_VM_PAGE(PTE_TO_PHYS(l3e)); | m = PHYS_TO_VM_PAGE(PTE_TO_PHYS(l3e)); | ||||
vm_page_dirty(m); | vm_page_dirty(m); | ||||
} | } | ||||
if (!atomic_fcmpset_long(l3, &l3e, l3e & ~mask)) | if (!atomic_fcmpset_long(l3, &l3e, l3e & ~mask)) | ||||
goto retry; | goto retryl3; | ||||
/* XXX: Use pmap_invalidate_range */ | anychanged = true; | ||||
pmap_invalidate_page(pmap, sva); | |||||
} | } | ||||
} | } | ||||
if (anychanged) | |||||
pmap_invalidate_all(pmap); | |||||
jhb: Should this be a pmap_invalidate_range() of the original sva and eva (you'd have to save the… | |||||
Done Inline ActionsThat's probably the right long-term direction, but I'd rather not do it in this change, as I'm just following amd64's example and I have no good way of testing the change (pmap_invalidate_range() just executes sfence.vma today). markj: That's probably the right long-term direction, but I'd rather not do it in this change, as I'm… | |||||
Done Inline Actions(BTW, there was some discussion along these lines in D12725.) markj: (BTW, there was some discussion along these lines in D12725.) | |||||
if (pv_lists_locked) | |||||
rw_runlock(&pvh_global_lock); | |||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
int | int | ||||
pmap_fault_fixup(pmap_t pmap, vm_offset_t va, vm_prot_t ftype) | pmap_fault_fixup(pmap_t pmap, vm_offset_t va, vm_prot_t ftype) | ||||
{ | { | ||||
pt_entry_t orig_l3; | pt_entry_t orig_l3; | ||||
pt_entry_t new_l3; | pt_entry_t new_l3; | ||||
▲ Show 20 Lines • Show All 840 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* The wired attribute of the page table entry is not a hardware feature, | * The wired attribute of the page table entry is not a hardware feature, | ||||
* so there is no need to invalidate any TLB entries. | * so there is no need to invalidate any TLB entries. | ||||
*/ | */ | ||||
void | void | ||||
pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
{ | { | ||||
vm_offset_t va_next; | vm_offset_t va_next; | ||||
pd_entry_t *l1, *l2; | pd_entry_t *l1, *l2, l2e; | ||||
pt_entry_t *l3; | pt_entry_t *l3, l3e; | ||||
boolean_t pv_lists_locked; | bool pv_lists_locked; | ||||
pv_lists_locked = FALSE; | pv_lists_locked = false; | ||||
retry: | |||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
for (; sva < eva; sva = va_next) { | for (; sva < eva; sva = va_next) { | ||||
l1 = pmap_l1(pmap, sva); | l1 = pmap_l1(pmap, sva); | ||||
if (pmap_load(l1) == 0) { | if (pmap_load(l1) == 0) { | ||||
va_next = (sva + L1_SIZE) & ~L1_OFFSET; | va_next = (sva + L1_SIZE) & ~L1_OFFSET; | ||||
if (va_next < sva) | if (va_next < sva) | ||||
va_next = eva; | va_next = eva; | ||||
continue; | continue; | ||||
} | } | ||||
va_next = (sva + L2_SIZE) & ~L2_OFFSET; | va_next = (sva + L2_SIZE) & ~L2_OFFSET; | ||||
if (va_next < sva) | if (va_next < sva) | ||||
va_next = eva; | va_next = eva; | ||||
l2 = pmap_l1_to_l2(l1, sva); | l2 = pmap_l1_to_l2(l1, sva); | ||||
if (pmap_load(l2) == 0) | if ((l2e = pmap_load(l2)) == 0) | ||||
continue; | continue; | ||||
if ((l2e & PTE_RWX) != 0) { | |||||
if (sva + L2_SIZE == va_next && eva >= va_next) { | |||||
if ((l2e & PTE_SW_WIRED) == 0) | |||||
panic("pmap_unwire: l2 %#jx is missing " | |||||
"PTE_SW_WIRED", (uintmax_t)l2e); | |||||
pmap_clear_bits(l2, PTE_SW_WIRED); | |||||
continue; | |||||
} else { | |||||
if (!pv_lists_locked) { | |||||
pv_lists_locked = true; | |||||
if (!rw_try_rlock(&pvh_global_lock)) { | |||||
PMAP_UNLOCK(pmap); | |||||
rw_rlock(&pvh_global_lock); | |||||
/* Repeat sva. */ | |||||
goto retry; | |||||
} | |||||
} | |||||
if (!pmap_demote_l2(pmap, l2, sva)) | |||||
panic("pmap_unwire: demotion failed"); | |||||
} | |||||
} | |||||
if (va_next > eva) | if (va_next > eva) | ||||
va_next = eva; | va_next = eva; | ||||
for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, | for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, | ||||
sva += L3_SIZE) { | sva += L3_SIZE) { | ||||
if (pmap_load(l3) == 0) | if ((l3e = pmap_load(l3)) == 0) | ||||
continue; | continue; | ||||
if ((pmap_load(l3) & PTE_SW_WIRED) == 0) | if ((l3e & PTE_SW_WIRED) == 0) | ||||
panic("pmap_unwire: l3 %#jx is missing " | panic("pmap_unwire: l3 %#jx is missing " | ||||
"PTE_SW_WIRED", (uintmax_t)*l3); | "PTE_SW_WIRED", (uintmax_t)l3e); | ||||
/* | /* | ||||
* PG_W must be cleared atomically. Although the pmap | * PG_W must be cleared atomically. Although the pmap | ||||
* lock synchronizes access to PG_W, another processor | * lock synchronizes access to PG_W, another processor | ||||
* could be setting PG_M and/or PG_A concurrently. | * could be setting PG_M and/or PG_A concurrently. | ||||
*/ | */ | ||||
atomic_clear_long(l3, PTE_SW_WIRED); | pmap_clear_bits(l3, PTE_SW_WIRED); | ||||
pmap->pm_stats.wired_count--; | pmap->pm_stats.wired_count--; | ||||
} | } | ||||
} | } | ||||
if (pv_lists_locked) | if (pv_lists_locked) | ||||
rw_runlock(&pvh_global_lock); | rw_runlock(&pvh_global_lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 377 Lines • ▼ Show 20 Lines | pmap_remove_pages(pmap_t pmap) | ||||
} | } | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
rw_runlock(&pvh_global_lock); | rw_runlock(&pvh_global_lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
vm_page_free_pages_toq(&free, false); | vm_page_free_pages_toq(&free, false); | ||||
} | } | ||||
/* | static bool | ||||
Done Inline ActionsDoes this comment still apply? I'm not sure what the difference is between dirty and modified is in this context, or where the read/write check is happening. mhorne063_gmail.com: Does this comment still apply? I'm not sure what the difference is between dirty and modified… | |||||
Done Inline ActionsDirty and modified mean the same thing in this context, they're used interchangeably. (amd64 has a "modified" bit whereas riscv has a "dirty" bit.) Indeed, the comment should be removed, it was copied from arm64. markj: Dirty and modified mean the same thing in this context, they're used interchangeably. (amd64… | |||||
* This is used to check if a page has been accessed or modified. As we | |||||
* don't have a bit to see if it has been modified we have to assume it | |||||
* has been if the page is read/write. | |||||
*/ | |||||
static boolean_t | |||||
pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) | pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) | ||||
{ | { | ||||
struct md_page *pvh; | |||||
struct rwlock *lock; | struct rwlock *lock; | ||||
pd_entry_t *l2; | |||||
pt_entry_t *l3, mask; | |||||
pv_entry_t pv; | pv_entry_t pv; | ||||
pt_entry_t *l3, mask, value; | |||||
pmap_t pmap; | pmap_t pmap; | ||||
int md_gen; | int md_gen, pvh_gen; | ||||
boolean_t rv; | bool rv; | ||||
mask = 0; | |||||
if (modified) | |||||
mask |= PTE_D; | |||||
if (accessed) | |||||
mask |= PTE_A; | |||||
Done Inline ActionsYou may wish to convert this to bool as you did in pmap_unwire(), and same with the above signature. Not necessary as part of this patch however. mhorne063_gmail.com: You may wish to convert this to `bool` as you did in `pmap_unwire()`, and same with the above… | |||||
Done Inline ActionsThanks, might as well do it here. markj: Thanks, might as well do it here. | |||||
rv = FALSE; | rv = FALSE; | ||||
rw_rlock(&pvh_global_lock); | rw_rlock(&pvh_global_lock); | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
rw_rlock(lock); | rw_rlock(lock); | ||||
restart: | restart: | ||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
rw_runlock(lock); | rw_runlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_rlock(lock); | rw_rlock(lock); | ||||
if (md_gen != m->md.pv_gen) { | if (md_gen != m->md.pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto restart; | goto restart; | ||||
} | } | ||||
} | } | ||||
l3 = pmap_l3(pmap, pv->pv_va); | l3 = pmap_l3(pmap, pv->pv_va); | ||||
mask = 0; | rv = (pmap_load(l3) & mask) == mask; | ||||
value = 0; | PMAP_UNLOCK(pmap); | ||||
if (modified) { | if (rv) | ||||
mask |= PTE_D; | goto out; | ||||
value |= PTE_D; | |||||
} | } | ||||
if (accessed) { | if ((m->flags & PG_FICTITIOUS) == 0) { | ||||
mask |= PTE_A; | pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
value |= PTE_A; | TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | |||||
if (!PMAP_TRYLOCK(pmap)) { | |||||
md_gen = m->md.pv_gen; | |||||
pvh_gen = pvh->pv_gen; | |||||
rw_runlock(lock); | |||||
PMAP_LOCK(pmap); | |||||
rw_rlock(lock); | |||||
if (md_gen != m->md.pv_gen || | |||||
pvh_gen != pvh->pv_gen) { | |||||
PMAP_UNLOCK(pmap); | |||||
goto restart; | |||||
} | } | ||||
#if 0 | |||||
if (modified) { | |||||
mask |= ATTR_AP_RW_BIT; | |||||
value |= ATTR_AP(ATTR_AP_RW); | |||||
} | } | ||||
if (accessed) { | l2 = pmap_l2(pmap, pv->pv_va); | ||||
mask |= ATTR_AF | ATTR_DESCR_MASK; | rv = (pmap_load(l2) & mask) == mask; | ||||
value |= ATTR_AF | L3_PAGE; | |||||
} | |||||
#endif | |||||
rv = (pmap_load(l3) & mask) == value; | |||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
if (rv) | if (rv) | ||||
goto out; | goto out; | ||||
} | |||||
} | } | ||||
out: | out: | ||||
rw_runlock(lock); | rw_runlock(lock); | ||||
rw_runlock(&pvh_global_lock); | rw_runlock(&pvh_global_lock); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 494 Lines • Show Last 20 Lines |
Should this be a pmap_invalidate_range() of the original sva and eva (you'd have to save the original sva somewhere)? Presumably at some point we could have pmap_invalidate_range() apply some logic to use pmap_invalidate_all() if there were too many pages? (Right now for bbl you might as well always use invalidate_all, but someday that might not be true.)