Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_fault.c
Show First 20 Lines • Show All 536 Lines • ▼ Show 20 Lines | if (map_last < pager_last) { | ||||
vm_fault_populate_cleanup(fs->first_object, map_last + 1, | vm_fault_populate_cleanup(fs->first_object, map_last + 1, | ||||
pager_last); | pager_last); | ||||
pager_last = map_last; | pager_last = map_last; | ||||
} | } | ||||
for (pidx = pager_first, m = vm_page_lookup(fs->first_object, pidx); | for (pidx = pager_first, m = vm_page_lookup(fs->first_object, pidx); | ||||
pidx <= pager_last; | pidx <= pager_last; | ||||
pidx += npages, m = vm_page_next(&m[npages - 1])) { | pidx += npages, m = vm_page_next(&m[npages - 1])) { | ||||
vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset; | vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset; | ||||
#if defined(__aarch64__) || defined(__amd64__) || (defined(__arm__) && \ | |||||
__ARM_ARCH >= 6) || defined(__i386__) || defined(__riscv) || \ | |||||
defined(__powerpc64__) | |||||
psind = m->psind; | psind = m->psind; | ||||
if (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 || | if (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 || | ||||
pidx + OFF_TO_IDX(pagesizes[psind]) - 1 > pager_last || | pidx + OFF_TO_IDX(pagesizes[psind]) - 1 > pager_last || | ||||
!pmap_ps_enabled(fs->map->pmap) || fs->wired)) | !pmap_ps_enabled(fs->map->pmap) || fs->wired)) | ||||
psind = 0; | psind = 0; | ||||
#else | |||||
psind = 0; | |||||
#endif | |||||
npages = atop(pagesizes[psind]); | npages = atop(pagesizes[psind]); | ||||
for (i = 0; i < npages; i++) { | for (i = 0; i < npages; i++) { | ||||
vm_fault_populate_check_page(&m[i]); | vm_fault_populate_check_page(&m[i]); | ||||
vm_fault_dirty(fs, &m[i]); | vm_fault_dirty(fs, &m[i]); | ||||
} | } | ||||
VM_OBJECT_WUNLOCK(fs->first_object); | VM_OBJECT_WUNLOCK(fs->first_object); | ||||
rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot, fs->fault_type | | rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot, fs->fault_type | | ||||
(fs->wired ? PMAP_ENTER_WIRED : 0), psind); | (fs->wired ? PMAP_ENTER_WIRED : 0), psind); | ||||
#if defined(__amd64__) | |||||
if (psind > 0 && rv == KERN_FAILURE) { | /* | ||||
* pmap_enter() may fail for a superpage mapping if additional | |||||
kib: I would say 'lower-level', since PKRU is somewhere below pmap. Or remove the 'higher-level'… | |||||
* protection policies prevent the full mapping. | |||||
* For example, this will happen on amd64 if the entire | |||||
* address range does not share the same userspace protection | |||||
* key. Revert to single-page mappings if this happens. | |||||
*/ | |||||
MPASS(rv == KERN_SUCCESS || | |||||
(psind > 0 && rv == KERN_PROTECTION_FAILURE)); | |||||
if (__predict_false(psind > 0 && | |||||
rv == KERN_PROTECTION_FAILURE)) { | |||||
for (i = 0; i < npages; i++) { | for (i = 0; i < npages; i++) { | ||||
rv = pmap_enter(fs->map->pmap, vaddr + ptoa(i), | rv = pmap_enter(fs->map->pmap, vaddr + ptoa(i), | ||||
&m[i], fs->prot, fs->fault_type | | &m[i], fs->prot, fs->fault_type | | ||||
(fs->wired ? PMAP_ENTER_WIRED : 0), 0); | (fs->wired ? PMAP_ENTER_WIRED : 0), 0); | ||||
MPASS(rv == KERN_SUCCESS); | MPASS(rv == KERN_SUCCESS); | ||||
} | } | ||||
} | } | ||||
#else | |||||
Not Done Inline ActionsIMO it would be cleaner to assert rv value right after pmap_enter(). Something like kib: IMO it would be cleaner to assert rv value right after pmap_enter(). Something like
`MPASS(rv… | |||||
MPASS(rv == KERN_SUCCESS); | |||||
#endif | |||||
VM_OBJECT_WLOCK(fs->first_object); | VM_OBJECT_WLOCK(fs->first_object); | ||||
for (i = 0; i < npages; i++) { | for (i = 0; i < npages; i++) { | ||||
if ((fs->fault_flags & VM_FAULT_WIRE) != 0) | if ((fs->fault_flags & VM_FAULT_WIRE) != 0) | ||||
vm_page_wire(&m[i]); | vm_page_wire(&m[i]); | ||||
else | else | ||||
vm_page_activate(&m[i]); | vm_page_activate(&m[i]); | ||||
if (fs->m_hold != NULL && m[i].pindex == fs->first_pindex) { | if (fs->m_hold != NULL && m[i].pindex == fs->first_pindex) { | ||||
(*fs->m_hold) = &m[i]; | (*fs->m_hold) = &m[i]; | ||||
▲ Show 20 Lines • Show All 1,501 Lines • Show Last 20 Lines |
I would say 'lower-level', since PKRU is somewhere below pmap. Or remove the 'higher-level' words.