Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_fault.c
Show First 20 Lines • Show All 414 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static int | static int | ||||
vm_fault_populate(struct faultstate *fs) | vm_fault_populate(struct faultstate *fs) | ||||
{ | { | ||||
vm_offset_t vaddr; | vm_offset_t vaddr; | ||||
vm_page_t m; | vm_page_t m; | ||||
vm_pindex_t map_first, map_last, pager_first, pager_last, pidx; | vm_pindex_t map_first, map_last, pager_first, pager_last, pidx; | ||||
int i, npages, psind, rv; | int bdry_idx, i, npages, psind, rv; | ||||
MPASS(fs->object == fs->first_object); | MPASS(fs->object == fs->first_object); | ||||
VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | ||||
MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); | MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); | ||||
MPASS(fs->first_object->backing_object == NULL); | MPASS(fs->first_object->backing_object == NULL); | ||||
MPASS(fs->lookup_still_valid); | MPASS(fs->lookup_still_valid); | ||||
pager_first = OFF_TO_IDX(fs->entry->offset); | pager_first = OFF_TO_IDX(fs->entry->offset); | ||||
pager_last = pager_first + atop(fs->entry->end - fs->entry->start) - 1; | pager_last = pager_first + atop(fs->entry->end - fs->entry->start) - 1; | ||||
unlock_map(fs); | unlock_map(fs); | ||||
unlock_vp(fs); | unlock_vp(fs); | ||||
/* | /* | ||||
* Call the pager (driver) populate() method. | * Call the pager (driver) populate() method. | ||||
* | * | ||||
* There is no guarantee that the method will be called again | * There is no guarantee that the method will be called again | ||||
* if the current fault is for read, and a future fault is | * if the current fault is for read, and a future fault is | ||||
* for write. Report the entry's maximum allowed protection | * for write. Report the entry's maximum allowed protection | ||||
* to the driver. | * to the driver. | ||||
*/ | */ | ||||
rv = vm_pager_populate(fs->first_object, fs->first_pindex, | rv = vm_pager_populate(fs->first_object, fs->first_pindex, | ||||
fs->fault_type, fs->entry->max_protection, &pager_first, &pager_last); | fs->fault_type, fs->entry->max_protection, &pager_first, | ||||
&pager_last); | |||||
VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | ||||
if (rv == VM_PAGER_BAD) { | if (rv == VM_PAGER_BAD) { | ||||
/* | /* | ||||
* VM_PAGER_BAD is the backdoor for a pager to request | * VM_PAGER_BAD is the backdoor for a pager to request | ||||
* normal fault handling. | * normal fault handling. | ||||
*/ | */ | ||||
vm_fault_restore_map_lock(fs); | vm_fault_restore_map_lock(fs); | ||||
if (fs->map->timestamp != fs->map_generation) | if (fs->map->timestamp != fs->map_generation) | ||||
return (KERN_RESTART); | return (KERN_RESTART); | ||||
return (KERN_NOT_RECEIVER); | return (KERN_NOT_RECEIVER); | ||||
} | } | ||||
if (rv != VM_PAGER_OK) | if (rv != VM_PAGER_OK) | ||||
return (KERN_FAILURE); /* AKA SIGSEGV */ | return (KERN_FAILURE); /* AKA SIGSEGV */ | ||||
/* Ensure that the driver is obeying the interface. */ | /* Ensure that the driver is obeying the interface. */ | ||||
MPASS(pager_first <= pager_last); | MPASS(pager_first <= pager_last); | ||||
MPASS(fs->first_pindex <= pager_last); | MPASS(fs->first_pindex <= pager_last); | ||||
MPASS(fs->first_pindex >= pager_first); | MPASS(fs->first_pindex >= pager_first); | ||||
MPASS(pager_last < fs->first_object->size); | MPASS(pager_last < fs->first_object->size); | ||||
vm_fault_restore_map_lock(fs); | vm_fault_restore_map_lock(fs); | ||||
bdry_idx = (fs->entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) >> | |||||
MAP_ENTRY_SPLIT_BOUNDARY_SHIFT; | |||||
if (fs->map->timestamp != fs->map_generation) { | if (fs->map->timestamp != fs->map_generation) { | ||||
if (bdry_idx == 0) { | |||||
vm_fault_populate_cleanup(fs->first_object, pager_first, | vm_fault_populate_cleanup(fs->first_object, pager_first, | ||||
pager_last); | pager_last); | ||||
} else { | |||||
m = vm_page_lookup(fs->first_object, pager_first); | |||||
if (m != fs->m) | |||||
vm_page_xunbusy(m); | |||||
} | |||||
return (KERN_RESTART); | return (KERN_RESTART); | ||||
} | } | ||||
/* | /* | ||||
* The map is unchanged after our last unlock. Process the fault. | * The map is unchanged after our last unlock. Process the fault. | ||||
* | * | ||||
* First, the special case of largepage mappings, where | |||||
* populate only busies the first page in superpage run. | |||||
*/ | |||||
if (bdry_idx != 0) { | |||||
m = vm_page_lookup(fs->first_object, pager_first); | |||||
vm_fault_populate_check_page(m); | |||||
VM_OBJECT_WUNLOCK(fs->first_object); | |||||
vaddr = fs->entry->start + IDX_TO_OFF(pager_first) - | |||||
fs->entry->offset; | |||||
/* assert alignment for entry */ | |||||
KASSERT((vaddr & (pagesizes[bdry_idx] - 1)) == 0, | |||||
("unaligned superpage start %#jx pager_first %#jx offset %#jx vaddr %#jx", | |||||
(uintmax_t)fs->entry->start, (uintmax_t)pager_first, | |||||
(uintmax_t)fs->entry->offset, (uintmax_t)vaddr)); | |||||
KASSERT((VM_PAGE_TO_PHYS(m) & (pagesizes[bdry_idx] - 1)) == 0, | |||||
("unaligned superpage m %p %#jx", m, | |||||
(uintmax_t)VM_PAGE_TO_PHYS(m))); | |||||
rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot, | |||||
fs->fault_type | (fs->wired ? PMAP_ENTER_WIRED : 0) | | |||||
markj: We are not passing PMAP_ENTER_WIRED even if the fault is from mlock(). More on that below. | |||||
PMAP_ENTER_LARGEPAGE, bdry_idx); | |||||
VM_OBJECT_WLOCK(fs->first_object); | |||||
vm_page_xunbusy(m); | |||||
if ((fs->fault_flags & VM_FAULT_WIRE) != 0) { | |||||
for (i = 0; i < atop(pagesizes[bdry_idx]); i++) | |||||
vm_page_wire(m + i); | |||||
} | |||||
if (fs->m_hold != NULL) { | |||||
*fs->m_hold = m + (fs->first_pindex - pager_first); | |||||
vm_page_wire(*fs->m_hold); | |||||
} | |||||
goto out; | |||||
} | |||||
/* | |||||
* The range [pager_first, pager_last] that is given to the | * The range [pager_first, pager_last] that is given to the | ||||
* pager is only a hint. The pager may populate any range | * pager is only a hint. The pager may populate any range | ||||
* within the object that includes the requested page index. | * within the object that includes the requested page index. | ||||
* In case the pager expanded the range, clip it to fit into | * In case the pager expanded the range, clip it to fit into | ||||
* the map entry. | * the map entry. | ||||
*/ | */ | ||||
map_first = OFF_TO_IDX(fs->entry->offset); | map_first = OFF_TO_IDX(fs->entry->offset); | ||||
if (map_first > pager_first) { | if (map_first > pager_first) { | ||||
vm_fault_populate_cleanup(fs->first_object, pager_first, | vm_fault_populate_cleanup(fs->first_object, pager_first, | ||||
map_first - 1); | map_first - 1); | ||||
pager_first = map_first; | pager_first = map_first; | ||||
} | } | ||||
map_last = map_first + atop(fs->entry->end - fs->entry->start) - 1; | map_last = map_first + atop(fs->entry->end - fs->entry->start) - 1; | ||||
if (map_last < pager_last) { | if (map_last < pager_last) { | ||||
vm_fault_populate_cleanup(fs->first_object, map_last + 1, | vm_fault_populate_cleanup(fs->first_object, map_last + 1, | ||||
pager_last); | pager_last); | ||||
pager_last = map_last; | pager_last = map_last; | ||||
} | } | ||||
for (pidx = pager_first, m = vm_page_lookup(fs->first_object, pidx); | for (pidx = pager_first, m = vm_page_lookup(fs->first_object, pidx); | ||||
Done Inline Actions/* should be on its own line. markj: `/*` should be on its own line. | |||||
pidx <= pager_last; | pidx <= pager_last; | ||||
pidx += npages, m = vm_page_next(&m[npages - 1])) { | pidx += npages, m = vm_page_next(&m[npages - 1])) { | ||||
vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset; | vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset; | ||||
#if defined(__aarch64__) || defined(__amd64__) || (defined(__arm__) && \ | #if defined(__aarch64__) || defined(__amd64__) || (defined(__arm__) && \ | ||||
__ARM_ARCH >= 6) || defined(__i386__) || defined(__riscv) | __ARM_ARCH >= 6) || defined(__i386__) || defined(__riscv) | ||||
psind = m->psind; | psind = m->psind; | ||||
if (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 || | if (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 || | ||||
pidx + OFF_TO_IDX(pagesizes[psind]) - 1 > pager_last || | pidx + OFF_TO_IDX(pagesizes[psind]) - 1 > pager_last || | ||||
Show All 30 Lines | for (i = 0; i < npages; i++) { | ||||
vm_page_activate(&m[i]); | vm_page_activate(&m[i]); | ||||
if (fs->m_hold != NULL && m[i].pindex == fs->first_pindex) { | if (fs->m_hold != NULL && m[i].pindex == fs->first_pindex) { | ||||
(*fs->m_hold) = &m[i]; | (*fs->m_hold) = &m[i]; | ||||
vm_page_wire(&m[i]); | vm_page_wire(&m[i]); | ||||
} | } | ||||
vm_page_xunbusy(&m[i]); | vm_page_xunbusy(&m[i]); | ||||
} | } | ||||
} | } | ||||
out: | |||||
curthread->td_ru.ru_majflt++; | curthread->td_ru.ru_majflt++; | ||||
return (KERN_SUCCESS); | return (KERN_SUCCESS); | ||||
} | } | ||||
static int prot_fault_translation; | static int prot_fault_translation; | ||||
SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RWTUN, | SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RWTUN, | ||||
&prot_fault_translation, 0, | &prot_fault_translation, 0, | ||||
"Control signal to deliver on protection fault"); | "Control signal to deliver on protection fault"); | ||||
▲ Show 20 Lines • Show All 700 Lines • ▼ Show 20 Lines | RetryFault: | ||||
/* | /* | ||||
* Try to avoid lock contention on the top-level object through | * Try to avoid lock contention on the top-level object through | ||||
* special-case handling of some types of page faults, specifically, | * special-case handling of some types of page faults, specifically, | ||||
* those that are mapping an existing page from the top-level object. | * those that are mapping an existing page from the top-level object. | ||||
* Under this condition, a read lock on the object suffices, allowing | * Under this condition, a read lock on the object suffices, allowing | ||||
* multiple page faults of a similar type to run in parallel. | * multiple page faults of a similar type to run in parallel. | ||||
*/ | */ | ||||
if (fs.vp == NULL /* avoid locked vnode leak */ && | if (fs.vp == NULL /* avoid locked vnode leak */ && | ||||
(fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) == 0 && | |||||
(fs.fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { | (fs.fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { | ||||
VM_OBJECT_RLOCK(fs.first_object); | VM_OBJECT_RLOCK(fs.first_object); | ||||
rv = vm_fault_soft_fast(&fs); | rv = vm_fault_soft_fast(&fs); | ||||
if (rv == KERN_SUCCESS) | if (rv == KERN_SUCCESS) | ||||
return (rv); | return (rv); | ||||
if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) { | if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) { | ||||
VM_OBJECT_RUNLOCK(fs.first_object); | VM_OBJECT_RUNLOCK(fs.first_object); | ||||
VM_OBJECT_WLOCK(fs.first_object); | VM_OBJECT_WLOCK(fs.first_object); | ||||
Show All 16 Lines | RetryFault: | ||||
fs.m_cow = fs.m = fs.first_m = NULL; | fs.m_cow = fs.m = fs.first_m = NULL; | ||||
/* | /* | ||||
* Search for the page at object/offset. | * Search for the page at object/offset. | ||||
*/ | */ | ||||
fs.object = fs.first_object; | fs.object = fs.first_object; | ||||
fs.pindex = fs.first_pindex; | fs.pindex = fs.first_pindex; | ||||
if ((fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) != 0) { | |||||
rv = vm_fault_allocate(&fs); | |||||
switch (rv) { | |||||
case KERN_RESTART: | |||||
unlock_and_deallocate(&fs); | |||||
/* FALLTHROUGH */ | |||||
case KERN_RESOURCE_SHORTAGE: | |||||
goto RetryFault; | |||||
case KERN_SUCCESS: | |||||
case KERN_FAILURE: | |||||
case KERN_OUT_OF_BOUNDS: | |||||
unlock_and_deallocate(&fs); | |||||
return (rv); | |||||
case KERN_NOT_RECEIVER: | |||||
break; | |||||
default: | |||||
panic("vm_fault: Unhandled rv %d", rv); | |||||
} | |||||
} | |||||
while (TRUE) { | while (TRUE) { | ||||
KASSERT(fs.m == NULL, | KASSERT(fs.m == NULL, | ||||
("page still set %p at loop start", fs.m)); | ("page still set %p at loop start", fs.m)); | ||||
/* | /* | ||||
* If the object is marked for imminent termination, | * If the object is marked for imminent termination, | ||||
* we retry here, since the collapse pass has raced | * we retry here, since the collapse pass has raced | ||||
* with us. Otherwise, if we see terminally dead | * with us. Otherwise, if we see terminally dead | ||||
* object, return fail. | * object, return fail. | ||||
▲ Show 20 Lines • Show All 710 Lines • Show Last 20 Lines |
We are not passing PMAP_ENTER_WIRED even if the fault is from mlock(). More on that below.