Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_fault.c
Show First 20 Lines • Show All 149 Lines • ▼ Show 20 Lines | struct faultstate { | ||||
vm_map_entry_t entry; | vm_map_entry_t entry; | ||||
int map_generation; | int map_generation; | ||||
bool lookup_still_valid; | bool lookup_still_valid; | ||||
/* Vnode if locked. */ | /* Vnode if locked. */ | ||||
struct vnode *vp; | struct vnode *vp; | ||||
}; | }; | ||||
/* | |||||
* Return codes for internal fault routines. | |||||
*/ | |||||
enum fault_status { | |||||
FAULT_SUCCESS = 1, /* Return success to user. */ | |||||
FAULT_FAILURE, /* Return failure to user. */ | |||||
FAULT_CONTINUE, /* Continue faulting. */ | |||||
FAULT_RESTART, /* Restart fault. */ | |||||
FAULT_OUT_OF_BOUNDS, /* Invalid address for pager. */ | |||||
FAULT_HARD, /* Performed I/O. */ | |||||
FAULT_SOFT, /* Found valid page. */ | |||||
}; | |||||
static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr, | static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr, | ||||
int ahead); | int ahead); | ||||
static void vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra, | static void vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra, | ||||
int backward, int forward, bool obj_locked); | int backward, int forward, bool obj_locked); | ||||
static int vm_pfault_oom_attempts = 3; | static int vm_pfault_oom_attempts = 3; | ||||
SYSCTL_INT(_vm, OID_AUTO, pfault_oom_attempts, CTLFLAG_RWTUN, | SYSCTL_INT(_vm, OID_AUTO, pfault_oom_attempts, CTLFLAG_RWTUN, | ||||
&vm_pfault_oom_attempts, 0, | &vm_pfault_oom_attempts, 0, | ||||
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines | else | ||||
vm_page_aflag_clear(m, PGA_NOSYNC); | vm_page_aflag_clear(m, PGA_NOSYNC); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Unlocks fs.first_object and fs.map on success. | * Unlocks fs.first_object and fs.map on success. | ||||
*/ | */ | ||||
static int | static enum fault_status | ||||
vm_fault_soft_fast(struct faultstate *fs) | vm_fault_soft_fast(struct faultstate *fs) | ||||
{ | { | ||||
vm_page_t m, m_map; | vm_page_t m, m_map; | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
vm_page_t m_super; | vm_page_t m_super; | ||||
int flags; | int flags; | ||||
#endif | #endif | ||||
int psind, rv; | int psind, rv; | ||||
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | if (psind == 0 && !fs->wired) | ||||
vm_fault_prefault(fs, vaddr, PFBAK, PFFOR, true); | vm_fault_prefault(fs, vaddr, PFBAK, PFFOR, true); | ||||
VM_OBJECT_RUNLOCK(fs->first_object); | VM_OBJECT_RUNLOCK(fs->first_object); | ||||
vm_fault_dirty(fs, m); | vm_fault_dirty(fs, m); | ||||
vm_map_lookup_done(fs->map, fs->entry); | vm_map_lookup_done(fs->map, fs->entry); | ||||
curthread->td_ru.ru_minflt++; | curthread->td_ru.ru_minflt++; | ||||
out: | out: | ||||
vm_object_unbusy(fs->first_object); | vm_object_unbusy(fs->first_object); | ||||
return (rv); | return (rv == KERN_SUCCESS ? FAULT_SUCCESS : FAULT_FAILURE); | ||||
} | } | ||||
kib: I would convert this function more thoroughly. If you introduce `enum fault_status res`… | |||||
static void | static void | ||||
vm_fault_restore_map_lock(struct faultstate *fs) | vm_fault_restore_map_lock(struct faultstate *fs) | ||||
{ | { | ||||
VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | ||||
MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); | MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); | ||||
Show All 31 Lines | vm_fault_populate_cleanup(vm_object_t object, vm_pindex_t first, | ||||
for (pidx = first, m = vm_page_lookup(object, pidx); | for (pidx = first, m = vm_page_lookup(object, pidx); | ||||
pidx <= last; pidx++, m = vm_page_next(m)) { | pidx <= last; pidx++, m = vm_page_next(m)) { | ||||
vm_fault_populate_check_page(m); | vm_fault_populate_check_page(m); | ||||
vm_page_deactivate(m); | vm_page_deactivate(m); | ||||
vm_page_xunbusy(m); | vm_page_xunbusy(m); | ||||
} | } | ||||
} | } | ||||
static int | static enum fault_status | ||||
vm_fault_populate(struct faultstate *fs) | vm_fault_populate(struct faultstate *fs) | ||||
{ | { | ||||
vm_offset_t vaddr; | vm_offset_t vaddr; | ||||
vm_page_t m; | vm_page_t m; | ||||
vm_pindex_t map_first, map_last, pager_first, pager_last, pidx; | vm_pindex_t map_first, map_last, pager_first, pager_last, pidx; | ||||
int bdry_idx, i, npages, psind, rv; | int bdry_idx, i, npages, psind, rv; | ||||
MPASS(fs->object == fs->first_object); | MPASS(fs->object == fs->first_object); | ||||
Show All 22 Lines | vm_fault_populate(struct faultstate *fs) | ||||
VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | ||||
if (rv == VM_PAGER_BAD) { | if (rv == VM_PAGER_BAD) { | ||||
/* | /* | ||||
* VM_PAGER_BAD is the backdoor for a pager to request | * VM_PAGER_BAD is the backdoor for a pager to request | ||||
* normal fault handling. | * normal fault handling. | ||||
*/ | */ | ||||
vm_fault_restore_map_lock(fs); | vm_fault_restore_map_lock(fs); | ||||
if (fs->map->timestamp != fs->map_generation) | if (fs->map->timestamp != fs->map_generation) | ||||
return (KERN_RESTART); | return (FAULT_RESTART); | ||||
return (KERN_NOT_RECEIVER); | return (FAULT_CONTINUE); | ||||
} | } | ||||
if (rv != VM_PAGER_OK) | if (rv != VM_PAGER_OK) | ||||
return (KERN_FAILURE); /* AKA SIGSEGV */ | return (FAULT_FAILURE); /* AKA SIGSEGV */ | ||||
/* Ensure that the driver is obeying the interface. */ | /* Ensure that the driver is obeying the interface. */ | ||||
MPASS(pager_first <= pager_last); | MPASS(pager_first <= pager_last); | ||||
MPASS(fs->first_pindex <= pager_last); | MPASS(fs->first_pindex <= pager_last); | ||||
MPASS(fs->first_pindex >= pager_first); | MPASS(fs->first_pindex >= pager_first); | ||||
MPASS(pager_last < fs->first_object->size); | MPASS(pager_last < fs->first_object->size); | ||||
vm_fault_restore_map_lock(fs); | vm_fault_restore_map_lock(fs); | ||||
bdry_idx = (fs->entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) >> | bdry_idx = (fs->entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) >> | ||||
MAP_ENTRY_SPLIT_BOUNDARY_SHIFT; | MAP_ENTRY_SPLIT_BOUNDARY_SHIFT; | ||||
if (fs->map->timestamp != fs->map_generation) { | if (fs->map->timestamp != fs->map_generation) { | ||||
if (bdry_idx == 0) { | if (bdry_idx == 0) { | ||||
vm_fault_populate_cleanup(fs->first_object, pager_first, | vm_fault_populate_cleanup(fs->first_object, pager_first, | ||||
pager_last); | pager_last); | ||||
} else { | } else { | ||||
m = vm_page_lookup(fs->first_object, pager_first); | m = vm_page_lookup(fs->first_object, pager_first); | ||||
if (m != fs->m) | if (m != fs->m) | ||||
vm_page_xunbusy(m); | vm_page_xunbusy(m); | ||||
} | } | ||||
return (KERN_RESTART); | return (FAULT_RESTART); | ||||
} | } | ||||
/* | /* | ||||
* The map is unchanged after our last unlock. Process the fault. | * The map is unchanged after our last unlock. Process the fault. | ||||
* | * | ||||
* First, the special case of largepage mappings, where | * First, the special case of largepage mappings, where | ||||
* populate only busies the first page in superpage run. | * populate only busies the first page in superpage run. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | for (i = 0; i < npages; i++) { | ||||
(*fs->m_hold) = &m[i]; | (*fs->m_hold) = &m[i]; | ||||
vm_page_wire(&m[i]); | vm_page_wire(&m[i]); | ||||
} | } | ||||
vm_page_xunbusy(&m[i]); | vm_page_xunbusy(&m[i]); | ||||
} | } | ||||
} | } | ||||
out: | out: | ||||
curthread->td_ru.ru_majflt++; | curthread->td_ru.ru_majflt++; | ||||
return (rv); | return (rv == KERN_SUCCESS ? FAULT_SUCCESS : FAULT_FAILURE); | ||||
} | } | ||||
kibUnsubmitted Done Inline ActionsI would also introduce a res for this function. kib: I would also introduce a `res` for this function. | |||||
static int prot_fault_translation; | static int prot_fault_translation; | ||||
SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RWTUN, | SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RWTUN, | ||||
&prot_fault_translation, 0, | &prot_fault_translation, 0, | ||||
"Control signal to deliver on protection fault"); | "Control signal to deliver on protection fault"); | ||||
/* compat definition to keep common code for signal translation */ | /* compat definition to keep common code for signal translation */ | ||||
#define UCODE_PAGEFLT 12 | #define UCODE_PAGEFLT 12 | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | default: | ||||
KASSERT(0, ("Unexpected Mach error %d from vm_fault()", | KASSERT(0, ("Unexpected Mach error %d from vm_fault()", | ||||
result)); | result)); | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
return (result); | return (result); | ||||
} | } | ||||
static int | static enum fault_status | ||||
vm_fault_lock_vnode(struct faultstate *fs, bool objlocked) | vm_fault_lock_vnode(struct faultstate *fs, bool objlocked) | ||||
{ | { | ||||
struct vnode *vp; | struct vnode *vp; | ||||
int error, locked; | int error, locked; | ||||
if (fs->object->type != OBJT_VNODE) | if (fs->object->type != OBJT_VNODE) | ||||
return (KERN_SUCCESS); | return (FAULT_CONTINUE); | ||||
vp = fs->object->handle; | vp = fs->object->handle; | ||||
if (vp == fs->vp) { | if (vp == fs->vp) { | ||||
ASSERT_VOP_LOCKED(vp, "saved vnode is not locked"); | ASSERT_VOP_LOCKED(vp, "saved vnode is not locked"); | ||||
return (KERN_SUCCESS); | return (FAULT_CONTINUE); | ||||
} | } | ||||
/* | /* | ||||
* Perform an unlock in case the desired vnode changed while | * Perform an unlock in case the desired vnode changed while | ||||
* the map was unlocked during a retry. | * the map was unlocked during a retry. | ||||
*/ | */ | ||||
unlock_vp(fs); | unlock_vp(fs); | ||||
locked = VOP_ISLOCKED(vp); | locked = VOP_ISLOCKED(vp); | ||||
if (locked != LK_EXCLUSIVE) | if (locked != LK_EXCLUSIVE) | ||||
locked = LK_SHARED; | locked = LK_SHARED; | ||||
/* | /* | ||||
* We must not sleep acquiring the vnode lock while we have | * We must not sleep acquiring the vnode lock while we have | ||||
* the page exclusive busied or the object's | * the page exclusive busied or the object's | ||||
* paging-in-progress count incremented. Otherwise, we could | * paging-in-progress count incremented. Otherwise, we could | ||||
* deadlock. | * deadlock. | ||||
*/ | */ | ||||
error = vget(vp, locked | LK_CANRECURSE | LK_NOWAIT); | error = vget(vp, locked | LK_CANRECURSE | LK_NOWAIT); | ||||
if (error == 0) { | if (error == 0) { | ||||
fs->vp = vp; | fs->vp = vp; | ||||
return (KERN_SUCCESS); | return (FAULT_CONTINUE); | ||||
} | } | ||||
vhold(vp); | vhold(vp); | ||||
if (objlocked) | if (objlocked) | ||||
unlock_and_deallocate(fs); | unlock_and_deallocate(fs); | ||||
else | else | ||||
fault_deallocate(fs); | fault_deallocate(fs); | ||||
error = vget(vp, locked | LK_RETRY | LK_CANRECURSE); | error = vget(vp, locked | LK_RETRY | LK_CANRECURSE); | ||||
vdrop(vp); | vdrop(vp); | ||||
fs->vp = vp; | fs->vp = vp; | ||||
KASSERT(error == 0, ("vm_fault: vget failed %d", error)); | KASSERT(error == 0, ("vm_fault: vget failed %d", error)); | ||||
return (KERN_RESOURCE_SHORTAGE); | return (FAULT_RESTART); | ||||
} | } | ||||
/* | /* | ||||
* Calculate the desired readahead. Handle drop-behind. | * Calculate the desired readahead. Handle drop-behind. | ||||
* | * | ||||
* Returns the number of readahead blocks to pass to the pager. | * Returns the number of readahead blocks to pass to the pager. | ||||
*/ | */ | ||||
static int | static int | ||||
▲ Show 20 Lines • Show All 351 Lines • ▼ Show 20 Lines | vm_fault_allocate_oom(struct faultstate *fs) | ||||
vm_pageout_oom(VM_OOM_MEM_PF); | vm_pageout_oom(VM_OOM_MEM_PF); | ||||
fs->oom_started = false; | fs->oom_started = false; | ||||
return (false); | return (false); | ||||
} | } | ||||
/* | /* | ||||
* Allocate a page directly or via the object populate method. | * Allocate a page directly or via the object populate method. | ||||
*/ | */ | ||||
static int | static enum fault_status | ||||
vm_fault_allocate(struct faultstate *fs) | vm_fault_allocate(struct faultstate *fs) | ||||
{ | { | ||||
struct domainset *dset; | struct domainset *dset; | ||||
int rv; | enum fault_status rv; | ||||
kibUnsubmitted Done Inline ActionsI would prefer to keep rv name for Mach errors, and use some other name (res?) for fault_status. kib: I would prefer to keep rv name for Mach errors, and use some other name (res?) for fault_status. | |||||
if ((fs->object->flags & OBJ_SIZEVNLOCK) != 0) { | if ((fs->object->flags & OBJ_SIZEVNLOCK) != 0) { | ||||
rv = vm_fault_lock_vnode(fs, true); | rv = vm_fault_lock_vnode(fs, true); | ||||
MPASS(rv == KERN_SUCCESS || rv == KERN_RESOURCE_SHORTAGE); | MPASS(rv == FAULT_CONTINUE || rv == FAULT_RESTART); | ||||
if (rv == KERN_RESOURCE_SHORTAGE) | if (rv == FAULT_RESTART) | ||||
return (rv); | return (rv); | ||||
} | } | ||||
if (fs->pindex >= fs->object->size) | if (fs->pindex >= fs->object->size) { | ||||
return (KERN_OUT_OF_BOUNDS); | unlock_and_deallocate(fs); | ||||
return (FAULT_OUT_OF_BOUNDS); | |||||
} | |||||
if (fs->object == fs->first_object && | if (fs->object == fs->first_object && | ||||
(fs->first_object->flags & OBJ_POPULATE) != 0 && | (fs->first_object->flags & OBJ_POPULATE) != 0 && | ||||
fs->first_object->shadow_count == 0) { | fs->first_object->shadow_count == 0) { | ||||
rv = vm_fault_populate(fs); | rv = vm_fault_populate(fs); | ||||
switch (rv) { | switch (rv) { | ||||
case KERN_SUCCESS: | case FAULT_SUCCESS: | ||||
case KERN_FAILURE: | case FAULT_FAILURE: | ||||
case KERN_PROTECTION_FAILURE: | case FAULT_RESTART: | ||||
case KERN_RESTART: | unlock_and_deallocate(fs); | ||||
return (rv); | return (rv); | ||||
case KERN_NOT_RECEIVER: | case FAULT_CONTINUE: | ||||
/* | /* | ||||
* Pager's populate() method | * Pager's populate() method | ||||
* returned VM_PAGER_BAD. | * returned VM_PAGER_BAD. | ||||
*/ | */ | ||||
break; | break; | ||||
default: | default: | ||||
panic("inconsistent return codes"); | panic("inconsistent return codes"); | ||||
} | } | ||||
Show All 19 Lines | #if VM_NRESERVLEVEL > 0 | ||||
vm_object_color(fs->object, atop(fs->vaddr) - fs->pindex); | vm_object_color(fs->object, atop(fs->vaddr) - fs->pindex); | ||||
#endif | #endif | ||||
fs->m = vm_page_alloc(fs->object, fs->pindex, | fs->m = vm_page_alloc(fs->object, fs->pindex, | ||||
P_KILLED(curproc) ? VM_ALLOC_SYSTEM : 0); | P_KILLED(curproc) ? VM_ALLOC_SYSTEM : 0); | ||||
} | } | ||||
if (fs->m == NULL) { | if (fs->m == NULL) { | ||||
if (vm_fault_allocate_oom(fs)) | if (vm_fault_allocate_oom(fs)) | ||||
vm_waitpfault(dset, vm_pfault_oom_wait * hz); | vm_waitpfault(dset, vm_pfault_oom_wait * hz); | ||||
return (KERN_RESOURCE_SHORTAGE); | return (FAULT_RESTART); | ||||
} | } | ||||
fs->oom_started = false; | fs->oom_started = false; | ||||
return (KERN_NOT_RECEIVER); | return (FAULT_CONTINUE); | ||||
} | } | ||||
/* | /* | ||||
* Call the pager to retrieve the page if there is a chance | * Call the pager to retrieve the page if there is a chance | ||||
* that the pager has it, and potentially retrieve additional | * that the pager has it, and potentially retrieve additional | ||||
* pages at the same time. | * pages at the same time. | ||||
*/ | */ | ||||
static int | static enum fault_status | ||||
vm_fault_getpages(struct faultstate *fs, int *behindp, int *aheadp) | vm_fault_getpages(struct faultstate *fs, int *behindp, int *aheadp) | ||||
{ | { | ||||
vm_offset_t e_end, e_start; | vm_offset_t e_end, e_start; | ||||
int ahead, behind, cluster_offset, rv; | int ahead, behind, cluster_offset, rv; | ||||
enum fault_status status; | |||||
u_char behavior; | u_char behavior; | ||||
/* | /* | ||||
* Prepare for unlocking the map. Save the map | * Prepare for unlocking the map. Save the map | ||||
* entry's start and end addresses, which are used to | * entry's start and end addresses, which are used to | ||||
* optimize the size of the pager operation below. | * optimize the size of the pager operation below. | ||||
* Even if the map entry's addresses change after | * Even if the map entry's addresses change after | ||||
* unlocking the map, using the saved addresses is | * unlocking the map, using the saved addresses is | ||||
Show All 20 Lines | vm_fault_getpages(struct faultstate *fs, int *behindp, int *aheadp) | ||||
/* | /* | ||||
* Release the map lock before locking the vnode or | * Release the map lock before locking the vnode or | ||||
* sleeping in the pager. (If the current object has | * sleeping in the pager. (If the current object has | ||||
* a shadow, then an earlier iteration of this loop | * a shadow, then an earlier iteration of this loop | ||||
* may have already unlocked the map.) | * may have already unlocked the map.) | ||||
*/ | */ | ||||
unlock_map(fs); | unlock_map(fs); | ||||
rv = vm_fault_lock_vnode(fs, false); | status = vm_fault_lock_vnode(fs, false); | ||||
MPASS(rv == KERN_SUCCESS || rv == KERN_RESOURCE_SHORTAGE); | MPASS(status == FAULT_CONTINUE || status == FAULT_RESTART); | ||||
if (rv == KERN_RESOURCE_SHORTAGE) | if (status == FAULT_RESTART) | ||||
return (rv); | return (status); | ||||
KASSERT(fs->vp == NULL || !fs->map->system_map, | KASSERT(fs->vp == NULL || !fs->map->system_map, | ||||
("vm_fault: vnode-backed object mapped by system map")); | ("vm_fault: vnode-backed object mapped by system map")); | ||||
/* | /* | ||||
* Page in the requested page and hint the pager, | * Page in the requested page and hint the pager, | ||||
* that it may bring up surrounding pages. | * that it may bring up surrounding pages. | ||||
*/ | */ | ||||
if (fs->nera == -1 || behavior == MAP_ENTRY_BEHAV_RANDOM || | if (fs->nera == -1 || behavior == MAP_ENTRY_BEHAV_RANDOM || | ||||
Show All 22 Lines | if (fs->nera > 0) { | ||||
ahead = VM_FAULT_READ_DEFAULT - 1 - cluster_offset; | ahead = VM_FAULT_READ_DEFAULT - 1 - cluster_offset; | ||||
} | } | ||||
ahead = ulmin(ahead, atop(e_end - fs->vaddr) - 1); | ahead = ulmin(ahead, atop(e_end - fs->vaddr) - 1); | ||||
} | } | ||||
*behindp = behind; | *behindp = behind; | ||||
*aheadp = ahead; | *aheadp = ahead; | ||||
rv = vm_pager_get_pages(fs->object, &fs->m, 1, behindp, aheadp); | rv = vm_pager_get_pages(fs->object, &fs->m, 1, behindp, aheadp); | ||||
if (rv == VM_PAGER_OK) | if (rv == VM_PAGER_OK) | ||||
return (KERN_SUCCESS); | return (FAULT_HARD); | ||||
if (rv == VM_PAGER_ERROR) | if (rv == VM_PAGER_ERROR) | ||||
printf("vm_fault: pager read error, pid %d (%s)\n", | printf("vm_fault: pager read error, pid %d (%s)\n", | ||||
curproc->p_pid, curproc->p_comm); | curproc->p_pid, curproc->p_comm); | ||||
/* | /* | ||||
* If an I/O error occurred or the requested page was | * If an I/O error occurred or the requested page was | ||||
* outside the range of the pager, clean up and return | * outside the range of the pager, clean up and return | ||||
* an error. | * an error. | ||||
*/ | */ | ||||
if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) { | if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) { | ||||
VM_OBJECT_WLOCK(fs->object); | VM_OBJECT_WLOCK(fs->object); | ||||
fault_page_free(&fs->m); | fault_page_free(&fs->m); | ||||
unlock_and_deallocate(fs); | unlock_and_deallocate(fs); | ||||
return (KERN_OUT_OF_BOUNDS); | return (FAULT_OUT_OF_BOUNDS); | ||||
} | } | ||||
KASSERT(rv == VM_PAGER_FAIL, | KASSERT(rv == VM_PAGER_FAIL, | ||||
("%s: unepxected pager error %d", __func__, rv)); | ("%s: unepxected pager error %d", __func__, rv)); | ||||
return (KERN_NOT_RECEIVER); | return (FAULT_CONTINUE); | ||||
} | } | ||||
/* | /* | ||||
* Wait/Retry if the page is busy. We have to do this if the page is | * Wait/Retry if the page is busy. We have to do this if the page is | ||||
* either exclusive or shared busy because the vm_pager may be using | * either exclusive or shared busy because the vm_pager may be using | ||||
* read busy for pageouts (and even pageins if it is the vnode pager), | * read busy for pageouts (and even pageins if it is the vnode pager), | ||||
* and we could end up trying to pagein and pageout the same page | * and we could end up trying to pagein and pageout the same page | ||||
* simultaneously. | * simultaneously. | ||||
Show All 27 Lines | |||||
} | } | ||||
int | int | ||||
vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | ||||
int fault_flags, vm_page_t *m_hold) | int fault_flags, vm_page_t *m_hold) | ||||
{ | { | ||||
struct faultstate fs; | struct faultstate fs; | ||||
int ahead, behind, faultcount; | int ahead, behind, faultcount; | ||||
int result, rv; | int result; | ||||
enum fault_status rv; | |||||
bool dead, hardfault; | bool dead, hardfault; | ||||
VM_CNT_INC(v_vm_faults); | VM_CNT_INC(v_vm_faults); | ||||
if ((curthread->td_pflags & TDP_NOFAULTING) != 0) | if ((curthread->td_pflags & TDP_NOFAULTING) != 0) | ||||
return (KERN_PROTECTION_FAILURE); | return (KERN_PROTECTION_FAILURE); | ||||
fs.vp = NULL; | fs.vp = NULL; | ||||
Show All 28 Lines | RetryFault: | ||||
* Under this condition, a read lock on the object suffices, allowing | * Under this condition, a read lock on the object suffices, allowing | ||||
* multiple page faults of a similar type to run in parallel. | * multiple page faults of a similar type to run in parallel. | ||||
*/ | */ | ||||
if (fs.vp == NULL /* avoid locked vnode leak */ && | if (fs.vp == NULL /* avoid locked vnode leak */ && | ||||
(fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) == 0 && | (fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) == 0 && | ||||
(fs.fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { | (fs.fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { | ||||
VM_OBJECT_RLOCK(fs.first_object); | VM_OBJECT_RLOCK(fs.first_object); | ||||
rv = vm_fault_soft_fast(&fs); | rv = vm_fault_soft_fast(&fs); | ||||
if (rv == KERN_SUCCESS) | if (rv == FAULT_SUCCESS) | ||||
return (rv); | return (KERN_SUCCESS); | ||||
if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) { | if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) { | ||||
VM_OBJECT_RUNLOCK(fs.first_object); | VM_OBJECT_RUNLOCK(fs.first_object); | ||||
VM_OBJECT_WLOCK(fs.first_object); | VM_OBJECT_WLOCK(fs.first_object); | ||||
} | } | ||||
} else { | } else { | ||||
VM_OBJECT_WLOCK(fs.first_object); | VM_OBJECT_WLOCK(fs.first_object); | ||||
} | } | ||||
Show All 15 Lines | RetryFault: | ||||
* Search for the page at object/offset. | * Search for the page at object/offset. | ||||
*/ | */ | ||||
fs.object = fs.first_object; | fs.object = fs.first_object; | ||||
fs.pindex = fs.first_pindex; | fs.pindex = fs.first_pindex; | ||||
if ((fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) != 0) { | if ((fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) != 0) { | ||||
rv = vm_fault_allocate(&fs); | rv = vm_fault_allocate(&fs); | ||||
switch (rv) { | switch (rv) { | ||||
case KERN_RESTART: | case FAULT_RESTART: | ||||
unlock_and_deallocate(&fs); | |||||
/* FALLTHROUGH */ | |||||
case KERN_RESOURCE_SHORTAGE: | |||||
goto RetryFault; | goto RetryFault; | ||||
case KERN_SUCCESS: | case FAULT_SUCCESS: | ||||
case KERN_FAILURE: | return (KERN_SUCCESS); | ||||
case KERN_PROTECTION_FAILURE: | case FAULT_FAILURE: | ||||
case KERN_OUT_OF_BOUNDS: | return (KERN_FAILURE); | ||||
unlock_and_deallocate(&fs); | case FAULT_OUT_OF_BOUNDS: | ||||
return (rv); | return (KERN_OUT_OF_BOUNDS); | ||||
case KERN_NOT_RECEIVER: | case FAULT_CONTINUE: | ||||
break; | break; | ||||
default: | default: | ||||
panic("vm_fault: Unhandled rv %d", rv); | panic("vm_fault: Unhandled rv %d", rv); | ||||
} | } | ||||
} | } | ||||
while (TRUE) { | while (TRUE) { | ||||
KASSERT(fs.m == NULL, | KASSERT(fs.m == NULL, | ||||
Show All 40 Lines | while (TRUE) { | ||||
* or this is the beginning of the search, allocate a new | * or this is the beginning of the search, allocate a new | ||||
* page. (Default objects are zero-fill, so there is no real | * page. (Default objects are zero-fill, so there is no real | ||||
* pager for them.) | * pager for them.) | ||||
*/ | */ | ||||
if (fs.m == NULL && (fs.object->type != OBJT_DEFAULT || | if (fs.m == NULL && (fs.object->type != OBJT_DEFAULT || | ||||
fs.object == fs.first_object)) { | fs.object == fs.first_object)) { | ||||
rv = vm_fault_allocate(&fs); | rv = vm_fault_allocate(&fs); | ||||
switch (rv) { | switch (rv) { | ||||
case KERN_RESTART: | case FAULT_RESTART: | ||||
unlock_and_deallocate(&fs); | |||||
/* FALLTHROUGH */ | |||||
case KERN_RESOURCE_SHORTAGE: | |||||
goto RetryFault; | goto RetryFault; | ||||
case KERN_SUCCESS: | case FAULT_SUCCESS: | ||||
case KERN_FAILURE: | return (KERN_SUCCESS); | ||||
case KERN_PROTECTION_FAILURE: | case FAULT_FAILURE: | ||||
case KERN_OUT_OF_BOUNDS: | return (KERN_FAILURE); | ||||
unlock_and_deallocate(&fs); | case FAULT_OUT_OF_BOUNDS: | ||||
return (rv); | return (KERN_OUT_OF_BOUNDS); | ||||
case KERN_NOT_RECEIVER: | case FAULT_CONTINUE: | ||||
break; | break; | ||||
default: | default: | ||||
panic("vm_fault: Unhandled rv %d", rv); | panic("vm_fault: Unhandled rv %d", rv); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Default objects have no pager so no exclusive busy exists | * Default objects have no pager so no exclusive busy exists | ||||
▲ Show 20 Lines • Show All 638 Lines • Show Last 20 Lines |
I would convert this function more thoroughly. If you introduce enum fault_status res variable inited with FAULT_SUCCESS, then you can directly assign FAULT_FAILURE to it in places where rv is tested for != KERN_SUCCESS, and just return res.
IMO it would feel less hackish.