diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -155,6 +155,19 @@ struct vnode *vp; }; +/* + * Return codes for internal fault routines. + */ +enum fault_status { + FAULT_SUCCESS = 1, /* Return success to user. */ + FAULT_FAILURE, /* Return failure to user. */ + FAULT_CONTINUE, /* Continue faulting. */ + FAULT_RESTART, /* Restart fault. */ + FAULT_OUT_OF_BOUNDS, /* Invalid address for pager. */ + FAULT_HARD, /* Performed I/O. */ + FAULT_SOFT, /* Found valid page. */ +}; + static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr, int ahead); static void vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra, @@ -298,7 +311,7 @@ /* * Unlocks fs.first_object and fs.map on success. */ -static int +static enum fault_status vm_fault_soft_fast(struct faultstate *fs) { vm_page_t m, m_map; @@ -306,17 +319,20 @@ vm_page_t m_super; int flags; #endif - int psind, rv; + int psind; vm_offset_t vaddr; + enum fault_status res; MPASS(fs->vp == NULL); + + res = FAULT_SUCCESS; vaddr = fs->vaddr; vm_object_busy(fs->first_object); m = vm_page_lookup(fs->first_object, fs->first_pindex); /* A busy page can be mapped for read|execute access. */ if (m == NULL || ((fs->prot & VM_PROT_WRITE) != 0 && vm_page_busied(m)) || !vm_page_all_valid(m)) { - rv = KERN_FAILURE; + res = FAULT_FAILURE; goto out; } m_map = m; @@ -351,10 +367,12 @@ } } #endif - rv = pmap_enter(fs->map->pmap, vaddr, m_map, fs->prot, fs->fault_type | - PMAP_ENTER_NOSLEEP | (fs->wired ? PMAP_ENTER_WIRED : 0), psind); - if (rv != KERN_SUCCESS) + if (pmap_enter(fs->map->pmap, vaddr, m_map, fs->prot, fs->fault_type | + PMAP_ENTER_NOSLEEP | (fs->wired ? PMAP_ENTER_WIRED : 0), psind) != + KERN_SUCCESS) { + res = FAULT_FAILURE; goto out; + } if (fs->m_hold != NULL) { (*fs->m_hold) = m; vm_page_wire(m); @@ -368,7 +386,7 @@ out: vm_object_unbusy(fs->first_object); - return (rv); + return (res); } static void @@ -417,13 +435,14 @@ } } -static int +static enum fault_status vm_fault_populate(struct faultstate *fs) { vm_offset_t vaddr; vm_page_t m; vm_pindex_t map_first, map_last, pager_first, pager_last, pidx; int bdry_idx, i, npages, psind, rv; + enum fault_status res; MPASS(fs->object == fs->first_object); VM_OBJECT_ASSERT_WLOCKED(fs->first_object); @@ -436,6 +455,8 @@ unlock_map(fs); unlock_vp(fs); + res = FAULT_SUCCESS; + /* * Call the pager (driver) populate() method. * @@ -456,11 +477,11 @@ */ vm_fault_restore_map_lock(fs); if (fs->map->timestamp != fs->map_generation) - return (KERN_RESTART); - return (KERN_NOT_RECEIVER); + return (FAULT_RESTART); + return (FAULT_CONTINUE); } if (rv != VM_PAGER_OK) - return (KERN_FAILURE); /* AKA SIGSEGV */ + return (FAULT_FAILURE); /* AKA SIGSEGV */ /* Ensure that the driver is obeying the interface. */ MPASS(pager_first <= pager_last); @@ -480,7 +501,7 @@ if (m != fs->m) vm_page_xunbusy(m); } - return (KERN_RESTART); + return (FAULT_RESTART); } /* @@ -510,8 +531,10 @@ PMAP_ENTER_LARGEPAGE, bdry_idx); VM_OBJECT_WLOCK(fs->first_object); vm_page_xunbusy(m); - if (rv != KERN_SUCCESS) + if (rv != KERN_SUCCESS) { + res = FAULT_FAILURE; goto out; + } if ((fs->fault_flags & VM_FAULT_WIRE) != 0) { for (i = 0; i < atop(pagesizes[bdry_idx]); i++) vm_page_wire(m + i); @@ -596,7 +619,7 @@ } out: curthread->td_ru.ru_majflt++; - return (rv); + return (res); } static int prot_fault_translation; @@ -699,18 +722,18 @@ return (result); } -static int +static enum fault_status vm_fault_lock_vnode(struct faultstate *fs, bool objlocked) { struct vnode *vp; int error, locked; if (fs->object->type != OBJT_VNODE) - return (KERN_SUCCESS); + return (FAULT_CONTINUE); vp = fs->object->handle; if (vp == fs->vp) { ASSERT_VOP_LOCKED(vp, "saved vnode is not locked"); - return (KERN_SUCCESS); + return (FAULT_CONTINUE); } /* @@ -732,7 +755,7 @@ error = vget(vp, locked | LK_CANRECURSE | LK_NOWAIT); if (error == 0) { fs->vp = vp; - return (KERN_SUCCESS); + return (FAULT_CONTINUE); } vhold(vp); @@ -744,7 +767,7 @@ vdrop(vp); fs->vp = vp; KASSERT(error == 0, ("vm_fault: vget failed %d", error)); - return (KERN_RESOURCE_SHORTAGE); + return (FAULT_RESTART); } /* @@ -1112,33 +1135,35 @@ /* * Allocate a page directly or via the object populate method. */ -static int +static enum fault_status vm_fault_allocate(struct faultstate *fs) { struct domainset *dset; - int rv; + enum fault_status res; if ((fs->object->flags & OBJ_SIZEVNLOCK) != 0) { - rv = vm_fault_lock_vnode(fs, true); - MPASS(rv == KERN_SUCCESS || rv == KERN_RESOURCE_SHORTAGE); - if (rv == KERN_RESOURCE_SHORTAGE) - return (rv); + res = vm_fault_lock_vnode(fs, true); + MPASS(res == FAULT_CONTINUE || res == FAULT_RESTART); + if (res == FAULT_RESTART) + return (res); } - if (fs->pindex >= fs->object->size) - return (KERN_OUT_OF_BOUNDS); + if (fs->pindex >= fs->object->size) { + unlock_and_deallocate(fs); + return (FAULT_OUT_OF_BOUNDS); + } if (fs->object == fs->first_object && (fs->first_object->flags & OBJ_POPULATE) != 0 && fs->first_object->shadow_count == 0) { - rv = vm_fault_populate(fs); - switch (rv) { - case KERN_SUCCESS: - case KERN_FAILURE: - case KERN_PROTECTION_FAILURE: - case KERN_RESTART: - return (rv); - case KERN_NOT_RECEIVER: + res = vm_fault_populate(fs); + switch (res) { + case FAULT_SUCCESS: + case FAULT_FAILURE: + case FAULT_RESTART: + unlock_and_deallocate(fs); + return (res); + case FAULT_CONTINUE: /* * Pager's populate() method * returned VM_PAGER_BAD. @@ -1174,11 +1199,11 @@ if (fs->m == NULL) { if (vm_fault_allocate_oom(fs)) vm_waitpfault(dset, vm_pfault_oom_wait * hz); - return (KERN_RESOURCE_SHORTAGE); + return (FAULT_RESTART); } fs->oom_started = false; - return (KERN_NOT_RECEIVER); + return (FAULT_CONTINUE); } /* @@ -1186,11 +1211,12 @@ * that the pager has it, and potentially retrieve additional * pages at the same time. */ -static int +static enum fault_status vm_fault_getpages(struct faultstate *fs, int *behindp, int *aheadp) { vm_offset_t e_end, e_start; int ahead, behind, cluster_offset, rv; + enum fault_status status; u_char behavior; /* @@ -1227,10 +1253,10 @@ */ unlock_map(fs); - rv = vm_fault_lock_vnode(fs, false); - MPASS(rv == KERN_SUCCESS || rv == KERN_RESOURCE_SHORTAGE); - if (rv == KERN_RESOURCE_SHORTAGE) - return (rv); + status = vm_fault_lock_vnode(fs, false); + MPASS(status == FAULT_CONTINUE || status == FAULT_RESTART); + if (status == FAULT_RESTART) + return (status); KASSERT(fs->vp == NULL || !fs->map->system_map, ("vm_fault: vnode-backed object mapped by system map")); @@ -1269,7 +1295,7 @@ *aheadp = ahead; rv = vm_pager_get_pages(fs->object, &fs->m, 1, behindp, aheadp); if (rv == VM_PAGER_OK) - return (KERN_SUCCESS); + return (FAULT_HARD); if (rv == VM_PAGER_ERROR) printf("vm_fault: pager read error, pid %d (%s)\n", curproc->p_pid, curproc->p_comm); @@ -1282,11 +1308,11 @@ VM_OBJECT_WLOCK(fs->object); fault_page_free(&fs->m); unlock_and_deallocate(fs); - return (KERN_OUT_OF_BOUNDS); + return (FAULT_OUT_OF_BOUNDS); } KASSERT(rv == VM_PAGER_FAIL, ("%s: unepxected pager error %d", __func__, rv)); - return (KERN_NOT_RECEIVER); + return (FAULT_CONTINUE); } /* @@ -1329,8 +1355,8 @@ int fault_flags, vm_page_t *m_hold) { struct faultstate fs; - int ahead, behind, faultcount; - int result, rv; + int ahead, behind, faultcount, rv; + enum fault_status res; bool dead, hardfault; VM_CNT_INC(v_vm_faults); @@ -1356,11 +1382,11 @@ * Find the backing store object and offset into it to begin the * search. */ - result = vm_fault_lookup(&fs); - if (result != KERN_SUCCESS) { - if (result == KERN_RESOURCE_SHORTAGE) + rv = vm_fault_lookup(&fs); + if (rv != KERN_SUCCESS) { + if (rv == KERN_RESOURCE_SHORTAGE) goto RetryFault; - return (result); + return (rv); } /* @@ -1374,9 +1400,9 @@ (fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) == 0 && (fs.fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { VM_OBJECT_RLOCK(fs.first_object); - rv = vm_fault_soft_fast(&fs); - if (rv == KERN_SUCCESS) - return (rv); + res = vm_fault_soft_fast(&fs); + if (res == FAULT_SUCCESS) + return (KERN_SUCCESS); if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) { VM_OBJECT_RUNLOCK(fs.first_object); VM_OBJECT_WLOCK(fs.first_object); @@ -1406,23 +1432,20 @@ fs.pindex = fs.first_pindex; if ((fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) != 0) { - rv = vm_fault_allocate(&fs); - switch (rv) { - case KERN_RESTART: - unlock_and_deallocate(&fs); - /* FALLTHROUGH */ - case KERN_RESOURCE_SHORTAGE: + res = vm_fault_allocate(&fs); + switch (res) { + case FAULT_RESTART: goto RetryFault; - case KERN_SUCCESS: - case KERN_FAILURE: - case KERN_PROTECTION_FAILURE: - case KERN_OUT_OF_BOUNDS: - unlock_and_deallocate(&fs); - return (rv); - case KERN_NOT_RECEIVER: + case FAULT_SUCCESS: + return (KERN_SUCCESS); + case FAULT_FAILURE: + return (KERN_FAILURE); + case FAULT_OUT_OF_BOUNDS: + return (KERN_OUT_OF_BOUNDS); + case FAULT_CONTINUE: break; default: - panic("vm_fault: Unhandled rv %d", rv); + panic("vm_fault: Unhandled status %d", res); } } @@ -1474,23 +1497,20 @@ */ if (fs.m == NULL && (fs.object->type != OBJT_DEFAULT || fs.object == fs.first_object)) { - rv = vm_fault_allocate(&fs); - switch (rv) { - case KERN_RESTART: - unlock_and_deallocate(&fs); - /* FALLTHROUGH */ - case KERN_RESOURCE_SHORTAGE: + res = vm_fault_allocate(&fs); + switch (res) { + case FAULT_RESTART: goto RetryFault; - case KERN_SUCCESS: - case KERN_FAILURE: - case KERN_PROTECTION_FAILURE: - case KERN_OUT_OF_BOUNDS: - unlock_and_deallocate(&fs); - return (rv); - case KERN_NOT_RECEIVER: + case FAULT_SUCCESS: + return (KERN_SUCCESS); + case FAULT_FAILURE: + return (KERN_FAILURE); + case FAULT_OUT_OF_BOUNDS: + return (KERN_OUT_OF_BOUNDS); + case FAULT_CONTINUE: break; default: - panic("vm_fault: Unhandled rv %d", rv); + panic("vm_fault: Unhandled status %d", res); } } @@ -1513,16 +1533,16 @@ */ VM_OBJECT_WUNLOCK(fs.object); - rv = vm_fault_getpages(&fs, &behind, &ahead); - if (rv == KERN_SUCCESS) { + res = vm_fault_getpages(&fs, &behind, &ahead); + if (res == FAULT_SUCCESS) { faultcount = behind + 1 + ahead; hardfault = true; break; /* break to PAGE HAS BEEN FOUND. */ } - if (rv == KERN_RESOURCE_SHORTAGE) + if (res == FAULT_RESTART) goto RetryFault; - if (rv == KERN_OUT_OF_BOUNDS) - return (rv); + if (res == FAULT_OUT_OF_BOUNDS) + return (KERN_OUT_OF_BOUNDS); VM_OBJECT_WLOCK(fs.object); } @@ -1583,12 +1603,12 @@ * lookup. */ if (!fs.lookup_still_valid) { - result = vm_fault_relookup(&fs); - if (result != KERN_SUCCESS) { + rv = vm_fault_relookup(&fs); + if (rv != KERN_SUCCESS) { fault_deallocate(&fs); - if (result == KERN_RESTART) + if (rv == KERN_RESTART) goto RetryFault; - return (result); + return (rv); } } VM_OBJECT_ASSERT_UNLOCKED(fs.object);