Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_fault.c
Show First 20 Lines • Show All 160 Lines • ▼ Show 20 Lines | |||||
enum fault_status { | enum fault_status { | ||||
FAULT_SUCCESS = 1, /* Return success to user. */ | FAULT_SUCCESS = 1, /* Return success to user. */ | ||||
FAULT_FAILURE, /* Return failure to user. */ | FAULT_FAILURE, /* Return failure to user. */ | ||||
FAULT_CONTINUE, /* Continue faulting. */ | FAULT_CONTINUE, /* Continue faulting. */ | ||||
FAULT_RESTART, /* Restart fault. */ | FAULT_RESTART, /* Restart fault. */ | ||||
FAULT_OUT_OF_BOUNDS, /* Invalid address for pager. */ | FAULT_OUT_OF_BOUNDS, /* Invalid address for pager. */ | ||||
FAULT_HARD, /* Performed I/O. */ | FAULT_HARD, /* Performed I/O. */ | ||||
FAULT_SOFT, /* Found valid page. */ | FAULT_SOFT, /* Found valid page. */ | ||||
FAULT_PROTECTION_FAILURE, /* Invalid access. */ | |||||
}; | }; | ||||
static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr, | static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr, | ||||
int ahead); | int ahead); | ||||
static void vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra, | static void vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra, | ||||
int backward, int forward, bool obj_locked); | int backward, int forward, bool obj_locked); | ||||
static int vm_pfault_oom_attempts = 3; | static int vm_pfault_oom_attempts = 3; | ||||
▲ Show 20 Lines • Show All 1,168 Lines • ▼ Show 20 Lines | vm_fault_busy_sleep(struct faultstate *fs) | ||||
unlock_map(fs); | unlock_map(fs); | ||||
if (fs->m != vm_page_lookup(fs->object, fs->pindex) || | if (fs->m != vm_page_lookup(fs->object, fs->pindex) || | ||||
!vm_page_busy_sleep(fs->m, "vmpfw", 0)) | !vm_page_busy_sleep(fs->m, "vmpfw", 0)) | ||||
VM_OBJECT_WUNLOCK(fs->object); | VM_OBJECT_WUNLOCK(fs->object); | ||||
VM_CNT_INC(v_intrans); | VM_CNT_INC(v_intrans); | ||||
vm_object_deallocate(fs->first_object); | vm_object_deallocate(fs->first_object); | ||||
} | } | ||||
/* | |||||
* Handle page lookup, populate, allocate, page-in for the current | |||||
* object. | |||||
* | |||||
* The object is locked on entry and will remain locked with a return | |||||
* code of FAULT_CONTINUE so that fault may follow the shadow chain. | |||||
* Otherwise, the object will be unlocked upon return. | |||||
*/ | |||||
static enum fault_status | |||||
vm_fault_object(struct faultstate *fs, int *behindp, int *aheadp) | |||||
{ | |||||
enum fault_status res; | |||||
bool dead; | |||||
/* | |||||
* If the object is marked for imminent termination, we retry | |||||
* here, since the collapse pass has raced with us. Otherwise, | |||||
* if we see terminally dead object, return fail. | |||||
*/ | |||||
if ((fs->object->flags & OBJ_DEAD) != 0) { | |||||
dead = fs->object->type == OBJT_DEAD; | |||||
unlock_and_deallocate(fs); | |||||
if (dead) | |||||
return (FAULT_PROTECTION_FAILURE); | |||||
pause("vmf_de", 1); | |||||
return (FAULT_RESTART); | |||||
} | |||||
/* | |||||
* See if the page is resident. | |||||
*/ | |||||
fs->m = vm_page_lookup(fs->object, fs->pindex); | |||||
if (fs->m != NULL) { | |||||
if (!vm_page_tryxbusy(fs->m)) { | |||||
vm_fault_busy_sleep(fs); | |||||
return (FAULT_RESTART); | |||||
} | |||||
/* | |||||
* The page is marked busy for other processes and the | |||||
* pagedaemon. If it is still completely valid we are | |||||
* done. | |||||
*/ | |||||
if (vm_page_all_valid(fs->m)) { | |||||
VM_OBJECT_WUNLOCK(fs->object); | |||||
return (FAULT_SOFT); | |||||
} | |||||
} | |||||
VM_OBJECT_ASSERT_WLOCKED(fs->object); | |||||
/* | |||||
* Page is not resident. If the pager might contain the page | |||||
* or this is the beginning of the search, allocate a new | |||||
* page. (Default objects are zero-fill, so there is no real | |||||
* pager for them.) | |||||
*/ | |||||
if (fs->m == NULL && (fs->object->type != OBJT_DEFAULT || | |||||
fs->object == fs->first_object)) { | |||||
res = vm_fault_allocate(fs); | |||||
if (res != FAULT_CONTINUE) | |||||
return (res); | |||||
} | |||||
/* | |||||
* Default objects have no pager so no exclusive busy exists | |||||
* to protect this page in the chain. Skip to the next | |||||
* object without dropping the lock to preserve atomicity of | |||||
* shadow faults. | |||||
*/ | |||||
if (fs->object->type != OBJT_DEFAULT) { | |||||
/* | |||||
* At this point, we have either allocated a new page | |||||
* or found an existing page that is only partially | |||||
* valid. | |||||
* | |||||
* We hold a reference on the current object and the | |||||
* page is exclusive busied. The exclusive busy | |||||
* prevents simultaneous faults and collapses while | |||||
* the object lock is dropped. | |||||
*/ | |||||
VM_OBJECT_WUNLOCK(fs->object); | |||||
res = vm_fault_getpages(fs, behindp, aheadp); | |||||
if (res == FAULT_CONTINUE) | |||||
VM_OBJECT_WLOCK(fs->object); | |||||
} else { | |||||
res = FAULT_CONTINUE; | |||||
} | |||||
return (res); | |||||
} | |||||
int | int | ||||
vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | ||||
int fault_flags, vm_page_t *m_hold) | int fault_flags, vm_page_t *m_hold) | ||||
{ | { | ||||
struct faultstate fs; | struct faultstate fs; | ||||
int ahead, behind, faultcount, rv; | int ahead, behind, faultcount, rv; | ||||
enum fault_status res; | enum fault_status res; | ||||
bool dead, hardfault; | bool hardfault; | ||||
VM_CNT_INC(v_vm_faults); | VM_CNT_INC(v_vm_faults); | ||||
if ((curthread->td_pflags & TDP_NOFAULTING) != 0) | if ((curthread->td_pflags & TDP_NOFAULTING) != 0) | ||||
return (KERN_PROTECTION_FAILURE); | return (KERN_PROTECTION_FAILURE); | ||||
fs.vp = NULL; | fs.vp = NULL; | ||||
fs.vaddr = vaddr; | fs.vaddr = vaddr; | ||||
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | if ((fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) != 0) { | ||||
default: | default: | ||||
panic("vm_fault: Unhandled status %d", res); | panic("vm_fault: Unhandled status %d", res); | ||||
} | } | ||||
} | } | ||||
while (TRUE) { | while (TRUE) { | ||||
KASSERT(fs.m == NULL, | KASSERT(fs.m == NULL, | ||||
("page still set %p at loop start", fs.m)); | ("page still set %p at loop start", fs.m)); | ||||
/* | |||||
* If the object is marked for imminent termination, | |||||
* we retry here, since the collapse pass has raced | |||||
* with us. Otherwise, if we see terminally dead | |||||
* object, return fail. | |||||
*/ | |||||
if ((fs.object->flags & OBJ_DEAD) != 0) { | |||||
dead = fs.object->type == OBJT_DEAD; | |||||
unlock_and_deallocate(&fs); | |||||
if (dead) | |||||
return (KERN_PROTECTION_FAILURE); | |||||
pause("vmf_de", 1); | |||||
goto RetryFault; | |||||
} | |||||
/* | res = vm_fault_object(&fs, &behind, &ahead); | ||||
* See if page is resident | |||||
*/ | |||||
fs.m = vm_page_lookup(fs.object, fs.pindex); | |||||
if (fs.m != NULL) { | |||||
if (vm_page_tryxbusy(fs.m) == 0) { | |||||
vm_fault_busy_sleep(&fs); | |||||
goto RetryFault; | |||||
} | |||||
/* | |||||
* The page is marked busy for other processes and the | |||||
* pagedaemon. If it still is completely valid we | |||||
* are done. | |||||
*/ | |||||
if (vm_page_all_valid(fs.m)) { | |||||
VM_OBJECT_WUNLOCK(fs.object); | |||||
break; /* break to PAGE HAS BEEN FOUND. */ | |||||
} | |||||
} | |||||
VM_OBJECT_ASSERT_WLOCKED(fs.object); | |||||
/* | |||||
* Page is not resident. If the pager might contain the page | |||||
* or this is the beginning of the search, allocate a new | |||||
* page. (Default objects are zero-fill, so there is no real | |||||
* pager for them.) | |||||
*/ | |||||
if (fs.m == NULL && (fs.object->type != OBJT_DEFAULT || | |||||
fs.object == fs.first_object)) { | |||||
res = vm_fault_allocate(&fs); | |||||
switch (res) { | switch (res) { | ||||
case FAULT_SOFT: | |||||
goto found; | |||||
kib: I liked the old `/* break to PAGE HAS BEEN FOUND. */` comment. I wonder if you could just… | |||||
Done Inline ActionsWell, it sounds somewhat obvious now that the lookup loop shrunk from 400LOC (in stable/12) to 45LOC. This fragment could be written like this: res = vm_fault_object(&fs, &behind, &ahead); if (res == FAULT_SOFT) break; /* page has been found */ if (res == FAULT_HARD) { faultcount = behind + 1 + ahead; hardfault = true; break; /* page has been found */ } switch (res) { <handle other statuses> } Or do you prefer to fully restore the old "PAGE HAS BEEN FOUND" label? I don't have strong feelings about it. markj: Well, it sounds somewhat obvious now that the lookup loop shrunk from 400LOC (in stable/12) to… | |||||
Not Done Inline ActionsI used this comment for orientation in the file. I remembered to search for it to find the core anchor basically splitting the fault handling into 'before we have a page' (lookup/validation) and 'after' (doing pmap work and post-accounting). May be I can use FAULT_SOFT/FAULT_HARD for this after your change. kib: I used this comment for orientation in the file. I remembered to search for it to find the… | |||||
case FAULT_HARD: | |||||
Not Done Inline ActionsThis is from vm_fault_getpages()? kib: This is from vm_fault_getpages()? | |||||
Done Inline ActionsYes. markj: Yes. | |||||
faultcount = behind + 1 + ahead; | |||||
hardfault = true; | |||||
goto found; | |||||
case FAULT_RESTART: | case FAULT_RESTART: | ||||
goto RetryFault; | goto RetryFault; | ||||
case FAULT_SUCCESS: | case FAULT_SUCCESS: | ||||
return (KERN_SUCCESS); | return (KERN_SUCCESS); | ||||
case FAULT_FAILURE: | case FAULT_FAILURE: | ||||
return (KERN_FAILURE); | return (KERN_FAILURE); | ||||
case FAULT_OUT_OF_BOUNDS: | case FAULT_OUT_OF_BOUNDS: | ||||
return (KERN_OUT_OF_BOUNDS); | return (KERN_OUT_OF_BOUNDS); | ||||
case FAULT_PROTECTION_FAILURE: | |||||
return (KERN_PROTECTION_FAILURE); | |||||
case FAULT_CONTINUE: | case FAULT_CONTINUE: | ||||
break; | break; | ||||
default: | default: | ||||
panic("vm_fault: Unhandled status %d", res); | panic("vm_fault: Unhandled status %d", res); | ||||
} | } | ||||
} | |||||
/* | /* | ||||
* Default objects have no pager so no exclusive busy exists | |||||
* to protect this page in the chain. Skip to the next | |||||
* object without dropping the lock to preserve atomicity of | |||||
* shadow faults. | |||||
*/ | |||||
if (fs.object->type != OBJT_DEFAULT) { | |||||
/* | |||||
* At this point, we have either allocated a new page | |||||
* or found an existing page that is only partially | |||||
* valid. | |||||
* | |||||
* We hold a reference on the current object and the | |||||
* page is exclusive busied. The exclusive busy | |||||
* prevents simultaneous faults and collapses while | |||||
* the object lock is dropped. | |||||
*/ | |||||
VM_OBJECT_WUNLOCK(fs.object); | |||||
res = vm_fault_getpages(&fs, &behind, &ahead); | |||||
if (res == FAULT_SUCCESS) { | |||||
faultcount = behind + 1 + ahead; | |||||
hardfault = true; | |||||
break; /* break to PAGE HAS BEEN FOUND. */ | |||||
} | |||||
if (res == FAULT_RESTART) | |||||
goto RetryFault; | |||||
if (res == FAULT_OUT_OF_BOUNDS) | |||||
return (KERN_OUT_OF_BOUNDS); | |||||
VM_OBJECT_WLOCK(fs.object); | |||||
} | |||||
/* | |||||
* The page was not found in the current object. Try to | * The page was not found in the current object. Try to | ||||
* traverse into a backing object or zero fill if none is | * traverse into a backing object or zero fill if none is | ||||
* found. | * found. | ||||
*/ | */ | ||||
if (vm_fault_next(&fs)) | if (vm_fault_next(&fs)) | ||||
continue; | continue; | ||||
if ((fs.fault_flags & VM_FAULT_NOFILL) != 0) { | if ((fs.fault_flags & VM_FAULT_NOFILL) != 0) { | ||||
if (fs.first_object == fs.object) | if (fs.first_object == fs.object) | ||||
fault_page_free(&fs.first_m); | fault_page_free(&fs.first_m); | ||||
unlock_and_deallocate(&fs); | unlock_and_deallocate(&fs); | ||||
return (KERN_OUT_OF_BOUNDS); | return (KERN_OUT_OF_BOUNDS); | ||||
} | } | ||||
VM_OBJECT_WUNLOCK(fs.object); | VM_OBJECT_WUNLOCK(fs.object); | ||||
vm_fault_zerofill(&fs); | vm_fault_zerofill(&fs); | ||||
/* Don't try to prefault neighboring pages. */ | /* Don't try to prefault neighboring pages. */ | ||||
faultcount = 1; | faultcount = 1; | ||||
break; /* break to PAGE HAS BEEN FOUND. */ | break; | ||||
} | } | ||||
found: | |||||
/* | /* | ||||
* PAGE HAS BEEN FOUND. A valid page has been found and exclusively | * A valid page has been found and exclusively busied. The | ||||
* busied. The object lock must no longer be held. | * object lock must no longer be held. | ||||
*/ | */ | ||||
vm_page_assert_xbusied(fs.m); | vm_page_assert_xbusied(fs.m); | ||||
VM_OBJECT_ASSERT_UNLOCKED(fs.object); | VM_OBJECT_ASSERT_UNLOCKED(fs.object); | ||||
/* | /* | ||||
* If the page is being written, but isn't already owned by the | * If the page is being written, but isn't already owned by the | ||||
* top-level object, we have to copy it into a new page owned by the | * top-level object, we have to copy it into a new page owned by the | ||||
* top-level object. | * top-level object. | ||||
▲ Show 20 Lines • Show All 577 Lines • Show Last 20 Lines |
I liked the old /* break to PAGE HAS BEEN FOUND. */ comment. I wonder if you could just paste it there.