Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_fault.c
Show First 20 Lines • Show All 119 Lines • ▼ Show 20 Lines | |||||
struct faultstate { | struct faultstate { | ||||
/* Fault parameters. */ | /* Fault parameters. */ | ||||
vm_offset_t vaddr; | vm_offset_t vaddr; | ||||
vm_page_t *m_hold; | vm_page_t *m_hold; | ||||
vm_prot_t fault_type; | vm_prot_t fault_type; | ||||
vm_prot_t prot; | vm_prot_t prot; | ||||
int fault_flags; | int fault_flags; | ||||
boolean_t wired; | |||||
/* Control state. */ | |||||
struct timeval oom_start_time; | struct timeval oom_start_time; | ||||
bool oom_started; | bool oom_started; | ||||
boolean_t wired; | int nera; | ||||
/* Page reference for cow. */ | /* Page reference for cow. */ | ||||
vm_page_t m_cow; | vm_page_t m_cow; | ||||
/* Current object. */ | /* Current object. */ | ||||
vm_object_t object; | vm_object_t object; | ||||
vm_pindex_t pindex; | vm_pindex_t pindex; | ||||
vm_page_t m; | vm_page_t m; | ||||
▲ Show 20 Lines • Show All 1,040 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/* | /* | ||||
* Call the pager to retrieve the page if there is a chance | * Call the pager to retrieve the page if there is a chance | ||||
* that the pager has it, and potentially retrieve additional | * that the pager has it, and potentially retrieve additional | ||||
* pages at the same time. | * pages at the same time. | ||||
*/ | */ | ||||
static int | static int | ||||
vm_fault_getpages(struct faultstate *fs, int nera, int *behindp, int *aheadp) | vm_fault_getpages(struct faultstate *fs, int *behindp, int *aheadp) | ||||
{ | { | ||||
vm_offset_t e_end, e_start; | vm_offset_t e_end, e_start; | ||||
int ahead, behind, cluster_offset, rv; | int ahead, behind, cluster_offset, rv; | ||||
u_char behavior; | u_char behavior; | ||||
/* | /* | ||||
* Prepare for unlocking the map. Save the map | * Prepare for unlocking the map. Save the map | ||||
* entry's start and end addresses, which are used to | * entry's start and end addresses, which are used to | ||||
* optimize the size of the pager operation below. | * optimize the size of the pager operation below. | ||||
* Even if the map entry's addresses change after | * Even if the map entry's addresses change after | ||||
* unlocking the map, using the saved addresses is | * unlocking the map, using the saved addresses is | ||||
* safe. | * safe. | ||||
*/ | */ | ||||
e_start = fs->entry->start; | e_start = fs->entry->start; | ||||
e_end = fs->entry->end; | e_end = fs->entry->end; | ||||
behavior = vm_map_entry_behavior(fs->entry); | behavior = vm_map_entry_behavior(fs->entry); | ||||
/* | /* | ||||
* If the pager for the current object might have | |||||
* the page, then determine the number of additional | |||||
* pages to read and potentially reprioritize | |||||
* previously read pages for earlier reclamation. | |||||
* These operations should only be performed once per | |||||
* page fault. Even if the current pager doesn't | |||||
* have the page, the number of additional pages to | |||||
* read will apply to subsequent objects in the | |||||
* shadow chain. | |||||
*/ | |||||
if (fs->nera == -1 && !P_KILLED(curproc)) | |||||
fs->nera = vm_fault_readahead(fs); | |||||
/* | |||||
* Release the map lock before locking the vnode or | * Release the map lock before locking the vnode or | ||||
* sleeping in the pager. (If the current object has | * sleeping in the pager. (If the current object has | ||||
* a shadow, then an earlier iteration of this loop | * a shadow, then an earlier iteration of this loop | ||||
* may have already unlocked the map.) | * may have already unlocked the map.) | ||||
*/ | */ | ||||
unlock_map(fs); | unlock_map(fs); | ||||
rv = vm_fault_lock_vnode(fs, false); | rv = vm_fault_lock_vnode(fs, false); | ||||
MPASS(rv == KERN_SUCCESS || rv == KERN_RESOURCE_SHORTAGE); | MPASS(rv == KERN_SUCCESS || rv == KERN_RESOURCE_SHORTAGE); | ||||
if (rv == KERN_RESOURCE_SHORTAGE) | if (rv == KERN_RESOURCE_SHORTAGE) | ||||
return (rv); | return (rv); | ||||
KASSERT(fs->vp == NULL || !fs->map->system_map, | KASSERT(fs->vp == NULL || !fs->map->system_map, | ||||
("vm_fault: vnode-backed object mapped by system map")); | ("vm_fault: vnode-backed object mapped by system map")); | ||||
/* | /* | ||||
* Page in the requested page and hint the pager, | * Page in the requested page and hint the pager, | ||||
* that it may bring up surrounding pages. | * that it may bring up surrounding pages. | ||||
*/ | */ | ||||
if (nera == -1 || behavior == MAP_ENTRY_BEHAV_RANDOM || | if (fs->nera == -1 || behavior == MAP_ENTRY_BEHAV_RANDOM || | ||||
P_KILLED(curproc)) { | P_KILLED(curproc)) { | ||||
behind = 0; | behind = 0; | ||||
ahead = 0; | ahead = 0; | ||||
} else { | } else { | ||||
/* Is this a sequential fault? */ | /* Is this a sequential fault? */ | ||||
if (nera > 0) { | if (fs->nera > 0) { | ||||
behind = 0; | behind = 0; | ||||
ahead = nera; | ahead = fs->nera; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Request a cluster of pages that is | * Request a cluster of pages that is | ||||
* aligned to a VM_FAULT_READ_DEFAULT | * aligned to a VM_FAULT_READ_DEFAULT | ||||
* page offset boundary within the | * page offset boundary within the | ||||
* object. Alignment to a page offset | * object. Alignment to a page offset | ||||
* boundary is more likely to coincide | * boundary is more likely to coincide | ||||
* with the underlying file system | * with the underlying file system | ||||
Show All 15 Lines | vm_fault_getpages(struct faultstate *fs, int *behindp, int *aheadp) | ||||
if (rv == VM_PAGER_ERROR) | if (rv == VM_PAGER_ERROR) | ||||
printf("vm_fault: pager read error, pid %d (%s)\n", | printf("vm_fault: pager read error, pid %d (%s)\n", | ||||
curproc->p_pid, curproc->p_comm); | curproc->p_pid, curproc->p_comm); | ||||
/* | /* | ||||
* If an I/O error occurred or the requested page was | * If an I/O error occurred or the requested page was | ||||
* outside the range of the pager, clean up and return | * outside the range of the pager, clean up and return | ||||
* an error. | * an error. | ||||
*/ | */ | ||||
if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) | if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) { | ||||
VM_OBJECT_WLOCK(fs->object); | |||||
fault_page_free(&fs->m); | |||||
unlock_and_deallocate(fs); | |||||
return (KERN_OUT_OF_BOUNDS); | return (KERN_OUT_OF_BOUNDS); | ||||
} | |||||
KASSERT(rv == VM_PAGER_FAIL, | |||||
("%s: unepxected pager error %d", __func__, rv)); | |||||
return (KERN_NOT_RECEIVER); | return (KERN_NOT_RECEIVER); | ||||
} | } | ||||
/* | /* | ||||
* Wait/Retry if the page is busy. We have to do this if the page is | * Wait/Retry if the page is busy. We have to do this if the page is | ||||
* either exclusive or shared busy because the vm_pager may be using | * either exclusive or shared busy because the vm_pager may be using | ||||
* read busy for pageouts (and even pageins if it is the vnode pager), | * read busy for pageouts (and even pageins if it is the vnode pager), | ||||
* and we could end up trying to pagein and pageout the same page | * and we could end up trying to pagein and pageout the same page | ||||
Show All 28 Lines | |||||
} | } | ||||
int | int | ||||
vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | ||||
int fault_flags, vm_page_t *m_hold) | int fault_flags, vm_page_t *m_hold) | ||||
{ | { | ||||
struct faultstate fs; | struct faultstate fs; | ||||
int ahead, behind, faultcount; | int ahead, behind, faultcount; | ||||
int nera, result, rv; | int result, rv; | ||||
bool dead, hardfault; | bool dead, hardfault; | ||||
VM_CNT_INC(v_vm_faults); | VM_CNT_INC(v_vm_faults); | ||||
if ((curthread->td_pflags & TDP_NOFAULTING) != 0) | if ((curthread->td_pflags & TDP_NOFAULTING) != 0) | ||||
return (KERN_PROTECTION_FAILURE); | return (KERN_PROTECTION_FAILURE); | ||||
fs.vp = NULL; | fs.vp = NULL; | ||||
fs.vaddr = vaddr; | fs.vaddr = vaddr; | ||||
fs.m_hold = m_hold; | fs.m_hold = m_hold; | ||||
fs.fault_flags = fault_flags; | fs.fault_flags = fault_flags; | ||||
fs.map = map; | fs.map = map; | ||||
fs.lookup_still_valid = false; | fs.lookup_still_valid = false; | ||||
fs.oom_started = false; | fs.oom_started = false; | ||||
fs.nera = -1; | |||||
faultcount = 0; | faultcount = 0; | ||||
nera = -1; | |||||
hardfault = false; | hardfault = false; | ||||
RetryFault: | RetryFault: | ||||
fs.fault_type = fault_type; | fs.fault_type = fault_type; | ||||
/* | /* | ||||
* Find the backing store object and offset into it to begin the | * Find the backing store object and offset into it to begin the | ||||
* search. | * search. | ||||
▲ Show 20 Lines • Show All 150 Lines • ▼ Show 20 Lines | if (fs.object->type != OBJT_DEFAULT) { | ||||
* | * | ||||
* We hold a reference on the current object and the | * We hold a reference on the current object and the | ||||
* page is exclusive busied. The exclusive busy | * page is exclusive busied. The exclusive busy | ||||
* prevents simultaneous faults and collapses while | * prevents simultaneous faults and collapses while | ||||
* the object lock is dropped. | * the object lock is dropped. | ||||
*/ | */ | ||||
VM_OBJECT_WUNLOCK(fs.object); | VM_OBJECT_WUNLOCK(fs.object); | ||||
/* | rv = vm_fault_getpages(&fs, &behind, &ahead); | ||||
* If the pager for the current object might have | |||||
* the page, then determine the number of additional | |||||
* pages to read and potentially reprioritize | |||||
* previously read pages for earlier reclamation. | |||||
* These operations should only be performed once per | |||||
* page fault. Even if the current pager doesn't | |||||
* have the page, the number of additional pages to | |||||
* read will apply to subsequent objects in the | |||||
* shadow chain. | |||||
*/ | |||||
if (nera == -1 && !P_KILLED(curproc)) | |||||
nera = vm_fault_readahead(&fs); | |||||
rv = vm_fault_getpages(&fs, nera, &behind, &ahead); | |||||
if (rv == KERN_SUCCESS) { | if (rv == KERN_SUCCESS) { | ||||
faultcount = behind + 1 + ahead; | faultcount = behind + 1 + ahead; | ||||
hardfault = true; | hardfault = true; | ||||
break; /* break to PAGE HAS BEEN FOUND. */ | break; /* break to PAGE HAS BEEN FOUND. */ | ||||
} | } | ||||
if (rv == KERN_RESOURCE_SHORTAGE) | if (rv == KERN_RESOURCE_SHORTAGE) | ||||
goto RetryFault; | goto RetryFault; | ||||
VM_OBJECT_WLOCK(fs.object); | if (rv == KERN_OUT_OF_BOUNDS) | ||||
if (rv == KERN_OUT_OF_BOUNDS) { | |||||
fault_page_free(&fs.m); | |||||
unlock_and_deallocate(&fs); | |||||
return (rv); | return (rv); | ||||
} | VM_OBJECT_WLOCK(fs.object); | ||||
} | } | ||||
/* | /* | ||||
* The page was not found in the current object. Try to | * The page was not found in the current object. Try to | ||||
* traverse into a backing object or zero fill if none is | * traverse into a backing object or zero fill if none is | ||||
* found. | * found. | ||||
*/ | */ | ||||
if (vm_fault_next(&fs)) | if (vm_fault_next(&fs)) | ||||
▲ Show 20 Lines • Show All 602 Lines • Show Last 20 Lines |