Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_fault.c
Show First 20 Lines • Show All 995 Lines • ▼ Show 20 Lines | if (fs->object != fs->first_object) | ||||
vm_object_pip_wakeup(fs->object); | vm_object_pip_wakeup(fs->object); | ||||
fs->pindex += OFF_TO_IDX(fs->object->backing_object_offset); | fs->pindex += OFF_TO_IDX(fs->object->backing_object_offset); | ||||
VM_OBJECT_WUNLOCK(fs->object); | VM_OBJECT_WUNLOCK(fs->object); | ||||
fs->object = next_object; | fs->object = next_object; | ||||
return (true); | return (true); | ||||
} | } | ||||
/* | /* | ||||
* Call the pager to retrieve the page if there is a chance | |||||
* that the pager has it, and potentially retrieve additional | |||||
* pages at the same time. | |||||
*/ | |||||
static int | |||||
vm_fault_getpages(struct faultstate *fs, int nera, int *behindp, int *aheadp) | |||||
{ | |||||
vm_offset_t e_end, e_start; | |||||
int ahead, behind, cluster_offset, rv; | |||||
u_char behavior; | |||||
/* | |||||
* Prepare for unlocking the map. Save the map | |||||
* entry's start and end addresses, which are used to | |||||
* optimize the size of the pager operation below. | |||||
* Even if the map entry's addresses change after | |||||
* unlocking the map, using the saved addresses is | |||||
* safe. | |||||
*/ | |||||
e_start = fs->entry->start; | |||||
e_end = fs->entry->end; | |||||
behavior = vm_map_entry_behavior(fs->entry); | |||||
/* | |||||
* Release the map lock before locking the vnode or | |||||
* sleeping in the pager. (If the current object has | |||||
* a shadow, then an earlier iteration of this loop | |||||
* may have already unlocked the map.) | |||||
*/ | |||||
unlock_map(fs); | |||||
rv = vm_fault_lock_vnode(fs, false); | |||||
MPASS(rv == KERN_SUCCESS || rv == KERN_RESOURCE_SHORTAGE); | |||||
if (rv == KERN_RESOURCE_SHORTAGE) | |||||
return (rv); | |||||
KASSERT(fs->vp == NULL || !fs->map->system_map, | |||||
("vm_fault: vnode-backed object mapped by system map")); | |||||
/* | |||||
* Page in the requested page and hint the pager, | |||||
* that it may bring up surrounding pages. | |||||
*/ | |||||
if (nera == -1 || behavior == MAP_ENTRY_BEHAV_RANDOM || | |||||
P_KILLED(curproc)) { | |||||
behind = 0; | |||||
ahead = 0; | |||||
} else { | |||||
/* Is this a sequential fault? */ | |||||
if (nera > 0) { | |||||
behind = 0; | |||||
ahead = nera; | |||||
} else { | |||||
/* | |||||
* Request a cluster of pages that is | |||||
* aligned to a VM_FAULT_READ_DEFAULT | |||||
* page offset boundary within the | |||||
* object. Alignment to a page offset | |||||
* boundary is more likely to coincide | |||||
* with the underlying file system | |||||
* block than alignment to a virtual | |||||
* address boundary. | |||||
*/ | |||||
cluster_offset = fs->pindex % VM_FAULT_READ_DEFAULT; | |||||
behind = ulmin(cluster_offset, | |||||
atop(fs->vaddr - e_start)); | |||||
ahead = VM_FAULT_READ_DEFAULT - 1 - cluster_offset; | |||||
} | |||||
ahead = ulmin(ahead, atop(e_end - fs->vaddr) - 1); | |||||
} | |||||
*behindp = behind; | |||||
*aheadp = ahead; | |||||
rv = vm_pager_get_pages(fs->object, &fs->m, 1, behindp, aheadp); | |||||
if (rv == VM_PAGER_OK) | |||||
return (KERN_SUCCESS); | |||||
if (rv == VM_PAGER_ERROR) | |||||
printf("vm_fault: pager read error, pid %d (%s)\n", | |||||
curproc->p_pid, curproc->p_comm); | |||||
/* | |||||
* If an I/O error occurred or the requested page was | |||||
* outside the range of the pager, clean up and return | |||||
* an error. | |||||
*/ | |||||
if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) | |||||
return (KERN_OUT_OF_BOUNDS); | |||||
return (KERN_NOT_RECEIVER); | |||||
} | |||||
/* | |||||
* Wait/Retry if the page is busy. We have to do this if the page is | * Wait/Retry if the page is busy. We have to do this if the page is | ||||
* either exclusive or shared busy because the vm_pager may be using | * either exclusive or shared busy because the vm_pager may be using | ||||
* read busy for pageouts (and even pageins if it is the vnode pager), | * read busy for pageouts (and even pageins if it is the vnode pager), | ||||
* and we could end up trying to pagein and pageout the same page | * and we could end up trying to pagein and pageout the same page | ||||
* simultaneously. | * simultaneously. | ||||
* | * | ||||
* We can theoretically allow the busy case on a read fault if the page | * We can theoretically allow the busy case on a read fault if the page | ||||
* is marked valid, but since such pages are typically already pmap'd, | * is marked valid, but since such pages are typically already pmap'd, | ||||
Show All 25 Lines | |||||
} | } | ||||
int | int | ||||
vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | ||||
int fault_flags, vm_page_t *m_hold) | int fault_flags, vm_page_t *m_hold) | ||||
{ | { | ||||
struct faultstate fs; | struct faultstate fs; | ||||
struct domainset *dset; | struct domainset *dset; | ||||
vm_offset_t e_end, e_start; | int ahead, alloc_req, behind, faultcount; | ||||
int ahead, alloc_req, behind, cluster_offset, faultcount; | |||||
int nera, oom, result, rv; | int nera, oom, result, rv; | ||||
u_char behavior; | |||||
bool dead, hardfault; | bool dead, hardfault; | ||||
VM_CNT_INC(v_vm_faults); | VM_CNT_INC(v_vm_faults); | ||||
if ((curthread->td_pflags & TDP_NOFAULTING) != 0) | if ((curthread->td_pflags & TDP_NOFAULTING) != 0) | ||||
return (KERN_PROTECTION_FAILURE); | return (KERN_PROTECTION_FAILURE); | ||||
fs.vp = NULL; | fs.vp = NULL; | ||||
▲ Show 20 Lines • Show All 219 Lines • ▼ Show 20 Lines | readrest: | ||||
* If the pager for the current object might have the page, | * If the pager for the current object might have the page, | ||||
* then determine the number of additional pages to read and | * then determine the number of additional pages to read and | ||||
* potentially reprioritize previously read pages for earlier | * potentially reprioritize previously read pages for earlier | ||||
* reclamation. These operations should only be performed | * reclamation. These operations should only be performed | ||||
* once per page fault. Even if the current pager doesn't | * once per page fault. Even if the current pager doesn't | ||||
* have the page, the number of additional pages to read will | * have the page, the number of additional pages to read will | ||||
* apply to subsequent objects in the shadow chain. | * apply to subsequent objects in the shadow chain. | ||||
*/ | */ | ||||
if (nera == -1 && !P_KILLED(curproc)) { | if (nera == -1 && !P_KILLED(curproc)) | ||||
nera = vm_fault_readahead(&fs); | nera = vm_fault_readahead(&fs); | ||||
/* | |||||
* Prepare for unlocking the map. Save the map | |||||
* entry's start and end addresses, which are used to | |||||
* optimize the size of the pager operation below. | |||||
* Even if the map entry's addresses change after | |||||
* unlocking the map, using the saved addresses is | |||||
* safe. | |||||
*/ | |||||
e_start = fs.entry->start; | |||||
e_end = fs.entry->end; | |||||
behavior = vm_map_entry_behavior(fs.entry); | |||||
} | |||||
/* | rv = vm_fault_getpages(&fs, nera, &behind, &ahead); | ||||
* Call the pager to retrieve the page if there is a chance | if (rv == KERN_SUCCESS) { | ||||
* that the pager has it, and potentially retrieve additional | |||||
* pages at the same time. | |||||
*/ | |||||
if (fs.object->type != OBJT_DEFAULT) { | |||||
/* | |||||
* Release the map lock before locking the vnode or | |||||
* sleeping in the pager. (If the current object has | |||||
* a shadow, then an earlier iteration of this loop | |||||
* may have already unlocked the map.) | |||||
*/ | |||||
unlock_map(&fs); | |||||
rv = vm_fault_lock_vnode(&fs, false); | |||||
MPASS(rv == KERN_SUCCESS || | |||||
rv == KERN_RESOURCE_SHORTAGE); | |||||
if (rv == KERN_RESOURCE_SHORTAGE) | |||||
goto RetryFault; | |||||
KASSERT(fs.vp == NULL || !fs.map->system_map, | |||||
("vm_fault: vnode-backed object mapped by system map")); | |||||
/* | |||||
* Page in the requested page and hint the pager, | |||||
* that it may bring up surrounding pages. | |||||
*/ | |||||
if (nera == -1 || behavior == MAP_ENTRY_BEHAV_RANDOM || | |||||
P_KILLED(curproc)) { | |||||
behind = 0; | |||||
ahead = 0; | |||||
} else { | |||||
/* Is this a sequential fault? */ | |||||
if (nera > 0) { | |||||
behind = 0; | |||||
ahead = nera; | |||||
} else { | |||||
/* | |||||
* Request a cluster of pages that is | |||||
* aligned to a VM_FAULT_READ_DEFAULT | |||||
* page offset boundary within the | |||||
* object. Alignment to a page offset | |||||
* boundary is more likely to coincide | |||||
* with the underlying file system | |||||
* block than alignment to a virtual | |||||
* address boundary. | |||||
*/ | |||||
cluster_offset = fs.pindex % | |||||
VM_FAULT_READ_DEFAULT; | |||||
behind = ulmin(cluster_offset, | |||||
atop(vaddr - e_start)); | |||||
ahead = VM_FAULT_READ_DEFAULT - 1 - | |||||
cluster_offset; | |||||
} | |||||
ahead = ulmin(ahead, atop(e_end - vaddr) - 1); | |||||
} | |||||
rv = vm_pager_get_pages(fs.object, &fs.m, 1, | |||||
&behind, &ahead); | |||||
if (rv == VM_PAGER_OK) { | |||||
faultcount = behind + 1 + ahead; | faultcount = behind + 1 + ahead; | ||||
hardfault = true; | hardfault = true; | ||||
break; /* break to PAGE HAS BEEN FOUND. */ | break; /* break to PAGE HAS BEEN FOUND. */ | ||||
} | } | ||||
if (rv == KERN_RESOURCE_SHORTAGE) | |||||
goto RetryFault; | |||||
VM_OBJECT_WLOCK(fs.object); | VM_OBJECT_WLOCK(fs.object); | ||||
if (rv == VM_PAGER_ERROR) | if (rv == KERN_OUT_OF_BOUNDS) { | ||||
printf("vm_fault: pager read error, pid %d (%s)\n", | |||||
curproc->p_pid, curproc->p_comm); | |||||
/* | |||||
* If an I/O error occurred or the requested page was | |||||
* outside the range of the pager, clean up and return | |||||
* an error. | |||||
*/ | |||||
if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) { | |||||
fault_page_free(&fs.m); | fault_page_free(&fs.m); | ||||
unlock_and_deallocate(&fs); | unlock_and_deallocate(&fs); | ||||
return (KERN_OUT_OF_BOUNDS); | return (rv); | ||||
} | } | ||||
} | |||||
/* | /* | ||||
* The page was not found in the current object. Try to traverse | * The page was not found in the current object. Try to | ||||
* into a backing object or zero fill if none is found. | * traverse into a backing object or zero fill if none is | ||||
* found. | |||||
*/ | */ | ||||
if (!vm_fault_next(&fs)) { | if (!vm_fault_next(&fs)) { | ||||
/* Don't try to prefault neighboring pages. */ | /* Don't try to prefault neighboring pages. */ | ||||
faultcount = 1; | faultcount = 1; | ||||
break; /* break to PAGE HAS BEEN FOUND. */ | break; /* break to PAGE HAS BEEN FOUND. */ | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 591 Lines • Show Last 20 Lines |