Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_fault.c
Show First 20 Lines • Show All 491 Lines • ▼ Show 20 Lines | vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | ||||
vm_pindex_t retry_pindex; | vm_pindex_t retry_pindex; | ||||
vm_prot_t prot, retry_prot; | vm_prot_t prot, retry_prot; | ||||
int ahead, alloc_req, behind, cluster_offset, error, era, faultcount; | int ahead, alloc_req, behind, cluster_offset, error, era, faultcount; | ||||
int locked, nera, result, rv; | int locked, nera, result, rv; | ||||
u_char behavior; | u_char behavior; | ||||
boolean_t wired; /* Passed by reference. */ | boolean_t wired; /* Passed by reference. */ | ||||
bool dead, growstack, hardfault, is_first_object_locked; | bool dead, growstack, hardfault, is_first_object_locked; | ||||
PCPU_INC(cnt.v_vm_faults); | VM_CNT_INC(v_vm_faults); | ||||
fs.vp = NULL; | fs.vp = NULL; | ||||
faultcount = 0; | faultcount = 0; | ||||
nera = -1; | nera = -1; | ||||
growstack = true; | growstack = true; | ||||
hardfault = false; | hardfault = false; | ||||
RetryFault:; | RetryFault:; | ||||
▲ Show 20 Lines • Show All 159 Lines • ▼ Show 20 Lines | if (fs.m != NULL) { | ||||
} | } | ||||
unlock_map(&fs); | unlock_map(&fs); | ||||
if (fs.m == vm_page_lookup(fs.object, | if (fs.m == vm_page_lookup(fs.object, | ||||
fs.pindex)) { | fs.pindex)) { | ||||
vm_page_sleep_if_busy(fs.m, "vmpfw"); | vm_page_sleep_if_busy(fs.m, "vmpfw"); | ||||
} | } | ||||
vm_object_pip_wakeup(fs.object); | vm_object_pip_wakeup(fs.object); | ||||
VM_OBJECT_WUNLOCK(fs.object); | VM_OBJECT_WUNLOCK(fs.object); | ||||
PCPU_INC(cnt.v_intrans); | VM_CNT_INC(v_intrans); | ||||
vm_object_deallocate(fs.first_object); | vm_object_deallocate(fs.first_object); | ||||
goto RetryFault; | goto RetryFault; | ||||
} | } | ||||
vm_page_lock(fs.m); | vm_page_lock(fs.m); | ||||
vm_page_remque(fs.m); | vm_page_remque(fs.m); | ||||
vm_page_unlock(fs.m); | vm_page_unlock(fs.m); | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 309 Lines • ▼ Show 20 Lines | if (next_object == NULL) { | ||||
fs.first_m = NULL; | fs.first_m = NULL; | ||||
/* | /* | ||||
* Zero the page if necessary and mark it valid. | * Zero the page if necessary and mark it valid. | ||||
*/ | */ | ||||
if ((fs.m->flags & PG_ZERO) == 0) { | if ((fs.m->flags & PG_ZERO) == 0) { | ||||
pmap_zero_page(fs.m); | pmap_zero_page(fs.m); | ||||
} else { | } else { | ||||
PCPU_INC(cnt.v_ozfod); | VM_CNT_INC(v_ozfod); | ||||
} | } | ||||
PCPU_INC(cnt.v_zfod); | VM_CNT_INC(v_zfod); | ||||
fs.m->valid = VM_PAGE_BITS_ALL; | fs.m->valid = VM_PAGE_BITS_ALL; | ||||
/* Don't try to prefault neighboring pages. */ | /* Don't try to prefault neighboring pages. */ | ||||
faultcount = 1; | faultcount = 1; | ||||
break; /* break to PAGE HAS BEEN FOUND */ | break; /* break to PAGE HAS BEEN FOUND */ | ||||
} else { | } else { | ||||
KASSERT(fs.object != next_object, | KASSERT(fs.object != next_object, | ||||
("object loop %p", next_object)); | ("object loop %p", next_object)); | ||||
VM_OBJECT_WLOCK(next_object); | VM_OBJECT_WLOCK(next_object); | ||||
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
/* | /* | ||||
* Removing the page from the backing object | * Removing the page from the backing object | ||||
* unbusied it. | * unbusied it. | ||||
*/ | */ | ||||
vm_page_xbusy(fs.m); | vm_page_xbusy(fs.m); | ||||
fs.first_m = fs.m; | fs.first_m = fs.m; | ||||
fs.m = NULL; | fs.m = NULL; | ||||
PCPU_INC(cnt.v_cow_optim); | VM_CNT_INC(v_cow_optim); | ||||
} else { | } else { | ||||
/* | /* | ||||
* Oh, well, lets copy it. | * Oh, well, lets copy it. | ||||
*/ | */ | ||||
pmap_copy_page(fs.m, fs.first_m); | pmap_copy_page(fs.m, fs.first_m); | ||||
fs.first_m->valid = VM_PAGE_BITS_ALL; | fs.first_m->valid = VM_PAGE_BITS_ALL; | ||||
if (wired && (fault_flags & | if (wired && (fault_flags & | ||||
VM_FAULT_WIRE) == 0) { | VM_FAULT_WIRE) == 0) { | ||||
Show All 19 Lines | #endif | ||||
/* | /* | ||||
* Only use the new page below... | * Only use the new page below... | ||||
*/ | */ | ||||
fs.object = fs.first_object; | fs.object = fs.first_object; | ||||
fs.pindex = fs.first_pindex; | fs.pindex = fs.first_pindex; | ||||
fs.m = fs.first_m; | fs.m = fs.first_m; | ||||
if (!is_first_object_locked) | if (!is_first_object_locked) | ||||
VM_OBJECT_WLOCK(fs.object); | VM_OBJECT_WLOCK(fs.object); | ||||
PCPU_INC(cnt.v_cow_faults); | VM_CNT_INC(v_cow_faults); | ||||
curthread->td_cow++; | curthread->td_cow++; | ||||
} else { | } else { | ||||
prot &= ~VM_PROT_WRITE; | prot &= ~VM_PROT_WRITE; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* We must verify that the maps have not changed since our last | * We must verify that the maps have not changed since our last | ||||
▲ Show 20 Lines • Show All 98 Lines • ▼ Show 20 Lines | #endif | ||||
vm_page_unlock(fs.m); | vm_page_unlock(fs.m); | ||||
vm_page_xunbusy(fs.m); | vm_page_xunbusy(fs.m); | ||||
/* | /* | ||||
* Unlock everything, and return | * Unlock everything, and return | ||||
*/ | */ | ||||
unlock_and_deallocate(&fs); | unlock_and_deallocate(&fs); | ||||
if (hardfault) { | if (hardfault) { | ||||
PCPU_INC(cnt.v_io_faults); | VM_CNT_INC(v_io_faults); | ||||
curthread->td_ru.ru_majflt++; | curthread->td_ru.ru_majflt++; | ||||
#ifdef RACCT | #ifdef RACCT | ||||
if (racct_enable && fs.object->type == OBJT_VNODE) { | if (racct_enable && fs.object->type == OBJT_VNODE) { | ||||
PROC_LOCK(curproc); | PROC_LOCK(curproc); | ||||
if ((fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) { | if ((fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) { | ||||
racct_add_force(curproc, RACCT_WRITEBPS, | racct_add_force(curproc, RACCT_WRITEBPS, | ||||
PAGE_SIZE + behind * PAGE_SIZE); | PAGE_SIZE + behind * PAGE_SIZE); | ||||
racct_add_force(curproc, RACCT_WRITEIOPS, 1); | racct_add_force(curproc, RACCT_WRITEIOPS, 1); | ||||
▲ Show 20 Lines • Show All 462 Lines • Show Last 20 Lines |