Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_fault.c
Show First 20 Lines • Show All 180 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
unlock_and_deallocate(struct faultstate *fs) | unlock_and_deallocate(struct faultstate *fs) | ||||
{ | { | ||||
vm_object_pip_wakeup(fs->object); | vm_object_pip_wakeup(fs->object); | ||||
VM_OBJECT_WUNLOCK(fs->object); | VM_OBJECT_WUNLOCK(fs->object); | ||||
if (fs->object != fs->first_object) { | if (fs->object != fs->first_object) { | ||||
VM_OBJECT_WLOCK(fs->first_object); | VM_OBJECT_WLOCK(fs->first_object); | ||||
vm_page_lock(fs->first_m); | |||||
vm_page_free(fs->first_m); | vm_page_free(fs->first_m); | ||||
vm_page_unlock(fs->first_m); | |||||
vm_object_pip_wakeup(fs->first_object); | vm_object_pip_wakeup(fs->first_object); | ||||
VM_OBJECT_WUNLOCK(fs->first_object); | VM_OBJECT_WUNLOCK(fs->first_object); | ||||
fs->first_m = NULL; | fs->first_m = NULL; | ||||
} | } | ||||
vm_object_deallocate(fs->first_object); | vm_object_deallocate(fs->first_object); | ||||
unlock_map(fs); | unlock_map(fs); | ||||
unlock_vp(fs); | unlock_vp(fs); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_prot_t prot, | ||||
if (need_dirty) | if (need_dirty) | ||||
vm_page_dirty(m); | vm_page_dirty(m); | ||||
if (!set_wd) | if (!set_wd) | ||||
vm_page_unlock(m); | vm_page_unlock(m); | ||||
else if (need_dirty) | else if (need_dirty) | ||||
vm_pager_page_unswapped(m); | vm_pager_page_unswapped(m); | ||||
} | } | ||||
static void | |||||
vm_fault_fill_hold(vm_page_t *m_hold, vm_page_t m) | |||||
{ | |||||
if (m_hold != NULL) { | |||||
*m_hold = m; | |||||
vm_page_lock(m); | |||||
vm_page_wire(m); | |||||
vm_page_unlock(m); | |||||
} | |||||
} | |||||
/* | /* | ||||
* Unlocks fs.first_object and fs.map on success. | * Unlocks fs.first_object and fs.map on success. | ||||
*/ | */ | ||||
static int | static int | ||||
vm_fault_soft_fast(struct faultstate *fs, vm_offset_t vaddr, vm_prot_t prot, | vm_fault_soft_fast(struct faultstate *fs, vm_offset_t vaddr, vm_prot_t prot, | ||||
int fault_type, int fault_flags, boolean_t wired, vm_page_t *m_hold) | int fault_type, int fault_flags, boolean_t wired, vm_page_t *m_hold) | ||||
{ | { | ||||
vm_page_t m, m_map; | vm_page_t m, m_map; | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | if (vm_page_ps_test(m_super, flags, m)) { | ||||
fault_type |= VM_PROT_WRITE; | fault_type |= VM_PROT_WRITE; | ||||
} | } | ||||
} | } | ||||
#endif | #endif | ||||
rv = pmap_enter(fs->map->pmap, vaddr, m_map, prot, fault_type | | rv = pmap_enter(fs->map->pmap, vaddr, m_map, prot, fault_type | | ||||
PMAP_ENTER_NOSLEEP | (wired ? PMAP_ENTER_WIRED : 0), psind); | PMAP_ENTER_NOSLEEP | (wired ? PMAP_ENTER_WIRED : 0), psind); | ||||
if (rv != KERN_SUCCESS) | if (rv != KERN_SUCCESS) | ||||
return (rv); | return (rv); | ||||
vm_fault_fill_hold(m_hold, m); | if (m_hold != NULL) { | ||||
*m_hold = m; | |||||
vm_page_wire(m); | |||||
} | |||||
vm_fault_dirty(fs->entry, m, prot, fault_type, fault_flags, false); | vm_fault_dirty(fs->entry, m, prot, fault_type, fault_flags, false); | ||||
if (psind == 0 && !wired) | if (psind == 0 && !wired) | ||||
vm_fault_prefault(fs, vaddr, PFBAK, PFFOR, true); | vm_fault_prefault(fs, vaddr, PFBAK, PFFOR, true); | ||||
VM_OBJECT_RUNLOCK(fs->first_object); | VM_OBJECT_RUNLOCK(fs->first_object); | ||||
vm_map_lookup_done(fs->map, fs->entry); | vm_map_lookup_done(fs->map, fs->entry); | ||||
curthread->td_ru.ru_minflt++; | curthread->td_ru.ru_minflt++; | ||||
return (KERN_SUCCESS); | return (KERN_SUCCESS); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 159 Lines • ▼ Show 20 Lines | if (psind > 0 && rv == KERN_FAILURE) { | ||||
} | } | ||||
} | } | ||||
#else | #else | ||||
MPASS(rv == KERN_SUCCESS); | MPASS(rv == KERN_SUCCESS); | ||||
#endif | #endif | ||||
VM_OBJECT_WLOCK(fs->first_object); | VM_OBJECT_WLOCK(fs->first_object); | ||||
m_mtx = NULL; | m_mtx = NULL; | ||||
for (i = 0; i < npages; i++) { | for (i = 0; i < npages; i++) { | ||||
vm_page_change_lock(&m[i], &m_mtx); | if ((fault_flags & VM_FAULT_WIRE) != 0) { | ||||
if ((fault_flags & VM_FAULT_WIRE) != 0) | |||||
vm_page_wire(&m[i]); | vm_page_wire(&m[i]); | ||||
else | } else { | ||||
vm_page_change_lock(&m[i], &m_mtx); | |||||
vm_page_activate(&m[i]); | vm_page_activate(&m[i]); | ||||
} | |||||
if (m_hold != NULL && m[i].pindex == fs->first_pindex) { | if (m_hold != NULL && m[i].pindex == fs->first_pindex) { | ||||
*m_hold = &m[i]; | *m_hold = &m[i]; | ||||
vm_page_wire(&m[i]); | vm_page_wire(&m[i]); | ||||
} | } | ||||
vm_page_xunbusy_maybelocked(&m[i]); | vm_page_xunbusy_maybelocked(&m[i]); | ||||
} | } | ||||
if (m_mtx != NULL) | if (m_mtx != NULL) | ||||
mtx_unlock(m_mtx); | mtx_unlock(m_mtx); | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, | ||||
int fault_flags, vm_page_t *m_hold) | int fault_flags, vm_page_t *m_hold) | ||||
{ | { | ||||
struct faultstate fs; | struct faultstate fs; | ||||
struct vnode *vp; | struct vnode *vp; | ||||
struct domainset *dset; | struct domainset *dset; | ||||
struct mtx *mtx; | |||||
vm_object_t next_object, retry_object; | vm_object_t next_object, retry_object; | ||||
vm_offset_t e_end, e_start; | vm_offset_t e_end, e_start; | ||||
vm_pindex_t retry_pindex; | vm_pindex_t retry_pindex; | ||||
vm_prot_t prot, retry_prot; | vm_prot_t prot, retry_prot; | ||||
int ahead, alloc_req, behind, cluster_offset, error, era, faultcount; | int ahead, alloc_req, behind, cluster_offset, error, era, faultcount; | ||||
int locked, nera, oom, result, rv; | int locked, nera, oom, result, rv; | ||||
u_char behavior; | u_char behavior; | ||||
boolean_t wired; /* Passed by reference. */ | boolean_t wired; /* Passed by reference. */ | ||||
▲ Show 20 Lines • Show All 152 Lines • ▼ Show 20 Lines | if (fs.m != NULL) { | ||||
vm_page_aflag_set(fs.m, PGA_REFERENCED); | vm_page_aflag_set(fs.m, PGA_REFERENCED); | ||||
if (fs.object != fs.first_object) { | if (fs.object != fs.first_object) { | ||||
if (!VM_OBJECT_TRYWLOCK( | if (!VM_OBJECT_TRYWLOCK( | ||||
fs.first_object)) { | fs.first_object)) { | ||||
VM_OBJECT_WUNLOCK(fs.object); | VM_OBJECT_WUNLOCK(fs.object); | ||||
VM_OBJECT_WLOCK(fs.first_object); | VM_OBJECT_WLOCK(fs.first_object); | ||||
VM_OBJECT_WLOCK(fs.object); | VM_OBJECT_WLOCK(fs.object); | ||||
} | } | ||||
vm_page_lock(fs.first_m); | |||||
vm_page_free(fs.first_m); | vm_page_free(fs.first_m); | ||||
vm_page_unlock(fs.first_m); | |||||
vm_object_pip_wakeup(fs.first_object); | vm_object_pip_wakeup(fs.first_object); | ||||
VM_OBJECT_WUNLOCK(fs.first_object); | VM_OBJECT_WUNLOCK(fs.first_object); | ||||
fs.first_m = NULL; | fs.first_m = NULL; | ||||
} | } | ||||
unlock_map(&fs); | unlock_map(&fs); | ||||
if (fs.m == vm_page_lookup(fs.object, | if (fs.m == vm_page_lookup(fs.object, | ||||
fs.pindex)) { | fs.pindex)) { | ||||
vm_page_sleep_if_busy(fs.m, "vmpfw"); | vm_page_sleep_if_busy(fs.m, "vmpfw"); | ||||
▲ Show 20 Lines • Show All 266 Lines • ▼ Show 20 Lines | if (fs.object->type != OBJT_DEFAULT) { | ||||
curproc->p_pid, curproc->p_comm); | curproc->p_pid, curproc->p_comm); | ||||
/* | /* | ||||
* If an I/O error occurred or the requested page was | * If an I/O error occurred or the requested page was | ||||
* outside the range of the pager, clean up and return | * outside the range of the pager, clean up and return | ||||
* an error. | * an error. | ||||
*/ | */ | ||||
if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) { | if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) { | ||||
vm_page_lock(fs.m); | |||||
if (!vm_page_wired(fs.m)) | if (!vm_page_wired(fs.m)) | ||||
vm_page_free(fs.m); | vm_page_free(fs.m); | ||||
else | else | ||||
vm_page_xunbusy_maybelocked(fs.m); | vm_page_xunbusy(fs.m); | ||||
vm_page_unlock(fs.m); | |||||
fs.m = NULL; | fs.m = NULL; | ||||
unlock_and_deallocate(&fs); | unlock_and_deallocate(&fs); | ||||
return (rv == VM_PAGER_ERROR ? KERN_FAILURE : | return (rv == VM_PAGER_ERROR ? KERN_FAILURE : | ||||
KERN_PROTECTION_FAILURE); | KERN_PROTECTION_FAILURE); | ||||
} | } | ||||
/* | /* | ||||
* The requested page does not exist at this object/ | * The requested page does not exist at this object/ | ||||
* offset. Remove the invalid page from the object, | * offset. Remove the invalid page from the object, | ||||
* waking up anyone waiting for it, and continue on to | * waking up anyone waiting for it, and continue on to | ||||
* the next object. However, if this is the top-level | * the next object. However, if this is the top-level | ||||
* object, we must leave the busy page in place to | * object, we must leave the busy page in place to | ||||
* prevent another process from rushing past us, and | * prevent another process from rushing past us, and | ||||
* inserting the page in that object at the same time | * inserting the page in that object at the same time | ||||
* that we are. | * that we are. | ||||
*/ | */ | ||||
if (fs.object != fs.first_object) { | if (fs.object != fs.first_object) { | ||||
vm_page_lock(fs.m); | |||||
if (!vm_page_wired(fs.m)) | if (!vm_page_wired(fs.m)) | ||||
vm_page_free(fs.m); | vm_page_free(fs.m); | ||||
else | else | ||||
vm_page_xunbusy_maybelocked(fs.m); | vm_page_xunbusy(fs.m); | ||||
vm_page_unlock(fs.m); | |||||
fs.m = NULL; | fs.m = NULL; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* We get here if the object has default pager (or unwiring) | * We get here if the object has default pager (or unwiring) | ||||
* or the pager doesn't have the page. | * or the pager doesn't have the page. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | if ((fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) { | ||||
*/ | */ | ||||
((fs.object->type == OBJT_DEFAULT) || | ((fs.object->type == OBJT_DEFAULT) || | ||||
(fs.object->type == OBJT_SWAP)) && | (fs.object->type == OBJT_SWAP)) && | ||||
(is_first_object_locked = VM_OBJECT_TRYWLOCK(fs.first_object)) && | (is_first_object_locked = VM_OBJECT_TRYWLOCK(fs.first_object)) && | ||||
/* | /* | ||||
* We don't chase down the shadow chain | * We don't chase down the shadow chain | ||||
*/ | */ | ||||
fs.object == fs.first_object->backing_object) { | fs.object == fs.first_object->backing_object) { | ||||
/* | |||||
* Keep the page wired to ensure that it is not | |||||
* freed by another thread, such as the page | |||||
* daemon, while it is disassociated from an | |||||
* object. | |||||
*/ | |||||
mtx = NULL; | |||||
vm_page_change_lock(fs.m, &mtx); | |||||
vm_page_wire(fs.m); | |||||
(void)vm_page_remove(fs.m); | (void)vm_page_remove(fs.m); | ||||
vm_page_change_lock(fs.first_m, &mtx); | |||||
vm_page_replace_checked(fs.m, fs.first_object, | vm_page_replace_checked(fs.m, fs.first_object, | ||||
fs.first_pindex, fs.first_m); | fs.first_pindex, fs.first_m); | ||||
vm_page_free(fs.first_m); | vm_page_free(fs.first_m); | ||||
vm_page_change_lock(fs.m, &mtx); | |||||
vm_page_unwire(fs.m, PQ_ACTIVE); | |||||
mtx_unlock(mtx); | |||||
vm_page_dirty(fs.m); | vm_page_dirty(fs.m); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
/* | /* | ||||
* Rename the reservation. | * Rename the reservation. | ||||
*/ | */ | ||||
vm_reserv_rename(fs.m, fs.first_object, | vm_reserv_rename(fs.m, fs.first_object, | ||||
fs.object, OFF_TO_IDX( | fs.object, OFF_TO_IDX( | ||||
fs.first_object->backing_object_offset)); | fs.first_object->backing_object_offset)); | ||||
Show All 9 Lines | #endif | ||||
} else { | } else { | ||||
/* | /* | ||||
* Oh, well, lets copy it. | * Oh, well, lets copy it. | ||||
*/ | */ | ||||
pmap_copy_page(fs.m, fs.first_m); | pmap_copy_page(fs.m, fs.first_m); | ||||
fs.first_m->valid = VM_PAGE_BITS_ALL; | fs.first_m->valid = VM_PAGE_BITS_ALL; | ||||
if (wired && (fault_flags & | if (wired && (fault_flags & | ||||
VM_FAULT_WIRE) == 0) { | VM_FAULT_WIRE) == 0) { | ||||
vm_page_lock(fs.first_m); | |||||
vm_page_wire(fs.first_m); | vm_page_wire(fs.first_m); | ||||
vm_page_unlock(fs.first_m); | |||||
vm_page_lock(fs.m); | |||||
vm_page_unwire(fs.m, PQ_INACTIVE); | vm_page_unwire(fs.m, PQ_INACTIVE); | ||||
vm_page_unlock(fs.m); | |||||
} | } | ||||
/* | /* | ||||
* We no longer need the old page or object. | * We no longer need the old page or object. | ||||
*/ | */ | ||||
release_page(&fs); | release_page(&fs); | ||||
} | } | ||||
/* | /* | ||||
* fs.object != fs.first_object due to above | * fs.object != fs.first_object due to above | ||||
▲ Show 20 Lines • Show All 116 Lines • ▼ Show 20 Lines | #endif | ||||
pmap_enter(fs.map->pmap, vaddr, fs.m, prot, | pmap_enter(fs.map->pmap, vaddr, fs.m, prot, | ||||
fault_type | (wired ? PMAP_ENTER_WIRED : 0), 0); | fault_type | (wired ? PMAP_ENTER_WIRED : 0), 0); | ||||
if (faultcount != 1 && (fault_flags & VM_FAULT_WIRE) == 0 && | if (faultcount != 1 && (fault_flags & VM_FAULT_WIRE) == 0 && | ||||
wired == 0) | wired == 0) | ||||
vm_fault_prefault(&fs, vaddr, | vm_fault_prefault(&fs, vaddr, | ||||
faultcount > 0 ? behind : PFBAK, | faultcount > 0 ? behind : PFBAK, | ||||
faultcount > 0 ? ahead : PFFOR, false); | faultcount > 0 ? ahead : PFFOR, false); | ||||
VM_OBJECT_WLOCK(fs.object); | VM_OBJECT_WLOCK(fs.object); | ||||
vm_page_lock(fs.m); | |||||
/* | /* | ||||
* If the page is not wired down, then put it where the pageout daemon | * If the page is not wired down, then put it where the pageout daemon | ||||
* can find it. | * can find it. | ||||
*/ | */ | ||||
if ((fault_flags & VM_FAULT_WIRE) != 0) | if ((fault_flags & VM_FAULT_WIRE) != 0) { | ||||
vm_page_wire(fs.m); | vm_page_wire(fs.m); | ||||
else | } else { | ||||
vm_page_lock(fs.m); | |||||
vm_page_activate(fs.m); | vm_page_activate(fs.m); | ||||
vm_page_unlock(fs.m); | |||||
} | |||||
if (m_hold != NULL) { | if (m_hold != NULL) { | ||||
*m_hold = fs.m; | *m_hold = fs.m; | ||||
vm_page_wire(fs.m); | vm_page_wire(fs.m); | ||||
} | } | ||||
vm_page_unlock(fs.m); | |||||
vm_page_xunbusy(fs.m); | vm_page_xunbusy(fs.m); | ||||
/* | /* | ||||
* Unlock everything, and return | * Unlock everything, and return | ||||
*/ | */ | ||||
unlock_and_deallocate(&fs); | unlock_and_deallocate(&fs); | ||||
if (hardfault) { | if (hardfault) { | ||||
VM_CNT_INC(v_io_faults); | VM_CNT_INC(v_io_faults); | ||||
▲ Show 20 Lines • Show All 252 Lines • ▼ Show 20 Lines | if (pmap_failed) { | ||||
for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) | for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) | ||||
if (*mp == NULL && vm_fault_hold(map, va, prot, | if (*mp == NULL && vm_fault_hold(map, va, prot, | ||||
VM_FAULT_NORMAL, mp) != KERN_SUCCESS) | VM_FAULT_NORMAL, mp) != KERN_SUCCESS) | ||||
goto error; | goto error; | ||||
} | } | ||||
return (count); | return (count); | ||||
error: | error: | ||||
for (mp = ma; mp < ma + count; mp++) | for (mp = ma; mp < ma + count; mp++) | ||||
if (*mp != NULL) { | if (*mp != NULL) | ||||
vm_page_lock(*mp); | vm_page_unwire(*mp, PQ_INACTIVE); | ||||
if (vm_page_unwire(*mp, PQ_INACTIVE) && | |||||
(*mp)->object == NULL) | |||||
vm_page_free(*mp); | |||||
vm_page_unlock(*mp); | |||||
} | |||||
return (-1); | return (-1); | ||||
} | } | ||||
/* | /* | ||||
* Routine: | * Routine: | ||||
* vm_fault_copy_entry | * vm_fault_copy_entry | ||||
* Function: | * Function: | ||||
* Create new shadow object backing dst_entry with private copy of | * Create new shadow object backing dst_entry with private copy of | ||||
▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | again: | ||||
/* | /* | ||||
* Mark it no longer busy, and put it on the active list. | * Mark it no longer busy, and put it on the active list. | ||||
*/ | */ | ||||
VM_OBJECT_WLOCK(dst_object); | VM_OBJECT_WLOCK(dst_object); | ||||
if (upgrade) { | if (upgrade) { | ||||
if (src_m != dst_m) { | if (src_m != dst_m) { | ||||
vm_page_lock(src_m); | |||||
vm_page_unwire(src_m, PQ_INACTIVE); | vm_page_unwire(src_m, PQ_INACTIVE); | ||||
vm_page_unlock(src_m); | |||||
vm_page_lock(dst_m); | |||||
vm_page_wire(dst_m); | vm_page_wire(dst_m); | ||||
vm_page_unlock(dst_m); | |||||
} else { | } else { | ||||
KASSERT(vm_page_wired(dst_m), | KASSERT(vm_page_wired(dst_m), | ||||
("dst_m %p is not wired", dst_m)); | ("dst_m %p is not wired", dst_m)); | ||||
} | } | ||||
} else { | } else { | ||||
vm_page_lock(dst_m); | vm_page_lock(dst_m); | ||||
vm_page_activate(dst_m); | vm_page_activate(dst_m); | ||||
vm_page_unlock(dst_m); | vm_page_unlock(dst_m); | ||||
Show All 29 Lines |