Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_object.c
Show First 20 Lines • Show All 234 Lines • ▼ Show 20 Lines | |||||
_vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags, | _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags, | ||||
vm_object_t object, void *handle) | vm_object_t object, void *handle) | ||||
{ | { | ||||
TAILQ_INIT(&object->memq); | TAILQ_INIT(&object->memq); | ||||
LIST_INIT(&object->shadow_head); | LIST_INIT(&object->shadow_head); | ||||
object->type = type; | object->type = type; | ||||
if (type == OBJT_SWAP) | if ((flags & OBJ_SWAP) != 0) | ||||
markj: `(flags & OBJ_SWAPPING) != 0`? | |||||
Done Inline ActionsIt is too early to check the flag, with the current code structure. I set OBJ_SWAPPING in swap_pager_alloc_init() right after vm_object_allocate() call. I can move setting of the flag into vm_object_allocate(). kib: It is too early to check the flag, with the current code structure. I set OBJ_SWAPPING in… | |||||
pctrie_init(&object->un_pager.swp.swp_blks); | pctrie_init(&object->un_pager.swp.swp_blks); | ||||
/* | /* | ||||
* Ensure that swap_pager_swapoff() iteration over object_list | * Ensure that swap_pager_swapoff() iteration over object_list | ||||
* sees up to date type and pctrie head if it observed | * sees up to date type and pctrie head if it observed | ||||
* non-dead object. | * non-dead object. | ||||
*/ | */ | ||||
atomic_thread_fence_rel(); | atomic_thread_fence_rel(); | ||||
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines | vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr) | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
switch (object->type) { | switch (object->type) { | ||||
case OBJT_DEFAULT: | case OBJT_DEFAULT: | ||||
case OBJT_DEVICE: | case OBJT_DEVICE: | ||||
case OBJT_MGTDEVICE: | case OBJT_MGTDEVICE: | ||||
case OBJT_PHYS: | case OBJT_PHYS: | ||||
case OBJT_SG: | case OBJT_SG: | ||||
case OBJT_SWAP: | case OBJT_SWAP: | ||||
case OBJT_SWAP_TMPFS: | |||||
case OBJT_VNODE: | case OBJT_VNODE: | ||||
if (!TAILQ_EMPTY(&object->memq)) | if (!TAILQ_EMPTY(&object->memq)) | ||||
return (KERN_FAILURE); | return (KERN_FAILURE); | ||||
break; | break; | ||||
case OBJT_DEAD: | case OBJT_DEAD: | ||||
return (KERN_INVALID_ARGUMENT); | return (KERN_INVALID_ARGUMENT); | ||||
default: | default: | ||||
panic("vm_object_set_memattr: object %p is of undefined type", | panic("vm_object_set_memattr: object %p is of undefined type", | ||||
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
vm_object_t object; | vm_object_t object; | ||||
u_short flags; | u_short flags; | ||||
switch (type) { | switch (type) { | ||||
case OBJT_DEAD: | case OBJT_DEAD: | ||||
panic("vm_object_allocate: can't create OBJT_DEAD"); | panic("vm_object_allocate: can't create OBJT_DEAD"); | ||||
case OBJT_DEFAULT: | case OBJT_DEFAULT: | ||||
case OBJT_SWAP: | |||||
flags = OBJ_COLORED; | flags = OBJ_COLORED; | ||||
break; | break; | ||||
case OBJT_SWAP: | |||||
case OBJT_SWAP_TMPFS: | |||||
flags = OBJ_COLORED | OBJ_SWAP; | |||||
break; | |||||
case OBJT_DEVICE: | case OBJT_DEVICE: | ||||
case OBJT_SG: | case OBJT_SG: | ||||
flags = OBJ_FICTITIOUS | OBJ_UNMANAGED; | flags = OBJ_FICTITIOUS | OBJ_UNMANAGED; | ||||
break; | break; | ||||
case OBJT_MGTDEVICE: | case OBJT_MGTDEVICE: | ||||
flags = OBJ_FICTITIOUS; | flags = OBJ_FICTITIOUS; | ||||
break; | break; | ||||
case OBJT_PHYS: | case OBJT_PHYS: | ||||
▲ Show 20 Lines • Show All 133 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
vm_object_t object; | vm_object_t object; | ||||
/* Fetch the final shadow. */ | /* Fetch the final shadow. */ | ||||
object = LIST_FIRST(&backing_object->shadow_head); | object = LIST_FIRST(&backing_object->shadow_head); | ||||
KASSERT(object != NULL && backing_object->shadow_count == 1, | KASSERT(object != NULL && backing_object->shadow_count == 1, | ||||
("vm_object_anon_deallocate: ref_count: %d, shadow_count: %d", | ("vm_object_anon_deallocate: ref_count: %d, shadow_count: %d", | ||||
backing_object->ref_count, backing_object->shadow_count)); | backing_object->ref_count, backing_object->shadow_count)); | ||||
KASSERT((object->flags & (OBJ_TMPFS_NODE | OBJ_ANON)) == OBJ_ANON, | KASSERT((object->flags & OBJ_ANON) != 0, | ||||
("invalid shadow object %p", object)); | ("invalid shadow object %p", object)); | ||||
if (!VM_OBJECT_TRYWLOCK(object)) { | if (!VM_OBJECT_TRYWLOCK(object)) { | ||||
/* | /* | ||||
* Prevent object from disappearing since we do not have a | * Prevent object from disappearing since we do not have a | ||||
* reference. | * reference. | ||||
*/ | */ | ||||
vm_object_pip_add(object, 1); | vm_object_pip_add(object, 1); | ||||
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines | while (object != NULL) { | ||||
/* | /* | ||||
* Handle the final reference to an object. We restart | * Handle the final reference to an object. We restart | ||||
* the loop with the backing object to avoid recursion. | * the loop with the backing object to avoid recursion. | ||||
*/ | */ | ||||
umtx_shm_object_terminated(object); | umtx_shm_object_terminated(object); | ||||
temp = object->backing_object; | temp = object->backing_object; | ||||
if (temp != NULL) { | if (temp != NULL) { | ||||
KASSERT((object->flags & OBJ_TMPFS_NODE) == 0, | KASSERT(object->type != OBJT_SWAP_TMPFS, | ||||
("shadowed tmpfs v_object 2 %p", object)); | ("shadowed tmpfs v_object 2 %p", object)); | ||||
vm_object_backing_remove(object); | vm_object_backing_remove(object); | ||||
} | } | ||||
KASSERT((object->flags & OBJ_DEAD) == 0, | KASSERT((object->flags & OBJ_DEAD) == 0, | ||||
("vm_object_deallocate: Terminating dead object.")); | ("vm_object_deallocate: Terminating dead object.")); | ||||
vm_object_set_flag(object, OBJ_DEAD); | vm_object_set_flag(object, OBJ_DEAD); | ||||
vm_object_terminate(object); | vm_object_terminate(object); | ||||
▲ Show 20 Lines • Show All 264 Lines • ▼ Show 20 Lines | if ((object->flags & OBJ_PG_DTOR) == 0) | ||||
vm_object_terminate_pages(object); | vm_object_terminate_pages(object); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (__predict_false(!LIST_EMPTY(&object->rvq))) | if (__predict_false(!LIST_EMPTY(&object->rvq))) | ||||
vm_reserv_break_all(object); | vm_reserv_break_all(object); | ||||
#endif | #endif | ||||
KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT || | KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT || | ||||
object->type == OBJT_SWAP, | object->type == OBJT_SWAP || object->type == OBJT_SWAP_TMPFS, | ||||
("%s: non-swap obj %p has cred", __func__, object)); | ("%s: non-swap obj %p has cred", __func__, object)); | ||||
/* | /* | ||||
* Let the pager know object is dead. | * Let the pager know object is dead. | ||||
*/ | */ | ||||
vm_pager_deallocate(object); | vm_pager_deallocate(object); | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
▲ Show 20 Lines • Show All 302 Lines • ▼ Show 20 Lines | return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) == | ||||
(OBJ_ONEMAPPING | OBJ_ANON)); | (OBJ_ONEMAPPING | OBJ_ANON)); | ||||
} | } | ||||
static void | static void | ||||
vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex, | vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex, | ||||
vm_size_t size) | vm_size_t size) | ||||
{ | { | ||||
if (advice == MADV_FREE && object->type == OBJT_SWAP) | if (advice == MADV_FREE) | ||||
swap_pager_freespace(object, pindex, size); | vm_pager_freespace(object, pindex, size); | ||||
} | } | ||||
/* | /* | ||||
* vm_object_madvise: | * vm_object_madvise: | ||||
* | * | ||||
* Implements the madvise function at the object/page level. | * Implements the madvise function at the object/page level. | ||||
* | * | ||||
* MADV_WILLNEED (any object) | * MADV_WILLNEED (any object) | ||||
▲ Show 20 Lines • Show All 332 Lines • ▼ Show 20 Lines | #endif | ||||
* orig_object's type may change while sleeping, so keep track | * orig_object's type may change while sleeping, so keep track | ||||
* of the beginning of the busied range. | * of the beginning of the busied range. | ||||
*/ | */ | ||||
if (orig_object->type != OBJT_SWAP) | if (orig_object->type != OBJT_SWAP) | ||||
vm_page_xunbusy(m); | vm_page_xunbusy(m); | ||||
else if (m_busy == NULL) | else if (m_busy == NULL) | ||||
m_busy = m; | m_busy = m; | ||||
} | } | ||||
if (orig_object->type == OBJT_SWAP) { | if ((orig_object->flags & OBJ_SWAP) != 0) { | ||||
/* | /* | ||||
* swap_pager_copy() can sleep, in which case the orig_object's | * swap_pager_copy() can sleep, in which case the orig_object's | ||||
* and new_object's locks are released and reacquired. | * and new_object's locks are released and reacquired. | ||||
*/ | */ | ||||
swap_pager_copy(orig_object, new_object, offidxstart, 0); | swap_pager_copy(orig_object, new_object, offidxstart, 0); | ||||
if (m_busy != NULL) | if (m_busy != NULL) | ||||
TAILQ_FOREACH_FROM(m_busy, &new_object->memq, listq) | TAILQ_FOREACH_FROM(m_busy, &new_object->memq, listq) | ||||
vm_page_xunbusy(m_busy); | vm_page_xunbusy(m_busy); | ||||
▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Lines | KASSERT(object->backing_object == backing_object, | ||||
("vm_object_collapse_scan: backing object mismatch %p != %p", | ("vm_object_collapse_scan: backing object mismatch %p != %p", | ||||
object->backing_object, backing_object)); | object->backing_object, backing_object)); | ||||
KASSERT(p->object == backing_object, | KASSERT(p->object == backing_object, | ||||
("vm_object_collapse_scan: object mismatch %p != %p", | ("vm_object_collapse_scan: object mismatch %p != %p", | ||||
p->object, backing_object)); | p->object, backing_object)); | ||||
if (p->pindex < backing_offset_index || | if (p->pindex < backing_offset_index || | ||||
new_pindex >= object->size) { | new_pindex >= object->size) { | ||||
if (backing_object->type == OBJT_SWAP) | vm_pager_freespace(backing_object, p->pindex, 1); | ||||
swap_pager_freespace(backing_object, p->pindex, | |||||
1); | |||||
KASSERT(!pmap_page_is_mapped(p), | KASSERT(!pmap_page_is_mapped(p), | ||||
("freeing mapped page %p", p)); | ("freeing mapped page %p", p)); | ||||
if (vm_page_remove(p)) | if (vm_page_remove(p)) | ||||
vm_page_free(p); | vm_page_free(p); | ||||
continue; | continue; | ||||
} | } | ||||
Show All 32 Lines | for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) { | ||||
if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL, | if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL, | ||||
NULL)) { | NULL)) { | ||||
/* | /* | ||||
* The page already exists in the parent OR swap exists | * The page already exists in the parent OR swap exists | ||||
* for this location in the parent. Leave the parent's | * for this location in the parent. Leave the parent's | ||||
* page alone. Destroy the original page from the | * page alone. Destroy the original page from the | ||||
* backing object. | * backing object. | ||||
*/ | */ | ||||
if (backing_object->type == OBJT_SWAP) | vm_pager_freespace(backing_object, p->pindex, 1); | ||||
swap_pager_freespace(backing_object, p->pindex, | |||||
1); | |||||
KASSERT(!pmap_page_is_mapped(p), | KASSERT(!pmap_page_is_mapped(p), | ||||
("freeing mapped page %p", p)); | ("freeing mapped page %p", p)); | ||||
if (vm_page_remove(p)) | if (vm_page_remove(p)) | ||||
vm_page_free(p); | vm_page_free(p); | ||||
if (pp != NULL) | if (pp != NULL) | ||||
vm_page_xunbusy(pp); | vm_page_xunbusy(pp); | ||||
continue; | continue; | ||||
} | } | ||||
/* | /* | ||||
* Page does not exist in parent, rename the page from the | * Page does not exist in parent, rename the page from the | ||||
* backing object to the main object. | * backing object to the main object. | ||||
* | * | ||||
* If the page was mapped to a process, it can remain mapped | * If the page was mapped to a process, it can remain mapped | ||||
* through the rename. vm_page_rename() will dirty the page. | * through the rename. vm_page_rename() will dirty the page. | ||||
*/ | */ | ||||
if (vm_page_rename(p, object, new_pindex)) { | if (vm_page_rename(p, object, new_pindex)) { | ||||
vm_page_xunbusy(p); | vm_page_xunbusy(p); | ||||
next = vm_object_collapse_scan_wait(object, NULL); | next = vm_object_collapse_scan_wait(object, NULL); | ||||
continue; | continue; | ||||
} | } | ||||
/* Use the old pindex to free the right page. */ | /* Use the old pindex to free the right page. */ | ||||
if (backing_object->type == OBJT_SWAP) | vm_pager_freespace(backing_object, new_pindex + | ||||
swap_pager_freespace(backing_object, | backing_offset_index, 1); | ||||
new_pindex + backing_offset_index, 1); | |||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
/* | /* | ||||
* Rename the reservation. | * Rename the reservation. | ||||
*/ | */ | ||||
vm_reserv_rename(p, object, backing_object, | vm_reserv_rename(p, object, backing_object, | ||||
backing_offset_index); | backing_offset_index); | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | #if VM_NRESERVLEVEL > 0 | ||||
*/ | */ | ||||
if (__predict_false(!LIST_EMPTY(&backing_object->rvq))) | if (__predict_false(!LIST_EMPTY(&backing_object->rvq))) | ||||
vm_reserv_break_all(backing_object); | vm_reserv_break_all(backing_object); | ||||
#endif | #endif | ||||
/* | /* | ||||
* Move the pager from backing_object to object. | * Move the pager from backing_object to object. | ||||
*/ | */ | ||||
if (backing_object->type == OBJT_SWAP) { | if ((backing_object->flags & OBJ_SWAP) != 0) { | ||||
/* | /* | ||||
* swap_pager_copy() can sleep, in which case | * swap_pager_copy() can sleep, in which case | ||||
* the backing_object's and object's locks are | * the backing_object's and object's locks are | ||||
* released and reacquired. | * released and reacquired. | ||||
* Since swap_pager_copy() is being asked to | * Since swap_pager_copy() is being asked to | ||||
* destroy backing_object, it will change the | * destroy backing_object, it will change the | ||||
* type to OBJT_DEFAULT. | * type to OBJT_DEFAULT. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Lines | wired: | ||||
} | } | ||||
if ((options & OBJPR_NOTMAPPED) == 0 && | if ((options & OBJPR_NOTMAPPED) == 0 && | ||||
object->ref_count != 0 && !vm_page_try_remove_all(p)) | object->ref_count != 0 && !vm_page_try_remove_all(p)) | ||||
goto wired; | goto wired; | ||||
vm_page_free(p); | vm_page_free(p); | ||||
} | } | ||||
vm_object_pip_wakeup(object); | vm_object_pip_wakeup(object); | ||||
if (object->type == OBJT_SWAP) { | vm_pager_freespace(object, start, (end == 0 ? object->size : end) - | ||||
if (end == 0) | start); | ||||
end = object->size; | |||||
swap_pager_freespace(object, start, end - start); | |||||
} | } | ||||
} | |||||
/* | /* | ||||
* vm_object_page_noreuse: | * vm_object_page_noreuse: | ||||
* | * | ||||
* For the given object, attempt to move the specified pages to | * For the given object, attempt to move the specified pages to | ||||
* the head of the inactive queue. This bypasses regular LRU | * the head of the inactive queue. This bypasses regular LRU | ||||
* operation and allows the pages to be reused quickly under memory | * operation and allows the pages to be reused quickly under memory | ||||
* pressure. If a page is wired for any reason, then it will not | * pressure. If a page is wired for any reason, then it will not | ||||
▲ Show 20 Lines • Show All 172 Lines • ▼ Show 20 Lines | #endif | ||||
if (next_pindex + next_size > prev_object->size) | if (next_pindex + next_size > prev_object->size) | ||||
prev_object->size = next_pindex + next_size; | prev_object->size = next_pindex + next_size; | ||||
VM_OBJECT_WUNLOCK(prev_object); | VM_OBJECT_WUNLOCK(prev_object); | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
void | void | ||||
vm_object_set_writeable_dirty(vm_object_t object) | vm_object_set_writeable_dirty_(vm_object_t object) | ||||
{ | { | ||||
/* Only set for vnodes & tmpfs */ | |||||
if (object->type != OBJT_VNODE && | |||||
(object->flags & OBJ_TMPFS_NODE) == 0) | |||||
return; | |||||
atomic_add_int(&object->generation, 1); | atomic_add_int(&object->generation, 1); | ||||
} | } | ||||
bool | |||||
vm_object_mightbedirty_(vm_object_t object) | |||||
{ | |||||
return (object->generation != object->cleangeneration); | |||||
} | |||||
/* | /* | ||||
* vm_object_unwire: | * vm_object_unwire: | ||||
* | * | ||||
* For each page offset within the specified range of the given object, | * For each page offset within the specified range of the given object, | ||||
* find the highest-level page in the shadow chain and unwire it. A page | * find the highest-level page in the shadow chain and unwire it. A page | ||||
* must exist at every page offset, and the highest-level page must be | * must exist at every page offset, and the highest-level page must be | ||||
* wired. | * wired. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | |||||
* no vnode allocated at the time of the call. | * no vnode allocated at the time of the call. | ||||
*/ | */ | ||||
struct vnode * | struct vnode * | ||||
vm_object_vnode(vm_object_t object) | vm_object_vnode(vm_object_t object) | ||||
{ | { | ||||
struct vnode *vp; | struct vnode *vp; | ||||
VM_OBJECT_ASSERT_LOCKED(object); | VM_OBJECT_ASSERT_LOCKED(object); | ||||
if (object->type == OBJT_VNODE) { | vm_pager_getvp(object, &vp, NULL); | ||||
vp = object->handle; | |||||
KASSERT(vp != NULL, ("%s: OBJT_VNODE has no vnode", __func__)); | |||||
} else if (object->type == OBJT_SWAP && | |||||
(object->flags & OBJ_TMPFS) != 0) { | |||||
vp = object->un_pager.swp.swp_tmpfs; | |||||
KASSERT(vp != NULL, ("%s: OBJT_TMPFS has no vnode", __func__)); | |||||
} else { | |||||
vp = NULL; | |||||
} | |||||
return (vp); | return (vp); | ||||
} | } | ||||
/* | /* | ||||
* Busy the vm object. This prevents new pages belonging to the object from | * Busy the vm object. This prevents new pages belonging to the object from | ||||
* becoming busy. Existing pages persist as busy. Callers are responsible | * becoming busy. Existing pages persist as busy. Callers are responsible | ||||
* for checking page state before proceeding. | * for checking page state before proceeding. | ||||
*/ | */ | ||||
Show All 36 Lines | vm_object_kvme_type(vm_object_t object, struct vnode **vpp) | ||||
if (vpp != NULL) | if (vpp != NULL) | ||||
*vpp = vm_object_vnode(object); | *vpp = vm_object_vnode(object); | ||||
switch (object->type) { | switch (object->type) { | ||||
case OBJT_DEFAULT: | case OBJT_DEFAULT: | ||||
return (KVME_TYPE_DEFAULT); | return (KVME_TYPE_DEFAULT); | ||||
case OBJT_VNODE: | case OBJT_VNODE: | ||||
return (KVME_TYPE_VNODE); | return (KVME_TYPE_VNODE); | ||||
case OBJT_SWAP: | case OBJT_SWAP: | ||||
if ((object->flags & OBJ_TMPFS_NODE) != 0) | |||||
return (KVME_TYPE_VNODE); | |||||
return (KVME_TYPE_SWAP); | return (KVME_TYPE_SWAP); | ||||
case OBJT_SWAP_TMPFS: | |||||
return (KVME_TYPE_VNODE); | |||||
case OBJT_DEVICE: | case OBJT_DEVICE: | ||||
return (KVME_TYPE_DEVICE); | return (KVME_TYPE_DEVICE); | ||||
case OBJT_PHYS: | case OBJT_PHYS: | ||||
return (KVME_TYPE_PHYS); | return (KVME_TYPE_PHYS); | ||||
case OBJT_DEAD: | case OBJT_DEAD: | ||||
return (KVME_TYPE_DEAD); | return (KVME_TYPE_DEAD); | ||||
case OBJT_SG: | case OBJT_SG: | ||||
return (KVME_TYPE_SG); | return (KVME_TYPE_SG); | ||||
▲ Show 20 Lines • Show All 360 Lines • Show Last 20 Lines |
(flags & OBJ_SWAPPING) != 0?