Page MenuHomeFreeBSD
Differential D22119 Diff 64589 head/sys/vm/vm_object.c

Changeset View

Standalone View

View Options

head/sys/vm/vm_object.c

Show First 20 Lines • Show All 233 Lines • ▼ Show 20 Linesvm_object_zinit(void *mem, int size, int flags)
mtx_lock(&vm_object_list_mtx); mtx_lock(&vm_object_list_mtx);
TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
mtx_unlock(&vm_object_list_mtx); mtx_unlock(&vm_object_list_mtx);
return (0); return (0);
} }
static void static void
_vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object) _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
vm_object_t object)
{ {
TAILQ_INIT(&object->memq); TAILQ_INIT(&object->memq);
LIST_INIT(&object->shadow_head); LIST_INIT(&object->shadow_head);
object->type = type; object->type = type;
if (type == OBJT_SWAP) if (type == OBJT_SWAP)
pctrie_init(&object->un_pager.swp.swp_blks); pctrie_init(&object->un_pager.swp.swp_blks);
/* /*
* Ensure that swap_pager_swapoff() iteration over object_list * Ensure that swap_pager_swapoff() iteration over object_list
* sees up to date type and pctrie head if it observed * sees up to date type and pctrie head if it observed
* non-dead object. * non-dead object.
*/ */
atomic_thread_fence_rel(); atomic_thread_fence_rel();
switch (type) { object->pg_color = 0;
case OBJT_DEAD: object->flags = flags;
panic("_vm_object_allocate: can't create OBJT_DEAD");
case OBJT_DEFAULT:
case OBJT_SWAP:
object->flags = OBJ_ONEMAPPING;
break;
case OBJT_DEVICE:
case OBJT_SG:
object->flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
break;
case OBJT_MGTDEVICE:
object->flags = OBJ_FICTITIOUS;
break;
case OBJT_PHYS:
object->flags = OBJ_UNMANAGED;
break;
case OBJT_VNODE:
object->flags = 0;
break;
default:
panic("_vm_object_allocate: type %d is undefined", type);
}
object->size = size; object->size = size;
object->domain.dr_policy = NULL; object->domain.dr_policy = NULL;
object->generation = 1; object->generation = 1;
object->cleangeneration = 1; object->cleangeneration = 1;
refcount_init(&object->ref_count, 1); refcount_init(&object->ref_count, 1);
object->memattr = VM_MEMATTR_DEFAULT; object->memattr = VM_MEMATTR_DEFAULT;
object->cred = NULL; object->cred = NULL;
object->charge = 0; object->charge = 0;
Show All 14 Lines
void void
vm_object_init(void) vm_object_init(void)
{ {
TAILQ_INIT(&vm_object_list); TAILQ_INIT(&vm_object_list);
mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF); mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
rw_init(&kernel_object->lock, "kernel vm object"); rw_init(&kernel_object->lock, "kernel vm object");
_vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS - _vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
VM_MIN_KERNEL_ADDRESS), kernel_object); VM_MIN_KERNEL_ADDRESS), OBJ_UNMANAGED, kernel_object);
#if VM_NRESERVLEVEL > 0 #if VM_NRESERVLEVEL > 0
kernel_object->flags |= OBJ_COLORED; kernel_object->flags |= OBJ_COLORED;
kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS); kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
#endif #endif
/* /*
* The lock portion of struct vm_object must be type stable due * The lock portion of struct vm_object must be type stable due
* to vm_pageout_fallback_object_lock locking a vm object * to vm_pageout_fallback_object_lock locking a vm object
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines
* vm_object_allocate: * vm_object_allocate:
* *
* Returns a new object with the given size. * Returns a new object with the given size.
*/ */
vm_object_t vm_object_t
vm_object_allocate(objtype_t type, vm_pindex_t size) vm_object_allocate(objtype_t type, vm_pindex_t size)
{ {
vm_object_t object; vm_object_t object;
u_short flags;
switch (type) {
case OBJT_DEAD:
panic("vm_object_allocate: can't create OBJT_DEAD");
case OBJT_DEFAULT:
case OBJT_SWAP:
flags = OBJ_COLORED;
break;
case OBJT_DEVICE:
case OBJT_SG:
flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
break;
case OBJT_MGTDEVICE:
flags = OBJ_FICTITIOUS;
break;
case OBJT_PHYS:
flags = OBJ_UNMANAGED;
break;
case OBJT_VNODE:
flags = 0;
break;
default:
panic("vm_object_allocate: type %d is undefined", type);
}
object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK); object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
_vm_object_allocate(type, size, object); _vm_object_allocate(type, size, flags, object);
return (object); return (object);
} }
/*
* vm_object_allocate_anon:
*
* Returns a new default object of the given size and marked as
* anonymous memory for special split/collapse handling. Color
* to be initialized by the caller.
*/
vm_object_t
vm_object_allocate_anon(vm_pindex_t size)
{
vm_object_t object;
object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
_vm_object_allocate(OBJT_DEFAULT, size, OBJ_ANON | OBJ_ONEMAPPING,
object);
return (object);
}
/* /*
* vm_object_reference: * vm_object_reference:
* *
* Gets another reference to the given object. Note: OBJ_DEAD * Gets another reference to the given object. Note: OBJ_DEAD
* objects can be referenced during final cleaning. * objects can be referenced during final cleaning.
*/ */
void void
vm_object_reference(vm_object_t object) vm_object_reference(vm_object_t object)
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines while (object != NULL) {
/* /*
* If the reference count goes to 0 we start calling * If the reference count goes to 0 we start calling
* vm_object_terminate() on the object chain. A ref count * vm_object_terminate() on the object chain. A ref count
* of 1 may be a special case depending on the shadow count * of 1 may be a special case depending on the shadow count
* being 0 or 1. These cases require a write lock on the * being 0 or 1. These cases require a write lock on the
* object. * object.
*/ */
if ((object->flags & OBJ_ANON) == 0)
released = refcount_release_if_gt(&object->ref_count, 1);
else
released = refcount_release_if_gt(&object->ref_count, 2); released = refcount_release_if_gt(&object->ref_count, 2);
VM_OBJECT_RUNLOCK(object); VM_OBJECT_RUNLOCK(object);
if (released) if (released)
return; return;
VM_OBJECT_WLOCK(object); VM_OBJECT_WLOCK(object);
KASSERT(object->ref_count != 0, KASSERT(object->ref_count != 0,
("vm_object_deallocate: object deallocated too many times: %d", object->type)); ("vm_object_deallocate: object deallocated too many times: %d", object->type));
refcount_release(&object->ref_count); refcount_release(&object->ref_count);
if (object->ref_count > 1) { if (object->ref_count > 1) {
VM_OBJECT_WUNLOCK(object); VM_OBJECT_WUNLOCK(object);
return; return;
} else if (object->ref_count == 1) { } else if (object->ref_count == 1) {
if (object->shadow_count == 0 && if (object->shadow_count == 0 &&
object->handle == NULL && object->handle == NULL &&
(object->type == OBJT_DEFAULT || (object->flags & OBJ_ANON) != 0) {
(object->type == OBJT_SWAP &&
(object->flags & OBJ_TMPFS_NODE) == 0))) {
vm_object_set_flag(object, OBJ_ONEMAPPING); vm_object_set_flag(object, OBJ_ONEMAPPING);
} else if ((object->shadow_count == 1) && } else if ((object->shadow_count == 1) &&
(object->handle == NULL) && (object->handle == NULL) &&
(object->type == OBJT_DEFAULT || (object->flags & OBJ_ANON) != 0) {
object->type == OBJT_SWAP)) {
vm_object_t robject; vm_object_t robject;
robject = LIST_FIRST(&object->shadow_head); robject = LIST_FIRST(&object->shadow_head);
KASSERT(robject != NULL, KASSERT(robject != NULL,
("vm_object_deallocate: ref_count: %d, shadow_count: %d", ("vm_object_deallocate: ref_count: %d, shadow_count: %d",
object->ref_count, object->ref_count,
object->shadow_count)); object->shadow_count));
KASSERT((robject->flags & OBJ_TMPFS_NODE) == 0, KASSERT((robject->flags & OBJ_TMPFS_NODE) == 0,
Show All 14 Lines if (object->ref_count > 1) {
continue; continue;
} }
/* /*
* Collapse object into its shadow unless its * Collapse object into its shadow unless its
* shadow is dead. In that case, object will * shadow is dead. In that case, object will
* be deallocated by the thread that is * be deallocated by the thread that is
* deallocating its shadow. * deallocating its shadow.
*/ */
if ((robject->flags & OBJ_DEAD) == 0 && if ((robject->flags &
(robject->handle == NULL) && (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON &&
(robject->type == OBJT_DEFAULT || robject->handle == NULL) {
robject->type == OBJT_SWAP)) {
refcount_acquire(&robject->ref_count); refcount_acquire(&robject->ref_count);
retry: retry:
if (REFCOUNT_COUNT(robject->paging_in_progress) > 0) { if (REFCOUNT_COUNT(robject->paging_in_progress) > 0) {
VM_OBJECT_WUNLOCK(object); VM_OBJECT_WUNLOCK(object);
vm_object_pip_wait(robject, vm_object_pip_wait(robject,
"objde1"); "objde1");
temp = robject->backing_object; temp = robject->backing_object;
▲ Show 20 Lines • Show All 453 Lines • ▼ Show 20 Lines
static bool static bool
vm_object_advice_applies(vm_object_t object, int advice) vm_object_advice_applies(vm_object_t object, int advice)
{ {
if ((object->flags & OBJ_UNMANAGED) != 0) if ((object->flags & OBJ_UNMANAGED) != 0)
return (false); return (false);
if (advice != MADV_FREE) if (advice != MADV_FREE)
return (true); return (true);
return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) && return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==
(object->flags & OBJ_ONEMAPPING) != 0); (OBJ_ONEMAPPING | OBJ_ANON));
} }
static void static void
vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex, vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
vm_size_t size) vm_size_t size)
{ {
if (advice == MADV_FREE && object->type == OBJT_SWAP) if (advice == MADV_FREE && object->type == OBJT_SWAP)
▲ Show 20 Lines • Show All 144 Lines • ▼ Show 20 Lines
{ {
vm_object_t source; vm_object_t source;
vm_object_t result; vm_object_t result;
source = *object; source = *object;
/* /*
* Don't create the new object if the old object isn't shared. * Don't create the new object if the old object isn't shared.
*
* If we hold the only reference we can guarantee that it won't
* increase while we have the map locked. Otherwise the race is
* harmless and we will end up with an extra shadow object that
* will be collapsed later.
*/ */
if (source != NULL) { if (source != NULL && source->ref_count == 1 &&
VM_OBJECT_RLOCK(source); source->handle == NULL && (source->flags & OBJ_ANON) != 0)
if (source->ref_count == 1 &&
source->handle == NULL &&
(source->type == OBJT_DEFAULT ||
source->type == OBJT_SWAP)) {
VM_OBJECT_RUNLOCK(source);
return; return;
}
VM_OBJECT_RUNLOCK(source);
}
/* /*
* Allocate a new object with the given length. * Allocate a new object with the given length.
*/ */
result = vm_object_allocate(OBJT_DEFAULT, atop(length)); result = vm_object_allocate_anon(atop(length));
/* /*
* The new object shadows the source object, adding a reference to it. * The new object shadows the source object, adding a reference to it.
* Our caller changes his reference to point to the new object, * Our caller changes his reference to point to the new object,
* removing a reference to the source object. Net result: no change * removing a reference to the source object. Net result: no change
* of reference count. * of reference count.
* *
* Try to optimize the result object's page color when shadowing * Try to optimize the result object's page color when shadowing
Show All 38 Lines
vm_object_split(vm_map_entry_t entry) vm_object_split(vm_map_entry_t entry)
{ {
vm_page_t m, m_next; vm_page_t m, m_next;
vm_object_t orig_object, new_object, source; vm_object_t orig_object, new_object, source;
vm_pindex_t idx, offidxstart; vm_pindex_t idx, offidxstart;
vm_size_t size; vm_size_t size;
orig_object = entry->object.vm_object; orig_object = entry->object.vm_object;
if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP) if ((orig_object->flags & OBJ_ANON) == 0)
return; return;
if (orig_object->ref_count <= 1) if (orig_object->ref_count <= 1)
return; return;
VM_OBJECT_WUNLOCK(orig_object); VM_OBJECT_WUNLOCK(orig_object);
offidxstart = OFF_TO_IDX(entry->offset); offidxstart = OFF_TO_IDX(entry->offset);
size = atop(entry->end - entry->start); size = atop(entry->end - entry->start);
/* /*
* If swap_pager_copy() is later called, it will convert new_object * If swap_pager_copy() is later called, it will convert new_object
* into a swap object. * into a swap object.
*/ */
new_object = vm_object_allocate(OBJT_DEFAULT, size); new_object = vm_object_allocate_anon(size);
/* /*
* At this point, the new object is still private, so the order in * At this point, the new object is still private, so the order in
* which the original and new objects are locked does not matter. * which the original and new objects are locked does not matter.
*/ */
VM_OBJECT_WLOCK(new_object); VM_OBJECT_WLOCK(new_object);
VM_OBJECT_WLOCK(orig_object); VM_OBJECT_WLOCK(orig_object);
new_object->domain = orig_object->domain; new_object->domain = orig_object->domain;
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Linesvm_object_scan_all_shadowed(vm_object_t object)
vm_page_t p, pp; vm_page_t p, pp;
vm_pindex_t backing_offset_index, new_pindex, pi, ps; vm_pindex_t backing_offset_index, new_pindex, pi, ps;
VM_OBJECT_ASSERT_WLOCKED(object); VM_OBJECT_ASSERT_WLOCKED(object);
VM_OBJECT_ASSERT_WLOCKED(object->backing_object); VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
backing_object = object->backing_object; backing_object = object->backing_object;
if (backing_object->type != OBJT_DEFAULT && if ((backing_object->flags & OBJ_ANON) == 0)
backing_object->type != OBJT_SWAP)
return (false); return (false);
pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset); pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
p = vm_page_find_least(backing_object, pi); p = vm_page_find_least(backing_object, pi);
ps = swap_pager_find_least(backing_object, pi); ps = swap_pager_find_least(backing_object, pi);
/* /*
* Only check pages inside the parent object's range and * Only check pages inside the parent object's range and
▲ Show 20 Lines • Show All 207 Lines • ▼ Show 20 Lines while (TRUE) {
*/ */
if ((backing_object = object->backing_object) == NULL) if ((backing_object = object->backing_object) == NULL)
break; break;
/* /*
* we check the backing object first, because it is most likely * we check the backing object first, because it is most likely
* not collapsable. * not collapsable.
*/ */
if ((backing_object->flags & OBJ_ANON) == 0)
break;
VM_OBJECT_WLOCK(backing_object); VM_OBJECT_WLOCK(backing_object);
if (backing_object->handle != NULL || if (backing_object->handle != NULL ||
(backing_object->type != OBJT_DEFAULT && (backing_object->flags & OBJ_DEAD) != 0 ||
backing_object->type != OBJT_SWAP) ||
(backing_object->flags & (OBJ_DEAD | OBJ_NOSPLIT)) != 0 ||
object->handle != NULL || object->handle != NULL ||
(object->type != OBJT_DEFAULT && (object->flags & OBJ_DEAD) != 0) {
object->type != OBJT_SWAP) ||
(object->flags & OBJ_DEAD)) {
VM_OBJECT_WUNLOCK(backing_object); VM_OBJECT_WUNLOCK(backing_object);
break; break;
} }
if (REFCOUNT_COUNT(object->paging_in_progress) > 0 || if (REFCOUNT_COUNT(object->paging_in_progress) > 0 ||
REFCOUNT_COUNT(backing_object->paging_in_progress) > 0) { REFCOUNT_COUNT(backing_object->paging_in_progress) > 0) {
vm_object_qcollapse(object); vm_object_qcollapse(object);
VM_OBJECT_WUNLOCK(backing_object); VM_OBJECT_WUNLOCK(backing_object);
▲ Show 20 Lines • Show All 334 Lines • ▼ Show 20 Lines
boolean_t boolean_t
vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset, vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
vm_size_t prev_size, vm_size_t next_size, boolean_t reserved) vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
{ {
vm_pindex_t next_pindex; vm_pindex_t next_pindex;
if (prev_object == NULL) if (prev_object == NULL)
return (TRUE); return (TRUE);
VM_OBJECT_WLOCK(prev_object); if ((prev_object->flags & OBJ_ANON) == 0)
if ((prev_object->type != OBJT_DEFAULT &&
prev_object->type != OBJT_SWAP) ||
(prev_object->flags & OBJ_NOSPLIT) != 0) {
VM_OBJECT_WUNLOCK(prev_object);
return (FALSE); return (FALSE);
}
VM_OBJECT_WLOCK(prev_object);
/* /*
* Try to collapse the object first * Try to collapse the object first
*/ */
vm_object_collapse(prev_object); vm_object_collapse(prev_object);
/* /*
* Can't coalesce if: . more than one reference . paged out . shadows * Can't coalesce if: . more than one reference . paged out . shadows
* another object . has a copy elsewhere (any of which mean that the * another object . has a copy elsewhere (any of which mean that the
▲ Show 20 Lines • Show All 602 Lines • Show Last 20 Lines