Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_map.c
Show First 20 Lines • Show All 501 Lines • ▼ Show 20 Lines | _vm_map_lock(vm_map_t map, const char *file, int line) | ||||
if (map->system_map) | if (map->system_map) | ||||
mtx_lock_flags_(&map->system_mtx, 0, file, line); | mtx_lock_flags_(&map->system_mtx, 0, file, line); | ||||
else | else | ||||
sx_xlock_(&map->lock, file, line); | sx_xlock_(&map->lock, file, line); | ||||
map->timestamp++; | map->timestamp++; | ||||
} | } | ||||
static void | static void | ||||
vm_map_entry_inc_vnode_text(vm_map_entry_t entry, bool add) | |||||
{ | |||||
vm_object_t object, object1; | |||||
struct vnode *vp; | |||||
if ((entry->eflags & MAP_ENTRY_VN_EXEC) == 0) | |||||
return; | |||||
KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, | |||||
("Submap with execs")); | |||||
object = entry->object.vm_object; | |||||
KASSERT(object != NULL, ("No object for text, entry %p", entry)); | |||||
VM_OBJECT_RLOCK(object); | |||||
while ((object1 = object->backing_object) != NULL) { | |||||
VM_OBJECT_RLOCK(object1); | |||||
VM_OBJECT_RUNLOCK(object); | |||||
object = object1; | |||||
} | |||||
vp = (object->flags & OBJ_TMPFS) == 0 ? object->handle : | |||||
object->un_pager.swp.swp_tmpfs; | |||||
if (add) | |||||
VOP_SET_TEXT_SUCCEED(vp); | |||||
else | |||||
VOP_UNSET_TEXT_SUCCEED(vp); | |||||
VM_OBJECT_RUNLOCK(object); | |||||
} | |||||
static void | |||||
vm_map_process_deferred(void) | vm_map_process_deferred(void) | ||||
{ | { | ||||
struct thread *td; | struct thread *td; | ||||
vm_map_entry_t entry, next; | vm_map_entry_t entry, next; | ||||
vm_object_t object; | vm_object_t object; | ||||
td = curthread; | td = curthread; | ||||
entry = td->td_map_def_user; | entry = td->td_map_def_user; | ||||
td->td_map_def_user = NULL; | td->td_map_def_user = NULL; | ||||
while (entry != NULL) { | while (entry != NULL) { | ||||
next = entry->next; | next = entry->next; | ||||
MPASS((entry->eflags & (MAP_ENTRY_VN_WRITECNT | | |||||
MAP_ENTRY_VN_EXEC)) != (MAP_ENTRY_VN_WRITECNT | | |||||
MAP_ENTRY_VN_EXEC)); | |||||
if ((entry->eflags & MAP_ENTRY_VN_WRITECNT) != 0) { | if ((entry->eflags & MAP_ENTRY_VN_WRITECNT) != 0) { | ||||
/* | /* | ||||
* Decrement the object's writemappings and | * Decrement the object's writemappings and | ||||
* possibly the vnode's v_writecount. | * possibly the vnode's v_writecount. | ||||
*/ | */ | ||||
KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, | KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, | ||||
("Submap with writecount")); | ("Submap with writecount")); | ||||
object = entry->object.vm_object; | object = entry->object.vm_object; | ||||
KASSERT(object != NULL, ("No object for writecount")); | KASSERT(object != NULL, ("No object for writecount")); | ||||
vnode_pager_release_writecount(object, entry->start, | vnode_pager_release_writecount(object, entry->start, | ||||
entry->end); | entry->end); | ||||
} | } | ||||
vm_map_entry_inc_vnode_text(entry, false); | |||||
vm_map_entry_deallocate(entry, FALSE); | vm_map_entry_deallocate(entry, FALSE); | ||||
entry = next; | entry = next; | ||||
} | } | ||||
} | } | ||||
void | void | ||||
_vm_map_unlock(vm_map_t map, const char *file, int line) | _vm_map_unlock(vm_map_t map, const char *file, int line) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 815 Lines • ▼ Show 20 Lines | vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | ||||
if (cow & MAP_DISABLE_COREDUMP) | if (cow & MAP_DISABLE_COREDUMP) | ||||
protoeflags |= MAP_ENTRY_NOCOREDUMP; | protoeflags |= MAP_ENTRY_NOCOREDUMP; | ||||
if (cow & MAP_STACK_GROWS_DOWN) | if (cow & MAP_STACK_GROWS_DOWN) | ||||
protoeflags |= MAP_ENTRY_GROWS_DOWN; | protoeflags |= MAP_ENTRY_GROWS_DOWN; | ||||
if (cow & MAP_STACK_GROWS_UP) | if (cow & MAP_STACK_GROWS_UP) | ||||
protoeflags |= MAP_ENTRY_GROWS_UP; | protoeflags |= MAP_ENTRY_GROWS_UP; | ||||
if (cow & MAP_VN_WRITECOUNT) | if (cow & MAP_VN_WRITECOUNT) | ||||
protoeflags |= MAP_ENTRY_VN_WRITECNT; | protoeflags |= MAP_ENTRY_VN_WRITECNT; | ||||
if (cow & MAP_VN_EXEC) | |||||
protoeflags |= MAP_ENTRY_VN_EXEC; | |||||
if ((cow & MAP_CREATE_GUARD) != 0) | if ((cow & MAP_CREATE_GUARD) != 0) | ||||
protoeflags |= MAP_ENTRY_GUARD; | protoeflags |= MAP_ENTRY_GUARD; | ||||
if ((cow & MAP_CREATE_STACK_GAP_DN) != 0) | if ((cow & MAP_CREATE_STACK_GAP_DN) != 0) | ||||
protoeflags |= MAP_ENTRY_STACK_GAP_DN; | protoeflags |= MAP_ENTRY_STACK_GAP_DN; | ||||
if ((cow & MAP_CREATE_STACK_GAP_UP) != 0) | if ((cow & MAP_CREATE_STACK_GAP_UP) != 0) | ||||
protoeflags |= MAP_ENTRY_STACK_GAP_UP; | protoeflags |= MAP_ENTRY_STACK_GAP_UP; | ||||
if (cow & MAP_INHERIT_SHARE) | if (cow & MAP_INHERIT_SHARE) | ||||
inheritance = VM_INHERIT_SHARE; | inheritance = VM_INHERIT_SHARE; | ||||
Show All 27 Lines | if (object != NULL) { | ||||
* aliases with precision.) | * aliases with precision.) | ||||
*/ | */ | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
if (object->ref_count > 1 || object->shadow_count != 0) | if (object->ref_count > 1 || object->shadow_count != 0) | ||||
vm_object_clear_flag(object, OBJ_ONEMAPPING); | vm_object_clear_flag(object, OBJ_ONEMAPPING); | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
} else if ((prev_entry->eflags & ~MAP_ENTRY_USER_WIRED) == | } else if ((prev_entry->eflags & ~MAP_ENTRY_USER_WIRED) == | ||||
protoeflags && | protoeflags && | ||||
(cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 && | (cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP | | ||||
MAP_VN_EXEC)) == 0 && | |||||
prev_entry->end == start && (prev_entry->cred == cred || | prev_entry->end == start && (prev_entry->cred == cred || | ||||
(prev_entry->object.vm_object != NULL && | (prev_entry->object.vm_object != NULL && | ||||
prev_entry->object.vm_object->cred == cred)) && | prev_entry->object.vm_object->cred == cred)) && | ||||
vm_object_coalesce(prev_entry->object.vm_object, | vm_object_coalesce(prev_entry->object.vm_object, | ||||
prev_entry->offset, | prev_entry->offset, | ||||
(vm_size_t)(prev_entry->end - prev_entry->start), | (vm_size_t)(prev_entry->end - prev_entry->start), | ||||
(vm_size_t)(end - prev_entry->end), cred != NULL && | (vm_size_t)(end - prev_entry->end), cred != NULL && | ||||
(protoeflags & MAP_ENTRY_NEEDS_COPY) == 0)) { | (protoeflags & MAP_ENTRY_NEEDS_COPY) == 0)) { | ||||
▲ Show 20 Lines • Show All 505 Lines • ▼ Show 20 Lines | vm_map_find_min(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* A map entry with any of the following flags set must not be merged with | * A map entry with any of the following flags set must not be merged with | ||||
* another entry. | * another entry. | ||||
*/ | */ | ||||
#define MAP_ENTRY_NOMERGE_MASK (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP | \ | #define MAP_ENTRY_NOMERGE_MASK (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP | \ | ||||
MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP) | MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP | MAP_ENTRY_VN_EXEC) | ||||
static bool | static bool | ||||
vm_map_mergeable_neighbors(vm_map_entry_t prev, vm_map_entry_t entry) | vm_map_mergeable_neighbors(vm_map_entry_t prev, vm_map_entry_t entry) | ||||
{ | { | ||||
KASSERT((prev->eflags & MAP_ENTRY_NOMERGE_MASK) == 0 || | KASSERT((prev->eflags & MAP_ENTRY_NOMERGE_MASK) == 0 || | ||||
(entry->eflags & MAP_ENTRY_NOMERGE_MASK) == 0, | (entry->eflags & MAP_ENTRY_NOMERGE_MASK) == 0, | ||||
("vm_map_mergeable_neighbors: neither %p nor %p are mergeable", | ("vm_map_mergeable_neighbors: neither %p nor %p are mergeable", | ||||
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines | _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start) | ||||
entry->start = start; | entry->start = start; | ||||
if (new_entry->cred != NULL) | if (new_entry->cred != NULL) | ||||
crhold(entry->cred); | crhold(entry->cred); | ||||
vm_map_entry_link(map, new_entry); | vm_map_entry_link(map, new_entry); | ||||
if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { | if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { | ||||
vm_object_reference(new_entry->object.vm_object); | vm_object_reference(new_entry->object.vm_object); | ||||
vm_map_entry_inc_vnode_text(new_entry, true); | |||||
/* | /* | ||||
* The object->un_pager.vnp.writemappings for the | * The object->un_pager.vnp.writemappings for the | ||||
* object of MAP_ENTRY_VN_WRITECNT type entry shall be | * object of MAP_ENTRY_VN_WRITECNT type entry shall be | ||||
* kept as is here. The virtual pages are | * kept as is here. The virtual pages are | ||||
* re-distributed among the clipped entries, so the sum is | * re-distributed among the clipped entries, so the sum is | ||||
* left the same. | * left the same. | ||||
*/ | */ | ||||
} | } | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end) | ||||
new_entry->offset += (end - entry->start); | new_entry->offset += (end - entry->start); | ||||
if (new_entry->cred != NULL) | if (new_entry->cred != NULL) | ||||
crhold(entry->cred); | crhold(entry->cred); | ||||
vm_map_entry_link(map, new_entry); | vm_map_entry_link(map, new_entry); | ||||
if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { | if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { | ||||
vm_object_reference(new_entry->object.vm_object); | vm_object_reference(new_entry->object.vm_object); | ||||
vm_map_entry_inc_vnode_text(new_entry, true); | |||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* vm_map_submap: [ kernel use only ] | * vm_map_submap: [ kernel use only ] | ||||
* | * | ||||
* Mark the given range as handled by a subordinate map. | * Mark the given range as handled by a subordinate map. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 1,654 Lines • ▼ Show 20 Lines | case VM_INHERIT_SHARE: | ||||
new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED | | new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED | | ||||
MAP_ENTRY_IN_TRANSITION); | MAP_ENTRY_IN_TRANSITION); | ||||
new_entry->wiring_thread = NULL; | new_entry->wiring_thread = NULL; | ||||
new_entry->wired_count = 0; | new_entry->wired_count = 0; | ||||
if (new_entry->eflags & MAP_ENTRY_VN_WRITECNT) { | if (new_entry->eflags & MAP_ENTRY_VN_WRITECNT) { | ||||
vnode_pager_update_writecount(object, | vnode_pager_update_writecount(object, | ||||
new_entry->start, new_entry->end); | new_entry->start, new_entry->end); | ||||
} | } | ||||
vm_map_entry_inc_vnode_text(new_entry, true); | |||||
/* | /* | ||||
* Insert the entry into the new map -- we know we're | * Insert the entry into the new map -- we know we're | ||||
* inserting at the end of the new map. | * inserting at the end of the new map. | ||||
*/ | */ | ||||
vm_map_entry_link(new_map, new_entry); | vm_map_entry_link(new_map, new_entry); | ||||
vmspace_map_entry_forked(vm1, vm2, new_entry); | vmspace_map_entry_forked(vm1, vm2, new_entry); | ||||
Show All 20 Lines | case VM_INHERIT_COPY: | ||||
new_entry->wiring_thread = NULL; | new_entry->wiring_thread = NULL; | ||||
new_entry->wired_count = 0; | new_entry->wired_count = 0; | ||||
new_entry->object.vm_object = NULL; | new_entry->object.vm_object = NULL; | ||||
new_entry->cred = NULL; | new_entry->cred = NULL; | ||||
vm_map_entry_link(new_map, new_entry); | vm_map_entry_link(new_map, new_entry); | ||||
vmspace_map_entry_forked(vm1, vm2, new_entry); | vmspace_map_entry_forked(vm1, vm2, new_entry); | ||||
vm_map_copy_entry(old_map, new_map, old_entry, | vm_map_copy_entry(old_map, new_map, old_entry, | ||||
new_entry, fork_charge); | new_entry, fork_charge); | ||||
vm_map_entry_inc_vnode_text(new_entry, true); | |||||
break; | break; | ||||
case VM_INHERIT_ZERO: | case VM_INHERIT_ZERO: | ||||
/* | /* | ||||
* Create a new anonymous mapping entry modelled from | * Create a new anonymous mapping entry modelled from | ||||
* the old one. | * the old one. | ||||
*/ | */ | ||||
new_entry = vm_map_entry_create(new_map); | new_entry = vm_map_entry_create(new_map); | ||||
memset(new_entry, 0, sizeof(*new_entry)); | memset(new_entry, 0, sizeof(*new_entry)); | ||||
new_entry->start = old_entry->start; | new_entry->start = old_entry->start; | ||||
new_entry->end = old_entry->end; | new_entry->end = old_entry->end; | ||||
new_entry->eflags = old_entry->eflags & | new_entry->eflags = old_entry->eflags & | ||||
~(MAP_ENTRY_USER_WIRED | MAP_ENTRY_IN_TRANSITION | | ~(MAP_ENTRY_USER_WIRED | MAP_ENTRY_IN_TRANSITION | | ||||
MAP_ENTRY_VN_WRITECNT); | MAP_ENTRY_VN_WRITECNT | MAP_ENTRY_VN_EXEC); | ||||
new_entry->protection = old_entry->protection; | new_entry->protection = old_entry->protection; | ||||
new_entry->max_protection = old_entry->max_protection; | new_entry->max_protection = old_entry->max_protection; | ||||
new_entry->inheritance = VM_INHERIT_ZERO; | new_entry->inheritance = VM_INHERIT_ZERO; | ||||
vm_map_entry_link(new_map, new_entry); | vm_map_entry_link(new_map, new_entry); | ||||
vmspace_map_entry_forked(vm1, vm2, new_entry); | vmspace_map_entry_forked(vm1, vm2, new_entry); | ||||
new_entry->cred = curthread->td_ucred; | new_entry->cred = curthread->td_ucred; | ||||
▲ Show 20 Lines • Show All 848 Lines • Show Last 20 Lines |