Changeset View
Standalone View
sys/vm/vm_map.c
Show First 20 Lines • Show All 1,597 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | ||||
vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max, int cow) | vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max, int cow) | ||||
{ | { | ||||
vm_map_entry_t new_entry, next_entry, prev_entry; | vm_map_entry_t new_entry, next_entry, prev_entry; | ||||
struct ucred *cred; | struct ucred *cred; | ||||
vm_eflags_t protoeflags; | vm_eflags_t protoeflags; | ||||
vm_inherit_t inheritance; | vm_inherit_t inheritance; | ||||
u_long bdry; | |||||
u_int bidx; | |||||
VM_MAP_ASSERT_LOCKED(map); | VM_MAP_ASSERT_LOCKED(map); | ||||
KASSERT(object != kernel_object || | KASSERT(object != kernel_object || | ||||
(cow & MAP_COPY_ON_WRITE) == 0, | (cow & MAP_COPY_ON_WRITE) == 0, | ||||
("vm_map_insert: kernel object and COW")); | ("vm_map_insert: kernel object and COW")); | ||||
KASSERT(object == NULL || (cow & MAP_NOFAULT) == 0, | KASSERT(object == NULL || (cow & MAP_NOFAULT) == 0 || | ||||
("vm_map_insert: paradoxical MAP_NOFAULT request")); | (cow & MAP_SPLIT_BOUNDARY_MASK) != 0, | ||||
("vm_map_insert: paradoxical MAP_NOFAULT request, obj %p cow %#x", | |||||
object, cow)); | |||||
KASSERT((prot & ~max) == 0, | KASSERT((prot & ~max) == 0, | ||||
("prot %#x is not subset of max_prot %#x", prot, max)); | ("prot %#x is not subset of max_prot %#x", prot, max)); | ||||
/* | /* | ||||
* Check that the start and end points are not bogus. | * Check that the start and end points are not bogus. | ||||
*/ | */ | ||||
if (start < vm_map_min(map) || end > vm_map_max(map) || | if (start < vm_map_min(map) || end > vm_map_max(map) || | ||||
start >= end) | start >= end) | ||||
Show All 39 Lines | vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | ||||
if ((cow & MAP_CREATE_STACK_GAP_DN) != 0) | if ((cow & MAP_CREATE_STACK_GAP_DN) != 0) | ||||
protoeflags |= MAP_ENTRY_STACK_GAP_DN; | protoeflags |= MAP_ENTRY_STACK_GAP_DN; | ||||
if ((cow & MAP_CREATE_STACK_GAP_UP) != 0) | if ((cow & MAP_CREATE_STACK_GAP_UP) != 0) | ||||
protoeflags |= MAP_ENTRY_STACK_GAP_UP; | protoeflags |= MAP_ENTRY_STACK_GAP_UP; | ||||
if (cow & MAP_INHERIT_SHARE) | if (cow & MAP_INHERIT_SHARE) | ||||
inheritance = VM_INHERIT_SHARE; | inheritance = VM_INHERIT_SHARE; | ||||
else | else | ||||
inheritance = VM_INHERIT_DEFAULT; | inheritance = VM_INHERIT_DEFAULT; | ||||
if ((cow & MAP_SPLIT_BOUNDARY_MASK) != 0) { | |||||
/* This magically ignores index 0, for usual page size. */ | |||||
bidx = (cow & MAP_SPLIT_BOUNDARY_MASK) >> | |||||
MAP_SPLIT_BOUNDARY_SHIFT; | |||||
if (bidx >= MAXPAGESIZES) | |||||
return (KERN_INVALID_ARGUMENT); | |||||
bdry = pagesizes[bidx] - 1; | |||||
if ((start & bdry) != 0 || (end & bdry) != 0) | |||||
return (KERN_INVALID_ARGUMENT); | |||||
protoeflags |= bidx << MAP_ENTRY_SPLIT_BOUNDARY_SHIFT; | |||||
} | |||||
cred = NULL; | cred = NULL; | ||||
if ((cow & (MAP_ACC_NO_CHARGE | MAP_NOFAULT | MAP_CREATE_GUARD)) != 0) | if ((cow & (MAP_ACC_NO_CHARGE | MAP_NOFAULT | MAP_CREATE_GUARD)) != 0) | ||||
goto charged; | goto charged; | ||||
if ((cow & MAP_ACC_CHARGED) || ((prot & VM_PROT_WRITE) && | if ((cow & MAP_ACC_CHARGED) || ((prot & VM_PROT_WRITE) && | ||||
((protoeflags & MAP_ENTRY_NEEDS_COPY) || object == NULL))) { | ((protoeflags & MAP_ENTRY_NEEDS_COPY) || object == NULL))) { | ||||
if (!(cow & MAP_ACC_CHARGED) && !swap_reserve(end - start)) | if (!(cow & MAP_ACC_CHARGED) && !swap_reserve(end - start)) | ||||
return (KERN_RESOURCE_SHORTAGE); | return (KERN_RESOURCE_SHORTAGE); | ||||
▲ Show 20 Lines • Show All 238 Lines • ▼ Show 20 Lines | vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | ||||
end = start + length; | end = start + length; | ||||
KASSERT((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 || | KASSERT((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 || | ||||
object == NULL, | object == NULL, | ||||
("vm_map_fixed: non-NULL backing object for stack")); | ("vm_map_fixed: non-NULL backing object for stack")); | ||||
vm_map_lock(map); | vm_map_lock(map); | ||||
VM_MAP_RANGE_CHECK(map, start, end); | VM_MAP_RANGE_CHECK(map, start, end); | ||||
if ((cow & MAP_CHECK_EXCL) == 0) | if ((cow & MAP_CHECK_EXCL) == 0) | ||||
vm_map_delete(map, start, end); | vm_map_delete(map, start, end); | ||||
markj: We are not checking the return value. vm_map_insert() will return an error if the space is… | |||||
Done Inline ActionsI did the pass over all vm_map_delete()s. kib: I did the pass over all vm_map_delete()s. | |||||
if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) != 0) { | if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) != 0) { | ||||
result = vm_map_stack_locked(map, start, length, sgrowsiz, | result = vm_map_stack_locked(map, start, length, sgrowsiz, | ||||
prot, max, cow); | prot, max, cow); | ||||
} else { | } else { | ||||
result = vm_map_insert(map, object, offset, start, end, | result = vm_map_insert(map, object, offset, start, end, | ||||
prot, max, cow); | prot, max, cow); | ||||
} | } | ||||
vm_map_unlock(map); | vm_map_unlock(map); | ||||
Show All 23 Lines | clustering_anon_allowed(vm_offset_t addr) | ||||
} | } | ||||
} | } | ||||
static long aslr_restarts; | static long aslr_restarts; | ||||
SYSCTL_LONG(_vm, OID_AUTO, aslr_restarts, CTLFLAG_RD, | SYSCTL_LONG(_vm, OID_AUTO, aslr_restarts, CTLFLAG_RD, | ||||
&aslr_restarts, 0, | &aslr_restarts, 0, | ||||
"Number of aslr failures"); | "Number of aslr failures"); | ||||
#define MAP_32BIT_MAX_ADDR ((vm_offset_t)1 << 31) | |||||
/* | /* | ||||
* Searches for the specified amount of free space in the given map with the | * Searches for the specified amount of free space in the given map with the | ||||
* specified alignment. Performs an address-ordered, first-fit search from | * specified alignment. Performs an address-ordered, first-fit search from | ||||
* the given address "*addr", with an optional upper bound "max_addr". If the | * the given address "*addr", with an optional upper bound "max_addr". If the | ||||
* parameter "alignment" is zero, then the alignment is computed from the | * parameter "alignment" is zero, then the alignment is computed from the | ||||
* given (object, offset) pair so as to enable the greatest possible use of | * given (object, offset) pair so as to enable the greatest possible use of | ||||
* superpage mappings. Returns KERN_SUCCESS and the address of the free space | * superpage mappings. Returns KERN_SUCCESS and the address of the free space | ||||
* in "*addr" if successful. Otherwise, returns KERN_NO_SPACE. | * in "*addr" if successful. Otherwise, returns KERN_NO_SPACE. | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | if (free_addr == aligned_addr) { | ||||
* change "*addr", then "*addr" must still be aligned | * change "*addr", then "*addr" must still be aligned | ||||
* and provide sufficient free space. | * and provide sufficient free space. | ||||
*/ | */ | ||||
return (KERN_SUCCESS); | return (KERN_SUCCESS); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
int | |||||
vm_map_find_aligned(vm_map_t map, vm_offset_t *addr, vm_size_t length, | |||||
vm_offset_t max_addr, vm_offset_t alignment) | |||||
{ | |||||
/* XXXKIB ASLR eh ? */ | |||||
*addr = vm_map_findspace(map, *addr, length); | |||||
if (*addr + length > vm_map_max(map) || | |||||
(max_addr != 0 && *addr + length > max_addr)) | |||||
return (KERN_NO_SPACE); | |||||
return (vm_map_alignspace(map, NULL, 0, addr, length, max_addr, | |||||
alignment)); | |||||
} | |||||
/* | /* | ||||
* vm_map_find finds an unallocated region in the target address | * vm_map_find finds an unallocated region in the target address | ||||
* map with the given length. The search is defined to be | * map with the given length. The search is defined to be | ||||
* first-fit from the specified address; the region found is | * first-fit from the specified address; the region found is | ||||
* returned in the same parameter. | * returned in the same parameter. | ||||
* | * | ||||
* If object is non-NULL, ref count must be bumped by caller | * If object is non-NULL, ref count must be bumped by caller | ||||
* prior to making call to account for the new entry. | * prior to making call to account for the new entry. | ||||
▲ Show 20 Lines • Show All 327 Lines • ▼ Show 20 Lines | if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { | ||||
* virtual pages are re-distributed among the clipped entries, | * virtual pages are re-distributed among the clipped entries, | ||||
* so the sum is left the same. | * so the sum is left the same. | ||||
*/ | */ | ||||
} | } | ||||
return (new_entry); | return (new_entry); | ||||
} | } | ||||
/* | /* | ||||
* vm_map_clip_start: [ internal use only ] | |||||
* | |||||
* Asserts that the given entry begins at or after | |||||
* the specified address; if necessary, | |||||
* it splits the entry into two. | |||||
*/ | |||||
#define vm_map_clip_start(map, entry, startaddr) \ | |||||
{ \ | |||||
if (startaddr > entry->start) \ | |||||
_vm_map_clip_start(map, entry, startaddr); \ | |||||
} | |||||
/* | |||||
* This routine is called only when it is known that | * This routine is called only when it is known that | ||||
* the entry must be split. | * the entry must be split. | ||||
*/ | */ | ||||
static inline void | static inline void | ||||
_vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start) | _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start) | ||||
{ | { | ||||
vm_map_entry_t new_entry; | vm_map_entry_t new_entry; | ||||
VM_MAP_ASSERT_LOCKED(map); | VM_MAP_ASSERT_LOCKED(map); | ||||
KASSERT(entry->end > start && entry->start < start, | KASSERT(entry->end > start && entry->start < start, | ||||
("_vm_map_clip_start: invalid clip of entry %p", entry)); | ("_vm_map_clip_start: invalid clip of entry %p", entry)); | ||||
new_entry = vm_map_entry_clone(map, entry); | new_entry = vm_map_entry_clone(map, entry); | ||||
/* | /* | ||||
* Split off the front portion. Insert the new entry BEFORE this one, | * Split off the front portion. Insert the new entry BEFORE this one, | ||||
* so that this entry has the specified starting address. | * so that this entry has the specified starting address. | ||||
*/ | */ | ||||
new_entry->end = start; | new_entry->end = start; | ||||
vm_map_entry_link(map, new_entry); | vm_map_entry_link(map, new_entry); | ||||
} | } | ||||
/* | /* | ||||
* vm_map_clip_start: [ internal use only ] | |||||
* | |||||
* Asserts that the given entry begins at or after | |||||
* the specified address; if necessary, | |||||
* it splits the entry into two. | |||||
*/ | |||||
static inline int | |||||
vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t startaddr) | |||||
{ | |||||
int bdry_idx; | |||||
if (startaddr <= entry->start) | |||||
return (KERN_SUCCESS); | |||||
bdry_idx = (entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) >> | |||||
MAP_ENTRY_SPLIT_BOUNDARY_SHIFT; | |||||
if (bdry_idx != 0) { | |||||
if ((startaddr & (pagesizes[bdry_idx] - 1)) != 0) | |||||
return (KERN_INVALID_ARGUMENT); | |||||
} | |||||
_vm_map_clip_start(map, entry, startaddr); | |||||
return (KERN_SUCCESS); | |||||
} | |||||
/* | |||||
* vm_map_lookup_clip_start: | * vm_map_lookup_clip_start: | ||||
* | * | ||||
* Find the entry at or just after 'start', and clip it if 'start' is in | * Find the entry at or just after 'start', and clip it if 'start' is in | ||||
* the interior of the entry. Return entry after 'start', and in | * the interior of the entry. Return entry after 'start', and in | ||||
* prev_entry set the entry before 'start'. | * prev_entry set the entry before 'start'. | ||||
*/ | */ | ||||
static inline vm_map_entry_t | static inline int | ||||
vm_map_lookup_clip_start(vm_map_t map, vm_offset_t start, | vm_map_lookup_clip_start(vm_map_t map, vm_offset_t start, | ||||
vm_map_entry_t *prev_entry) | vm_map_entry_t *res_entry, vm_map_entry_t *prev_entry) | ||||
{ | { | ||||
vm_map_entry_t entry; | vm_map_entry_t entry; | ||||
int rv; | |||||
if (vm_map_lookup_entry(map, start, prev_entry)) { | if (vm_map_lookup_entry(map, start, prev_entry)) { | ||||
entry = *prev_entry; | entry = *prev_entry; | ||||
vm_map_clip_start(map, entry, start); | rv = vm_map_clip_start(map, entry, start); | ||||
if (rv != KERN_SUCCESS) | |||||
return (rv); | |||||
*prev_entry = vm_map_entry_pred(entry); | *prev_entry = vm_map_entry_pred(entry); | ||||
} else | } else | ||||
entry = vm_map_entry_succ(*prev_entry); | entry = vm_map_entry_succ(*prev_entry); | ||||
return (entry); | *res_entry = entry; | ||||
return (KERN_SUCCESS); | |||||
} | } | ||||
/* | /* | ||||
* vm_map_clip_end: [ internal use only ] | |||||
* | |||||
* Asserts that the given entry ends at or before | |||||
* the specified address; if necessary, | |||||
* it splits the entry into two. | |||||
*/ | |||||
#define vm_map_clip_end(map, entry, endaddr) \ | |||||
{ \ | |||||
if ((endaddr) < (entry->end)) \ | |||||
_vm_map_clip_end((map), (entry), (endaddr)); \ | |||||
} | |||||
/* | |||||
* This routine is called only when it is known that | * This routine is called only when it is known that | ||||
* the entry must be split. | * the entry must be split. | ||||
*/ | */ | ||||
static inline void | static inline void | ||||
_vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end) | _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end) | ||||
{ | { | ||||
vm_map_entry_t new_entry; | vm_map_entry_t new_entry; | ||||
VM_MAP_ASSERT_LOCKED(map); | VM_MAP_ASSERT_LOCKED(map); | ||||
KASSERT(entry->start < end && entry->end > end, | KASSERT(entry->start < end && entry->end > end, | ||||
("_vm_map_clip_end: invalid clip of entry %p", entry)); | ("_vm_map_clip_end: invalid clip of entry %p", entry)); | ||||
new_entry = vm_map_entry_clone(map, entry); | new_entry = vm_map_entry_clone(map, entry); | ||||
/* | /* | ||||
* Split off the back portion. Insert the new entry AFTER this one, | * Split off the back portion. Insert the new entry AFTER this one, | ||||
* so that this entry has the specified ending address. | * so that this entry has the specified ending address. | ||||
*/ | */ | ||||
new_entry->start = end; | new_entry->start = end; | ||||
vm_map_entry_link(map, new_entry); | vm_map_entry_link(map, new_entry); | ||||
} | } | ||||
/* | /* | ||||
* vm_map_clip_end: [ internal use only ] | |||||
* | |||||
* Asserts that the given entry ends at or before | |||||
* the specified address; if necessary, | |||||
* it splits the entry into two. | |||||
*/ | |||||
static inline int | |||||
vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t endaddr) | |||||
{ | |||||
int bdry_idx; | |||||
if (endaddr >= entry->end) | |||||
return (KERN_SUCCESS); | |||||
bdry_idx = (entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) >> | |||||
MAP_ENTRY_SPLIT_BOUNDARY_SHIFT; | |||||
if (bdry_idx != 0) { | |||||
if ((endaddr & (pagesizes[bdry_idx] - 1)) != 0) | |||||
return (KERN_INVALID_ARGUMENT); | |||||
} | |||||
_vm_map_clip_end(map, entry, endaddr); | |||||
return (KERN_SUCCESS); | |||||
} | |||||
/* | |||||
* vm_map_submap: [ kernel use only ] | * vm_map_submap: [ kernel use only ] | ||||
* | * | ||||
* Mark the given range as handled by a subordinate map. | * Mark the given range as handled by a subordinate map. | ||||
* | * | ||||
* This range must have been created with vm_map_find, | * This range must have been created with vm_map_find, | ||||
* and no other operations may have been performed on this | * and no other operations may have been performed on this | ||||
* range prior to calling vm_map_submap. | * range prior to calling vm_map_submap. | ||||
* | * | ||||
Show All 22 Lines | vm_map_submap( | ||||
submap->flags |= MAP_IS_SUB_MAP; | submap->flags |= MAP_IS_SUB_MAP; | ||||
vm_map_unlock(submap); | vm_map_unlock(submap); | ||||
vm_map_lock(map); | vm_map_lock(map); | ||||
VM_MAP_RANGE_CHECK(map, start, end); | VM_MAP_RANGE_CHECK(map, start, end); | ||||
if (vm_map_lookup_entry(map, start, &entry) && entry->end >= end && | if (vm_map_lookup_entry(map, start, &entry) && entry->end >= end && | ||||
(entry->eflags & MAP_ENTRY_COW) == 0 && | (entry->eflags & MAP_ENTRY_COW) == 0 && | ||||
entry->object.vm_object == NULL) { | entry->object.vm_object == NULL) { | ||||
vm_map_clip_start(map, entry, start); | result = vm_map_clip_start(map, entry, start); | ||||
vm_map_clip_end(map, entry, end); | if (result != KERN_SUCCESS) | ||||
goto unlock; | |||||
result = vm_map_clip_end(map, entry, end); | |||||
if (result != KERN_SUCCESS) | |||||
goto unlock; | |||||
entry->object.sub_map = submap; | entry->object.sub_map = submap; | ||||
entry->eflags |= MAP_ENTRY_IS_SUB_MAP; | entry->eflags |= MAP_ENTRY_IS_SUB_MAP; | ||||
result = KERN_SUCCESS; | result = KERN_SUCCESS; | ||||
} | } | ||||
unlock: | |||||
vm_map_unlock(map); | vm_map_unlock(map); | ||||
if (result != KERN_SUCCESS) { | if (result != KERN_SUCCESS) { | ||||
vm_map_lock(submap); | vm_map_lock(submap); | ||||
submap->flags &= ~MAP_IS_SUB_MAP; | submap->flags &= ~MAP_IS_SUB_MAP; | ||||
vm_map_unlock(submap); | vm_map_unlock(submap); | ||||
} | } | ||||
return (result); | return (result); | ||||
▲ Show 20 Lines • Show All 170 Lines • ▼ Show 20 Lines | again: | ||||
/* | /* | ||||
* Before changing the protections, try to reserve swap space for any | * Before changing the protections, try to reserve swap space for any | ||||
* private (i.e., copy-on-write) mappings that are transitioning from | * private (i.e., copy-on-write) mappings that are transitioning from | ||||
* read-only to read/write access. If a reservation fails, break out | * read-only to read/write access. If a reservation fails, break out | ||||
* of this loop early and let the next loop simplify the entries, since | * of this loop early and let the next loop simplify the entries, since | ||||
* some may now be mergeable. | * some may now be mergeable. | ||||
*/ | */ | ||||
rv = KERN_SUCCESS; | rv = vm_map_clip_start(map, first_entry, start); | ||||
vm_map_clip_start(map, first_entry, start); | if (rv != KERN_SUCCESS) { | ||||
vm_map_unlock(map); | |||||
return (rv); | |||||
} | |||||
for (entry = first_entry; entry->start < end; | for (entry = first_entry; entry->start < end; | ||||
entry = vm_map_entry_succ(entry)) { | entry = vm_map_entry_succ(entry)) { | ||||
vm_map_clip_end(map, entry, end); | rv = vm_map_clip_end(map, entry, end); | ||||
if (rv != KERN_SUCCESS) { | |||||
vm_map_unlock(map); | |||||
return (rv); | |||||
} | |||||
if (set_max || | if (set_max || | ||||
((new_prot & ~entry->protection) & VM_PROT_WRITE) == 0 || | ((new_prot & ~entry->protection) & VM_PROT_WRITE) == 0 || | ||||
ENTRY_CHARGED(entry) || | ENTRY_CHARGED(entry) || | ||||
(entry->eflags & MAP_ENTRY_GUARD) != 0) { | (entry->eflags & MAP_ENTRY_GUARD) != 0) { | ||||
continue; | continue; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
vm_map_madvise( | vm_map_madvise( | ||||
vm_map_t map, | vm_map_t map, | ||||
vm_offset_t start, | vm_offset_t start, | ||||
vm_offset_t end, | vm_offset_t end, | ||||
int behav) | int behav) | ||||
{ | { | ||||
vm_map_entry_t entry, prev_entry; | vm_map_entry_t entry, prev_entry; | ||||
int rv; | |||||
bool modify_map; | bool modify_map; | ||||
/* | /* | ||||
* Some madvise calls directly modify the vm_map_entry, in which case | * Some madvise calls directly modify the vm_map_entry, in which case | ||||
* we need to use an exclusive lock on the map and we need to perform | * we need to use an exclusive lock on the map and we need to perform | ||||
* various clipping operations. Otherwise we only need a read-lock | * various clipping operations. Otherwise we only need a read-lock | ||||
* on the map. | * on the map. | ||||
*/ | */ | ||||
Show All 29 Lines | vm_map_madvise( | ||||
if (modify_map) { | if (modify_map) { | ||||
/* | /* | ||||
* madvise behaviors that are implemented in the vm_map_entry. | * madvise behaviors that are implemented in the vm_map_entry. | ||||
* | * | ||||
* We clip the vm_map_entry so that behavioral changes are | * We clip the vm_map_entry so that behavioral changes are | ||||
* limited to the specified address range. | * limited to the specified address range. | ||||
*/ | */ | ||||
for (entry = vm_map_lookup_clip_start(map, start, &prev_entry); | rv = vm_map_lookup_clip_start(map, start, &entry, &prev_entry); | ||||
entry->start < end; | if (rv != KERN_SUCCESS) { | ||||
prev_entry = entry, entry = vm_map_entry_succ(entry)) { | vm_map_unlock(map); | ||||
return (vm_mmap_to_errno(rv)); | |||||
} | |||||
for (; entry->start < end; prev_entry = entry, | |||||
entry = vm_map_entry_succ(entry)) { | |||||
if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) | if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) | ||||
continue; | continue; | ||||
vm_map_clip_end(map, entry, end); | rv = vm_map_clip_end(map, entry, end); | ||||
if (rv != KERN_SUCCESS) { | |||||
vm_map_unlock(map); | |||||
return (vm_mmap_to_errno(rv)); | |||||
} | |||||
switch (behav) { | switch (behav) { | ||||
case MADV_NORMAL: | case MADV_NORMAL: | ||||
vm_map_entry_set_behavior(entry, | vm_map_entry_set_behavior(entry, | ||||
MAP_ENTRY_BEHAV_NORMAL); | MAP_ENTRY_BEHAV_NORMAL); | ||||
break; | break; | ||||
case MADV_SEQUENTIAL: | case MADV_SEQUENTIAL: | ||||
vm_map_entry_set_behavior(entry, | vm_map_entry_set_behavior(entry, | ||||
▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Lines | |||||
* affects how the map will be shared with | * affects how the map will be shared with | ||||
* child maps at the time of vmspace_fork. | * child maps at the time of vmspace_fork. | ||||
*/ | */ | ||||
int | int | ||||
vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, | vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, | ||||
vm_inherit_t new_inheritance) | vm_inherit_t new_inheritance) | ||||
{ | { | ||||
vm_map_entry_t entry, prev_entry; | vm_map_entry_t entry, prev_entry; | ||||
int rv; | |||||
switch (new_inheritance) { | switch (new_inheritance) { | ||||
case VM_INHERIT_NONE: | case VM_INHERIT_NONE: | ||||
case VM_INHERIT_COPY: | case VM_INHERIT_COPY: | ||||
case VM_INHERIT_SHARE: | case VM_INHERIT_SHARE: | ||||
case VM_INHERIT_ZERO: | case VM_INHERIT_ZERO: | ||||
break; | break; | ||||
default: | default: | ||||
return (KERN_INVALID_ARGUMENT); | return (KERN_INVALID_ARGUMENT); | ||||
} | } | ||||
rv = KERN_SUCCESS; | |||||
if (start == end) | if (start == end) | ||||
return (KERN_SUCCESS); | return (rv); | ||||
vm_map_lock(map); | vm_map_lock(map); | ||||
Done Inline ActionsI don't see the point of assigning rv here, it is assigned again a few lines below. markj: I don't see the point of assigning `rv` here, it is assigned again a few lines below. | |||||
VM_MAP_RANGE_CHECK(map, start, end); | VM_MAP_RANGE_CHECK(map, start, end); | ||||
for (entry = vm_map_lookup_clip_start(map, start, &prev_entry); | rv = vm_map_lookup_clip_start(map, start, &entry, &prev_entry); | ||||
entry->start < end; | if (rv != KERN_SUCCESS) | ||||
prev_entry = entry, entry = vm_map_entry_succ(entry)) { | goto unlock; | ||||
vm_map_clip_end(map, entry, end); | for (; entry->start < end; prev_entry = entry, | ||||
entry = vm_map_entry_succ(entry)) { | |||||
rv = vm_map_clip_end(map, entry, end); | |||||
Done Inline ActionsI think it should look up the entry containing end - 1. Otherwise the vm_map_clip_end() call might try to clip lentry to a length of 0. markj: I think it should look up the entry containing `end - 1`. Otherwise the vm_map_clip_end() call… | |||||
Done Inline ActionsI meant that it should be vm_map_lookup_entry(map, end - 1, &lentry). markj: I meant that it should be `vm_map_lookup_entry(map, end - 1, &lentry)`. | |||||
if (rv != KERN_SUCCESS) | |||||
goto unlock; | |||||
Done Inline ActionsSo now we can fail part-way through an minherit() call and leave the range in an inconsistent state. It looks like we should do clipping first, in a separate loop. markj: So now we can fail part-way through an minherit() call and leave the range in an inconsistent… | |||||
if ((entry->eflags & MAP_ENTRY_GUARD) == 0 || | if ((entry->eflags & MAP_ENTRY_GUARD) == 0 || | ||||
Done Inline ActionsWhy not allow VM_INHERIT_NONE? markj: Why not allow VM_INHERIT_NONE? | |||||
Done Inline ActionsI believe VM_INHERIT_ZERO should also be prohibited. markj: I believe VM_INHERIT_ZERO should also be prohibited. | |||||
Done Inline ActionsCan you explain why ? I clear MAP_ENTRY_SPLIT_BOUNDARY_MASK on fork for INHERIT_ZERO. kib: Can you explain why ?
I clear MAP_ENTRY_SPLIT_BOUNDARY_MASK on fork for INHERIT_ZERO. | |||||
Done Inline ActionsIt is just kind of weird to me - we are mapping and using a shm object because its mappings have some special properties, and after fork() it is replaced by a plain anonymous mapping. I guess INHERIT_ZERO is permitted for vnodes as well, so there is some precedent for this. I'll just modify my test for this case. markj: It is just kind of weird to me - we are mapping and using a shm object because its mappings… | |||||
Done Inline ActionsBut this *is* the semantic of VM_INHERIT_ZERO. kib: But this *is* the semantic of VM_INHERIT_ZERO. | |||||
Done Inline ActionsRight, and I think the semantic makes no sense for shared mappings. We already have INHERIT_NONE for them. markj: Right, and I think the semantic makes no sense for shared mappings. We already have… | |||||
new_inheritance != VM_INHERIT_ZERO) | new_inheritance != VM_INHERIT_ZERO) | ||||
entry->inheritance = new_inheritance; | entry->inheritance = new_inheritance; | ||||
vm_map_try_merge_entries(map, prev_entry, entry); | vm_map_try_merge_entries(map, prev_entry, entry); | ||||
} | } | ||||
vm_map_try_merge_entries(map, prev_entry, entry); | vm_map_try_merge_entries(map, prev_entry, entry); | ||||
vm_map_unlock(map); | vm_map_unlock(map); | ||||
return (KERN_SUCCESS); | unlock: | ||||
Done Inline ActionsI think we need to prohibit VM_INHERIT_COPY and perhaps _ZERO for large pages. Currently vmspace_fork() copies the boundary eflags from the old entry. With _COPY inheritance the page fault handler may busy individual 4KB pages in the large page, but because we avoid busying all 4KB pages in a large page there is no synchronization between them. markj: I think we need to prohibit VM_INHERIT_COPY and perhaps _ZERO for large pages. Currently… | |||||
Done Inline ActionsThen MAP_PRIVATE should be disabled as well. kib: Then MAP_PRIVATE should be disabled as well. | |||||
Done Inline ActionsYes, I think so. There might be a useful semantic for MAP_PRIVATE. Suppose I want to reserve a pool of large pages at boot, before physical memory is fragmented. I can create a set of named shm objects for that purpose, but I can't see a way for applications to atomically "claim" a particular object. Some malicious application could open all of them and map them, so there is no way to ensure that a legitimate consumer gets a private set of large pages from the pool. It would be good to provide some interface that guarantees that a largepage object is not mapped by any other process. I haven't thought a lot about it so far so I don't want to make any suggestions yet. Maybe I am missing something. markj: Yes, I think so.
There might be a useful semantic for MAP_PRIVATE. Suppose I want to reserve a… | |||||
return (rv); | |||||
Done Inline ActionsThe label should be one line before. kib: The label should be one line before. | |||||
} | } | ||||
/* | /* | ||||
* vm_map_entry_in_transition: | * vm_map_entry_in_transition: | ||||
* | * | ||||
* Release the map lock, and sleep until the entry is no longer in | * Release the map lock, and sleep until the entry is no longer in | ||||
* transition. Awake and acquire the map lock. If the map changed while | * transition. Awake and acquire the map lock. If the map changed while | ||||
* another held the lock, lookup a possibly-changed entry at or after the | * another held the lock, lookup a possibly-changed entry at or after the | ||||
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines | if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { | ||||
} | } | ||||
rv = KERN_INVALID_ADDRESS; | rv = KERN_INVALID_ADDRESS; | ||||
break; | break; | ||||
} | } | ||||
first_entry = (entry == first_entry) ? | first_entry = (entry == first_entry) ? | ||||
next_entry : NULL; | next_entry : NULL; | ||||
continue; | continue; | ||||
} | } | ||||
vm_map_clip_start(map, entry, start); | rv = vm_map_clip_start(map, entry, start); | ||||
vm_map_clip_end(map, entry, end); | if (rv != KERN_SUCCESS) | ||||
break; | |||||
rv = vm_map_clip_end(map, entry, end); | |||||
if (rv != KERN_SUCCESS) | |||||
break; | |||||
/* | /* | ||||
* Mark the entry in case the map lock is released. (See | * Mark the entry in case the map lock is released. (See | ||||
* above.) | * above.) | ||||
*/ | */ | ||||
KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 && | KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 && | ||||
entry->wiring_thread == NULL, | entry->wiring_thread == NULL, | ||||
("owned map entry %p", entry)); | ("owned map entry %p", entry)); | ||||
entry->eflags |= MAP_ENTRY_IN_TRANSITION; | entry->eflags |= MAP_ENTRY_IN_TRANSITION; | ||||
▲ Show 20 Lines • Show All 190 Lines • ▼ Show 20 Lines | if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { | ||||
return (KERN_INVALID_ADDRESS); | return (KERN_INVALID_ADDRESS); | ||||
rv = KERN_INVALID_ADDRESS; | rv = KERN_INVALID_ADDRESS; | ||||
goto done; | goto done; | ||||
} | } | ||||
first_entry = (entry == first_entry) ? | first_entry = (entry == first_entry) ? | ||||
next_entry : NULL; | next_entry : NULL; | ||||
continue; | continue; | ||||
} | } | ||||
vm_map_clip_start(map, entry, start); | rv = vm_map_clip_start(map, entry, start); | ||||
vm_map_clip_end(map, entry, end); | if (rv != KERN_SUCCESS) | ||||
break; | |||||
rv = vm_map_clip_end(map, entry, end); | |||||
if (rv != KERN_SUCCESS) | |||||
break; | |||||
Done Inline ActionsShould be goto done. markj: Should be `goto done`. | |||||
/* | /* | ||||
* Mark the entry in case the map lock is released. (See | * Mark the entry in case the map lock is released. (See | ||||
* above.) | * above.) | ||||
*/ | */ | ||||
KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 && | KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 && | ||||
entry->wiring_thread == NULL, | entry->wiring_thread == NULL, | ||||
("owned map entry %p", entry)); | ("owned map entry %p", entry)); | ||||
entry->eflags |= MAP_ENTRY_IN_TRANSITION; | entry->eflags |= MAP_ENTRY_IN_TRANSITION; | ||||
▲ Show 20 Lines • Show All 109 Lines • ▼ Show 20 Lines | for (; entry->start < end; | ||||
* If holes_ok was specified, an empty | * If holes_ok was specified, an empty | ||||
* space in the unwired region could have been mapped | * space in the unwired region could have been mapped | ||||
* while the map lock was dropped for faulting in the | * while the map lock was dropped for faulting in the | ||||
* pages or draining MAP_ENTRY_IN_TRANSITION. | * pages or draining MAP_ENTRY_IN_TRANSITION. | ||||
* Moreover, another thread could be simultaneously | * Moreover, another thread could be simultaneously | ||||
* wiring this new mapping entry. Detect these cases | * wiring this new mapping entry. Detect these cases | ||||
* and skip any entries marked as in transition not by us. | * and skip any entries marked as in transition not by us. | ||||
*/ | */ | ||||
if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || | if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || | ||||
Done Inline ActionsNow vm_map_wire() can fail during clipping, which happens before MAP_ENTRY_IN_TRANSITION is set. I suspect that the assertion condition below should be holes_ok || rv != KERN_SUCCESS or even holes_ok || rv == KERN_INVALID_ARGUMENT. markj: Now vm_map_wire() can fail during clipping, which happens before MAP_ENTRY_IN_TRANSITION is set. | |||||
entry->wiring_thread != curthread) { | entry->wiring_thread != curthread) { | ||||
KASSERT(holes_ok, | KASSERT(holes_ok, | ||||
("vm_map_wire: !HOLESOK and new/changed entry")); | ("vm_map_wire: !HOLESOK and new/changed entry")); | ||||
continue; | continue; | ||||
} | } | ||||
if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0) { | if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0) { | ||||
/* do nothing */ | /* do nothing */ | ||||
▲ Show 20 Lines • Show All 92 Lines • ▼ Show 20 Lines | for (entry = first_entry; entry->start < end; entry = next_entry) { | ||||
if (end > entry->end && | if (end > entry->end && | ||||
entry->end != next_entry->start) { | entry->end != next_entry->start) { | ||||
vm_map_unlock_read(map); | vm_map_unlock_read(map); | ||||
return (KERN_INVALID_ADDRESS); | return (KERN_INVALID_ADDRESS); | ||||
} | } | ||||
} | } | ||||
if (invalidate) | if (invalidate) | ||||
pmap_remove(map->pmap, start, end); | pmap_remove(map->pmap, start, end); | ||||
Done Inline ActionsWe need to make sure that a largepage mapping doesn't get partially removed. markj: We need to make sure that a largepage mapping doesn't get partially removed. | |||||
failed = FALSE; | failed = FALSE; | ||||
/* | /* | ||||
* Make a second pass, cleaning/uncaching pages from the indicated | * Make a second pass, cleaning/uncaching pages from the indicated | ||||
* objects as we go. | * objects as we go. | ||||
*/ | */ | ||||
for (entry = first_entry; entry->start < end;) { | for (entry = first_entry; entry->start < end;) { | ||||
offset = entry->offset + (start - entry->start); | offset = entry->offset + (start - entry->start); | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) | ||||
KASSERT(entry->wired_count > 0, | KASSERT(entry->wired_count > 0, | ||||
("vm_map_entry_unwire: entry %p isn't wired", entry)); | ("vm_map_entry_unwire: entry %p isn't wired", entry)); | ||||
size = entry->end - entry->start; | size = entry->end - entry->start; | ||||
if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0) | if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0) | ||||
vm_map_wire_user_count_sub(atop(size)); | vm_map_wire_user_count_sub(atop(size)); | ||||
pmap_unwire(map->pmap, entry->start, entry->end); | pmap_unwire(map->pmap, entry->start, entry->end); | ||||
vm_object_unwire(entry->object.vm_object, entry->offset, size, | vm_object_unwire(entry->object.vm_object, entry->offset, size, | ||||
PQ_ACTIVE); | PQ_ACTIVE); | ||||
Done Inline Actionss/SHIFT/MASK/ markj: s/SHIFT/MASK/ | |||||
entry->wired_count = 0; | entry->wired_count = 0; | ||||
} | } | ||||
static void | static void | ||||
vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map) | vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map) | ||||
{ | { | ||||
if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) | if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) | ||||
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines | |||||
* vm_map_delete: [ internal use only ] | * vm_map_delete: [ internal use only ] | ||||
* | * | ||||
* Deallocates the given address range from the target | * Deallocates the given address range from the target | ||||
* map. | * map. | ||||
*/ | */ | ||||
int | int | ||||
vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) | vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) | ||||
{ | { | ||||
vm_map_entry_t entry, next_entry; | vm_map_entry_t entry, next_entry, scratch_entry; | ||||
int rv; | |||||
VM_MAP_ASSERT_LOCKED(map); | VM_MAP_ASSERT_LOCKED(map); | ||||
rv = KERN_SUCCESS; | |||||
if (start == end) | if (start == end) | ||||
return (KERN_SUCCESS); | return (rv); | ||||
/* | /* | ||||
* Find the start of the region, and clip it. | * Find the start of the region, and clip it. | ||||
* Step through all entries in this region. | * Step through all entries in this region. | ||||
*/ | */ | ||||
for (entry = vm_map_lookup_clip_start(map, start, &entry); | rv = vm_map_lookup_clip_start(map, start, &entry, &scratch_entry); | ||||
entry->start < end; entry = next_entry) { | if (rv != KERN_SUCCESS) | ||||
return (rv); | |||||
for (; entry->start < end; entry = next_entry) { | |||||
/* | /* | ||||
* Wait for wiring or unwiring of an entry to complete. | * Wait for wiring or unwiring of an entry to complete. | ||||
* Also wait for any system wirings to disappear on | * Also wait for any system wirings to disappear on | ||||
* user maps. | * user maps. | ||||
*/ | */ | ||||
if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 || | if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 || | ||||
(vm_map_pmap(map) != kernel_pmap && | (vm_map_pmap(map) != kernel_pmap && | ||||
vm_map_entry_system_wired_count(entry) != 0)) { | vm_map_entry_system_wired_count(entry) != 0)) { | ||||
unsigned int last_timestamp; | unsigned int last_timestamp; | ||||
vm_offset_t saved_start; | vm_offset_t saved_start; | ||||
saved_start = entry->start; | saved_start = entry->start; | ||||
entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; | entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; | ||||
last_timestamp = map->timestamp; | last_timestamp = map->timestamp; | ||||
(void) vm_map_unlock_and_wait(map, 0); | (void) vm_map_unlock_and_wait(map, 0); | ||||
vm_map_lock(map); | vm_map_lock(map); | ||||
if (last_timestamp + 1 != map->timestamp) { | if (last_timestamp + 1 != map->timestamp) { | ||||
/* | /* | ||||
* Look again for the entry because the map was | * Look again for the entry because the map was | ||||
* modified while it was unlocked. | * modified while it was unlocked. | ||||
* Specifically, the entry may have been | * Specifically, the entry may have been | ||||
* clipped, merged, or deleted. | * clipped, merged, or deleted. | ||||
*/ | */ | ||||
next_entry = vm_map_lookup_clip_start(map, | rv = vm_map_lookup_clip_start(map, saved_start, | ||||
saved_start, &next_entry); | &next_entry, &scratch_entry); | ||||
if (rv != KERN_SUCCESS) | |||||
break; | |||||
} else | } else | ||||
next_entry = entry; | next_entry = entry; | ||||
continue; | continue; | ||||
} | } | ||||
vm_map_clip_end(map, entry, end); | |||||
/* XXXKIB or delete to the upper superpage boundary ? */ | |||||
rv = vm_map_clip_end(map, entry, end); | |||||
if (rv != KERN_SUCCESS) | |||||
break; | |||||
next_entry = vm_map_entry_succ(entry); | next_entry = vm_map_entry_succ(entry); | ||||
/* | /* | ||||
* Unwire before removing addresses from the pmap; otherwise, | * Unwire before removing addresses from the pmap; otherwise, | ||||
* unwiring will put the entries back in the pmap. | * unwiring will put the entries back in the pmap. | ||||
*/ | */ | ||||
if (entry->wired_count != 0) | if (entry->wired_count != 0) | ||||
vm_map_entry_unwire(map, entry); | vm_map_entry_unwire(map, entry); | ||||
Show All 13 Lines | for (; entry->start < end; entry = next_entry) { | ||||
/* | /* | ||||
* Delete the entry only after removing all pmap | * Delete the entry only after removing all pmap | ||||
* entries pointing to its pages. (Otherwise, its | * entries pointing to its pages. (Otherwise, its | ||||
* page frames may be reallocated, and any modify bits | * page frames may be reallocated, and any modify bits | ||||
* will be set in the wrong object!) | * will be set in the wrong object!) | ||||
*/ | */ | ||||
vm_map_entry_delete(map, entry); | vm_map_entry_delete(map, entry); | ||||
} | } | ||||
return (KERN_SUCCESS); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* vm_map_remove: | * vm_map_remove: | ||||
* | * | ||||
* Remove the given address range from the target map. | * Remove the given address range from the target map. | ||||
* This is the exported form of vm_map_delete. | * This is the exported form of vm_map_delete. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 1,365 Lines • Show Last 20 Lines |
We are not checking the return value. vm_map_insert() will return an error if the space is occupied by a largepage mapping, but the errno value will be wrong (it should be EINVAL).