Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_map.c
Show First 20 Lines • Show All 795 Lines • ▼ Show 20 Lines | _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) | ||||
map->system_map = 0; | map->system_map = 0; | ||||
map->pmap = pmap; | map->pmap = pmap; | ||||
map->header.end = min; | map->header.end = min; | ||||
map->header.start = max; | map->header.start = max; | ||||
map->flags = 0; | map->flags = 0; | ||||
map->root = NULL; | map->root = NULL; | ||||
map->timestamp = 0; | map->timestamp = 0; | ||||
map->busy = 0; | map->busy = 0; | ||||
map->anon_loc = 0; | |||||
} | } | ||||
void | void | ||||
vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) | vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) | ||||
{ | { | ||||
_vm_map_init(map, pmap, min, max); | _vm_map_init(map, pmap, min, max); | ||||
mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK); | mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK); | ||||
▲ Show 20 Lines • Show All 663 Lines • ▼ Show 20 Lines | vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | ||||
} else { | } else { | ||||
result = vm_map_insert(map, object, offset, start, end, | result = vm_map_insert(map, object, offset, start, end, | ||||
prot, max, cow); | prot, max, cow); | ||||
} | } | ||||
vm_map_unlock(map); | vm_map_unlock(map); | ||||
return (result); | return (result); | ||||
} | } | ||||
static const int aslr_pages_rnd_64[2] = {0x1000, 0x10}; | |||||
static const int aslr_pages_rnd_32[2] = {0x100, 0x4}; | |||||
static int cluster_anon = 1; | |||||
SYSCTL_INT(_vm, OID_AUTO, cluster_anon, CTLFLAG_RW, | |||||
&cluster_anon, 0, | |||||
"Cluster anonymous mappings"); | |||||
static long aslr_restarts; | |||||
SYSCTL_LONG(_vm, OID_AUTO, aslr_restarts, CTLFLAG_RD, | |||||
&aslr_restarts, 0, | |||||
"Number of aslr failures"); | |||||
#define MAP_32BIT_MAX_ADDR ((vm_offset_t)1 << 31) | |||||
/* | /* | ||||
* Searches for the specified amount of free space in the given map with the | * Searches for the specified amount of free space in the given map with the | ||||
* specified alignment. Performs an address-ordered, first-fit search from | * specified alignment. Performs an address-ordered, first-fit search from | ||||
* the given address "*addr", with an optional upper bound "max_addr". If the | * the given address "*addr", with an optional upper bound "max_addr". If the | ||||
* parameter "alignment" is zero, then the alignment is computed from the | * parameter "alignment" is zero, then the alignment is computed from the | ||||
* given (object, offset) pair so as to enable the greatest possible use of | * given (object, offset) pair so as to enable the greatest possible use of | ||||
* superpage mappings. Returns KERN_SUCCESS and the address of the free space | * superpage mappings. Returns KERN_SUCCESS and the address of the free space | ||||
* in "*addr" if successful. Otherwise, returns KERN_NO_SPACE. | * in "*addr" if successful. Otherwise, returns KERN_NO_SPACE. | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | |||||
* prior to making call to account for the new entry. | * prior to making call to account for the new entry. | ||||
*/ | */ | ||||
int | int | ||||
vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, | ||||
vm_offset_t *addr, /* IN/OUT */ | vm_offset_t *addr, /* IN/OUT */ | ||||
vm_size_t length, vm_offset_t max_addr, int find_space, | vm_size_t length, vm_offset_t max_addr, int find_space, | ||||
vm_prot_t prot, vm_prot_t max, int cow) | vm_prot_t prot, vm_prot_t max, int cow) | ||||
{ | { | ||||
vm_offset_t alignment, min_addr; | vm_offset_t alignment, curr_min_addr, min_addr; | ||||
int rv; | int gap, pidx, rv, try; | ||||
bool cluster, en_aslr, update_anon; | |||||
KASSERT((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 || | KASSERT((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 || | ||||
object == NULL, | object == NULL, | ||||
("vm_map_find: non-NULL backing object for stack")); | ("vm_map_find: non-NULL backing object for stack")); | ||||
MPASS((cow & MAP_REMAP) == 0 || (find_space == VMFS_NO_SPACE && | MPASS((cow & MAP_REMAP) == 0 || (find_space == VMFS_NO_SPACE && | ||||
(cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0)); | (cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0)); | ||||
if (find_space == VMFS_OPTIMAL_SPACE && (object == NULL || | if (find_space == VMFS_OPTIMAL_SPACE && (object == NULL || | ||||
(object->flags & OBJ_COLORED) == 0)) | (object->flags & OBJ_COLORED) == 0)) | ||||
find_space = VMFS_ANY_SPACE; | find_space = VMFS_ANY_SPACE; | ||||
if (find_space >> 8 != 0) { | if (find_space >> 8 != 0) { | ||||
KASSERT((find_space & 0xff) == 0, ("bad VMFS flags")); | KASSERT((find_space & 0xff) == 0, ("bad VMFS flags")); | ||||
alignment = (vm_offset_t)1 << (find_space >> 8); | alignment = (vm_offset_t)1 << (find_space >> 8); | ||||
} else | } else | ||||
alignment = 0; | alignment = 0; | ||||
en_aslr = (map->flags & MAP_ASLR) != 0; | |||||
update_anon = cluster = cluster_anon != 0 && | |||||
(map->flags & MAP_IS_SUB_MAP) == 0 && max_addr == 0 && | |||||
find_space != VMFS_NO_SPACE && object == NULL && | |||||
(cow & (MAP_INHERIT_SHARE | MAP_STACK_GROWS_UP | | |||||
MAP_STACK_GROWS_DOWN)) == 0 && prot != PROT_NONE; | |||||
curr_min_addr = min_addr = *addr; | |||||
if (en_aslr && min_addr == 0 && !cluster && | |||||
find_space != VMFS_NO_SPACE && | |||||
(map->flags & MAP_ASLR_IGNSTART) != 0) | |||||
curr_min_addr = min_addr = vm_map_min(map); | |||||
try = 0; | |||||
vm_map_lock(map); | vm_map_lock(map); | ||||
if (cluster) { | |||||
curr_min_addr = map->anon_loc; | |||||
if (curr_min_addr == 0) | |||||
cluster = false; | |||||
} | |||||
if (find_space != VMFS_NO_SPACE) { | if (find_space != VMFS_NO_SPACE) { | ||||
KASSERT(find_space == VMFS_ANY_SPACE || | KASSERT(find_space == VMFS_ANY_SPACE || | ||||
find_space == VMFS_OPTIMAL_SPACE || | find_space == VMFS_OPTIMAL_SPACE || | ||||
find_space == VMFS_SUPER_SPACE || | find_space == VMFS_SUPER_SPACE || | ||||
alignment != 0, ("unexpected VMFS flag")); | alignment != 0, ("unexpected VMFS flag")); | ||||
min_addr = *addr; | |||||
again: | again: | ||||
if (vm_map_findspace(map, min_addr, length, addr) || | /* | ||||
* When creating an anonymous mapping, try clustering | |||||
* with an existing anonymous mapping first. | |||||
* | |||||
* We make up to two attempts to find address space | |||||
* for a given find_space value. The first attempt may | |||||
* apply randomization or may cluster with an existing | |||||
* anonymous mapping. If this first attempt fails, | |||||
* perform a first-fit search of the available address | |||||
* space. | |||||
* | |||||
* If all tries failed, and find_space is | |||||
* VMFS_OPTIMAL_SPACE, fallback to VMFS_ANY_SPACE. | |||||
* Again enable clustering and randomization. | |||||
*/ | |||||
try++; | |||||
MPASS(try <= 2); | |||||
if (try == 2) { | |||||
/* | |||||
* Second try: we failed either to find a | |||||
* suitable region for randomizing the | |||||
* allocation, or to cluster with an existing | |||||
* mapping. Retry with free run. | |||||
*/ | |||||
curr_min_addr = (map->flags & MAP_ASLR_IGNSTART) != 0 ? | |||||
vm_map_min(map) : min_addr; | |||||
atomic_add_long(&aslr_restarts, 1); | |||||
} | |||||
if (try == 1 && en_aslr && !cluster) { | |||||
/* | |||||
* Find space for allocation, including | |||||
* gap needed for later randomization. | |||||
*/ | |||||
pidx = MAXPAGESIZES > 1 && pagesizes[1] != 0 && | |||||
(find_space == VMFS_SUPER_SPACE || find_space == | |||||
VMFS_OPTIMAL_SPACE) ? 1 : 0; | |||||
gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR && | |||||
(max_addr == 0 || max_addr > MAP_32BIT_MAX_ADDR) ? | |||||
aslr_pages_rnd_64[pidx] : aslr_pages_rnd_32[pidx]; | |||||
if (vm_map_findspace(map, curr_min_addr, length + | |||||
gap * pagesizes[pidx], addr) || | |||||
(max_addr != 0 && *addr + length > max_addr)) | |||||
goto again; | |||||
/* And randomize the start address. */ | |||||
*addr += (arc4random() % gap) * pagesizes[pidx]; | |||||
dougm: If max_addr != 0, you've guaranteed that *addr + length <= max_addr, but after this… | |||||
kibAuthorUnsubmitted Done Inline ActionsYou mean, in the situation where vm_map_maxaddr(map) > max_addr. Please see D19688. kib: You mean, in the situation where vm_map_maxaddr(map) > max_addr. Please see D19688. | |||||
} else if (vm_map_findspace(map, curr_min_addr, length, addr) || | |||||
(max_addr != 0 && *addr + length > max_addr)) { | (max_addr != 0 && *addr + length > max_addr)) { | ||||
if (cluster) { | |||||
cluster = false; | |||||
MPASS(try == 1); | |||||
goto again; | |||||
} | |||||
rv = KERN_NO_SPACE; | rv = KERN_NO_SPACE; | ||||
goto done; | goto done; | ||||
} | } | ||||
if (find_space != VMFS_ANY_SPACE && | if (find_space != VMFS_ANY_SPACE && | ||||
(rv = vm_map_alignspace(map, object, offset, addr, length, | (rv = vm_map_alignspace(map, object, offset, addr, length, | ||||
max_addr, alignment)) != KERN_SUCCESS) { | max_addr, alignment)) != KERN_SUCCESS) { | ||||
if (find_space == VMFS_OPTIMAL_SPACE) { | if (find_space == VMFS_OPTIMAL_SPACE) { | ||||
find_space = VMFS_ANY_SPACE; | find_space = VMFS_ANY_SPACE; | ||||
curr_min_addr = min_addr; | |||||
cluster = update_anon; | |||||
try = 0; | |||||
goto again; | goto again; | ||||
} | } | ||||
goto done; | goto done; | ||||
} | } | ||||
} else if ((cow & MAP_REMAP) != 0) { | } else if ((cow & MAP_REMAP) != 0) { | ||||
if (*addr < vm_map_min(map) || | if (*addr < vm_map_min(map) || | ||||
*addr + length > vm_map_max(map) || | *addr + length > vm_map_max(map) || | ||||
*addr + length <= length) { | *addr + length <= length) { | ||||
rv = KERN_INVALID_ADDRESS; | rv = KERN_INVALID_ADDRESS; | ||||
goto done; | goto done; | ||||
} | } | ||||
vm_map_delete(map, *addr, *addr + length); | vm_map_delete(map, *addr, *addr + length); | ||||
} | } | ||||
if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) != 0) { | if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) != 0) { | ||||
rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot, | rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot, | ||||
max, cow); | max, cow); | ||||
} else { | } else { | ||||
rv = vm_map_insert(map, object, offset, *addr, *addr + length, | rv = vm_map_insert(map, object, offset, *addr, *addr + length, | ||||
prot, max, cow); | prot, max, cow); | ||||
} | } | ||||
if (rv == KERN_SUCCESS && update_anon) | |||||
map->anon_loc = *addr + length; | |||||
done: | done: | ||||
vm_map_unlock(map); | vm_map_unlock(map); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* vm_map_find_min() is a variant of vm_map_find() that takes an | * vm_map_find_min() is a variant of vm_map_find() that takes an | ||||
* additional parameter (min_addr) and treats the given address | * additional parameter (min_addr) and treats the given address | ||||
▲ Show 20 Lines • Show All 293 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
vm_map_submap( | vm_map_submap( | ||||
vm_map_t map, | vm_map_t map, | ||||
vm_offset_t start, | vm_offset_t start, | ||||
vm_offset_t end, | vm_offset_t end, | ||||
vm_map_t submap) | vm_map_t submap) | ||||
{ | { | ||||
vm_map_entry_t entry; | vm_map_entry_t entry; | ||||
int result = KERN_INVALID_ARGUMENT; | int result; | ||||
result = KERN_INVALID_ARGUMENT; | |||||
vm_map_lock(submap); | |||||
submap->flags |= MAP_IS_SUB_MAP; | |||||
vm_map_unlock(submap); | |||||
vm_map_lock(map); | vm_map_lock(map); | ||||
VM_MAP_RANGE_CHECK(map, start, end); | VM_MAP_RANGE_CHECK(map, start, end); | ||||
if (vm_map_lookup_entry(map, start, &entry)) { | if (vm_map_lookup_entry(map, start, &entry)) { | ||||
vm_map_clip_start(map, entry, start); | vm_map_clip_start(map, entry, start); | ||||
} else | } else | ||||
entry = entry->next; | entry = entry->next; | ||||
vm_map_clip_end(map, entry, end); | vm_map_clip_end(map, entry, end); | ||||
if ((entry->start == start) && (entry->end == end) && | if ((entry->start == start) && (entry->end == end) && | ||||
((entry->eflags & MAP_ENTRY_COW) == 0) && | ((entry->eflags & MAP_ENTRY_COW) == 0) && | ||||
(entry->object.vm_object == NULL)) { | (entry->object.vm_object == NULL)) { | ||||
entry->object.sub_map = submap; | entry->object.sub_map = submap; | ||||
entry->eflags |= MAP_ENTRY_IS_SUB_MAP; | entry->eflags |= MAP_ENTRY_IS_SUB_MAP; | ||||
result = KERN_SUCCESS; | result = KERN_SUCCESS; | ||||
} | } | ||||
vm_map_unlock(map); | vm_map_unlock(map); | ||||
if (result != KERN_SUCCESS) { | |||||
vm_map_lock(submap); | |||||
submap->flags &= ~MAP_IS_SUB_MAP; | |||||
vm_map_unlock(submap); | |||||
} | |||||
return (result); | return (result); | ||||
} | } | ||||
/* | /* | ||||
* The maximum number of pages to map if MAP_PREFAULT_PARTIAL is specified | * The maximum number of pages to map if MAP_PREFAULT_PARTIAL is specified | ||||
*/ | */ | ||||
#define MAX_INIT_PT 96 | #define MAX_INIT_PT 96 | ||||
▲ Show 20 Lines • Show All 1,210 Lines • ▼ Show 20 Lines | while (entry->start < end) { | ||||
* Remove mappings for the pages, but only if the | * Remove mappings for the pages, but only if the | ||||
* mappings could exist. For instance, it does not | * mappings could exist. For instance, it does not | ||||
* make sense to call pmap_remove() for guard entries. | * make sense to call pmap_remove() for guard entries. | ||||
*/ | */ | ||||
if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0 || | if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0 || | ||||
entry->object.vm_object != NULL) | entry->object.vm_object != NULL) | ||||
pmap_remove(map->pmap, entry->start, entry->end); | pmap_remove(map->pmap, entry->start, entry->end); | ||||
if (entry->end == map->anon_loc) | |||||
map->anon_loc = entry->start; | |||||
/* | /* | ||||
* Delete the entry only after removing all pmap | * Delete the entry only after removing all pmap | ||||
* entries pointing to its pages. (Otherwise, its | * entries pointing to its pages. (Otherwise, its | ||||
* page frames may be reallocated, and any modify bits | * page frames may be reallocated, and any modify bits | ||||
* will be set in the wrong object!) | * will be set in the wrong object!) | ||||
*/ | */ | ||||
vm_map_entry_delete(map, entry); | vm_map_entry_delete(map, entry); | ||||
entry = next; | entry = next; | ||||
▲ Show 20 Lines • Show All 257 Lines • ▼ Show 20 Lines | vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge) | ||||
vm2->vm_maxsaddr = vm1->vm_maxsaddr; | vm2->vm_maxsaddr = vm1->vm_maxsaddr; | ||||
vm_map_lock(old_map); | vm_map_lock(old_map); | ||||
if (old_map->busy) | if (old_map->busy) | ||||
vm_map_wait_busy(old_map); | vm_map_wait_busy(old_map); | ||||
new_map = &vm2->vm_map; | new_map = &vm2->vm_map; | ||||
locked = vm_map_trylock(new_map); /* trylock to silence WITNESS */ | locked = vm_map_trylock(new_map); /* trylock to silence WITNESS */ | ||||
KASSERT(locked, ("vmspace_fork: lock failed")); | KASSERT(locked, ("vmspace_fork: lock failed")); | ||||
new_map->anon_loc = old_map->anon_loc; | |||||
old_entry = old_map->header.next; | old_entry = old_map->header.next; | ||||
while (old_entry != &old_map->header) { | while (old_entry != &old_map->header) { | ||||
if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP) | if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP) | ||||
panic("vm_map_fork: encountered a submap"); | panic("vm_map_fork: encountered a submap"); | ||||
inh = old_entry->inheritance; | inh = old_entry->inheritance; | ||||
if ((old_entry->eflags & MAP_ENTRY_GUARD) != 0 && | if ((old_entry->eflags & MAP_ENTRY_GUARD) != 0 && | ||||
▲ Show 20 Lines • Show All 996 Lines • Show Last 20 Lines |
If max_addr != 0, you've guaranteed that *addr + length <= max_addr, but after this modification to *addr, the guarantee won't hold and you may return an address beyond max_addr. Or so it seems to me.