Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_map.c
Show First 20 Lines • Show All 590 Lines • ▼ Show 20 Lines | if ((entry->eflags & MAP_ENTRY_WRITECNT) != 0) { | ||||
entry->end); | entry->end); | ||||
} | } | ||||
vm_map_entry_set_vnode_text(entry, false); | vm_map_entry_set_vnode_text(entry, false); | ||||
vm_map_entry_deallocate(entry, FALSE); | vm_map_entry_deallocate(entry, FALSE); | ||||
entry = next; | entry = next; | ||||
} | } | ||||
} | } | ||||
#ifdef INVARIANTS | |||||
static void | |||||
_vm_map_assert_locked(vm_map_t map, const char *file, int line) | |||||
{ | |||||
if (map->system_map) | |||||
mtx_assert_(&map->system_mtx, MA_OWNED, file, line); | |||||
else | |||||
sx_assert_(&map->lock, SA_XLOCKED, file, line); | |||||
} | |||||
#define VM_MAP_ASSERT_LOCKED(map) \ | |||||
_vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE) | |||||
enum { VMMAP_CHECK_NONE, VMMAP_CHECK_UNLOCK, VMMAP_CHECK_ALL }; | |||||
#ifdef DIAGNOSTIC | |||||
static int enable_vmmap_check = VMMAP_CHECK_UNLOCK; | |||||
#else | |||||
static int enable_vmmap_check = VMMAP_CHECK_NONE; | |||||
#endif | |||||
SYSCTL_INT(_debug, OID_AUTO, vmmap_check, CTLFLAG_RWTUN, | |||||
&enable_vmmap_check, 0, "Enable vm map consistency checking"); | |||||
static void _vm_map_assert_consistent(vm_map_t map, int check); | |||||
#define VM_MAP_ASSERT_CONSISTENT(map) \ | |||||
_vm_map_assert_consistent(map, VMMAP_CHECK_ALL) | |||||
#define VM_MAP_UNLOCK_CONSISTENT(map) do { \ | |||||
if (map->updates > map->nentries) { \ | |||||
_vm_map_assert_consistent(map, VMMAP_CHECK_UNLOCK); \ | |||||
map->updates = 0; \ | |||||
} \ | |||||
} | |||||
#else | |||||
#define VM_MAP_ASSERT_LOCKED(map) | |||||
#define VM_MAP_ASSERT_CONSISTENT(map) | |||||
#define VM_MAP_UNLOCK_CONSISTENT(map) | |||||
#endif /* INVARIANTS */ | |||||
void | void | ||||
_vm_map_unlock(vm_map_t map, const char *file, int line) | _vm_map_unlock(vm_map_t map, const char *file, int line) | ||||
{ | { | ||||
VM_MAP_UNLOCK_CONSISTENT(map); | |||||
if (map->system_map) | if (map->system_map) | ||||
mtx_unlock_flags_(&map->system_mtx, 0, file, line); | mtx_unlock_flags_(&map->system_mtx, 0, file, line); | ||||
else { | else { | ||||
sx_xunlock_(&map->lock, file, line); | sx_xunlock_(&map->lock, file, line); | ||||
vm_map_process_deferred(); | vm_map_process_deferred(); | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
void | void | ||||
_vm_map_lock_downgrade(vm_map_t map, const char *file, int line) | _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) | ||||
{ | { | ||||
if (map->system_map) { | if (map->system_map) { | ||||
mtx_assert_(&map->system_mtx, MA_OWNED, file, line); | mtx_assert_(&map->system_mtx, MA_OWNED, file, line); | ||||
} else | } else { | ||||
VM_MAP_UNLOCK_CONSISTENT(map); | |||||
sx_downgrade_(&map->lock, file, line); | sx_downgrade_(&map->lock, file, line); | ||||
} | } | ||||
} | |||||
/* | /* | ||||
* vm_map_locked: | * vm_map_locked: | ||||
* | * | ||||
* Returns a non-zero value if the caller holds a write (exclusive) lock | * Returns a non-zero value if the caller holds a write (exclusive) lock | ||||
* on the specified map and the value "0" otherwise. | * on the specified map and the value "0" otherwise. | ||||
*/ | */ | ||||
int | int | ||||
vm_map_locked(vm_map_t map) | vm_map_locked(vm_map_t map) | ||||
{ | { | ||||
if (map->system_map) | if (map->system_map) | ||||
return (mtx_owned(&map->system_mtx)); | return (mtx_owned(&map->system_mtx)); | ||||
else | else | ||||
return (sx_xlocked(&map->lock)); | return (sx_xlocked(&map->lock)); | ||||
} | } | ||||
#ifdef INVARIANTS | |||||
static void | |||||
_vm_map_assert_locked(vm_map_t map, const char *file, int line) | |||||
{ | |||||
if (map->system_map) | |||||
mtx_assert_(&map->system_mtx, MA_OWNED, file, line); | |||||
else | |||||
sx_assert_(&map->lock, SA_XLOCKED, file, line); | |||||
} | |||||
#define VM_MAP_ASSERT_LOCKED(map) \ | |||||
_vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE) | |||||
#ifdef DIAGNOSTIC | |||||
static int enable_vmmap_check = 1; | |||||
#else | |||||
static int enable_vmmap_check = 0; | |||||
#endif | |||||
SYSCTL_INT(_debug, OID_AUTO, vmmap_check, CTLFLAG_RWTUN, | |||||
&enable_vmmap_check, 0, "Enable vm map consistency checking"); | |||||
static void _vm_map_assert_consistent(vm_map_t map); | |||||
#define VM_MAP_ASSERT_CONSISTENT(map) \ | |||||
_vm_map_assert_consistent(map) | |||||
#else | |||||
#define VM_MAP_ASSERT_LOCKED(map) | |||||
#define VM_MAP_ASSERT_CONSISTENT(map) | |||||
#endif /* INVARIANTS */ | |||||
/* | /* | ||||
* _vm_map_unlock_and_wait: | * _vm_map_unlock_and_wait: | ||||
* | * | ||||
* Atomically releases the lock on the specified map and puts the calling | * Atomically releases the lock on the specified map and puts the calling | ||||
* thread to sleep. The calling thread will remain asleep until either | * thread to sleep. The calling thread will remain asleep until either | ||||
* vm_map_wakeup() is performed on the map or the specified timeout is | * vm_map_wakeup() is performed on the map or the specified timeout is | ||||
* exceeded. | * exceeded. | ||||
* | * | ||||
* WARNING! This function does not perform deferred deallocations of | * WARNING! This function does not perform deferred deallocations of | ||||
* objects and map entries. Therefore, the calling thread is expected to | * objects and map entries. Therefore, the calling thread is expected to | ||||
* reacquire the map lock after reawakening and later perform an ordinary | * reacquire the map lock after reawakening and later perform an ordinary | ||||
* unlock operation, such as vm_map_unlock(), before completing its | * unlock operation, such as vm_map_unlock(), before completing its | ||||
* operation on the map. | * operation on the map. | ||||
*/ | */ | ||||
int | int | ||||
_vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) | _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) | ||||
{ | { | ||||
VM_MAP_UNLOCK_CONSISTENT(map); | |||||
mtx_lock(&map_sleep_mtx); | mtx_lock(&map_sleep_mtx); | ||||
if (map->system_map) | if (map->system_map) | ||||
mtx_unlock_flags_(&map->system_mtx, 0, file, line); | mtx_unlock_flags_(&map->system_mtx, 0, file, line); | ||||
else | else | ||||
sx_xunlock_(&map->lock, file, line); | sx_xunlock_(&map->lock, file, line); | ||||
return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", | return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", | ||||
timo)); | timo)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 350 Lines • ▼ Show 20 Lines | do { | ||||
MAX(rlist->max_free, max_free_right); | MAX(rlist->max_free, max_free_right); | ||||
vm_map_entry_swap(&rlist->left, &prev); | vm_map_entry_swap(&rlist->left, &prev); | ||||
vm_map_entry_swap(&prev, &rlist); | vm_map_entry_swap(&prev, &rlist); | ||||
} while (rlist != &map->header); | } while (rlist != &map->header); | ||||
root->right = prev; | root->right = prev; | ||||
} | } | ||||
root->max_free = MAX(max_free_left, max_free_right); | root->max_free = MAX(max_free_left, max_free_right); | ||||
map->root = root; | map->root = root; | ||||
#ifdef DIAGNOSTIC | |||||
++map->nupdates; | |||||
#endif | |||||
} | } | ||||
/* | /* | ||||
* vm_map_splay: | * vm_map_splay: | ||||
* | * | ||||
* The Sleator and Tarjan top-down splay algorithm with the | * The Sleator and Tarjan top-down splay algorithm with the | ||||
* following variation. Max_free must be computed bottom-up, so | * following variation. Max_free must be computed bottom-up, so | ||||
* on the downward pass, maintain the left and right spines in | * on the downward pass, maintain the left and right spines in | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | if ((locked = vm_map_locked(map)) || | ||||
sx_try_upgrade(&map->lock)) { | sx_try_upgrade(&map->lock)) { | ||||
/* | /* | ||||
* Splay requires a write lock on the map. However, it only | * Splay requires a write lock on the map. However, it only | ||||
* restructures the binary search tree; it does not otherwise | * restructures the binary search tree; it does not otherwise | ||||
* change the map. Thus, the map's timestamp need not change | * change the map. Thus, the map's timestamp need not change | ||||
* on a temporary upgrade. | * on a temporary upgrade. | ||||
*/ | */ | ||||
cur = vm_map_splay(map, address); | cur = vm_map_splay(map, address); | ||||
if (!locked) | if (!locked) { | ||||
VM_MAP_UNLOCK_CONSISTENT(map); | |||||
sx_downgrade(&map->lock); | sx_downgrade(&map->lock); | ||||
} | |||||
/* | /* | ||||
* If "address" is contained within a map entry, the new root | * If "address" is contained within a map entry, the new root | ||||
* is that map entry. Otherwise, the new root is a map entry | * is that map entry. Otherwise, the new root is a map entry | ||||
* immediately before or after "address". | * immediately before or after "address". | ||||
*/ | */ | ||||
if (address < cur->start) { | if (address < cur->start) { | ||||
*entry = &map->header; | *entry = &map->header; | ||||
▲ Show 20 Lines • Show All 3,438 Lines • ▼ Show 20 Lines | |||||
vm_map_pmap_KBI(vm_map_t map) | vm_map_pmap_KBI(vm_map_t map) | ||||
{ | { | ||||
return (map->pmap); | return (map->pmap); | ||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
static void | static void | ||||
_vm_map_assert_consistent(vm_map_t map) | _vm_map_assert_consistent(vm_map_t map, int check) | ||||
{ | { | ||||
vm_map_entry_t entry, prev; | vm_map_entry_t entry, prev; | ||||
vm_size_t max_left, max_right; | vm_size_t max_left, max_right; | ||||
if (!enable_vmmap_check) | if (enable_vmmap_check != check) | ||||
return; | return; | ||||
prev = &map->header; | prev = &map->header; | ||||
VM_MAP_ENTRY_FOREACH(entry, map) { | VM_MAP_ENTRY_FOREACH(entry, map) { | ||||
KASSERT(prev->end <= entry->start, | KASSERT(prev->end <= entry->start, | ||||
("map %p prev->end = %jx, start = %jx", map, | ("map %p prev->end = %jx, start = %jx", map, | ||||
(uintmax_t)prev->end, (uintmax_t)entry->start)); | (uintmax_t)prev->end, (uintmax_t)entry->start)); | ||||
KASSERT(entry->start < entry->end, | KASSERT(entry->start < entry->end, | ||||
▲ Show 20 Lines • Show All 129 Lines • Show Last 20 Lines |