Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_map.c
Show First 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | |||||
#include <sys/sysent.h> | #include <sys/sysent.h> | ||||
#include <sys/shm.h> | #include <sys/shm.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_param.h> | #include <vm/vm_param.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <vm/vm_map.h> | #include <vm/vm_map.h> | ||||
#include <vm/vm_page.h> | #include <vm/vm_page.h> | ||||
#include <vm/vm_pageout.h> | |||||
#include <vm/vm_object.h> | #include <vm/vm_object.h> | ||||
#include <vm/vm_pager.h> | #include <vm/vm_pager.h> | ||||
#include <vm/vm_kern.h> | #include <vm/vm_kern.h> | ||||
#include <vm/vm_extern.h> | #include <vm/vm_extern.h> | ||||
#include <vm/vnode_pager.h> | #include <vm/vnode_pager.h> | ||||
#include <vm/swap_pager.h> | #include <vm/swap_pager.h> | ||||
#include <vm/uma.h> | #include <vm/uma.h> | ||||
▲ Show 20 Lines • Show All 2,744 Lines • ▼ Show 20 Lines | if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || | ||||
entry->wiring_thread != curthread) { | entry->wiring_thread != curthread) { | ||||
KASSERT((flags & VM_MAP_WIRE_HOLESOK) != 0, | KASSERT((flags & VM_MAP_WIRE_HOLESOK) != 0, | ||||
("vm_map_unwire: !HOLESOK and new/changed entry")); | ("vm_map_unwire: !HOLESOK and new/changed entry")); | ||||
continue; | continue; | ||||
} | } | ||||
if (rv == KERN_SUCCESS && (!user_unwire || | if (rv == KERN_SUCCESS && (!user_unwire || | ||||
(entry->eflags & MAP_ENTRY_USER_WIRED))) { | (entry->eflags & MAP_ENTRY_USER_WIRED))) { | ||||
if (user_unwire) | |||||
entry->eflags &= ~MAP_ENTRY_USER_WIRED; | |||||
if (entry->wired_count == 1) | if (entry->wired_count == 1) | ||||
vm_map_entry_unwire(map, entry); | vm_map_entry_unwire(map, entry); | ||||
else | else | ||||
entry->wired_count--; | entry->wired_count--; | ||||
if (user_unwire) | |||||
entry->eflags &= ~MAP_ENTRY_USER_WIRED; | |||||
} | } | ||||
KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, | KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, | ||||
("vm_map_unwire: in-transition flag missing %p", entry)); | ("vm_map_unwire: in-transition flag missing %p", entry)); | ||||
KASSERT(entry->wiring_thread == curthread, | KASSERT(entry->wiring_thread == curthread, | ||||
("vm_map_unwire: alien wire %p", entry)); | ("vm_map_unwire: alien wire %p", entry)); | ||||
entry->eflags &= ~MAP_ENTRY_IN_TRANSITION; | entry->eflags &= ~MAP_ENTRY_IN_TRANSITION; | ||||
entry->wiring_thread = NULL; | entry->wiring_thread = NULL; | ||||
if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { | if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { | ||||
entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; | entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; | ||||
need_wakeup = TRUE; | need_wakeup = TRUE; | ||||
} | } | ||||
vm_map_simplify_entry(map, entry); | vm_map_simplify_entry(map, entry); | ||||
} | } | ||||
vm_map_unlock(map); | vm_map_unlock(map); | ||||
if (need_wakeup) | if (need_wakeup) | ||||
vm_map_wakeup(map); | vm_map_wakeup(map); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
static void | |||||
vm_map_wire_user_count_sub(u_long npages) | |||||
{ | |||||
atomic_subtract_long(&vm_user_wire_count, npages); | |||||
} | |||||
static bool | |||||
vm_map_wire_user_count_add(u_long npages) | |||||
{ | |||||
u_long wired; | |||||
wired = vm_user_wire_count; | |||||
do { | |||||
if (npages + wired > vm_page_max_user_wired) | |||||
return (false); | |||||
} while (!atomic_fcmpset_long(&vm_user_wire_count, &wired, | |||||
npages + wired)); | |||||
return (true); | |||||
} | |||||
/* | /* | ||||
* vm_map_wire_entry_failure: | * vm_map_wire_entry_failure: | ||||
* | * | ||||
* Handle a wiring failure on the given entry. | * Handle a wiring failure on the given entry. | ||||
* | * | ||||
* The map should be locked. | * The map should be locked. | ||||
*/ | */ | ||||
static void | static void | ||||
Show All 20 Lines | vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, | ||||
/* | /* | ||||
* Assign an out-of-range value to represent the failure to wire this | * Assign an out-of-range value to represent the failure to wire this | ||||
* entry. | * entry. | ||||
*/ | */ | ||||
entry->wired_count = -1; | entry->wired_count = -1; | ||||
} | } | ||||
int | |||||
vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) | |||||
{ | |||||
int rv; | |||||
vm_map_lock(map); | |||||
rv = vm_map_wire_locked(map, start, end, flags); | |||||
vm_map_unlock(map); | |||||
return (rv); | |||||
} | |||||
/* | /* | ||||
* vm_map_wire: | * vm_map_wire_locked: | ||||
* | * | ||||
* Implements both kernel and user wiring. | * Implements both kernel and user wiring. Returns with the map locked, | ||||
* the map lock may be dropped. | |||||
*/ | */ | ||||
int | int | ||||
vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, | vm_map_wire_locked(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) | ||||
int flags) | |||||
{ | { | ||||
vm_map_entry_t entry, first_entry, tmp_entry; | vm_map_entry_t entry, first_entry, tmp_entry; | ||||
vm_offset_t faddr, saved_end, saved_start; | vm_offset_t faddr, saved_end, saved_start; | ||||
unsigned int last_timestamp; | u_long npages; | ||||
u_int last_timestamp; | |||||
int rv; | int rv; | ||||
boolean_t need_wakeup, result, user_wire; | boolean_t need_wakeup, result, user_wire; | ||||
vm_prot_t prot; | vm_prot_t prot; | ||||
VM_MAP_ASSERT_LOCKED(map); | |||||
if (start == end) | if (start == end) | ||||
return (KERN_SUCCESS); | return (KERN_SUCCESS); | ||||
prot = 0; | prot = 0; | ||||
if (flags & VM_MAP_WIRE_WRITE) | if (flags & VM_MAP_WIRE_WRITE) | ||||
prot |= VM_PROT_WRITE; | prot |= VM_PROT_WRITE; | ||||
user_wire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE; | user_wire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE; | ||||
vm_map_lock(map); | |||||
VM_MAP_RANGE_CHECK(map, start, end); | VM_MAP_RANGE_CHECK(map, start, end); | ||||
if (!vm_map_lookup_entry(map, start, &first_entry)) { | if (!vm_map_lookup_entry(map, start, &first_entry)) { | ||||
if (flags & VM_MAP_WIRE_HOLESOK) | if (flags & VM_MAP_WIRE_HOLESOK) | ||||
first_entry = first_entry->next; | first_entry = first_entry->next; | ||||
else { | else | ||||
vm_map_unlock(map); | |||||
return (KERN_INVALID_ADDRESS); | return (KERN_INVALID_ADDRESS); | ||||
} | } | ||||
} | |||||
last_timestamp = map->timestamp; | last_timestamp = map->timestamp; | ||||
entry = first_entry; | entry = first_entry; | ||||
while (entry->start < end) { | while (entry->start < end) { | ||||
if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { | if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { | ||||
/* | /* | ||||
* We have not yet clipped the entry. | * We have not yet clipped the entry. | ||||
*/ | */ | ||||
saved_start = (start >= entry->start) ? start : | saved_start = (start >= entry->start) ? start : | ||||
Show All 16 Lines | if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { | ||||
&tmp_entry)) { | &tmp_entry)) { | ||||
if (flags & VM_MAP_WIRE_HOLESOK) | if (flags & VM_MAP_WIRE_HOLESOK) | ||||
tmp_entry = tmp_entry->next; | tmp_entry = tmp_entry->next; | ||||
else { | else { | ||||
if (saved_start == start) { | if (saved_start == start) { | ||||
/* | /* | ||||
* first_entry has been deleted. | * first_entry has been deleted. | ||||
*/ | */ | ||||
vm_map_unlock(map); | |||||
return (KERN_INVALID_ADDRESS); | return (KERN_INVALID_ADDRESS); | ||||
} | } | ||||
end = saved_start; | end = saved_start; | ||||
rv = KERN_INVALID_ADDRESS; | rv = KERN_INVALID_ADDRESS; | ||||
goto done; | goto done; | ||||
} | } | ||||
} | } | ||||
if (entry == first_entry) | if (entry == first_entry) | ||||
Show All 23 Lines | if ((entry->protection & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 | ||||
end = entry->end; | end = entry->end; | ||||
rv = KERN_INVALID_ADDRESS; | rv = KERN_INVALID_ADDRESS; | ||||
goto done; | goto done; | ||||
} | } | ||||
goto next_entry; | goto next_entry; | ||||
} | } | ||||
if (entry->wired_count == 0) { | if (entry->wired_count == 0) { | ||||
entry->wired_count++; | entry->wired_count++; | ||||
saved_start = entry->start; | |||||
saved_end = entry->end; | |||||
npages = atop(entry->end - entry->start); | |||||
if (user_wire && !vm_map_wire_user_count_add(npages)) { | |||||
vm_map_wire_entry_failure(map, entry, | |||||
entry->start); | |||||
end = entry->end; | |||||
rv = KERN_RESOURCE_SHORTAGE; | |||||
goto done; | |||||
} | |||||
/* | /* | ||||
* Release the map lock, relying on the in-transition | * Release the map lock, relying on the in-transition | ||||
* mark. Mark the map busy for fork. | * mark. Mark the map busy for fork. | ||||
*/ | */ | ||||
saved_start = entry->start; | |||||
saved_end = entry->end; | |||||
vm_map_busy(map); | vm_map_busy(map); | ||||
vm_map_unlock(map); | vm_map_unlock(map); | ||||
faddr = saved_start; | faddr = saved_start; | ||||
do { | do { | ||||
/* | /* | ||||
* Simulate a fault to get the page and enter | * Simulate a fault to get the page and enter | ||||
* it into the physical map. | * it into the physical map. | ||||
Show All 31 Lines | if (entry->wired_count == 0) { | ||||
vm_map_wire_entry_failure(map, | vm_map_wire_entry_failure(map, | ||||
entry, faddr); | entry, faddr); | ||||
entry = entry->next; | entry = entry->next; | ||||
} | } | ||||
} | } | ||||
last_timestamp = map->timestamp; | last_timestamp = map->timestamp; | ||||
if (rv != KERN_SUCCESS) { | if (rv != KERN_SUCCESS) { | ||||
vm_map_wire_entry_failure(map, entry, faddr); | vm_map_wire_entry_failure(map, entry, faddr); | ||||
if (user_wire) | |||||
vm_map_wire_user_count_sub(npages); | |||||
end = entry->end; | end = entry->end; | ||||
goto done; | goto done; | ||||
} | } | ||||
} else if (!user_wire || | } else if (!user_wire || | ||||
(entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { | (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { | ||||
entry->wired_count++; | entry->wired_count++; | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | if (rv == KERN_SUCCESS) { | ||||
*/ | */ | ||||
entry->wired_count = 0; | entry->wired_count = 0; | ||||
} else if (!user_wire || | } else if (!user_wire || | ||||
(entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { | (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { | ||||
/* | /* | ||||
* Undo the wiring. Wiring succeeded on this entry | * Undo the wiring. Wiring succeeded on this entry | ||||
* but failed on a later entry. | * but failed on a later entry. | ||||
*/ | */ | ||||
if (entry->wired_count == 1) | if (entry->wired_count == 1) { | ||||
vm_map_entry_unwire(map, entry); | vm_map_entry_unwire(map, entry); | ||||
else | if (user_wire) | ||||
vm_map_wire_user_count_sub( | |||||
atop(entry->end - entry->start)); | |||||
} else | |||||
entry->wired_count--; | entry->wired_count--; | ||||
} | } | ||||
next_entry_done: | next_entry_done: | ||||
KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, | KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, | ||||
("vm_map_wire: in-transition flag missing %p", entry)); | ("vm_map_wire: in-transition flag missing %p", entry)); | ||||
KASSERT(entry->wiring_thread == curthread, | KASSERT(entry->wiring_thread == curthread, | ||||
("vm_map_wire: alien wire %p", entry)); | ("vm_map_wire: alien wire %p", entry)); | ||||
entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION | | entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION | | ||||
MAP_ENTRY_WIRE_SKIPPED); | MAP_ENTRY_WIRE_SKIPPED); | ||||
entry->wiring_thread = NULL; | entry->wiring_thread = NULL; | ||||
if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { | if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { | ||||
entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; | entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; | ||||
need_wakeup = TRUE; | need_wakeup = TRUE; | ||||
} | } | ||||
vm_map_simplify_entry(map, entry); | vm_map_simplify_entry(map, entry); | ||||
} | } | ||||
vm_map_unlock(map); | |||||
if (need_wakeup) | if (need_wakeup) | ||||
vm_map_wakeup(map); | vm_map_wakeup(map); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* vm_map_sync | * vm_map_sync | ||||
* | * | ||||
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | |||||
* Make the region specified by this entry pageable. | * Make the region specified by this entry pageable. | ||||
* | * | ||||
* The map in question should be locked. | * The map in question should be locked. | ||||
* [This is the reason for this routine's existence.] | * [This is the reason for this routine's existence.] | ||||
*/ | */ | ||||
static void | static void | ||||
vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) | vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) | ||||
{ | { | ||||
vm_size_t size; | |||||
VM_MAP_ASSERT_LOCKED(map); | VM_MAP_ASSERT_LOCKED(map); | ||||
KASSERT(entry->wired_count > 0, | KASSERT(entry->wired_count > 0, | ||||
("vm_map_entry_unwire: entry %p isn't wired", entry)); | ("vm_map_entry_unwire: entry %p isn't wired", entry)); | ||||
size = entry->end - entry->start; | |||||
if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0) | |||||
vm_map_wire_user_count_sub(atop(size)); | |||||
pmap_unwire(map->pmap, entry->start, entry->end); | pmap_unwire(map->pmap, entry->start, entry->end); | ||||
vm_object_unwire(entry->object.vm_object, entry->offset, entry->end - | vm_object_unwire(entry->object.vm_object, entry->offset, size, | ||||
entry->start, PQ_ACTIVE); | PQ_ACTIVE); | ||||
entry->wired_count = 0; | entry->wired_count = 0; | ||||
} | } | ||||
static void | static void | ||||
vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map) | vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map) | ||||
{ | { | ||||
if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) | if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) | ||||
▲ Show 20 Lines • Show All 948 Lines • ▼ Show 20 Lines | #endif | ||||
} | } | ||||
if (rv == KERN_SUCCESS && is_procstack) | if (rv == KERN_SUCCESS && is_procstack) | ||||
vm->vm_ssize += btoc(grow_amount); | vm->vm_ssize += btoc(grow_amount); | ||||
/* | /* | ||||
* Heed the MAP_WIREFUTURE flag if it was set for this process. | * Heed the MAP_WIREFUTURE flag if it was set for this process. | ||||
*/ | */ | ||||
if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) { | if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) { | ||||
vm_map_unlock(map); | rv = vm_map_wire_locked(map, grow_start, | ||||
vm_map_wire(map, grow_start, grow_start + grow_amount, | grow_start + grow_amount, | ||||
VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); | VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); | ||||
vm_map_lock_read(map); | } | ||||
} else | |||||
vm_map_lock_downgrade(map); | vm_map_lock_downgrade(map); | ||||
out: | out: | ||||
#ifdef RACCT | #ifdef RACCT | ||||
if (racct_enable && rv != KERN_SUCCESS) { | if (racct_enable && rv != KERN_SUCCESS) { | ||||
PROC_LOCK(p); | PROC_LOCK(p); | ||||
error = racct_set(p, RACCT_VMEM, map->size); | error = racct_set(p, RACCT_VMEM, map->size); | ||||
KASSERT(error == 0, ("decreasing RACCT_VMEM failed")); | KASSERT(error == 0, ("decreasing RACCT_VMEM failed")); | ||||
if (!old_mlock) { | if (!old_mlock) { | ||||
▲ Show 20 Lines • Show All 491 Lines • Show Last 20 Lines |