Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_page.c
Show First 20 Lines • Show All 168 Lines • ▼ Show 20 Lines | |||||
static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); | static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); | ||||
SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | | SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | | ||||
CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages"); | CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages"); | ||||
static uma_zone_t fakepg_zone; | static uma_zone_t fakepg_zone; | ||||
static void vm_page_alloc_check(vm_page_t m); | static void vm_page_alloc_check(vm_page_t m); | ||||
static bool _vm_page_busy_sleep(vm_object_t obj, vm_page_t m, | static bool _vm_page_busy_sleep(vm_object_t obj, vm_page_t m, | ||||
const char *wmesg, bool nonshared, bool locked); | vm_pindex_t pindex, const char *wmesg, int allocflags, bool locked); | ||||
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); | static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); | ||||
static void vm_page_enqueue(vm_page_t m, uint8_t queue); | static void vm_page_enqueue(vm_page_t m, uint8_t queue); | ||||
static bool vm_page_free_prep(vm_page_t m); | static bool vm_page_free_prep(vm_page_t m); | ||||
static void vm_page_free_toq(vm_page_t m); | static void vm_page_free_toq(vm_page_t m); | ||||
static void vm_page_init(void *dummy); | static void vm_page_init(void *dummy); | ||||
static int vm_page_insert_after(vm_page_t m, vm_object_t object, | static int vm_page_insert_after(vm_page_t m, vm_object_t object, | ||||
vm_pindex_t pindex, vm_page_t mpred); | vm_pindex_t pindex, vm_page_t mpred); | ||||
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object, | static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object, | ||||
▲ Show 20 Lines • Show All 655 Lines • ▼ Show 20 Lines | vm_page_acquire_flags(vm_page_t m, int allocflags) | ||||
if (locked && (allocflags & VM_ALLOC_WIRED) != 0) | if (locked && (allocflags & VM_ALLOC_WIRED) != 0) | ||||
vm_page_wire(m); | vm_page_wire(m); | ||||
return (locked); | return (locked); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_busy_sleep_flags | * vm_page_busy_sleep_flags | ||||
* | * | ||||
* Sleep for busy according to VM_ALLOC_ parameters. | * Sleep for busy according to VM_ALLOC_ parameters. Returns true | ||||
* if the caller should retry and false otherwise. | |||||
*/ | */ | ||||
static bool | static bool | ||||
vm_page_busy_sleep_flags(vm_object_t object, vm_page_t m, const char *wmesg, | vm_page_busy_sleep_flags(vm_object_t object, vm_page_t m, const char *wmesg, | ||||
int allocflags) | int allocflags) | ||||
{ | { | ||||
if ((allocflags & VM_ALLOC_NOWAIT) != 0) | if ((allocflags & VM_ALLOC_NOWAIT) != 0) | ||||
return (false); | return (false); | ||||
/* | /* | ||||
* Reference the page before unlocking and | * Reference the page before unlocking and sleeping so that | ||||
* sleeping so that the page daemon is less | * the page daemon is less likely to reclaim it. | ||||
* likely to reclaim it. | |||||
*/ | */ | ||||
if ((allocflags & VM_ALLOC_NOCREAT) == 0) | if ((allocflags & VM_ALLOC_NOCREAT) == 0) | ||||
vm_page_aflag_set(m, PGA_REFERENCED); | vm_page_reference(m); | ||||
if (_vm_page_busy_sleep(object, m, wmesg, (allocflags & | |||||
VM_ALLOC_IGN_SBUSY) != 0, true)) | if (_vm_page_busy_sleep(object, m, m->pindex, wmesg, allocflags, true)) | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | ||||
return (false); | return (false); | ||||
return (true); | return (true); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_busy_acquire: | * vm_page_busy_acquire: | ||||
* | * | ||||
* Acquire the busy lock as described by VM_ALLOC_* flags. Will loop | * Acquire the busy lock as described by VM_ALLOC_* flags. Will loop | ||||
* and drop the object lock if necessary. | * and drop the object lock if necessary. | ||||
Show All 17 Lines | if (vm_page_acquire_flags(m, allocflags)) | ||||
return (true); | return (true); | ||||
if ((allocflags & VM_ALLOC_NOWAIT) != 0) | if ((allocflags & VM_ALLOC_NOWAIT) != 0) | ||||
return (false); | return (false); | ||||
if (obj != NULL) | if (obj != NULL) | ||||
locked = VM_OBJECT_WOWNED(obj); | locked = VM_OBJECT_WOWNED(obj); | ||||
else | else | ||||
locked = false; | locked = false; | ||||
MPASS(locked || vm_page_wired(m)); | MPASS(locked || vm_page_wired(m)); | ||||
if (_vm_page_busy_sleep(obj, m, "vmpba", | if (_vm_page_busy_sleep(obj, m, m->pindex, "vmpba", allocflags, | ||||
(allocflags & VM_ALLOC_SBUSY) != 0, locked)) | locked) && locked) | ||||
VM_OBJECT_WLOCK(obj); | VM_OBJECT_WLOCK(obj); | ||||
if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | ||||
return (false); | return (false); | ||||
KASSERT(m->object == obj || m->object == NULL, | KASSERT(m->object == obj || m->object == NULL, | ||||
("vm_page_busy_acquire: page %p does not belong to %p", | ("vm_page_busy_acquire: page %p does not belong to %p", | ||||
m, obj)); | m, obj)); | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 108 Lines • ▼ Show 20 Lines | |||||
vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared) | vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared) | ||||
{ | { | ||||
vm_object_t obj; | vm_object_t obj; | ||||
obj = m->object; | obj = m->object; | ||||
VM_OBJECT_ASSERT_LOCKED(obj); | VM_OBJECT_ASSERT_LOCKED(obj); | ||||
vm_page_lock_assert(m, MA_NOTOWNED); | vm_page_lock_assert(m, MA_NOTOWNED); | ||||
if (!_vm_page_busy_sleep(obj, m, wmesg, nonshared, true)) | if (!_vm_page_busy_sleep(obj, m, m->pindex, wmesg, | ||||
nonshared ? VM_ALLOC_SBUSY : 0 , true)) | |||||
VM_OBJECT_DROP(obj); | VM_OBJECT_DROP(obj); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_busy_sleep_unlocked: | |||||
* | |||||
* Sleep if the page is busy, using the page pointer as wchan. | |||||
* This is used to implement the hard-path of busying mechanism. | |||||
* | |||||
* If nonshared is true, sleep only if the page is xbusy. | |||||
* | |||||
* The object lock must not be held on entry. The operation will | |||||
* return if the page changes identity. | |||||
*/ | |||||
void | |||||
vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex, | |||||
const char *wmesg, bool nonshared) | |||||
{ | |||||
VM_OBJECT_ASSERT_UNLOCKED(obj); | |||||
vm_page_lock_assert(m, MA_NOTOWNED); | |||||
_vm_page_busy_sleep(obj, m, pindex, wmesg, | |||||
nonshared ? VM_ALLOC_SBUSY : 0, false); | |||||
} | |||||
/* | |||||
* _vm_page_busy_sleep: | * _vm_page_busy_sleep: | ||||
* | * | ||||
* Internal busy sleep function. | * Internal busy sleep function. Verifies the page identity and | ||||
* lockstate against parameters. Returns true if it sleeps and | |||||
* false otherwise. | |||||
* | |||||
* If locked is true the lock will be dropped for any true returns | |||||
* and held for any false returns. | |||||
*/ | */ | ||||
static bool | static bool | ||||
_vm_page_busy_sleep(vm_object_t obj, vm_page_t m, const char *wmesg, | _vm_page_busy_sleep(vm_object_t obj, vm_page_t m, vm_pindex_t pindex, | ||||
bool nonshared, bool locked) | const char *wmesg, int allocflags, bool locked) | ||||
{ | { | ||||
bool xsleep; | |||||
u_int x; | u_int x; | ||||
/* | /* | ||||
* If the object is busy we must wait for that to drain to zero | * If the object is busy we must wait for that to drain to zero | ||||
* before trying the page again. | * before trying the page again. | ||||
*/ | */ | ||||
if (obj != NULL && vm_object_busied(obj)) { | if (obj != NULL && vm_object_busied(obj)) { | ||||
if (locked) | if (locked) | ||||
VM_OBJECT_DROP(obj); | VM_OBJECT_DROP(obj); | ||||
vm_object_busy_wait(obj, wmesg); | vm_object_busy_wait(obj, wmesg); | ||||
return (locked); | return (true); | ||||
} | } | ||||
if (!vm_page_busied(m)) | |||||
return (false); | |||||
xsleep = (allocflags & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) != 0; | |||||
sleepq_lock(m); | sleepq_lock(m); | ||||
x = m->busy_lock; | x = atomic_load_int(&m->busy_lock); | ||||
if (x == VPB_UNBUSIED || (nonshared && (x & VPB_BIT_SHARED) != 0) || | do { | ||||
((x & VPB_BIT_WAITERS) == 0 && | /* | ||||
!atomic_cmpset_int(&m->busy_lock, x, x | VPB_BIT_WAITERS))) { | * If the page changes objects or becomes unlocked we can | ||||
* simply return. | |||||
*/ | |||||
if (x == VPB_UNBUSIED || | |||||
(xsleep && (x & VPB_BIT_SHARED) != 0) || | |||||
m->object != obj || m->pindex != pindex) { | |||||
sleepq_release(m); | sleepq_release(m); | ||||
return (false); | return (false); | ||||
} | } | ||||
if ((x & VPB_BIT_WAITERS) != 0) | |||||
break; | |||||
} while (!atomic_fcmpset_int(&m->busy_lock, &x, x | VPB_BIT_WAITERS)); | |||||
if (locked) | if (locked) | ||||
VM_OBJECT_DROP(obj); | VM_OBJECT_DROP(obj); | ||||
DROP_GIANT(); | DROP_GIANT(); | ||||
sleepq_add(m, NULL, wmesg, 0, 0); | sleepq_add(m, NULL, wmesg, 0, 0); | ||||
sleepq_wait(m, PVM); | sleepq_wait(m, PVM); | ||||
PICKUP_GIANT(); | PICKUP_GIANT(); | ||||
return (locked); | return (true); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_trysbusy: | * vm_page_trysbusy: | ||||
* | * | ||||
* Try to shared busy a page. | * Try to shared busy a page. | ||||
* If the operation succeeds 1 is returned otherwise 0. | * If the operation succeeds 1 is returned otherwise 0. | ||||
* The operation never sleeps. | * The operation never sleeps. | ||||
▲ Show 20 Lines • Show All 261 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* Sleep and release the object lock if the page is busied. | * Sleep and release the object lock if the page is busied. | ||||
* Returns TRUE if the thread slept. | * Returns TRUE if the thread slept. | ||||
* | * | ||||
* The given page must be unlocked and object containing it must | * The given page must be unlocked and object containing it must | ||||
* be locked. | * be locked. | ||||
*/ | */ | ||||
int | int | ||||
vm_page_sleep_if_busy(vm_page_t m, const char *msg) | vm_page_sleep_if_busy(vm_page_t m, const char *wmesg) | ||||
{ | { | ||||
vm_object_t obj; | vm_object_t obj; | ||||
vm_page_lock_assert(m, MA_NOTOWNED); | vm_page_lock_assert(m, MA_NOTOWNED); | ||||
VM_OBJECT_ASSERT_WLOCKED(m->object); | VM_OBJECT_ASSERT_WLOCKED(m->object); | ||||
/* | /* | ||||
* The page-specific object must be cached because page | * The page-specific object must be cached because page | ||||
* identity can change during the sleep, causing the | * identity can change during the sleep, causing the | ||||
* re-lock of a different object. | * re-lock of a different object. | ||||
* It is assumed that a reference to the object is already | * It is assumed that a reference to the object is already | ||||
* held by the callers. | * held by the callers. | ||||
*/ | */ | ||||
obj = m->object; | obj = m->object; | ||||
if (vm_page_busied(m) || (obj != NULL && obj->busy)) { | if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, 0, true)) { | ||||
vm_page_busy_sleep(m, msg, false); | |||||
VM_OBJECT_WLOCK(obj); | VM_OBJECT_WLOCK(obj); | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_sleep_if_xbusy: | * vm_page_sleep_if_xbusy: | ||||
* | * | ||||
* Sleep and release the object lock if the page is xbusied. | * Sleep and release the object lock if the page is xbusied. | ||||
* Returns TRUE if the thread slept. | * Returns TRUE if the thread slept. | ||||
* | * | ||||
* The given page must be unlocked and object containing it must | * The given page must be unlocked and object containing it must | ||||
* be locked. | * be locked. | ||||
*/ | */ | ||||
int | int | ||||
vm_page_sleep_if_xbusy(vm_page_t m, const char *msg) | vm_page_sleep_if_xbusy(vm_page_t m, const char *wmesg) | ||||
{ | { | ||||
vm_object_t obj; | vm_object_t obj; | ||||
vm_page_lock_assert(m, MA_NOTOWNED); | vm_page_lock_assert(m, MA_NOTOWNED); | ||||
VM_OBJECT_ASSERT_WLOCKED(m->object); | VM_OBJECT_ASSERT_WLOCKED(m->object); | ||||
/* | /* | ||||
* The page-specific object must be cached because page | * The page-specific object must be cached because page | ||||
* identity can change during the sleep, causing the | * identity can change during the sleep, causing the | ||||
* re-lock of a different object. | * re-lock of a different object. | ||||
* It is assumed that a reference to the object is already | * It is assumed that a reference to the object is already | ||||
* held by the callers. | * held by the callers. | ||||
*/ | */ | ||||
obj = m->object; | obj = m->object; | ||||
if (vm_page_xbusied(m) || (obj != NULL && obj->busy)) { | if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, VM_ALLOC_SBUSY, | ||||
vm_page_busy_sleep(m, msg, true); | true)) { | ||||
VM_OBJECT_WLOCK(obj); | VM_OBJECT_WLOCK(obj); | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_dirty_KBI: [ internal use only ] | * vm_page_dirty_KBI: [ internal use only ] | ||||
▲ Show 20 Lines • Show All 3,760 Lines • Show Last 20 Lines |