Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_page.c
Show First 20 Lines • Show All 856 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
vm_page_reference(vm_page_t m) | vm_page_reference(vm_page_t m) | ||||
{ | { | ||||
vm_page_aflag_set(m, PGA_REFERENCED); | vm_page_aflag_set(m, PGA_REFERENCED); | ||||
} | } | ||||
static bool | |||||
vm_page_acquire_flags(vm_page_t m, int allocflags) | |||||
{ | |||||
bool locked; | |||||
if ((allocflags & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) != 0) | |||||
locked = vm_page_trysbusy(m); | |||||
else | |||||
locked = vm_page_tryxbusy(m); | |||||
if (locked && (allocflags & VM_ALLOC_WIRED) != 0) | |||||
vm_page_wire(m); | |||||
return (locked); | |||||
} | |||||
static bool | |||||
vm_page_busy_sleep_flags(vm_object_t object, vm_page_t m, const char *wchan, | |||||
int allocflags) | |||||
{ | |||||
if ((allocflags & VM_ALLOC_NOWAIT) != 0) | |||||
return (false); | |||||
/* | /* | ||||
* Reference the page before unlocking and | |||||
* sleeping so that the page daemon is less | |||||
* likely to reclaim it. | |||||
*/ | |||||
if ((allocflags & VM_ALLOC_NOCREAT) == 0) | |||||
vm_page_aflag_set(m, PGA_REFERENCED); | |||||
vm_page_busy_sleep(m, wchan, (allocflags & | |||||
VM_ALLOC_IGN_SBUSY) != 0); | |||||
VM_OBJECT_WLOCK(object); | |||||
if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | |||||
return (false); | |||||
return (true); | |||||
} | |||||
/* | |||||
* vm_page_busy_acquire: | * vm_page_busy_acquire: | ||||
* | * | ||||
* Acquire the busy lock as described by VM_ALLOC_* flags. Will loop | * Acquire the busy lock as described by VM_ALLOC_* flags. Will loop | ||||
* and drop the object lock if necessary. | * and drop the object lock if necessary. | ||||
*/ | */ | ||||
int | bool | ||||
vm_page_busy_acquire(vm_page_t m, int allocflags) | vm_page_busy_acquire(vm_page_t m, int allocflags) | ||||
{ | { | ||||
vm_object_t obj; | vm_object_t obj; | ||||
bool locked; | bool locked; | ||||
/* | /* | ||||
* The page-specific object must be cached because page | * The page-specific object must be cached because page | ||||
* identity can change during the sleep, causing the | * identity can change during the sleep, causing the | ||||
* re-lock of a different object. | * re-lock of a different object. | ||||
* It is assumed that a reference to the object is already | * It is assumed that a reference to the object is already | ||||
* held by the callers. | * held by the callers. | ||||
*/ | */ | ||||
obj = m->object; | obj = m->object; | ||||
for (;;) { | for (;;) { | ||||
if ((allocflags & VM_ALLOC_SBUSY) == 0) { | if (vm_page_acquire_flags(m, allocflags)) | ||||
if (vm_page_tryxbusy(m)) | return (true); | ||||
return (TRUE); | |||||
} else { | |||||
if (vm_page_trysbusy(m)) | |||||
return (TRUE); | |||||
} | |||||
if ((allocflags & VM_ALLOC_NOWAIT) != 0) | if ((allocflags & VM_ALLOC_NOWAIT) != 0) | ||||
return (FALSE); | return (false); | ||||
if (obj != NULL) | if (obj != NULL) | ||||
locked = VM_OBJECT_WOWNED(obj); | locked = VM_OBJECT_WOWNED(obj); | ||||
else | else | ||||
locked = FALSE; | locked = false; | ||||
MPASS(locked || vm_page_wired(m)); | MPASS(locked || vm_page_wired(m)); | ||||
_vm_page_busy_sleep(obj, m, "vmpba", | _vm_page_busy_sleep(obj, m, "vmpba", | ||||
(allocflags & VM_ALLOC_SBUSY) != 0, locked); | (allocflags & VM_ALLOC_SBUSY) != 0, locked); | ||||
if (locked) | if (locked) | ||||
VM_OBJECT_WLOCK(obj); | VM_OBJECT_WLOCK(obj); | ||||
if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | ||||
return (FALSE); | return (false); | ||||
KASSERT(m->object == obj || m->object == NULL, | KASSERT(m->object == obj || m->object == NULL, | ||||
("vm_page_busy_acquire: page %p does not belong to %p", | ("vm_page_busy_acquire: page %p does not belong to %p", | ||||
m, obj)); | m, obj)); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* vm_page_busy_downgrade: | * vm_page_busy_downgrade: | ||||
▲ Show 20 Lines • Show All 3,308 Lines • ▼ Show 20 Lines | vm_page_advise(vm_page_t m, int advice) | ||||
* laundry are moved there. | * laundry are moved there. | ||||
*/ | */ | ||||
if (m->dirty == 0) | if (m->dirty == 0) | ||||
vm_page_deactivate_noreuse(m); | vm_page_deactivate_noreuse(m); | ||||
else if (!vm_page_in_laundry(m)) | else if (!vm_page_in_laundry(m)) | ||||
vm_page_launder(m); | vm_page_launder(m); | ||||
} | } | ||||
static inline int | |||||
vm_page_grab_pflags(int allocflags) | |||||
{ | |||||
int pflags; | |||||
KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 || | |||||
(allocflags & VM_ALLOC_WIRED) != 0, | |||||
("vm_page_grab_pflags: the pages must be busied or wired")); | |||||
KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || | |||||
(allocflags & VM_ALLOC_IGN_SBUSY) != 0, | |||||
("vm_page_grab_pflags: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY " | |||||
"mismatch")); | |||||
pflags = allocflags & | |||||
~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL | | |||||
VM_ALLOC_NOBUSY); | |||||
if ((allocflags & VM_ALLOC_NOWAIT) == 0) | |||||
pflags |= VM_ALLOC_WAITFAIL; | |||||
if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) | |||||
pflags |= VM_ALLOC_SBUSY; | |||||
return (pflags); | |||||
} | |||||
/* | /* | ||||
* Grab a page, waiting until we are waken up due to the page | * Grab a page, waiting until we are waken up due to the page | ||||
* changing state. We keep on waiting, if the page continues | * changing state. We keep on waiting, if the page continues | ||||
* to be in the object. If the page doesn't exist, first allocate it | * to be in the object. If the page doesn't exist, first allocate it | ||||
* and then conditionally zero it. | * and then conditionally zero it. | ||||
* | * | ||||
* This routine may sleep. | * This routine may sleep. | ||||
* | * | ||||
* The object must be locked on entry. The lock will, however, be released | * The object must be locked on entry. The lock will, however, be released | ||||
* and reacquired if the routine sleeps. | * and reacquired if the routine sleeps. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) | vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
int sleep; | |||||
int pflags; | int pflags; | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || | pflags = vm_page_grab_pflags(allocflags); | ||||
(allocflags & VM_ALLOC_IGN_SBUSY) != 0, | |||||
("vm_page_grab: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch")); | |||||
pflags = allocflags & | |||||
~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL | | |||||
VM_ALLOC_NOBUSY); | |||||
if ((allocflags & VM_ALLOC_NOWAIT) == 0) | |||||
pflags |= VM_ALLOC_WAITFAIL; | |||||
if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) | |||||
pflags |= VM_ALLOC_SBUSY; | |||||
retrylookup: | retrylookup: | ||||
if ((m = vm_page_lookup(object, pindex)) != NULL) { | if ((m = vm_page_lookup(object, pindex)) != NULL) { | ||||
if ((allocflags & (VM_ALLOC_IGN_SBUSY | VM_ALLOC_SBUSY)) != 0) | if (!vm_page_acquire_flags(m, allocflags)) { | ||||
sleep = !vm_page_trysbusy(m); | if (vm_page_busy_sleep_flags(object, m, "pgrbwt", | ||||
else | allocflags)) | ||||
sleep = !vm_page_tryxbusy(m); | |||||
if (sleep) { | |||||
if ((allocflags & VM_ALLOC_NOWAIT) != 0) | |||||
return (NULL); | |||||
/* | |||||
* Reference the page before unlocking and | |||||
* sleeping so that the page daemon is less | |||||
* likely to reclaim it. | |||||
*/ | |||||
if ((allocflags & VM_ALLOC_NOCREAT) == 0) | |||||
vm_page_aflag_set(m, PGA_REFERENCED); | |||||
vm_page_busy_sleep(m, "pgrbwt", (allocflags & | |||||
VM_ALLOC_IGN_SBUSY) != 0); | |||||
VM_OBJECT_WLOCK(object); | |||||
if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | |||||
return (NULL); | |||||
goto retrylookup; | goto retrylookup; | ||||
} else { | return (NULL); | ||||
if ((allocflags & VM_ALLOC_WIRED) != 0) | } | ||||
vm_page_wire(m); | |||||
goto out; | goto out; | ||||
} | } | ||||
} | |||||
if ((allocflags & VM_ALLOC_NOCREAT) != 0) | if ((allocflags & VM_ALLOC_NOCREAT) != 0) | ||||
return (NULL); | return (NULL); | ||||
m = vm_page_alloc(object, pindex, pflags); | m = vm_page_alloc(object, pindex, pflags); | ||||
if (m == NULL) { | if (m == NULL) { | ||||
if ((allocflags & VM_ALLOC_NOWAIT) != 0) | if ((allocflags & VM_ALLOC_NOWAIT) != 0) | ||||
return (NULL); | return (NULL); | ||||
goto retrylookup; | goto retrylookup; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | if ((m = vm_page_lookup(object, pindex)) != NULL) { | ||||
*/ | */ | ||||
if (!vm_page_all_valid(m) || | if (!vm_page_all_valid(m) || | ||||
(allocflags & (VM_ALLOC_IGN_SBUSY | VM_ALLOC_SBUSY)) == 0) { | (allocflags & (VM_ALLOC_IGN_SBUSY | VM_ALLOC_SBUSY)) == 0) { | ||||
sleep = !vm_page_tryxbusy(m); | sleep = !vm_page_tryxbusy(m); | ||||
xbusy = true; | xbusy = true; | ||||
} else | } else | ||||
sleep = !vm_page_trysbusy(m); | sleep = !vm_page_trysbusy(m); | ||||
if (sleep) { | if (sleep) { | ||||
/* | (void)vm_page_busy_sleep_flags(object, m, "pgrbwt", | ||||
* Reference the page before unlocking and | allocflags); | ||||
* sleeping so that the page daemon is less | |||||
* likely to reclaim it. | |||||
*/ | |||||
if ((allocflags & VM_ALLOC_NOCREAT) == 0) | |||||
vm_page_aflag_set(m, PGA_REFERENCED); | |||||
vm_page_busy_sleep(m, "pgrbwt", (allocflags & | |||||
VM_ALLOC_IGN_SBUSY) != 0); | |||||
VM_OBJECT_WLOCK(object); | |||||
goto retrylookup; | goto retrylookup; | ||||
} | } | ||||
if ((allocflags & VM_ALLOC_NOCREAT) != 0 && | if ((allocflags & VM_ALLOC_NOCREAT) != 0 && | ||||
!vm_page_all_valid(m)) { | !vm_page_all_valid(m)) { | ||||
if (xbusy) | if (xbusy) | ||||
vm_page_xunbusy(m); | vm_page_xunbusy(m); | ||||
else | else | ||||
vm_page_sunbusy(m); | vm_page_sunbusy(m); | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
int | int | ||||
vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, | vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, | ||||
vm_page_t *ma, int count) | vm_page_t *ma, int count) | ||||
{ | { | ||||
vm_page_t m, mpred; | vm_page_t m, mpred; | ||||
int pflags; | int pflags; | ||||
int i; | int i; | ||||
bool sleep; | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0, | KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0, | ||||
("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed")); | ("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed")); | ||||
KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 || | |||||
(allocflags & VM_ALLOC_WIRED) != 0, | pflags = vm_page_grab_pflags(allocflags); | ||||
("vm_page_grab_pages: the pages must be busied or wired")); | |||||
KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || | |||||
(allocflags & VM_ALLOC_IGN_SBUSY) != 0, | |||||
("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch")); | |||||
if (count == 0) | if (count == 0) | ||||
return (0); | return (0); | ||||
pflags = allocflags & | |||||
~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL | | |||||
VM_ALLOC_NOBUSY); | |||||
if ((allocflags & VM_ALLOC_NOWAIT) == 0) | |||||
pflags |= VM_ALLOC_WAITFAIL; | |||||
if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) | |||||
pflags |= VM_ALLOC_SBUSY; | |||||
i = 0; | i = 0; | ||||
retrylookup: | retrylookup: | ||||
m = vm_radix_lookup_le(&object->rtree, pindex + i); | m = vm_radix_lookup_le(&object->rtree, pindex + i); | ||||
if (m == NULL || m->pindex != pindex + i) { | if (m == NULL || m->pindex != pindex + i) { | ||||
mpred = m; | mpred = m; | ||||
m = NULL; | m = NULL; | ||||
} else | } else | ||||
mpred = TAILQ_PREV(m, pglist, listq); | mpred = TAILQ_PREV(m, pglist, listq); | ||||
for (; i < count; i++) { | for (; i < count; i++) { | ||||
if (m != NULL) { | if (m != NULL) { | ||||
if ((allocflags & | if (!vm_page_acquire_flags(m, allocflags)) { | ||||
(VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) != 0) | if (vm_page_busy_sleep_flags(object, m, | ||||
sleep = !vm_page_trysbusy(m); | "grbmaw", allocflags)) | ||||
else | |||||
sleep = !vm_page_tryxbusy(m); | |||||
if (sleep) { | |||||
if ((allocflags & VM_ALLOC_NOWAIT) != 0) | |||||
break; | |||||
/* | |||||
* Reference the page before unlocking and | |||||
* sleeping so that the page daemon is less | |||||
* likely to reclaim it. | |||||
*/ | |||||
if ((allocflags & VM_ALLOC_NOCREAT) == 0) | |||||
vm_page_aflag_set(m, PGA_REFERENCED); | |||||
vm_page_busy_sleep(m, "grbmaw", (allocflags & | |||||
VM_ALLOC_IGN_SBUSY) != 0); | |||||
VM_OBJECT_WLOCK(object); | |||||
goto retrylookup; | goto retrylookup; | ||||
break; | |||||
} | } | ||||
if ((allocflags & VM_ALLOC_WIRED) != 0) | |||||
vm_page_wire(m); | |||||
} else { | } else { | ||||
if ((allocflags & VM_ALLOC_NOCREAT) != 0) | if ((allocflags & VM_ALLOC_NOCREAT) != 0) | ||||
break; | break; | ||||
m = vm_page_alloc_after(object, pindex + i, | m = vm_page_alloc_after(object, pindex + i, | ||||
pflags | VM_ALLOC_COUNT(count - i), mpred); | pflags | VM_ALLOC_COUNT(count - i), mpred); | ||||
if (m == NULL) { | if (m == NULL) { | ||||
if ((allocflags & VM_ALLOC_NOWAIT) != 0) | if ((allocflags & VM_ALLOC_NOWAIT) != 0) | ||||
break; | break; | ||||
▲ Show 20 Lines • Show All 616 Lines • Show Last 20 Lines |