Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 418 Lines • ▼ Show 20 Lines | sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS) | ||||
error = sbuf_finish(&sbuf); | error = sbuf_finish(&sbuf); | ||||
sbuf_delete(&sbuf); | sbuf_delete(&sbuf); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Initialize a dummy page for use in scans of the specified paging queue. | * Initialize a dummy page for use in scans of the specified paging queue. | ||||
* In principle, this function only needs to set the flag PG_MARKER. | * In principle, this function only needs to set the flag PG_MARKER. | ||||
* Nonetheless, it write busies and initializes the hold count to one as | * Nonetheless, it write busies the page as a safety precaution. | ||||
* safety precautions. | |||||
*/ | */ | ||||
static void | static void | ||||
vm_page_init_marker(vm_page_t marker, int queue, uint8_t aflags) | vm_page_init_marker(vm_page_t marker, int queue, uint8_t aflags) | ||||
{ | { | ||||
bzero(marker, sizeof(*marker)); | bzero(marker, sizeof(*marker)); | ||||
marker->flags = PG_MARKER; | marker->flags = PG_MARKER; | ||||
marker->aflags = aflags; | marker->aflags = aflags; | ||||
marker->busy_lock = VPB_SINGLE_EXCLUSIVER; | marker->busy_lock = VPB_SINGLE_EXCLUSIVER; | ||||
marker->queue = queue; | marker->queue = queue; | ||||
marker->hold_count = 1; | |||||
} | } | ||||
static void | static void | ||||
vm_page_domain_init(int domain) | vm_page_domain_init(int domain) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
struct vm_pagequeue *pq; | struct vm_pagequeue *pq; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
static void | static void | ||||
vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind) | vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind) | ||||
{ | { | ||||
m->object = NULL; | m->object = NULL; | ||||
m->wire_count = 0; | m->wire_count = 0; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
m->hold_count = 0; | |||||
m->flags = m->aflags = 0; | m->flags = m->aflags = 0; | ||||
m->phys_addr = pa; | m->phys_addr = pa; | ||||
m->queue = PQ_NONE; | m->queue = PQ_NONE; | ||||
m->psind = 0; | m->psind = 0; | ||||
m->segind = segind; | m->segind = segind; | ||||
m->order = VM_NFREEORDER; | m->order = VM_NFREEORDER; | ||||
m->pool = VM_FREEPOOL_DEFAULT; | m->pool = VM_FREEPOOL_DEFAULT; | ||||
m->valid = m->dirty = 0; | m->valid = m->dirty = 0; | ||||
▲ Show 20 Lines • Show All 566 Lines • ▼ Show 20 Lines | if (*mtx == mtx1) | ||||
return; | return; | ||||
if (*mtx != NULL) | if (*mtx != NULL) | ||||
mtx_unlock(*mtx); | mtx_unlock(*mtx); | ||||
*mtx = mtx1; | *mtx = mtx1; | ||||
mtx_lock(mtx1); | mtx_lock(mtx1); | ||||
} | } | ||||
/* | /* | ||||
* Keep page from being freed by the page daemon | |||||
* much of the same effect as wiring, except much lower | |||||
* overhead and should be used only for *very* temporary | |||||
* holding ("wiring"). | |||||
*/ | |||||
void | |||||
vm_page_hold(vm_page_t mem) | |||||
{ | |||||
vm_page_lock_assert(mem, MA_OWNED); | |||||
mem->hold_count++; | |||||
} | |||||
void | |||||
vm_page_unhold(vm_page_t mem) | |||||
{ | |||||
vm_page_lock_assert(mem, MA_OWNED); | |||||
KASSERT(mem->hold_count >= 1, ("vm_page_unhold: hold count < 0!!!")); | |||||
--mem->hold_count; | |||||
if (mem->hold_count == 0 && (mem->flags & PG_UNHOLDFREE) != 0) | |||||
vm_page_free_toq(mem); | |||||
} | |||||
/* | |||||
* vm_page_unhold_pages: | * vm_page_unhold_pages: | ||||
* | * | ||||
* Unhold each of the pages that is referenced by the given array. | * Unhold each of the pages that is referenced by the given array. | ||||
*/ | */ | ||||
void | void | ||||
vm_page_unhold_pages(vm_page_t *ma, int count) | vm_page_unhold_pages(vm_page_t *ma, int count) | ||||
{ | { | ||||
struct mtx *mtx; | struct mtx *mtx; | ||||
mtx = NULL; | mtx = NULL; | ||||
for (; count != 0; count--) { | for (; count != 0; count--) { | ||||
vm_page_change_lock(*ma, &mtx); | vm_page_change_lock(*ma, &mtx); | ||||
vm_page_unhold(*ma); | if (vm_page_unwire(*ma, PQ_ACTIVE) && (*ma)->object == NULL) | ||||
vm_page_free(*ma); | |||||
ma++; | ma++; | ||||
} | } | ||||
if (mtx != NULL) | if (mtx != NULL) | ||||
mtx_unlock(mtx); | mtx_unlock(mtx); | ||||
} | } | ||||
vm_page_t | vm_page_t | ||||
PHYS_TO_VM_PAGE(vm_paddr_t pa) | PHYS_TO_VM_PAGE(vm_paddr_t pa) | ||||
▲ Show 20 Lines • Show All 445 Lines • ▼ Show 20 Lines | |||||
vm_page_t | vm_page_t | ||||
vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex) | vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex) | ||||
{ | { | ||||
vm_page_t mold; | vm_page_t mold; | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT(mnew->object == NULL, | KASSERT(mnew->object == NULL, | ||||
("vm_page_replace: page %p already in object", mnew)); | ("vm_page_replace: page %p already in object", mnew)); | ||||
KASSERT(mnew->queue == PQ_NONE, | KASSERT(mnew->queue == PQ_NONE || vm_page_wired(mnew), | ||||
("vm_page_replace: new page %p is on a paging queue", mnew)); | ("vm_page_replace: new page %p is on a paging queue", mnew)); | ||||
/* | /* | ||||
* This function mostly follows vm_page_insert() and | * This function mostly follows vm_page_insert() and | ||||
* vm_page_remove() without the radix, object count and vnode | * vm_page_remove() without the radix, object count and vnode | ||||
* dance. Double check such functions for more comments. | * dance. Double check such functions for more comments. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 531 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
vm_page_alloc_check(vm_page_t m) | vm_page_alloc_check(vm_page_t m) | ||||
{ | { | ||||
KASSERT(m->object == NULL, ("page %p has object", m)); | KASSERT(m->object == NULL, ("page %p has object", m)); | ||||
KASSERT(m->queue == PQ_NONE && (m->aflags & PGA_QUEUE_STATE_MASK) == 0, | KASSERT(m->queue == PQ_NONE && (m->aflags & PGA_QUEUE_STATE_MASK) == 0, | ||||
("page %p has unexpected queue %d, flags %#x", | ("page %p has unexpected queue %d, flags %#x", | ||||
m, m->queue, (m->aflags & PGA_QUEUE_STATE_MASK))); | m, m->queue, (m->aflags & PGA_QUEUE_STATE_MASK))); | ||||
KASSERT(!vm_page_held(m), ("page %p is held", m)); | KASSERT(!vm_page_wired(m), ("page %p is wired", m)); | ||||
KASSERT(!vm_page_busied(m), ("page %p is busy", m)); | KASSERT(!vm_page_busied(m), ("page %p is busy", m)); | ||||
KASSERT(m->dirty == 0, ("page %p is dirty", m)); | KASSERT(m->dirty == 0, ("page %p is dirty", m)); | ||||
KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, | KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, | ||||
("page %p has unexpected memattr %d", | ("page %p has unexpected memattr %d", | ||||
m, pmap_page_get_memattr(m))); | m, pmap_page_get_memattr(m))); | ||||
KASSERT(m->valid == 0, ("free page %p is valid", m)); | KASSERT(m->valid == 0, ("free page %p is valid", m)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 186 Lines • ▼ Show 20 Lines | if (run_len == 0) { | ||||
continue; | continue; | ||||
} | } | ||||
} else | } else | ||||
KASSERT(m_run != NULL, ("m_run == NULL")); | KASSERT(m_run != NULL, ("m_run == NULL")); | ||||
vm_page_change_lock(m, &m_mtx); | vm_page_change_lock(m, &m_mtx); | ||||
m_inc = 1; | m_inc = 1; | ||||
retry: | retry: | ||||
if (vm_page_held(m)) | if (vm_page_wired(m)) | ||||
run_ext = 0; | run_ext = 0; | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
else if ((level = vm_reserv_level(m)) >= 0 && | else if ((level = vm_reserv_level(m)) >= 0 && | ||||
(options & VPSC_NORESERV) != 0) { | (options & VPSC_NORESERV) != 0) { | ||||
run_ext = 0; | run_ext = 0; | ||||
/* Advance to the end of the reservation. */ | /* Advance to the end of the reservation. */ | ||||
pa = VM_PAGE_TO_PHYS(m); | pa = VM_PAGE_TO_PHYS(m); | ||||
m_inc = atop(roundup2(pa + 1, vm_reserv_size(level)) - | m_inc = atop(roundup2(pa + 1, vm_reserv_size(level)) - | ||||
Show All 11 Lines | else if ((object = m->object) != NULL) { | ||||
VM_OBJECT_RLOCK(object); | VM_OBJECT_RLOCK(object); | ||||
mtx_lock(m_mtx); | mtx_lock(m_mtx); | ||||
if (m->object != object) { | if (m->object != object) { | ||||
/* | /* | ||||
* The page may have been freed. | * The page may have been freed. | ||||
*/ | */ | ||||
VM_OBJECT_RUNLOCK(object); | VM_OBJECT_RUNLOCK(object); | ||||
goto retry; | goto retry; | ||||
} else if (vm_page_held(m)) { | } else if (vm_page_wired(m)) { | ||||
run_ext = 0; | run_ext = 0; | ||||
goto unlock; | goto unlock; | ||||
} | } | ||||
} | } | ||||
KASSERT((m->flags & PG_UNHOLDFREE) == 0, | |||||
("page %p is PG_UNHOLDFREE", m)); | |||||
/* Don't care: PG_NODUMP, PG_ZERO. */ | /* Don't care: PG_NODUMP, PG_ZERO. */ | ||||
if (object->type != OBJT_DEFAULT && | if (object->type != OBJT_DEFAULT && | ||||
object->type != OBJT_SWAP && | object->type != OBJT_SWAP && | ||||
object->type != OBJT_VNODE) { | object->type != OBJT_VNODE) { | ||||
run_ext = 0; | run_ext = 0; | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
} else if ((options & VPSC_NOSUPER) != 0 && | } else if ((options & VPSC_NOSUPER) != 0 && | ||||
(level = vm_reserv_level_iffullpop(m)) >= 0) { | (level = vm_reserv_level_iffullpop(m)) >= 0) { | ||||
▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines | for (; error == 0 && m < m_end; m++) { | ||||
KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0, | KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0, | ||||
("page %p is PG_FICTITIOUS or PG_MARKER", m)); | ("page %p is PG_FICTITIOUS or PG_MARKER", m)); | ||||
/* | /* | ||||
* Avoid releasing and reacquiring the same page lock. | * Avoid releasing and reacquiring the same page lock. | ||||
*/ | */ | ||||
vm_page_change_lock(m, &m_mtx); | vm_page_change_lock(m, &m_mtx); | ||||
retry: | retry: | ||||
if (vm_page_held(m)) | if (vm_page_wired(m)) | ||||
error = EBUSY; | error = EBUSY; | ||||
else if ((object = m->object) != NULL) { | else if ((object = m->object) != NULL) { | ||||
/* | /* | ||||
* The page is relocated if and only if it could be | * The page is relocated if and only if it could be | ||||
* laundered or reclaimed by the page daemon. | * laundered or reclaimed by the page daemon. | ||||
*/ | */ | ||||
if (!VM_OBJECT_TRYWLOCK(object)) { | if (!VM_OBJECT_TRYWLOCK(object)) { | ||||
mtx_unlock(m_mtx); | mtx_unlock(m_mtx); | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
mtx_lock(m_mtx); | mtx_lock(m_mtx); | ||||
if (m->object != object) { | if (m->object != object) { | ||||
/* | /* | ||||
* The page may have been freed. | * The page may have been freed. | ||||
*/ | */ | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
goto retry; | goto retry; | ||||
} else if (vm_page_held(m)) { | } else if (vm_page_wired(m)) { | ||||
error = EBUSY; | error = EBUSY; | ||||
goto unlock; | goto unlock; | ||||
} | } | ||||
} | } | ||||
KASSERT((m->flags & PG_UNHOLDFREE) == 0, | |||||
("page %p is PG_UNHOLDFREE", m)); | |||||
/* Don't care: PG_NODUMP, PG_ZERO. */ | /* Don't care: PG_NODUMP, PG_ZERO. */ | ||||
if (object->type != OBJT_DEFAULT && | if (object->type != OBJT_DEFAULT && | ||||
object->type != OBJT_SWAP && | object->type != OBJT_SWAP && | ||||
object->type != OBJT_VNODE) | object->type != OBJT_VNODE) | ||||
error = EINVAL; | error = EINVAL; | ||||
else if (object->memattr != VM_MEMATTR_DEFAULT) | else if (object->memattr != VM_MEMATTR_DEFAULT) | ||||
error = EINVAL; | error = EINVAL; | ||||
else if (vm_page_queue(m) != PQ_NONE && | else if (vm_page_queue(m) != PQ_NONE && | ||||
▲ Show 20 Lines • Show All 916 Lines • ▼ Show 20 Lines | #endif | ||||
if ((m->oflags & VPO_UNMANAGED) == 0) | if ((m->oflags & VPO_UNMANAGED) == 0) | ||||
vm_page_dequeue_deferred_free(m); | vm_page_dequeue_deferred_free(m); | ||||
m->valid = 0; | m->valid = 0; | ||||
vm_page_undirty(m); | vm_page_undirty(m); | ||||
if (vm_page_wired(m) != 0) | if (vm_page_wired(m) != 0) | ||||
panic("vm_page_free_prep: freeing wired page %p", m); | panic("vm_page_free_prep: freeing wired page %p", m); | ||||
if (m->hold_count != 0) { | |||||
m->flags &= ~PG_ZERO; | |||||
KASSERT((m->flags & PG_UNHOLDFREE) == 0, | |||||
("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m)); | |||||
m->flags |= PG_UNHOLDFREE; | |||||
return (false); | |||||
} | |||||
/* | /* | ||||
* Restore the default memory attribute to the page. | * Restore the default memory attribute to the page. | ||||
*/ | */ | ||||
if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT) | if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT) | ||||
pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT); | pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
▲ Show 20 Lines • Show All 293 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
bool | bool | ||||
vm_page_try_to_free(vm_page_t m) | vm_page_try_to_free(vm_page_t m) | ||||
{ | { | ||||
vm_page_assert_locked(m); | vm_page_assert_locked(m); | ||||
VM_OBJECT_ASSERT_WLOCKED(m->object); | VM_OBJECT_ASSERT_WLOCKED(m->object); | ||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("page %p is unmanaged", m)); | KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("page %p is unmanaged", m)); | ||||
if (m->dirty != 0 || vm_page_held(m) || vm_page_busied(m)) | if (m->dirty != 0 || vm_page_wired(m) || vm_page_busied(m)) | ||||
return (false); | return (false); | ||||
if (m->object->ref_count != 0) { | if (m->object->ref_count != 0) { | ||||
pmap_remove_all(m); | pmap_remove_all(m); | ||||
if (m->dirty != 0) | if (m->dirty != 0) | ||||
return (false); | return (false); | ||||
} | } | ||||
vm_page_free(m); | vm_page_free(m); | ||||
return (true); | return (true); | ||||
▲ Show 20 Lines • Show All 723 Lines • ▼ Show 20 Lines | DB_SHOW_COMMAND(pginfo, vm_page_print_pginfo) | ||||
virt = strchr(modif, 'v') != NULL; | virt = strchr(modif, 'v') != NULL; | ||||
if (virt) | if (virt) | ||||
m = PHYS_TO_VM_PAGE(pmap_kextract(addr)); | m = PHYS_TO_VM_PAGE(pmap_kextract(addr)); | ||||
else if (phys) | else if (phys) | ||||
m = PHYS_TO_VM_PAGE(addr); | m = PHYS_TO_VM_PAGE(addr); | ||||
else | else | ||||
m = (vm_page_t)addr; | m = (vm_page_t)addr; | ||||
db_printf( | db_printf( | ||||
"page %p obj %p pidx 0x%jx phys 0x%jx q %d hold %d wire %d\n" | "page %p obj %p pidx 0x%jx phys 0x%jx q %d wire %d\n" | ||||
" af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n", | " af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n", | ||||
m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr, | m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr, | ||||
m->queue, m->hold_count, m->wire_count, m->aflags, m->oflags, | m->queue, m->wire_count, m->aflags, m->oflags, | ||||
m->flags, m->act_count, m->busy_lock, m->valid, m->dirty); | m->flags, m->act_count, m->busy_lock, m->valid, m->dirty); | ||||
} | } | ||||
#endif /* DDB */ | #endif /* DDB */ |