Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_page.c
Show First 20 Lines • Show All 1,480 Lines • ▼ Show 20 Lines | |||||
* This routine may not sleep. | * This routine may not sleep. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) | vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) | ||||
{ | { | ||||
vm_page_t m, mpred; | vm_page_t m, mpred; | ||||
int flags, req_class; | int flags, req_class; | ||||
mpred = 0; /* XXX: pacify gcc */ | mpred = NULL; /* XXX: pacify gcc */ | ||||
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | ||||
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | ||||
((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ||||
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | ||||
("vm_page_alloc: inconsistent object(%p)/req(%x)", (void *)object, | ("vm_page_alloc: inconsistent object(%p)/req(%x)", object, req)); | ||||
req)); | |||||
if (object != NULL) | if (object != NULL) | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
req_class = req & VM_ALLOC_CLASS_MASK; | req_class = req & VM_ALLOC_CLASS_MASK; | ||||
/* | /* | ||||
* The page daemon is allowed to dig deeper into the free page list. | * The page daemon is allowed to dig deeper into the free page list. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines | #endif | ||||
if (object != NULL) { | if (object != NULL) { | ||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
pagedaemon_wakeup(); | pagedaemon_wakeup(); | ||||
if (req & VM_ALLOC_WIRED) { | if (req & VM_ALLOC_WIRED) { | ||||
atomic_subtract_int(&vm_cnt.v_wire_count, 1); | atomic_subtract_int(&vm_cnt.v_wire_count, 1); | ||||
m->wire_count = 0; | m->wire_count = 0; | ||||
} | } | ||||
m->object = NULL; | KASSERT(m->object == NULL, ("page %p has object", m)); | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
vm_page_free(m); | /* Don't change PG_ZERO. */ | ||||
vm_page_free_toq(m); | |||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* Ignore device objects; the pager sets "memattr" for them. */ | /* Ignore device objects; the pager sets "memattr" for them. */ | ||||
if (object->memattr != VM_MEMATTR_DEFAULT && | if (object->memattr != VM_MEMATTR_DEFAULT && | ||||
(object->flags & OBJ_FICTITIOUS) == 0) | (object->flags & OBJ_FICTITIOUS) == 0) | ||||
pmap_page_set_memattr(m, object->memattr); | pmap_page_set_memattr(m, object->memattr); | ||||
} else | } else | ||||
Show All 25 Lines | |||||
* then the memory attribute setting for the physical pages is configured | * then the memory attribute setting for the physical pages is configured | ||||
* to the object's memory attribute setting. Otherwise, the memory | * to the object's memory attribute setting. Otherwise, the memory | ||||
* attribute setting for the physical pages is configured to "memattr", | * attribute setting for the physical pages is configured to "memattr", | ||||
* overriding the object's memory attribute setting. However, if the | * overriding the object's memory attribute setting. However, if the | ||||
* object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the | * object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the | ||||
* memory attribute setting for the physical pages cannot be configured | * memory attribute setting for the physical pages cannot be configured | ||||
* to VM_MEMATTR_DEFAULT. | * to VM_MEMATTR_DEFAULT. | ||||
* | * | ||||
* The specified object may not contain fictitious pages. | |||||
* | |||||
* The caller must always specify an allocation class. | * The caller must always specify an allocation class. | ||||
* | * | ||||
* allocation classes: | * allocation classes: | ||||
* VM_ALLOC_NORMAL normal process request | * VM_ALLOC_NORMAL normal process request | ||||
* VM_ALLOC_SYSTEM system *really* needs a page | * VM_ALLOC_SYSTEM system *really* needs a page | ||||
* VM_ALLOC_INTERRUPT interrupt time request | * VM_ALLOC_INTERRUPT interrupt time request | ||||
* | * | ||||
* optional allocation flags: | * optional allocation flags: | ||||
* VM_ALLOC_NOBUSY do not exclusive busy the page | * VM_ALLOC_NOBUSY do not exclusive busy the page | ||||
* VM_ALLOC_NODUMP do not include the page in a kernel core dump | * VM_ALLOC_NODUMP do not include the page in a kernel core dump | ||||
* VM_ALLOC_NOOBJ page is not associated with an object and | * VM_ALLOC_NOOBJ page is not associated with an object and | ||||
* should not be exclusive busy | * should not be exclusive busy | ||||
* VM_ALLOC_SBUSY shared busy the allocated page | * VM_ALLOC_SBUSY shared busy the allocated page | ||||
* VM_ALLOC_WIRED wire the allocated page | * VM_ALLOC_WIRED wire the allocated page | ||||
* VM_ALLOC_ZERO prefer a zeroed page | * VM_ALLOC_ZERO prefer a zeroed page | ||||
* | * | ||||
* This routine may not sleep. | * This routine may not sleep. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | ||||
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | ||||
vm_paddr_t boundary, vm_memattr_t memattr) | vm_paddr_t boundary, vm_memattr_t memattr) | ||||
{ | { | ||||
vm_page_t m, m_tmp, m_ret; | vm_page_t m, m_ret, mpred; | ||||
u_int flags; | u_int busy_lock, flags, oflags; | ||||
int req_class; | int req_class; | ||||
mpred = NULL; /* XXX: pacify gcc */ | |||||
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | ||||
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | ||||
((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ||||
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | ||||
("vm_page_alloc: inconsistent object(%p)/req(%x)", (void *)object, | ("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object, | ||||
req)); | req)); | ||||
if (object != NULL) { | if (object != NULL) { | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT(object->type == OBJT_PHYS, | KASSERT((object->flags & OBJ_FICTITIOUS) == 0, | ||||
("vm_page_alloc_contig: object %p isn't OBJT_PHYS", | ("vm_page_alloc_contig: object %p has fictitious pages", | ||||
object)); | object)); | ||||
} | } | ||||
KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | ||||
req_class = req & VM_ALLOC_CLASS_MASK; | req_class = req & VM_ALLOC_CLASS_MASK; | ||||
/* | /* | ||||
* The page daemon is allowed to dig deeper into the free page list. | * The page daemon is allowed to dig deeper into the free page list. | ||||
*/ | */ | ||||
if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) | if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) | ||||
req_class = VM_ALLOC_SYSTEM; | req_class = VM_ALLOC_SYSTEM; | ||||
if (object != NULL) { | |||||
mpred = vm_radix_lookup_le(&object->rtree, pindex); | |||||
KASSERT(mpred == NULL || mpred->pindex != pindex, | |||||
("vm_page_alloc_contig: pindex already allocated")); | |||||
} | |||||
/* | |||||
* Can we allocate the pages without the number of free pages falling | |||||
* below the lower bound for the allocation class? | |||||
*/ | |||||
mtx_lock(&vm_page_queue_free_mtx); | mtx_lock(&vm_page_queue_free_mtx); | ||||
if (vm_cnt.v_free_count >= npages + vm_cnt.v_free_reserved || | if (vm_cnt.v_free_count >= npages + vm_cnt.v_free_reserved || | ||||
(req_class == VM_ALLOC_SYSTEM && | (req_class == VM_ALLOC_SYSTEM && | ||||
vm_cnt.v_free_count >= npages + vm_cnt.v_interrupt_free_min) || | vm_cnt.v_free_count >= npages + vm_cnt.v_interrupt_free_min) || | ||||
(req_class == VM_ALLOC_INTERRUPT && | (req_class == VM_ALLOC_INTERRUPT && | ||||
vm_cnt.v_free_count >= npages)) { | vm_cnt.v_free_count >= npages)) { | ||||
/* | |||||
* Can we allocate the pages from a reservation? | |||||
*/ | |||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
retry: | retry: | ||||
if (object == NULL || (object->flags & OBJ_COLORED) == 0 || | if (object == NULL || (object->flags & OBJ_COLORED) == 0 || | ||||
(m_ret = vm_reserv_alloc_contig(object, pindex, npages, | (m_ret = vm_reserv_alloc_contig(object, pindex, npages, | ||||
low, high, alignment, boundary)) == NULL) | low, high, alignment, boundary, mpred)) == NULL) | ||||
#endif | #endif | ||||
/* | |||||
* If not, allocate them from the free page queues. | |||||
*/ | |||||
m_ret = vm_phys_alloc_contig(npages, low, high, | m_ret = vm_phys_alloc_contig(npages, low, high, | ||||
alignment, boundary); | alignment, boundary); | ||||
} else { | } else { | ||||
mtx_unlock(&vm_page_queue_free_mtx); | mtx_unlock(&vm_page_queue_free_mtx); | ||||
atomic_add_int(&vm_pageout_deficit, npages); | atomic_add_int(&vm_pageout_deficit, npages); | ||||
pagedaemon_wakeup(); | pagedaemon_wakeup(); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
Show All 15 Lines | #endif | ||||
/* | /* | ||||
* Initialize the pages. Only the PG_ZERO flag is inherited. | * Initialize the pages. Only the PG_ZERO flag is inherited. | ||||
*/ | */ | ||||
flags = 0; | flags = 0; | ||||
if ((req & VM_ALLOC_ZERO) != 0) | if ((req & VM_ALLOC_ZERO) != 0) | ||||
flags = PG_ZERO; | flags = PG_ZERO; | ||||
if ((req & VM_ALLOC_NODUMP) != 0) | if ((req & VM_ALLOC_NODUMP) != 0) | ||||
flags |= PG_NODUMP; | flags |= PG_NODUMP; | ||||
oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | |||||
VPO_UNMANAGED : 0; | |||||
busy_lock = VPB_UNBUSIED; | |||||
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | |||||
busy_lock = VPB_SINGLE_EXCLUSIVER; | |||||
if ((req & VM_ALLOC_SBUSY) != 0) | |||||
busy_lock = VPB_SHARERS_WORD(1); | |||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
atomic_add_int(&vm_cnt.v_wire_count, npages); | atomic_add_int(&vm_cnt.v_wire_count, npages); | ||||
if (object != NULL) { | if (object != NULL) { | ||||
if (object->memattr != VM_MEMATTR_DEFAULT && | if (object->memattr != VM_MEMATTR_DEFAULT && | ||||
memattr == VM_MEMATTR_DEFAULT) | memattr == VM_MEMATTR_DEFAULT) | ||||
memattr = object->memattr; | memattr = object->memattr; | ||||
} | } | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
m->aflags = 0; | m->aflags = 0; | ||||
m->flags = (m->flags | PG_NODUMP) & flags; | m->flags = (m->flags | PG_NODUMP) & flags; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = busy_lock; | ||||
if (object != NULL) { | |||||
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) | |||||
m->busy_lock = VPB_SINGLE_EXCLUSIVER; | |||||
if ((req & VM_ALLOC_SBUSY) != 0) | |||||
m->busy_lock = VPB_SHARERS_WORD(1); | |||||
} | |||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
m->wire_count = 1; | m->wire_count = 1; | ||||
/* Unmanaged pages don't use "act_count". */ | m->act_count = 0; | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = oflags; | ||||
if (object != NULL) { | if (object != NULL) { | ||||
if (vm_page_insert(m, object, pindex)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
if (vm_paging_needed()) | |||||
pagedaemon_wakeup(); | pagedaemon_wakeup(); | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
atomic_subtract_int(&vm_cnt.v_wire_count, | atomic_subtract_int( | ||||
npages); | &vm_cnt.v_wire_count, npages); | ||||
for (m_tmp = m, m = m_ret; | KASSERT(m->object == NULL, | ||||
m < &m_ret[npages]; m++) { | ("page %p has object", m)); | ||||
if ((req & VM_ALLOC_WIRED) != 0) | mpred = m; | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | |||||
if (m <= mpred && | |||||
(req & VM_ALLOC_WIRED) != 0) | |||||
m->wire_count = 0; | m->wire_count = 0; | ||||
if (m >= m_tmp) { | m->oflags = VPO_UNMANAGED; | ||||
m->object = NULL; | |||||
m->oflags |= VPO_UNMANAGED; | |||||
} | |||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
vm_page_free(m); | /* Don't change PG_ZERO. */ | ||||
vm_page_free_toq(m); | |||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
mpred = m; | |||||
} else | } else | ||||
m->pindex = pindex; | m->pindex = pindex; | ||||
if (memattr != VM_MEMATTR_DEFAULT) | if (memattr != VM_MEMATTR_DEFAULT) | ||||
pmap_page_set_memattr(m, memattr); | pmap_page_set_memattr(m, memattr); | ||||
pindex++; | pindex++; | ||||
} | } | ||||
if (vm_paging_needed()) | if (vm_paging_needed()) | ||||
pagedaemon_wakeup(); | pagedaemon_wakeup(); | ||||
return (m_ret); | return (m_ret); | ||||
} | } | ||||
/* | /* | ||||
* Check a page that has been freshly dequeued from a freelist. | * Check a page that has been freshly dequeued from a freelist. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_alloc_check(vm_page_t m) | vm_page_alloc_check(vm_page_t m) | ||||
{ | { | ||||
KASSERT(m->object == NULL, ("page %p has object", m)); | |||||
KASSERT(m->queue == PQ_NONE, | KASSERT(m->queue == PQ_NONE, | ||||
("page %p has unexpected queue %d", m, m->queue)); | ("page %p has unexpected queue %d", m, m->queue)); | ||||
KASSERT(m->wire_count == 0, ("page %p is wired", m)); | KASSERT(m->wire_count == 0, ("page %p is wired", m)); | ||||
KASSERT(m->hold_count == 0, ("page %p is held", m)); | KASSERT(m->hold_count == 0, ("page %p is held", m)); | ||||
KASSERT(!vm_page_busied(m), ("page %p is busy", m)); | KASSERT(!vm_page_busied(m), ("page %p is busy", m)); | ||||
KASSERT(m->dirty == 0, ("page %p is dirty", m)); | KASSERT(m->dirty == 0, ("page %p is dirty", m)); | ||||
KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, | KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, | ||||
("page %p has unexpected memattr %d", | ("page %p has unexpected memattr %d", | ||||
▲ Show 20 Lines • Show All 1,742 Lines • Show Last 20 Lines |