Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 2,207 Lines • ▼ Show 20 Lines | |||||
* allocation classes: | * allocation classes: | ||||
* VM_ALLOC_NORMAL normal process request | * VM_ALLOC_NORMAL normal process request | ||||
* VM_ALLOC_SYSTEM system *really* needs a page | * VM_ALLOC_SYSTEM system *really* needs a page | ||||
* VM_ALLOC_INTERRUPT interrupt time request | * VM_ALLOC_INTERRUPT interrupt time request | ||||
* | * | ||||
* optional allocation flags: | * optional allocation flags: | ||||
* VM_ALLOC_NOBUSY do not exclusive busy the page | * VM_ALLOC_NOBUSY do not exclusive busy the page | ||||
* VM_ALLOC_NODUMP do not include the page in a kernel core dump | * VM_ALLOC_NODUMP do not include the page in a kernel core dump | ||||
* VM_ALLOC_NOOBJ page is not associated with an object and | |||||
* should not be exclusive busy | |||||
* VM_ALLOC_SBUSY shared busy the allocated page | * VM_ALLOC_SBUSY shared busy the allocated page | ||||
* VM_ALLOC_WIRED wire the allocated page | * VM_ALLOC_WIRED wire the allocated page | ||||
* VM_ALLOC_ZERO prefer a zeroed page | * VM_ALLOC_ZERO prefer a zeroed page | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | ||||
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | ||||
vm_paddr_t boundary, vm_memattr_t memattr) | vm_paddr_t boundary, vm_memattr_t memattr) | ||||
Show All 18 Lines | vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | ||||
vm_paddr_t boundary, vm_memattr_t memattr) | vm_paddr_t boundary, vm_memattr_t memattr) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m, m_ret, mpred; | vm_page_t m, m_ret, mpred; | ||||
u_int busy_lock, flags, oflags; | u_int busy_lock, flags, oflags; | ||||
mpred = NULL; /* XXX: pacify gcc */ | mpred = NULL; /* XXX: pacify gcc */ | ||||
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | KASSERT(((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ||||
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | |||||
((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | |||||
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | ||||
("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object, | ("invalid request %#x", req)); | ||||
req)); | KASSERT((req & VM_ALLOC_WAITOK) == 0, | ||||
KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | |||||
("Can't sleep and retry object insertion.")); | ("Can't sleep and retry object insertion.")); | ||||
if (object != NULL) { | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT((object->flags & OBJ_FICTITIOUS) == 0, | KASSERT((object->flags & OBJ_FICTITIOUS) == 0, | ||||
("vm_page_alloc_contig: object %p has fictitious pages", | ("vm_page_alloc_contig: object %p has fictitious pages", | ||||
object)); | object)); | ||||
} | |||||
KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | ||||
if (object != NULL) { | |||||
mpred = vm_radix_lookup_le(&object->rtree, pindex); | mpred = vm_radix_lookup_le(&object->rtree, pindex); | ||||
KASSERT(mpred == NULL || mpred->pindex != pindex, | KASSERT(mpred == NULL || mpred->pindex != pindex, | ||||
("vm_page_alloc_contig: pindex already allocated")); | ("vm_page_alloc_contig: pindex already allocated")); | ||||
} | |||||
/* | /* | ||||
* Can we allocate the pages without the number of free pages falling | * Can we allocate the pages without the number of free pages falling | ||||
* below the lower bound for the allocation class? | * below the lower bound for the allocation class? | ||||
*/ | */ | ||||
m_ret = NULL; | m_ret = NULL; | ||||
again: | again: | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
Show All 35 Lines | #endif | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
vm_page_dequeue(m); | vm_page_dequeue(m); | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
} | } | ||||
/* | /* | ||||
* Initialize the pages. Only the PG_ZERO flag is inherited. | * Initialize the pages. Only the PG_ZERO flag is inherited. | ||||
*/ | */ | ||||
flags = 0; | |||||
if ((req & VM_ALLOC_ZERO) != 0) | |||||
flags = PG_ZERO; | flags = PG_ZERO; | ||||
if ((req & VM_ALLOC_NODUMP) != 0) | if ((req & VM_ALLOC_NODUMP) != 0) | ||||
flags |= PG_NODUMP; | flags |= PG_NODUMP; | ||||
oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | oflags = (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0; | ||||
VPO_UNMANAGED : 0; | if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) | ||||
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | |||||
busy_lock = VPB_CURTHREAD_EXCLUSIVE; | busy_lock = VPB_CURTHREAD_EXCLUSIVE; | ||||
else if ((req & VM_ALLOC_SBUSY) != 0) | else if ((req & VM_ALLOC_SBUSY) != 0) | ||||
busy_lock = VPB_SHARERS_WORD(1); | busy_lock = VPB_SHARERS_WORD(1); | ||||
else | else | ||||
busy_lock = VPB_UNBUSIED; | busy_lock = VPB_UNBUSIED; | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
vm_wire_add(npages); | vm_wire_add(npages); | ||||
if (object != NULL) { | |||||
if (object->memattr != VM_MEMATTR_DEFAULT && | if (object->memattr != VM_MEMATTR_DEFAULT && | ||||
memattr == VM_MEMATTR_DEFAULT) | memattr == VM_MEMATTR_DEFAULT) | ||||
memattr = object->memattr; | memattr = object->memattr; | ||||
} | |||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
m->a.flags = 0; | m->a.flags = 0; | ||||
m->flags = (m->flags | PG_NODUMP) & flags; | m->flags = (m->flags | PG_NODUMP) & flags; | ||||
m->busy_lock = busy_lock; | m->busy_lock = busy_lock; | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
m->ref_count = 1; | m->ref_count = 1; | ||||
m->a.act_count = 0; | m->a.act_count = 0; | ||||
m->oflags = oflags; | m->oflags = oflags; | ||||
if (object != NULL) { | |||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
vm_wire_sub(npages); | vm_wire_sub(npages); | ||||
KASSERT(m->object == NULL, | KASSERT(m->object == NULL, | ||||
("page %p has object", m)); | ("page %p has object", m)); | ||||
mpred = m; | mpred = m; | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
if (m <= mpred && | if (m <= mpred && | ||||
(req & VM_ALLOC_WIRED) != 0) | (req & VM_ALLOC_WIRED) != 0) | ||||
m->ref_count = 0; | m->ref_count = 0; | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
/* Don't change PG_ZERO. */ | /* Don't change PG_ZERO. */ | ||||
vm_page_free_toq(m); | vm_page_free_toq(m); | ||||
} | } | ||||
if (req & VM_ALLOC_WAITFAIL) { | if (req & VM_ALLOC_WAITFAIL) { | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
vm_radix_wait(); | vm_radix_wait(); | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
mpred = m; | mpred = m; | ||||
} else | |||||
m->pindex = pindex; | |||||
if (memattr != VM_MEMATTR_DEFAULT) | if (memattr != VM_MEMATTR_DEFAULT) | ||||
pmap_page_set_memattr(m, memattr); | pmap_page_set_memattr(m, memattr); | ||||
pindex++; | pindex++; | ||||
} | } | ||||
return (m_ret); | return (m_ret); | ||||
} | } | ||||
/* | /* | ||||
* Allocate a physical page that is not intended to be inserted into a VM | * Allocate a physical page that is not intended to be inserted into a VM | ||||
* object. If the "freelist" parameter is not equal to VM_NFREELIST, then only | * object. If the "freelist" parameter is not equal to VM_NFREELIST, then only | ||||
* pages from the specified vm_phys freelist will be returned. | * pages from the specified vm_phys freelist will be returned. | ||||
*/ | */ | ||||
static __always_inline vm_page_t | static __always_inline vm_page_t | ||||
_vm_page_alloc_noobj_domain(int domain, const int freelist, int req) | _vm_page_alloc_noobj_domain(int domain, const int freelist, int req) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m; | vm_page_t m; | ||||
int flags; | int flags; | ||||
KASSERT((req & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY | | KASSERT((req & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) == 0, | ||||
VM_ALLOC_NOOBJ)) == 0, | |||||
("%s: invalid req %#x", __func__, req)); | ("%s: invalid req %#x", __func__, req)); | ||||
flags = (req & VM_ALLOC_NODUMP) != 0 ? PG_NODUMP : 0; | flags = (req & VM_ALLOC_NODUMP) != 0 ? PG_NODUMP : 0; | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
again: | again: | ||||
if (freelist == VM_NFREELIST && | if (freelist == VM_NFREELIST && | ||||
vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone != NULL) { | vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone != NULL) { | ||||
m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone, | m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone, | ||||
▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Lines | vm_page_alloc_noobj_contig(int req, u_long npages, vm_paddr_t low, | ||||
return (m); | return (m); | ||||
} | } | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_noobj_contig_domain(int domain, int req, u_long npages, | vm_page_alloc_noobj_contig_domain(int domain, int req, u_long npages, | ||||
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, | vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, | ||||
vm_memattr_t memattr) | vm_memattr_t memattr) | ||||
{ | { | ||||
vm_page_t m; | struct vm_domain *vmd; | ||||
u_long i; | vm_page_t m, m_ret; | ||||
u_int flags; | |||||
KASSERT((req & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY | | KASSERT(((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ||||
VM_ALLOC_NOOBJ)) == 0, | (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | ||||
("%s: invalid req %#x", __func__, req)); | ("invalid request %#x", req)); | ||||
KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | |||||
m = vm_page_alloc_contig_domain(NULL, 0, domain, req | VM_ALLOC_NOOBJ, | m_ret = NULL; | ||||
npages, low, high, alignment, boundary, memattr); | again: | ||||
if (m != NULL && (req & VM_ALLOC_ZERO) != 0) { | vmd = VM_DOMAIN(domain); | ||||
for (i = 0; i < npages; i++) { | if (vm_domain_allocate(vmd, req, npages)) { | ||||
if ((m[i].flags & PG_ZERO) == 0) | /* | ||||
pmap_zero_page(&m[i]); | * allocate them from the free page queues. | ||||
*/ | |||||
vm_domain_free_lock(vmd); | |||||
m_ret = vm_phys_alloc_contig(domain, npages, low, high, | |||||
alignment, boundary); | |||||
vm_domain_free_unlock(vmd); | |||||
if (m_ret == NULL) { | |||||
vm_domain_freecnt_inc(vmd, npages); | |||||
} | } | ||||
} | } | ||||
return (m); | if (m_ret == NULL) { | ||||
if (vm_domain_alloc_fail(vmd, NULL, req)) | |||||
goto again; | |||||
return (NULL); | |||||
} | |||||
/* | |||||
* Initialize the pages. Only the PG_ZERO flag is inherited. | |||||
*/ | |||||
flags = PG_ZERO; | |||||
if ((req & VM_ALLOC_NODUMP) != 0) | |||||
flags |= PG_NODUMP; | |||||
if ((req & VM_ALLOC_WIRED) != 0) | |||||
vm_wire_add(npages); | |||||
for (m = m_ret; m < &m_ret[npages]; m++) { | |||||
vm_page_alloc_check(m); | |||||
vm_page_dequeue(m); | |||||
m->pindex = 0xdeadc0dedeadc0de; | |||||
m->a.flags = 0; | |||||
m->flags = (m->flags | PG_NODUMP) & flags; | |||||
m->busy_lock = VPB_UNBUSIED; | |||||
if ((req & VM_ALLOC_WIRED) != 0) | |||||
m->ref_count = 1; | |||||
m->a.act_count = 0; | |||||
m->oflags = VPO_UNMANAGED; | |||||
if ((req & VM_ALLOC_ZERO) != 0 && (m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m); | |||||
} | |||||
return (m_ret); | |||||
} | } | ||||
markj: Missing a conditional pmap_page_set_memattr() call here. | |||||
/* | /* | ||||
Not Done Inline ActionsSince this function only applies to ordinary (DRAM) memory, I would do the zeroing before changing the memattr. Until we return from this function, the pages are not yet being shared with whatever device might have led us to, for example, make the pages uncacheable. alc: Since this function only applies to ordinary (DRAM) memory, I would do the zeroing before… | |||||
Done Inline ActionsI'd forgotten that pmap_page_set_memattr() flushes caches, otherwise it would seem incorrect in general to zero the page first. Maybe this deserves a comment. markj: I'd forgotten that pmap_page_set_memattr() flushes caches, otherwise it would seem incorrect in… | |||||
Done Inline ActionsYes, this deserves a comment. alc: Yes, this deserves a comment. | |||||
* Check a page that has been freshly dequeued from a freelist. | * Check a page that has been freshly dequeued from a freelist. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_alloc_check(vm_page_t m) | vm_page_alloc_check(vm_page_t m) | ||||
{ | { | ||||
KASSERT(m->object == NULL, ("page %p has object", m)); | KASSERT(m->object == NULL, ("page %p has object", m)); | ||||
KASSERT(m->a.queue == PQ_NONE && | KASSERT(m->a.queue == PQ_NONE && | ||||
▲ Show 20 Lines • Show All 3,055 Lines • Show Last 20 Lines |
Missing a conditional pmap_page_set_memattr() call here.