Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 2,211 Lines • ▼ Show 20 Lines | |||||
* allocation classes: | * allocation classes: | ||||
* VM_ALLOC_NORMAL normal process request | * VM_ALLOC_NORMAL normal process request | ||||
* VM_ALLOC_SYSTEM system *really* needs a page | * VM_ALLOC_SYSTEM system *really* needs a page | ||||
* VM_ALLOC_INTERRUPT interrupt time request | * VM_ALLOC_INTERRUPT interrupt time request | ||||
* | * | ||||
* optional allocation flags: | * optional allocation flags: | ||||
* VM_ALLOC_NOBUSY do not exclusive busy the page | * VM_ALLOC_NOBUSY do not exclusive busy the page | ||||
* VM_ALLOC_NODUMP do not include the page in a kernel core dump | * VM_ALLOC_NODUMP do not include the page in a kernel core dump | ||||
* VM_ALLOC_NOOBJ page is not associated with an object and | |||||
* should not be exclusive busy | |||||
* VM_ALLOC_SBUSY shared busy the allocated page | * VM_ALLOC_SBUSY shared busy the allocated page | ||||
* VM_ALLOC_WIRED wire the allocated page | * VM_ALLOC_WIRED wire the allocated page | ||||
* VM_ALLOC_ZERO prefer a zeroed page | * VM_ALLOC_ZERO prefer a zeroed page | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | ||||
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | ||||
vm_paddr_t boundary, vm_memattr_t memattr) | vm_paddr_t boundary, vm_memattr_t memattr) | ||||
Show All 17 Lines | |||||
vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain, | vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | ||||
vm_paddr_t boundary, vm_memattr_t memattr) | vm_paddr_t boundary, vm_memattr_t memattr) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m, m_ret, mpred; | vm_page_t m, m_ret, mpred; | ||||
u_int busy_lock, flags, oflags; | u_int busy_lock, flags, oflags; | ||||
mpred = NULL; /* XXX: pacify gcc */ | #define VPAC_FLAGS VPA_FLAGS | ||||
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | KASSERT((req & ~VPAC_FLAGS) == 0, | ||||
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | ("invalid request %#x", req)); | ||||
((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | KASSERT(((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ||||
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | ||||
("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object, | ("invalid request %#x", req)); | ||||
req)); | |||||
KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | |||||
("Can't sleep and retry object insertion.")); | |||||
if (object != NULL) { | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT((object->flags & OBJ_FICTITIOUS) == 0, | KASSERT((object->flags & OBJ_FICTITIOUS) == 0, | ||||
("vm_page_alloc_contig: object %p has fictitious pages", | ("vm_page_alloc_contig: object %p has fictitious pages", | ||||
object)); | object)); | ||||
} | |||||
KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | ||||
if (object != NULL) { | |||||
mpred = vm_radix_lookup_le(&object->rtree, pindex); | mpred = vm_radix_lookup_le(&object->rtree, pindex); | ||||
KASSERT(mpred == NULL || mpred->pindex != pindex, | KASSERT(mpred == NULL || mpred->pindex != pindex, | ||||
("vm_page_alloc_contig: pindex already allocated")); | ("vm_page_alloc_contig: pindex already allocated")); | ||||
} | |||||
/* | /* | ||||
* Can we allocate the pages without the number of free pages falling | * Can we allocate the pages without the number of free pages falling | ||||
* below the lower bound for the allocation class? | * below the lower bound for the allocation class? | ||||
*/ | */ | ||||
m_ret = NULL; | m_ret = NULL; | ||||
again: | again: | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
Show All 35 Lines | #endif | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
vm_page_dequeue(m); | vm_page_dequeue(m); | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
} | } | ||||
/* | /* | ||||
* Initialize the pages. Only the PG_ZERO flag is inherited. | * Initialize the pages. Only the PG_ZERO flag is inherited. | ||||
*/ | */ | ||||
flags = 0; | |||||
if ((req & VM_ALLOC_ZERO) != 0) | |||||
flags = PG_ZERO; | flags = PG_ZERO; | ||||
if ((req & VM_ALLOC_NODUMP) != 0) | if ((req & VM_ALLOC_NODUMP) != 0) | ||||
flags |= PG_NODUMP; | flags |= PG_NODUMP; | ||||
oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | oflags = (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0; | ||||
VPO_UNMANAGED : 0; | if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) | ||||
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | |||||
busy_lock = VPB_CURTHREAD_EXCLUSIVE; | busy_lock = VPB_CURTHREAD_EXCLUSIVE; | ||||
else if ((req & VM_ALLOC_SBUSY) != 0) | else if ((req & VM_ALLOC_SBUSY) != 0) | ||||
busy_lock = VPB_SHARERS_WORD(1); | busy_lock = VPB_SHARERS_WORD(1); | ||||
else | else | ||||
busy_lock = VPB_UNBUSIED; | busy_lock = VPB_UNBUSIED; | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
vm_wire_add(npages); | vm_wire_add(npages); | ||||
if (object != NULL) { | |||||
if (object->memattr != VM_MEMATTR_DEFAULT && | if (object->memattr != VM_MEMATTR_DEFAULT && | ||||
memattr == VM_MEMATTR_DEFAULT) | memattr == VM_MEMATTR_DEFAULT) | ||||
memattr = object->memattr; | memattr = object->memattr; | ||||
} | |||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
m->a.flags = 0; | m->a.flags = 0; | ||||
m->flags = (m->flags | PG_NODUMP) & flags; | m->flags = (m->flags | PG_NODUMP) & flags; | ||||
m->busy_lock = busy_lock; | m->busy_lock = busy_lock; | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
m->ref_count = 1; | m->ref_count = 1; | ||||
m->a.act_count = 0; | m->a.act_count = 0; | ||||
m->oflags = oflags; | m->oflags = oflags; | ||||
if (object != NULL) { | |||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
vm_wire_sub(npages); | vm_wire_sub(npages); | ||||
KASSERT(m->object == NULL, | KASSERT(m->object == NULL, | ||||
("page %p has object", m)); | ("page %p has object", m)); | ||||
mpred = m; | mpred = m; | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
if (m <= mpred && | if (m <= mpred && | ||||
(req & VM_ALLOC_WIRED) != 0) | (req & VM_ALLOC_WIRED) != 0) | ||||
m->ref_count = 0; | m->ref_count = 0; | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
/* Don't change PG_ZERO. */ | /* Don't change PG_ZERO. */ | ||||
vm_page_free_toq(m); | vm_page_free_toq(m); | ||||
} | } | ||||
if (req & VM_ALLOC_WAITFAIL) { | if (req & VM_ALLOC_WAITFAIL) { | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
vm_radix_wait(); | vm_radix_wait(); | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
mpred = m; | mpred = m; | ||||
} else | |||||
m->pindex = pindex; | |||||
if (memattr != VM_MEMATTR_DEFAULT) | if (memattr != VM_MEMATTR_DEFAULT) | ||||
pmap_page_set_memattr(m, memattr); | pmap_page_set_memattr(m, memattr); | ||||
pindex++; | pindex++; | ||||
} | } | ||||
return (m_ret); | return (m_ret); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 137 Lines • ▼ Show 20 Lines | vm_page_alloc_noobj_contig(int req, u_long npages, vm_paddr_t low, | ||||
return (m); | return (m); | ||||
} | } | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_noobj_contig_domain(int domain, int req, u_long npages, | vm_page_alloc_noobj_contig_domain(int domain, int req, u_long npages, | ||||
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, | vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, | ||||
vm_memattr_t memattr) | vm_memattr_t memattr) | ||||
{ | { | ||||
vm_page_t m; | struct vm_domain *vmd; | ||||
u_long i; | vm_page_t m, m_ret; | ||||
u_int flags; | |||||
KASSERT((req & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY | | #define VPANC_FLAGS VPAN_FLAGS | ||||
VM_ALLOC_NOOBJ)) == 0, | KASSERT((req & ~VPANC_FLAGS) == 0, | ||||
("%s: invalid req %#x", __func__, req)); | ("invalid request %#x", req)); | ||||
KASSERT(((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | |||||
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | |||||
("invalid request %#x", req)); | |||||
KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | |||||
m = vm_page_alloc_contig_domain(NULL, 0, domain, req | VM_ALLOC_NOOBJ, | m_ret = NULL; | ||||
npages, low, high, alignment, boundary, memattr); | again: | ||||
if (m != NULL && (req & VM_ALLOC_ZERO) != 0) { | vmd = VM_DOMAIN(domain); | ||||
for (i = 0; i < npages; i++) { | if (vm_domain_allocate(vmd, req, npages)) { | ||||
if ((m[i].flags & PG_ZERO) == 0) | /* | ||||
pmap_zero_page(&m[i]); | * allocate them from the free page queues. | ||||
*/ | |||||
vm_domain_free_lock(vmd); | |||||
m_ret = vm_phys_alloc_contig(domain, npages, low, high, | |||||
alignment, boundary); | |||||
vm_domain_free_unlock(vmd); | |||||
if (m_ret == NULL) { | |||||
vm_domain_freecnt_inc(vmd, npages); | |||||
} | } | ||||
} | } | ||||
return (m); | if (m_ret == NULL) { | ||||
if (vm_domain_alloc_fail(vmd, NULL, req)) | |||||
goto again; | |||||
return (NULL); | |||||
} | |||||
/* | |||||
* Initialize the pages. Only the PG_ZERO flag is inherited. | |||||
*/ | |||||
flags = PG_ZERO; | |||||
if ((req & VM_ALLOC_NODUMP) != 0) | |||||
flags |= PG_NODUMP; | |||||
if ((req & VM_ALLOC_WIRED) != 0) | |||||
vm_wire_add(npages); | |||||
for (m = m_ret; m < &m_ret[npages]; m++) { | |||||
vm_page_dequeue(m); | |||||
vm_page_alloc_check(m); | |||||
/* | |||||
* Consumers should not rely on a useful default pindex value. | |||||
*/ | |||||
m->pindex = 0xdeadc0dedeadc0de; | |||||
m->a.flags = 0; | |||||
m->flags = (m->flags | PG_NODUMP) & flags; | |||||
m->busy_lock = VPB_UNBUSIED; | |||||
if ((req & VM_ALLOC_WIRED) != 0) | |||||
m->ref_count = 1; | |||||
m->a.act_count = 0; | |||||
m->oflags = VPO_UNMANAGED; | |||||
/* | |||||
* Zero the page before updating any mappings since the page is | |||||
markj: Missing a conditional pmap_page_set_memattr() call here. | |||||
* not yet shared with any devices which might require the | |||||
* non-default memory attribute. pmap_page_set_memattr() | |||||
Not Done Inline ActionsSince this function only applies to ordinary (DRAM) memory, I would do the zeroing before changing the memattr. Until we return from this function, the pages are not yet being shared with whatever device might have led us to, for example, make the pages uncacheable. alc: Since this function only applies to ordinary (DRAM) memory, I would do the zeroing before… | |||||
Done Inline ActionsI'd forgotten that pmap_page_set_memattr() flushes caches, otherwise it would seem incorrect in general to zero the page first. Maybe this deserves a comment. markj: I'd forgotten that pmap_page_set_memattr() flushes caches, otherwise it would seem incorrect in… | |||||
Done Inline ActionsYes, this deserves a comment. alc: Yes, this deserves a comment. | |||||
* flushes data caches before returning. | |||||
*/ | |||||
if ((req & VM_ALLOC_ZERO) != 0 && (m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m); | |||||
if (memattr != VM_MEMATTR_DEFAULT) | |||||
pmap_page_set_memattr(m, memattr); | |||||
} | |||||
return (m_ret); | |||||
} | } | ||||
/* | /* | ||||
* Check a page that has been freshly dequeued from a freelist. | * Check a page that has been freshly dequeued from a freelist. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_alloc_check(vm_page_t m) | vm_page_alloc_check(vm_page_t m) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 3,058 Lines • Show Last 20 Lines |
Missing a conditional pmap_page_set_memattr() call here.