Changeset View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 1,943 Lines • ▼ Show 20 Lines | vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex) | ||||
m->pindex = new_pindex; | m->pindex = new_pindex; | ||||
m->object = new_object; | m->object = new_object; | ||||
vm_page_insert_radixdone(m, new_object, mpred); | vm_page_insert_radixdone(m, new_object, mpred); | ||||
vm_page_dirty(m); | vm_page_dirty(m); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | |||||
vm_page_alloc_zero(vm_page_t m, int req) | |||||
{ | |||||
if ((req & VM_ALLOC_ZERO) != 0) { | |||||
if ((m->flags & PG_ZERO) != 0) { | |||||
VM_CNT_INC(v_ozfod); | |||||
} else { | |||||
pmap_zero_page(m); | |||||
VM_CNT_INC(v_zfod); | |||||
alc: This is going to change the meaning of this counter from zero-fill on page faults (which is… | |||||
markjAuthorUnsubmitted Done Inline ActionsThat is true, I'm shoehorning them in here. My weak argument is that the vmstat -s descriptions are vague enough to be correct under this new usage. However with your proposal implemented I would move these increments back to their original location. markj: That is true, I'm shoehorning them in here. My weak argument is that the vmstat -s descriptions… | |||||
} | |||||
} | |||||
} | |||||
/* | /* | ||||
* vm_page_alloc: | * vm_page_alloc: | ||||
* | * | ||||
* Allocate and return a page that is associated with the specified | * Allocate and return a page that is associated with the specified | ||||
* object and offset pair. By default, this page is exclusive busied. | * object and offset pair. By default, this page is exclusive busied. | ||||
* | * | ||||
* The caller must always specify an allocation class. | * The caller must always specify an allocation class. | ||||
* | * | ||||
* allocation classes: | * allocation classes: | ||||
* VM_ALLOC_NORMAL normal process request | * VM_ALLOC_NORMAL normal process request | ||||
* VM_ALLOC_SYSTEM system *really* needs a page | * VM_ALLOC_SYSTEM system *really* needs a page | ||||
* VM_ALLOC_INTERRUPT interrupt time request | * VM_ALLOC_INTERRUPT interrupt time request | ||||
* | * | ||||
* optional allocation flags: | * optional allocation flags: | ||||
* VM_ALLOC_COUNT(number) the number of additional pages that the caller | * VM_ALLOC_COUNT(number) the number of additional pages that the caller | ||||
* intends to allocate | * intends to allocate | ||||
* VM_ALLOC_NOBUSY do not exclusive busy the page | * VM_ALLOC_NOBUSY do not exclusive busy the page | ||||
* VM_ALLOC_NODUMP do not include the page in a kernel core dump | * VM_ALLOC_NODUMP do not include the page in a kernel core dump | ||||
* VM_ALLOC_NOOBJ page is not associated with an object and | * VM_ALLOC_NOOBJ page is not associated with an object and | ||||
* should not be exclusive busy | * should not be exclusive busy | ||||
* | |||||
alcUnsubmitted Not Done Inline ActionsWhy the new "blank line" here? These are all optional flags, and there is no similar blank line added in similar comments below. alc: Why the new "blank line" here? These are all optional flags, and there is no similar blank… | |||||
* VM_ALLOC_SBUSY shared busy the allocated page | * VM_ALLOC_SBUSY shared busy the allocated page | ||||
* VM_ALLOC_WIRED wire the allocated page | * VM_ALLOC_WIRED wire the allocated page | ||||
* VM_ALLOC_ZERO prefer a zeroed page | * VM_ALLOC_ZERO return a zeroed page | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) | vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) | ||||
{ | { | ||||
return (vm_page_alloc_after(object, pindex, req, object != NULL ? | return (vm_page_alloc_after(object, pindex, req, object != NULL ? | ||||
vm_radix_lookup_le(&object->rtree, pindex) : NULL)); | vm_radix_lookup_le(&object->rtree, pindex) : NULL)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 156 Lines • ▼ Show 20 Lines | #endif | ||||
/* | /* | ||||
* At this point we had better have found a good page. | * At this point we had better have found a good page. | ||||
*/ | */ | ||||
found: | found: | ||||
vm_page_dequeue(m); | vm_page_dequeue(m); | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
/* | /* | ||||
* Initialize the page. Only the PG_ZERO flag is inherited. | * Initialize the page. Zero its contents if so requested. | ||||
*/ | */ | ||||
if ((req & VM_ALLOC_ZERO) != 0) | vm_page_alloc_zero(m, req); | ||||
flags |= (m->flags & PG_ZERO); | |||||
if ((req & VM_ALLOC_NODUMP) != 0) | if ((req & VM_ALLOC_NODUMP) != 0) | ||||
flags |= PG_NODUMP; | flags |= PG_NODUMP; | ||||
m->flags = flags; | m->flags = flags; | ||||
m->a.flags = 0; | m->a.flags = 0; | ||||
m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | ||||
VPO_UNMANAGED : 0; | VPO_UNMANAGED : 0; | ||||
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | ||||
m->busy_lock = VPB_CURTHREAD_EXCLUSIVE; | m->busy_lock = VPB_CURTHREAD_EXCLUSIVE; | ||||
Show All 11 Lines | if (object != NULL) { | ||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
if (req & VM_ALLOC_WIRED) { | if (req & VM_ALLOC_WIRED) { | ||||
vm_wire_sub(1); | vm_wire_sub(1); | ||||
m->ref_count = 0; | m->ref_count = 0; | ||||
} | } | ||||
KASSERT(m->object == NULL, ("page %p has object", m)); | KASSERT(m->object == NULL, ("page %p has object", m)); | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
/* Don't change PG_ZERO. */ | if ((req & VM_ALLOC_ZERO) != 0) | ||||
m->flags |= PG_ZERO; | |||||
vm_page_free_toq(m); | vm_page_free_toq(m); | ||||
if (req & VM_ALLOC_WAITFAIL) { | if (req & VM_ALLOC_WAITFAIL) { | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
vm_radix_wait(); | vm_radix_wait(); | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
Show All 40 Lines | |||||
* | * | ||||
* optional allocation flags: | * optional allocation flags: | ||||
* VM_ALLOC_NOBUSY do not exclusive busy the page | * VM_ALLOC_NOBUSY do not exclusive busy the page | ||||
* VM_ALLOC_NODUMP do not include the page in a kernel core dump | * VM_ALLOC_NODUMP do not include the page in a kernel core dump | ||||
* VM_ALLOC_NOOBJ page is not associated with an object and | * VM_ALLOC_NOOBJ page is not associated with an object and | ||||
* should not be exclusive busy | * should not be exclusive busy | ||||
* VM_ALLOC_SBUSY shared busy the allocated page | * VM_ALLOC_SBUSY shared busy the allocated page | ||||
* VM_ALLOC_WIRED wire the allocated page | * VM_ALLOC_WIRED wire the allocated page | ||||
* VM_ALLOC_ZERO prefer a zeroed page | * VM_ALLOC_ZERO return a zeroed page | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | ||||
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | ||||
vm_paddr_t boundary, vm_memattr_t memattr) | vm_paddr_t boundary, vm_memattr_t memattr) | ||||
{ | { | ||||
struct vm_domainset_iter di; | struct vm_domainset_iter di; | ||||
vm_page_t m; | vm_page_t m; | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | |||||
found: | found: | ||||
#endif | #endif | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
vm_page_dequeue(m); | vm_page_dequeue(m); | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
} | } | ||||
/* | /* | ||||
* Initialize the pages. Only the PG_ZERO flag is inherited. | * Initialize the pages. Zero their contents if so requested. | ||||
*/ | */ | ||||
flags = 0; | flags = 0; | ||||
if ((req & VM_ALLOC_ZERO) != 0) | |||||
flags = PG_ZERO; | |||||
if ((req & VM_ALLOC_NODUMP) != 0) | if ((req & VM_ALLOC_NODUMP) != 0) | ||||
flags |= PG_NODUMP; | flags |= PG_NODUMP; | ||||
oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | ||||
VPO_UNMANAGED : 0; | VPO_UNMANAGED : 0; | ||||
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | ||||
busy_lock = VPB_CURTHREAD_EXCLUSIVE; | busy_lock = VPB_CURTHREAD_EXCLUSIVE; | ||||
else if ((req & VM_ALLOC_SBUSY) != 0) | else if ((req & VM_ALLOC_SBUSY) != 0) | ||||
busy_lock = VPB_SHARERS_WORD(1); | busy_lock = VPB_SHARERS_WORD(1); | ||||
else | else | ||||
busy_lock = VPB_UNBUSIED; | busy_lock = VPB_UNBUSIED; | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
vm_wire_add(npages); | vm_wire_add(npages); | ||||
if (object != NULL) { | if (object != NULL) { | ||||
if (object->memattr != VM_MEMATTR_DEFAULT && | if (object->memattr != VM_MEMATTR_DEFAULT && | ||||
memattr == VM_MEMATTR_DEFAULT) | memattr == VM_MEMATTR_DEFAULT) | ||||
memattr = object->memattr; | memattr = object->memattr; | ||||
} | } | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
vm_page_alloc_zero(m, req); | |||||
alcUnsubmitted Not Done Inline ActionsJust an observation: Per table 5 (https://www.usenix.org/system/files/atc20-zhu-weixi_0.pdf), we should really stop zeroing one page at a time. On machines with a direct map, pmap_zero_page_area() would already "do the right thing". We would just need to modify the implementation on the other architectures. alc: Just an observation: Per table 5 (https://www.usenix.org/system/files/atc20-zhu-weixi_0.pdf)… | |||||
markjAuthorUnsubmitted Done Inline ActionsI think we'd want to additionally widen the types of the size and off to size_t. markj: I think we'd want to additionally widen the types of the `size` and `off` to size_t. | |||||
m->flags = flags; | |||||
m->a.flags = 0; | m->a.flags = 0; | ||||
m->flags = (m->flags | PG_NODUMP) & flags; | |||||
m->busy_lock = busy_lock; | m->busy_lock = busy_lock; | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
m->ref_count = 1; | m->ref_count = 1; | ||||
m->a.act_count = 0; | m->a.act_count = 0; | ||||
m->oflags = oflags; | m->oflags = oflags; | ||||
if (object != NULL) { | if (object != NULL) { | ||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
vm_wire_sub(npages); | vm_wire_sub(npages); | ||||
KASSERT(m->object == NULL, | KASSERT(m->object == NULL, | ||||
("page %p has object", m)); | ("page %p has object", m)); | ||||
mpred = m; | mpred = m; | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
if (m <= mpred && | if (m <= mpred && | ||||
(req & VM_ALLOC_WIRED) != 0) | (req & VM_ALLOC_WIRED) != 0) | ||||
m->ref_count = 0; | m->ref_count = 0; | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
/* Don't change PG_ZERO. */ | if ((req & VM_ALLOC_ZERO) != 0) | ||||
m->flags |= PG_ZERO; | |||||
Done Inline ActionsWouldn't it be useful to set PG_ZERO if VM_ALLOC_ZERO was set? At least the work we did would not be lost. kib: Wouldn't it be useful to set PG_ZERO if VM_ALLOC_ZERO was set? At least the work we did would… | |||||
vm_page_free_toq(m); | vm_page_free_toq(m); | ||||
} | } | ||||
if (req & VM_ALLOC_WAITFAIL) { | if (req & VM_ALLOC_WAITFAIL) { | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
vm_radix_wait(); | vm_radix_wait(); | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
Show All 40 Lines | |||||
* VM_ALLOC_NORMAL normal process request | * VM_ALLOC_NORMAL normal process request | ||||
* VM_ALLOC_SYSTEM system *really* needs a page | * VM_ALLOC_SYSTEM system *really* needs a page | ||||
* VM_ALLOC_INTERRUPT interrupt time request | * VM_ALLOC_INTERRUPT interrupt time request | ||||
* | * | ||||
* optional allocation flags: | * optional allocation flags: | ||||
* VM_ALLOC_COUNT(number) the number of additional pages that the caller | * VM_ALLOC_COUNT(number) the number of additional pages that the caller | ||||
* intends to allocate | * intends to allocate | ||||
* VM_ALLOC_WIRED wire the allocated page | * VM_ALLOC_WIRED wire the allocated page | ||||
* VM_ALLOC_ZERO prefer a zeroed page | * VM_ALLOC_ZERO return a zeroed page | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_freelist(int freelist, int req) | vm_page_alloc_freelist(int freelist, int req) | ||||
{ | { | ||||
struct vm_domainset_iter di; | struct vm_domainset_iter di; | ||||
vm_page_t m; | vm_page_t m; | ||||
int domain; | int domain; | ||||
vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); | vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); | ||||
do { | do { | ||||
m = vm_page_alloc_freelist_domain(domain, freelist, req); | m = vm_page_alloc_freelist_domain(domain, freelist, req); | ||||
if (m != NULL) | if (m != NULL) | ||||
break; | break; | ||||
} while (vm_domainset_iter_page(&di, NULL, &domain) == 0); | } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); | ||||
return (m); | return (m); | ||||
} | } | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_freelist_domain(int domain, int freelist, int req) | vm_page_alloc_freelist_domain(int domain, int freelist, int req) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m; | vm_page_t m; | ||||
u_int flags; | |||||
m = NULL; | m = NULL; | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
again: | again: | ||||
if (vm_domain_allocate(vmd, req, 1)) { | if (vm_domain_allocate(vmd, req, 1)) { | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
m = vm_phys_alloc_freelist_pages(domain, freelist, | m = vm_phys_alloc_freelist_pages(domain, freelist, | ||||
VM_FREEPOOL_DIRECT, 0); | VM_FREEPOOL_DIRECT, 0); | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
if (m == NULL) | if (m == NULL) | ||||
vm_domain_freecnt_inc(vmd, 1); | vm_domain_freecnt_inc(vmd, 1); | ||||
} | } | ||||
if (m == NULL) { | if (m == NULL) { | ||||
if (vm_domain_alloc_fail(vmd, NULL, req)) | if (vm_domain_alloc_fail(vmd, NULL, req)) | ||||
goto again; | goto again; | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
vm_page_dequeue(m); | vm_page_dequeue(m); | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
/* | /* | ||||
* Initialize the page. Only the PG_ZERO flag is inherited. | * Initialize the pages. Zero their contents if so requested. | ||||
alcUnsubmitted Not Done Inline Actions"pages" -> "page", "their" -> "its". alc: "pages" -> "page", "their" -> "its". | |||||
*/ | */ | ||||
vm_page_alloc_zero(m, req); | |||||
m->flags = 0; | |||||
m->a.flags = 0; | m->a.flags = 0; | ||||
flags = 0; | |||||
if ((req & VM_ALLOC_ZERO) != 0) | |||||
flags = PG_ZERO; | |||||
m->flags &= flags; | |||||
if ((req & VM_ALLOC_WIRED) != 0) { | if ((req & VM_ALLOC_WIRED) != 0) { | ||||
vm_wire_add(1); | vm_wire_add(1); | ||||
m->ref_count = 1; | m->ref_count = 1; | ||||
} | } | ||||
/* Unmanaged pages don't use "act_count". */ | /* Unmanaged pages don't use "act_count". */ | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
return (m); | return (m); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 3,034 Lines • Show Last 20 Lines |
This is going to change the meaning of this counter from zero-fill on page faults (which is where the "d" for "demand" as in "demand paging" comes from) to zero-fill on any allocation.