Changeset View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 2,391 Lines • ▼ Show 20 Lines | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
if (memattr != VM_MEMATTR_DEFAULT) | if (memattr != VM_MEMATTR_DEFAULT) | ||||
pmap_page_set_memattr(m, memattr); | pmap_page_set_memattr(m, memattr); | ||||
pindex++; | pindex++; | ||||
} | } | ||||
return (m_ret); | return (m_ret); | ||||
} | } | ||||
/* | /* | ||||
* Check a page that has been freshly dequeued from a freelist. | * Allocate a physical page that is not intended to be inserted into a VM | ||||
* object. If the "freelist" parameter is not equal to VM_NFREELIST, then only | |||||
* pages from the specified vm_phys freelist will be returned. | |||||
*/ | */ | ||||
static void | static __always_inline vm_page_t | ||||
vm_page_alloc_check(vm_page_t m) | _vm_page_alloc_noobj_domain(int domain, const int freelist, int req) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
vm_page_t m; | |||||
int flags; | |||||
KASSERT(m->object == NULL, ("page %p has object", m)); | KASSERT((req & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY | | ||||
KASSERT(m->a.queue == PQ_NONE && | VM_ALLOC_NOOBJ)) == 0, | ||||
kib: I would allow VM_ALLOC_NOBUSY | |||||
(m->a.flags & PGA_QUEUE_STATE_MASK) == 0, | ("%s: invalid req %#x", __func__, req)); | ||||
("page %p has unexpected queue %d, flags %#x", | |||||
m, m->a.queue, (m->a.flags & PGA_QUEUE_STATE_MASK))); | flags = (req & VM_ALLOC_NODUMP) != 0 ? PG_NODUMP : 0; | ||||
Done Inline Actions!= 0 kib: != 0 | |||||
KASSERT(m->ref_count == 0, ("page %p has references", m)); | vmd = VM_DOMAIN(domain); | ||||
KASSERT(vm_page_busy_freed(m), ("page %p is not freed", m)); | again: | ||||
KASSERT(m->dirty == 0, ("page %p is dirty", m)); | if (freelist == VM_NFREELIST && | ||||
alcUnsubmitted Done Inline ActionsSee my below comment about the _freelist functions. alc: See my below comment about the _freelist functions. | |||||
KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, | vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone != NULL) { | ||||
("page %p has unexpected memattr %d", | m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone, | ||||
m, pmap_page_get_memattr(m))); | M_NOWAIT | M_NOVM); | ||||
KASSERT(m->valid == 0, ("free page %p is valid", m)); | if (m != NULL) { | ||||
pmap_vm_page_alloc_check(m); | flags |= PG_PCPU_CACHE; | ||||
goto found; | |||||
} | } | ||||
} | |||||
/* | if (vm_domain_allocate(vmd, req, 1)) { | ||||
* vm_page_alloc_freelist: | vm_domain_free_lock(vmd); | ||||
* | if (freelist == VM_NFREELIST) | ||||
* Allocate a physical page from the specified free page list. | m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DIRECT, 0); | ||||
* | else | ||||
* The caller must always specify an allocation class. | m = vm_phys_alloc_freelist_pages(domain, freelist, | ||||
* | VM_FREEPOOL_DIRECT, 0); | ||||
* allocation classes: | vm_domain_free_unlock(vmd); | ||||
* VM_ALLOC_NORMAL normal process request | if (m == NULL) | ||||
* VM_ALLOC_SYSTEM system *really* needs a page | vm_domain_freecnt_inc(vmd, 1); | ||||
* VM_ALLOC_INTERRUPT interrupt time request | } | ||||
* | if (m == NULL) { | ||||
* optional allocation flags: | if (vm_domain_alloc_fail(vmd, NULL, req)) | ||||
* VM_ALLOC_COUNT(number) the number of additional pages that the caller | goto again; | ||||
* intends to allocate | return (NULL); | ||||
* VM_ALLOC_WIRED wire the allocated page | } | ||||
* VM_ALLOC_ZERO prefer a zeroed page | |||||
*/ | found: | ||||
vm_page_dequeue(m); | |||||
vm_page_alloc_check(m); | |||||
/* Consumers should not rely on a useful default pindex value. */ | |||||
m->pindex = 0xdeadc0dedeadc0de; | |||||
m->flags = (m->flags & PG_ZERO) | flags; | |||||
m->a.flags = 0; | |||||
m->oflags = VPO_UNMANAGED; | |||||
Done Inline Actions!= 0 kib: != 0 | |||||
m->busy_lock = VPB_UNBUSIED; | |||||
if ((req & VM_ALLOC_WIRED) != 0) { | |||||
vm_wire_add(1); | |||||
Done Inline ActionsI wonder if it would be useful to assign pindex some value? E.g. 0 or -1, or even some pattern like deadc0de, so that consumers that need pindex would note it. kib: I wonder if it would be useful to assign pindex some value? E.g. 0 or -1, or even some pattern… | |||||
Done Inline ActionsI like that idea. Maybe it should be INVARIANTS-only but I don't think there is any harm in having it be unconditional. markj: I like that idea. Maybe it should be INVARIANTS-only but I don't think there is any harm in… | |||||
m->ref_count = 1; | |||||
} | |||||
if ((req & VM_ALLOC_ZERO) != 0 && (m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m); | |||||
return (m); | |||||
} | |||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_freelist(int freelist, int req) | vm_page_alloc_freelist(int freelist, int req) | ||||
{ | { | ||||
struct vm_domainset_iter di; | struct vm_domainset_iter di; | ||||
vm_page_t m; | vm_page_t m; | ||||
int domain; | int domain; | ||||
vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); | vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); | ||||
do { | do { | ||||
m = vm_page_alloc_freelist_domain(domain, freelist, req); | m = vm_page_alloc_freelist_domain(domain, freelist, req); | ||||
if (m != NULL) | if (m != NULL) | ||||
break; | break; | ||||
} while (vm_domainset_iter_page(&di, NULL, &domain) == 0); | } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); | ||||
return (m); | return (m); | ||||
} | } | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_freelist_domain(int domain, int freelist, int req) | vm_page_alloc_freelist_domain(int domain, int freelist, int req) | ||||
{ | { | ||||
struct vm_domain *vmd; | KASSERT(freelist >= 0 && freelist < VM_NFREELIST, | ||||
("%s: invalid freelist %d", __func__, freelist)); | |||||
return (_vm_page_alloc_noobj_domain(domain, freelist, req)); | |||||
} | |||||
alcUnsubmitted Not Done Inline ActionsAs an aside, I am going to argue that these functions should be jettisoned. They are only used by MIPS, and those uses would be handled efficiently, i.e., without excessive iteration, by vm_page_alloc_noobj_contig(). alc: As an aside, I am going to argue that these functions should be jettisoned. They are only used… | |||||
impUnsubmitted Not Done Inline ActionsOne could just mark them #ifdef mips. then when the coming mips retirement happens, they will be removed. imp: One could just mark them #ifdef mips. then when the coming mips retirement happens, they will… | |||||
alcUnsubmitted Not Done Inline ActionsHow soon will that be? I brought up the elimination of the _freelist functions here because eliminating them would simplify the implementation of the helper function, _vm_page_alloc_noobj_domain(), that is introduced in this change. alc: How soon will that be? I brought up the elimination of the _freelist functions here because… | |||||
markjAuthorUnsubmitted Done Inline ActionsIt looks like Warner started the discussion, so I guess it would happen sometime before 14.x. We could in the meantime implement a MIPS-private vm_page_alloc_freelist() like this (together with a per-domain variant)? vm_page_t pmap_alloc_direct_page(vm_pindex_t index, int req) { vm_page_t m; req |= VM_ALLOC_WIRE | VM_ALLOC_ZERO; #if VM_FREELIST_DIRECT == VM_FREELIST_DEFAULT m = vm_page_alloc_noobj(req); #elif VM_FREELIST_DIRECT == VM_FREELIST_LOWMEM m = vm_page_alloc_noobj_contig(req, 1, 0, VM_LOWMEM_BOUNDARY, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); #else #error unhandled #endif m->pindex = index; return (m); } Alternately, just eliminate the _vm_page_alloc_noobj_domain() helper once MIPS is removed. Or, keep the vm_page_alloc_freelist() implementation separate from the vm_page_alloc_noobj() implementation so that the eventual removal of vm_page_alloc_freelist() needn't touch anything else. markj: It looks like Warner started the discussion, so I guess it would happen sometime before 14.x. | |||||
vm_page_t | |||||
vm_page_alloc_noobj(int req) | |||||
{ | |||||
struct vm_domainset_iter di; | |||||
vm_page_t m; | vm_page_t m; | ||||
u_int flags; | int domain; | ||||
m = NULL; | vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); | ||||
vmd = VM_DOMAIN(domain); | do { | ||||
again: | m = vm_page_alloc_noobj_domain(domain, req); | ||||
if (vm_domain_allocate(vmd, req, 1)) { | if (m != NULL) | ||||
vm_domain_free_lock(vmd); | break; | ||||
m = vm_phys_alloc_freelist_pages(domain, freelist, | } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); | ||||
VM_FREEPOOL_DIRECT, 0); | |||||
vm_domain_free_unlock(vmd); | return (m); | ||||
if (m == NULL) | |||||
vm_domain_freecnt_inc(vmd, 1); | |||||
} | } | ||||
if (m == NULL) { | |||||
if (vm_domain_alloc_fail(vmd, NULL, req)) | vm_page_t | ||||
goto again; | vm_page_alloc_noobj_domain(int domain, int req) | ||||
return (NULL); | { | ||||
return (_vm_page_alloc_noobj_domain(domain, VM_NFREELIST, req)); | |||||
} | } | ||||
vm_page_dequeue(m); | |||||
vm_page_alloc_check(m); | |||||
/* | /* | ||||
* Initialize the page. Only the PG_ZERO flag is inherited. | * Check a page that has been freshly dequeued from a freelist. | ||||
*/ | */ | ||||
m->a.flags = 0; | static void | ||||
flags = 0; | vm_page_alloc_check(vm_page_t m) | ||||
Done Inline ActionsWhy moving this function around? kib: Why moving this function around? | |||||
Done Inline ActionsI just wanted to group all of the allocator entry points together. markj: I just wanted to group all of the allocator entry points together. | |||||
Done Inline ActionsPerhaps commit the move separately? kib: Perhaps commit the move separately? | |||||
if ((req & VM_ALLOC_ZERO) != 0) | { | ||||
flags = PG_ZERO; | |||||
alcUnsubmitted Done Inline ActionsYou can drop the blank line here. alc: You can drop the blank line here. | |||||
m->flags &= flags; | KASSERT(m->object == NULL, ("page %p has object", m)); | ||||
if ((req & VM_ALLOC_WIRED) != 0) { | KASSERT(m->a.queue == PQ_NONE && | ||||
vm_wire_add(1); | (m->a.flags & PGA_QUEUE_STATE_MASK) == 0, | ||||
m->ref_count = 1; | ("page %p has unexpected queue %d, flags %#x", | ||||
} | m, m->a.queue, (m->a.flags & PGA_QUEUE_STATE_MASK))); | ||||
/* Unmanaged pages don't use "act_count". */ | KASSERT(m->ref_count == 0, ("page %p has references", m)); | ||||
m->oflags = VPO_UNMANAGED; | KASSERT(vm_page_busy_freed(m), ("page %p is not freed", m)); | ||||
return (m); | KASSERT(m->dirty == 0, ("page %p is dirty", m)); | ||||
KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, | |||||
("page %p has unexpected memattr %d", | |||||
m, pmap_page_get_memattr(m))); | |||||
KASSERT(m->valid == 0, ("free page %p is valid", m)); | |||||
pmap_vm_page_alloc_check(m); | |||||
alcUnsubmitted Done Inline ActionsIf I am allowed to be truly pedantic for a moment, I would suggest reordering these asserts to match the order of the fields in the struct. Then, a reader can more easily figure out which fields are being checked and which are not. alc: If I am allowed to be truly pedantic for a moment, I would suggest reordering these asserts to… | |||||
markjAuthorUnsubmitted Done Inline ActionsI split the vm_page_alloc_check() move into a separate commit already. This diff shows the modifications you suggested, but I will merge them with the separate commit before pushing. markj: I split the vm_page_alloc_check() move into a separate commit already. This diff shows the… | |||||
} | } | ||||
static int | static int | ||||
vm_page_zone_import(void *arg, void **store, int cnt, int domain, int flags) | vm_page_zone_import(void *arg, void **store, int cnt, int domain, int flags) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
struct vm_pgcache *pgcache; | struct vm_pgcache *pgcache; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 3,036 Lines • Show Last 20 Lines |
I would allow VM_ALLOC_NOBUSY