Changeset View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 2,389 Lines • ▼ Show 20 Lines | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
if (memattr != VM_MEMATTR_DEFAULT) | if (memattr != VM_MEMATTR_DEFAULT) | ||||
pmap_page_set_memattr(m, memattr); | pmap_page_set_memattr(m, memattr); | ||||
pindex++; | pindex++; | ||||
} | } | ||||
return (m_ret); | return (m_ret); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_alloc_freelist: | * Allocate a physical page that is not intended to be inserted into a VM | ||||
kib: I would allow VM_ALLOC_NOBUSY | |||||
* | * object. If the "freelist" parameter is not equal to VM_NFREELIST, then only | ||||
* Allocate a physical page from the specified free page list. | * pages from the specified vm_phys freelist will be returned. | ||||
* | |||||
* The caller must always specify an allocation class. | |||||
* | |||||
* allocation classes: | |||||
* VM_ALLOC_NORMAL normal process request | |||||
* VM_ALLOC_SYSTEM system *really* needs a page | |||||
* VM_ALLOC_INTERRUPT interrupt time request | |||||
* | |||||
* optional allocation flags: | |||||
* VM_ALLOC_COUNT(number) the number of additional pages that the caller | |||||
* intends to allocate | |||||
* VM_ALLOC_WIRED wire the allocated page | |||||
* VM_ALLOC_ZERO prefer a zeroed page | |||||
*/ | */ | ||||
Done Inline Actions!= 0 kib: != 0 | |||||
vm_page_t | static __always_inline vm_page_t | ||||
vm_page_alloc_freelist(int freelist, int req) | _vm_page_alloc_noobj_domain(int domain, const int freelist, int req) | ||||
{ | { | ||||
Done Inline ActionsSee my below comment about the _freelist functions. alc: See my below comment about the _freelist functions. | |||||
struct vm_domainset_iter di; | |||||
vm_page_t m; | |||||
int domain; | |||||
vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); | |||||
do { | |||||
m = vm_page_alloc_freelist_domain(domain, freelist, req); | |||||
if (m != NULL) | |||||
break; | |||||
} while (vm_domainset_iter_page(&di, NULL, &domain) == 0); | |||||
return (m); | |||||
} | |||||
vm_page_t | |||||
vm_page_alloc_freelist_domain(int domain, int freelist, int req) | |||||
{ | |||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m; | vm_page_t m; | ||||
Not Done Inline ActionsAs an aside, I am going to argue that these functions should be jettisoned. They are only used by MIPS, and those uses would be handled efficiently, i.e., without excessive iteration, by vm_page_alloc_noobj_contig(). alc: As an aside, I am going to argue that these functions should be jettisoned. They are only used… | |||||
Not Done Inline ActionsOne could just mark them #ifdef mips. then when the coming mips retirement happens, they will be removed. imp: One could just mark them #ifdef mips. then when the coming mips retirement happens, they will… | |||||
Not Done Inline ActionsHow soon will that be? I brought up the elimination of the _freelist functions here because eliminating them would simplify the implementation of the helper function, _vm_page_alloc_noobj_domain(), that is introduced in this change. alc: How soon will that be? I brought up the elimination of the _freelist functions here because… | |||||
Done Inline ActionsIt looks like Warner started the discussion, so I guess it would happen sometime before 14.x. We could in the meantime implement a MIPS-private vm_page_alloc_freelist() like this (together with a per-domain variant)? vm_page_t pmap_alloc_direct_page(vm_pindex_t index, int req) { vm_page_t m; req |= VM_ALLOC_WIRE | VM_ALLOC_ZERO; #if VM_FREELIST_DIRECT == VM_FREELIST_DEFAULT m = vm_page_alloc_noobj(req); #elif VM_FREELIST_DIRECT == VM_FREELIST_LOWMEM m = vm_page_alloc_noobj_contig(req, 1, 0, VM_LOWMEM_BOUNDARY, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); #else #error unhandled #endif m->pindex = index; return (m); } Alternately, just eliminate the _vm_page_alloc_noobj_domain() helper once MIPS is removed. Or, keep the vm_page_alloc_freelist() implementation separate from the vm_page_alloc_noobj() implementation so that the eventual removal of vm_page_alloc_freelist() needn't touch anything else. markj: It looks like Warner started the discussion, so I guess it would happen sometime before 14.x. | |||||
u_int flags; | int flags; | ||||
m = NULL; | KASSERT((req & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY | | ||||
VM_ALLOC_NOOBJ)) == 0, | |||||
("%s: invalid req %#x", __func__, req)); | |||||
flags = (req & VM_ALLOC_NODUMP) != 0 ? PG_NODUMP : 0; | |||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
again: | again: | ||||
if (freelist == VM_NFREELIST && | |||||
vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone != NULL) { | |||||
m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone, | |||||
M_NOWAIT | M_NOVM); | |||||
if (m != NULL) { | |||||
flags |= PG_PCPU_CACHE; | |||||
goto found; | |||||
} | |||||
} | |||||
if (vm_domain_allocate(vmd, req, 1)) { | if (vm_domain_allocate(vmd, req, 1)) { | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
if (freelist == VM_NFREELIST) | |||||
m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DIRECT, 0); | |||||
else | |||||
m = vm_phys_alloc_freelist_pages(domain, freelist, | m = vm_phys_alloc_freelist_pages(domain, freelist, | ||||
VM_FREEPOOL_DIRECT, 0); | VM_FREEPOOL_DIRECT, 0); | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
if (m == NULL) | if (m == NULL) | ||||
vm_domain_freecnt_inc(vmd, 1); | vm_domain_freecnt_inc(vmd, 1); | ||||
} | } | ||||
if (m == NULL) { | if (m == NULL) { | ||||
if (vm_domain_alloc_fail(vmd, NULL, req)) | if (vm_domain_alloc_fail(vmd, NULL, req)) | ||||
goto again; | goto again; | ||||
Done Inline Actions!= 0 kib: != 0 | |||||
return (NULL); | return (NULL); | ||||
} | } | ||||
Done Inline ActionsI wonder if it would be useful to assign pindex some value? E.g. 0 or -1, or even some pattern like deadc0de, so that consumers that need pindex would note it. kib: I wonder if it would be useful to assign pindex some value? E.g. 0 or -1, or even some pattern… | |||||
Done Inline ActionsI like that idea. Maybe it should be INVARIANTS-only but I don't think there is any harm in having it be unconditional. markj: I like that idea. Maybe it should be INVARIANTS-only but I don't think there is any harm in… | |||||
found: | |||||
vm_page_dequeue(m); | vm_page_dequeue(m); | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
/* | /* Consumers should not rely on a useful default pindex value. */ | ||||
* Initialize the page. Only the PG_ZERO flag is inherited. | m->pindex = 0xdeadc0dedeadc0de; | ||||
*/ | m->flags = (m->flags & PG_ZERO) | flags; | ||||
m->a.flags = 0; | m->a.flags = 0; | ||||
flags = 0; | m->oflags = VPO_UNMANAGED; | ||||
if ((req & VM_ALLOC_ZERO) != 0) | m->busy_lock = VPB_UNBUSIED; | ||||
flags = PG_ZERO; | |||||
m->flags &= flags; | |||||
if ((req & VM_ALLOC_WIRED) != 0) { | if ((req & VM_ALLOC_WIRED) != 0) { | ||||
vm_wire_add(1); | vm_wire_add(1); | ||||
m->ref_count = 1; | m->ref_count = 1; | ||||
} | } | ||||
/* Unmanaged pages don't use "act_count". */ | |||||
m->oflags = VPO_UNMANAGED; | if ((req & VM_ALLOC_ZERO) != 0 && (m->flags & PG_ZERO) == 0) | ||||
pmap_zero_page(m); | |||||
return (m); | return (m); | ||||
} | } | ||||
vm_page_t | |||||
vm_page_alloc_freelist(int freelist, int req) | |||||
{ | |||||
struct vm_domainset_iter di; | |||||
vm_page_t m; | |||||
int domain; | |||||
vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); | |||||
do { | |||||
m = vm_page_alloc_freelist_domain(domain, freelist, req); | |||||
if (m != NULL) | |||||
break; | |||||
} while (vm_domainset_iter_page(&di, NULL, &domain) == 0); | |||||
return (m); | |||||
} | |||||
vm_page_t | |||||
vm_page_alloc_freelist_domain(int domain, int freelist, int req) | |||||
{ | |||||
KASSERT(freelist >= 0 && freelist < VM_NFREELIST, | |||||
("%s: invalid freelist %d", __func__, freelist)); | |||||
return (_vm_page_alloc_noobj_domain(domain, freelist, req)); | |||||
} | |||||
vm_page_t | |||||
vm_page_alloc_noobj(int req) | |||||
{ | |||||
struct vm_domainset_iter di; | |||||
vm_page_t m; | |||||
int domain; | |||||
vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); | |||||
do { | |||||
m = vm_page_alloc_noobj_domain(domain, req); | |||||
if (m != NULL) | |||||
break; | |||||
} while (vm_domainset_iter_page(&di, NULL, &domain) == 0); | |||||
return (m); | |||||
} | |||||
vm_page_t | |||||
vm_page_alloc_noobj_domain(int domain, int req) | |||||
{ | |||||
return (_vm_page_alloc_noobj_domain(domain, VM_NFREELIST, req)); | |||||
} | |||||
/* | /* | ||||
* Check a page that has been freshly dequeued from a freelist. | * Check a page that has been freshly dequeued from a freelist. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_alloc_check(vm_page_t m) | vm_page_alloc_check(vm_page_t m) | ||||
{ | { | ||||
KASSERT(m->object == NULL, ("page %p has object", m)); | KASSERT(m->object == NULL, ("page %p has object", m)); | ||||
KASSERT(m->ref_count == 0, ("page %p has references", m)); | |||||
KASSERT(vm_page_busy_freed(m), ("page %p is not freed", m)); | |||||
KASSERT(m->a.queue == PQ_NONE && | KASSERT(m->a.queue == PQ_NONE && | ||||
(m->a.flags & PGA_QUEUE_STATE_MASK) == 0, | (m->a.flags & PGA_QUEUE_STATE_MASK) == 0, | ||||
("page %p has unexpected queue %d, flags %#x", | ("page %p has unexpected queue %d, flags %#x", | ||||
m, m->a.queue, (m->a.flags & PGA_QUEUE_STATE_MASK))); | m, m->a.queue, (m->a.flags & PGA_QUEUE_STATE_MASK))); | ||||
KASSERT(m->ref_count == 0, ("page %p has references", m)); | KASSERT(m->valid == 0, ("free page %p is valid", m)); | ||||
KASSERT(vm_page_busy_freed(m), ("page %p is not freed", m)); | |||||
KASSERT(m->dirty == 0, ("page %p is dirty", m)); | KASSERT(m->dirty == 0, ("page %p is dirty", m)); | ||||
KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, | KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, | ||||
("page %p has unexpected memattr %d", | ("page %p has unexpected memattr %d", | ||||
m, pmap_page_get_memattr(m))); | m, pmap_page_get_memattr(m))); | ||||
KASSERT(m->valid == 0, ("free page %p is valid", m)); | |||||
pmap_vm_page_alloc_check(m); | pmap_vm_page_alloc_check(m); | ||||
} | } | ||||
static int | static int | ||||
vm_page_zone_import(void *arg, void **store, int cnt, int domain, int flags) | vm_page_zone_import(void *arg, void **store, int cnt, int domain, int flags) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
struct vm_pgcache *pgcache; | struct vm_pgcache *pgcache; | ||||
int i; | int i; | ||||
pgcache = arg; | pgcache = arg; | ||||
vmd = VM_DOMAIN(pgcache->domain); | vmd = VM_DOMAIN(pgcache->domain); | ||||
/* | /* | ||||
Done Inline ActionsWhy moving this function around? kib: Why moving this function around? | |||||
Done Inline ActionsI just wanted to group all of the allocator entry points together. markj: I just wanted to group all of the allocator entry points together. | |||||
Done Inline ActionsPerhaps commit the move separately? kib: Perhaps commit the move separately? | |||||
* The page daemon should avoid creating extra memory pressure since its | * The page daemon should avoid creating extra memory pressure since its | ||||
* main purpose is to replenish the store of free pages. | * main purpose is to replenish the store of free pages. | ||||
Done Inline ActionsYou can drop the blank line here. alc: You can drop the blank line here. | |||||
*/ | */ | ||||
if (vmd->vmd_severeset || curproc == pageproc || | if (vmd->vmd_severeset || curproc == pageproc || | ||||
!_vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt)) | !_vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt)) | ||||
return (0); | return (0); | ||||
domain = vmd->vmd_domain; | domain = vmd->vmd_domain; | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
i = vm_phys_alloc_npages(domain, pgcache->pool, cnt, | i = vm_phys_alloc_npages(domain, pgcache->pool, cnt, | ||||
(vm_page_t *)store); | (vm_page_t *)store); | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
if (cnt != i) | if (cnt != i) | ||||
vm_domain_freecnt_inc(vmd, cnt - i); | vm_domain_freecnt_inc(vmd, cnt - i); | ||||
return (i); | return (i); | ||||
Done Inline ActionsIf I am allowed to be truly pedantic for a moment, I would suggest reordering these asserts to match the order of the fields in the struct. Then, a reader can more easily figure out which fields are being checked and which are not. alc: If I am allowed to be truly pedantic for a moment, I would suggest reordering these asserts to… | |||||
Done Inline ActionsI split the vm_page_alloc_check() move into a separate commit already. This diff shows the modifications you suggested, but I will merge them with the separate commit before pushing. markj: I split the vm_page_alloc_check() move into a separate commit already. This diff shows the… | |||||
} | } | ||||
static void | static void | ||||
vm_page_zone_release(void *arg, void **store, int cnt) | vm_page_zone_release(void *arg, void **store, int cnt) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
struct vm_pgcache *pgcache; | struct vm_pgcache *pgcache; | ||||
vm_page_t m; | vm_page_t m; | ||||
▲ Show 20 Lines • Show All 3,008 Lines • Show Last 20 Lines |
I would allow VM_ALLOC_NOBUSY