Changeset View
Changeset View
Standalone View
Standalone View
vm/vm_page.c
Context not available. | |||||
struct vm_domain vm_dom[MAXMEMDOM]; | struct vm_domain vm_dom[MAXMEMDOM]; | ||||
struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT]; | struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT]; | ||||
/* The following fields are protected by the domainset lock. */ | |||||
kib: I think it is better to move the comment one line below. | |||||
struct mtx_padalign __exclusive_cache_line vm_domainset_lock; | struct mtx_padalign __exclusive_cache_line vm_domainset_lock; | ||||
domainset_t __exclusive_cache_line vm_min_domains; | domainset_t __exclusive_cache_line vm_min_domains; | ||||
domainset_t __exclusive_cache_line vm_severe_domains; | domainset_t __exclusive_cache_line vm_severe_domains; | ||||
Context not available. | |||||
static int vm_severe_waiters; | static int vm_severe_waiters; | ||||
static int vm_pageproc_waiters; | static int vm_pageproc_waiters; | ||||
/* | /* | ||||
* bogus page -- for I/O to/from partially complete buffers, | * bogus page -- for I/O to/from partially complete buffers, | ||||
* or for paging into sparsely invalid regions. | * or for paging into sparsely invalid regions. | ||||
Context not available. | |||||
vm_page_t mpred); | vm_page_t mpred); | ||||
static int vm_page_reclaim_run(int req_class, int domain, u_long npages, | static int vm_page_reclaim_run(int req_class, int domain, u_long npages, | ||||
vm_page_t m_run, vm_paddr_t high); | vm_page_t m_run, vm_paddr_t high); | ||||
static void vm_domain_free_wakeup(struct vm_domain *); | |||||
static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, | static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, | ||||
int req); | int req); | ||||
Context not available. | |||||
MTX_DEF | MTX_DUPOK); | MTX_DEF | MTX_DUPOK); | ||||
} | } | ||||
mtx_init(&vmd->vmd_free_mtx, "vm page free queue", NULL, MTX_DEF); | mtx_init(&vmd->vmd_free_mtx, "vm page free queue", NULL, MTX_DEF); | ||||
mtx_init(&vmd->vmd_pageout_mtx, "vm pageout lock", NULL, MTX_DEF); | |||||
snprintf(vmd->vmd_name, sizeof(vmd->vmd_name), "%d", domain); | snprintf(vmd->vmd_name, sizeof(vmd->vmd_name), "%d", domain); | ||||
} | } | ||||
Context not available. | |||||
vmd = VM_DOMAIN(seg->domain); | vmd = VM_DOMAIN(seg->domain); | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
vm_phys_free_contig(m, pagecount); | vm_phys_free_contig(m, pagecount); | ||||
vm_domain_freecnt_adj(vmd, (int)pagecount); | |||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
vm_domain_freecnt_inc(vmd, (int)pagecount); | |||||
kibUnsubmitted Not Done Inline Actionsint cast is excessive. kib: int cast is excessive. | |||||
vm_cnt.v_page_count += (u_int)pagecount; | vm_cnt.v_page_count += (u_int)pagecount; | ||||
vmd = VM_DOMAIN(seg->domain); | vmd = VM_DOMAIN(seg->domain); | ||||
Context not available. | |||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m; | vm_page_t m; | ||||
int flags; | int flags; | ||||
u_int free_count; | |||||
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | ||||
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | ||||
Context not available. | |||||
#endif | #endif | ||||
} | } | ||||
} | } | ||||
if (m != NULL) | |||||
vm_domain_freecnt_dec(vmd, 1); | |||||
vm_domain_free_unlock(vmd); | |||||
if (m == NULL) { | if (m == NULL) { | ||||
/* | /* | ||||
* Not allocatable, give up. | * Not allocatable, give up. | ||||
Context not available. | |||||
* At this point we had better have found a good page. | * At this point we had better have found a good page. | ||||
*/ | */ | ||||
KASSERT(m != NULL, ("missing page")); | KASSERT(m != NULL, ("missing page")); | ||||
free_count = vm_domain_freecnt_adj(vmd, -1); | |||||
vm_domain_free_unlock(vmd); | |||||
/* | |||||
* Don't wakeup too often - wakeup the pageout daemon when | |||||
* we would be nearly out of memory. | |||||
*/ | |||||
if (vm_paging_needed(vmd, free_count)) | |||||
pagedaemon_wakeup(vmd->vmd_domain); | |||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
found: | found: | ||||
#endif | #endif | ||||
Context not available. | |||||
if (object != NULL) { | if (object != NULL) { | ||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
pagedaemon_wakeup(domain); | |||||
if (req & VM_ALLOC_WIRED) { | if (req & VM_ALLOC_WIRED) { | ||||
vm_wire_sub(1); | vm_wire_sub(1); | ||||
m->wire_count = 0; | m->wire_count = 0; | ||||
Context not available. | |||||
goto retry; | goto retry; | ||||
#endif | #endif | ||||
} | } | ||||
if (m_ret != NULL) | |||||
vm_domain_freecnt_dec(vmd, npages); | |||||
vm_domain_free_unlock(vmd); | |||||
if (m_ret == NULL) { | if (m_ret == NULL) { | ||||
if (vm_domain_alloc_fail(vmd, object, req)) | if (vm_domain_alloc_fail(vmd, object, req)) | ||||
goto again; | goto again; | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
vm_domain_freecnt_adj(vmd, -npages); | |||||
vm_domain_free_unlock(vmd); | |||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
found: | found: | ||||
#endif | #endif | ||||
Context not available. | |||||
m->oflags = oflags; | m->oflags = oflags; | ||||
if (object != NULL) { | if (object != NULL) { | ||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
pagedaemon_wakeup(domain); | |||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
vm_wire_sub(npages); | vm_wire_sub(npages); | ||||
KASSERT(m->object == NULL, | KASSERT(m->object == NULL, | ||||
Context not available. | |||||
pmap_page_set_memattr(m, memattr); | pmap_page_set_memattr(m, memattr); | ||||
pindex++; | pindex++; | ||||
} | } | ||||
vmd = VM_DOMAIN(domain); | |||||
if (vm_paging_needed(vmd, vmd->vmd_free_count)) | |||||
pagedaemon_wakeup(domain); | |||||
return (m_ret); | return (m_ret); | ||||
} | } | ||||
Context not available. | |||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m; | vm_page_t m; | ||||
u_int flags, free_count; | u_int flags; | ||||
/* | /* | ||||
* Do not allocate reserved pages unless the req has asked for it. | * Do not allocate reserved pages unless the req has asked for it. | ||||
Context not available. | |||||
if (vm_domain_available(vmd, req, 1)) | if (vm_domain_available(vmd, req, 1)) | ||||
m = vm_phys_alloc_freelist_pages(domain, freelist, | m = vm_phys_alloc_freelist_pages(domain, freelist, | ||||
VM_FREEPOOL_DIRECT, 0); | VM_FREEPOOL_DIRECT, 0); | ||||
if (m != NULL) | |||||
vm_domain_freecnt_dec(vmd, 1); | |||||
vm_domain_free_unlock(vmd); | |||||
if (m == NULL) { | if (m == NULL) { | ||||
if (vm_domain_alloc_fail(vmd, NULL, req)) | if (vm_domain_alloc_fail(vmd, NULL, req)) | ||||
goto again; | goto again; | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
free_count = vm_domain_freecnt_adj(vmd, -1); | |||||
vm_domain_free_unlock(vmd); | |||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
/* | /* | ||||
Context not available. | |||||
} | } | ||||
/* Unmanaged pages don't use "act_count". */ | /* Unmanaged pages don't use "act_count". */ | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
if (vm_paging_needed(vmd, free_count)) | |||||
pagedaemon_wakeup(domain); | |||||
return (m); | return (m); | ||||
} | } | ||||
Context not available. | |||||
/* | /* | ||||
* Clear the domain from the appropriate page level domainset. | * Clear the domain from the appropriate page level domainset. | ||||
*/ | */ | ||||
static void | void | ||||
vm_domain_clear(struct vm_domain *vmd) | vm_domain_clear(struct vm_domain *vmd) | ||||
{ | { | ||||
Context not available. | |||||
wakeup(&vm_severe_domains); | wakeup(&vm_severe_domains); | ||||
} | } | ||||
} | } | ||||
/* | |||||
* if pageout daemon needs pages, then tell it that there are | |||||
markjUnsubmitted Done Inline ActionsStyle: the first letter should be capitalized and there should be a newline before the comment. markj: Style: the first letter should be capitalized and there should be a newline before the comment. | |||||
* some free. | |||||
*/ | |||||
if (vmd->vmd_pageout_pages_needed && | |||||
vmd->vmd_free_count >= vmd->vmd_pageout_free_min) { | |||||
wakeup(&vmd->vmd_pageout_pages_needed); | |||||
vmd->vmd_pageout_pages_needed = 0; | |||||
} | |||||
/* See comments in vm_wait(); */ | |||||
markjUnsubmitted Done Inline ActionsSemicolon instead of a period. markj: Semicolon instead of a period. | |||||
if (vm_pageproc_waiters) { | |||||
vm_pageproc_waiters = 0; | |||||
wakeup(&vm_pageproc_waiters); | |||||
} | |||||
mtx_unlock(&vm_domainset_lock); | mtx_unlock(&vm_domainset_lock); | ||||
} | } | ||||
Context not available. | |||||
vm_wait_count(void) | vm_wait_count(void) | ||||
{ | { | ||||
return (vm_severe_waiters + vm_min_waiters); | return (vm_severe_waiters + vm_min_waiters + vm_pageproc_waiters); | ||||
} | } | ||||
static void | static void | ||||
Context not available. | |||||
if (curproc == pageproc) { | if (curproc == pageproc) { | ||||
mtx_lock(&vm_domainset_lock); | mtx_lock(&vm_domainset_lock); | ||||
vm_pageproc_waiters++; | vm_pageproc_waiters++; | ||||
msleep(&vm_pageproc_waiters, &vm_domainset_lock, PVM, | msleep(&vm_pageproc_waiters, &vm_domainset_lock, PVM | PDROP, | ||||
"pageprocwait", 1); | "pageprocwait", 1); | ||||
mtx_unlock(&vm_domainset_lock); | |||||
} else { | } else { | ||||
/* | /* | ||||
* XXX Ideally we would wait only until the allocation could | * XXX Ideally we would wait only until the allocation could | ||||
Context not available. | |||||
domainset_t wdom; | domainset_t wdom; | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
vm_domain_free_assert_locked(vmd); | vm_domain_free_assert_unlocked(vmd); | ||||
if (curproc == pageproc) { | if (curproc == pageproc) { | ||||
vmd->vmd_pageout_pages_needed = 1; | mtx_lock(&vm_domainset_lock); | ||||
msleep(&vmd->vmd_pageout_pages_needed, | if (vmd->vmd_free_count < vmd->vmd_pageout_free_min) { | ||||
vm_domain_free_lockptr(vmd), PDROP | PSWP, "VMWait", 0); | vmd->vmd_pageout_pages_needed = 1; | ||||
msleep(&vmd->vmd_pageout_pages_needed, | |||||
&vm_domainset_lock, PDROP | PSWP, "VMWait", 0); | |||||
} else | |||||
mtx_unlock(&vm_domainset_lock); | |||||
} else { | } else { | ||||
vm_domain_free_unlock(vmd); | |||||
if (pageproc == NULL) | if (pageproc == NULL) | ||||
panic("vm_wait in early boot"); | panic("vm_wait in early boot"); | ||||
DOMAINSET_ZERO(&wdom); | DOMAINSET_ZERO(&wdom); | ||||
Context not available. | |||||
vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, int req) | vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, int req) | ||||
{ | { | ||||
vm_domain_free_assert_locked(vmd); | vm_domain_free_assert_unlocked(vmd); | ||||
atomic_add_int(&vmd->vmd_pageout_deficit, | atomic_add_int(&vmd->vmd_pageout_deficit, | ||||
max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); | max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); | ||||
Context not available. | |||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
if (req & VM_ALLOC_WAITOK) | if (req & VM_ALLOC_WAITOK) | ||||
return (EAGAIN); | return (EAGAIN); | ||||
} else { | |||||
vm_domain_free_unlock(vmd); | |||||
pagedaemon_wakeup(vmd->vmd_domain); | |||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
Context not available. | |||||
} | } | ||||
/* | /* | ||||
* vm_domain_free_wakeup: | |||||
* | |||||
* Helper routine for vm_page_free_toq(). This routine is called | |||||
* when a page is added to the free queues. | |||||
* | |||||
* The page queues must be locked. | |||||
*/ | |||||
static void | |||||
vm_domain_free_wakeup(struct vm_domain *vmd) | |||||
{ | |||||
vm_domain_free_assert_locked(vmd); | |||||
/* | |||||
* if pageout daemon needs pages, then tell it that there are | |||||
* some free. | |||||
*/ | |||||
if (vmd->vmd_pageout_pages_needed && | |||||
vmd->vmd_free_count >= vmd->vmd_pageout_free_min) { | |||||
wakeup(&vmd->vmd_pageout_pages_needed); | |||||
vmd->vmd_pageout_pages_needed = 0; | |||||
} | |||||
/* | |||||
* wakeup processes that are waiting on memory if we hit a | |||||
* high water mark. And wakeup scheduler process if we have | |||||
* lots of memory. this process will swapin processes. | |||||
*/ | |||||
if ((vmd->vmd_minset && !vm_paging_min(vmd)) || | |||||
(vmd->vmd_severeset && !vm_paging_severe(vmd))) | |||||
vm_domain_clear(vmd); | |||||
/* See comments in vm_wait(); */ | |||||
if (vm_pageproc_waiters) { | |||||
vm_pageproc_waiters = 0; | |||||
wakeup(&vm_pageproc_waiters); | |||||
} | |||||
} | |||||
/* | |||||
* vm_page_free_prep: | * vm_page_free_prep: | ||||
* | * | ||||
* Prepares the given page to be put on the free list, | * Prepares the given page to be put on the free list, | ||||
Context not available. | |||||
/* | /* | ||||
* Insert the page into the physical memory allocator's free page | * Insert the page into the physical memory allocator's free page | ||||
* queues. This is the last step to free a page. | * queues. This is the last step to free a page. The caller is | ||||
* responsible for adjusting the free page count. | |||||
*/ | */ | ||||
static void | static void | ||||
vm_page_free_phys(struct vm_domain *vmd, vm_page_t m) | vm_page_free_phys(struct vm_domain *vmd, vm_page_t m) | ||||
Context not available. | |||||
vm_domain_free_assert_locked(vmd); | vm_domain_free_assert_locked(vmd); | ||||
vm_domain_freecnt_adj(vmd, 1); | |||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (!vm_reserv_free_page(m)) | if (!vm_reserv_free_page(m)) | ||||
#endif | #endif | ||||
Context not available. | |||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m; | vm_page_t m; | ||||
int cnt; | |||||
if (TAILQ_EMPTY(tq)) | if (TAILQ_EMPTY(tq)) | ||||
return; | return; | ||||
vmd = NULL; | vmd = NULL; | ||||
cnt = 0; | |||||
TAILQ_FOREACH(m, tq, listq) { | TAILQ_FOREACH(m, tq, listq) { | ||||
if (vmd != vm_pagequeue_domain(m)) { | if (vmd != vm_pagequeue_domain(m)) { | ||||
if (vmd != NULL) { | if (vmd != NULL) { | ||||
vm_domain_free_wakeup(vmd); | |||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
vm_domain_freecnt_inc(vmd, cnt); | |||||
cnt = 0; | |||||
} | } | ||||
vmd = vm_pagequeue_domain(m); | vmd = vm_pagequeue_domain(m); | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
} | } | ||||
vm_page_free_phys(vmd, m); | vm_page_free_phys(vmd, m); | ||||
cnt++; | |||||
} | } | ||||
if (vmd != NULL) { | if (vmd != NULL) { | ||||
vm_domain_free_wakeup(vmd); | |||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
vm_domain_freecnt_inc(vmd, cnt); | |||||
} | } | ||||
} | } | ||||
Context not available. | |||||
vmd = vm_pagequeue_domain(m); | vmd = vm_pagequeue_domain(m); | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
vm_page_free_phys(vmd, m); | vm_page_free_phys(vmd, m); | ||||
vm_domain_free_wakeup(vmd); | |||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
vm_domain_freecnt_inc(vmd, 1); | |||||
} | } | ||||
/* | /* | ||||
Context not available. |
I think it is better to move the comment one line below.