Changeset View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 109 Lines • ▼ Show 20 Lines | |||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <vm/vm_param.h> | #include <vm/vm_param.h> | ||||
#include <vm/vm_domainset.h> | #include <vm/vm_domainset.h> | ||||
#include <vm/vm_kern.h> | #include <vm/vm_kern.h> | ||||
#include <vm/vm_object.h> | #include <vm/vm_object.h> | ||||
#include <vm/vm_page.h> | #include <vm/vm_page.h> | ||||
#include <vm/vm_pageout.h> | #include <vm/vm_pageout.h> | ||||
#include <vm/vm_pager.h> | |||||
#include <vm/vm_phys.h> | #include <vm/vm_phys.h> | ||||
#include <vm/vm_pagequeue.h> | |||||
#include <vm/vm_pager.h> | |||||
#include <vm/vm_radix.h> | #include <vm/vm_radix.h> | ||||
#include <vm/vm_reserv.h> | #include <vm/vm_reserv.h> | ||||
#include <vm/vm_extern.h> | #include <vm/vm_extern.h> | ||||
#include <vm/uma.h> | #include <vm/uma.h> | ||||
#include <vm/uma_int.h> | #include <vm/uma_int.h> | ||||
#include <machine/md_var.h> | #include <machine/md_var.h> | ||||
/* | /* | ||||
* Associated with page of user-allocatable memory is a | * Associated with page of user-allocatable memory is a | ||||
* page structure. | * page structure. | ||||
*/ | */ | ||||
struct vm_domain vm_dom[MAXMEMDOM]; | struct vm_domain vm_dom[MAXMEMDOM]; | ||||
struct mtx_padalign __exclusive_cache_line vm_page_queue_free_mtx; | |||||
struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT]; | struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT]; | ||||
struct mtx_padalign __exclusive_cache_line vm_domainset_lock; | |||||
domainset_t __exclusive_cache_line vm_min_domains; | |||||
domainset_t __exclusive_cache_line vm_severe_domains; | |||||
static int vm_min_waiters; | |||||
static int vm_severe_waiters; | |||||
static int vm_pageproc_waiters; | |||||
/* | /* | ||||
* bogus page -- for I/O to/from partially complete buffers, | * bogus page -- for I/O to/from partially complete buffers, | ||||
* or for paging into sparsely invalid regions. | * or for paging into sparsely invalid regions. | ||||
*/ | */ | ||||
vm_page_t bogus_page; | vm_page_t bogus_page; | ||||
vm_page_t vm_page_array; | vm_page_t vm_page_array; | ||||
long vm_page_array_size; | long vm_page_array_size; | ||||
long first_page; | long first_page; | ||||
static int boot_pages = UMA_BOOT_PAGES; | static int boot_pages = UMA_BOOT_PAGES; | ||||
SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | ||||
&boot_pages, 0, | &boot_pages, 0, | ||||
"number of pages allocated for bootstrapping the VM system"); | "number of pages allocated for bootstrapping the VM system"); | ||||
static int pa_tryrelock_restart; | static int pa_tryrelock_restart; | ||||
SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFLAG_RD, | SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFLAG_RD, | ||||
&pa_tryrelock_restart, 0, "Number of tryrelock restarts"); | &pa_tryrelock_restart, 0, "Number of tryrelock restarts"); | ||||
static TAILQ_HEAD(, vm_page) blacklist_head; | static TAILQ_HEAD(, vm_page) blacklist_head; | ||||
static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); | static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); | ||||
SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | | SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | | ||||
CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages"); | CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages"); | ||||
/* Is the page daemon waiting for free pages? */ | |||||
static int vm_pageout_pages_needed; | |||||
static uma_zone_t fakepg_zone; | static uma_zone_t fakepg_zone; | ||||
static void vm_page_alloc_check(vm_page_t m); | static void vm_page_alloc_check(vm_page_t m); | ||||
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); | static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); | ||||
static void vm_page_enqueue(uint8_t queue, vm_page_t m); | static void vm_page_enqueue(uint8_t queue, vm_page_t m); | ||||
static void vm_page_free_phys(vm_page_t m); | static void vm_page_free_phys(vm_page_t m); | ||||
static void vm_page_free_wakeup(void); | |||||
static void vm_page_init(void *dummy); | static void vm_page_init(void *dummy); | ||||
static int vm_page_insert_after(vm_page_t m, vm_object_t object, | static int vm_page_insert_after(vm_page_t m, vm_object_t object, | ||||
vm_pindex_t pindex, vm_page_t mpred); | vm_pindex_t pindex, vm_page_t mpred); | ||||
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object, | static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object, | ||||
vm_page_t mpred); | vm_page_t mpred); | ||||
static int vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run, | static int vm_page_reclaim_run(int req_class, int domain, u_long npages, | ||||
vm_paddr_t high); | vm_page_t m_run, vm_paddr_t high); | ||||
static int vm_page_alloc_fail(vm_object_t object, int req); | static void vm_domain_free_wakeup(struct vm_domain *); | ||||
static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, | |||||
int req); | |||||
SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init, NULL); | SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init, NULL); | ||||
static void | static void | ||||
vm_page_init(void *dummy) | vm_page_init(void *dummy) | ||||
{ | { | ||||
fakepg_zone = uma_zcreate("fakepg", sizeof(struct vm_page), NULL, NULL, | fakepg_zone = uma_zcreate("fakepg", sizeof(struct vm_page), NULL, NULL, | ||||
▲ Show 20 Lines • Show All 120 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* Iterate through the provided string of blacklist addresses, pulling | * Iterate through the provided string of blacklist addresses, pulling | ||||
* each entry out of the physical allocator free list and putting it | * each entry out of the physical allocator free list and putting it | ||||
* onto a list for reporting via the vm.page_blacklist sysctl. | * onto a list for reporting via the vm.page_blacklist sysctl. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_blacklist_check(char *list, char *end) | vm_page_blacklist_check(char *list, char *end) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||
vm_page_t m; | vm_page_t m; | ||||
char *next; | char *next; | ||||
int ret; | int ret; | ||||
next = list; | next = list; | ||||
while (next != NULL) { | while (next != NULL) { | ||||
if ((pa = vm_page_blacklist_next(&next, end)) == 0) | if ((pa = vm_page_blacklist_next(&next, end)) == 0) | ||||
continue; | continue; | ||||
m = vm_phys_paddr_to_vm_page(pa); | m = vm_phys_paddr_to_vm_page(pa); | ||||
if (m == NULL) | if (m == NULL) | ||||
continue; | continue; | ||||
mtx_lock(&vm_page_queue_free_mtx); | vmd = vm_pagequeue_domain(m); | ||||
vm_domain_free_lock(vmd); | |||||
ret = vm_phys_unfree_page(m); | ret = vm_phys_unfree_page(m); | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
if (ret == TRUE) { | if (ret == TRUE) { | ||||
TAILQ_INSERT_TAIL(&blacklist_head, m, listq); | TAILQ_INSERT_TAIL(&blacklist_head, m, listq); | ||||
if (bootverbose) | if (bootverbose) | ||||
printf("Skipping page with pa 0x%jx\n", | printf("Skipping page with pa 0x%jx\n", | ||||
(uintmax_t)pa); | (uintmax_t)pa); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | TAILQ_FOREACH(m, &blacklist_head, listq) { | ||||
first = 0; | first = 0; | ||||
} | } | ||||
error = sbuf_finish(&sbuf); | error = sbuf_finish(&sbuf); | ||||
sbuf_delete(&sbuf); | sbuf_delete(&sbuf); | ||||
return (error); | return (error); | ||||
} | } | ||||
static void | static void | ||||
vm_page_domain_init(struct vm_domain *vmd) | vm_page_domain_init(int domain) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
struct vm_pagequeue *pq; | struct vm_pagequeue *pq; | ||||
int i; | int i; | ||||
vmd = VM_DOMAIN(domain); | |||||
bzero(vmd, sizeof(*vmd)); | |||||
*__DECONST(char **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_name) = | *__DECONST(char **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_name) = | ||||
"vm inactive pagequeue"; | "vm inactive pagequeue"; | ||||
*__DECONST(u_int **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_vcnt) = | |||||
&vm_cnt.v_inactive_count; | |||||
*__DECONST(char **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_name) = | *__DECONST(char **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_name) = | ||||
"vm active pagequeue"; | "vm active pagequeue"; | ||||
*__DECONST(u_int **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_vcnt) = | |||||
&vm_cnt.v_active_count; | |||||
*__DECONST(char **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_name) = | *__DECONST(char **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_name) = | ||||
"vm laundry pagequeue"; | "vm laundry pagequeue"; | ||||
*__DECONST(int **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_vcnt) = | |||||
&vm_cnt.v_laundry_count; | |||||
*__DECONST(char **, &vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_name) = | *__DECONST(char **, &vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_name) = | ||||
"vm unswappable pagequeue"; | "vm unswappable pagequeue"; | ||||
/* Unswappable dirty pages are counted as being in the laundry. */ | vmd->vmd_domain = domain; | ||||
*__DECONST(int **, &vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_vcnt) = | |||||
&vm_cnt.v_laundry_count; | |||||
vmd->vmd_page_count = 0; | vmd->vmd_page_count = 0; | ||||
vmd->vmd_free_count = 0; | vmd->vmd_free_count = 0; | ||||
vmd->vmd_segs = 0; | vmd->vmd_segs = 0; | ||||
vmd->vmd_oom = FALSE; | vmd->vmd_oom = FALSE; | ||||
for (i = 0; i < PQ_COUNT; i++) { | for (i = 0; i < PQ_COUNT; i++) { | ||||
pq = &vmd->vmd_pagequeues[i]; | pq = &vmd->vmd_pagequeues[i]; | ||||
TAILQ_INIT(&pq->pq_pl); | TAILQ_INIT(&pq->pq_pl); | ||||
mtx_init(&pq->pq_mutex, pq->pq_name, "vm pagequeue", | mtx_init(&pq->pq_mutex, pq->pq_name, "vm pagequeue", | ||||
MTX_DEF | MTX_DUPOK); | MTX_DEF | MTX_DUPOK); | ||||
} | } | ||||
mtx_init(&vmd->vmd_free_mtx, "vm page free queue", NULL, MTX_DEF); | |||||
} | } | ||||
/* | /* | ||||
* Initialize a physical page in preparation for adding it to the free | * Initialize a physical page in preparation for adding it to the free | ||||
* lists. | * lists. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind) | vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind) | ||||
Show All 20 Lines | |||||
* Initializes the resident memory module. Allocates physical memory for | * Initializes the resident memory module. Allocates physical memory for | ||||
* bootstrapping UMA and some data structures that are used to manage | * bootstrapping UMA and some data structures that are used to manage | ||||
* physical pages. Initializes these structures, and populates the free | * physical pages. Initializes these structures, and populates the free | ||||
* page queues. | * page queues. | ||||
*/ | */ | ||||
vm_offset_t | vm_offset_t | ||||
vm_page_startup(vm_offset_t vaddr) | vm_page_startup(vm_offset_t vaddr) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
struct vm_phys_seg *seg; | struct vm_phys_seg *seg; | ||||
vm_page_t m; | vm_page_t m; | ||||
char *list, *listend; | char *list, *listend; | ||||
vm_offset_t mapped; | vm_offset_t mapped; | ||||
vm_paddr_t end, high_avail, low_avail, new_end, page_range, size; | vm_paddr_t end, high_avail, low_avail, new_end, page_range, size; | ||||
vm_paddr_t biggestsize, last_pa, pa; | vm_paddr_t biggestsize, last_pa, pa; | ||||
u_long pagecount; | u_long pagecount; | ||||
int biggestone, i, pages_per_zone, segind; | int biggestone, i, pages_per_zone, segind; | ||||
Show All 14 Lines | for (i = 0; phys_avail[i + 1]; i += 2) { | ||||
} | } | ||||
} | } | ||||
end = phys_avail[biggestone+1]; | end = phys_avail[biggestone+1]; | ||||
/* | /* | ||||
* Initialize the page and queue locks. | * Initialize the page and queue locks. | ||||
*/ | */ | ||||
mtx_init(&vm_page_queue_free_mtx, "vm page free queue", NULL, MTX_DEF); | mtx_init(&vm_domainset_lock, "vm domainset lock", NULL, MTX_DEF); | ||||
for (i = 0; i < PA_LOCK_COUNT; i++) | for (i = 0; i < PA_LOCK_COUNT; i++) | ||||
mtx_init(&pa_lock[i], "vm page", NULL, MTX_DEF); | mtx_init(&pa_lock[i], "vm page", NULL, MTX_DEF); | ||||
for (i = 0; i < vm_ndomains; i++) | for (i = 0; i < vm_ndomains; i++) | ||||
vm_page_domain_init(&vm_dom[i]); | vm_page_domain_init(i); | ||||
/* | /* | ||||
* Almost all of the pages needed for bootstrapping UMA are used | * Almost all of the pages needed for bootstrapping UMA are used | ||||
* for zone structures, so if the number of CPUs results in those | * for zone structures, so if the number of CPUs results in those | ||||
* structures taking more than one page each, we set aside more pages | * structures taking more than one page each, we set aside more pages | ||||
* in proportion to the zone structure size. | * in proportion to the zone structure size. | ||||
*/ | */ | ||||
pages_per_zone = howmany(sizeof(struct uma_zone) + | pages_per_zone = howmany(sizeof(struct uma_zone) + | ||||
▲ Show 20 Lines • Show All 181 Lines • ▼ Show 20 Lines | #endif | ||||
*/ | */ | ||||
vm_phys_init(); | vm_phys_init(); | ||||
/* | /* | ||||
* Initialize the page structures and add every available page to the | * Initialize the page structures and add every available page to the | ||||
* physical memory allocator's free lists. | * physical memory allocator's free lists. | ||||
*/ | */ | ||||
vm_cnt.v_page_count = 0; | vm_cnt.v_page_count = 0; | ||||
vm_cnt.v_free_count = 0; | |||||
for (segind = 0; segind < vm_phys_nsegs; segind++) { | for (segind = 0; segind < vm_phys_nsegs; segind++) { | ||||
seg = &vm_phys_segs[segind]; | seg = &vm_phys_segs[segind]; | ||||
for (m = seg->first_page, pa = seg->start; pa < seg->end; | for (m = seg->first_page, pa = seg->start; pa < seg->end; | ||||
m++, pa += PAGE_SIZE) | m++, pa += PAGE_SIZE) | ||||
vm_page_init_page(m, pa, segind); | vm_page_init_page(m, pa, segind); | ||||
/* | /* | ||||
* Add the segment to the free lists only if it is covered by | * Add the segment to the free lists only if it is covered by | ||||
* one of the ranges in phys_avail. Because we've added the | * one of the ranges in phys_avail. Because we've added the | ||||
* ranges to the vm_phys_segs array, we can assume that each | * ranges to the vm_phys_segs array, we can assume that each | ||||
* segment is either entirely contained in one of the ranges, | * segment is either entirely contained in one of the ranges, | ||||
* or doesn't overlap any of them. | * or doesn't overlap any of them. | ||||
*/ | */ | ||||
for (i = 0; phys_avail[i + 1] != 0; i += 2) { | for (i = 0; phys_avail[i + 1] != 0; i += 2) { | ||||
struct vm_domain *vmd; | |||||
if (seg->start < phys_avail[i] || | if (seg->start < phys_avail[i] || | ||||
seg->end > phys_avail[i + 1]) | seg->end > phys_avail[i + 1]) | ||||
continue; | continue; | ||||
m = seg->first_page; | m = seg->first_page; | ||||
pagecount = (u_long)atop(seg->end - seg->start); | pagecount = (u_long)atop(seg->end - seg->start); | ||||
mtx_lock(&vm_page_queue_free_mtx); | vmd = VM_DOMAIN(seg->domain); | ||||
vm_domain_free_lock(vmd); | |||||
vm_phys_free_contig(m, pagecount); | vm_phys_free_contig(m, pagecount); | ||||
vm_phys_freecnt_adj(m, (int)pagecount); | vm_domain_freecnt_adj(vmd, (int)pagecount); | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
vm_cnt.v_page_count += (u_int)pagecount; | vm_cnt.v_page_count += (u_int)pagecount; | ||||
vmd = &vm_dom[seg->domain]; | vmd = VM_DOMAIN(seg->domain);; | ||||
markj: Extra semicolon. | |||||
vmd->vmd_page_count += (u_int)pagecount; | vmd->vmd_page_count += (u_int)pagecount; | ||||
vmd->vmd_segs |= 1UL << m->segind; | vmd->vmd_segs |= 1UL << m->segind; | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Remove blacklisted pages from the physical memory allocator. | * Remove blacklisted pages from the physical memory allocator. | ||||
▲ Show 20 Lines • Show All 908 Lines • ▼ Show 20 Lines | m = vm_page_alloc_domain_after(object, pindex, domain, req, | ||||
mpred); | mpred); | ||||
if (m != NULL) | if (m != NULL) | ||||
break; | break; | ||||
} while (vm_domainset_iter_page(&di, &domain, &req) == 0); | } while (vm_domainset_iter_page(&di, &domain, &req) == 0); | ||||
return (m); | return (m); | ||||
} | } | ||||
/* | |||||
* Returns true if the number of free pages exceeds the minimum | |||||
* for the request class and false otherwise. | |||||
*/ | |||||
int | |||||
vm_domain_available(struct vm_domain *vmd, int req, int npages) | |||||
{ | |||||
vm_domain_free_assert_locked(vmd); | |||||
req = req & VM_ALLOC_CLASS_MASK; | |||||
/* | |||||
* The page daemon is allowed to dig deeper into the free page list. | |||||
*/ | |||||
if (curproc == pageproc && req != VM_ALLOC_INTERRUPT) | |||||
req = VM_ALLOC_SYSTEM; | |||||
if (vmd->vmd_free_count >= npages + vmd->vmd_free_reserved || | |||||
(req == VM_ALLOC_SYSTEM && | |||||
vmd->vmd_free_count >= npages + vmd->vmd_interrupt_free_min) || | |||||
(req == VM_ALLOC_INTERRUPT && | |||||
vmd->vmd_free_count >= npages)) | |||||
return (1); | |||||
return (0); | |||||
} | |||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain, | vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
int req, vm_page_t mpred) | int req, vm_page_t mpred) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
vm_page_t m; | vm_page_t m; | ||||
int flags, req_class; | int flags; | ||||
u_int free_count; | u_int free_count; | ||||
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | ||||
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | ||||
Not Done Inline ActionsJust to be pedantic, I think we should re-evaluate this after dropping the object lock, i.e., move this to after the "again" label. markj: Just to be pedantic, I think we should re-evaluate this after dropping the object lock, i.e. | |||||
((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ||||
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | ||||
("inconsistent object(%p)/req(%x)", object, req)); | ("inconsistent object(%p)/req(%x)", object, req)); | ||||
KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | ||||
("Can't sleep and retry object insertion.")); | ("Can't sleep and retry object insertion.")); | ||||
KASSERT(mpred == NULL || mpred->pindex < pindex, | KASSERT(mpred == NULL || mpred->pindex < pindex, | ||||
("mpred %p doesn't precede pindex 0x%jx", mpred, | ("mpred %p doesn't precede pindex 0x%jx", mpred, | ||||
(uintmax_t)pindex)); | (uintmax_t)pindex)); | ||||
if (object != NULL) | if (object != NULL) | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
req_class = req & VM_ALLOC_CLASS_MASK; | |||||
/* | |||||
* The page daemon is allowed to dig deeper into the free page list. | |||||
*/ | |||||
if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) | |||||
req_class = VM_ALLOC_SYSTEM; | |||||
/* | |||||
* Allocate a page if the number of free pages exceeds the minimum | |||||
* for the request class. | |||||
*/ | |||||
again: | again: | ||||
m = NULL; | m = NULL; | ||||
mtx_lock(&vm_page_queue_free_mtx); | #if VM_NRESERVLEVEL > 0 | ||||
if (vm_cnt.v_free_count > vm_cnt.v_free_reserved || | if (vm_object_reserv(object) && | ||||
(req_class == VM_ALLOC_SYSTEM && | (m = vm_reserv_extend(req, object, pindex, domain, mpred)) | ||||
vm_cnt.v_free_count > vm_cnt.v_interrupt_free_min) || | != NULL) { | ||||
(req_class == VM_ALLOC_INTERRUPT && | domain = vm_phys_domain(m); | ||||
vm_cnt.v_free_count > 0)) { | vmd = VM_DOMAIN(domain); | ||||
goto found; | |||||
} | |||||
#endif | |||||
vmd = VM_DOMAIN(domain); | |||||
vm_domain_free_lock(vmd); | |||||
if (vm_domain_available(vmd, req, 1)) { | |||||
/* | /* | ||||
* Can we allocate the page from a reservation? | * Can we allocate the page from a reservation? | ||||
*/ | */ | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (object == NULL || (object->flags & (OBJ_COLORED | | if (!vm_object_reserv(object) || | ||||
OBJ_FICTITIOUS)) != OBJ_COLORED || (m = | (m = vm_reserv_alloc_page(object, pindex, | ||||
vm_reserv_alloc_page(object, pindex, domain, | domain, mpred)) == NULL) | ||||
mpred)) == NULL) | |||||
#endif | #endif | ||||
{ | { | ||||
/* | /* | ||||
* If not, allocate it from the free page queues. | * If not, allocate it from the free page queues. | ||||
*/ | */ | ||||
m = vm_phys_alloc_pages(domain, object != NULL ? | m = vm_phys_alloc_pages(domain, object != NULL ? | ||||
VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); | VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (m == NULL && vm_reserv_reclaim_inactive(domain)) { | if (m == NULL && vm_reserv_reclaim_inactive(domain)) { | ||||
m = vm_phys_alloc_pages(domain, | m = vm_phys_alloc_pages(domain, | ||||
object != NULL ? | object != NULL ? | ||||
VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, | VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, | ||||
0); | 0); | ||||
} | } | ||||
#endif | #endif | ||||
} | } | ||||
} | } | ||||
if (m == NULL) { | if (m == NULL) { | ||||
/* | /* | ||||
* Not allocatable, give up. | * Not allocatable, give up. | ||||
*/ | */ | ||||
if (vm_page_alloc_fail(object, req)) | if (vm_domain_alloc_fail(vmd, object, req)) | ||||
goto again; | goto again; | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | /* | ||||
* At this point we had better have found a good page. | * At this point we had better have found a good page. | ||||
*/ | */ | ||||
KASSERT(m != NULL, ("missing page")); | KASSERT(m != NULL, ("missing page")); | ||||
free_count = vm_phys_freecnt_adj(m, -1); | free_count = vm_domain_freecnt_adj(vmd, -1); | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
/* | |||||
* Don't wakeup too often - wakeup the pageout daemon when | |||||
* we would be nearly out of memory. | |||||
*/ | |||||
if (vm_paging_needed(vmd, free_count)) | |||||
pagedaemon_wakeup(vmd->vmd_domain); | |||||
#if VM_NRESERVLEVEL > 0 | |||||
found: | |||||
#endif | |||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
/* | /* | ||||
* Initialize the page. Only the PG_ZERO flag is inherited. | * Initialize the page. Only the PG_ZERO flag is inherited. | ||||
*/ | */ | ||||
flags = 0; | flags = 0; | ||||
if ((req & VM_ALLOC_ZERO) != 0) | if ((req & VM_ALLOC_ZERO) != 0) | ||||
flags = PG_ZERO; | flags = PG_ZERO; | ||||
Show All 16 Lines | if (req & VM_ALLOC_WIRED) { | ||||
*/ | */ | ||||
atomic_add_int(&vm_cnt.v_wire_count, 1); | atomic_add_int(&vm_cnt.v_wire_count, 1); | ||||
m->wire_count = 1; | m->wire_count = 1; | ||||
} | } | ||||
m->act_count = 0; | m->act_count = 0; | ||||
if (object != NULL) { | if (object != NULL) { | ||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
pagedaemon_wakeup(); | pagedaemon_wakeup(domain); | ||||
if (req & VM_ALLOC_WIRED) { | if (req & VM_ALLOC_WIRED) { | ||||
atomic_subtract_int(&vm_cnt.v_wire_count, 1); | atomic_subtract_int(&vm_cnt.v_wire_count, 1); | ||||
m->wire_count = 0; | m->wire_count = 0; | ||||
} | } | ||||
KASSERT(m->object == NULL, ("page %p has object", m)); | KASSERT(m->object == NULL, ("page %p has object", m)); | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
/* Don't change PG_ZERO. */ | /* Don't change PG_ZERO. */ | ||||
vm_page_free_toq(m); | vm_page_free_toq(m); | ||||
if (req & VM_ALLOC_WAITFAIL) { | if (req & VM_ALLOC_WAITFAIL) { | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
vm_radix_wait(); | vm_radix_wait(); | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* Ignore device objects; the pager sets "memattr" for them. */ | /* Ignore device objects; the pager sets "memattr" for them. */ | ||||
if (object->memattr != VM_MEMATTR_DEFAULT && | if (object->memattr != VM_MEMATTR_DEFAULT && | ||||
(object->flags & OBJ_FICTITIOUS) == 0) | (object->flags & OBJ_FICTITIOUS) == 0) | ||||
pmap_page_set_memattr(m, object->memattr); | pmap_page_set_memattr(m, object->memattr); | ||||
} else | } else | ||||
m->pindex = pindex; | m->pindex = pindex; | ||||
/* | |||||
* Don't wakeup too often - wakeup the pageout daemon when | |||||
* we would be nearly out of memory. | |||||
*/ | |||||
if (vm_paging_needed(free_count)) | |||||
pagedaemon_wakeup(); | |||||
return (m); | return (m); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_alloc_contig: | * vm_page_alloc_contig: | ||||
* | * | ||||
* Allocate a contiguous set of physical pages of the given size "npages" | * Allocate a contiguous set of physical pages of the given size "npages" | ||||
* from the free lists. All of the physical pages must be at or above | * from the free lists. All of the physical pages must be at or above | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, | ||||
return (m); | return (m); | ||||
} | } | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain, | vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | ||||
vm_paddr_t boundary, vm_memattr_t memattr) | vm_paddr_t boundary, vm_memattr_t memattr) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
vm_page_t m, m_ret, mpred; | vm_page_t m, m_ret, mpred; | ||||
u_int busy_lock, flags, oflags; | u_int busy_lock, flags, oflags; | ||||
int req_class; | |||||
mpred = NULL; /* XXX: pacify gcc */ | mpred = NULL; /* XXX: pacify gcc */ | ||||
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | ||||
Done Inline ActionsThis should probably be a vm_object inline. jeff: This should probably be a vm_object inline. | |||||
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | ||||
((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ||||
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | ||||
("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object, | ("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object, | ||||
req)); | req)); | ||||
KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | ||||
("Can't sleep and retry object insertion.")); | ("Can't sleep and retry object insertion.")); | ||||
if (object != NULL) { | if (object != NULL) { | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT((object->flags & OBJ_FICTITIOUS) == 0, | KASSERT((object->flags & OBJ_FICTITIOUS) == 0, | ||||
("vm_page_alloc_contig: object %p has fictitious pages", | ("vm_page_alloc_contig: object %p has fictitious pages", | ||||
object)); | object)); | ||||
} | } | ||||
KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); | ||||
req_class = req & VM_ALLOC_CLASS_MASK; | |||||
/* | |||||
* The page daemon is allowed to dig deeper into the free page list. | |||||
*/ | |||||
if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) | |||||
req_class = VM_ALLOC_SYSTEM; | |||||
if (object != NULL) { | if (object != NULL) { | ||||
mpred = vm_radix_lookup_le(&object->rtree, pindex); | mpred = vm_radix_lookup_le(&object->rtree, pindex); | ||||
KASSERT(mpred == NULL || mpred->pindex != pindex, | KASSERT(mpred == NULL || mpred->pindex != pindex, | ||||
("vm_page_alloc_contig: pindex already allocated")); | ("vm_page_alloc_contig: pindex already allocated")); | ||||
} | } | ||||
/* | /* | ||||
* Can we allocate the pages without the number of free pages falling | * Can we allocate the pages without the number of free pages falling | ||||
* below the lower bound for the allocation class? | * below the lower bound for the allocation class? | ||||
*/ | */ | ||||
again: | again: | ||||
#if VM_NRESERVLEVEL > 0 | |||||
if (vm_object_reserv(object) && | |||||
(m_ret = vm_reserv_extend_contig(req, object, pindex, domain, | |||||
npages, low, high, alignment, boundary, mpred)) != NULL) { | |||||
domain = vm_phys_domain(m_ret); | |||||
vmd = VM_DOMAIN(domain); | |||||
goto found; | |||||
} | |||||
#endif | |||||
m_ret = NULL; | m_ret = NULL; | ||||
mtx_lock(&vm_page_queue_free_mtx); | vmd = VM_DOMAIN(domain); | ||||
if (vm_cnt.v_free_count >= npages + vm_cnt.v_free_reserved || | vm_domain_free_lock(vmd); | ||||
(req_class == VM_ALLOC_SYSTEM && | if (vm_domain_available(vmd, req, npages)) { | ||||
vm_cnt.v_free_count >= npages + vm_cnt.v_interrupt_free_min) || | |||||
(req_class == VM_ALLOC_INTERRUPT && | |||||
vm_cnt.v_free_count >= npages)) { | |||||
/* | /* | ||||
* Can we allocate the pages from a reservation? | * Can we allocate the pages from a reservation? | ||||
*/ | */ | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
retry: | retry: | ||||
if (object == NULL || (object->flags & OBJ_COLORED) == 0 || | if (!vm_object_reserv(object) || | ||||
(m_ret = vm_reserv_alloc_contig(object, pindex, domain, | (m_ret = vm_reserv_alloc_contig(object, pindex, domain, | ||||
npages, low, high, alignment, boundary, mpred)) == NULL) | npages, low, high, alignment, boundary, mpred)) == NULL) | ||||
#endif | #endif | ||||
/* | /* | ||||
* If not, allocate them from the free page queues. | * If not, allocate them from the free page queues. | ||||
*/ | */ | ||||
m_ret = vm_phys_alloc_contig(domain, npages, low, high, | m_ret = vm_phys_alloc_contig(domain, npages, low, high, | ||||
alignment, boundary); | alignment, boundary); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (m_ret == NULL && vm_reserv_reclaim_contig( | if (m_ret == NULL && vm_reserv_reclaim_contig( | ||||
domain, npages, low, high, alignment, boundary)) | domain, npages, low, high, alignment, boundary)) | ||||
goto retry; | goto retry; | ||||
#endif | #endif | ||||
} | } | ||||
if (m_ret == NULL) { | if (m_ret == NULL) { | ||||
if (vm_page_alloc_fail(object, req)) | if (vm_domain_alloc_fail(vmd, object, req)) | ||||
kibUnsubmitted Done Inline ActionsThe issue with the unfair allocation after the wait is still there, and perhaps it is even exaggerated. After we did full iteration over all domains and all domains failed to provide an allocatable page, we sleep and try to allocate from the initial domain, forever. I believe that vm_domainset_lock and vm_min_domains bitset do allow to solve this correctly. kib: The issue with the unfair allocation after the wait is still there, and perhaps it is even… | |||||
jeffAuthorUnsubmitted Done Inline ActionsWhat I would like to do is further eliminate calls to VM_WAIT() and then pass the failing object in on the few that remain. If that is not safe I will pass the failing domainset which is. I didn't want to deviate too much from the existing code with this patch. I will refactor this entirely when I bring in the pid controller since you can rely on different behavior from the pagedaemon at the same time. jeff: What I would like to do is further eliminate calls to VM_WAIT() and then pass the failing… | |||||
kibUnsubmitted Done Inline ActionsI initially thought that implementing the fairness would be possible but somewhat intensive. Apparently, it seems to be quite easy. If you only put the thread to sleep when iterator restarts, instead of sleeping on each step after the first full iteration, I believe the issue of fairness will be solved. I mean that the thread would only sleep once between attempts to allocate from all possible domains, which is the same behavior as it has now with the non-numa config. kib: I initially thought that implementing the fairness would be possible but somewhat intensive. | |||||
jeffAuthorUnsubmitted Not Done Inline ActionsI agree. The addition of the bitsets makes this possible and it makes it possible to do precise sleeps on sets of domains. There is a global lock on memory exhaustion but I believe proper pageout regulation will make this quite rare in cases that are not I/O bound. If you're I/O bound and out of memory a global lock is inconsequential. I will make this change when I bring in the pid control so I can test the changes to the regulation mechanism in concert. jeff: I agree. The addition of the bitsets makes this possible and it makes it possible to do… | |||||
goto again; | goto again; | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
vm_phys_freecnt_adj(m_ret, -npages); | vm_domain_freecnt_adj(vmd, -npages); | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
#if VM_NRESERVLEVEL > 0 | |||||
found: | |||||
#endif | |||||
for (m = m_ret; m < &m_ret[npages]; m++) | for (m = m_ret; m < &m_ret[npages]; m++) | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
/* | /* | ||||
* Initialize the pages. Only the PG_ZERO flag is inherited. | * Initialize the pages. Only the PG_ZERO flag is inherited. | ||||
*/ | */ | ||||
flags = 0; | flags = 0; | ||||
if ((req & VM_ALLOC_ZERO) != 0) | if ((req & VM_ALLOC_ZERO) != 0) | ||||
Show All 19 Lines | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
m->flags = (m->flags | PG_NODUMP) & flags; | m->flags = (m->flags | PG_NODUMP) & flags; | ||||
m->busy_lock = busy_lock; | m->busy_lock = busy_lock; | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
m->wire_count = 1; | m->wire_count = 1; | ||||
m->act_count = 0; | m->act_count = 0; | ||||
m->oflags = oflags; | m->oflags = oflags; | ||||
if (object != NULL) { | if (object != NULL) { | ||||
if (vm_page_insert_after(m, object, pindex, mpred)) { | if (vm_page_insert_after(m, object, pindex, mpred)) { | ||||
pagedaemon_wakeup(); | pagedaemon_wakeup(domain); | ||||
if ((req & VM_ALLOC_WIRED) != 0) | if ((req & VM_ALLOC_WIRED) != 0) | ||||
atomic_subtract_int( | atomic_subtract_int( | ||||
&vm_cnt.v_wire_count, npages); | &vm_cnt.v_wire_count, npages); | ||||
KASSERT(m->object == NULL, | KASSERT(m->object == NULL, | ||||
("page %p has object", m)); | ("page %p has object", m)); | ||||
mpred = m; | mpred = m; | ||||
for (m = m_ret; m < &m_ret[npages]; m++) { | for (m = m_ret; m < &m_ret[npages]; m++) { | ||||
if (m <= mpred && | if (m <= mpred && | ||||
Show All 13 Lines | if (object != NULL) { | ||||
} | } | ||||
mpred = m; | mpred = m; | ||||
} else | } else | ||||
m->pindex = pindex; | m->pindex = pindex; | ||||
if (memattr != VM_MEMATTR_DEFAULT) | if (memattr != VM_MEMATTR_DEFAULT) | ||||
pmap_page_set_memattr(m, memattr); | pmap_page_set_memattr(m, memattr); | ||||
pindex++; | pindex++; | ||||
} | } | ||||
if (vm_paging_needed(vm_cnt.v_free_count)) | vmd = VM_DOMAIN(domain); | ||||
pagedaemon_wakeup(); | if (vm_paging_needed(vmd, vmd->vmd_free_count)) | ||||
pagedaemon_wakeup(domain); | |||||
return (m_ret); | return (m_ret); | ||||
} | } | ||||
/* | /* | ||||
* Check a page that has been freshly dequeued from a freelist. | * Check a page that has been freshly dequeued from a freelist. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_alloc_check(vm_page_t m) | vm_page_alloc_check(vm_page_t m) | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | vm_page_alloc_freelist(int freelist, int req) | ||||
} while (vm_domainset_iter_page(&di, &domain, &req) == 0); | } while (vm_domainset_iter_page(&di, &domain, &req) == 0); | ||||
return (m); | return (m); | ||||
} | } | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_freelist_domain(int domain, int freelist, int req) | vm_page_alloc_freelist_domain(int domain, int freelist, int req) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
vm_page_t m; | vm_page_t m; | ||||
u_int flags, free_count; | u_int flags, free_count; | ||||
int req_class; | |||||
req_class = req & VM_ALLOC_CLASS_MASK; | |||||
/* | /* | ||||
* The page daemon is allowed to dig deeper into the free page list. | |||||
*/ | |||||
if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) | |||||
req_class = VM_ALLOC_SYSTEM; | |||||
/* | |||||
* Do not allocate reserved pages unless the req has asked for it. | * Do not allocate reserved pages unless the req has asked for it. | ||||
*/ | */ | ||||
vmd = VM_DOMAIN(domain); | |||||
again: | again: | ||||
mtx_lock(&vm_page_queue_free_mtx); | vm_domain_free_lock(vmd); | ||||
if (vm_cnt.v_free_count > vm_cnt.v_free_reserved || | if (vm_domain_available(vmd, req, 1)) | ||||
(req_class == VM_ALLOC_SYSTEM && | |||||
vm_cnt.v_free_count > vm_cnt.v_interrupt_free_min) || | |||||
(req_class == VM_ALLOC_INTERRUPT && | |||||
vm_cnt.v_free_count > 0)) | |||||
m = vm_phys_alloc_freelist_pages(domain, freelist, | m = vm_phys_alloc_freelist_pages(domain, freelist, | ||||
VM_FREEPOOL_DIRECT, 0); | VM_FREEPOOL_DIRECT, 0); | ||||
if (m == NULL) { | if (m == NULL) { | ||||
if (vm_page_alloc_fail(NULL, req)) | if (vm_domain_alloc_fail(vmd, NULL, req)) | ||||
goto again; | goto again; | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
free_count = vm_phys_freecnt_adj(m, -1); | free_count = vm_domain_freecnt_adj(vmd, -1); | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
/* | /* | ||||
* Initialize the page. Only the PG_ZERO flag is inherited. | * Initialize the page. Only the PG_ZERO flag is inherited. | ||||
*/ | */ | ||||
m->aflags = 0; | m->aflags = 0; | ||||
flags = 0; | flags = 0; | ||||
if ((req & VM_ALLOC_ZERO) != 0) | if ((req & VM_ALLOC_ZERO) != 0) | ||||
flags = PG_ZERO; | flags = PG_ZERO; | ||||
m->flags &= flags; | m->flags &= flags; | ||||
if ((req & VM_ALLOC_WIRED) != 0) { | if ((req & VM_ALLOC_WIRED) != 0) { | ||||
/* | /* | ||||
* The page lock is not required for wiring a page that does | * The page lock is not required for wiring a page that does | ||||
* not belong to an object. | * not belong to an object. | ||||
*/ | */ | ||||
atomic_add_int(&vm_cnt.v_wire_count, 1); | atomic_add_int(&vm_cnt.v_wire_count, 1); | ||||
m->wire_count = 1; | m->wire_count = 1; | ||||
} | } | ||||
/* Unmanaged pages don't use "act_count". */ | /* Unmanaged pages don't use "act_count". */ | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
if (vm_paging_needed(free_count)) | if (vm_paging_needed(vmd, free_count)) | ||||
pagedaemon_wakeup(); | pagedaemon_wakeup(domain); | ||||
return (m); | return (m); | ||||
} | } | ||||
#define VPSC_ANY 0 /* No restrictions. */ | #define VPSC_ANY 0 /* No restrictions. */ | ||||
#define VPSC_NORESERV 1 /* Skip reservations; implies VPSC_NOSUPER. */ | #define VPSC_NORESERV 1 /* Skip reservations; implies VPSC_NOSUPER. */ | ||||
#define VPSC_NOSUPER 2 /* Skip superpages. */ | #define VPSC_NOSUPER 2 /* Skip superpages. */ | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 205 Lines • ▼ Show 20 Lines | |||||
* Returns 0 if every physical page within the run was already free or | * Returns 0 if every physical page within the run was already free or | ||||
* just freed by a successful relocation. Otherwise, returns a non-zero | * just freed by a successful relocation. Otherwise, returns a non-zero | ||||
* value indicating why the last attempt to relocate a virtual page was | * value indicating why the last attempt to relocate a virtual page was | ||||
* unsuccessful. | * unsuccessful. | ||||
* | * | ||||
* "req_class" must be an allocation class. | * "req_class" must be an allocation class. | ||||
*/ | */ | ||||
static int | static int | ||||
vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run, | vm_page_reclaim_run(int req_class, int domain, u_long npages, vm_page_t m_run, | ||||
vm_paddr_t high) | vm_paddr_t high) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
struct mtx *m_mtx; | struct mtx *m_mtx; | ||||
struct spglist free; | struct spglist free; | ||||
vm_object_t object; | vm_object_t object; | ||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||
vm_page_t m, m_end, m_new; | vm_page_t m, m_end, m_new; | ||||
int error, order, req; | int error, order, req; | ||||
KASSERT((req_class & VM_ALLOC_CLASS_MASK) == req_class, | KASSERT((req_class & VM_ALLOC_CLASS_MASK) == req_class, | ||||
▲ Show 20 Lines • Show All 133 Lines • ▼ Show 20 Lines | else if ((object = m->object) != NULL) { | ||||
("page %p is dirty", m)); | ("page %p is dirty", m)); | ||||
} | } | ||||
SLIST_INSERT_HEAD(&free, m, plinks.s.ss); | SLIST_INSERT_HEAD(&free, m, plinks.s.ss); | ||||
} else | } else | ||||
error = EBUSY; | error = EBUSY; | ||||
unlock: | unlock: | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
} else { | } else { | ||||
mtx_lock(&vm_page_queue_free_mtx); | MPASS(vm_phys_domain(m) == domain); | ||||
vmd = VM_DOMAIN(domain); | |||||
vm_domain_free_lock(vmd); | |||||
order = m->order; | order = m->order; | ||||
if (order < VM_NFREEORDER) { | if (order < VM_NFREEORDER) { | ||||
/* | /* | ||||
* The page is enqueued in the physical memory | * The page is enqueued in the physical memory | ||||
* allocator's free page queues. Moreover, it | * allocator's free page queues. Moreover, it | ||||
* is the first page in a power-of-two-sized | * is the first page in a power-of-two-sized | ||||
* run of contiguous free pages. Jump ahead | * run of contiguous free pages. Jump ahead | ||||
* to the last page within that run, and | * to the last page within that run, and | ||||
* continue from there. | * continue from there. | ||||
*/ | */ | ||||
m += (1 << order) - 1; | m += (1 << order) - 1; | ||||
} | } | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
else if (vm_reserv_is_page_free(m)) | else if (vm_reserv_is_page_free(m)) | ||||
order = 0; | order = 0; | ||||
#endif | #endif | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
if (order == VM_NFREEORDER) | if (order == VM_NFREEORDER) | ||||
error = EINVAL; | error = EINVAL; | ||||
} | } | ||||
} | } | ||||
if (m_mtx != NULL) | if (m_mtx != NULL) | ||||
mtx_unlock(m_mtx); | mtx_unlock(m_mtx); | ||||
if ((m = SLIST_FIRST(&free)) != NULL) { | if ((m = SLIST_FIRST(&free)) != NULL) { | ||||
mtx_lock(&vm_page_queue_free_mtx); | vmd = VM_DOMAIN(domain); | ||||
vm_domain_free_lock(vmd); | |||||
Done Inline ActionsI believe this is true since we are operating on a contiguous region of memory. Should probably assert below in the loop instead. jeff: I believe this is true since we are operating on a contiguous region of memory. Should… | |||||
do { | do { | ||||
MPASS(vm_phys_domain(m) == domain); | |||||
SLIST_REMOVE_HEAD(&free, plinks.s.ss); | SLIST_REMOVE_HEAD(&free, plinks.s.ss); | ||||
vm_page_free_phys(m); | vm_page_free_phys(m); | ||||
} while ((m = SLIST_FIRST(&free)) != NULL); | } while ((m = SLIST_FIRST(&free)) != NULL); | ||||
vm_page_free_wakeup(); | vm_domain_free_wakeup(vmd); | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
#define NRUNS 16 | #define NRUNS 16 | ||||
CTASSERT(powerof2(NRUNS)); | CTASSERT(powerof2(NRUNS)); | ||||
Show All 23 Lines | |||||
* | * | ||||
* "npages" must be greater than zero. Both "alignment" and "boundary" | * "npages" must be greater than zero. Both "alignment" and "boundary" | ||||
* must be a power of two. | * must be a power of two. | ||||
*/ | */ | ||||
bool | bool | ||||
vm_page_reclaim_contig_domain(int domain, int req, u_long npages, | vm_page_reclaim_contig_domain(int domain, int req, u_long npages, | ||||
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
vm_paddr_t curr_low; | vm_paddr_t curr_low; | ||||
vm_page_t m_run, m_runs[NRUNS]; | vm_page_t m_run, m_runs[NRUNS]; | ||||
u_long count, reclaimed; | u_long count, reclaimed; | ||||
int error, i, options, req_class; | int error, i, options, req_class; | ||||
KASSERT(npages > 0, ("npages is 0")); | KASSERT(npages > 0, ("npages is 0")); | ||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | ||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | ||||
req_class = req & VM_ALLOC_CLASS_MASK; | req_class = req & VM_ALLOC_CLASS_MASK; | ||||
/* | /* | ||||
* The page daemon is allowed to dig deeper into the free page list. | * The page daemon is allowed to dig deeper into the free page list. | ||||
*/ | */ | ||||
if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) | if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) | ||||
req_class = VM_ALLOC_SYSTEM; | req_class = VM_ALLOC_SYSTEM; | ||||
/* | /* | ||||
* Return if the number of free pages cannot satisfy the requested | * Return if the number of free pages cannot satisfy the requested | ||||
* allocation. | * allocation. | ||||
*/ | */ | ||||
count = vm_cnt.v_free_count; | vmd = VM_DOMAIN(domain); | ||||
if (count < npages + vm_cnt.v_free_reserved || (count < npages + | count = vmd->vmd_free_count; | ||||
vm_cnt.v_interrupt_free_min && req_class == VM_ALLOC_SYSTEM) || | if (count < npages + vmd->vmd_free_reserved || (count < npages + | ||||
vmd->vmd_interrupt_free_min && req_class == VM_ALLOC_SYSTEM) || | |||||
(count < npages && req_class == VM_ALLOC_INTERRUPT)) | (count < npages && req_class == VM_ALLOC_INTERRUPT)) | ||||
return (false); | return (false); | ||||
/* | /* | ||||
* Scan up to three times, relaxing the restrictions ("options") on | * Scan up to three times, relaxing the restrictions ("options") on | ||||
* the reclamation of reservations and superpages each time. | * the reclamation of reservations and superpages each time. | ||||
*/ | */ | ||||
for (options = VPSC_NORESERV;;) { | for (options = VPSC_NORESERV;;) { | ||||
Show All 19 Lines | for (options = VPSC_NORESERV;;) { | ||||
* MIN_RECLAIM. Reset "reclaimed" each time because each | * MIN_RECLAIM. Reset "reclaimed" each time because each | ||||
* reclamation is idempotent, and runs will (likely) recur | * reclamation is idempotent, and runs will (likely) recur | ||||
* from one scan to the next as restrictions are relaxed. | * from one scan to the next as restrictions are relaxed. | ||||
*/ | */ | ||||
reclaimed = 0; | reclaimed = 0; | ||||
for (i = 0; count > 0 && i < NRUNS; i++) { | for (i = 0; count > 0 && i < NRUNS; i++) { | ||||
count--; | count--; | ||||
m_run = m_runs[RUN_INDEX(count)]; | m_run = m_runs[RUN_INDEX(count)]; | ||||
error = vm_page_reclaim_run(req_class, npages, m_run, | error = vm_page_reclaim_run(req_class, domain, npages, | ||||
high); | m_run, high); | ||||
if (error == 0) { | if (error == 0) { | ||||
reclaimed += npages; | reclaimed += npages; | ||||
if (reclaimed >= MIN_RECLAIM) | if (reclaimed >= MIN_RECLAIM) | ||||
return (true); | return (true); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
Show All 23 Lines | ret = vm_page_reclaim_contig_domain(domain, req, npages, low, | ||||
high, alignment, boundary); | high, alignment, boundary); | ||||
if (ret) | if (ret) | ||||
break; | break; | ||||
} while (vm_domainset_iter_page(&di, &domain, &req) == 0); | } while (vm_domainset_iter_page(&di, &domain, &req) == 0); | ||||
return (ret); | return (ret); | ||||
} | } | ||||
/* | |||||
* Set the domain in the appropriate page level domainset. | |||||
*/ | |||||
void | |||||
vm_domain_set(struct vm_domain *vmd) | |||||
{ | |||||
mtx_lock(&vm_domainset_lock); | |||||
if (!vmd->vmd_minset && vm_paging_min(vmd)) { | |||||
vmd->vmd_minset = 1; | |||||
DOMAINSET_SET(vmd->vmd_domain, &vm_min_domains); | |||||
} | |||||
if (!vmd->vmd_severeset && vm_paging_severe(vmd)) { | |||||
vmd->vmd_severeset = 1; | |||||
DOMAINSET_CLR(vmd->vmd_domain, &vm_severe_domains); | |||||
} | |||||
mtx_unlock(&vm_domainset_lock); | |||||
} | |||||
/* | /* | ||||
* vm_wait: (also see VM_WAIT macro) | * Clear the domain from the appropriate page level domainset. | ||||
*/ | |||||
static void | |||||
vm_domain_clear(struct vm_domain *vmd) | |||||
{ | |||||
mtx_lock(&vm_domainset_lock); | |||||
if (vmd->vmd_minset && !vm_paging_min(vmd)) { | |||||
vmd->vmd_minset = 0; | |||||
DOMAINSET_CLR(vmd->vmd_domain, &vm_min_domains); | |||||
if (vm_min_waiters != 0) { | |||||
vm_min_waiters = 0; | |||||
wakeup(&vm_min_domains); | |||||
} | |||||
} | |||||
if (vmd->vmd_severeset && !vm_paging_severe(vmd)) { | |||||
vmd->vmd_severeset = 0; | |||||
DOMAINSET_CLR(vmd->vmd_domain, &vm_severe_domains); | |||||
if (vm_severe_waiters != 0) { | |||||
vm_severe_waiters = 0; | |||||
wakeup(&vm_severe_domains); | |||||
} | |||||
} | |||||
mtx_unlock(&vm_domainset_lock); | |||||
} | |||||
/* | |||||
* Wait for free pages to exceed the min threshold globally. | |||||
*/ | |||||
void | |||||
vm_wait_min(void) | |||||
{ | |||||
mtx_lock(&vm_domainset_lock); | |||||
while (vm_page_count_min()) { | |||||
vm_min_waiters++; | |||||
msleep(&vm_min_domains, &vm_domainset_lock, PVM, "vmwait", 0); | |||||
} | |||||
mtx_unlock(&vm_domainset_lock); | |||||
} | |||||
/* | |||||
* Wait for free pages to exceed the severe threshold globally. | |||||
*/ | |||||
void | |||||
vm_wait_severe(void) | |||||
{ | |||||
mtx_lock(&vm_domainset_lock); | |||||
while (vm_page_count_severe()) { | |||||
Done Inline ActionsThis should be vm_page_count_severe(). markj: This should be vm_page_count_severe(). | |||||
vm_severe_waiters++; | |||||
msleep(&vm_min_domains, &vm_domainset_lock, PVM, "vmwait", 0); | |||||
kibUnsubmitted Done Inline ActionsShould this wait performed on the &vm_severe_domains ? kib: Should this wait performed on the &vm_severe_domains ? | |||||
} | |||||
mtx_unlock(&vm_domainset_lock); | |||||
} | |||||
u_int | |||||
vm_wait_count(void) | |||||
{ | |||||
u_int cnt; | |||||
int i; | |||||
cnt = 0; | |||||
for (i = 0; i < vm_ndomains; i++) | |||||
cnt += VM_DOMAIN(i)->vmd_waiters; | |||||
cnt += vm_severe_waiters + vm_min_waiters; | |||||
return (cnt); | |||||
} | |||||
/* | |||||
* vm_wait_domain: | |||||
* | * | ||||
* Sleep until free pages are available for allocation. | * Sleep until free pages are available for allocation. | ||||
* - Called in various places before memory allocations. | * - Called in various places after failed memory allocations. | ||||
*/ | */ | ||||
static void | void | ||||
_vm_wait(void) | vm_wait_domain(int domain) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vmd = VM_DOMAIN(domain); | ||||
vm_domain_free_assert_locked(vmd); | |||||
if (curproc == pageproc) { | if (curproc == pageproc) { | ||||
vm_pageout_pages_needed = 1; | vmd->vmd_pageout_pages_needed = 1; | ||||
msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx, | msleep(&vmd->vmd_pageout_pages_needed, | ||||
PDROP | PSWP, "VMWait", 0); | vm_domain_free_lockptr(vmd), PDROP | PSWP, "VMWait", 0); | ||||
} else { | } else { | ||||
if (pageproc == NULL) | if (pageproc == NULL) | ||||
panic("vm_wait in early boot"); | panic("vm_wait in early boot"); | ||||
pagedaemon_wait(PVM, "vmwait"); | pagedaemon_wait(domain, PVM, "vmwait"); | ||||
} | } | ||||
} | } | ||||
/* | |||||
* vm_wait: (also see VM_WAIT macro) | |||||
* | |||||
* Sleep until free pages are available for allocation. | |||||
* - Called in various places after failed memory allocations. | |||||
*/ | |||||
void | void | ||||
vm_wait(void) | vm_wait(void) | ||||
{ | { | ||||
mtx_lock(&vm_page_queue_free_mtx); | /* | ||||
_vm_wait(); | * We use racey wakeup synchronization to avoid expensive global | ||||
* locking for the pageproc when sleeping with a non-specific vm_wait. | |||||
* To handle this, we only sleep for one tick in this instance. It | |||||
* is expected that most allocations for the pageproc will come from | |||||
* kmem or vm_page_grab* which will use the more specific and | |||||
* race-free vm_wait_domain(). | |||||
*/ | |||||
if (curproc == pageproc) { | |||||
mtx_lock(&vm_domainset_lock); | |||||
vm_pageproc_waiters++; | |||||
msleep(&vm_pageproc_waiters, &vm_domainset_lock, PVM, | |||||
"pageprocwait", 1); | |||||
mtx_unlock(&vm_domainset_lock); | |||||
} else { | |||||
/* | |||||
* XXX Ideally we would wait only until the allocation could | |||||
* be satisfied. This condition can cause new allocators to | |||||
* consume all freed pages while old allocators wait. | |||||
*/ | |||||
mtx_lock(&vm_domainset_lock); | |||||
if (vm_page_count_min()) { | |||||
vm_min_waiters++; | |||||
msleep(&vm_min_domains, &vm_domainset_lock, PVM, | |||||
"vmwait", 0); | |||||
} | } | ||||
mtx_unlock(&vm_domainset_lock); | |||||
} | |||||
} | |||||
Not Done Inline ActionsPlease review this in particular. jeff: Please review this in particular. | |||||
/* | /* | ||||
* vm_page_alloc_fail: | * vm_domain_alloc_fail: | ||||
* | * | ||||
* Called when a page allocation function fails. Informs the | * Called when a page allocation function fails. Informs the | ||||
* pagedaemon and performs the requested wait. Requires the | * pagedaemon and performs the requested wait. Requires the | ||||
* page_queue_free and object lock on entry. Returns with the | * domain_free and object lock on entry. Returns with the | ||||
* object lock held and free lock released. Returns an error when | * object lock held and free lock released. Returns an error when | ||||
* retry is necessary. | * retry is necessary. | ||||
* | * | ||||
*/ | */ | ||||
static int | static int | ||||
kibUnsubmitted Not Done Inline ActionsI suggest changing the return type to bool. kib: I suggest changing the return type to bool. | |||||
vm_page_alloc_fail(vm_object_t object, int req) | vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, int req) | ||||
{ | { | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(vmd); | ||||
atomic_add_int(&vm_pageout_deficit, | atomic_add_int(&vmd->vmd_pageout_deficit, | ||||
max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); | max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); | ||||
if (req & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) { | if (req & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) { | ||||
if (object != NULL) | if (object != NULL) | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
_vm_wait(); | vm_wait_domain(vmd->vmd_domain); | ||||
if (object != NULL) | if (object != NULL) | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
if (req & VM_ALLOC_WAITOK) | if (req & VM_ALLOC_WAITOK) | ||||
return (EAGAIN); | return (EAGAIN); | ||||
} else { | } else { | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
pagedaemon_wakeup(); | pagedaemon_wakeup(vmd->vmd_domain); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* vm_waitpfault: (also see VM_WAITPFAULT macro) | * vm_waitpfault: (also see VM_WAITPFAULT macro) | ||||
* | * | ||||
* Sleep until free pages are available for allocation. | * Sleep until free pages are available for allocation. | ||||
* - Called only in vm_fault so that processes page faulting | * - Called only in vm_fault so that processes page faulting | ||||
* can be easily tracked. | * can be easily tracked. | ||||
* - Sleeps at a lower priority than vm_wait() so that vm_wait()ing | * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing | ||||
* processes will be able to grab memory first. Do not change | * processes will be able to grab memory first. Do not change | ||||
* this balance without careful testing first. | * this balance without careful testing first. | ||||
*/ | */ | ||||
void | void | ||||
vm_waitpfault(void) | vm_waitpfault(void) | ||||
{ | { | ||||
mtx_lock(&vm_page_queue_free_mtx); | mtx_lock(&vm_domainset_lock); | ||||
pagedaemon_wait(PUSER, "pfault"); | if (vm_page_count_min()) { | ||||
Done Inline ActionsThis seems overly conservative, though maybe not a problem in practice. Suppose that a thread's allocation policy restricts it to a single domain. Then it only makes sense to check that domain's free count. We could use the thread and object domain allocation policies to derive the set of domains that we care about, and restrict out attention to that set. Also, this is identical to vm_wait_min(); we might consider parameterizing the prio and wmesg. markj: This seems overly conservative, though maybe not a problem in practice. Suppose that a thread's… | |||||
Done Inline ActionsYou are right. I think if we sleep until any domain transitions above min we will be close to the original intent. I can wakeup on any bit being cleared in the set instead of every. Long term I would like to totally eliminate naked vm_wait() calls for something with more information. We could do better than sleeping for min if we knew the original 'req' parameter. It is only used in vm_fault and pmap now where locking is too complex to sleep in vm_page_alloc. vm_wait_min() still wants every. jeff: You are right. I think if we sleep until any domain transitions above min we will be close to… | |||||
vm_min_waiters++; | |||||
msleep(&vm_min_domains, &vm_domainset_lock, PUSER, "pfault", 0); | |||||
} | } | ||||
mtx_unlock(&vm_domainset_lock); | |||||
} | |||||
struct vm_pagequeue * | struct vm_pagequeue * | ||||
vm_page_pagequeue(vm_page_t m) | vm_page_pagequeue(vm_page_t m) | ||||
{ | { | ||||
if (vm_page_in_laundry(m)) | return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]); | ||||
Not Done Inline ActionsCould be an inline now? jeff: Could be an inline now? | |||||
return (&vm_dom[0].vmd_pagequeues[m->queue]); | |||||
else | |||||
return (&vm_phys_domain(m)->vmd_pagequeues[m->queue]); | |||||
} | } | ||||
/* | /* | ||||
* vm_page_dequeue: | * vm_page_dequeue: | ||||
* | * | ||||
* Remove the given page from its current page queue. | * Remove the given page from its current page queue. | ||||
* | * | ||||
* The page must be locked. | * The page must be locked. | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
vm_page_enqueue(uint8_t queue, vm_page_t m) | vm_page_enqueue(uint8_t queue, vm_page_t m) | ||||
{ | { | ||||
struct vm_pagequeue *pq; | struct vm_pagequeue *pq; | ||||
vm_page_lock_assert(m, MA_OWNED); | vm_page_lock_assert(m, MA_OWNED); | ||||
KASSERT(queue < PQ_COUNT, | KASSERT(queue < PQ_COUNT, | ||||
("vm_page_enqueue: invalid queue %u request for page %p", | ("vm_page_enqueue: invalid queue %u request for page %p", | ||||
queue, m)); | queue, m)); | ||||
if (queue == PQ_LAUNDRY || queue == PQ_UNSWAPPABLE) | pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue]; | ||||
pq = &vm_dom[0].vmd_pagequeues[queue]; | |||||
else | |||||
pq = &vm_phys_domain(m)->vmd_pagequeues[queue]; | |||||
vm_pagequeue_lock(pq); | vm_pagequeue_lock(pq); | ||||
m->queue = queue; | m->queue = queue; | ||||
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); | TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); | ||||
vm_pagequeue_cnt_inc(pq); | vm_pagequeue_cnt_inc(pq); | ||||
vm_pagequeue_unlock(pq); | vm_pagequeue_unlock(pq); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) { | ||||
("vm_page_activate: wired page %p is queued", m)); | ("vm_page_activate: wired page %p is queued", m)); | ||||
} else { | } else { | ||||
if (m->act_count < ACT_INIT) | if (m->act_count < ACT_INIT) | ||||
m->act_count = ACT_INIT; | m->act_count = ACT_INIT; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* vm_page_free_wakeup: | * vm_domain_free_wakeup: | ||||
* | * | ||||
* Helper routine for vm_page_free_toq(). This routine is called | * Helper routine for vm_page_free_toq(). This routine is called | ||||
* when a page is added to the free queues. | * when a page is added to the free queues. | ||||
* | * | ||||
* The page queues must be locked. | * The page queues must be locked. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_free_wakeup(void) | vm_domain_free_wakeup(struct vm_domain *vmd) | ||||
{ | { | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(vmd); | ||||
/* | /* | ||||
* if pageout daemon needs pages, then tell it that there are | * if pageout daemon needs pages, then tell it that there are | ||||
* some free. | * some free. | ||||
*/ | */ | ||||
if (vm_pageout_pages_needed && | if (vmd->vmd_pageout_pages_needed && | ||||
vm_cnt.v_free_count >= vm_cnt.v_pageout_free_min) { | vmd->vmd_free_count >= vmd->vmd_pageout_free_min) { | ||||
wakeup(&vm_pageout_pages_needed); | wakeup(&vmd->vmd_pageout_pages_needed); | ||||
vm_pageout_pages_needed = 0; | vmd->vmd_pageout_pages_needed = 0; | ||||
} | } | ||||
/* | /* | ||||
* wakeup processes that are waiting on memory if we hit a | * wakeup processes that are waiting on memory if we hit a | ||||
* high water mark. And wakeup scheduler process if we have | * high water mark. And wakeup scheduler process if we have | ||||
* lots of memory. this process will swapin processes. | * lots of memory. this process will swapin processes. | ||||
*/ | */ | ||||
if (vm_pages_needed && !vm_page_count_min()) { | if (vmd->vmd_pages_needed && !vm_paging_min(vmd)) { | ||||
vm_pages_needed = false; | vmd->vmd_pages_needed = false; | ||||
wakeup(&vm_cnt.v_free_count); | wakeup(&vmd->vmd_free_count); | ||||
} | } | ||||
if ((vmd->vmd_minset && !vm_paging_min(vmd)) || | |||||
(vmd->vmd_severeset && !vm_paging_severe(vmd))) | |||||
vm_domain_clear(vmd); | |||||
/* See comments in vm_wait(); */ | |||||
if (vm_pageproc_waiters) { | |||||
vm_pageproc_waiters = 0; | |||||
wakeup(&vm_pageproc_waiters); | |||||
} | } | ||||
} | |||||
/* | /* | ||||
* vm_page_free_prep: | * vm_page_free_prep: | ||||
* | * | ||||
* Prepares the given page to be put on the free list, | * Prepares the given page to be put on the free list, | ||||
* disassociating it from any VM object. The caller may return | * disassociating it from any VM object. The caller may return | ||||
* the page to the free list only if this function returns true. | * the page to the free list only if this function returns true. | ||||
* | * | ||||
* The object must be locked. The page must be locked if it is | * The object must be locked. The page must be locked if it is | ||||
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Insert the page into the physical memory allocator's free page | * Insert the page into the physical memory allocator's free page | ||||
* queues. This is the last step to free a page. | * queues. This is the last step to free a page. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_free_phys(vm_page_t m) | vm_page_free_phys(vm_page_t m) | ||||
{ | { | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(vm_pagequeue_domain(m)); | ||||
vm_phys_freecnt_adj(m, 1); | vm_domain_freecnt_adj(vm_pagequeue_domain(m), 1); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (!vm_reserv_free_page(m)) | if (!vm_reserv_free_page(m)) | ||||
#endif | #endif | ||||
vm_phys_free_pages(m, 0); | vm_phys_free_pages(m, 0); | ||||
} | } | ||||
void | void | ||||
vm_page_free_phys_pglist(struct pglist *tq) | vm_page_free_phys_pglist(struct pglist *tq) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
vm_page_t m; | vm_page_t m; | ||||
if (TAILQ_EMPTY(tq)) | if (TAILQ_EMPTY(tq)) | ||||
return; | return; | ||||
mtx_lock(&vm_page_queue_free_mtx); | vmd = NULL; | ||||
TAILQ_FOREACH(m, tq, listq) | TAILQ_FOREACH(m, tq, listq) { | ||||
if (vmd != vm_pagequeue_domain(m)) { | |||||
if (vmd != NULL) { | |||||
vm_domain_free_wakeup(vmd); | |||||
vm_domain_free_unlock(vmd); | |||||
} | |||||
vmd = vm_pagequeue_domain(m); | |||||
vm_domain_free_lock(vmd); | |||||
} | |||||
vm_page_free_phys(m); | vm_page_free_phys(m); | ||||
vm_page_free_wakeup(); | |||||
mtx_unlock(&vm_page_queue_free_mtx); | |||||
} | } | ||||
if (vmd != NULL) { | |||||
vm_domain_free_wakeup(vmd); | |||||
vm_domain_free_unlock(vmd); | |||||
} | |||||
} | |||||
/* | /* | ||||
* vm_page_free_toq: | * vm_page_free_toq: | ||||
* | * | ||||
* Returns the given page to the free list, disassociating it | * Returns the given page to the free list, disassociating it | ||||
* from any VM object. | * from any VM object. | ||||
* | * | ||||
* The object must be locked. The page must be locked if it is | * The object must be locked. The page must be locked if it is | ||||
* managed. | * managed. | ||||
*/ | */ | ||||
void | void | ||||
vm_page_free_toq(vm_page_t m) | vm_page_free_toq(vm_page_t m) | ||||
{ | { | ||||
struct vm_domain *vmd; | |||||
if (!vm_page_free_prep(m, false)) | if (!vm_page_free_prep(m, false)) | ||||
return; | return; | ||||
mtx_lock(&vm_page_queue_free_mtx); | vmd = vm_pagequeue_domain(m); | ||||
vm_domain_free_lock(vmd); | |||||
vm_page_free_phys(m); | vm_page_free_phys(m); | ||||
vm_page_free_wakeup(); | vm_domain_free_wakeup(vmd); | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(vmd); | ||||
} | } | ||||
/* | /* | ||||
* vm_page_wire: | * vm_page_wire: | ||||
* | * | ||||
* Mark this page as wired down by yet | * Mark this page as wired down by yet | ||||
* another map, removing it from paging queues | * another map, removing it from paging queues | ||||
* as necessary. | * as necessary. | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | _vm_page_deactivate(vm_page_t m, boolean_t noreuse) | ||||
/* | /* | ||||
* Ignore if the page is already inactive, unless it is unlikely to be | * Ignore if the page is already inactive, unless it is unlikely to be | ||||
* reactivated. | * reactivated. | ||||
*/ | */ | ||||
if ((queue = m->queue) == PQ_INACTIVE && !noreuse) | if ((queue = m->queue) == PQ_INACTIVE && !noreuse) | ||||
return; | return; | ||||
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) { | if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) { | ||||
pq = &vm_phys_domain(m)->vmd_pagequeues[PQ_INACTIVE]; | pq = &vm_pagequeue_domain(m)->vmd_pagequeues[PQ_INACTIVE]; | ||||
/* Avoid multiple acquisitions of the inactive queue lock. */ | /* Avoid multiple acquisitions of the inactive queue lock. */ | ||||
if (queue == PQ_INACTIVE) { | if (queue == PQ_INACTIVE) { | ||||
vm_pagequeue_lock(pq); | vm_pagequeue_lock(pq); | ||||
vm_page_dequeue_locked(m); | vm_page_dequeue_locked(m); | ||||
} else { | } else { | ||||
if (queue != PQ_NONE) | if (queue != PQ_NONE) | ||||
vm_page_dequeue(m); | vm_page_dequeue(m); | ||||
vm_pagequeue_lock(pq); | vm_pagequeue_lock(pq); | ||||
} | } | ||||
m->queue = PQ_INACTIVE; | m->queue = PQ_INACTIVE; | ||||
if (noreuse) | if (noreuse) | ||||
TAILQ_INSERT_BEFORE(&vm_phys_domain(m)->vmd_inacthead, | TAILQ_INSERT_BEFORE( | ||||
m, plinks.q); | &vm_pagequeue_domain(m)->vmd_inacthead, m, | ||||
plinks.q); | |||||
else | else | ||||
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); | TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); | ||||
vm_pagequeue_cnt_inc(pq); | vm_pagequeue_cnt_inc(pq); | ||||
vm_pagequeue_unlock(pq); | vm_pagequeue_unlock(pq); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 760 Lines • ▼ Show 20 Lines | |||||
#ifdef DDB | #ifdef DDB | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <ddb/ddb.h> | #include <ddb/ddb.h> | ||||
DB_SHOW_COMMAND(page, vm_page_print_page_info) | DB_SHOW_COMMAND(page, vm_page_print_page_info) | ||||
{ | { | ||||
db_printf("vm_cnt.v_free_count: %d\n", vm_cnt.v_free_count); | db_printf("vm_cnt.v_free_count: %d\n", vm_free_count()); | ||||
db_printf("vm_cnt.v_inactive_count: %d\n", vm_cnt.v_inactive_count); | db_printf("vm_cnt.v_inactive_count: %d\n", vm_inactive_count()); | ||||
db_printf("vm_cnt.v_active_count: %d\n", vm_cnt.v_active_count); | db_printf("vm_cnt.v_active_count: %d\n", vm_active_count()); | ||||
db_printf("vm_cnt.v_laundry_count: %d\n", vm_cnt.v_laundry_count); | db_printf("vm_cnt.v_laundry_count: %d\n", vm_laundry_count()); | ||||
db_printf("vm_cnt.v_wire_count: %d\n", vm_cnt.v_wire_count); | db_printf("vm_cnt.v_wire_count: %d\n", vm_cnt.v_wire_count); | ||||
db_printf("vm_cnt.v_free_reserved: %d\n", vm_cnt.v_free_reserved); | db_printf("vm_cnt.v_free_reserved: %d\n", vm_cnt.v_free_reserved); | ||||
db_printf("vm_cnt.v_free_min: %d\n", vm_cnt.v_free_min); | db_printf("vm_cnt.v_free_min: %d\n", vm_cnt.v_free_min); | ||||
db_printf("vm_cnt.v_free_target: %d\n", vm_cnt.v_free_target); | db_printf("vm_cnt.v_free_target: %d\n", vm_cnt.v_free_target); | ||||
db_printf("vm_cnt.v_inactive_target: %d\n", vm_cnt.v_inactive_target); | db_printf("vm_cnt.v_inactive_target: %d\n", vm_cnt.v_inactive_target); | ||||
} | } | ||||
DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info) | DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info) | ||||
{ | { | ||||
int dom; | int dom; | ||||
db_printf("pq_free %d\n", vm_cnt.v_free_count); | db_printf("pq_free %d\n", vm_free_count()); | ||||
for (dom = 0; dom < vm_ndomains; dom++) { | for (dom = 0; dom < vm_ndomains; dom++) { | ||||
db_printf( | db_printf( | ||||
"dom %d page_cnt %d free %d pq_act %d pq_inact %d pq_laund %d pq_unsw %d\n", | "dom %d page_cnt %d free %d pq_act %d pq_inact %d pq_laund %d pq_unsw %d\n", | ||||
dom, | dom, | ||||
vm_dom[dom].vmd_page_count, | vm_dom[dom].vmd_page_count, | ||||
vm_dom[dom].vmd_free_count, | vm_dom[dom].vmd_free_count, | ||||
vm_dom[dom].vmd_pagequeues[PQ_ACTIVE].pq_cnt, | vm_dom[dom].vmd_pagequeues[PQ_ACTIVE].pq_cnt, | ||||
vm_dom[dom].vmd_pagequeues[PQ_INACTIVE].pq_cnt, | vm_dom[dom].vmd_pagequeues[PQ_INACTIVE].pq_cnt, | ||||
Show All 28 Lines |
Extra semicolon.