Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | |||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include "opt_vm.h" | #include "opt_vm.h" | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/lock.h> | #include <sys/counter.h> | ||||
#include <sys/domainset.h> | #include <sys/domainset.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/limits.h> | #include <sys/limits.h> | ||||
#include <sys/linker.h> | #include <sys/linker.h> | ||||
#include <sys/lock.h> | |||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mman.h> | #include <sys/mman.h> | ||||
#include <sys/msgbuf.h> | #include <sys/msgbuf.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sleepqueue.h> | #include <sys/sleepqueue.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
Show All 36 Lines | |||||
struct mtx_padalign __exclusive_cache_line vm_domainset_lock; | struct mtx_padalign __exclusive_cache_line vm_domainset_lock; | ||||
/* The following fields are protected by the domainset lock. */ | /* The following fields are protected by the domainset lock. */ | ||||
domainset_t __exclusive_cache_line vm_min_domains; | domainset_t __exclusive_cache_line vm_min_domains; | ||||
domainset_t __exclusive_cache_line vm_severe_domains; | domainset_t __exclusive_cache_line vm_severe_domains; | ||||
static int vm_min_waiters; | static int vm_min_waiters; | ||||
static int vm_severe_waiters; | static int vm_severe_waiters; | ||||
static int vm_pageproc_waiters; | static int vm_pageproc_waiters; | ||||
static SYSCTL_NODE(_vm_stats, OID_AUTO, page, CTLFLAG_RD, 0, | |||||
"VM page statistics"); | |||||
static counter_u64_t queue_ops = EARLY_COUNTER; | |||||
SYSCTL_COUNTER_U64(_vm_stats_page, OID_AUTO, queue_ops, | |||||
CTLFLAG_RD, &queue_ops, | |||||
"Number of batched queue operations"); | |||||
static counter_u64_t queue_nops = EARLY_COUNTER; | |||||
SYSCTL_COUNTER_U64(_vm_stats_page, OID_AUTO, queue_nops, | |||||
CTLFLAG_RD, &queue_nops, | |||||
"Number of batched queue operations with no effects"); | |||||
static void | |||||
counter_startup(void) | |||||
{ | |||||
queue_ops = counter_u64_alloc(M_WAITOK); | |||||
queue_nops = counter_u64_alloc(M_WAITOK); | |||||
} | |||||
SYSINIT(page_counters, SI_SUB_CPU, SI_ORDER_ANY, counter_startup, NULL); | |||||
/* | /* | ||||
* bogus page -- for I/O to/from partially complete buffers, | * bogus page -- for I/O to/from partially complete buffers, | ||||
* or for paging into sparsely invalid regions. | * or for paging into sparsely invalid regions. | ||||
*/ | */ | ||||
vm_page_t bogus_page; | vm_page_t bogus_page; | ||||
vm_page_t __read_mostly vm_page_array; | vm_page_t __read_mostly vm_page_array; | ||||
long __read_mostly vm_page_array_size; | long __read_mostly vm_page_array_size; | ||||
▲ Show 20 Lines • Show All 2,971 Lines • ▼ Show 20 Lines | KASSERT(pq == vm_page_pagequeue(m) || | ||||
(qflags & PGA_QUEUE_STATE_MASK) == 0, | (qflags & PGA_QUEUE_STATE_MASK) == 0, | ||||
("page %p doesn't belong to queue %p but has aflags %#x", | ("page %p doesn't belong to queue %p but has aflags %#x", | ||||
m, pq, qflags)); | m, pq, qflags)); | ||||
if ((qflags & PGA_DEQUEUE) != 0) { | if ((qflags & PGA_DEQUEUE) != 0) { | ||||
if (__predict_true((qflags & PGA_ENQUEUED) != 0)) | if (__predict_true((qflags & PGA_ENQUEUED) != 0)) | ||||
vm_pagequeue_remove(pq, m); | vm_pagequeue_remove(pq, m); | ||||
vm_page_dequeue_complete(m); | vm_page_dequeue_complete(m); | ||||
counter_u64_add(queue_ops, 1); | |||||
} else if ((qflags & (PGA_REQUEUE | PGA_REQUEUE_HEAD)) != 0) { | } else if ((qflags & (PGA_REQUEUE | PGA_REQUEUE_HEAD)) != 0) { | ||||
if ((qflags & PGA_ENQUEUED) != 0) | if ((qflags & PGA_ENQUEUED) != 0) | ||||
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); | TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); | ||||
else { | else { | ||||
vm_pagequeue_cnt_inc(pq); | vm_pagequeue_cnt_inc(pq); | ||||
vm_page_aflag_set(m, PGA_ENQUEUED); | vm_page_aflag_set(m, PGA_ENQUEUED); | ||||
} | } | ||||
/* | /* | ||||
* Give PGA_REQUEUE_HEAD precedence over PGA_REQUEUE. | * Give PGA_REQUEUE_HEAD precedence over PGA_REQUEUE. | ||||
* In particular, if both flags are set in close succession, | * In particular, if both flags are set in close succession, | ||||
* only PGA_REQUEUE_HEAD will be applied, even if it was set | * only PGA_REQUEUE_HEAD will be applied, even if it was set | ||||
* first. | * first. | ||||
*/ | */ | ||||
if ((qflags & PGA_REQUEUE_HEAD) != 0) { | if ((qflags & PGA_REQUEUE_HEAD) != 0) { | ||||
KASSERT(m->queue == PQ_INACTIVE, | KASSERT(m->queue == PQ_INACTIVE, | ||||
("head enqueue not supported for page %p", m)); | ("head enqueue not supported for page %p", m)); | ||||
vmd = vm_pagequeue_domain(m); | vmd = vm_pagequeue_domain(m); | ||||
TAILQ_INSERT_BEFORE(&vmd->vmd_inacthead, m, plinks.q); | TAILQ_INSERT_BEFORE(&vmd->vmd_inacthead, m, plinks.q); | ||||
} else | } else | ||||
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); | TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); | ||||
vm_page_aflag_clear(m, qflags & (PGA_REQUEUE | | vm_page_aflag_clear(m, qflags & (PGA_REQUEUE | | ||||
PGA_REQUEUE_HEAD)); | PGA_REQUEUE_HEAD)); | ||||
counter_u64_add(queue_ops, 1); | |||||
} else { | |||||
counter_u64_add(queue_nops, 1); | |||||
} | } | ||||
} | } | ||||
static void | static void | ||||
vm_pqbatch_process(struct vm_pagequeue *pq, struct vm_batchqueue *bq, | vm_pqbatch_process(struct vm_pagequeue *pq, struct vm_batchqueue *bq, | ||||
uint8_t queue) | uint8_t queue) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
▲ Show 20 Lines • Show All 1,752 Lines • Show Last 20 Lines |