Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_page.c
Show First 20 Lines • Show All 3,050 Lines • ▼ Show 20 Lines | vm_waitpfault(struct domainset *dset, int timo) | ||||
if (vm_page_count_min_set(&dset->ds_mask)) { | if (vm_page_count_min_set(&dset->ds_mask)) { | ||||
vm_min_waiters++; | vm_min_waiters++; | ||||
msleep(&vm_min_domains, &vm_domainset_lock, PUSER | PDROP, | msleep(&vm_min_domains, &vm_domainset_lock, PUSER | PDROP, | ||||
"pfault", timo); | "pfault", timo); | ||||
} else | } else | ||||
mtx_unlock(&vm_domainset_lock); | mtx_unlock(&vm_domainset_lock); | ||||
} | } | ||||
struct vm_pagequeue * | static struct vm_pagequeue * | ||||
vm_page_pagequeue(vm_page_t m) | vm_page_pagequeue(vm_page_t m) | ||||
{ | { | ||||
return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]); | |||||
} | |||||
static struct mtx * | |||||
vm_page_pagequeue_lockptr(vm_page_t m) | |||||
{ | |||||
uint8_t queue; | uint8_t queue; | ||||
if ((queue = atomic_load_8(&m->queue)) == PQ_NONE) | if ((queue = atomic_load_8(&m->queue)) == PQ_NONE) | ||||
return (NULL); | return (NULL); | ||||
return (&vm_pagequeue_domain(m)->vmd_pagequeues[queue].pq_mutex); | return (&vm_pagequeue_domain(m)->vmd_pagequeues[queue]); | ||||
} | } | ||||
static inline void | static inline void | ||||
vm_pqbatch_process_page(struct vm_pagequeue *pq, vm_page_t m) | vm_pqbatch_process_page(struct vm_pagequeue *pq, vm_page_t m) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
uint8_t qflags; | uint8_t qflags; | ||||
CRITICAL_ASSERT(curthread); | CRITICAL_ASSERT(curthread); | ||||
vm_pagequeue_assert_locked(pq); | vm_pagequeue_assert_locked(pq); | ||||
/* | /* | ||||
* The page daemon is allowed to set m->queue = PQ_NONE without | * The page daemon is allowed to set m->queue = PQ_NONE without | ||||
* the page queue lock held. In this case it is about to free the page, | * the page queue lock held. In this case it is about to free the page, | ||||
* which must not have any queue state. | * which must not have any queue state. | ||||
*/ | */ | ||||
qflags = atomic_load_8(&m->aflags) & PGA_QUEUE_STATE_MASK; | qflags = atomic_load_8(&m->aflags) & PGA_QUEUE_STATE_MASK; | ||||
KASSERT(pq == vm_page_pagequeue(m) || qflags == 0, | KASSERT(pq == vm_page_pagequeue(m) || qflags == 0, | ||||
("page %p doesn't belong to queue %p but has queue state %#x", | ("page %p doesn't belong to queue %p but has queue state %#x", | ||||
m, pq, qflags)); | m, pq, qflags)); | ||||
if ((qflags & PGA_DEQUEUE) != 0) { | if ((qflags & PGA_DEQUEUE) != 0) { | ||||
if (__predict_true((qflags & PGA_ENQUEUED) != 0)) { | if (__predict_true((qflags & PGA_ENQUEUED) != 0)) | ||||
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); | vm_pagequeue_remove(pq, m); | ||||
vm_pagequeue_cnt_dec(pq); | |||||
} | |||||
vm_page_dequeue_complete(m); | vm_page_dequeue_complete(m); | ||||
} else if ((qflags & (PGA_REQUEUE | PGA_REQUEUE_HEAD)) != 0) { | } else if ((qflags & (PGA_REQUEUE | PGA_REQUEUE_HEAD)) != 0) { | ||||
if ((qflags & PGA_ENQUEUED) != 0) | if ((qflags & PGA_ENQUEUED) != 0) | ||||
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); | TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); | ||||
else { | else { | ||||
vm_pagequeue_cnt_inc(pq); | vm_pagequeue_cnt_inc(pq); | ||||
vm_page_aflag_set(m, PGA_ENQUEUED); | vm_page_aflag_set(m, PGA_ENQUEUED); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 186 Lines • ▼ Show 20 Lines | |||||
* Remove the page from whichever page queue it's in, if any. | * Remove the page from whichever page queue it's in, if any. | ||||
* The page must either be locked or unallocated. This constraint | * The page must either be locked or unallocated. This constraint | ||||
* ensures that the queue state of the page will remain consistent | * ensures that the queue state of the page will remain consistent | ||||
* after this function returns. | * after this function returns. | ||||
*/ | */ | ||||
void | void | ||||
vm_page_dequeue(vm_page_t m) | vm_page_dequeue(vm_page_t m) | ||||
{ | { | ||||
struct mtx *lock, *lock1; | struct vm_pagequeue *pq, *pq1; | ||||
struct vm_pagequeue *pq; | |||||
uint8_t aflags; | uint8_t aflags; | ||||
KASSERT(mtx_owned(vm_page_lockptr(m)) || m->order == VM_NFREEORDER, | KASSERT(mtx_owned(vm_page_lockptr(m)) || m->object == NULL, | ||||
("page %p is allocated and unlocked", m)); | ("page %p is allocated and unlocked", m)); | ||||
for (;;) { | for (pq = vm_page_pagequeue(m);; pq = pq1) { | ||||
lock = vm_page_pagequeue_lockptr(m); | if (pq == NULL) { | ||||
if (lock == NULL) { | |||||
/* | /* | ||||
* A thread may be concurrently executing | * A thread may be concurrently executing | ||||
* vm_page_dequeue_complete(). Ensure that all queue | * vm_page_dequeue_complete(). Ensure that all queue | ||||
* state is cleared before we return. | * state is cleared before we return. | ||||
*/ | */ | ||||
aflags = atomic_load_8(&m->aflags); | aflags = atomic_load_8(&m->aflags); | ||||
if ((aflags & PGA_QUEUE_STATE_MASK) == 0) | if ((aflags & PGA_QUEUE_STATE_MASK) == 0) | ||||
return; | return; | ||||
KASSERT((aflags & PGA_DEQUEUE) != 0, | KASSERT((aflags & PGA_DEQUEUE) != 0, | ||||
("page %p has unexpected queue state flags %#x", | ("page %p has unexpected queue state flags %#x", | ||||
m, aflags)); | m, aflags)); | ||||
/* | /* | ||||
* Busy wait until the thread updating queue state is | * Busy wait until the thread updating queue state is | ||||
* finished. Such a thread must be executing in a | * finished. Such a thread must be executing in a | ||||
* critical section. | * critical section. | ||||
*/ | */ | ||||
cpu_spinwait(); | cpu_spinwait(); | ||||
pq1 = vm_page_pagequeue(m); | |||||
continue; | continue; | ||||
} | } | ||||
mtx_lock(lock); | vm_pagequeue_lock(pq); | ||||
if ((lock1 = vm_page_pagequeue_lockptr(m)) == lock) | if ((pq1 = vm_page_pagequeue(m)) == pq) | ||||
break; | break; | ||||
mtx_unlock(lock); | vm_pagequeue_unlock(pq); | ||||
lock = lock1; | |||||
} | } | ||||
KASSERT(lock == vm_page_pagequeue_lockptr(m), | KASSERT(pq == vm_page_pagequeue(m), | ||||
("%s: page %p migrated directly between queues", __func__, m)); | ("%s: page %p migrated directly between queues", __func__, m)); | ||||
KASSERT((m->aflags & PGA_DEQUEUE) != 0 || | KASSERT((m->aflags & PGA_DEQUEUE) != 0 || | ||||
mtx_owned(vm_page_lockptr(m)), | mtx_owned(vm_page_lockptr(m)), | ||||
("%s: queued unlocked page %p", __func__, m)); | ("%s: queued unlocked page %p", __func__, m)); | ||||
if ((m->aflags & PGA_ENQUEUED) != 0) { | if ((m->aflags & PGA_ENQUEUED) != 0) | ||||
pq = vm_page_pagequeue(m); | vm_pagequeue_remove(pq, m); | ||||
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); | |||||
vm_pagequeue_cnt_dec(pq); | |||||
} | |||||
vm_page_dequeue_complete(m); | vm_page_dequeue_complete(m); | ||||
mtx_unlock(lock); | vm_pagequeue_unlock(pq); | ||||
} | } | ||||
/* | /* | ||||
* Schedule the given page for insertion into the specified page queue. | * Schedule the given page for insertion into the specified page queue. | ||||
* Physical insertion of the page may be deferred indefinitely. | * Physical insertion of the page may be deferred indefinitely. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_enqueue(vm_page_t m, uint8_t queue) | vm_page_enqueue(vm_page_t m, uint8_t queue) | ||||
▲ Show 20 Lines • Show All 1,222 Lines • Show Last 20 Lines |