Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_page.c
Show First 20 Lines • Show All 3,169 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
vm_pqbatch_submit_page(vm_page_t m, uint8_t queue) | vm_pqbatch_submit_page(vm_page_t m, uint8_t queue) | ||||
{ | { | ||||
struct vm_batchqueue *bq; | struct vm_batchqueue *bq; | ||||
struct vm_pagequeue *pq; | struct vm_pagequeue *pq; | ||||
int domain; | int domain; | ||||
vm_page_assert_locked(m); | KASSERT((m->oflags & VPO_UNMANAGED) == 0, | ||||
("page %p is unmanaged", m)); | |||||
KASSERT(mtx_owned(vm_page_lockptr(m)) || | |||||
(m->object == NULL && (m->aflags & PGA_DEQUEUE) != 0), | |||||
("missing synchronization for page %p", m)); | |||||
KASSERT(queue < PQ_COUNT, ("invalid queue %d", queue)); | KASSERT(queue < PQ_COUNT, ("invalid queue %d", queue)); | ||||
domain = vm_phys_domain(m); | domain = vm_phys_domain(m); | ||||
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue]; | pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue]; | ||||
critical_enter(); | critical_enter(); | ||||
bq = DPCPU_PTR(pqbatch[domain][queue]); | bq = DPCPU_PTR(pqbatch[domain][queue]); | ||||
if (vm_batchqueue_insert(bq, m)) { | if (vm_batchqueue_insert(bq, m)) { | ||||
critical_exit(); | critical_exit(); | ||||
return; | return; | ||||
} | } | ||||
if (!vm_pagequeue_trylock(pq)) { | if (!vm_pagequeue_trylock(pq)) { | ||||
critical_exit(); | critical_exit(); | ||||
vm_pagequeue_lock(pq); | vm_pagequeue_lock(pq); | ||||
critical_enter(); | critical_enter(); | ||||
bq = DPCPU_PTR(pqbatch[domain][queue]); | bq = DPCPU_PTR(pqbatch[domain][queue]); | ||||
} | } | ||||
vm_pqbatch_process(pq, bq, queue); | vm_pqbatch_process(pq, bq, queue); | ||||
/* | /* | ||||
* The page may have been logically dequeued before we acquired the | * The page may have been logically dequeued before we acquired the | ||||
* page queue lock. In this case, the page lock prevents the page | * page queue lock. In this case, since we either hold the page lock | ||||
* from being logically enqueued elsewhere. | * or the page is being freed, a different thread cannot be concurrently | ||||
* enqueuing the page. | |||||
*/ | */ | ||||
if (__predict_true(m->queue == queue)) | if (__predict_true(m->queue == queue)) | ||||
vm_pqbatch_process_page(pq, m); | vm_pqbatch_process_page(pq, m); | ||||
else { | else { | ||||
KASSERT(m->queue == PQ_NONE, | KASSERT(m->queue == PQ_NONE, | ||||
("invalid queue transition for page %p", m)); | ("invalid queue transition for page %p", m)); | ||||
KASSERT((m->aflags & PGA_ENQUEUED) == 0, | KASSERT((m->aflags & PGA_ENQUEUED) == 0, | ||||
("page %p is enqueued with invalid queue index", m)); | ("page %p is enqueued with invalid queue index", m)); | ||||
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines | vm_page_dequeue_deferred(vm_page_t m) | ||||
if ((queue = vm_page_queue(m)) == PQ_NONE) | if ((queue = vm_page_queue(m)) == PQ_NONE) | ||||
return; | return; | ||||
vm_page_aflag_set(m, PGA_DEQUEUE); | vm_page_aflag_set(m, PGA_DEQUEUE); | ||||
vm_pqbatch_submit_page(m, queue); | vm_pqbatch_submit_page(m, queue); | ||||
} | } | ||||
/* | /* | ||||
* A variant of vm_page_dequeue_deferred() that does not assert the page | |||||
* lock and is only to be called from vm_page_free_prep(). It is just an | |||||
* open-coded implementation of vm_page_dequeue_deferred(). Because the | |||||
* page is being freed, we can assume that nothing else is scheduling queue | |||||
* operations on this page, so we get for free the mutual exclusion that | |||||
* is otherwise provided by the page lock. | |||||
*/ | |||||
static void | |||||
vm_page_dequeue_deferred_free(vm_page_t m) | |||||
{ | |||||
uint8_t queue; | |||||
KASSERT(m->object == NULL, ("page %p has an object reference", m)); | |||||
if ((m->aflags & PGA_DEQUEUE) != 0) | |||||
return; | |||||
atomic_thread_fence_acq(); | |||||
if ((queue = m->queue) == PQ_NONE) | |||||
return; | |||||
vm_page_aflag_set(m, PGA_DEQUEUE); | |||||
vm_pqbatch_submit_page(m, queue); | |||||
} | |||||
/* | |||||
* vm_page_dequeue: | * vm_page_dequeue: | ||||
* | * | ||||
* Remove the page from whichever page queue it's in, if any. | * Remove the page from whichever page queue it's in, if any. | ||||
* The page must either be locked or unallocated. This constraint | * The page must either be locked or unallocated. This constraint | ||||
* ensures that the queue state of the page will remain consistent | * ensures that the queue state of the page will remain consistent | ||||
* after this function returns. | * after this function returns. | ||||
*/ | */ | ||||
void | void | ||||
▲ Show 20 Lines • Show All 168 Lines • ▼ Show 20 Lines | #endif | ||||
} | } | ||||
/* | /* | ||||
* Pages need not be dequeued before they are returned to the physical | * Pages need not be dequeued before they are returned to the physical | ||||
* memory allocator, but they must at least be marked for a deferred | * memory allocator, but they must at least be marked for a deferred | ||||
* dequeue. | * dequeue. | ||||
*/ | */ | ||||
if ((m->oflags & VPO_UNMANAGED) == 0) | if ((m->oflags & VPO_UNMANAGED) == 0) | ||||
vm_page_dequeue_deferred(m); | vm_page_dequeue_deferred_free(m); | ||||
m->valid = 0; | m->valid = 0; | ||||
vm_page_undirty(m); | vm_page_undirty(m); | ||||
if (m->wire_count != 0) | if (m->wire_count != 0) | ||||
panic("vm_page_free_prep: freeing wired page %p", m); | panic("vm_page_free_prep: freeing wired page %p", m); | ||||
if (m->hold_count != 0) { | if (m->hold_count != 0) { | ||||
m->flags &= ~PG_ZERO; | m->flags &= ~PG_ZERO; | ||||
▲ Show 20 Lines • Show All 1,032 Lines • Show Last 20 Lines |