Diff 20384

sys/conf/files

Show First 20 Lines • Show All 125 Lines • ▼ Show 20 Lines
#include <vm/uma_int.h>		#include <vm/uma_int.h>

#include <machine/md_var.h>		#include <machine/md_var.h>

extern int uma_startup_count(int);		extern int uma_startup_count(int);
extern void uma_startup(void *, int);		extern void uma_startup(void *, int);
extern int vmem_startup_count(void);		extern int vmem_startup_count(void);

/*
* Associated with page of user-allocatable memory is a
* page structure.
*/

struct vm_domain vm_dom[MAXMEMDOM];		struct vm_domain vm_dom[MAXMEMDOM];

		static DPCPU_DEFINE(struct vm_batchqueue, pqbatch[MAXMEMDOM][PQ_COUNT]);
		static DPCPU_DEFINE(struct vm_batchqueue, noreuseq[MAXMEMDOM]);
		jeffUnsubmitted Not Done Inline Actions What is the reason for not using just another bit in the aflags to indicate head vs tail insertion? jeff: What is the reason for not using just another bit in the aflags to indicate head vs tail…
		markjAuthorUnsubmitted Not Done Inline Actions It was mostly because we only have two atomic bits left. Another solution would be to have a separate inactive pagequeue for pages that are bypassing global LRU. Because pq->pq_mtx is a pointer, both queues could be protected by the same lock, making transfers between the two queues relatively efficient. What do you think? markj: It was mostly because we only have two atomic bits left. Another solution would be to have a…
		kibUnsubmitted Not Done Inline Actions Why not put this into static pcpu ? kib: Why not put this into static pcpu ?
		markjAuthorUnsubmitted Not Done Inline Actions Is there any downside to using the dynamic per-CPU section? I like the fact that the definition is visible only in vm_page.c. markj: Is there any downside to using the dynamic per-CPU section? I like the fact that the definition…
		kibUnsubmitted Not Done Inline Actions It is one more arithmetic op on pointers to get to dpcu from pcpu. But my point is that the batch queues are not dynamic, they exists for the whole duration of the system operation. I do not insist either way. kib: It is one more arithmetic op on pointers to get to dpcu from pcpu. But my point is that the…

struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];		struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];

struct mtx_padalign __exclusive_cache_line vm_domainset_lock;		struct mtx_padalign __exclusive_cache_line vm_domainset_lock;
/* The following fields are protected by the domainset lock. */		/* The following fields are protected by the domainset lock. */
domainset_t __exclusive_cache_line vm_min_domains;		domainset_t __exclusive_cache_line vm_min_domains;
domainset_t __exclusive_cache_line vm_severe_domains;		domainset_t __exclusive_cache_line vm_severe_domains;
static int vm_min_waiters;		static int vm_min_waiters;
static int vm_severe_waiters;		static int vm_severe_waiters;
Show All 22 Lines
static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);		static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|		SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|
CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");		CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");

static uma_zone_t fakepg_zone;		static uma_zone_t fakepg_zone;

static void vm_page_alloc_check(vm_page_t m);		static void vm_page_alloc_check(vm_page_t m);
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);		static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
static void vm_page_enqueue(uint8_t queue, vm_page_t m);		static void vm_page_enqueue_lazy(vm_page_t m, uint8_t queue);
static void vm_page_init(void *dummy);		static void vm_page_init(void *dummy);
static int vm_page_insert_after(vm_page_t m, vm_object_t object,		static int vm_page_insert_after(vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred);		vm_pindex_t pindex, vm_page_t mpred);
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,		static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
vm_page_t mpred);		vm_page_t mpred);
static int vm_page_reclaim_run(int req_class, int domain, u_long npages,		static int vm_page_reclaim_run(int req_class, int domain, u_long npages,
vm_page_t m_run, vm_paddr_t high);		vm_page_t m_run, vm_paddr_t high);
static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,		static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,
▲ Show 20 Lines • Show All 1,623 Lines • ▼ Show 20 Lines	#endif
/*		/*
* At this point we had better have found a good page.		* At this point we had better have found a good page.
*/		*/
KASSERT(m != NULL, ("missing page"));		KASSERT(m != NULL, ("missing page"));

#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
found:		found:
#endif		#endif
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
flags = PG_ZERO;		flags = PG_ZERO;
▲ Show 20 Lines • Show All 180 Lines • ▼ Show 20 Lines	#endif
if (m_ret == NULL) {		if (m_ret == NULL) {
if (vm_domain_alloc_fail(vmd, object, req))		if (vm_domain_alloc_fail(vmd, object, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}
#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
found:		found:
#endif		#endif
for (m = m_ret; m < &m_ret[npages]; m++)		for (m = m_ret; m < &m_ret[npages]; m++) {
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);
		}

/*		/*
* Initialize the pages. Only the PG_ZERO flag is inherited.		* Initialize the pages. Only the PG_ZERO flag is inherited.
*/		*/
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
flags = PG_ZERO;		flags = PG_ZERO;
if ((req & VM_ALLOC_NODUMP) != 0)		if ((req & VM_ALLOC_NODUMP) != 0)
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines	if (vm_domain_allocate(vmd, req, 1)) {
if (m == NULL)		if (m == NULL)
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
}		}
if (m == NULL) {		if (m == NULL) {
if (vm_domain_alloc_fail(vmd, NULL, req))		if (vm_domain_alloc_fail(vmd, NULL, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
m->aflags = 0;		m->aflags = 0;
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
▲ Show 20 Lines • Show All 176 Lines • ▼ Show 20 Lines	#if VM_NRESERVLEVEL > 0
(level = vm_reserv_level_iffullpop(m)) >= 0) {		(level = vm_reserv_level_iffullpop(m)) >= 0) {
run_ext = 0;		run_ext = 0;
/* Advance to the end of the superpage. */		/* Advance to the end of the superpage. */
pa = VM_PAGE_TO_PHYS(m);		pa = VM_PAGE_TO_PHYS(m);
m_inc = atop(roundup2(pa + 1,		m_inc = atop(roundup2(pa + 1,
vm_reserv_size(level)) - pa);		vm_reserv_size(level)) - pa);
#endif		#endif
} else if (object->memattr == VM_MEMATTR_DEFAULT &&		} else if (object->memattr == VM_MEMATTR_DEFAULT &&
m->queue != PQ_NONE && !vm_page_busied(m)) {		m->queue != PQ_NONE &&
		(m->aflags & PGA_DEQUEUE) == 0 && !vm_page_busied(m)) {
		kibUnsubmitted Done Inline Actions Did you considered creating macros to detect logical enqueue conditions ? Like logically non-queued == (queue != PG_NONE && PGA_DEQUEUE == 0) ? kib: Did you considered creating macros to detect logical enqueue conditions ? Like logically non…
		markjAuthorUnsubmitted Done Inline Actions Yes, I should do that. I think this is missing an acquire barrier too. markj: Yes, I should do that. I think this is missing an acquire barrier too.
/*		/*
* The page is allocated but eligible for		* The page is allocated but eligible for
* relocation. Extend the current run by one		* relocation. Extend the current run by one
* page.		* page.
*/		*/
KASSERT(pmap_page_get_memattr(m) ==		KASSERT(pmap_page_get_memattr(m) ==
VM_MEMATTR_DEFAULT,		VM_MEMATTR_DEFAULT,
("page %p has an unexpected memattr", m));		("page %p has an unexpected memattr", m));
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines	else if ((object = m->object) != NULL) {
("page %p is PG_UNHOLDFREE", m));		("page %p is PG_UNHOLDFREE", m));
/* Don't care: PG_NODUMP, PG_ZERO. */		/* Don't care: PG_NODUMP, PG_ZERO. */
if (object->type != OBJT_DEFAULT &&		if (object->type != OBJT_DEFAULT &&
object->type != OBJT_SWAP &&		object->type != OBJT_SWAP &&
object->type != OBJT_VNODE)		object->type != OBJT_VNODE)
error = EINVAL;		error = EINVAL;
else if (object->memattr != VM_MEMATTR_DEFAULT)		else if (object->memattr != VM_MEMATTR_DEFAULT)
error = EINVAL;		error = EINVAL;
else if (m->queue != PQ_NONE && !vm_page_busied(m)) {		else if (m->queue != PQ_NONE &&
		(m->aflags & PGA_DEQUEUE) == 0 &&
		!vm_page_busied(m)) {
KASSERT(pmap_page_get_memattr(m) ==		KASSERT(pmap_page_get_memattr(m) ==
VM_MEMATTR_DEFAULT,		VM_MEMATTR_DEFAULT,
("page %p has an unexpected memattr", m));		("page %p has an unexpected memattr", m));
KASSERT((m->oflags & (VPO_SWAPINPROG \|		KASSERT((m->oflags & (VPO_SWAPINPROG \|
VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,		VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,
("page %p has unexpected oflags", m));		("page %p has unexpected oflags", m));
/* Don't care: VPO_NOSYNC. */		/* Don't care: VPO_NOSYNC. */
if (m->valid != 0) {		if (m->valid != 0) {
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines	else if ((object = m->object) != NULL) {
/*		/*
* Replace "m" with the new page. For		* Replace "m" with the new page. For
* vm_page_replace(), "m" must be busy		* vm_page_replace(), "m" must be busy
* and dequeued. Finally, change "m"		* and dequeued. Finally, change "m"
* as if vm_page_free() was called.		* as if vm_page_free() was called.
*/		*/
if (object->ref_count != 0)		if (object->ref_count != 0)
pmap_remove_all(m);		pmap_remove_all(m);
m_new->aflags = m->aflags;		m_new->aflags = m->aflags &
		~PGA_QUEUE_STATE_MASK;
KASSERT(m_new->oflags == VPO_UNMANAGED,		KASSERT(m_new->oflags == VPO_UNMANAGED,
("page %p is managed", m_new));		("page %p is managed", m_new));
m_new->oflags = m->oflags & VPO_NOSYNC;		m_new->oflags = m->oflags & VPO_NOSYNC;
pmap_copy_page(m, m_new);		pmap_copy_page(m, m_new);
m_new->valid = m->valid;		m_new->valid = m->valid;
m_new->dirty = m->dirty;		m_new->dirty = m->dirty;
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
vm_page_xbusy(m);		vm_page_xbusy(m);
vm_page_remque(m);		vm_page_remque(m);
vm_page_replace_checked(m_new, object,		vm_page_replace_checked(m_new, object,
m->pindex, m);		m->pindex, m);
if (vm_page_free_prep(m, false))		if (vm_page_free_prep(m))
SLIST_INSERT_HEAD(&free, m,		SLIST_INSERT_HEAD(&free, m,
plinks.s.ss);		plinks.s.ss);

/*		/*
* The new page must be deactivated		* The new page must be deactivated
* before the object is unlocked.		* before the object is unlocked.
*/		*/
vm_page_change_lock(m_new, &m_mtx);		vm_page_change_lock(m_new, &m_mtx);
vm_page_deactivate(m_new);		vm_page_deactivate(m_new);
} else {		} else {
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
vm_page_remque(m);		vm_page_remque(m);
vm_page_remove(m);		vm_page_remove(m);
if (vm_page_free_prep(m, false))		if (vm_page_free_prep(m))
SLIST_INSERT_HEAD(&free, m,		SLIST_INSERT_HEAD(&free, m,
plinks.s.ss);		plinks.s.ss);
KASSERT(m->dirty == 0,		KASSERT(m->dirty == 0,
("page %p is dirty", m));		("page %p is dirty", m));
}		}
} else		} else
error = EBUSY;		error = EBUSY;
unlock:		unlock:
▲ Show 20 Lines • Show All 426 Lines • ▼ Show 20 Lines

struct vm_pagequeue *		struct vm_pagequeue *
vm_page_pagequeue(vm_page_t m)		vm_page_pagequeue(vm_page_t m)
{		{

return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]);		return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]);
}		}

		static struct mtx *
		vm_page_pagequeue_lockptr(vm_page_t m)
		{

		if (m->queue == PQ_NONE)
		return (NULL);
		return (&vm_page_pagequeue(m)->pq_mutex);
		}

		static void
		vm_pqbatch_process(struct vm_pagequeue pq, struct vm_batchqueue bq,
		uint8_t queue)
		{
		vm_page_t m;
		int delta, i;
		uint8_t aflags;

		vm_pagequeue_assert_locked(pq);

		delta = 0;
		for (i = 0; i < bq->bq_cnt; i++) {
		m = bq->bq_pa[i];
		if (__predict_false(m->queue != queue))
		continue;

		aflags = m->aflags;
		if ((aflags & PGA_DEQUEUE) != 0) {
		if (__predict_true((aflags & PGA_ENQUEUED) != 0)) {
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		delta--;
		}

/*		/*
* vm_page_dequeue:		* Synchronize with the page daemon, which may be
		* simultaneously scanning this page with only the page
		* lock held. We must be careful to avoid leaving the
		* page in a state where it appears to belong to a page
		* queue.
		*/
		m->queue = PQ_NONE;
		atomic_thread_fence_rel();
		vm_page_aflag_clear(m, PGA_QUEUE_STATE_MASK);
		} else if ((aflags & PGA_ENQUEUED) == 0) {
		TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
		delta++;
		vm_page_aflag_set(m, PGA_ENQUEUED);
		if ((aflags & PGA_REQUEUE) != 0)
		vm_page_aflag_clear(m, PGA_REQUEUE);
		} else if ((aflags & PGA_REQUEUE) != 0) {
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
		vm_page_aflag_clear(m, PGA_REQUEUE);
		}
		}
		vm_batchqueue_init(bq);
		vm_pagequeue_cnt_add(pq, delta);
		}

		/*
		* vm_page_dequeue_lazy: [ internal use only ]
*		*
* Remove the given page from its current page queue.		* Request removal of the given page from its current page
		* queue. Physical removal from the queue may be deferred
		* arbitrarily, and may be cancelled by later queue operations
		* on that page.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_dequeue(vm_page_t m)		vm_page_dequeue_lazy(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain, queue;

vm_page_assert_locked(m);		vm_page_assert_locked(m);
KASSERT(m->queue < PQ_COUNT, ("vm_page_dequeue: page %p is not queued",
m));		queue = m->queue;
pq = vm_page_pagequeue(m);		if (queue == PQ_NONE)
		return;
		domain = vm_phys_domain(m);
		pq = &VM_DOMAIN(domain)->vmd_pagequeues[queue];

		vm_page_aflag_set(m, PGA_DEQUEUE);

		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (vm_batchqueue_insert(bq, m)) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		kibUnsubmitted Not Done Inline Actions Did you considered copying the pcpu pqbatch into a local array for processing and enabling preemption immediately after that ? I think that on full pqbatch, we spend a time which is enough to adversely affect the system responsivness. kib: Did you considered copying the pcpu pqbatch into a local array for processing and enabling…
		markjAuthorUnsubmitted Not Done Inline Actions I did, but there is a non-trivial stack space cost (256 bytes on LP64) in addition to that of the copy. The cost of processing 32 pages (8 on !amd64) ought to be quite low. I ran Jeff's dd benchmark (where parallel dd processes read from a sparse file and thus frequently enqueue pages) and used DTrace to count the number of preemptions that occurred in vm_page_enqueue(); excluding preemptions of the idle threads, less than 0.1% of preemptions occurred during the critical_exit() here. This is also an extreme case and in pretty much any workload we will not be performing queue operations at nearly that high a rate. For example, when running a 16-thread pgbench select-only benchmark on an SSD array with about 1GB/s throughput, the number of preemptions that occur here is effectively 0. Preemptions in UMA are much more common. markj: I did, but there is a non-trivial stack space cost (256 bytes on LP64) in addition to that of…
vm_pagequeue_lock(pq);		vm_pagequeue_lock(pq);
m->queue = PQ_NONE;		critical_enter();
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		bq = DPCPU_PTR(pqbatch[domain][queue]);
vm_pagequeue_cnt_dec(pq);		}
		vm_pqbatch_process(pq, bq, queue);

		/*
		* The page may have been dequeued by another thread before we
		* acquired the page queue lock. However, since we hold the
		* page lock, the page's queue field cannot change a second
		* time and we can safely clear PGA_DEQUEUE.
		*/
		KASSERT(m->queue == queue \|\| m->queue == PQ_NONE,
		("%s: page %p migrated between queues", __func__, m));
		if (m->queue == queue) {
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		} else
		vm_page_aflag_clear(m, PGA_DEQUEUE);
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		critical_exit();
}		}

/*		/*
* vm_page_dequeue_locked:		* vm_page_dequeue_locked:
*		*
* Remove the given page from its current page queue.		* Remove the page from its page queue, which must be locked.
		* If the page lock is not held, there is no guarantee that the
		* page will not be enqueued by another thread before this function
		* returns. In this case, it is up to the caller to ensure that
		* no other threads hold a reference to the page.
*		*
* The page and page queue must be locked.		* The page queue lock must be held. If the page is not already
		* logically dequeued, the page lock must be held as well.
*/		*/
void		void
vm_page_dequeue_locked(vm_page_t m)		vm_page_dequeue_locked(vm_page_t m)
{		{
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;

vm_page_lock_assert(m, MA_OWNED);		KASSERT(m->queue != PQ_NONE,
		("%s: page %p queue field is PQ_NONE", __func__, m));
		vm_pagequeue_assert_locked(vm_page_pagequeue(m));
		KASSERT((m->aflags & PGA_DEQUEUE) != 0 \|\|
		mtx_owned(vm_page_lockptr(m)),
		("%s: queued unlocked page %p", __func__, m));

		if ((m->aflags & PGA_ENQUEUED) != 0) {
pq = vm_page_pagequeue(m);		pq = vm_page_pagequeue(m);
vm_pagequeue_assert_locked(pq);
m->queue = PQ_NONE;
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_dec(pq);		vm_pagequeue_cnt_dec(pq);
}		}

/*		/*
* vm_page_enqueue:		* Synchronize with the page daemon, which may be simultaneously
		* scanning this page with only the page lock held. We must be careful
		* to avoid leaving the page in a state where it appears to belong to a
		* page queue.
		*/
		m->queue = PQ_NONE;
		atomic_thread_fence_rel();
		vm_page_aflag_clear(m, PGA_QUEUE_STATE_MASK);
		kibUnsubmitted Not Done Inline Actions I think you need acquire fence before reading aflags there. kib: I think you need acquire fence before reading aflags there.
		markjAuthorUnsubmitted Not Done Inline Actions I am not sure about this. Right now, we are using barriers to synchronize the following operations: T1 dequeues a page with the queue lock held: set m->queue = PQ_NONE, release barrier, clear PGA_DEQUEUE in aflags T2 checks to see if the page is dequeued with page lock held: check for PGA_DEQUEUE in aflags, acquire barrier, check m->queue == PQ_NONE (I don't know what kind of notation is best here.) Here we hold the queue lock, so m->queue is stable, and any thread which updated it must have implicitly issued a release barrier when it unlocked the queue. So I don't see why we need an acquire fence. markj: I am not sure about this. Right now, we are using barriers to synchronize the following…
		kibUnsubmitted Not Done Inline Actions Ok, thank you for the explanation. kib: Ok, thank you for the explanation.
		}

		/*
		* vm_page_dequeue:
*		*
* Add the given page to the specified page queue.		* Remove the page from whichever page queue it's in, if any.
		* If the page lock is not held, there is no guarantee that the
		* page will not be enqueued by another thread before this function
		* returns. In this case, it is up to the caller to ensure that
		* no other threads hold a reference to the page.
		*/
		void
		vm_page_dequeue(vm_page_t m)
		{
		struct mtx lock, lock1;

		lock = vm_page_pagequeue_lockptr(m);
		for (;;) {
		if (lock == NULL)
		return;
		mtx_lock(lock);
		if ((lock1 = vm_page_pagequeue_lockptr(m)) == lock)
		break;
		mtx_unlock(lock);
		lock = lock1;
		}
		KASSERT(lock == vm_page_pagequeue_lockptr(m),
		("%s: page %p migrated directly between queues", __func__, m));
		vm_page_dequeue_locked(m);
		mtx_unlock(lock);
		}

		/*
		* vm_page_enqueue_lazy:
*		*
		* Schedule the given page for insertion into the specified page queue.
		* Physical insertion of the page may be deferred indefinitely.
		*
* The page must be locked.		* The page must be locked.
*/		*/
static void		static void
vm_page_enqueue(uint8_t queue, vm_page_t m)		vm_page_enqueue_lazy(vm_page_t m, uint8_t queue)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain;

vm_page_lock_assert(m, MA_OWNED);		vm_page_assert_locked(m);
KASSERT(queue < PQ_COUNT,		KASSERT(m->queue == PQ_NONE && (m->aflags & PGA_QUEUE_STATE_MASK) == 0,
("vm_page_enqueue: invalid queue %u request for page %p",		("%s: page %p is already enqueued", __func__, m));
queue, m));
		domain = vm_phys_domain(m);
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];		pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];
vm_pagequeue_lock(pq);
		/*
		* The queue field might be changed back to PQ_NONE by a concurrent
		* call to vm_page_dequeue(). In that case the batch queue entry will
		* be a no-op.
		*/
m->queue = queue;		m->queue = queue;
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_inc(pq);		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (__predict_true(vm_batchqueue_insert(bq, m))) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		vm_pagequeue_lock(pq);
		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		}
		glebiusUnsubmitted Not Done Inline Actions What if after critical_exit we have changed the CPU? In this case proceeding further to the end of the function doesn't seem optimal. I'd suggest just to put a page on regular queue, since we already got lock. glebius: What if after critical_exit we have changed the CPU? In this case proceeding further to the end…
		markjAuthorUnsubmitted Not Done Inline Actions Why not? We now hold the page queue lock, so we might as well drain the current queue. markj: Why not? We now hold the page queue lock, so we might as well drain the current queue.
		vm_pqbatch_process(pq, bq, queue);
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		kibUnsubmitted Not Done Inline Actions Don't you need the release fence between setting page domain and setting PGA_REQUEUE ? kib: Don't you need the release fence between setting page domain and setting PGA_REQUEUE ?
		markjAuthorUnsubmitted Not Done Inline Actions See above. We should already be synchronized by the page lock. markj: See above. We should already be synchronized by the page lock.
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		glebiusUnsubmitted Not Done Inline Actions Can you please explain the need for the second call to vm_pqbatch_process? glebius: Can you please explain the need for the second call to vm_pqbatch_process?
		markjAuthorUnsubmitted Done Inline Actions It processes solely the page m. We're at this point because the queue is full, so we have to drain it first by calling `vm_pqbatch_process()`. Probably there should be a `vm_pqbatch_process_page()` which processes a single page, and we just call that on `m`. markj: It processes solely the page m. We're at this point because the queue is full, so we have to…
		critical_exit();
}		}

/*		/*
* vm_page_requeue:		* vm_page_requeue:
*		*
* Move the given page to the tail of its current page queue.		* Schedule a requeue of the given page.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_requeue(vm_page_t m)		vm_page_requeue(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain, queue;

vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
KASSERT(m->queue != PQ_NONE,		KASSERT(m->queue != PQ_NONE,
("vm_page_requeue: page %p is not queued", m));		("%s: page %p is not enqueued", __func__, m));

		domain = vm_phys_domain(m);
		queue = m->queue;
pq = vm_page_pagequeue(m);		pq = vm_page_pagequeue(m);

		if (queue == PQ_NONE)
		return;

		vm_page_aflag_set(m, PGA_REQUEUE);
		jeffUnsubmitted Done Inline Actions Does it make sense to test for this flag before setting it? If it's already set there is not much value in doing it again. jeff: Does it make sense to test for this flag before setting it? If it's already set there is not…
		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (__predict_true(vm_batchqueue_insert(bq, m))) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
vm_pagequeue_lock(pq);		vm_pagequeue_lock(pq);
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		critical_enter();
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);		bq = DPCPU_PTR(pqbatch[domain][queue]);
		}
		vm_pqbatch_process(pq, bq, queue);
		KASSERT(m->queue == queue \|\| m->queue == PQ_NONE,
		("%s: page %p migrated between queues", __func__, m));
		if (m->queue == queue) {
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		} else
		vm_page_aflag_clear(m, PGA_REQUEUE);
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		critical_exit();
}		}

/*		/*
* vm_page_requeue_locked:
*
* Move the given page to the tail of its current page queue.
*
* The page queue must be locked.
*/
void
vm_page_requeue_locked(vm_page_t m)
{
struct vm_pagequeue *pq;

KASSERT(m->queue != PQ_NONE,
("vm_page_requeue_locked: page %p is not queued", m));
pq = vm_page_pagequeue(m);
vm_pagequeue_assert_locked(pq);
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
}

/*
* vm_page_activate:		* vm_page_activate:
*		*
* Put the specified page on the active list (if appropriate).		* Put the specified page on the active list (if appropriate).
* Ensure that act_count is at least ACT_INIT but do not otherwise		* Ensure that act_count is at least ACT_INIT but do not otherwise
* mess with it.		* mess with it.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_activate(vm_page_t m)		vm_page_activate(vm_page_t m)
{		{
int queue;		int queue;

vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
if ((queue = m->queue) != PQ_ACTIVE) {
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if ((queue = m->queue) == PQ_ACTIVE \|\| m->wire_count > 0 \|\|
if (m->act_count < ACT_INIT)		(m->oflags & VPO_UNMANAGED) != 0) {
		if (queue == PQ_ACTIVE && m->act_count < ACT_INIT)
m->act_count = ACT_INIT;		m->act_count = ACT_INIT;
if (queue != PQ_NONE)		return;
vm_page_dequeue(m);
vm_page_enqueue(PQ_ACTIVE, m);
}		}
} else {
		vm_page_remque(m);
if (m->act_count < ACT_INIT)		if (m->act_count < ACT_INIT)
m->act_count = ACT_INIT;		m->act_count = ACT_INIT;
		vm_page_enqueue_lazy(m, PQ_ACTIVE);
}		}
}

/*		/*
* vm_page_free_prep:		* vm_page_free_prep:
*		*
* Prepares the given page to be put on the free list,		* Prepares the given page to be put on the free list,
* disassociating it from any VM object. The caller may return		* disassociating it from any VM object. The caller may return
* the page to the free list only if this function returns true.		* the page to the free list only if this function returns true.
*		*
* The object must be locked. The page must be locked if it is		* The object must be locked. The page must be locked if it is
* managed. For a queued managed page, the pagequeue_locked		* managed.
* argument specifies whether the page queue is already locked.
*/		*/
bool		bool
vm_page_free_prep(vm_page_t m, bool pagequeue_locked)		vm_page_free_prep(vm_page_t m)
{		{

#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)		#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)
if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {		if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {
uint64_t *p;		uint64_t *p;
int i;		int i;
p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));		p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)		for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)
KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",		KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",
m, i, (uintmax_t)*p));		m, i, (uintmax_t)*p));
}		}
#endif		#endif
if ((m->oflags & VPO_UNMANAGED) == 0) {		if ((m->oflags & VPO_UNMANAGED) == 0) {
vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
KASSERT(!pmap_page_is_mapped(m),		KASSERT(!pmap_page_is_mapped(m),
("vm_page_free_toq: freeing mapped page %p", m));		("vm_page_free_prep: freeing mapped page %p", m));
} else		} else
KASSERT(m->queue == PQ_NONE,		KASSERT(m->queue == PQ_NONE,
("vm_page_free_toq: unmanaged page %p is queued", m));		("vm_page_free_prep: unmanaged page %p is queued", m));
VM_CNT_INC(v_tfree);		VM_CNT_INC(v_tfree);

if (vm_page_sbusied(m))		if (vm_page_sbusied(m))
panic("vm_page_free: freeing busy page %p", m);		panic("vm_page_free_prep: freeing busy page %p", m);

vm_page_remove(m);		vm_page_remove(m);

/*		/*
* If fictitious remove object association and		* If fictitious remove object association and
* return.		* return.
*/		*/
if ((m->flags & PG_FICTITIOUS) != 0) {		if ((m->flags & PG_FICTITIOUS) != 0) {
KASSERT(m->wire_count == 1,		KASSERT(m->wire_count == 1,
("fictitious page %p is not wired", m));		("fictitious page %p is not wired", m));
KASSERT(m->queue == PQ_NONE,		KASSERT(m->queue == PQ_NONE,
("fictitious page %p is queued", m));		("fictitious page %p is queued", m));
return (false);		return (false);
}		}

if (m->queue != PQ_NONE) {		/*
if (pagequeue_locked)		* Pages need not be dequeued before they are returned to the physical
vm_page_dequeue_locked(m);		* memory allocator, but they must at least be marked for a deferred
else		* dequeue.
vm_page_dequeue(m);		*/
}		if ((m->oflags & VPO_UNMANAGED) == 0)
		vm_page_dequeue_lazy(m);

m->valid = 0;		m->valid = 0;
vm_page_undirty(m);		vm_page_undirty(m);

if (m->wire_count != 0)		if (m->wire_count != 0)
panic("vm_page_free: freeing wired page %p", m);		panic("vm_page_free_prep: freeing wired page %p", m);
if (m->hold_count != 0) {		if (m->hold_count != 0) {
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
KASSERT((m->flags & PG_UNHOLDFREE) == 0,		KASSERT((m->flags & PG_UNHOLDFREE) == 0,
("vm_page_free: freeing PG_UNHOLDFREE page %p", m));		("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m));
m->flags \|= PG_UNHOLDFREE;		m->flags \|= PG_UNHOLDFREE;
return (false);		return (false);
}		}

/*		/*
* Restore the default memory attribute to the page.		* Restore the default memory attribute to the page.
*/		*/
if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)		if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);		pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);

#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
if (vm_reserv_free_page(m))		if (vm_reserv_free_page(m))
return (false);		return (false);
#endif		#endif

return (true);		return (true);
}		}

void
vm_page_free_phys_pglist(struct pglist *tq)
{
struct vm_domain *vmd;
vm_page_t m;
int cnt;

if (TAILQ_EMPTY(tq))
return;
vmd = NULL;
cnt = 0;
TAILQ_FOREACH(m, tq, listq) {
if (vmd != vm_pagequeue_domain(m)) {
if (vmd != NULL) {
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, cnt);
cnt = 0;
}
vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);
}
vm_phys_free_pages(m, 0);
cnt++;
}
if (vmd != NULL) {
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, cnt);
}
}

/*		/*
* vm_page_free_toq:		* vm_page_free_toq:
*		*
* Returns the given page to the free list, disassociating it		* Returns the given page to the free list, disassociating it
* from any VM object.		* from any VM object.
*		*
* The object must be locked. The page must be locked if it is		* The object must be locked. The page must be locked if it is
* managed.		* managed.
*/		*/
void		void
vm_page_free_toq(vm_page_t m)		vm_page_free_toq(vm_page_t m)
{		{
struct vm_domain *vmd;		struct vm_domain *vmd;

if (!vm_page_free_prep(m, false))		if (!vm_page_free_prep(m))
return;		return;

vmd = vm_pagequeue_domain(m);		vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);		vm_domain_free_lock(vmd);
vm_phys_free_pages(m, 0);		vm_phys_free_pages(m, 0);
vm_domain_free_unlock(vmd);		vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
}		}

/*		/*
* vm_page_free_pages_toq:		* vm_page_free_pages_toq:
*		*
* Returns a list of pages to the free list, disassociating it		* Returns a list of pages to the free list, disassociating it
* from any VM object. In other words, this is equivalent to		* from any VM object. In other words, this is equivalent to
* calling vm_page_free_toq() for each page of a list of VM objects.		* calling vm_page_free_toq() for each page of a list of VM objects.
*		*
* The objects must be locked. The pages must be locked if it is		* The objects must be locked. The pages must be locked if it is
* managed.		* managed.
*/		*/
void		void
vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)		vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)
{		{
vm_page_t m;		vm_page_t m;
struct pglist pgl;
int count;		int count;

if (SLIST_EMPTY(free))		if (SLIST_EMPTY(free))
return;		return;

count = 0;		count = 0;
TAILQ_INIT(&pgl);
while ((m = SLIST_FIRST(free)) != NULL) {		while ((m = SLIST_FIRST(free)) != NULL) {
count++;		count++;
SLIST_REMOVE_HEAD(free, plinks.s.ss);		SLIST_REMOVE_HEAD(free, plinks.s.ss);
if (vm_page_free_prep(m, false))		vm_page_free_toq(m);
TAILQ_INSERT_TAIL(&pgl, m, listq);
}		}

vm_page_free_phys_pglist(&pgl);

if (update_wire_count)		if (update_wire_count)
vm_wire_sub(count);		vm_wire_sub(count);
}		}

/*		/*
* vm_page_wire:		* vm_page_wire:
*		*
* Mark this page as wired down. If the page is fictitious, then		* Mark this page as wired down. If the page is fictitious, then
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
bool		bool
vm_page_unwire(vm_page_t m, uint8_t queue)		vm_page_unwire(vm_page_t m, uint8_t queue)
{		{
bool unwired;		bool unwired;

KASSERT(queue < PQ_COUNT \|\| queue == PQ_NONE,		KASSERT(queue < PQ_COUNT \|\| queue == PQ_NONE,
("vm_page_unwire: invalid queue %u request for page %p",		("vm_page_unwire: invalid queue %u request for page %p",
queue, m));		queue, m));
		if ((m->oflags & VPO_UNMANAGED) == 0)
		vm_page_assert_locked(m);

unwired = vm_page_unwire_noq(m);		unwired = vm_page_unwire_noq(m);
if (unwired && (m->oflags & VPO_UNMANAGED) == 0 && m->object != NULL) {		if (!unwired \|\| (m->oflags & VPO_UNMANAGED) != 0 \|\| m->object == NULL)
		return (unwired);

if (m->queue == queue) {		if (m->queue == queue) {
if (queue == PQ_ACTIVE)		if (queue == PQ_ACTIVE)
vm_page_reference(m);		vm_page_reference(m);
else if (queue != PQ_NONE)		else if (queue != PQ_NONE)
vm_page_requeue(m);		vm_page_requeue(m);
} else {		} else {
vm_page_remque(m);		vm_page_dequeue(m);
if (queue != PQ_NONE) {		if (queue != PQ_NONE) {
vm_page_enqueue(queue, m);		vm_page_enqueue_lazy(m, queue);
if (queue == PQ_ACTIVE)		if (queue == PQ_ACTIVE)
/* Initialize act_count. */		/* Initialize act_count. */
vm_page_activate(m);		vm_page_activate(m);
}		}
}		}
}
return (unwired);		return (unwired);
}		}

/*		/*
*		*
* vm_page_unwire_noq:		* vm_page_unwire_noq:
*		*
* Unwire a page without (re-)inserting it into a page queue. It is up		* Unwire a page without (re-)inserting it into a page queue. It is up
Show All 17 Lines	vm_page_unwire_noq(vm_page_t m)
if (m->wire_count == 0) {		if (m->wire_count == 0) {
vm_wire_sub(1);		vm_wire_sub(1);
return (true);		return (true);
} else		} else
return (false);		return (false);
}		}

/*		/*
* Move the specified page to the inactive queue, or requeue the page if it is		* Move the specified page to the tail of the inactive queue, or requeue
* already in the inactive queue.		* the page if it is already in the inactive queue.
*		*
* Normally, "noreuse" is FALSE, resulting in LRU ordering of the inactive
* queue. However, setting "noreuse" to TRUE will accelerate the specified
* page's reclamation, but it will not unmap the page from any address space.
* This is implemented by inserting the page near the head of the inactive
* queue, using a marker page to guide FIFO insertion ordering.
*
* The page must be locked.		* The page must be locked.
*/		*/
static inline void		void
_vm_page_deactivate(vm_page_t m, boolean_t noreuse)		vm_page_deactivate(vm_page_t m)
{		{
struct vm_pagequeue *pq;
int queue;

vm_page_assert_locked(m);		vm_page_assert_locked(m);

if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[PQ_INACTIVE];		return;
/* Avoid multiple acquisitions of the inactive queue lock. */
queue = m->queue;
if (queue == PQ_INACTIVE) {
vm_pagequeue_lock(pq);
vm_page_dequeue_locked(m);
} else {
if (queue != PQ_NONE)
vm_page_dequeue(m);
vm_pagequeue_lock(pq);
}
m->queue = PQ_INACTIVE;
if (noreuse)
TAILQ_INSERT_BEFORE(
&vm_pagequeue_domain(m)->vmd_inacthead, m,
plinks.q);
else
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_inc(pq);
vm_pagequeue_unlock(pq);
}
}

/*		if (!vm_page_inactive(m)) {
		jeffUnsubmitted Done Inline Actions Should be an XXX comment. This path likely needs to be fast. jeff: Should be an XXX comment. This path likely needs to be fast.
* Move the specified page to the inactive queue, or requeue the page if it is		vm_page_remque(m);
* already in the inactive queue.		vm_page_enqueue_lazy(m, PQ_INACTIVE);
*		} else
* The page must be locked.		vm_page_requeue(m);
*/
void
vm_page_deactivate(vm_page_t m)
{

_vm_page_deactivate(m, FALSE);
}		}

/*		/*
* Move the specified page to the inactive queue with the expectation		* Move the specified page close to the head of the inactive queue,
* that it is unlikely to be reused.		* bypassing LRU. A marker page is used to maintain FIFO ordering.
		* As with regular enqueues, we use a per-CPU batch queue to reduce
		* contention on the page queue lock.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_deactivate_noreuse(vm_page_t m)		vm_page_deactivate_noreuse(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
		struct vm_domain *vmd;
		struct vm_pagequeue *pq;
		vm_page_t marker;
		int domain;

_vm_page_deactivate(m, TRUE);		vm_page_assert_locked(m);

		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
		return;

		domain = vm_phys_domain(m);
		vmd = VM_DOMAIN(domain);
		pq = &vmd->vmd_pagequeues[PQ_INACTIVE];

		if (!vm_page_inactive(m))
		vm_page_remque(m);

		m->queue = PQ_INACTIVE;

		critical_enter();
		bq = DPCPU_PTR(noreuseq[domain]);
		if (vm_batchqueue_insert(bq, m)) {
		critical_exit();
		return;
}		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		vm_pagequeue_lock(pq);
		critical_enter();
		bq = DPCPU_PTR(noreuseq[domain]);
		}
		marker = &vmd->vmd_inacthead;
		do {
		if (m->queue != PQ_INACTIVE)
		continue;
		if ((m->aflags & PGA_ENQUEUED) != 0)
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		else {
		vm_page_aflag_set(m, PGA_ENQUEUED);
		vm_pagequeue_cnt_inc(pq);
		}
		TAILQ_INSERT_BEFORE(marker, m, plinks.q);
		} while ((m = vm_batchqueue_pop(bq)) != NULL);
		vm_pagequeue_unlock(pq);
		vm_batchqueue_init(bq);
		critical_exit();
		}

/*		/*
* vm_page_launder		* vm_page_launder
*		*
* Put a page in the laundry, or requeue it if it is already there.		* Put a page in the laundry, or requeue it if it is already there.
*/		*/
void		void
vm_page_launder(vm_page_t m)		vm_page_launder(vm_page_t m)
{		{

vm_page_assert_locked(m);		vm_page_assert_locked(m);
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
		return;

if (m->queue == PQ_LAUNDRY)		if (m->queue == PQ_LAUNDRY)
vm_page_requeue(m);		vm_page_requeue(m);
else {		else {
vm_page_remque(m);		vm_page_remque(m);
vm_page_enqueue(PQ_LAUNDRY, m);		vm_page_enqueue_lazy(m, PQ_LAUNDRY);
}		}
}		}
}

/*		/*
* vm_page_unswappable		* vm_page_unswappable
*		*
* Put a page in the PQ_UNSWAPPABLE holding queue.		* Put a page in the PQ_UNSWAPPABLE holding queue.
*/		*/
void		void
vm_page_unswappable(vm_page_t m)		vm_page_unswappable(vm_page_t m)
{		{

vm_page_assert_locked(m);		vm_page_assert_locked(m);
KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,		KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,
("page %p already unswappable", m));		("page %p already unswappable", m));
if (m->queue != PQ_NONE)
vm_page_dequeue(m);		vm_page_remque(m);
vm_page_enqueue(PQ_UNSWAPPABLE, m);		vm_page_enqueue_lazy(m, PQ_UNSWAPPABLE);
}		}

/*		/*
* Attempt to free the page. If it cannot be freed, do nothing. Returns true		* Attempt to free the page. If it cannot be freed, do nothing. Returns true
* if the page is freed and false otherwise.		* if the page is freed and false otherwise.
*		*
* The page must be managed. The page and its containing object must be		* The page must be managed. The page and its containing object must be
* locked.		* locked.
▲ Show 20 Lines • Show All 750 Lines • Show Last 20 Lines

View Options

sys/dev/mlx5/mlx5_ib/mlx5_ah.c

Show First 20 Lines • Show All 125 Lines • ▼ Show 20 Lines
#include <vm/uma_int.h>		#include <vm/uma_int.h>

#include <machine/md_var.h>		#include <machine/md_var.h>

extern int uma_startup_count(int);		extern int uma_startup_count(int);
extern void uma_startup(void *, int);		extern void uma_startup(void *, int);
extern int vmem_startup_count(void);		extern int vmem_startup_count(void);

/*
* Associated with page of user-allocatable memory is a
* page structure.
*/

struct vm_domain vm_dom[MAXMEMDOM];		struct vm_domain vm_dom[MAXMEMDOM];

		static DPCPU_DEFINE(struct vm_batchqueue, pqbatch[MAXMEMDOM][PQ_COUNT]);
		static DPCPU_DEFINE(struct vm_batchqueue, noreuseq[MAXMEMDOM]);
		jeffUnsubmitted Not Done Inline Actions What is the reason for not using just another bit in the aflags to indicate head vs tail insertion? jeff: What is the reason for not using just another bit in the aflags to indicate head vs tail…
		markjAuthorUnsubmitted Not Done Inline Actions It was mostly because we only have two atomic bits left. Another solution would be to have a separate inactive pagequeue for pages that are bypassing global LRU. Because pq->pq_mtx is a pointer, both queues could be protected by the same lock, making transfers between the two queues relatively efficient. What do you think? markj: It was mostly because we only have two atomic bits left. Another solution would be to have a…
		kibUnsubmitted Not Done Inline Actions Why not put this into static pcpu ? kib: Why not put this into static pcpu ?
		markjAuthorUnsubmitted Not Done Inline Actions Is there any downside to using the dynamic per-CPU section? I like the fact that the definition is visible only in vm_page.c. markj: Is there any downside to using the dynamic per-CPU section? I like the fact that the definition…
		kibUnsubmitted Not Done Inline Actions It is one more arithmetic op on pointers to get to dpcu from pcpu. But my point is that the batch queues are not dynamic, they exists for the whole duration of the system operation. I do not insist either way. kib: It is one more arithmetic op on pointers to get to dpcu from pcpu. But my point is that the…

struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];		struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];

struct mtx_padalign __exclusive_cache_line vm_domainset_lock;		struct mtx_padalign __exclusive_cache_line vm_domainset_lock;
/* The following fields are protected by the domainset lock. */		/* The following fields are protected by the domainset lock. */
domainset_t __exclusive_cache_line vm_min_domains;		domainset_t __exclusive_cache_line vm_min_domains;
domainset_t __exclusive_cache_line vm_severe_domains;		domainset_t __exclusive_cache_line vm_severe_domains;
static int vm_min_waiters;		static int vm_min_waiters;
static int vm_severe_waiters;		static int vm_severe_waiters;
Show All 22 Lines
static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);		static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|		SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|
CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");		CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");

static uma_zone_t fakepg_zone;		static uma_zone_t fakepg_zone;

static void vm_page_alloc_check(vm_page_t m);		static void vm_page_alloc_check(vm_page_t m);
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);		static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
static void vm_page_enqueue(uint8_t queue, vm_page_t m);		static void vm_page_enqueue_lazy(vm_page_t m, uint8_t queue);
static void vm_page_init(void *dummy);		static void vm_page_init(void *dummy);
static int vm_page_insert_after(vm_page_t m, vm_object_t object,		static int vm_page_insert_after(vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred);		vm_pindex_t pindex, vm_page_t mpred);
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,		static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
vm_page_t mpred);		vm_page_t mpred);
static int vm_page_reclaim_run(int req_class, int domain, u_long npages,		static int vm_page_reclaim_run(int req_class, int domain, u_long npages,
vm_page_t m_run, vm_paddr_t high);		vm_page_t m_run, vm_paddr_t high);
static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,		static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,
▲ Show 20 Lines • Show All 1,623 Lines • ▼ Show 20 Lines	#endif
/*		/*
* At this point we had better have found a good page.		* At this point we had better have found a good page.
*/		*/
KASSERT(m != NULL, ("missing page"));		KASSERT(m != NULL, ("missing page"));

#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
found:		found:
#endif		#endif
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
flags = PG_ZERO;		flags = PG_ZERO;
▲ Show 20 Lines • Show All 180 Lines • ▼ Show 20 Lines	#endif
if (m_ret == NULL) {		if (m_ret == NULL) {
if (vm_domain_alloc_fail(vmd, object, req))		if (vm_domain_alloc_fail(vmd, object, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}
#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
found:		found:
#endif		#endif
for (m = m_ret; m < &m_ret[npages]; m++)		for (m = m_ret; m < &m_ret[npages]; m++) {
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);
		}

/*		/*
* Initialize the pages. Only the PG_ZERO flag is inherited.		* Initialize the pages. Only the PG_ZERO flag is inherited.
*/		*/
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
flags = PG_ZERO;		flags = PG_ZERO;
if ((req & VM_ALLOC_NODUMP) != 0)		if ((req & VM_ALLOC_NODUMP) != 0)
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines	if (vm_domain_allocate(vmd, req, 1)) {
if (m == NULL)		if (m == NULL)
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
}		}
if (m == NULL) {		if (m == NULL) {
if (vm_domain_alloc_fail(vmd, NULL, req))		if (vm_domain_alloc_fail(vmd, NULL, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
m->aflags = 0;		m->aflags = 0;
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
▲ Show 20 Lines • Show All 176 Lines • ▼ Show 20 Lines	#if VM_NRESERVLEVEL > 0
(level = vm_reserv_level_iffullpop(m)) >= 0) {		(level = vm_reserv_level_iffullpop(m)) >= 0) {
run_ext = 0;		run_ext = 0;
/* Advance to the end of the superpage. */		/* Advance to the end of the superpage. */
pa = VM_PAGE_TO_PHYS(m);		pa = VM_PAGE_TO_PHYS(m);
m_inc = atop(roundup2(pa + 1,		m_inc = atop(roundup2(pa + 1,
vm_reserv_size(level)) - pa);		vm_reserv_size(level)) - pa);
#endif		#endif
} else if (object->memattr == VM_MEMATTR_DEFAULT &&		} else if (object->memattr == VM_MEMATTR_DEFAULT &&
m->queue != PQ_NONE && !vm_page_busied(m)) {		m->queue != PQ_NONE &&
		(m->aflags & PGA_DEQUEUE) == 0 && !vm_page_busied(m)) {
		kibUnsubmitted Done Inline Actions Did you considered creating macros to detect logical enqueue conditions ? Like logically non-queued == (queue != PG_NONE && PGA_DEQUEUE == 0) ? kib: Did you considered creating macros to detect logical enqueue conditions ? Like logically non…
		markjAuthorUnsubmitted Done Inline Actions Yes, I should do that. I think this is missing an acquire barrier too. markj: Yes, I should do that. I think this is missing an acquire barrier too.
/*		/*
* The page is allocated but eligible for		* The page is allocated but eligible for
* relocation. Extend the current run by one		* relocation. Extend the current run by one
* page.		* page.
*/		*/
KASSERT(pmap_page_get_memattr(m) ==		KASSERT(pmap_page_get_memattr(m) ==
VM_MEMATTR_DEFAULT,		VM_MEMATTR_DEFAULT,
("page %p has an unexpected memattr", m));		("page %p has an unexpected memattr", m));
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines	else if ((object = m->object) != NULL) {
("page %p is PG_UNHOLDFREE", m));		("page %p is PG_UNHOLDFREE", m));
/* Don't care: PG_NODUMP, PG_ZERO. */		/* Don't care: PG_NODUMP, PG_ZERO. */
if (object->type != OBJT_DEFAULT &&		if (object->type != OBJT_DEFAULT &&
object->type != OBJT_SWAP &&		object->type != OBJT_SWAP &&
object->type != OBJT_VNODE)		object->type != OBJT_VNODE)
error = EINVAL;		error = EINVAL;
else if (object->memattr != VM_MEMATTR_DEFAULT)		else if (object->memattr != VM_MEMATTR_DEFAULT)
error = EINVAL;		error = EINVAL;
else if (m->queue != PQ_NONE && !vm_page_busied(m)) {		else if (m->queue != PQ_NONE &&
		(m->aflags & PGA_DEQUEUE) == 0 &&
		!vm_page_busied(m)) {
KASSERT(pmap_page_get_memattr(m) ==		KASSERT(pmap_page_get_memattr(m) ==
VM_MEMATTR_DEFAULT,		VM_MEMATTR_DEFAULT,
("page %p has an unexpected memattr", m));		("page %p has an unexpected memattr", m));
KASSERT((m->oflags & (VPO_SWAPINPROG \|		KASSERT((m->oflags & (VPO_SWAPINPROG \|
VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,		VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,
("page %p has unexpected oflags", m));		("page %p has unexpected oflags", m));
/* Don't care: VPO_NOSYNC. */		/* Don't care: VPO_NOSYNC. */
if (m->valid != 0) {		if (m->valid != 0) {
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines	else if ((object = m->object) != NULL) {
/*		/*
* Replace "m" with the new page. For		* Replace "m" with the new page. For
* vm_page_replace(), "m" must be busy		* vm_page_replace(), "m" must be busy
* and dequeued. Finally, change "m"		* and dequeued. Finally, change "m"
* as if vm_page_free() was called.		* as if vm_page_free() was called.
*/		*/
if (object->ref_count != 0)		if (object->ref_count != 0)
pmap_remove_all(m);		pmap_remove_all(m);
m_new->aflags = m->aflags;		m_new->aflags = m->aflags &
		~PGA_QUEUE_STATE_MASK;
KASSERT(m_new->oflags == VPO_UNMANAGED,		KASSERT(m_new->oflags == VPO_UNMANAGED,
("page %p is managed", m_new));		("page %p is managed", m_new));
m_new->oflags = m->oflags & VPO_NOSYNC;		m_new->oflags = m->oflags & VPO_NOSYNC;
pmap_copy_page(m, m_new);		pmap_copy_page(m, m_new);
m_new->valid = m->valid;		m_new->valid = m->valid;
m_new->dirty = m->dirty;		m_new->dirty = m->dirty;
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
vm_page_xbusy(m);		vm_page_xbusy(m);
vm_page_remque(m);		vm_page_remque(m);
vm_page_replace_checked(m_new, object,		vm_page_replace_checked(m_new, object,
m->pindex, m);		m->pindex, m);
if (vm_page_free_prep(m, false))		if (vm_page_free_prep(m))
SLIST_INSERT_HEAD(&free, m,		SLIST_INSERT_HEAD(&free, m,
plinks.s.ss);		plinks.s.ss);

/*		/*
* The new page must be deactivated		* The new page must be deactivated
* before the object is unlocked.		* before the object is unlocked.
*/		*/
vm_page_change_lock(m_new, &m_mtx);		vm_page_change_lock(m_new, &m_mtx);
vm_page_deactivate(m_new);		vm_page_deactivate(m_new);
} else {		} else {
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
vm_page_remque(m);		vm_page_remque(m);
vm_page_remove(m);		vm_page_remove(m);
if (vm_page_free_prep(m, false))		if (vm_page_free_prep(m))
SLIST_INSERT_HEAD(&free, m,		SLIST_INSERT_HEAD(&free, m,
plinks.s.ss);		plinks.s.ss);
KASSERT(m->dirty == 0,		KASSERT(m->dirty == 0,
("page %p is dirty", m));		("page %p is dirty", m));
}		}
} else		} else
error = EBUSY;		error = EBUSY;
unlock:		unlock:
▲ Show 20 Lines • Show All 426 Lines • ▼ Show 20 Lines

struct vm_pagequeue *		struct vm_pagequeue *
vm_page_pagequeue(vm_page_t m)		vm_page_pagequeue(vm_page_t m)
{		{

return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]);		return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]);
}		}

		static struct mtx *
		vm_page_pagequeue_lockptr(vm_page_t m)
		{

		if (m->queue == PQ_NONE)
		return (NULL);
		return (&vm_page_pagequeue(m)->pq_mutex);
		}

		static void
		vm_pqbatch_process(struct vm_pagequeue pq, struct vm_batchqueue bq,
		uint8_t queue)
		{
		vm_page_t m;
		int delta, i;
		uint8_t aflags;

		vm_pagequeue_assert_locked(pq);

		delta = 0;
		for (i = 0; i < bq->bq_cnt; i++) {
		m = bq->bq_pa[i];
		if (__predict_false(m->queue != queue))
		continue;

		aflags = m->aflags;
		if ((aflags & PGA_DEQUEUE) != 0) {
		if (__predict_true((aflags & PGA_ENQUEUED) != 0)) {
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		delta--;
		}

/*		/*
* vm_page_dequeue:		* Synchronize with the page daemon, which may be
		* simultaneously scanning this page with only the page
		* lock held. We must be careful to avoid leaving the
		* page in a state where it appears to belong to a page
		* queue.
		*/
		m->queue = PQ_NONE;
		atomic_thread_fence_rel();
		vm_page_aflag_clear(m, PGA_QUEUE_STATE_MASK);
		} else if ((aflags & PGA_ENQUEUED) == 0) {
		TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
		delta++;
		vm_page_aflag_set(m, PGA_ENQUEUED);
		if ((aflags & PGA_REQUEUE) != 0)
		vm_page_aflag_clear(m, PGA_REQUEUE);
		} else if ((aflags & PGA_REQUEUE) != 0) {
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
		vm_page_aflag_clear(m, PGA_REQUEUE);
		}
		}
		vm_batchqueue_init(bq);
		vm_pagequeue_cnt_add(pq, delta);
		}

		/*
		* vm_page_dequeue_lazy: [ internal use only ]
*		*
* Remove the given page from its current page queue.		* Request removal of the given page from its current page
		* queue. Physical removal from the queue may be deferred
		* arbitrarily, and may be cancelled by later queue operations
		* on that page.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_dequeue(vm_page_t m)		vm_page_dequeue_lazy(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain, queue;

vm_page_assert_locked(m);		vm_page_assert_locked(m);
KASSERT(m->queue < PQ_COUNT, ("vm_page_dequeue: page %p is not queued",
m));		queue = m->queue;
pq = vm_page_pagequeue(m);		if (queue == PQ_NONE)
		return;
		domain = vm_phys_domain(m);
		pq = &VM_DOMAIN(domain)->vmd_pagequeues[queue];

		vm_page_aflag_set(m, PGA_DEQUEUE);

		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (vm_batchqueue_insert(bq, m)) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		kibUnsubmitted Not Done Inline Actions Did you considered copying the pcpu pqbatch into a local array for processing and enabling preemption immediately after that ? I think that on full pqbatch, we spend a time which is enough to adversely affect the system responsivness. kib: Did you considered copying the pcpu pqbatch into a local array for processing and enabling…
		markjAuthorUnsubmitted Not Done Inline Actions I did, but there is a non-trivial stack space cost (256 bytes on LP64) in addition to that of the copy. The cost of processing 32 pages (8 on !amd64) ought to be quite low. I ran Jeff's dd benchmark (where parallel dd processes read from a sparse file and thus frequently enqueue pages) and used DTrace to count the number of preemptions that occurred in vm_page_enqueue(); excluding preemptions of the idle threads, less than 0.1% of preemptions occurred during the critical_exit() here. This is also an extreme case and in pretty much any workload we will not be performing queue operations at nearly that high a rate. For example, when running a 16-thread pgbench select-only benchmark on an SSD array with about 1GB/s throughput, the number of preemptions that occur here is effectively 0. Preemptions in UMA are much more common. markj: I did, but there is a non-trivial stack space cost (256 bytes on LP64) in addition to that of…
vm_pagequeue_lock(pq);		vm_pagequeue_lock(pq);
m->queue = PQ_NONE;		critical_enter();
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		bq = DPCPU_PTR(pqbatch[domain][queue]);
vm_pagequeue_cnt_dec(pq);		}
		vm_pqbatch_process(pq, bq, queue);

		/*
		* The page may have been dequeued by another thread before we
		* acquired the page queue lock. However, since we hold the
		* page lock, the page's queue field cannot change a second
		* time and we can safely clear PGA_DEQUEUE.
		*/
		KASSERT(m->queue == queue \|\| m->queue == PQ_NONE,
		("%s: page %p migrated between queues", __func__, m));
		if (m->queue == queue) {
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		} else
		vm_page_aflag_clear(m, PGA_DEQUEUE);
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		critical_exit();
}		}

/*		/*
* vm_page_dequeue_locked:		* vm_page_dequeue_locked:
*		*
* Remove the given page from its current page queue.		* Remove the page from its page queue, which must be locked.
		* If the page lock is not held, there is no guarantee that the
		* page will not be enqueued by another thread before this function
		* returns. In this case, it is up to the caller to ensure that
		* no other threads hold a reference to the page.
*		*
* The page and page queue must be locked.		* The page queue lock must be held. If the page is not already
		* logically dequeued, the page lock must be held as well.
*/		*/
void		void
vm_page_dequeue_locked(vm_page_t m)		vm_page_dequeue_locked(vm_page_t m)
{		{
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;

vm_page_lock_assert(m, MA_OWNED);		KASSERT(m->queue != PQ_NONE,
		("%s: page %p queue field is PQ_NONE", __func__, m));
		vm_pagequeue_assert_locked(vm_page_pagequeue(m));
		KASSERT((m->aflags & PGA_DEQUEUE) != 0 \|\|
		mtx_owned(vm_page_lockptr(m)),
		("%s: queued unlocked page %p", __func__, m));

		if ((m->aflags & PGA_ENQUEUED) != 0) {
pq = vm_page_pagequeue(m);		pq = vm_page_pagequeue(m);
vm_pagequeue_assert_locked(pq);
m->queue = PQ_NONE;
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_dec(pq);		vm_pagequeue_cnt_dec(pq);
}		}

/*		/*
* vm_page_enqueue:		* Synchronize with the page daemon, which may be simultaneously
		* scanning this page with only the page lock held. We must be careful
		* to avoid leaving the page in a state where it appears to belong to a
		* page queue.
		*/
		m->queue = PQ_NONE;
		atomic_thread_fence_rel();
		vm_page_aflag_clear(m, PGA_QUEUE_STATE_MASK);
		kibUnsubmitted Not Done Inline Actions I think you need acquire fence before reading aflags there. kib: I think you need acquire fence before reading aflags there.
		markjAuthorUnsubmitted Not Done Inline Actions I am not sure about this. Right now, we are using barriers to synchronize the following operations: T1 dequeues a page with the queue lock held: set m->queue = PQ_NONE, release barrier, clear PGA_DEQUEUE in aflags T2 checks to see if the page is dequeued with page lock held: check for PGA_DEQUEUE in aflags, acquire barrier, check m->queue == PQ_NONE (I don't know what kind of notation is best here.) Here we hold the queue lock, so m->queue is stable, and any thread which updated it must have implicitly issued a release barrier when it unlocked the queue. So I don't see why we need an acquire fence. markj: I am not sure about this. Right now, we are using barriers to synchronize the following…
		kibUnsubmitted Not Done Inline Actions Ok, thank you for the explanation. kib: Ok, thank you for the explanation.
		}

		/*
		* vm_page_dequeue:
*		*
* Add the given page to the specified page queue.		* Remove the page from whichever page queue it's in, if any.
		* If the page lock is not held, there is no guarantee that the
		* page will not be enqueued by another thread before this function
		* returns. In this case, it is up to the caller to ensure that
		* no other threads hold a reference to the page.
		*/
		void
		vm_page_dequeue(vm_page_t m)
		{
		struct mtx lock, lock1;

		lock = vm_page_pagequeue_lockptr(m);
		for (;;) {
		if (lock == NULL)
		return;
		mtx_lock(lock);
		if ((lock1 = vm_page_pagequeue_lockptr(m)) == lock)
		break;
		mtx_unlock(lock);
		lock = lock1;
		}
		KASSERT(lock == vm_page_pagequeue_lockptr(m),
		("%s: page %p migrated directly between queues", __func__, m));
		vm_page_dequeue_locked(m);
		mtx_unlock(lock);
		}

		/*
		* vm_page_enqueue_lazy:
*		*
		* Schedule the given page for insertion into the specified page queue.
		* Physical insertion of the page may be deferred indefinitely.
		*
* The page must be locked.		* The page must be locked.
*/		*/
static void		static void
vm_page_enqueue(uint8_t queue, vm_page_t m)		vm_page_enqueue_lazy(vm_page_t m, uint8_t queue)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain;

vm_page_lock_assert(m, MA_OWNED);		vm_page_assert_locked(m);
KASSERT(queue < PQ_COUNT,		KASSERT(m->queue == PQ_NONE && (m->aflags & PGA_QUEUE_STATE_MASK) == 0,
("vm_page_enqueue: invalid queue %u request for page %p",		("%s: page %p is already enqueued", __func__, m));
queue, m));
		domain = vm_phys_domain(m);
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];		pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];
vm_pagequeue_lock(pq);
		/*
		* The queue field might be changed back to PQ_NONE by a concurrent
		* call to vm_page_dequeue(). In that case the batch queue entry will
		* be a no-op.
		*/
m->queue = queue;		m->queue = queue;
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_inc(pq);		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (__predict_true(vm_batchqueue_insert(bq, m))) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		vm_pagequeue_lock(pq);
		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		}
		glebiusUnsubmitted Not Done Inline Actions What if after critical_exit we have changed the CPU? In this case proceeding further to the end of the function doesn't seem optimal. I'd suggest just to put a page on regular queue, since we already got lock. glebius: What if after critical_exit we have changed the CPU? In this case proceeding further to the end…
		markjAuthorUnsubmitted Not Done Inline Actions Why not? We now hold the page queue lock, so we might as well drain the current queue. markj: Why not? We now hold the page queue lock, so we might as well drain the current queue.
		vm_pqbatch_process(pq, bq, queue);
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		kibUnsubmitted Not Done Inline Actions Don't you need the release fence between setting page domain and setting PGA_REQUEUE ? kib: Don't you need the release fence between setting page domain and setting PGA_REQUEUE ?
		markjAuthorUnsubmitted Not Done Inline Actions See above. We should already be synchronized by the page lock. markj: See above. We should already be synchronized by the page lock.
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		glebiusUnsubmitted Not Done Inline Actions Can you please explain the need for the second call to vm_pqbatch_process? glebius: Can you please explain the need for the second call to vm_pqbatch_process?
		markjAuthorUnsubmitted Done Inline Actions It processes solely the page m. We're at this point because the queue is full, so we have to drain it first by calling `vm_pqbatch_process()`. Probably there should be a `vm_pqbatch_process_page()` which processes a single page, and we just call that on `m`. markj: It processes solely the page m. We're at this point because the queue is full, so we have to…
		critical_exit();
}		}

/*		/*
* vm_page_requeue:		* vm_page_requeue:
*		*
* Move the given page to the tail of its current page queue.		* Schedule a requeue of the given page.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_requeue(vm_page_t m)		vm_page_requeue(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain, queue;

vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
KASSERT(m->queue != PQ_NONE,		KASSERT(m->queue != PQ_NONE,
("vm_page_requeue: page %p is not queued", m));		("%s: page %p is not enqueued", __func__, m));

		domain = vm_phys_domain(m);
		queue = m->queue;
pq = vm_page_pagequeue(m);		pq = vm_page_pagequeue(m);

		if (queue == PQ_NONE)
		return;

		vm_page_aflag_set(m, PGA_REQUEUE);
		jeffUnsubmitted Done Inline Actions Does it make sense to test for this flag before setting it? If it's already set there is not much value in doing it again. jeff: Does it make sense to test for this flag before setting it? If it's already set there is not…
		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (__predict_true(vm_batchqueue_insert(bq, m))) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
vm_pagequeue_lock(pq);		vm_pagequeue_lock(pq);
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		critical_enter();
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);		bq = DPCPU_PTR(pqbatch[domain][queue]);
		}
		vm_pqbatch_process(pq, bq, queue);
		KASSERT(m->queue == queue \|\| m->queue == PQ_NONE,
		("%s: page %p migrated between queues", __func__, m));
		if (m->queue == queue) {
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		} else
		vm_page_aflag_clear(m, PGA_REQUEUE);
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		critical_exit();
}		}

/*		/*
* vm_page_requeue_locked:
*
* Move the given page to the tail of its current page queue.
*
* The page queue must be locked.
*/
void
vm_page_requeue_locked(vm_page_t m)
{
struct vm_pagequeue *pq;

KASSERT(m->queue != PQ_NONE,
("vm_page_requeue_locked: page %p is not queued", m));
pq = vm_page_pagequeue(m);
vm_pagequeue_assert_locked(pq);
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
}

/*
* vm_page_activate:		* vm_page_activate:
*		*
* Put the specified page on the active list (if appropriate).		* Put the specified page on the active list (if appropriate).
* Ensure that act_count is at least ACT_INIT but do not otherwise		* Ensure that act_count is at least ACT_INIT but do not otherwise
* mess with it.		* mess with it.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_activate(vm_page_t m)		vm_page_activate(vm_page_t m)
{		{
int queue;		int queue;

vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
if ((queue = m->queue) != PQ_ACTIVE) {
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if ((queue = m->queue) == PQ_ACTIVE \|\| m->wire_count > 0 \|\|
if (m->act_count < ACT_INIT)		(m->oflags & VPO_UNMANAGED) != 0) {
		if (queue == PQ_ACTIVE && m->act_count < ACT_INIT)
m->act_count = ACT_INIT;		m->act_count = ACT_INIT;
if (queue != PQ_NONE)		return;
vm_page_dequeue(m);
vm_page_enqueue(PQ_ACTIVE, m);
}		}
} else {
		vm_page_remque(m);
if (m->act_count < ACT_INIT)		if (m->act_count < ACT_INIT)
m->act_count = ACT_INIT;		m->act_count = ACT_INIT;
		vm_page_enqueue_lazy(m, PQ_ACTIVE);
}		}
}

/*		/*
* vm_page_free_prep:		* vm_page_free_prep:
*		*
* Prepares the given page to be put on the free list,		* Prepares the given page to be put on the free list,
* disassociating it from any VM object. The caller may return		* disassociating it from any VM object. The caller may return
* the page to the free list only if this function returns true.		* the page to the free list only if this function returns true.
*		*
* The object must be locked. The page must be locked if it is		* The object must be locked. The page must be locked if it is
* managed. For a queued managed page, the pagequeue_locked		* managed.
* argument specifies whether the page queue is already locked.
*/		*/
bool		bool
vm_page_free_prep(vm_page_t m, bool pagequeue_locked)		vm_page_free_prep(vm_page_t m)
{		{

#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)		#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)
if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {		if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {
uint64_t *p;		uint64_t *p;
int i;		int i;
p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));		p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)		for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)
KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",		KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",
m, i, (uintmax_t)*p));		m, i, (uintmax_t)*p));
}		}
#endif		#endif
if ((m->oflags & VPO_UNMANAGED) == 0) {		if ((m->oflags & VPO_UNMANAGED) == 0) {
vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
KASSERT(!pmap_page_is_mapped(m),		KASSERT(!pmap_page_is_mapped(m),
("vm_page_free_toq: freeing mapped page %p", m));		("vm_page_free_prep: freeing mapped page %p", m));
} else		} else
KASSERT(m->queue == PQ_NONE,		KASSERT(m->queue == PQ_NONE,
("vm_page_free_toq: unmanaged page %p is queued", m));		("vm_page_free_prep: unmanaged page %p is queued", m));
VM_CNT_INC(v_tfree);		VM_CNT_INC(v_tfree);

if (vm_page_sbusied(m))		if (vm_page_sbusied(m))
panic("vm_page_free: freeing busy page %p", m);		panic("vm_page_free_prep: freeing busy page %p", m);

vm_page_remove(m);		vm_page_remove(m);

/*		/*
* If fictitious remove object association and		* If fictitious remove object association and
* return.		* return.
*/		*/
if ((m->flags & PG_FICTITIOUS) != 0) {		if ((m->flags & PG_FICTITIOUS) != 0) {
KASSERT(m->wire_count == 1,		KASSERT(m->wire_count == 1,
("fictitious page %p is not wired", m));		("fictitious page %p is not wired", m));
KASSERT(m->queue == PQ_NONE,		KASSERT(m->queue == PQ_NONE,
("fictitious page %p is queued", m));		("fictitious page %p is queued", m));
return (false);		return (false);
}		}

if (m->queue != PQ_NONE) {		/*
if (pagequeue_locked)		* Pages need not be dequeued before they are returned to the physical
vm_page_dequeue_locked(m);		* memory allocator, but they must at least be marked for a deferred
else		* dequeue.
vm_page_dequeue(m);		*/
}		if ((m->oflags & VPO_UNMANAGED) == 0)
		vm_page_dequeue_lazy(m);

m->valid = 0;		m->valid = 0;
vm_page_undirty(m);		vm_page_undirty(m);

if (m->wire_count != 0)		if (m->wire_count != 0)
panic("vm_page_free: freeing wired page %p", m);		panic("vm_page_free_prep: freeing wired page %p", m);
if (m->hold_count != 0) {		if (m->hold_count != 0) {
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
KASSERT((m->flags & PG_UNHOLDFREE) == 0,		KASSERT((m->flags & PG_UNHOLDFREE) == 0,
("vm_page_free: freeing PG_UNHOLDFREE page %p", m));		("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m));
m->flags \|= PG_UNHOLDFREE;		m->flags \|= PG_UNHOLDFREE;
return (false);		return (false);
}		}

/*		/*
* Restore the default memory attribute to the page.		* Restore the default memory attribute to the page.
*/		*/
if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)		if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);		pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);

#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
if (vm_reserv_free_page(m))		if (vm_reserv_free_page(m))
return (false);		return (false);
#endif		#endif

return (true);		return (true);
}		}

void
vm_page_free_phys_pglist(struct pglist *tq)
{
struct vm_domain *vmd;
vm_page_t m;
int cnt;

if (TAILQ_EMPTY(tq))
return;
vmd = NULL;
cnt = 0;
TAILQ_FOREACH(m, tq, listq) {
if (vmd != vm_pagequeue_domain(m)) {
if (vmd != NULL) {
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, cnt);
cnt = 0;
}
vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);
}
vm_phys_free_pages(m, 0);
cnt++;
}
if (vmd != NULL) {
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, cnt);
}
}

/*		/*
* vm_page_free_toq:		* vm_page_free_toq:
*		*
* Returns the given page to the free list, disassociating it		* Returns the given page to the free list, disassociating it
* from any VM object.		* from any VM object.
*		*
* The object must be locked. The page must be locked if it is		* The object must be locked. The page must be locked if it is
* managed.		* managed.
*/		*/
void		void
vm_page_free_toq(vm_page_t m)		vm_page_free_toq(vm_page_t m)
{		{
struct vm_domain *vmd;		struct vm_domain *vmd;

if (!vm_page_free_prep(m, false))		if (!vm_page_free_prep(m))
return;		return;

vmd = vm_pagequeue_domain(m);		vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);		vm_domain_free_lock(vmd);
vm_phys_free_pages(m, 0);		vm_phys_free_pages(m, 0);
vm_domain_free_unlock(vmd);		vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
}		}

/*		/*
* vm_page_free_pages_toq:		* vm_page_free_pages_toq:
*		*
* Returns a list of pages to the free list, disassociating it		* Returns a list of pages to the free list, disassociating it
* from any VM object. In other words, this is equivalent to		* from any VM object. In other words, this is equivalent to
* calling vm_page_free_toq() for each page of a list of VM objects.		* calling vm_page_free_toq() for each page of a list of VM objects.
*		*
* The objects must be locked. The pages must be locked if it is		* The objects must be locked. The pages must be locked if it is
* managed.		* managed.
*/		*/
void		void
vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)		vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)
{		{
vm_page_t m;		vm_page_t m;
struct pglist pgl;
int count;		int count;

if (SLIST_EMPTY(free))		if (SLIST_EMPTY(free))
return;		return;

count = 0;		count = 0;
TAILQ_INIT(&pgl);
while ((m = SLIST_FIRST(free)) != NULL) {		while ((m = SLIST_FIRST(free)) != NULL) {
count++;		count++;
SLIST_REMOVE_HEAD(free, plinks.s.ss);		SLIST_REMOVE_HEAD(free, plinks.s.ss);
if (vm_page_free_prep(m, false))		vm_page_free_toq(m);
TAILQ_INSERT_TAIL(&pgl, m, listq);
}		}

vm_page_free_phys_pglist(&pgl);

if (update_wire_count)		if (update_wire_count)
vm_wire_sub(count);		vm_wire_sub(count);
}		}

/*		/*
* vm_page_wire:		* vm_page_wire:
*		*
* Mark this page as wired down. If the page is fictitious, then		* Mark this page as wired down. If the page is fictitious, then
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
bool		bool
vm_page_unwire(vm_page_t m, uint8_t queue)		vm_page_unwire(vm_page_t m, uint8_t queue)
{		{
bool unwired;		bool unwired;

KASSERT(queue < PQ_COUNT \|\| queue == PQ_NONE,		KASSERT(queue < PQ_COUNT \|\| queue == PQ_NONE,
("vm_page_unwire: invalid queue %u request for page %p",		("vm_page_unwire: invalid queue %u request for page %p",
queue, m));		queue, m));
		if ((m->oflags & VPO_UNMANAGED) == 0)
		vm_page_assert_locked(m);

unwired = vm_page_unwire_noq(m);		unwired = vm_page_unwire_noq(m);
if (unwired && (m->oflags & VPO_UNMANAGED) == 0 && m->object != NULL) {		if (!unwired \|\| (m->oflags & VPO_UNMANAGED) != 0 \|\| m->object == NULL)
		return (unwired);

if (m->queue == queue) {		if (m->queue == queue) {
if (queue == PQ_ACTIVE)		if (queue == PQ_ACTIVE)
vm_page_reference(m);		vm_page_reference(m);
else if (queue != PQ_NONE)		else if (queue != PQ_NONE)
vm_page_requeue(m);		vm_page_requeue(m);
} else {		} else {
vm_page_remque(m);		vm_page_dequeue(m);
if (queue != PQ_NONE) {		if (queue != PQ_NONE) {
vm_page_enqueue(queue, m);		vm_page_enqueue_lazy(m, queue);
if (queue == PQ_ACTIVE)		if (queue == PQ_ACTIVE)
/* Initialize act_count. */		/* Initialize act_count. */
vm_page_activate(m);		vm_page_activate(m);
}		}
}		}
}
return (unwired);		return (unwired);
}		}

/*		/*
*		*
* vm_page_unwire_noq:		* vm_page_unwire_noq:
*		*
* Unwire a page without (re-)inserting it into a page queue. It is up		* Unwire a page without (re-)inserting it into a page queue. It is up
Show All 17 Lines	vm_page_unwire_noq(vm_page_t m)
if (m->wire_count == 0) {		if (m->wire_count == 0) {
vm_wire_sub(1);		vm_wire_sub(1);
return (true);		return (true);
} else		} else
return (false);		return (false);
}		}

/*		/*
* Move the specified page to the inactive queue, or requeue the page if it is		* Move the specified page to the tail of the inactive queue, or requeue
* already in the inactive queue.		* the page if it is already in the inactive queue.
*		*
* Normally, "noreuse" is FALSE, resulting in LRU ordering of the inactive
* queue. However, setting "noreuse" to TRUE will accelerate the specified
* page's reclamation, but it will not unmap the page from any address space.
* This is implemented by inserting the page near the head of the inactive
* queue, using a marker page to guide FIFO insertion ordering.
*
* The page must be locked.		* The page must be locked.
*/		*/
static inline void		void
_vm_page_deactivate(vm_page_t m, boolean_t noreuse)		vm_page_deactivate(vm_page_t m)
{		{
struct vm_pagequeue *pq;
int queue;

vm_page_assert_locked(m);		vm_page_assert_locked(m);

if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[PQ_INACTIVE];		return;
/* Avoid multiple acquisitions of the inactive queue lock. */
queue = m->queue;
if (queue == PQ_INACTIVE) {
vm_pagequeue_lock(pq);
vm_page_dequeue_locked(m);
} else {
if (queue != PQ_NONE)
vm_page_dequeue(m);
vm_pagequeue_lock(pq);
}
m->queue = PQ_INACTIVE;
if (noreuse)
TAILQ_INSERT_BEFORE(
&vm_pagequeue_domain(m)->vmd_inacthead, m,
plinks.q);
else
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_inc(pq);
vm_pagequeue_unlock(pq);
}
}

/*		if (!vm_page_inactive(m)) {
		jeffUnsubmitted Done Inline Actions Should be an XXX comment. This path likely needs to be fast. jeff: Should be an XXX comment. This path likely needs to be fast.
* Move the specified page to the inactive queue, or requeue the page if it is		vm_page_remque(m);
* already in the inactive queue.		vm_page_enqueue_lazy(m, PQ_INACTIVE);
*		} else
* The page must be locked.		vm_page_requeue(m);
*/
void
vm_page_deactivate(vm_page_t m)
{

_vm_page_deactivate(m, FALSE);
}		}

/*		/*
* Move the specified page to the inactive queue with the expectation		* Move the specified page close to the head of the inactive queue,
* that it is unlikely to be reused.		* bypassing LRU. A marker page is used to maintain FIFO ordering.
		* As with regular enqueues, we use a per-CPU batch queue to reduce
		* contention on the page queue lock.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_deactivate_noreuse(vm_page_t m)		vm_page_deactivate_noreuse(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
		struct vm_domain *vmd;
		struct vm_pagequeue *pq;
		vm_page_t marker;
		int domain;

_vm_page_deactivate(m, TRUE);		vm_page_assert_locked(m);

		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
		return;

		domain = vm_phys_domain(m);
		vmd = VM_DOMAIN(domain);
		pq = &vmd->vmd_pagequeues[PQ_INACTIVE];

		if (!vm_page_inactive(m))
		vm_page_remque(m);

		m->queue = PQ_INACTIVE;

		critical_enter();
		bq = DPCPU_PTR(noreuseq[domain]);
		if (vm_batchqueue_insert(bq, m)) {
		critical_exit();
		return;
}		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		vm_pagequeue_lock(pq);
		critical_enter();
		bq = DPCPU_PTR(noreuseq[domain]);
		}
		marker = &vmd->vmd_inacthead;
		do {
		if (m->queue != PQ_INACTIVE)
		continue;
		if ((m->aflags & PGA_ENQUEUED) != 0)
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		else {
		vm_page_aflag_set(m, PGA_ENQUEUED);
		vm_pagequeue_cnt_inc(pq);
		}
		TAILQ_INSERT_BEFORE(marker, m, plinks.q);
		} while ((m = vm_batchqueue_pop(bq)) != NULL);
		vm_pagequeue_unlock(pq);
		vm_batchqueue_init(bq);
		critical_exit();
		}

/*		/*
* vm_page_launder		* vm_page_launder
*		*
* Put a page in the laundry, or requeue it if it is already there.		* Put a page in the laundry, or requeue it if it is already there.
*/		*/
void		void
vm_page_launder(vm_page_t m)		vm_page_launder(vm_page_t m)
{		{

vm_page_assert_locked(m);		vm_page_assert_locked(m);
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
		return;

if (m->queue == PQ_LAUNDRY)		if (m->queue == PQ_LAUNDRY)
vm_page_requeue(m);		vm_page_requeue(m);
else {		else {
vm_page_remque(m);		vm_page_remque(m);
vm_page_enqueue(PQ_LAUNDRY, m);		vm_page_enqueue_lazy(m, PQ_LAUNDRY);
}		}
}		}
}

/*		/*
* vm_page_unswappable		* vm_page_unswappable
*		*
* Put a page in the PQ_UNSWAPPABLE holding queue.		* Put a page in the PQ_UNSWAPPABLE holding queue.
*/		*/
void		void
vm_page_unswappable(vm_page_t m)		vm_page_unswappable(vm_page_t m)
{		{

vm_page_assert_locked(m);		vm_page_assert_locked(m);
KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,		KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,
("page %p already unswappable", m));		("page %p already unswappable", m));
if (m->queue != PQ_NONE)
vm_page_dequeue(m);		vm_page_remque(m);
vm_page_enqueue(PQ_UNSWAPPABLE, m);		vm_page_enqueue_lazy(m, PQ_UNSWAPPABLE);
}		}

/*		/*
* Attempt to free the page. If it cannot be freed, do nothing. Returns true		* Attempt to free the page. If it cannot be freed, do nothing. Returns true
* if the page is freed and false otherwise.		* if the page is freed and false otherwise.
*		*
* The page must be managed. The page and its containing object must be		* The page must be managed. The page and its containing object must be
* locked.		* locked.
▲ Show 20 Lines • Show All 750 Lines • Show Last 20 Lines

View Options

sys/dev/mlx5/mlx5_ib/mlx5_cq.c

Show First 20 Lines • Show All 125 Lines • ▼ Show 20 Lines
#include <vm/uma_int.h>		#include <vm/uma_int.h>

#include <machine/md_var.h>		#include <machine/md_var.h>

extern int uma_startup_count(int);		extern int uma_startup_count(int);
extern void uma_startup(void *, int);		extern void uma_startup(void *, int);
extern int vmem_startup_count(void);		extern int vmem_startup_count(void);

/*
* Associated with page of user-allocatable memory is a
* page structure.
*/

struct vm_domain vm_dom[MAXMEMDOM];		struct vm_domain vm_dom[MAXMEMDOM];

		static DPCPU_DEFINE(struct vm_batchqueue, pqbatch[MAXMEMDOM][PQ_COUNT]);
		static DPCPU_DEFINE(struct vm_batchqueue, noreuseq[MAXMEMDOM]);
		jeffUnsubmitted Not Done Inline Actions What is the reason for not using just another bit in the aflags to indicate head vs tail insertion? jeff: What is the reason for not using just another bit in the aflags to indicate head vs tail…
		markjAuthorUnsubmitted Not Done Inline Actions It was mostly because we only have two atomic bits left. Another solution would be to have a separate inactive pagequeue for pages that are bypassing global LRU. Because pq->pq_mtx is a pointer, both queues could be protected by the same lock, making transfers between the two queues relatively efficient. What do you think? markj: It was mostly because we only have two atomic bits left. Another solution would be to have a…
		kibUnsubmitted Not Done Inline Actions Why not put this into static pcpu ? kib: Why not put this into static pcpu ?
		markjAuthorUnsubmitted Not Done Inline Actions Is there any downside to using the dynamic per-CPU section? I like the fact that the definition is visible only in vm_page.c. markj: Is there any downside to using the dynamic per-CPU section? I like the fact that the definition…
		kibUnsubmitted Not Done Inline Actions It is one more arithmetic op on pointers to get to dpcu from pcpu. But my point is that the batch queues are not dynamic, they exists for the whole duration of the system operation. I do not insist either way. kib: It is one more arithmetic op on pointers to get to dpcu from pcpu. But my point is that the…

struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];		struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];

struct mtx_padalign __exclusive_cache_line vm_domainset_lock;		struct mtx_padalign __exclusive_cache_line vm_domainset_lock;
/* The following fields are protected by the domainset lock. */		/* The following fields are protected by the domainset lock. */
domainset_t __exclusive_cache_line vm_min_domains;		domainset_t __exclusive_cache_line vm_min_domains;
domainset_t __exclusive_cache_line vm_severe_domains;		domainset_t __exclusive_cache_line vm_severe_domains;
static int vm_min_waiters;		static int vm_min_waiters;
static int vm_severe_waiters;		static int vm_severe_waiters;
Show All 22 Lines
static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);		static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|		SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|
CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");		CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");

static uma_zone_t fakepg_zone;		static uma_zone_t fakepg_zone;

static void vm_page_alloc_check(vm_page_t m);		static void vm_page_alloc_check(vm_page_t m);
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);		static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
static void vm_page_enqueue(uint8_t queue, vm_page_t m);		static void vm_page_enqueue_lazy(vm_page_t m, uint8_t queue);
static void vm_page_init(void *dummy);		static void vm_page_init(void *dummy);
static int vm_page_insert_after(vm_page_t m, vm_object_t object,		static int vm_page_insert_after(vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred);		vm_pindex_t pindex, vm_page_t mpred);
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,		static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
vm_page_t mpred);		vm_page_t mpred);
static int vm_page_reclaim_run(int req_class, int domain, u_long npages,		static int vm_page_reclaim_run(int req_class, int domain, u_long npages,
vm_page_t m_run, vm_paddr_t high);		vm_page_t m_run, vm_paddr_t high);
static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,		static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,
▲ Show 20 Lines • Show All 1,623 Lines • ▼ Show 20 Lines	#endif
/*		/*
* At this point we had better have found a good page.		* At this point we had better have found a good page.
*/		*/
KASSERT(m != NULL, ("missing page"));		KASSERT(m != NULL, ("missing page"));

#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
found:		found:
#endif		#endif
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
flags = PG_ZERO;		flags = PG_ZERO;
▲ Show 20 Lines • Show All 180 Lines • ▼ Show 20 Lines	#endif
if (m_ret == NULL) {		if (m_ret == NULL) {
if (vm_domain_alloc_fail(vmd, object, req))		if (vm_domain_alloc_fail(vmd, object, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}
#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
found:		found:
#endif		#endif
for (m = m_ret; m < &m_ret[npages]; m++)		for (m = m_ret; m < &m_ret[npages]; m++) {
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);
		}

/*		/*
* Initialize the pages. Only the PG_ZERO flag is inherited.		* Initialize the pages. Only the PG_ZERO flag is inherited.
*/		*/
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
flags = PG_ZERO;		flags = PG_ZERO;
if ((req & VM_ALLOC_NODUMP) != 0)		if ((req & VM_ALLOC_NODUMP) != 0)
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines	if (vm_domain_allocate(vmd, req, 1)) {
if (m == NULL)		if (m == NULL)
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
}		}
if (m == NULL) {		if (m == NULL) {
if (vm_domain_alloc_fail(vmd, NULL, req))		if (vm_domain_alloc_fail(vmd, NULL, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
m->aflags = 0;		m->aflags = 0;
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
▲ Show 20 Lines • Show All 176 Lines • ▼ Show 20 Lines	#if VM_NRESERVLEVEL > 0
(level = vm_reserv_level_iffullpop(m)) >= 0) {		(level = vm_reserv_level_iffullpop(m)) >= 0) {
run_ext = 0;		run_ext = 0;
/* Advance to the end of the superpage. */		/* Advance to the end of the superpage. */
pa = VM_PAGE_TO_PHYS(m);		pa = VM_PAGE_TO_PHYS(m);
m_inc = atop(roundup2(pa + 1,		m_inc = atop(roundup2(pa + 1,
vm_reserv_size(level)) - pa);		vm_reserv_size(level)) - pa);
#endif		#endif
} else if (object->memattr == VM_MEMATTR_DEFAULT &&		} else if (object->memattr == VM_MEMATTR_DEFAULT &&
m->queue != PQ_NONE && !vm_page_busied(m)) {		m->queue != PQ_NONE &&
		(m->aflags & PGA_DEQUEUE) == 0 && !vm_page_busied(m)) {
		kibUnsubmitted Done Inline Actions Did you considered creating macros to detect logical enqueue conditions ? Like logically non-queued == (queue != PG_NONE && PGA_DEQUEUE == 0) ? kib: Did you considered creating macros to detect logical enqueue conditions ? Like logically non…
		markjAuthorUnsubmitted Done Inline Actions Yes, I should do that. I think this is missing an acquire barrier too. markj: Yes, I should do that. I think this is missing an acquire barrier too.
/*		/*
* The page is allocated but eligible for		* The page is allocated but eligible for
* relocation. Extend the current run by one		* relocation. Extend the current run by one
* page.		* page.
*/		*/
KASSERT(pmap_page_get_memattr(m) ==		KASSERT(pmap_page_get_memattr(m) ==
VM_MEMATTR_DEFAULT,		VM_MEMATTR_DEFAULT,
("page %p has an unexpected memattr", m));		("page %p has an unexpected memattr", m));
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines	else if ((object = m->object) != NULL) {
("page %p is PG_UNHOLDFREE", m));		("page %p is PG_UNHOLDFREE", m));
/* Don't care: PG_NODUMP, PG_ZERO. */		/* Don't care: PG_NODUMP, PG_ZERO. */
if (object->type != OBJT_DEFAULT &&		if (object->type != OBJT_DEFAULT &&
object->type != OBJT_SWAP &&		object->type != OBJT_SWAP &&
object->type != OBJT_VNODE)		object->type != OBJT_VNODE)
error = EINVAL;		error = EINVAL;
else if (object->memattr != VM_MEMATTR_DEFAULT)		else if (object->memattr != VM_MEMATTR_DEFAULT)
error = EINVAL;		error = EINVAL;
else if (m->queue != PQ_NONE && !vm_page_busied(m)) {		else if (m->queue != PQ_NONE &&
		(m->aflags & PGA_DEQUEUE) == 0 &&
		!vm_page_busied(m)) {
KASSERT(pmap_page_get_memattr(m) ==		KASSERT(pmap_page_get_memattr(m) ==
VM_MEMATTR_DEFAULT,		VM_MEMATTR_DEFAULT,
("page %p has an unexpected memattr", m));		("page %p has an unexpected memattr", m));
KASSERT((m->oflags & (VPO_SWAPINPROG \|		KASSERT((m->oflags & (VPO_SWAPINPROG \|
VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,		VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,
("page %p has unexpected oflags", m));		("page %p has unexpected oflags", m));
/* Don't care: VPO_NOSYNC. */		/* Don't care: VPO_NOSYNC. */
if (m->valid != 0) {		if (m->valid != 0) {
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines	else if ((object = m->object) != NULL) {
/*		/*
* Replace "m" with the new page. For		* Replace "m" with the new page. For
* vm_page_replace(), "m" must be busy		* vm_page_replace(), "m" must be busy
* and dequeued. Finally, change "m"		* and dequeued. Finally, change "m"
* as if vm_page_free() was called.		* as if vm_page_free() was called.
*/		*/
if (object->ref_count != 0)		if (object->ref_count != 0)
pmap_remove_all(m);		pmap_remove_all(m);
m_new->aflags = m->aflags;		m_new->aflags = m->aflags &
		~PGA_QUEUE_STATE_MASK;
KASSERT(m_new->oflags == VPO_UNMANAGED,		KASSERT(m_new->oflags == VPO_UNMANAGED,
("page %p is managed", m_new));		("page %p is managed", m_new));
m_new->oflags = m->oflags & VPO_NOSYNC;		m_new->oflags = m->oflags & VPO_NOSYNC;
pmap_copy_page(m, m_new);		pmap_copy_page(m, m_new);
m_new->valid = m->valid;		m_new->valid = m->valid;
m_new->dirty = m->dirty;		m_new->dirty = m->dirty;
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
vm_page_xbusy(m);		vm_page_xbusy(m);
vm_page_remque(m);		vm_page_remque(m);
vm_page_replace_checked(m_new, object,		vm_page_replace_checked(m_new, object,
m->pindex, m);		m->pindex, m);
if (vm_page_free_prep(m, false))		if (vm_page_free_prep(m))
SLIST_INSERT_HEAD(&free, m,		SLIST_INSERT_HEAD(&free, m,
plinks.s.ss);		plinks.s.ss);

/*		/*
* The new page must be deactivated		* The new page must be deactivated
* before the object is unlocked.		* before the object is unlocked.
*/		*/
vm_page_change_lock(m_new, &m_mtx);		vm_page_change_lock(m_new, &m_mtx);
vm_page_deactivate(m_new);		vm_page_deactivate(m_new);
} else {		} else {
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
vm_page_remque(m);		vm_page_remque(m);
vm_page_remove(m);		vm_page_remove(m);
if (vm_page_free_prep(m, false))		if (vm_page_free_prep(m))
SLIST_INSERT_HEAD(&free, m,		SLIST_INSERT_HEAD(&free, m,
plinks.s.ss);		plinks.s.ss);
KASSERT(m->dirty == 0,		KASSERT(m->dirty == 0,
("page %p is dirty", m));		("page %p is dirty", m));
}		}
} else		} else
error = EBUSY;		error = EBUSY;
unlock:		unlock:
▲ Show 20 Lines • Show All 426 Lines • ▼ Show 20 Lines

struct vm_pagequeue *		struct vm_pagequeue *
vm_page_pagequeue(vm_page_t m)		vm_page_pagequeue(vm_page_t m)
{		{

return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]);		return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]);
}		}

		static struct mtx *
		vm_page_pagequeue_lockptr(vm_page_t m)
		{

		if (m->queue == PQ_NONE)
		return (NULL);
		return (&vm_page_pagequeue(m)->pq_mutex);
		}

		static void
		vm_pqbatch_process(struct vm_pagequeue pq, struct vm_batchqueue bq,
		uint8_t queue)
		{
		vm_page_t m;
		int delta, i;
		uint8_t aflags;

		vm_pagequeue_assert_locked(pq);

		delta = 0;
		for (i = 0; i < bq->bq_cnt; i++) {
		m = bq->bq_pa[i];
		if (__predict_false(m->queue != queue))
		continue;

		aflags = m->aflags;
		if ((aflags & PGA_DEQUEUE) != 0) {
		if (__predict_true((aflags & PGA_ENQUEUED) != 0)) {
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		delta--;
		}

/*		/*
* vm_page_dequeue:		* Synchronize with the page daemon, which may be
		* simultaneously scanning this page with only the page
		* lock held. We must be careful to avoid leaving the
		* page in a state where it appears to belong to a page
		* queue.
		*/
		m->queue = PQ_NONE;
		atomic_thread_fence_rel();
		vm_page_aflag_clear(m, PGA_QUEUE_STATE_MASK);
		} else if ((aflags & PGA_ENQUEUED) == 0) {
		TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
		delta++;
		vm_page_aflag_set(m, PGA_ENQUEUED);
		if ((aflags & PGA_REQUEUE) != 0)
		vm_page_aflag_clear(m, PGA_REQUEUE);
		} else if ((aflags & PGA_REQUEUE) != 0) {
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
		vm_page_aflag_clear(m, PGA_REQUEUE);
		}
		}
		vm_batchqueue_init(bq);
		vm_pagequeue_cnt_add(pq, delta);
		}

		/*
		* vm_page_dequeue_lazy: [ internal use only ]
*		*
* Remove the given page from its current page queue.		* Request removal of the given page from its current page
		* queue. Physical removal from the queue may be deferred
		* arbitrarily, and may be cancelled by later queue operations
		* on that page.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_dequeue(vm_page_t m)		vm_page_dequeue_lazy(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain, queue;

vm_page_assert_locked(m);		vm_page_assert_locked(m);
KASSERT(m->queue < PQ_COUNT, ("vm_page_dequeue: page %p is not queued",
m));		queue = m->queue;
pq = vm_page_pagequeue(m);		if (queue == PQ_NONE)
		return;
		domain = vm_phys_domain(m);
		pq = &VM_DOMAIN(domain)->vmd_pagequeues[queue];

		vm_page_aflag_set(m, PGA_DEQUEUE);

		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (vm_batchqueue_insert(bq, m)) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		kibUnsubmitted Not Done Inline Actions Did you considered copying the pcpu pqbatch into a local array for processing and enabling preemption immediately after that ? I think that on full pqbatch, we spend a time which is enough to adversely affect the system responsivness. kib: Did you considered copying the pcpu pqbatch into a local array for processing and enabling…
		markjAuthorUnsubmitted Not Done Inline Actions I did, but there is a non-trivial stack space cost (256 bytes on LP64) in addition to that of the copy. The cost of processing 32 pages (8 on !amd64) ought to be quite low. I ran Jeff's dd benchmark (where parallel dd processes read from a sparse file and thus frequently enqueue pages) and used DTrace to count the number of preemptions that occurred in vm_page_enqueue(); excluding preemptions of the idle threads, less than 0.1% of preemptions occurred during the critical_exit() here. This is also an extreme case and in pretty much any workload we will not be performing queue operations at nearly that high a rate. For example, when running a 16-thread pgbench select-only benchmark on an SSD array with about 1GB/s throughput, the number of preemptions that occur here is effectively 0. Preemptions in UMA are much more common. markj: I did, but there is a non-trivial stack space cost (256 bytes on LP64) in addition to that of…
vm_pagequeue_lock(pq);		vm_pagequeue_lock(pq);
m->queue = PQ_NONE;		critical_enter();
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		bq = DPCPU_PTR(pqbatch[domain][queue]);
vm_pagequeue_cnt_dec(pq);		}
		vm_pqbatch_process(pq, bq, queue);

		/*
		* The page may have been dequeued by another thread before we
		* acquired the page queue lock. However, since we hold the
		* page lock, the page's queue field cannot change a second
		* time and we can safely clear PGA_DEQUEUE.
		*/
		KASSERT(m->queue == queue \|\| m->queue == PQ_NONE,
		("%s: page %p migrated between queues", __func__, m));
		if (m->queue == queue) {
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		} else
		vm_page_aflag_clear(m, PGA_DEQUEUE);
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		critical_exit();
}		}

/*		/*
* vm_page_dequeue_locked:		* vm_page_dequeue_locked:
*		*
* Remove the given page from its current page queue.		* Remove the page from its page queue, which must be locked.
		* If the page lock is not held, there is no guarantee that the
		* page will not be enqueued by another thread before this function
		* returns. In this case, it is up to the caller to ensure that
		* no other threads hold a reference to the page.
*		*
* The page and page queue must be locked.		* The page queue lock must be held. If the page is not already
		* logically dequeued, the page lock must be held as well.
*/		*/
void		void
vm_page_dequeue_locked(vm_page_t m)		vm_page_dequeue_locked(vm_page_t m)
{		{
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;

vm_page_lock_assert(m, MA_OWNED);		KASSERT(m->queue != PQ_NONE,
		("%s: page %p queue field is PQ_NONE", __func__, m));
		vm_pagequeue_assert_locked(vm_page_pagequeue(m));
		KASSERT((m->aflags & PGA_DEQUEUE) != 0 \|\|
		mtx_owned(vm_page_lockptr(m)),
		("%s: queued unlocked page %p", __func__, m));

		if ((m->aflags & PGA_ENQUEUED) != 0) {
pq = vm_page_pagequeue(m);		pq = vm_page_pagequeue(m);
vm_pagequeue_assert_locked(pq);
m->queue = PQ_NONE;
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_dec(pq);		vm_pagequeue_cnt_dec(pq);
}		}

/*		/*
* vm_page_enqueue:		* Synchronize with the page daemon, which may be simultaneously
		* scanning this page with only the page lock held. We must be careful
		* to avoid leaving the page in a state where it appears to belong to a
		* page queue.
		*/
		m->queue = PQ_NONE;
		atomic_thread_fence_rel();
		vm_page_aflag_clear(m, PGA_QUEUE_STATE_MASK);
		kibUnsubmitted Not Done Inline Actions I think you need acquire fence before reading aflags there. kib: I think you need acquire fence before reading aflags there.
		markjAuthorUnsubmitted Not Done Inline Actions I am not sure about this. Right now, we are using barriers to synchronize the following operations: T1 dequeues a page with the queue lock held: set m->queue = PQ_NONE, release barrier, clear PGA_DEQUEUE in aflags T2 checks to see if the page is dequeued with page lock held: check for PGA_DEQUEUE in aflags, acquire barrier, check m->queue == PQ_NONE (I don't know what kind of notation is best here.) Here we hold the queue lock, so m->queue is stable, and any thread which updated it must have implicitly issued a release barrier when it unlocked the queue. So I don't see why we need an acquire fence. markj: I am not sure about this. Right now, we are using barriers to synchronize the following…
		kibUnsubmitted Not Done Inline Actions Ok, thank you for the explanation. kib: Ok, thank you for the explanation.
		}

		/*
		* vm_page_dequeue:
*		*
* Add the given page to the specified page queue.		* Remove the page from whichever page queue it's in, if any.
		* If the page lock is not held, there is no guarantee that the
		* page will not be enqueued by another thread before this function
		* returns. In this case, it is up to the caller to ensure that
		* no other threads hold a reference to the page.
		*/
		void
		vm_page_dequeue(vm_page_t m)
		{
		struct mtx lock, lock1;

		lock = vm_page_pagequeue_lockptr(m);
		for (;;) {
		if (lock == NULL)
		return;
		mtx_lock(lock);
		if ((lock1 = vm_page_pagequeue_lockptr(m)) == lock)
		break;
		mtx_unlock(lock);
		lock = lock1;
		}
		KASSERT(lock == vm_page_pagequeue_lockptr(m),
		("%s: page %p migrated directly between queues", __func__, m));
		vm_page_dequeue_locked(m);
		mtx_unlock(lock);
		}

		/*
		* vm_page_enqueue_lazy:
*		*
		* Schedule the given page for insertion into the specified page queue.
		* Physical insertion of the page may be deferred indefinitely.
		*
* The page must be locked.		* The page must be locked.
*/		*/
static void		static void
vm_page_enqueue(uint8_t queue, vm_page_t m)		vm_page_enqueue_lazy(vm_page_t m, uint8_t queue)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain;

vm_page_lock_assert(m, MA_OWNED);		vm_page_assert_locked(m);
KASSERT(queue < PQ_COUNT,		KASSERT(m->queue == PQ_NONE && (m->aflags & PGA_QUEUE_STATE_MASK) == 0,
("vm_page_enqueue: invalid queue %u request for page %p",		("%s: page %p is already enqueued", __func__, m));
queue, m));
		domain = vm_phys_domain(m);
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];		pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];
vm_pagequeue_lock(pq);
		/*
		* The queue field might be changed back to PQ_NONE by a concurrent
		* call to vm_page_dequeue(). In that case the batch queue entry will
		* be a no-op.
		*/
m->queue = queue;		m->queue = queue;
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_inc(pq);		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (__predict_true(vm_batchqueue_insert(bq, m))) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		vm_pagequeue_lock(pq);
		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		}
		glebiusUnsubmitted Not Done Inline Actions What if after critical_exit we have changed the CPU? In this case proceeding further to the end of the function doesn't seem optimal. I'd suggest just to put a page on regular queue, since we already got lock. glebius: What if after critical_exit we have changed the CPU? In this case proceeding further to the end…
		markjAuthorUnsubmitted Not Done Inline Actions Why not? We now hold the page queue lock, so we might as well drain the current queue. markj: Why not? We now hold the page queue lock, so we might as well drain the current queue.
		vm_pqbatch_process(pq, bq, queue);
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		kibUnsubmitted Not Done Inline Actions Don't you need the release fence between setting page domain and setting PGA_REQUEUE ? kib: Don't you need the release fence between setting page domain and setting PGA_REQUEUE ?
		markjAuthorUnsubmitted Not Done Inline Actions See above. We should already be synchronized by the page lock. markj: See above. We should already be synchronized by the page lock.
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		glebiusUnsubmitted Not Done Inline Actions Can you please explain the need for the second call to vm_pqbatch_process? glebius: Can you please explain the need for the second call to vm_pqbatch_process?
		markjAuthorUnsubmitted Done Inline Actions It processes solely the page m. We're at this point because the queue is full, so we have to drain it first by calling `vm_pqbatch_process()`. Probably there should be a `vm_pqbatch_process_page()` which processes a single page, and we just call that on `m`. markj: It processes solely the page m. We're at this point because the queue is full, so we have to…
		critical_exit();
}		}

/*		/*
* vm_page_requeue:		* vm_page_requeue:
*		*
* Move the given page to the tail of its current page queue.		* Schedule a requeue of the given page.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_requeue(vm_page_t m)		vm_page_requeue(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain, queue;

vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
KASSERT(m->queue != PQ_NONE,		KASSERT(m->queue != PQ_NONE,
("vm_page_requeue: page %p is not queued", m));		("%s: page %p is not enqueued", __func__, m));

		domain = vm_phys_domain(m);
		queue = m->queue;
pq = vm_page_pagequeue(m);		pq = vm_page_pagequeue(m);

		if (queue == PQ_NONE)
		return;

		vm_page_aflag_set(m, PGA_REQUEUE);
		jeffUnsubmitted Done Inline Actions Does it make sense to test for this flag before setting it? If it's already set there is not much value in doing it again. jeff: Does it make sense to test for this flag before setting it? If it's already set there is not…
		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (__predict_true(vm_batchqueue_insert(bq, m))) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
vm_pagequeue_lock(pq);		vm_pagequeue_lock(pq);
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		critical_enter();
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);		bq = DPCPU_PTR(pqbatch[domain][queue]);
		}
		vm_pqbatch_process(pq, bq, queue);
		KASSERT(m->queue == queue \|\| m->queue == PQ_NONE,
		("%s: page %p migrated between queues", __func__, m));
		if (m->queue == queue) {
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		} else
		vm_page_aflag_clear(m, PGA_REQUEUE);
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		critical_exit();
}		}

/*		/*
* vm_page_requeue_locked:
*
* Move the given page to the tail of its current page queue.
*
* The page queue must be locked.
*/
void
vm_page_requeue_locked(vm_page_t m)
{
struct vm_pagequeue *pq;

KASSERT(m->queue != PQ_NONE,
("vm_page_requeue_locked: page %p is not queued", m));
pq = vm_page_pagequeue(m);
vm_pagequeue_assert_locked(pq);
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
}

/*
* vm_page_activate:		* vm_page_activate:
*		*
* Put the specified page on the active list (if appropriate).		* Put the specified page on the active list (if appropriate).
* Ensure that act_count is at least ACT_INIT but do not otherwise		* Ensure that act_count is at least ACT_INIT but do not otherwise
* mess with it.		* mess with it.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_activate(vm_page_t m)		vm_page_activate(vm_page_t m)
{		{
int queue;		int queue;

vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
if ((queue = m->queue) != PQ_ACTIVE) {
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if ((queue = m->queue) == PQ_ACTIVE \|\| m->wire_count > 0 \|\|
if (m->act_count < ACT_INIT)		(m->oflags & VPO_UNMANAGED) != 0) {
		if (queue == PQ_ACTIVE && m->act_count < ACT_INIT)
m->act_count = ACT_INIT;		m->act_count = ACT_INIT;
if (queue != PQ_NONE)		return;
vm_page_dequeue(m);
vm_page_enqueue(PQ_ACTIVE, m);
}		}
} else {
		vm_page_remque(m);
if (m->act_count < ACT_INIT)		if (m->act_count < ACT_INIT)
m->act_count = ACT_INIT;		m->act_count = ACT_INIT;
		vm_page_enqueue_lazy(m, PQ_ACTIVE);
}		}
}

/*		/*
* vm_page_free_prep:		* vm_page_free_prep:
*		*
* Prepares the given page to be put on the free list,		* Prepares the given page to be put on the free list,
* disassociating it from any VM object. The caller may return		* disassociating it from any VM object. The caller may return
* the page to the free list only if this function returns true.		* the page to the free list only if this function returns true.
*		*
* The object must be locked. The page must be locked if it is		* The object must be locked. The page must be locked if it is
* managed. For a queued managed page, the pagequeue_locked		* managed.
* argument specifies whether the page queue is already locked.
*/		*/
bool		bool
vm_page_free_prep(vm_page_t m, bool pagequeue_locked)		vm_page_free_prep(vm_page_t m)
{		{

#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)		#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)
if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {		if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {
uint64_t *p;		uint64_t *p;
int i;		int i;
p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));		p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)		for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)
KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",		KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",
m, i, (uintmax_t)*p));		m, i, (uintmax_t)*p));
}		}
#endif		#endif
if ((m->oflags & VPO_UNMANAGED) == 0) {		if ((m->oflags & VPO_UNMANAGED) == 0) {
vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
KASSERT(!pmap_page_is_mapped(m),		KASSERT(!pmap_page_is_mapped(m),
("vm_page_free_toq: freeing mapped page %p", m));		("vm_page_free_prep: freeing mapped page %p", m));
} else		} else
KASSERT(m->queue == PQ_NONE,		KASSERT(m->queue == PQ_NONE,
("vm_page_free_toq: unmanaged page %p is queued", m));		("vm_page_free_prep: unmanaged page %p is queued", m));
VM_CNT_INC(v_tfree);		VM_CNT_INC(v_tfree);

if (vm_page_sbusied(m))		if (vm_page_sbusied(m))
panic("vm_page_free: freeing busy page %p", m);		panic("vm_page_free_prep: freeing busy page %p", m);

vm_page_remove(m);		vm_page_remove(m);

/*		/*
* If fictitious remove object association and		* If fictitious remove object association and
* return.		* return.
*/		*/
if ((m->flags & PG_FICTITIOUS) != 0) {		if ((m->flags & PG_FICTITIOUS) != 0) {
KASSERT(m->wire_count == 1,		KASSERT(m->wire_count == 1,
("fictitious page %p is not wired", m));		("fictitious page %p is not wired", m));
KASSERT(m->queue == PQ_NONE,		KASSERT(m->queue == PQ_NONE,
("fictitious page %p is queued", m));		("fictitious page %p is queued", m));
return (false);		return (false);
}		}

if (m->queue != PQ_NONE) {		/*
if (pagequeue_locked)		* Pages need not be dequeued before they are returned to the physical
vm_page_dequeue_locked(m);		* memory allocator, but they must at least be marked for a deferred
else		* dequeue.
vm_page_dequeue(m);		*/
}		if ((m->oflags & VPO_UNMANAGED) == 0)
		vm_page_dequeue_lazy(m);

m->valid = 0;		m->valid = 0;
vm_page_undirty(m);		vm_page_undirty(m);

if (m->wire_count != 0)		if (m->wire_count != 0)
panic("vm_page_free: freeing wired page %p", m);		panic("vm_page_free_prep: freeing wired page %p", m);
if (m->hold_count != 0) {		if (m->hold_count != 0) {
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
KASSERT((m->flags & PG_UNHOLDFREE) == 0,		KASSERT((m->flags & PG_UNHOLDFREE) == 0,
("vm_page_free: freeing PG_UNHOLDFREE page %p", m));		("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m));
m->flags \|= PG_UNHOLDFREE;		m->flags \|= PG_UNHOLDFREE;
return (false);		return (false);
}		}

/*		/*
* Restore the default memory attribute to the page.		* Restore the default memory attribute to the page.
*/		*/
if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)		if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);		pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);

#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
if (vm_reserv_free_page(m))		if (vm_reserv_free_page(m))
return (false);		return (false);
#endif		#endif

return (true);		return (true);
}		}

void
vm_page_free_phys_pglist(struct pglist *tq)
{
struct vm_domain *vmd;
vm_page_t m;
int cnt;

if (TAILQ_EMPTY(tq))
return;
vmd = NULL;
cnt = 0;
TAILQ_FOREACH(m, tq, listq) {
if (vmd != vm_pagequeue_domain(m)) {
if (vmd != NULL) {
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, cnt);
cnt = 0;
}
vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);
}
vm_phys_free_pages(m, 0);
cnt++;
}
if (vmd != NULL) {
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, cnt);
}
}

/*		/*
* vm_page_free_toq:		* vm_page_free_toq:
*		*
* Returns the given page to the free list, disassociating it		* Returns the given page to the free list, disassociating it
* from any VM object.		* from any VM object.
*		*
* The object must be locked. The page must be locked if it is		* The object must be locked. The page must be locked if it is
* managed.		* managed.
*/		*/
void		void
vm_page_free_toq(vm_page_t m)		vm_page_free_toq(vm_page_t m)
{		{
struct vm_domain *vmd;		struct vm_domain *vmd;

if (!vm_page_free_prep(m, false))		if (!vm_page_free_prep(m))
return;		return;

vmd = vm_pagequeue_domain(m);		vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);		vm_domain_free_lock(vmd);
vm_phys_free_pages(m, 0);		vm_phys_free_pages(m, 0);
vm_domain_free_unlock(vmd);		vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
}		}

/*		/*
* vm_page_free_pages_toq:		* vm_page_free_pages_toq:
*		*
* Returns a list of pages to the free list, disassociating it		* Returns a list of pages to the free list, disassociating it
* from any VM object. In other words, this is equivalent to		* from any VM object. In other words, this is equivalent to
* calling vm_page_free_toq() for each page of a list of VM objects.		* calling vm_page_free_toq() for each page of a list of VM objects.
*		*
* The objects must be locked. The pages must be locked if it is		* The objects must be locked. The pages must be locked if it is
* managed.		* managed.
*/		*/
void		void
vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)		vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)
{		{
vm_page_t m;		vm_page_t m;
struct pglist pgl;
int count;		int count;

if (SLIST_EMPTY(free))		if (SLIST_EMPTY(free))
return;		return;

count = 0;		count = 0;
TAILQ_INIT(&pgl);
while ((m = SLIST_FIRST(free)) != NULL) {		while ((m = SLIST_FIRST(free)) != NULL) {
count++;		count++;
SLIST_REMOVE_HEAD(free, plinks.s.ss);		SLIST_REMOVE_HEAD(free, plinks.s.ss);
if (vm_page_free_prep(m, false))		vm_page_free_toq(m);
TAILQ_INSERT_TAIL(&pgl, m, listq);
}		}

vm_page_free_phys_pglist(&pgl);

if (update_wire_count)		if (update_wire_count)
vm_wire_sub(count);		vm_wire_sub(count);
}		}

/*		/*
* vm_page_wire:		* vm_page_wire:
*		*
* Mark this page as wired down. If the page is fictitious, then		* Mark this page as wired down. If the page is fictitious, then
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
bool		bool
vm_page_unwire(vm_page_t m, uint8_t queue)		vm_page_unwire(vm_page_t m, uint8_t queue)
{		{
bool unwired;		bool unwired;

KASSERT(queue < PQ_COUNT \|\| queue == PQ_NONE,		KASSERT(queue < PQ_COUNT \|\| queue == PQ_NONE,
("vm_page_unwire: invalid queue %u request for page %p",		("vm_page_unwire: invalid queue %u request for page %p",
queue, m));		queue, m));
		if ((m->oflags & VPO_UNMANAGED) == 0)
		vm_page_assert_locked(m);

unwired = vm_page_unwire_noq(m);		unwired = vm_page_unwire_noq(m);
if (unwired && (m->oflags & VPO_UNMANAGED) == 0 && m->object != NULL) {		if (!unwired \|\| (m->oflags & VPO_UNMANAGED) != 0 \|\| m->object == NULL)
		return (unwired);

if (m->queue == queue) {		if (m->queue == queue) {
if (queue == PQ_ACTIVE)		if (queue == PQ_ACTIVE)
vm_page_reference(m);		vm_page_reference(m);
else if (queue != PQ_NONE)		else if (queue != PQ_NONE)
vm_page_requeue(m);		vm_page_requeue(m);
} else {		} else {
vm_page_remque(m);		vm_page_dequeue(m);
if (queue != PQ_NONE) {		if (queue != PQ_NONE) {
vm_page_enqueue(queue, m);		vm_page_enqueue_lazy(m, queue);
if (queue == PQ_ACTIVE)		if (queue == PQ_ACTIVE)
/* Initialize act_count. */		/* Initialize act_count. */
vm_page_activate(m);		vm_page_activate(m);
}		}
}		}
}
return (unwired);		return (unwired);
}		}

/*		/*
*		*
* vm_page_unwire_noq:		* vm_page_unwire_noq:
*		*
* Unwire a page without (re-)inserting it into a page queue. It is up		* Unwire a page without (re-)inserting it into a page queue. It is up
Show All 17 Lines	vm_page_unwire_noq(vm_page_t m)
if (m->wire_count == 0) {		if (m->wire_count == 0) {
vm_wire_sub(1);		vm_wire_sub(1);
return (true);		return (true);
} else		} else
return (false);		return (false);
}		}

/*		/*
* Move the specified page to the inactive queue, or requeue the page if it is		* Move the specified page to the tail of the inactive queue, or requeue
* already in the inactive queue.		* the page if it is already in the inactive queue.
*		*
* Normally, "noreuse" is FALSE, resulting in LRU ordering of the inactive
* queue. However, setting "noreuse" to TRUE will accelerate the specified
* page's reclamation, but it will not unmap the page from any address space.
* This is implemented by inserting the page near the head of the inactive
* queue, using a marker page to guide FIFO insertion ordering.
*
* The page must be locked.		* The page must be locked.
*/		*/
static inline void		void
_vm_page_deactivate(vm_page_t m, boolean_t noreuse)		vm_page_deactivate(vm_page_t m)
{		{
struct vm_pagequeue *pq;
int queue;

vm_page_assert_locked(m);		vm_page_assert_locked(m);

if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[PQ_INACTIVE];		return;
/* Avoid multiple acquisitions of the inactive queue lock. */
queue = m->queue;
if (queue == PQ_INACTIVE) {
vm_pagequeue_lock(pq);
vm_page_dequeue_locked(m);
} else {
if (queue != PQ_NONE)
vm_page_dequeue(m);
vm_pagequeue_lock(pq);
}
m->queue = PQ_INACTIVE;
if (noreuse)
TAILQ_INSERT_BEFORE(
&vm_pagequeue_domain(m)->vmd_inacthead, m,
plinks.q);
else
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_inc(pq);
vm_pagequeue_unlock(pq);
}
}

/*		if (!vm_page_inactive(m)) {
		jeffUnsubmitted Done Inline Actions Should be an XXX comment. This path likely needs to be fast. jeff: Should be an XXX comment. This path likely needs to be fast.
* Move the specified page to the inactive queue, or requeue the page if it is		vm_page_remque(m);
* already in the inactive queue.		vm_page_enqueue_lazy(m, PQ_INACTIVE);
*		} else
* The page must be locked.		vm_page_requeue(m);
*/
void
vm_page_deactivate(vm_page_t m)
{

_vm_page_deactivate(m, FALSE);
}		}

/*		/*
* Move the specified page to the inactive queue with the expectation		* Move the specified page close to the head of the inactive queue,
* that it is unlikely to be reused.		* bypassing LRU. A marker page is used to maintain FIFO ordering.
		* As with regular enqueues, we use a per-CPU batch queue to reduce
		* contention on the page queue lock.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_deactivate_noreuse(vm_page_t m)		vm_page_deactivate_noreuse(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
		struct vm_domain *vmd;
		struct vm_pagequeue *pq;
		vm_page_t marker;
		int domain;

_vm_page_deactivate(m, TRUE);		vm_page_assert_locked(m);

		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
		return;

		domain = vm_phys_domain(m);
		vmd = VM_DOMAIN(domain);
		pq = &vmd->vmd_pagequeues[PQ_INACTIVE];

		if (!vm_page_inactive(m))
		vm_page_remque(m);

		m->queue = PQ_INACTIVE;

		critical_enter();
		bq = DPCPU_PTR(noreuseq[domain]);
		if (vm_batchqueue_insert(bq, m)) {
		critical_exit();
		return;
}		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		vm_pagequeue_lock(pq);
		critical_enter();
		bq = DPCPU_PTR(noreuseq[domain]);
		}
		marker = &vmd->vmd_inacthead;
		do {
		if (m->queue != PQ_INACTIVE)
		continue;
		if ((m->aflags & PGA_ENQUEUED) != 0)
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		else {
		vm_page_aflag_set(m, PGA_ENQUEUED);
		vm_pagequeue_cnt_inc(pq);
		}
		TAILQ_INSERT_BEFORE(marker, m, plinks.q);
		} while ((m = vm_batchqueue_pop(bq)) != NULL);
		vm_pagequeue_unlock(pq);
		vm_batchqueue_init(bq);
		critical_exit();
		}

/*		/*
* vm_page_launder		* vm_page_launder
*		*
* Put a page in the laundry, or requeue it if it is already there.		* Put a page in the laundry, or requeue it if it is already there.
*/		*/
void		void
vm_page_launder(vm_page_t m)		vm_page_launder(vm_page_t m)
{		{

vm_page_assert_locked(m);		vm_page_assert_locked(m);
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
		return;

if (m->queue == PQ_LAUNDRY)		if (m->queue == PQ_LAUNDRY)
vm_page_requeue(m);		vm_page_requeue(m);
else {		else {
vm_page_remque(m);		vm_page_remque(m);
vm_page_enqueue(PQ_LAUNDRY, m);		vm_page_enqueue_lazy(m, PQ_LAUNDRY);
}		}
}		}
}

/*		/*
* vm_page_unswappable		* vm_page_unswappable
*		*
* Put a page in the PQ_UNSWAPPABLE holding queue.		* Put a page in the PQ_UNSWAPPABLE holding queue.
*/		*/
void		void
vm_page_unswappable(vm_page_t m)		vm_page_unswappable(vm_page_t m)
{		{

vm_page_assert_locked(m);		vm_page_assert_locked(m);
KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,		KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,
("page %p already unswappable", m));		("page %p already unswappable", m));
if (m->queue != PQ_NONE)
vm_page_dequeue(m);		vm_page_remque(m);
vm_page_enqueue(PQ_UNSWAPPABLE, m);		vm_page_enqueue_lazy(m, PQ_UNSWAPPABLE);
}		}

/*		/*
* Attempt to free the page. If it cannot be freed, do nothing. Returns true		* Attempt to free the page. If it cannot be freed, do nothing. Returns true
* if the page is freed and false otherwise.		* if the page is freed and false otherwise.
*		*
* The page must be managed. The page and its containing object must be		* The page must be managed. The page and its containing object must be
* locked.		* locked.
▲ Show 20 Lines • Show All 750 Lines • Show Last 20 Lines

View Options

sys/dev/mlx5/mlx5_ib/mlx5_doorbell.c

Show First 20 Lines • Show All 125 Lines • ▼ Show 20 Lines
#include <vm/uma_int.h>		#include <vm/uma_int.h>

#include <machine/md_var.h>		#include <machine/md_var.h>

extern int uma_startup_count(int);		extern int uma_startup_count(int);
extern void uma_startup(void *, int);		extern void uma_startup(void *, int);
extern int vmem_startup_count(void);		extern int vmem_startup_count(void);

/*
* Associated with page of user-allocatable memory is a
* page structure.
*/

struct vm_domain vm_dom[MAXMEMDOM];		struct vm_domain vm_dom[MAXMEMDOM];

		static DPCPU_DEFINE(struct vm_batchqueue, pqbatch[MAXMEMDOM][PQ_COUNT]);
		static DPCPU_DEFINE(struct vm_batchqueue, noreuseq[MAXMEMDOM]);
		jeffUnsubmitted Not Done Inline Actions What is the reason for not using just another bit in the aflags to indicate head vs tail insertion? jeff: What is the reason for not using just another bit in the aflags to indicate head vs tail…
		markjAuthorUnsubmitted Not Done Inline Actions It was mostly because we only have two atomic bits left. Another solution would be to have a separate inactive pagequeue for pages that are bypassing global LRU. Because pq->pq_mtx is a pointer, both queues could be protected by the same lock, making transfers between the two queues relatively efficient. What do you think? markj: It was mostly because we only have two atomic bits left. Another solution would be to have a…
		kibUnsubmitted Not Done Inline Actions Why not put this into static pcpu ? kib: Why not put this into static pcpu ?
		markjAuthorUnsubmitted Not Done Inline Actions Is there any downside to using the dynamic per-CPU section? I like the fact that the definition is visible only in vm_page.c. markj: Is there any downside to using the dynamic per-CPU section? I like the fact that the definition…
		kibUnsubmitted Not Done Inline Actions It is one more arithmetic op on pointers to get to dpcu from pcpu. But my point is that the batch queues are not dynamic, they exists for the whole duration of the system operation. I do not insist either way. kib: It is one more arithmetic op on pointers to get to dpcu from pcpu. But my point is that the…

struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];		struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];

struct mtx_padalign __exclusive_cache_line vm_domainset_lock;		struct mtx_padalign __exclusive_cache_line vm_domainset_lock;
/* The following fields are protected by the domainset lock. */		/* The following fields are protected by the domainset lock. */
domainset_t __exclusive_cache_line vm_min_domains;		domainset_t __exclusive_cache_line vm_min_domains;
domainset_t __exclusive_cache_line vm_severe_domains;		domainset_t __exclusive_cache_line vm_severe_domains;
static int vm_min_waiters;		static int vm_min_waiters;
static int vm_severe_waiters;		static int vm_severe_waiters;
Show All 22 Lines
static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);		static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|		SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|
CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");		CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");

static uma_zone_t fakepg_zone;		static uma_zone_t fakepg_zone;

static void vm_page_alloc_check(vm_page_t m);		static void vm_page_alloc_check(vm_page_t m);
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);		static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
static void vm_page_enqueue(uint8_t queue, vm_page_t m);		static void vm_page_enqueue_lazy(vm_page_t m, uint8_t queue);
static void vm_page_init(void *dummy);		static void vm_page_init(void *dummy);
static int vm_page_insert_after(vm_page_t m, vm_object_t object,		static int vm_page_insert_after(vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred);		vm_pindex_t pindex, vm_page_t mpred);
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,		static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
vm_page_t mpred);		vm_page_t mpred);
static int vm_page_reclaim_run(int req_class, int domain, u_long npages,		static int vm_page_reclaim_run(int req_class, int domain, u_long npages,
vm_page_t m_run, vm_paddr_t high);		vm_page_t m_run, vm_paddr_t high);
static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,		static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,
▲ Show 20 Lines • Show All 1,623 Lines • ▼ Show 20 Lines	#endif
/*		/*
* At this point we had better have found a good page.		* At this point we had better have found a good page.
*/		*/
KASSERT(m != NULL, ("missing page"));		KASSERT(m != NULL, ("missing page"));

#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
found:		found:
#endif		#endif
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
flags = PG_ZERO;		flags = PG_ZERO;
▲ Show 20 Lines • Show All 180 Lines • ▼ Show 20 Lines	#endif
if (m_ret == NULL) {		if (m_ret == NULL) {
if (vm_domain_alloc_fail(vmd, object, req))		if (vm_domain_alloc_fail(vmd, object, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}
#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
found:		found:
#endif		#endif
for (m = m_ret; m < &m_ret[npages]; m++)		for (m = m_ret; m < &m_ret[npages]; m++) {
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);
		}

/*		/*
* Initialize the pages. Only the PG_ZERO flag is inherited.		* Initialize the pages. Only the PG_ZERO flag is inherited.
*/		*/
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
flags = PG_ZERO;		flags = PG_ZERO;
if ((req & VM_ALLOC_NODUMP) != 0)		if ((req & VM_ALLOC_NODUMP) != 0)
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines	if (vm_domain_allocate(vmd, req, 1)) {
if (m == NULL)		if (m == NULL)
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
}		}
if (m == NULL) {		if (m == NULL) {
if (vm_domain_alloc_fail(vmd, NULL, req))		if (vm_domain_alloc_fail(vmd, NULL, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}
		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
m->aflags = 0;		m->aflags = 0;
flags = 0;		flags = 0;
if ((req & VM_ALLOC_ZERO) != 0)		if ((req & VM_ALLOC_ZERO) != 0)
▲ Show 20 Lines • Show All 176 Lines • ▼ Show 20 Lines	#if VM_NRESERVLEVEL > 0
(level = vm_reserv_level_iffullpop(m)) >= 0) {		(level = vm_reserv_level_iffullpop(m)) >= 0) {
run_ext = 0;		run_ext = 0;
/* Advance to the end of the superpage. */		/* Advance to the end of the superpage. */
pa = VM_PAGE_TO_PHYS(m);		pa = VM_PAGE_TO_PHYS(m);
m_inc = atop(roundup2(pa + 1,		m_inc = atop(roundup2(pa + 1,
vm_reserv_size(level)) - pa);		vm_reserv_size(level)) - pa);
#endif		#endif
} else if (object->memattr == VM_MEMATTR_DEFAULT &&		} else if (object->memattr == VM_MEMATTR_DEFAULT &&
m->queue != PQ_NONE && !vm_page_busied(m)) {		m->queue != PQ_NONE &&
		(m->aflags & PGA_DEQUEUE) == 0 && !vm_page_busied(m)) {
		kibUnsubmitted Done Inline Actions Did you considered creating macros to detect logical enqueue conditions ? Like logically non-queued == (queue != PG_NONE && PGA_DEQUEUE == 0) ? kib: Did you considered creating macros to detect logical enqueue conditions ? Like logically non…
		markjAuthorUnsubmitted Done Inline Actions Yes, I should do that. I think this is missing an acquire barrier too. markj: Yes, I should do that. I think this is missing an acquire barrier too.
/*		/*
* The page is allocated but eligible for		* The page is allocated but eligible for
* relocation. Extend the current run by one		* relocation. Extend the current run by one
* page.		* page.
*/		*/
KASSERT(pmap_page_get_memattr(m) ==		KASSERT(pmap_page_get_memattr(m) ==
VM_MEMATTR_DEFAULT,		VM_MEMATTR_DEFAULT,
("page %p has an unexpected memattr", m));		("page %p has an unexpected memattr", m));
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines	else if ((object = m->object) != NULL) {
("page %p is PG_UNHOLDFREE", m));		("page %p is PG_UNHOLDFREE", m));
/* Don't care: PG_NODUMP, PG_ZERO. */		/* Don't care: PG_NODUMP, PG_ZERO. */
if (object->type != OBJT_DEFAULT &&		if (object->type != OBJT_DEFAULT &&
object->type != OBJT_SWAP &&		object->type != OBJT_SWAP &&
object->type != OBJT_VNODE)		object->type != OBJT_VNODE)
error = EINVAL;		error = EINVAL;
else if (object->memattr != VM_MEMATTR_DEFAULT)		else if (object->memattr != VM_MEMATTR_DEFAULT)
error = EINVAL;		error = EINVAL;
else if (m->queue != PQ_NONE && !vm_page_busied(m)) {		else if (m->queue != PQ_NONE &&
		(m->aflags & PGA_DEQUEUE) == 0 &&
		!vm_page_busied(m)) {
KASSERT(pmap_page_get_memattr(m) ==		KASSERT(pmap_page_get_memattr(m) ==
VM_MEMATTR_DEFAULT,		VM_MEMATTR_DEFAULT,
("page %p has an unexpected memattr", m));		("page %p has an unexpected memattr", m));
KASSERT((m->oflags & (VPO_SWAPINPROG \|		KASSERT((m->oflags & (VPO_SWAPINPROG \|
VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,		VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,
("page %p has unexpected oflags", m));		("page %p has unexpected oflags", m));
/* Don't care: VPO_NOSYNC. */		/* Don't care: VPO_NOSYNC. */
if (m->valid != 0) {		if (m->valid != 0) {
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines	else if ((object = m->object) != NULL) {
/*		/*
* Replace "m" with the new page. For		* Replace "m" with the new page. For
* vm_page_replace(), "m" must be busy		* vm_page_replace(), "m" must be busy
* and dequeued. Finally, change "m"		* and dequeued. Finally, change "m"
* as if vm_page_free() was called.		* as if vm_page_free() was called.
*/		*/
if (object->ref_count != 0)		if (object->ref_count != 0)
pmap_remove_all(m);		pmap_remove_all(m);
m_new->aflags = m->aflags;		m_new->aflags = m->aflags &
		~PGA_QUEUE_STATE_MASK;
KASSERT(m_new->oflags == VPO_UNMANAGED,		KASSERT(m_new->oflags == VPO_UNMANAGED,
("page %p is managed", m_new));		("page %p is managed", m_new));
m_new->oflags = m->oflags & VPO_NOSYNC;		m_new->oflags = m->oflags & VPO_NOSYNC;
pmap_copy_page(m, m_new);		pmap_copy_page(m, m_new);
m_new->valid = m->valid;		m_new->valid = m->valid;
m_new->dirty = m->dirty;		m_new->dirty = m->dirty;
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
vm_page_xbusy(m);		vm_page_xbusy(m);
vm_page_remque(m);		vm_page_remque(m);
vm_page_replace_checked(m_new, object,		vm_page_replace_checked(m_new, object,
m->pindex, m);		m->pindex, m);
if (vm_page_free_prep(m, false))		if (vm_page_free_prep(m))
SLIST_INSERT_HEAD(&free, m,		SLIST_INSERT_HEAD(&free, m,
plinks.s.ss);		plinks.s.ss);

/*		/*
* The new page must be deactivated		* The new page must be deactivated
* before the object is unlocked.		* before the object is unlocked.
*/		*/
vm_page_change_lock(m_new, &m_mtx);		vm_page_change_lock(m_new, &m_mtx);
vm_page_deactivate(m_new);		vm_page_deactivate(m_new);
} else {		} else {
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
vm_page_remque(m);		vm_page_remque(m);
vm_page_remove(m);		vm_page_remove(m);
if (vm_page_free_prep(m, false))		if (vm_page_free_prep(m))
SLIST_INSERT_HEAD(&free, m,		SLIST_INSERT_HEAD(&free, m,
plinks.s.ss);		plinks.s.ss);
KASSERT(m->dirty == 0,		KASSERT(m->dirty == 0,
("page %p is dirty", m));		("page %p is dirty", m));
}		}
} else		} else
error = EBUSY;		error = EBUSY;
unlock:		unlock:
▲ Show 20 Lines • Show All 426 Lines • ▼ Show 20 Lines

struct vm_pagequeue *		struct vm_pagequeue *
vm_page_pagequeue(vm_page_t m)		vm_page_pagequeue(vm_page_t m)
{		{

return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]);		return (&vm_pagequeue_domain(m)->vmd_pagequeues[m->queue]);
}		}

		static struct mtx *
		vm_page_pagequeue_lockptr(vm_page_t m)
		{

		if (m->queue == PQ_NONE)
		return (NULL);
		return (&vm_page_pagequeue(m)->pq_mutex);
		}

		static void
		vm_pqbatch_process(struct vm_pagequeue pq, struct vm_batchqueue bq,
		uint8_t queue)
		{
		vm_page_t m;
		int delta, i;
		uint8_t aflags;

		vm_pagequeue_assert_locked(pq);

		delta = 0;
		for (i = 0; i < bq->bq_cnt; i++) {
		m = bq->bq_pa[i];
		if (__predict_false(m->queue != queue))
		continue;

		aflags = m->aflags;
		if ((aflags & PGA_DEQUEUE) != 0) {
		if (__predict_true((aflags & PGA_ENQUEUED) != 0)) {
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		delta--;
		}

/*		/*
* vm_page_dequeue:		* Synchronize with the page daemon, which may be
		* simultaneously scanning this page with only the page
		* lock held. We must be careful to avoid leaving the
		* page in a state where it appears to belong to a page
		* queue.
		*/
		m->queue = PQ_NONE;
		atomic_thread_fence_rel();
		vm_page_aflag_clear(m, PGA_QUEUE_STATE_MASK);
		} else if ((aflags & PGA_ENQUEUED) == 0) {
		TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
		delta++;
		vm_page_aflag_set(m, PGA_ENQUEUED);
		if ((aflags & PGA_REQUEUE) != 0)
		vm_page_aflag_clear(m, PGA_REQUEUE);
		} else if ((aflags & PGA_REQUEUE) != 0) {
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
		vm_page_aflag_clear(m, PGA_REQUEUE);
		}
		}
		vm_batchqueue_init(bq);
		vm_pagequeue_cnt_add(pq, delta);
		}

		/*
		* vm_page_dequeue_lazy: [ internal use only ]
*		*
* Remove the given page from its current page queue.		* Request removal of the given page from its current page
		* queue. Physical removal from the queue may be deferred
		* arbitrarily, and may be cancelled by later queue operations
		* on that page.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_dequeue(vm_page_t m)		vm_page_dequeue_lazy(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain, queue;

vm_page_assert_locked(m);		vm_page_assert_locked(m);
KASSERT(m->queue < PQ_COUNT, ("vm_page_dequeue: page %p is not queued",
m));		queue = m->queue;
pq = vm_page_pagequeue(m);		if (queue == PQ_NONE)
		return;
		domain = vm_phys_domain(m);
		pq = &VM_DOMAIN(domain)->vmd_pagequeues[queue];

		vm_page_aflag_set(m, PGA_DEQUEUE);

		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (vm_batchqueue_insert(bq, m)) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		kibUnsubmitted Not Done Inline Actions Did you considered copying the pcpu pqbatch into a local array for processing and enabling preemption immediately after that ? I think that on full pqbatch, we spend a time which is enough to adversely affect the system responsivness. kib: Did you considered copying the pcpu pqbatch into a local array for processing and enabling…
		markjAuthorUnsubmitted Not Done Inline Actions I did, but there is a non-trivial stack space cost (256 bytes on LP64) in addition to that of the copy. The cost of processing 32 pages (8 on !amd64) ought to be quite low. I ran Jeff's dd benchmark (where parallel dd processes read from a sparse file and thus frequently enqueue pages) and used DTrace to count the number of preemptions that occurred in vm_page_enqueue(); excluding preemptions of the idle threads, less than 0.1% of preemptions occurred during the critical_exit() here. This is also an extreme case and in pretty much any workload we will not be performing queue operations at nearly that high a rate. For example, when running a 16-thread pgbench select-only benchmark on an SSD array with about 1GB/s throughput, the number of preemptions that occur here is effectively 0. Preemptions in UMA are much more common. markj: I did, but there is a non-trivial stack space cost (256 bytes on LP64) in addition to that of…
vm_pagequeue_lock(pq);		vm_pagequeue_lock(pq);
m->queue = PQ_NONE;		critical_enter();
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		bq = DPCPU_PTR(pqbatch[domain][queue]);
vm_pagequeue_cnt_dec(pq);		}
		vm_pqbatch_process(pq, bq, queue);

		/*
		* The page may have been dequeued by another thread before we
		* acquired the page queue lock. However, since we hold the
		* page lock, the page's queue field cannot change a second
		* time and we can safely clear PGA_DEQUEUE.
		*/
		KASSERT(m->queue == queue \|\| m->queue == PQ_NONE,
		("%s: page %p migrated between queues", __func__, m));
		if (m->queue == queue) {
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		} else
		vm_page_aflag_clear(m, PGA_DEQUEUE);
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		critical_exit();
}		}

/*		/*
* vm_page_dequeue_locked:		* vm_page_dequeue_locked:
*		*
* Remove the given page from its current page queue.		* Remove the page from its page queue, which must be locked.
		* If the page lock is not held, there is no guarantee that the
		* page will not be enqueued by another thread before this function
		* returns. In this case, it is up to the caller to ensure that
		* no other threads hold a reference to the page.
*		*
* The page and page queue must be locked.		* The page queue lock must be held. If the page is not already
		* logically dequeued, the page lock must be held as well.
*/		*/
void		void
vm_page_dequeue_locked(vm_page_t m)		vm_page_dequeue_locked(vm_page_t m)
{		{
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;

vm_page_lock_assert(m, MA_OWNED);		KASSERT(m->queue != PQ_NONE,
		("%s: page %p queue field is PQ_NONE", __func__, m));
		vm_pagequeue_assert_locked(vm_page_pagequeue(m));
		KASSERT((m->aflags & PGA_DEQUEUE) != 0 \|\|
		mtx_owned(vm_page_lockptr(m)),
		("%s: queued unlocked page %p", __func__, m));

		if ((m->aflags & PGA_ENQUEUED) != 0) {
pq = vm_page_pagequeue(m);		pq = vm_page_pagequeue(m);
vm_pagequeue_assert_locked(pq);
m->queue = PQ_NONE;
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_dec(pq);		vm_pagequeue_cnt_dec(pq);
}		}

/*		/*
* vm_page_enqueue:		* Synchronize with the page daemon, which may be simultaneously
		* scanning this page with only the page lock held. We must be careful
		* to avoid leaving the page in a state where it appears to belong to a
		* page queue.
		*/
		m->queue = PQ_NONE;
		atomic_thread_fence_rel();
		vm_page_aflag_clear(m, PGA_QUEUE_STATE_MASK);
		kibUnsubmitted Not Done Inline Actions I think you need acquire fence before reading aflags there. kib: I think you need acquire fence before reading aflags there.
		markjAuthorUnsubmitted Not Done Inline Actions I am not sure about this. Right now, we are using barriers to synchronize the following operations: T1 dequeues a page with the queue lock held: set m->queue = PQ_NONE, release barrier, clear PGA_DEQUEUE in aflags T2 checks to see if the page is dequeued with page lock held: check for PGA_DEQUEUE in aflags, acquire barrier, check m->queue == PQ_NONE (I don't know what kind of notation is best here.) Here we hold the queue lock, so m->queue is stable, and any thread which updated it must have implicitly issued a release barrier when it unlocked the queue. So I don't see why we need an acquire fence. markj: I am not sure about this. Right now, we are using barriers to synchronize the following…
		kibUnsubmitted Not Done Inline Actions Ok, thank you for the explanation. kib: Ok, thank you for the explanation.
		}

		/*
		* vm_page_dequeue:
*		*
* Add the given page to the specified page queue.		* Remove the page from whichever page queue it's in, if any.
		* If the page lock is not held, there is no guarantee that the
		* page will not be enqueued by another thread before this function
		* returns. In this case, it is up to the caller to ensure that
		* no other threads hold a reference to the page.
		*/
		void
		vm_page_dequeue(vm_page_t m)
		{
		struct mtx lock, lock1;

		lock = vm_page_pagequeue_lockptr(m);
		for (;;) {
		if (lock == NULL)
		return;
		mtx_lock(lock);
		if ((lock1 = vm_page_pagequeue_lockptr(m)) == lock)
		break;
		mtx_unlock(lock);
		lock = lock1;
		}
		KASSERT(lock == vm_page_pagequeue_lockptr(m),
		("%s: page %p migrated directly between queues", __func__, m));
		vm_page_dequeue_locked(m);
		mtx_unlock(lock);
		}

		/*
		* vm_page_enqueue_lazy:
*		*
		* Schedule the given page for insertion into the specified page queue.
		* Physical insertion of the page may be deferred indefinitely.
		*
* The page must be locked.		* The page must be locked.
*/		*/
static void		static void
vm_page_enqueue(uint8_t queue, vm_page_t m)		vm_page_enqueue_lazy(vm_page_t m, uint8_t queue)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain;

vm_page_lock_assert(m, MA_OWNED);		vm_page_assert_locked(m);
KASSERT(queue < PQ_COUNT,		KASSERT(m->queue == PQ_NONE && (m->aflags & PGA_QUEUE_STATE_MASK) == 0,
("vm_page_enqueue: invalid queue %u request for page %p",		("%s: page %p is already enqueued", __func__, m));
queue, m));
		domain = vm_phys_domain(m);
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];		pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];
vm_pagequeue_lock(pq);
		/*
		* The queue field might be changed back to PQ_NONE by a concurrent
		* call to vm_page_dequeue(). In that case the batch queue entry will
		* be a no-op.
		*/
m->queue = queue;		m->queue = queue;
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_inc(pq);		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (__predict_true(vm_batchqueue_insert(bq, m))) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		vm_pagequeue_lock(pq);
		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		}
		glebiusUnsubmitted Not Done Inline Actions What if after critical_exit we have changed the CPU? In this case proceeding further to the end of the function doesn't seem optimal. I'd suggest just to put a page on regular queue, since we already got lock. glebius: What if after critical_exit we have changed the CPU? In this case proceeding further to the end…
		markjAuthorUnsubmitted Not Done Inline Actions Why not? We now hold the page queue lock, so we might as well drain the current queue. markj: Why not? We now hold the page queue lock, so we might as well drain the current queue.
		vm_pqbatch_process(pq, bq, queue);
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		kibUnsubmitted Not Done Inline Actions Don't you need the release fence between setting page domain and setting PGA_REQUEUE ? kib: Don't you need the release fence between setting page domain and setting PGA_REQUEUE ?
		markjAuthorUnsubmitted Not Done Inline Actions See above. We should already be synchronized by the page lock. markj: See above. We should already be synchronized by the page lock.
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		glebiusUnsubmitted Not Done Inline Actions Can you please explain the need for the second call to vm_pqbatch_process? glebius: Can you please explain the need for the second call to vm_pqbatch_process?
		markjAuthorUnsubmitted Done Inline Actions It processes solely the page m. We're at this point because the queue is full, so we have to drain it first by calling `vm_pqbatch_process()`. Probably there should be a `vm_pqbatch_process_page()` which processes a single page, and we just call that on `m`. markj: It processes solely the page m. We're at this point because the queue is full, so we have to…
		critical_exit();
}		}

/*		/*
* vm_page_requeue:		* vm_page_requeue:
*		*
* Move the given page to the tail of its current page queue.		* Schedule a requeue of the given page.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_requeue(vm_page_t m)		vm_page_requeue(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
struct vm_pagequeue *pq;		struct vm_pagequeue *pq;
		int domain, queue;

vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
KASSERT(m->queue != PQ_NONE,		KASSERT(m->queue != PQ_NONE,
("vm_page_requeue: page %p is not queued", m));		("%s: page %p is not enqueued", __func__, m));

		domain = vm_phys_domain(m);
		queue = m->queue;
pq = vm_page_pagequeue(m);		pq = vm_page_pagequeue(m);

		if (queue == PQ_NONE)
		return;

		vm_page_aflag_set(m, PGA_REQUEUE);
		jeffUnsubmitted Done Inline Actions Does it make sense to test for this flag before setting it? If it's already set there is not much value in doing it again. jeff: Does it make sense to test for this flag before setting it? If it's already set there is not…
		critical_enter();
		bq = DPCPU_PTR(pqbatch[domain][queue]);
		if (__predict_true(vm_batchqueue_insert(bq, m))) {
		critical_exit();
		return;
		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
vm_pagequeue_lock(pq);		vm_pagequeue_lock(pq);
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);		critical_enter();
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);		bq = DPCPU_PTR(pqbatch[domain][queue]);
		}
		vm_pqbatch_process(pq, bq, queue);
		KASSERT(m->queue == queue \|\| m->queue == PQ_NONE,
		("%s: page %p migrated between queues", __func__, m));
		if (m->queue == queue) {
		(void)vm_batchqueue_insert(bq, m);
		vm_pqbatch_process(pq, bq, queue);
		} else
		vm_page_aflag_clear(m, PGA_REQUEUE);
vm_pagequeue_unlock(pq);		vm_pagequeue_unlock(pq);
		critical_exit();
}		}

/*		/*
* vm_page_requeue_locked:
*
* Move the given page to the tail of its current page queue.
*
* The page queue must be locked.
*/
void
vm_page_requeue_locked(vm_page_t m)
{
struct vm_pagequeue *pq;

KASSERT(m->queue != PQ_NONE,
("vm_page_requeue_locked: page %p is not queued", m));
pq = vm_page_pagequeue(m);
vm_pagequeue_assert_locked(pq);
TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
}

/*
* vm_page_activate:		* vm_page_activate:
*		*
* Put the specified page on the active list (if appropriate).		* Put the specified page on the active list (if appropriate).
* Ensure that act_count is at least ACT_INIT but do not otherwise		* Ensure that act_count is at least ACT_INIT but do not otherwise
* mess with it.		* mess with it.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_activate(vm_page_t m)		vm_page_activate(vm_page_t m)
{		{
int queue;		int queue;

vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
if ((queue = m->queue) != PQ_ACTIVE) {
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if ((queue = m->queue) == PQ_ACTIVE \|\| m->wire_count > 0 \|\|
if (m->act_count < ACT_INIT)		(m->oflags & VPO_UNMANAGED) != 0) {
		if (queue == PQ_ACTIVE && m->act_count < ACT_INIT)
m->act_count = ACT_INIT;		m->act_count = ACT_INIT;
if (queue != PQ_NONE)		return;
vm_page_dequeue(m);
vm_page_enqueue(PQ_ACTIVE, m);
}		}
} else {
		vm_page_remque(m);
if (m->act_count < ACT_INIT)		if (m->act_count < ACT_INIT)
m->act_count = ACT_INIT;		m->act_count = ACT_INIT;
		vm_page_enqueue_lazy(m, PQ_ACTIVE);
}		}
}

/*		/*
* vm_page_free_prep:		* vm_page_free_prep:
*		*
* Prepares the given page to be put on the free list,		* Prepares the given page to be put on the free list,
* disassociating it from any VM object. The caller may return		* disassociating it from any VM object. The caller may return
* the page to the free list only if this function returns true.		* the page to the free list only if this function returns true.
*		*
* The object must be locked. The page must be locked if it is		* The object must be locked. The page must be locked if it is
* managed. For a queued managed page, the pagequeue_locked		* managed.
* argument specifies whether the page queue is already locked.
*/		*/
bool		bool
vm_page_free_prep(vm_page_t m, bool pagequeue_locked)		vm_page_free_prep(vm_page_t m)
{		{

#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)		#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)
if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {		if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {
uint64_t *p;		uint64_t *p;
int i;		int i;
p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));		p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)		for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)
KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",		KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",
m, i, (uintmax_t)*p));		m, i, (uintmax_t)*p));
}		}
#endif		#endif
if ((m->oflags & VPO_UNMANAGED) == 0) {		if ((m->oflags & VPO_UNMANAGED) == 0) {
vm_page_lock_assert(m, MA_OWNED);		vm_page_lock_assert(m, MA_OWNED);
KASSERT(!pmap_page_is_mapped(m),		KASSERT(!pmap_page_is_mapped(m),
("vm_page_free_toq: freeing mapped page %p", m));		("vm_page_free_prep: freeing mapped page %p", m));
} else		} else
KASSERT(m->queue == PQ_NONE,		KASSERT(m->queue == PQ_NONE,
("vm_page_free_toq: unmanaged page %p is queued", m));		("vm_page_free_prep: unmanaged page %p is queued", m));
VM_CNT_INC(v_tfree);		VM_CNT_INC(v_tfree);

if (vm_page_sbusied(m))		if (vm_page_sbusied(m))
panic("vm_page_free: freeing busy page %p", m);		panic("vm_page_free_prep: freeing busy page %p", m);

vm_page_remove(m);		vm_page_remove(m);

/*		/*
* If fictitious remove object association and		* If fictitious remove object association and
* return.		* return.
*/		*/
if ((m->flags & PG_FICTITIOUS) != 0) {		if ((m->flags & PG_FICTITIOUS) != 0) {
KASSERT(m->wire_count == 1,		KASSERT(m->wire_count == 1,
("fictitious page %p is not wired", m));		("fictitious page %p is not wired", m));
KASSERT(m->queue == PQ_NONE,		KASSERT(m->queue == PQ_NONE,
("fictitious page %p is queued", m));		("fictitious page %p is queued", m));
return (false);		return (false);
}		}

if (m->queue != PQ_NONE) {		/*
if (pagequeue_locked)		* Pages need not be dequeued before they are returned to the physical
vm_page_dequeue_locked(m);		* memory allocator, but they must at least be marked for a deferred
else		* dequeue.
vm_page_dequeue(m);		*/
}		if ((m->oflags & VPO_UNMANAGED) == 0)
		vm_page_dequeue_lazy(m);

m->valid = 0;		m->valid = 0;
vm_page_undirty(m);		vm_page_undirty(m);

if (m->wire_count != 0)		if (m->wire_count != 0)
panic("vm_page_free: freeing wired page %p", m);		panic("vm_page_free_prep: freeing wired page %p", m);
if (m->hold_count != 0) {		if (m->hold_count != 0) {
m->flags &= ~PG_ZERO;		m->flags &= ~PG_ZERO;
KASSERT((m->flags & PG_UNHOLDFREE) == 0,		KASSERT((m->flags & PG_UNHOLDFREE) == 0,
("vm_page_free: freeing PG_UNHOLDFREE page %p", m));		("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m));
m->flags \|= PG_UNHOLDFREE;		m->flags \|= PG_UNHOLDFREE;
return (false);		return (false);
}		}

/*		/*
* Restore the default memory attribute to the page.		* Restore the default memory attribute to the page.
*/		*/
if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)		if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);		pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);

#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
if (vm_reserv_free_page(m))		if (vm_reserv_free_page(m))
return (false);		return (false);
#endif		#endif

return (true);		return (true);
}		}

void
vm_page_free_phys_pglist(struct pglist *tq)
{
struct vm_domain *vmd;
vm_page_t m;
int cnt;

if (TAILQ_EMPTY(tq))
return;
vmd = NULL;
cnt = 0;
TAILQ_FOREACH(m, tq, listq) {
if (vmd != vm_pagequeue_domain(m)) {
if (vmd != NULL) {
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, cnt);
cnt = 0;
}
vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);
}
vm_phys_free_pages(m, 0);
cnt++;
}
if (vmd != NULL) {
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, cnt);
}
}

/*		/*
* vm_page_free_toq:		* vm_page_free_toq:
*		*
* Returns the given page to the free list, disassociating it		* Returns the given page to the free list, disassociating it
* from any VM object.		* from any VM object.
*		*
* The object must be locked. The page must be locked if it is		* The object must be locked. The page must be locked if it is
* managed.		* managed.
*/		*/
void		void
vm_page_free_toq(vm_page_t m)		vm_page_free_toq(vm_page_t m)
{		{
struct vm_domain *vmd;		struct vm_domain *vmd;

if (!vm_page_free_prep(m, false))		if (!vm_page_free_prep(m))
return;		return;

vmd = vm_pagequeue_domain(m);		vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);		vm_domain_free_lock(vmd);
vm_phys_free_pages(m, 0);		vm_phys_free_pages(m, 0);
vm_domain_free_unlock(vmd);		vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
}		}

/*		/*
* vm_page_free_pages_toq:		* vm_page_free_pages_toq:
*		*
* Returns a list of pages to the free list, disassociating it		* Returns a list of pages to the free list, disassociating it
* from any VM object. In other words, this is equivalent to		* from any VM object. In other words, this is equivalent to
* calling vm_page_free_toq() for each page of a list of VM objects.		* calling vm_page_free_toq() for each page of a list of VM objects.
*		*
* The objects must be locked. The pages must be locked if it is		* The objects must be locked. The pages must be locked if it is
* managed.		* managed.
*/		*/
void		void
vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)		vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)
{		{
vm_page_t m;		vm_page_t m;
struct pglist pgl;
int count;		int count;

if (SLIST_EMPTY(free))		if (SLIST_EMPTY(free))
return;		return;

count = 0;		count = 0;
TAILQ_INIT(&pgl);
while ((m = SLIST_FIRST(free)) != NULL) {		while ((m = SLIST_FIRST(free)) != NULL) {
count++;		count++;
SLIST_REMOVE_HEAD(free, plinks.s.ss);		SLIST_REMOVE_HEAD(free, plinks.s.ss);
if (vm_page_free_prep(m, false))		vm_page_free_toq(m);
TAILQ_INSERT_TAIL(&pgl, m, listq);
}		}

vm_page_free_phys_pglist(&pgl);

if (update_wire_count)		if (update_wire_count)
vm_wire_sub(count);		vm_wire_sub(count);
}		}

/*		/*
* vm_page_wire:		* vm_page_wire:
*		*
* Mark this page as wired down. If the page is fictitious, then		* Mark this page as wired down. If the page is fictitious, then
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
bool		bool
vm_page_unwire(vm_page_t m, uint8_t queue)		vm_page_unwire(vm_page_t m, uint8_t queue)
{		{
bool unwired;		bool unwired;

KASSERT(queue < PQ_COUNT \|\| queue == PQ_NONE,		KASSERT(queue < PQ_COUNT \|\| queue == PQ_NONE,
("vm_page_unwire: invalid queue %u request for page %p",		("vm_page_unwire: invalid queue %u request for page %p",
queue, m));		queue, m));
		if ((m->oflags & VPO_UNMANAGED) == 0)
		vm_page_assert_locked(m);

unwired = vm_page_unwire_noq(m);		unwired = vm_page_unwire_noq(m);
if (unwired && (m->oflags & VPO_UNMANAGED) == 0 && m->object != NULL) {		if (!unwired \|\| (m->oflags & VPO_UNMANAGED) != 0 \|\| m->object == NULL)
		return (unwired);

if (m->queue == queue) {		if (m->queue == queue) {
if (queue == PQ_ACTIVE)		if (queue == PQ_ACTIVE)
vm_page_reference(m);		vm_page_reference(m);
else if (queue != PQ_NONE)		else if (queue != PQ_NONE)
vm_page_requeue(m);		vm_page_requeue(m);
} else {		} else {
vm_page_remque(m);		vm_page_dequeue(m);
if (queue != PQ_NONE) {		if (queue != PQ_NONE) {
vm_page_enqueue(queue, m);		vm_page_enqueue_lazy(m, queue);
if (queue == PQ_ACTIVE)		if (queue == PQ_ACTIVE)
/* Initialize act_count. */		/* Initialize act_count. */
vm_page_activate(m);		vm_page_activate(m);
}		}
}		}
}
return (unwired);		return (unwired);
}		}

/*		/*
*		*
* vm_page_unwire_noq:		* vm_page_unwire_noq:
*		*
* Unwire a page without (re-)inserting it into a page queue. It is up		* Unwire a page without (re-)inserting it into a page queue. It is up
Show All 17 Lines	vm_page_unwire_noq(vm_page_t m)
if (m->wire_count == 0) {		if (m->wire_count == 0) {
vm_wire_sub(1);		vm_wire_sub(1);
return (true);		return (true);
} else		} else
return (false);		return (false);
}		}

/*		/*
* Move the specified page to the inactive queue, or requeue the page if it is		* Move the specified page to the tail of the inactive queue, or requeue
* already in the inactive queue.		* the page if it is already in the inactive queue.
*		*
* Normally, "noreuse" is FALSE, resulting in LRU ordering of the inactive
* queue. However, setting "noreuse" to TRUE will accelerate the specified
* page's reclamation, but it will not unmap the page from any address space.
* This is implemented by inserting the page near the head of the inactive
* queue, using a marker page to guide FIFO insertion ordering.
*
* The page must be locked.		* The page must be locked.
*/		*/
static inline void		void
_vm_page_deactivate(vm_page_t m, boolean_t noreuse)		vm_page_deactivate(vm_page_t m)
{		{
struct vm_pagequeue *pq;
int queue;

vm_page_assert_locked(m);		vm_page_assert_locked(m);

if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
pq = &vm_pagequeue_domain(m)->vmd_pagequeues[PQ_INACTIVE];		return;
/* Avoid multiple acquisitions of the inactive queue lock. */
queue = m->queue;
if (queue == PQ_INACTIVE) {
vm_pagequeue_lock(pq);
vm_page_dequeue_locked(m);
} else {
if (queue != PQ_NONE)
vm_page_dequeue(m);
vm_pagequeue_lock(pq);
}
m->queue = PQ_INACTIVE;
if (noreuse)
TAILQ_INSERT_BEFORE(
&vm_pagequeue_domain(m)->vmd_inacthead, m,
plinks.q);
else
TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
vm_pagequeue_cnt_inc(pq);
vm_pagequeue_unlock(pq);
}
}

/*		if (!vm_page_inactive(m)) {
		jeffUnsubmitted Done Inline Actions Should be an XXX comment. This path likely needs to be fast. jeff: Should be an XXX comment. This path likely needs to be fast.
* Move the specified page to the inactive queue, or requeue the page if it is		vm_page_remque(m);
* already in the inactive queue.		vm_page_enqueue_lazy(m, PQ_INACTIVE);
*		} else
* The page must be locked.		vm_page_requeue(m);
*/
void
vm_page_deactivate(vm_page_t m)
{

_vm_page_deactivate(m, FALSE);
}		}

/*		/*
* Move the specified page to the inactive queue with the expectation		* Move the specified page close to the head of the inactive queue,
* that it is unlikely to be reused.		* bypassing LRU. A marker page is used to maintain FIFO ordering.
		* As with regular enqueues, we use a per-CPU batch queue to reduce
		* contention on the page queue lock.
*		*
* The page must be locked.		* The page must be locked.
*/		*/
void		void
vm_page_deactivate_noreuse(vm_page_t m)		vm_page_deactivate_noreuse(vm_page_t m)
{		{
		struct vm_batchqueue *bq;
		struct vm_domain *vmd;
		struct vm_pagequeue *pq;
		vm_page_t marker;
		int domain;

_vm_page_deactivate(m, TRUE);		vm_page_assert_locked(m);

		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
		return;

		domain = vm_phys_domain(m);
		vmd = VM_DOMAIN(domain);
		pq = &vmd->vmd_pagequeues[PQ_INACTIVE];

		if (!vm_page_inactive(m))
		vm_page_remque(m);

		m->queue = PQ_INACTIVE;

		critical_enter();
		bq = DPCPU_PTR(noreuseq[domain]);
		if (vm_batchqueue_insert(bq, m)) {
		critical_exit();
		return;
}		}
		if (!vm_pagequeue_trylock(pq)) {
		critical_exit();
		vm_pagequeue_lock(pq);
		critical_enter();
		bq = DPCPU_PTR(noreuseq[domain]);
		}
		marker = &vmd->vmd_inacthead;
		do {
		if (m->queue != PQ_INACTIVE)
		continue;
		if ((m->aflags & PGA_ENQUEUED) != 0)
		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
		else {
		vm_page_aflag_set(m, PGA_ENQUEUED);
		vm_pagequeue_cnt_inc(pq);
		}
		TAILQ_INSERT_BEFORE(marker, m, plinks.q);
		} while ((m = vm_batchqueue_pop(bq)) != NULL);
		vm_pagequeue_unlock(pq);
		vm_batchqueue_init(bq);
		critical_exit();
		}

/*		/*
* vm_page_launder		* vm_page_launder
*		*
* Put a page in the laundry, or requeue it if it is already there.		* Put a page in the laundry, or requeue it if it is already there.
*/		*/
void		void
vm_page_launder(vm_page_t m)		vm_page_launder(vm_page_t m)
{		{

vm_page_assert_locked(m);		vm_page_assert_locked(m);
if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {		if (m->wire_count > 0 \|\| (m->oflags & VPO_UNMANAGED) != 0)
		return;

if (m->queue == PQ_LAUNDRY)		if (m->queue == PQ_LAUNDRY)
vm_page_requeue(m);		vm_page_requeue(m);
else {		else {
vm_page_remque(m);		vm_page_remque(m);
vm_page_enqueue(PQ_LAUNDRY, m);		vm_page_enqueue_lazy(m, PQ_LAUNDRY);
}		}
}		}
}

/*		/*
* vm_page_unswappable		* vm_page_unswappable
*		*
* Put a page in the PQ_UNSWAPPABLE holding queue.		* Put a page in the PQ_UNSWAPPABLE holding queue.
*/		*/
void		void
vm_page_unswappable(vm_page_t m)		vm_page_unswappable(vm_page_t m)
{		{

vm_page_assert_locked(m);		vm_page_assert_locked(m);
KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,		KASSERT(m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0,
("page %p already unswappable", m));		("page %p already unswappable", m));
if (m->queue != PQ_NONE)
vm_page_dequeue(m);		vm_page_remque(m);
vm_page_enqueue(PQ_UNSWAPPABLE, m);		vm_page_enqueue_lazy(m, PQ_UNSWAPPABLE);
}		}

/*		/*
* Attempt to free the page. If it cannot be freed, do nothing. Returns true		* Attempt to free the page. If it cannot be freed, do nothing. Returns true
* if the page is freed and false otherwise.		* if the page is freed and false otherwise.
*		*
* The page must be managed. The page and its containing object must be		* The page must be managed. The page and its containing object must be
* locked.		* locked.
▲ Show 20 Lines • Show All 750 Lines • Show Last 20 Lines