Differential D23449 Diff 68789 sys/vm/vm_page.c

Changeset View

Standalone View

sys/vm/vm_page.c

Show First 20 Lines • Show All 824 Lines • ▼ Show 20 Lines

void		void
vm_page_reference(vm_page_t m)		vm_page_reference(vm_page_t m)
{		{

vm_page_aflag_set(m, PGA_REFERENCED);		vm_page_aflag_set(m, PGA_REFERENCED);
}		}

		/*
		* vm_page_grab_trybusy
		*
		* Helper routine for grab functions to trylock busy.
		*
		* Returns true on success and false on failure.
		*/
static bool		static bool
vm_page_acquire_flags(vm_page_t m, int allocflags)		vm_page_grab_trybusy(vm_page_t m, int allocflags)
		markjUnsubmitted Not Done Inline Actions vm_page_grab_trybusy() is also used by non-grab functions. You could rename it to vm_page_trybusy() and group it with the other functions in this file that deal with page busying. markj: vm_page_grab_trybusy() is also used by non-grab functions. You could rename it to…
{		{
bool locked;

if ((allocflags & (VM_ALLOC_SBUSY \| VM_ALLOC_IGN_SBUSY)) != 0)		if ((allocflags & (VM_ALLOC_SBUSY \| VM_ALLOC_IGN_SBUSY)) != 0)
locked = vm_page_trysbusy(m);		return (vm_page_trysbusy(m));
else		else
locked = vm_page_tryxbusy(m);		return (vm_page_tryxbusy(m));
		}

		/*
		* vm_page_grab_tryacquire
		*
		* Helper routine for grab functions to trylock busy and wire.
		*
		* Returns true on success and false on failure.
		*/
		static inline bool
		vm_page_grab_tryacquire(vm_page_t m, int allocflags)
		{
		bool locked;

		locked = vm_page_grab_trybusy(m, allocflags);
if (locked && (allocflags & VM_ALLOC_WIRED) != 0)		if (locked && (allocflags & VM_ALLOC_WIRED) != 0)
vm_page_wire(m);		vm_page_wire(m);
return (locked);		return (locked);
}		}

/*		/*
* vm_page_busy_sleep_flags		* vm_page_grab_release
*		*
		* Helper routine for grab functions to release busy on return.
		*/
		static inline void
		vm_page_grab_release(vm_page_t m, int allocflags)
		{

		if ((allocflags & VM_ALLOC_NOBUSY) != 0) {
		if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
		vm_page_sunbusy(m);
		else
		vm_page_xunbusy(m);
		}
		}

		/*
		* vm_page_grab_sleep
		*
* Sleep for busy according to VM_ALLOC_ parameters. Returns true		* Sleep for busy according to VM_ALLOC_ parameters. Returns true
* if the caller should retry and false otherwise.		* if the caller should retry and false otherwise.
		kibUnsubmitted Not Done Inline Actions Although it is obvious for careful reader that the function drops the object lock, it might be surprising to not so careful one. Also we traditionally point that out. So I think it is useful to note that the function might drop the object lock if owned. kib: Although it is obvious for careful reader that the function drops the object lock, it might be…
*/		*/
static bool		static bool
vm_page_busy_sleep_flags(vm_object_t object, vm_page_t m, const char *wmesg,		vm_page_grab_sleep(vm_object_t object, vm_page_t m, vm_pindex_t pindex,
int allocflags)		const char *wmesg, int allocflags, bool locked)
{		{

if ((allocflags & VM_ALLOC_NOWAIT) != 0)		if ((allocflags & VM_ALLOC_NOWAIT) != 0)
return (false);		return (false);

/*		/*
* Reference the page before unlocking and sleeping so that		* Reference the page before unlocking and sleeping so that
* the page daemon is less likely to reclaim it.		* the page daemon is less likely to reclaim it.
*/		*/
if ((allocflags & VM_ALLOC_NOCREAT) == 0)		if (locked && (allocflags & VM_ALLOC_NOCREAT) == 0)
vm_page_reference(m);		vm_page_reference(m);

if (_vm_page_busy_sleep(object, m, m->pindex, wmesg, allocflags, true))		if (_vm_page_busy_sleep(object, m, m->pindex, wmesg, allocflags,
		locked) && locked)
VM_OBJECT_WLOCK(object);		VM_OBJECT_WLOCK(object);
if ((allocflags & VM_ALLOC_WAITFAIL) != 0)		if ((allocflags & VM_ALLOC_WAITFAIL) != 0)
return (false);		return (false);

return (true);		return (true);
}		}

/*		/*
Show All 12 Lines	vm_page_busy_acquire(vm_page_t m, int allocflags)
* The page-specific object must be cached because page		* The page-specific object must be cached because page
* identity can change during the sleep, causing the		* identity can change during the sleep, causing the
* re-lock of a different object.		* re-lock of a different object.
* It is assumed that a reference to the object is already		* It is assumed that a reference to the object is already
* held by the callers.		* held by the callers.
*/		*/
obj = m->object;		obj = m->object;
for (;;) {		for (;;) {
if (vm_page_acquire_flags(m, allocflags))		if (vm_page_grab_tryacquire(m, allocflags))
		markjUnsubmitted Not Done Inline Actions Why tryacquire instead of trybusy? I don't think anything should be using this function to wire the page. If not, then we should rename the function to e.g., vm_page_acquire(). markj: Why tryacquire instead of trybusy? I don't think anything should be using this function to wire…
		jeffAuthorUnsubmitted Done Inline Actions I found no reason to preclude wiring the page although it would be unusual. jeff: I found no reason to preclude wiring the page although it would be unusual.
return (true);		return (true);
if ((allocflags & VM_ALLOC_NOWAIT) != 0)		if ((allocflags & VM_ALLOC_NOWAIT) != 0)
return (false);		return (false);
if (obj != NULL)		if (obj != NULL)
locked = VM_OBJECT_WOWNED(obj);		locked = VM_OBJECT_WOWNED(obj);
else		else
locked = false;		locked = false;
MPASS(locked \|\| vm_page_wired(m));		MPASS(locked \|\| vm_page_wired(m));
▲ Show 20 Lines • Show All 693 Lines • ▼ Show 20 Lines	vm_page_object_remove(vm_page_t m)
VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT((m->ref_count & VPRC_OBJREF) != 0,		KASSERT((m->ref_count & VPRC_OBJREF) != 0,
("page %p is missing its object ref", m));		("page %p is missing its object ref", m));

/* Deferred free of swap space. */		/* Deferred free of swap space. */
if ((m->a.flags & PGA_SWAP_FREE) != 0)		if ((m->a.flags & PGA_SWAP_FREE) != 0)
vm_pager_page_unswapped(m);		vm_pager_page_unswapped(m);

		/*
		* Don't allow the compiler to re-order clearing the object
		* pointer after the radix_remove/TAILQ. This allows lockless
		* users to discover without looping. The radix code
		* provides the release barrier.
		*/
		kibUnsubmitted Not Done Inline Actions Is this comment still relevant ? I believe it described a release store. Right now there is nothing preventing observer from seeing updated radix with old m->object. kib: Is this comment still relevant ? I believe it described a release store. Right now there is…
		jeffAuthorUnsubmitted Done Inline Actions The release store is in vm_radix_remove(). The acquire is in vm_radix_lookup_unlocked(). jeff: The release store is in vm_radix_remove(). The acquire is in vm_radix_lookup_unlocked().
		atomic_store_ptr(&m->object, NULL);

mrem = vm_radix_remove(&object->rtree, m->pindex);		mrem = vm_radix_remove(&object->rtree, m->pindex);
KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m));		KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m));

/*		/*
* Now remove from the object's list of backed pages.		* Now remove from the object's list of backed pages.
*/		*/
TAILQ_REMOVE(&object->memq, m, listq);		TAILQ_REMOVE(&object->memq, m, listq);

Show All 38 Lines
* Removes the page but leaves the xbusy held. Returns true if this		* Removes the page but leaves the xbusy held. Returns true if this
* removed the final ref and false otherwise.		* removed the final ref and false otherwise.
*/		*/
bool		bool
vm_page_remove_xbusy(vm_page_t m)		vm_page_remove_xbusy(vm_page_t m)
{		{

vm_page_object_remove(m);		vm_page_object_remove(m);
m->object = NULL;
return (vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF);		return (vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF);
}		}

/*		/*
* vm_page_lookup:		* vm_page_lookup:
*		*
* Returns the page associated with the object/offset		* Returns the page associated with the object/offset
* pair specified; if none is found, NULL is returned.		* pair specified; if none is found, NULL is returned.
*		*
* The object must be locked.		* The object must be locked.
*/		*/
vm_page_t		vm_page_t
vm_page_lookup(vm_object_t object, vm_pindex_t pindex)		vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
{		{

VM_OBJECT_ASSERT_LOCKED(object);		VM_OBJECT_ASSERT_LOCKED(object);
return (vm_radix_lookup(&object->rtree, pindex));		return (vm_radix_lookup(&object->rtree, pindex));
}		}

/*		/*
		* This should only be used by lockless functions for releasing transient
		* incorrect acquires. The page may have been freed after we acquired a
		* busy lock. In this case busy_lock == VPB_FREED and we have nothing
		* further to do.
		*/
		static void
		vm_page_busy_release(vm_page_t m)
		{
		u_int x;

		x = atomic_load_int(&m->busy_lock);
		markjUnsubmitted Not Done Inline Actions This should be an atomic load. markj: This should be an atomic load.
		jeffAuthorUnsubmitted Done Inline Actions For the reader or the machine? jeff: For the reader or the machine?
		kibUnsubmitted Not Done Inline Actions For compiler. Atomic_load() does not add anything to the generated load, except it ensures that the load indeed happen (but could be reordered or coalesced). I think it would be somewhat clearer to see atomic_load() for lock-kind word, as well. kib: For compiler. Atomic_load() does not add anything to the generated load, except it ensures…
		for (;;) {
		if (x == VPB_FREED)
		break;
		if ((x & VPB_BIT_SHARED) != 0 && VPB_SHARERS(x) > 1) {
		if (atomic_fcmpset_int(&m->busy_lock, &x,
		x - VPB_ONE_SHARER))
		break;
		markjUnsubmitted Not Done Inline Actions Don't we potentially need to wake up waiters if we are the last sharer? markj: Don't we potentially need to wake up waiters if we are the last sharer?
		jeffAuthorUnsubmitted Done Inline Actions If we are the last sharer we should execute the case below. jeff: If we are the last sharer we should execute the case below.
		continue;
		}
		KASSERT((x & VPB_BIT_SHARED) != 0 \|\|
		kibUnsubmitted Not Done Inline Actions Should we assert that x is exclusive busy owner ? kib: Should we assert that x is exclusive busy owner ?
		(x & ~VPB_BIT_WAITERS) == VPB_CURTHREAD_EXCLUSIVE,
		("vm_page_busy_release: %p xbusy not owned.", m));
		if (!atomic_fcmpset_rel_int(&m->busy_lock, &x, VPB_UNBUSIED))
		continue;
		if ((x & VPB_BIT_WAITERS) != 0)
		wakeup(m);
		break;
		}
		}

		/*
* vm_page_find_least:		* vm_page_find_least:
*		*
* Returns the page associated with the object with least pindex		* Returns the page associated with the object with least pindex
* greater than or equal to the parameter pindex, or NULL.		* greater than or equal to the parameter pindex, or NULL.
*		*
* The object must be locked.		* The object must be locked.
*/		*/
vm_page_t		vm_page_t
▲ Show 20 Lines • Show All 1,993 Lines • ▼ Show 20 Lines	if (m->object != NULL) {
* The object reference can be released without an atomic		* The object reference can be released without an atomic
* operation.		* operation.
*/		*/
KASSERT((m->flags & PG_FICTITIOUS) != 0 \|\|		KASSERT((m->flags & PG_FICTITIOUS) != 0 \|\|
m->ref_count == VPRC_OBJREF,		m->ref_count == VPRC_OBJREF,
("vm_page_free_prep: page %p has unexpected ref_count %u",		("vm_page_free_prep: page %p has unexpected ref_count %u",
m, m->ref_count));		m, m->ref_count));
vm_page_object_remove(m);		vm_page_object_remove(m);
m->object = NULL;
m->ref_count -= VPRC_OBJREF;		m->ref_count -= VPRC_OBJREF;
} else		} else
vm_page_assert_unbusied(m);		vm_page_assert_unbusied(m);

vm_page_busy_free(m);		vm_page_busy_free(m);

/*		/*
* If fictitious remove object association and		* If fictitious remove object association and
▲ Show 20 Lines • Show All 540 Lines • ▼ Show 20 Lines	vm_page_advise(vm_page_t m, int advice)
* laundry are moved there.		* laundry are moved there.
*/		*/
if (m->dirty == 0)		if (m->dirty == 0)
vm_page_deactivate_noreuse(m);		vm_page_deactivate_noreuse(m);
else if (!vm_page_in_laundry(m))		else if (!vm_page_in_laundry(m))
vm_page_launder(m);		vm_page_launder(m);
}		}

static inline int		/*
vm_page_grab_pflags(int allocflags)		* Assert that the grab flags are valid.
		*/
		static inline void
		vm_page_grab_check(int allocflags)
		markjUnsubmitted Not Done Inline Actions vm_page_grab_check() might be a better name. It is consistent with vm_page_alloc_check() and vm_page_grab_asserts sounds like "asserts" is the object, not "vm_page". markj: vm_page_grab_check() might be a better name. It is consistent with vm_page_alloc_check() and…
		jeffAuthorUnsubmitted Done Inline Actions ok. The naming I really don't like is _unlocked after everything. I could probably do a sweep of the tree and convert everything to _unlocked and drop it from the name. Any outside sources doing direct object manipulation will suddenly be wrong though. If you have suggestions I'm all ears. jeff: ok. The naming I really don't like is _unlocked after everything. I could probably do a sweep…
{		{
int pflags;

KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 \|\|		KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 \|\|
(allocflags & VM_ALLOC_WIRED) != 0,		(allocflags & VM_ALLOC_WIRED) != 0,
("vm_page_grab_pflags: the pages must be busied or wired"));		("vm_page_grab*: the pages must be busied or wired"));

KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|		KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|
(allocflags & VM_ALLOC_IGN_SBUSY) != 0,		(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
("vm_page_grab_pflags: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY "		("vm_page_grab*: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));
"mismatch"));		}

		/*
		* Calculate the page allocation flags for grab.
		*/
		static inline int
		vm_page_grab_pflags(int allocflags)
		{
		int pflags;

		markjUnsubmitted Not Done Inline Actions You can get rid of the extra quotes here. markj: You can get rid of the extra quotes here.
pflags = allocflags &		pflags = allocflags &
~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL \|		~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL \|
VM_ALLOC_NOBUSY);		VM_ALLOC_NOBUSY);
if ((allocflags & VM_ALLOC_NOWAIT) == 0)		if ((allocflags & VM_ALLOC_NOWAIT) == 0)
pflags \|= VM_ALLOC_WAITFAIL;		pflags \|= VM_ALLOC_WAITFAIL;
if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)		if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
pflags \|= VM_ALLOC_SBUSY;		pflags \|= VM_ALLOC_SBUSY;

Show All 10 Lines
*		*
* The object must be locked on entry. The lock will, however, be released		* The object must be locked on entry. The lock will, however, be released
* and reacquired if the routine sleeps.		* and reacquired if the routine sleeps.
*/		*/
vm_page_t		vm_page_t
vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)		vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
{		{
vm_page_t m;		vm_page_t m;
int pflags;

VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);
pflags = vm_page_grab_pflags(allocflags);		vm_page_grab_check(allocflags);

retrylookup:		retrylookup:
if ((m = vm_page_lookup(object, pindex)) != NULL) {		if ((m = vm_page_lookup(object, pindex)) != NULL) {
if (!vm_page_acquire_flags(m, allocflags)) {		if (!vm_page_grab_tryacquire(m, allocflags)) {
if (vm_page_busy_sleep_flags(object, m, "pgrbwt",		if (vm_page_grab_sleep(object, m, pindex, "pgrbwt",
allocflags))		allocflags, true))
goto retrylookup;		goto retrylookup;
return (NULL);		return (NULL);
}		}
goto out;		goto out;
}		}
if ((allocflags & VM_ALLOC_NOCREAT) != 0)		if ((allocflags & VM_ALLOC_NOCREAT) != 0)
return (NULL);		return (NULL);
m = vm_page_alloc(object, pindex, pflags);		m = vm_page_alloc(object, pindex, vm_page_grab_pflags(allocflags));
if (m == NULL) {		if (m == NULL) {
if ((allocflags & (VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL)) != 0)		if ((allocflags & (VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL)) != 0)
return (NULL);		return (NULL);
goto retrylookup;		goto retrylookup;
}		}
if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)		if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
pmap_zero_page(m);		pmap_zero_page(m);

out:		out:
if ((allocflags & VM_ALLOC_NOBUSY) != 0) {		vm_page_grab_release(m, allocflags);
if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
vm_page_sunbusy(m);		return (m);
else
vm_page_xunbusy(m);
}		}

		/*
		* Locklessly acquire a page while ensuring the identity does not change.
		* This routine will sleep according to flags. The return value indicates
		* whether the caller should retry. The return is tri-state with mp:
		*
		* (false, *mp != NULL) - The operation was successful.
		* (false, *mp == NULL) - Retry will not succeed (SLEEPFAIL\|NOWAIT)
		* (true, *mp == NULL) - Retry may succeed but the page has not been acquired.
		*/
		static bool
		vm_page_busy_acquire_unlocked(vm_object_t object, vm_pindex_t pindex,
		vm_page_t *mp, int allocflags)
		{
		vm_page_t m;

		m = *mp;
		*mp = NULL;
		for (;;) {
		if (vm_page_grab_trybusy(m, allocflags)) {
		if (m->object == object && m->pindex == pindex)
		break;
		/* Identity changed, caller must relookup. */
		vm_page_busy_release(m);
		return (true);
		}
		if (!vm_page_grab_sleep(object, m, pindex, "pgbau",
		allocflags, false))
		return (false);
		/* Identity changed, caller must relookup. */
		if (m->object != object \|\| m->pindex != pindex)
		return (true);
		}
		if ((allocflags & VM_ALLOC_WIRED) != 0)
		vm_page_wire(m);
		markjUnsubmitted Not Done Inline Actions The use of TAILQ_NEXT here is technically fine but breaks the locking protocol. In the absence of an "atomic" TAILQ_NEXT it would probably be good to note this in the comment. markj: The use of TAILQ_NEXT here is technically fine but breaks the locking protocol. In the absence…
		jeffAuthorUnsubmitted Done Inline Actions That was the intent of the comment but I can clarify. jeff: That was the intent of the comment but I can clarify.
		vm_page_grab_release(m, allocflags);
		*mp = m;

		return (false);
		}

		/*
		* Try to locklessly grab a page and fall back to the object lock if NOCREAT
		* is not set.
		*/
		vm_page_t
		vm_page_grab_unlocked(vm_object_t object, vm_pindex_t pindex, int allocflags)
		{
		markjUnsubmitted Not Done Inline Actions Maybe use __predict_true here. markj: Maybe use __predict_true here.
		vm_page_t m;

		vm_page_grab_check(allocflags);

		for (;;) {
		m = vm_radix_lookup_unlocked(&object->rtree, pindex);
		if (m == NULL)
		break;
		if (!vm_page_busy_acquire_unlocked(object, pindex, &m,
		allocflags))
return (m);		return (m);
}		}

/*		/*
		* The radix lockless lookup should never return spurious errors. If
		* the user specifies NOCREAT they are guaranteed there was no page
		* present at the instant of the call. A NOCREAT caller must handle
		* create races gracefully.
		*/
		if ((allocflags & VM_ALLOC_NOCREAT) != 0)
		return (NULL);

		VM_OBJECT_WLOCK(object);
		m = vm_page_grab(object, pindex, allocflags);
		VM_OBJECT_WUNLOCK(object);

		return (m);
		}

		/*
* Grab a page and make it valid, paging in if necessary. Pages missing from		* Grab a page and make it valid, paging in if necessary. Pages missing from
* their pager are zero filled and validated. If a VM_ALLOC_COUNT is supplied		* their pager are zero filled and validated. If a VM_ALLOC_COUNT is supplied
* and the page is not valid as many as VM_INITIAL_PAGEIN pages can be brought		* and the page is not valid as many as VM_INITIAL_PAGEIN pages can be brought
* in simultaneously. Additional pages will be left on a paging queue but		* in simultaneously. Additional pages will be left on a paging queue but
* will neither be wired nor busy regardless of allocflags.		* will neither be wired nor busy regardless of allocflags.
*/		*/
int		int
		markjUnsubmitted Not Done Inline Actions "Spurious errors" is kind of ambiguous. "False negatives"? markj: "Spurious errors" is kind of ambiguous. "False negatives"?
vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags)		vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags)
{		{
vm_page_t m;		vm_page_t m;
vm_page_t ma[VM_INITIAL_PAGEIN];		vm_page_t ma[VM_INITIAL_PAGEIN];
bool sleep, xbusy;
int after, i, pflags, rv;		int after, i, pflags, rv;

KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|		KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|
(allocflags & VM_ALLOC_IGN_SBUSY) != 0,		(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));		("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));
KASSERT((allocflags &		KASSERT((allocflags &
(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL \| VM_ALLOC_ZERO)) == 0,		(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL \| VM_ALLOC_ZERO)) == 0,
("vm_page_grab_valid: Invalid flags 0x%X", allocflags));		("vm_page_grab_valid: Invalid flags 0x%X", allocflags));
VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);
pflags = allocflags & ~(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY);		pflags = allocflags & ~(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY);
pflags \|= VM_ALLOC_WAITFAIL;		pflags \|= VM_ALLOC_WAITFAIL;

retrylookup:		retrylookup:
xbusy = false;
if ((m = vm_page_lookup(object, pindex)) != NULL) {		if ((m = vm_page_lookup(object, pindex)) != NULL) {
/*		/*
* If the page is fully valid it can only become invalid		* If the page is fully valid it can only become invalid
* with the object lock held. If it is not valid it can		* with the object lock held. If it is not valid it can
* become valid with the busy lock held. Therefore, we		* become valid with the busy lock held. Therefore, we
* may unnecessarily lock the exclusive busy here if we		* may unnecessarily lock the exclusive busy here if we
* race with I/O completion not using the object lock.		* race with I/O completion not using the object lock.
* However, we will not end up with an invalid page and a		* However, we will not end up with an invalid page and a
* shared lock.		* shared lock.
*/		*/
if (!vm_page_all_valid(m) \|\|		if (!vm_page_grab_trybusy(m,
(allocflags & (VM_ALLOC_IGN_SBUSY \| VM_ALLOC_SBUSY)) == 0) {		vm_page_all_valid(m) ? allocflags : 0)) {
sleep = !vm_page_tryxbusy(m);		(void)vm_page_grab_sleep(object, m, pindex, "pgrbwt",
xbusy = true;		allocflags, true);
} else
sleep = !vm_page_trysbusy(m);
if (sleep) {
(void)vm_page_busy_sleep_flags(object, m, "pgrbwt",
allocflags);
goto retrylookup;		goto retrylookup;
}		}
if ((allocflags & VM_ALLOC_NOCREAT) != 0 &&		if (vm_page_all_valid(m))
!vm_page_all_valid(m)) {		goto out;
if (xbusy)		if ((allocflags & VM_ALLOC_NOCREAT) != 0) {
vm_page_xunbusy(m);		vm_page_busy_release(m);
else
vm_page_sunbusy(m);
*mp = NULL;		*mp = NULL;
return (VM_PAGER_FAIL);		return (VM_PAGER_FAIL);
}		}
if ((allocflags & VM_ALLOC_WIRED) != 0)
vm_page_wire(m);
if (vm_page_all_valid(m))
goto out;
} else if ((allocflags & VM_ALLOC_NOCREAT) != 0) {		} else if ((allocflags & VM_ALLOC_NOCREAT) != 0) {
*mp = NULL;		*mp = NULL;
return (VM_PAGER_FAIL);		return (VM_PAGER_FAIL);
} else if ((m = vm_page_alloc(object, pindex, pflags)) != NULL) {		} else if ((m = vm_page_alloc(object, pindex, pflags)) == NULL) {
xbusy = true;
} else {
goto retrylookup;		goto retrylookup;
}		}

vm_page_assert_xbusied(m);		vm_page_assert_xbusied(m);
MPASS(xbusy);
if (vm_pager_has_page(object, pindex, NULL, &after)) {		if (vm_pager_has_page(object, pindex, NULL, &after)) {
after = MIN(after, VM_INITIAL_PAGEIN);		after = MIN(after, VM_INITIAL_PAGEIN);
after = MIN(after, allocflags >> VM_ALLOC_COUNT_SHIFT);		after = MIN(after, allocflags >> VM_ALLOC_COUNT_SHIFT);
after = MAX(after, 1);		after = MAX(after, 1);
ma[0] = m;		ma[0] = m;
for (i = 1; i < after; i++) {		for (i = 1; i < after; i++) {
if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) {		if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) {
if (ma[i]->valid \|\| !vm_page_tryxbusy(ma[i]))		if (ma[i]->valid \|\| !vm_page_tryxbusy(ma[i]))
Show All 9 Lines	if (vm_pager_has_page(object, pindex, NULL, &after)) {
vm_object_pip_add(object, after);		vm_object_pip_add(object, after);
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);
rv = vm_pager_get_pages(object, ma, after, NULL, NULL);		rv = vm_pager_get_pages(object, ma, after, NULL, NULL);
VM_OBJECT_WLOCK(object);		VM_OBJECT_WLOCK(object);
vm_object_pip_wakeupn(object, after);		vm_object_pip_wakeupn(object, after);
/* Pager may have replaced a page. */		/* Pager may have replaced a page. */
m = ma[0];		m = ma[0];
if (rv != VM_PAGER_OK) {		if (rv != VM_PAGER_OK) {
if ((allocflags & VM_ALLOC_WIRED) != 0)
vm_page_unwire_noq(m);
for (i = 0; i < after; i++) {		for (i = 0; i < after; i++) {
if (!vm_page_wired(ma[i]))		if (!vm_page_wired(ma[i]))
vm_page_free(ma[i]);		vm_page_free(ma[i]);
else		else
vm_page_xunbusy(ma[i]);		vm_page_xunbusy(ma[i]);
}		}
*mp = NULL;		*mp = NULL;
return (rv);		return (rv);
}		}
for (i = 1; i < after; i++)		for (i = 1; i < after; i++)
vm_page_readahead_finish(ma[i]);		vm_page_readahead_finish(ma[i]);
MPASS(vm_page_all_valid(m));		MPASS(vm_page_all_valid(m));
} else {		} else {
vm_page_zero_invalid(m, TRUE);		vm_page_zero_invalid(m, TRUE);
}		}
out:		out:
if ((allocflags & VM_ALLOC_NOBUSY) != 0) {		if ((allocflags & VM_ALLOC_WIRED) != 0)
if (xbusy)		vm_page_wire(m);
vm_page_xunbusy(m);		if ((allocflags & VM_ALLOC_SBUSY) != 0 && vm_page_xbusied(m))
else
vm_page_sunbusy(m);
}
if ((allocflags & VM_ALLOC_SBUSY) != 0 && xbusy)
vm_page_busy_downgrade(m);		vm_page_busy_downgrade(m);
		else if ((allocflags & VM_ALLOC_NOBUSY) != 0)
		vm_page_busy_release(m);
*mp = m;		*mp = m;
return (VM_PAGER_OK);		return (VM_PAGER_OK);
}		}

/*		/*
		* Locklessly grab a valid page. If the page is not valid or not yet
		* allocated this will fall back to the object lock method.
		*/
		int
		vm_page_grab_valid_unlocked(vm_page_t *mp, vm_object_t object,
		vm_pindex_t pindex, int allocflags)
		{
		vm_page_t m;
		int flags;
		int error;

		KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|
		(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
		("vm_page_grab_valid_unlocked: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY "
		"mismatch"));
		KASSERT((allocflags &
		(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL \| VM_ALLOC_ZERO)) == 0,
		("vm_page_grab_valid_unlocked: Invalid flags 0x%X", allocflags));

		/*
		* Attempt a lockless lookup and busy. We need at least an sbusy
		* before we can inspect the valid field and return a wired page.
		*/
		flags = allocflags & ~(VM_ALLOC_NOBUSY \| VM_ALLOC_WIRED);
		for (;;) {
		m = vm_radix_lookup_unlocked(&object->rtree, pindex);
		if (m == NULL)
		break;
		if (vm_page_busy_acquire_unlocked(object, pindex, &m, flags))
		continue;
		markjUnsubmitted Not Done Inline Actions You might add a two line vm_page_grab_wire() for this, since it gets copied around a lot. markj: You might add a two line vm_page_grab_wire() for this, since it gets copied around a lot.
		jeffAuthorUnsubmitted Done Inline Actions This and _valid() don't want to do it via _acquire or tryacquire because they may decide not to keep the page and this simplified error handling. So they should be the only two IIRC. jeff: This and _valid() don't want to do it via _acquire or tryacquire because they may decide not to…
		if (m == NULL)
		return (VM_PAGER_FAIL);
		if (!vm_page_all_valid(m)) {
		vm_page_busy_release(m);
		break;
		}
		if ((allocflags & VM_ALLOC_WIRED) != 0)
		vm_page_wire(m);
		vm_page_grab_release(m, allocflags);
		*mp = m;
		return (VM_PAGER_OK);
		}
		VM_OBJECT_WLOCK(object);
		error = vm_page_grab_valid(mp, object, pindex, allocflags);
		VM_OBJECT_WUNLOCK(object);

		return (error);
		}
		markjUnsubmitted Not Done Inline Actions Why is it sufficient to hold a wiring? Without the object lock or page busied the page's <obj, pindex> identity is not stable. markj: Why is it sufficient to hold a wiring? Without the object lock or page busied the page's <obj…
		jeffAuthorUnsubmitted Done Inline Actions I think this is actually harmless because the page in question will have a NULL object pointer and its next pointer will either be clear or point to a page without a NULL object pointer. pindex will remain unchanged. I suppose that could create a spin loop until the NULL pointer makes it to the list. So from that pov I need to be more careful here and perhaps retry less. Another option would be to pass the expected object pointer in with the page for validation. Once grab_next fails grab_pages_unlocked will fall back to the locked loop. So the subsequent pages will be valid. I don't think it's possible to get incorrect results this way. jeff: I think this is actually harmless because the page in question will have a NULL object pointer…

		/*
		* Locklessly attempt to acquire a pindex sequential page. The
		* (object, pindex) tuple is used to validate the origin page and
		* returned page. mp is an in/out field. The predecessor is supplied
		* on input and on output it will be NULL or the desired page.
		*
		* We can not guarantee that an existing page will not return NULL. Callers
		* are responsible for retrying under the object lock.
		*
		* The return value indicates whether the caller should retry. The return
		* is tri-state with mp:
		*
		* (false, *mp != NULL) - The operation was successful.
		* (false, *mp == NULL) - Retry will not succeed (WAITFAIL\|NOWAIT)
		* (true, *mp == NULL) - Retry may succeed but the page has not been acquired.
		*/
		static bool
		vm_page_grab_next_unlocked(vm_object_t object, vm_pindex_t pindex,
		vm_page_t *mp, int allocflags)
		{
		vm_page_t m;
		vm_page_t next;

		m = *mp;
		vm_page_grab_check(allocflags);
		MPASS(vm_page_busied(m) \|\| vm_page_wired(m));

		for (;;) {
		/*
		* We may get a spurious NULL here because the previous
		* page has been removed from the list.
		*/
		*mp = next = TAILQ_NEXT(m, listq);
		if (next == NULL \|\| next->pindex != pindex)
		jeffAuthorUnsubmitted Done Inline Actions We could actually handle more cases by falling back to a lockless lookup. Then the grab_next_unlocked return would be conclusive. jeff: We could actually handle more cases by falling back to a lockless lookup. Then the…
		break;
		if (!vm_page_busy_acquire_unlocked(object, pindex, mp,
		allocflags))
		return (false);
		}
		*mp = NULL;
		return (true);
		}

		/*
* Return the specified range of pages from the given object. For each		* Return the specified range of pages from the given object. For each
* page offset within the range, if a page already exists within the object		* page offset within the range, if a page already exists within the object
* at that offset and it is busy, then wait for it to change state. If,		* at that offset and it is busy, then wait for it to change state. If,
* instead, the page doesn't exist, then allocate it.		* instead, the page doesn't exist, then allocate it.
*		*
* The caller must always specify an allocation class.		* The caller must always specify an allocation class.
*		*
* allocation classes:		* allocation classes:
Show All 20 Lines
{		{
vm_page_t m, mpred;		vm_page_t m, mpred;
int pflags;		int pflags;
int i;		int i;

VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0,		KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0,
("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed"));		("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed"));
		vm_page_grab_check(allocflags);

pflags = vm_page_grab_pflags(allocflags);		pflags = vm_page_grab_pflags(allocflags);
if (count == 0)		if (count == 0)
return (0);		return (0);

i = 0;		i = 0;
retrylookup:		retrylookup:
m = vm_radix_lookup_le(&object->rtree, pindex + i);		m = vm_radix_lookup_le(&object->rtree, pindex + i);
if (m == NULL \|\| m->pindex != pindex + i) {		if (m == NULL \|\| m->pindex != pindex + i) {
mpred = m;		mpred = m;
m = NULL;		m = NULL;
} else		} else
mpred = TAILQ_PREV(m, pglist, listq);		mpred = TAILQ_PREV(m, pglist, listq);
for (; i < count; i++) {		for (; i < count; i++) {
if (m != NULL) {		if (m != NULL) {
if (!vm_page_acquire_flags(m, allocflags)) {		if (!vm_page_grab_tryacquire(m, allocflags)) {
if (vm_page_busy_sleep_flags(object, m,		if (vm_page_grab_sleep(object, m, pindex,
"grbmaw", allocflags))		"grbmaw", allocflags, true))
goto retrylookup;		goto retrylookup;
break;		break;
}		}
} else {		} else {
if ((allocflags & VM_ALLOC_NOCREAT) != 0)		if ((allocflags & VM_ALLOC_NOCREAT) != 0)
break;		break;
m = vm_page_alloc_after(object, pindex + i,		m = vm_page_alloc_after(object, pindex + i,
pflags \| VM_ALLOC_COUNT(count - i), mpred);		pflags \| VM_ALLOC_COUNT(count - i), mpred);
if (m == NULL) {		if (m == NULL) {
if ((allocflags & (VM_ALLOC_NOWAIT \|		if ((allocflags & (VM_ALLOC_NOWAIT \|
VM_ALLOC_WAITFAIL)) != 0)		VM_ALLOC_WAITFAIL)) != 0)
break;		break;
goto retrylookup;		goto retrylookup;
}		}
}		}
if (vm_page_none_valid(m) &&		if (vm_page_none_valid(m) &&
(allocflags & VM_ALLOC_ZERO) != 0) {		(allocflags & VM_ALLOC_ZERO) != 0) {
if ((m->flags & PG_ZERO) == 0)		if ((m->flags & PG_ZERO) == 0)
pmap_zero_page(m);		pmap_zero_page(m);
vm_page_valid(m);		vm_page_valid(m);
}		}
if ((allocflags & VM_ALLOC_NOBUSY) != 0) {		vm_page_grab_release(m, allocflags);
if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
vm_page_sunbusy(m);
else
vm_page_xunbusy(m);
}
ma[i] = mpred = m;		ma[i] = mpred = m;
m = vm_page_next(m);		m = vm_page_next(m);
}		}
		return (i);
		}

		/*
		* Unlocked variant of vm_page_grab_pages(). This accepts the same flags
		* and will fall back to the locked variant to handle allocation.
		*/
		int
		vm_page_grab_pages_unlocked(vm_object_t object, vm_pindex_t pindex,
		int allocflags, vm_page_t *ma, int count)
		{
		vm_page_t m;
		int flags;
		int i;

		vm_page_grab_check(allocflags);

		/*
		* Modify flags for lockless acquire to hold the page until we
		* set it valid if necessary.
		*/
		flags = allocflags & ~VM_ALLOC_NOBUSY;
		for (;;) {
		markjUnsubmitted Not Done Inline Actions I think "next" should be called "pred" or "prev". markj: I think "next" should be called "pred" or "prev".
		m = vm_radix_lookup_unlocked(&object->rtree, pindex);
		if (m == NULL)
		break;
		if (vm_page_busy_acquire_unlocked(object, pindex, &m, flags))
		continue;
		if (m == NULL)
		return (0);
		break;
		}
		i = 0;
		while (m != NULL) {
		if ((flags & VM_ALLOC_ZERO) != 0 && vm_page_none_valid(m)) {
		if ((m->flags & PG_ZERO) == 0)
		pmap_zero_page(m);
		vm_page_valid(m);
		}
		/* m will still be wired or busy according to flags. */
		vm_page_grab_release(m, allocflags);
		ma[i] = m;
		pindex++;
		if (++i == count)
		return (i);
		/* Any retry must be handled with the lock. */
		if (vm_page_grab_next_unlocked(object, pindex, &m, flags))
		break;
		if (m == NULL)
		return (i);
		}
		count -= i;
		VM_OBJECT_WLOCK(object);
		i += vm_page_grab_pages(object, pindex, allocflags, &ma[i], count);
		VM_OBJECT_WUNLOCK(object);

return (i);		return (i);
}		}

/*		/*
* Mapping function for valid or dirty bits in a page.		* Mapping function for valid or dirty bits in a page.
*		*
* Inputs are required to range within a page.		* Inputs are required to range within a page.
*/		*/
▲ Show 20 Lines • Show All 668 Lines • Show Last 20 Lines