Diff 97019

sys/vm/vm_page.h

	Show First 20 Lines • Show All 543 Lines • ▼ Show 20 Lines
	#define VM_ALLOC_NORECLAIM 0x0080 /* (c) Do not reclaim after failure */			#define VM_ALLOC_NORECLAIM 0x0080 /* (c) Do not reclaim after failure */
	#define VM_ALLOC_NOOBJ 0x0100 /* (acg) No associated object */			#define VM_ALLOC_NOOBJ 0x0100 /* (acg) No associated object */
	#define VM_ALLOC_NOBUSY 0x0200 /* (acgp) Do not excl busy the page */			#define VM_ALLOC_NOBUSY 0x0200 /* (acgp) Do not excl busy the page */
	#define VM_ALLOC_NOCREAT 0x0400 /* (gp) Don't create a page */			#define VM_ALLOC_NOCREAT 0x0400 /* (gp) Don't create a page */
	#define VM_ALLOC_IGN_SBUSY 0x1000 /* (gp) Ignore shared busy flag */			#define VM_ALLOC_IGN_SBUSY 0x1000 /* (gp) Ignore shared busy flag */
	#define VM_ALLOC_NODUMP 0x2000 /* (ag) don't include in dump */			#define VM_ALLOC_NODUMP 0x2000 /* (ag) don't include in dump */
	#define VM_ALLOC_SBUSY 0x4000 /* (acgp) Shared busy the page */			#define VM_ALLOC_SBUSY 0x4000 /* (acgp) Shared busy the page */
	#define VM_ALLOC_NOWAIT 0x8000 /* (acgnp) Do not sleep */			#define VM_ALLOC_NOWAIT 0x8000 /* (acgnp) Do not sleep */
				#define VM_ALLOC_COUNT_MAX 0xffff
	#define VM_ALLOC_COUNT_SHIFT 16			#define VM_ALLOC_COUNT_SHIFT 16
	#define VM_ALLOC_COUNT(count) ((count) << VM_ALLOC_COUNT_SHIFT)			#define VM_ALLOC_COUNT_MASK (VM_ALLOC_COUNT(VM_ALLOC_COUNT_MAX))
				kibUnsubmitted Done Inline Actions Could this be VM_ALLOC_COUNT(VM_ALLOC_COUNT_MAX - 1) and count < VM_ALLOC_COUNT_MAX could be used for assert in VM_ALLOC_COUNT? kib: Could this be VM_ALLOC_COUNT(VM_ALLOC_COUNT_MAX - 1) and count < VM_ALLOC_COUNT_MAX could be…
				markjAuthorUnsubmitted Done Inline Actions Are you suggesting using a statement expression there, e.g., #define VM_ALLOC_COUNT(count) ({ MPASS(count < VM_ALLOC_COUNT_MAX); count << VM_ALLOC_COUNT_SHIFT; }) ? markj: Are you suggesting using a statement expression there, e.g., ``` #define VM_ALLOC_COUNT(count)…
				kibUnsubmitted Done Inline Actions Yes something along this line for VM_ALLOC_COUNT(). If vm/vm_page.h is used in usermode (even rarely) then `__extension__` would be useful/needed as well. kib: Yes something along this line for VM_ALLOC_COUNT(). If vm/vm_page.h is used in usermode (even…
				markjAuthorUnsubmitted Done Inline Actions This chunk is kernel-only. I am not 100% sure about adding the assertion, it means that VM_ALLOC_COUNT() cannot be assigned to a global var. But I don't think this is likely to arise... markj: This chunk is kernel-only. I am not 100% sure about adding the assertion, it means that…
				#define VM_ALLOC_COUNT(count) ({ \
				KASSERT((count) <= VM_ALLOC_COUNT_MAX, \
				("%s: invalid VM_ALLOC_COUNT value", __func__)); \
				(count) << VM_ALLOC_COUNT_SHIFT; \
				})

	#ifdef M_NOWAIT			#ifdef M_NOWAIT
	static inline int			static inline int
	malloc2vm_flags(int malloc_flags)			malloc2vm_flags(int malloc_flags)
	{			{
	int pflags;			int pflags;

	KASSERT((malloc_flags & M_USE_RESERVE) == 0 \|\|			KASSERT((malloc_flags & M_USE_RESERVE) == 0 \|\|
	▲ Show 20 Lines • Show All 467 Lines • Show Last 20 Lines

View Options

sys/vm/vm_page.c

Show First 20 Lines • Show All 1,957 Lines • ▼ Show 20 Lines
* VM_ALLOC_SYSTEM system really needs a page		* VM_ALLOC_SYSTEM system really needs a page
* VM_ALLOC_INTERRUPT interrupt time request		* VM_ALLOC_INTERRUPT interrupt time request
*		*
* optional allocation flags:		* optional allocation flags:
* VM_ALLOC_COUNT(number) the number of additional pages that the caller		* VM_ALLOC_COUNT(number) the number of additional pages that the caller
* intends to allocate		* intends to allocate
* VM_ALLOC_NOBUSY do not exclusive busy the page		* VM_ALLOC_NOBUSY do not exclusive busy the page
* VM_ALLOC_NODUMP do not include the page in a kernel core dump		* VM_ALLOC_NODUMP do not include the page in a kernel core dump
* VM_ALLOC_NOOBJ page is not associated with an object and
* should not be exclusive busy
* VM_ALLOC_SBUSY shared busy the allocated page		* VM_ALLOC_SBUSY shared busy the allocated page
* VM_ALLOC_WIRED wire the allocated page		* VM_ALLOC_WIRED wire the allocated page
* VM_ALLOC_ZERO prefer a zeroed page		* VM_ALLOC_ZERO prefer a zeroed page
*/		*/
vm_page_t		vm_page_t
vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)		vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
{		{

return (vm_page_alloc_after(object, pindex, req, object != NULL ?		return (vm_page_alloc_after(object, pindex, req,
vm_radix_lookup_le(&object->rtree, pindex) : NULL));		vm_radix_lookup_le(&object->rtree, pindex)));
}		}

vm_page_t		vm_page_t
vm_page_alloc_domain(vm_object_t object, vm_pindex_t pindex, int domain,		vm_page_alloc_domain(vm_object_t object, vm_pindex_t pindex, int domain,
int req)		int req)
{		{

return (vm_page_alloc_domain_after(object, pindex, domain, req,		return (vm_page_alloc_domain_after(object, pindex, domain, req,
object != NULL ? vm_radix_lookup_le(&object->rtree, pindex) :		vm_radix_lookup_le(&object->rtree, pindex)));
NULL));
}		}

/*		/*
* Allocate a page in the specified object with the given page index. To		* Allocate a page in the specified object with the given page index. To
* optimize insertion of the page into the object, the caller must also specifiy		* optimize insertion of the page into the object, the caller must also specifiy
* the resident page in the object with largest index smaller than the given		* the resident page in the object with largest index smaller than the given
* page index, or NULL if no such page exists.		* page index, or NULL if no such page exists.
*/		*/
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines
}		}

vm_page_t		vm_page_t
vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain,		vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain,
int req, vm_page_t mpred)		int req, vm_page_t mpred)
{		{
struct vm_domain *vmd;		struct vm_domain *vmd;
vm_page_t m;		vm_page_t m;
int flags, pool;		int flags;

KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&		#define VPA_FLAGS (VM_ALLOC_CLASS_MASK \| VM_ALLOC_WAITFAIL \| \
(object != NULL \|\| (req & VM_ALLOC_SBUSY) == 0) &&		VM_ALLOC_NOWAIT \| VM_ALLOC_NOBUSY \| \
((req & (VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)) !=		VM_ALLOC_SBUSY \| VM_ALLOC_WIRED \| \
		VM_ALLOC_NODUMP \| VM_ALLOC_ZERO \| VM_ALLOC_COUNT_MASK)
		KASSERT((req & ~VPA_FLAGS) == 0,
		("invalid request %#x", req));
		KASSERT(((req & (VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)) !=
		kibUnsubmitted Not Done Inline Actions I wonder if it makes sense to add initial assert that checks that req contains only flags that the function can handle. For this and all other vm_page_alloc* functions. kib: I wonder if it makes sense to add initial assert that checks that req contains only flags that…
		markjAuthorUnsubmitted Done Inline Actions Yes, I think it's a good idea. markj: Yes, I think it's a good idea.
(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)),		(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)),
("inconsistent object(%p)/req(%x)", object, req));		("invalid request %#x", req));
KASSERT(object == NULL \|\| (req & VM_ALLOC_WAITOK) == 0,
("Can't sleep and retry object insertion."));
KASSERT(mpred == NULL \|\| mpred->pindex < pindex,		KASSERT(mpred == NULL \|\| mpred->pindex < pindex,
("mpred %p doesn't precede pindex 0x%jx", mpred,		("mpred %p doesn't precede pindex 0x%jx", mpred,
(uintmax_t)pindex));		(uintmax_t)pindex));
if (object != NULL)
VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);

flags = 0;		flags = 0;
m = NULL;		m = NULL;
pool = object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT;
again:		again:
		alcUnsubmitted Not Done Inline Actions We really don't need this variable any longer. alc: We really don't need this variable any longer.
#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
/*		/*
* Can we allocate the page from a reservation?		* Can we allocate the page from a reservation?
*/		*/
if (vm_object_reserv(object) &&		if (vm_object_reserv(object) &&
(m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) !=		(m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) !=
NULL) {		NULL) {
goto found;		goto found;
}		}
#endif		#endif
vmd = VM_DOMAIN(domain);		vmd = VM_DOMAIN(domain);
if (vmd->vmd_pgcache[pool].zone != NULL) {		if (vmd->vmd_pgcache[VM_FREEPOOL_DEFAULT].zone != NULL) {
m = uma_zalloc(vmd->vmd_pgcache[pool].zone, M_NOWAIT \| M_NOVM);		m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DEFAULT].zone,
		M_NOWAIT \| M_NOVM);
if (m != NULL) {		if (m != NULL) {
flags \|= PG_PCPU_CACHE;		flags \|= PG_PCPU_CACHE;
goto found;		goto found;
}		}
}		}
if (vm_domain_allocate(vmd, req, 1)) {		if (vm_domain_allocate(vmd, req, 1)) {
/*		/*
* If not, allocate it from the free page queues.		* If not, allocate it from the free page queues.
*/		*/
vm_domain_free_lock(vmd);		vm_domain_free_lock(vmd);
m = vm_phys_alloc_pages(domain, pool, 0);		m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT, 0);
vm_domain_free_unlock(vmd);		vm_domain_free_unlock(vmd);
if (m == NULL) {		if (m == NULL) {
vm_domain_freecnt_inc(vmd, 1);		vm_domain_freecnt_inc(vmd, 1);
#if VM_NRESERVLEVEL > 0		#if VM_NRESERVLEVEL > 0
if (vm_reserv_reclaim_inactive(domain))		if (vm_reserv_reclaim_inactive(domain))
goto again;		goto again;
#endif		#endif
}		}
Show All 12 Lines	#endif
*/		*/
found:		found:
vm_page_dequeue(m);		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/*		/*
* Initialize the page. Only the PG_ZERO flag is inherited.		* Initialize the page. Only the PG_ZERO flag is inherited.
*/		*/
if ((req & VM_ALLOC_ZERO) != 0)		flags \|= m->flags & PG_ZERO;
		kibUnsubmitted Done Inline Actions I do not think () are needed. kib: I do not think () are needed.
flags \|= (m->flags & PG_ZERO);
if ((req & VM_ALLOC_NODUMP) != 0)		if ((req & VM_ALLOC_NODUMP) != 0)
flags \|= PG_NODUMP;		flags \|= PG_NODUMP;
m->flags = flags;		m->flags = flags;
m->a.flags = 0;		m->a.flags = 0;
m->oflags = object == NULL \|\| (object->flags & OBJ_UNMANAGED) != 0 ?		m->oflags = (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0;
VPO_UNMANAGED : 0;		if ((req & (VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)) == 0)
if ((req & (VM_ALLOC_NOBUSY \| VM_ALLOC_NOOBJ \| VM_ALLOC_SBUSY)) == 0)
m->busy_lock = VPB_CURTHREAD_EXCLUSIVE;		m->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
else if ((req & VM_ALLOC_SBUSY) != 0)		else if ((req & VM_ALLOC_SBUSY) != 0)
m->busy_lock = VPB_SHARERS_WORD(1);		m->busy_lock = VPB_SHARERS_WORD(1);
else		else
m->busy_lock = VPB_UNBUSIED;		m->busy_lock = VPB_UNBUSIED;
if (req & VM_ALLOC_WIRED) {		if (req & VM_ALLOC_WIRED) {
vm_wire_add(1);		vm_wire_add(1);
m->ref_count = 1;		m->ref_count = 1;
}		}
m->a.act_count = 0;		m->a.act_count = 0;

if (object != NULL) {
if (vm_page_insert_after(m, object, pindex, mpred)) {		if (vm_page_insert_after(m, object, pindex, mpred)) {
if (req & VM_ALLOC_WIRED) {		if (req & VM_ALLOC_WIRED) {
vm_wire_sub(1);		vm_wire_sub(1);
m->ref_count = 0;		m->ref_count = 0;
}		}
KASSERT(m->object == NULL, ("page %p has object", m));		KASSERT(m->object == NULL, ("page %p has object", m));
m->oflags = VPO_UNMANAGED;		m->oflags = VPO_UNMANAGED;
m->busy_lock = VPB_UNBUSIED;		m->busy_lock = VPB_UNBUSIED;
/* Don't change PG_ZERO. */		/* Don't change PG_ZERO. */
vm_page_free_toq(m);		vm_page_free_toq(m);
if (req & VM_ALLOC_WAITFAIL) {		if (req & VM_ALLOC_WAITFAIL) {
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);
vm_radix_wait();		vm_radix_wait();
VM_OBJECT_WLOCK(object);		VM_OBJECT_WLOCK(object);
}		}
return (NULL);		return (NULL);
}		}

/* Ignore device objects; the pager sets "memattr" for them. */		/* Ignore device objects; the pager sets "memattr" for them. */
if (object->memattr != VM_MEMATTR_DEFAULT &&		if (object->memattr != VM_MEMATTR_DEFAULT &&
(object->flags & OBJ_FICTITIOUS) == 0)		(object->flags & OBJ_FICTITIOUS) == 0)
pmap_page_set_memattr(m, object->memattr);		pmap_page_set_memattr(m, object->memattr);
} else
m->pindex = pindex;

return (m);		return (m);
}		}

/*		/*
* vm_page_alloc_contig:		* vm_page_alloc_contig:
*		*
* Allocate a contiguous set of physical pages of the given size "npages"		* Allocate a contiguous set of physical pages of the given size "npages"
▲ Show 20 Lines • Show All 202 Lines • ▼ Show 20 Lines
*/		*/
static __always_inline vm_page_t		static __always_inline vm_page_t
_vm_page_alloc_noobj_domain(int domain, const int freelist, int req)		_vm_page_alloc_noobj_domain(int domain, const int freelist, int req)
{		{
struct vm_domain *vmd;		struct vm_domain *vmd;
vm_page_t m;		vm_page_t m;
int flags;		int flags;

KASSERT((req & (VM_ALLOC_SBUSY \| VM_ALLOC_IGN_SBUSY \|		#define VPAN_FLAGS (VM_ALLOC_CLASS_MASK \| VM_ALLOC_WAITFAIL \| \
VM_ALLOC_NOOBJ)) == 0,		VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| \
("%s: invalid req %#x", __func__, req));		VM_ALLOC_NOBUSY \| VM_ALLOC_WIRED \| \
		VM_ALLOC_NODUMP \| VM_ALLOC_ZERO \| VM_ALLOC_COUNT_MASK)
		KASSERT((req & ~VPAN_FLAGS) == 0,
		("invalid request %#x", req));

flags = (req & VM_ALLOC_NODUMP) != 0 ? PG_NODUMP : 0;		flags = (req & VM_ALLOC_NODUMP) != 0 ? PG_NODUMP : 0;
vmd = VM_DOMAIN(domain);		vmd = VM_DOMAIN(domain);
again:		again:
if (freelist == VM_NFREELIST &&		if (freelist == VM_NFREELIST &&
vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone != NULL) {		vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone != NULL) {
m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone,		m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone,
M_NOWAIT \| M_NOVM);		M_NOWAIT \| M_NOVM);
Show All 19 Lines	if (vm_domain_alloc_fail(vmd, NULL, req))
goto again;		goto again;
return (NULL);		return (NULL);
}		}

found:		found:
vm_page_dequeue(m);		vm_page_dequeue(m);
vm_page_alloc_check(m);		vm_page_alloc_check(m);

/* Consumers should not rely on a useful default pindex value. */		/*
		* Consumers should not rely on a useful default pindex value.
		*/
m->pindex = 0xdeadc0dedeadc0de;		m->pindex = 0xdeadc0dedeadc0de;
m->flags = (m->flags & PG_ZERO) \| flags;		m->flags = (m->flags & PG_ZERO) \| flags;
m->a.flags = 0;		m->a.flags = 0;
m->oflags = VPO_UNMANAGED;		m->oflags = VPO_UNMANAGED;
m->busy_lock = VPB_UNBUSIED;		m->busy_lock = VPB_UNBUSIED;
if ((req & VM_ALLOC_WIRED) != 0) {		if ((req & VM_ALLOC_WIRED) != 0) {
vm_wire_add(1);		vm_wire_add(1);
m->ref_count = 1;		m->ref_count = 1;
▲ Show 20 Lines • Show All 2,048 Lines • ▼ Show 20 Lines
*/		*/
static inline int		static inline int
vm_page_grab_pflags(int allocflags)		vm_page_grab_pflags(int allocflags)
{		{
int pflags;		int pflags;

pflags = allocflags &		pflags = allocflags &
~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL \|		~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL \|
VM_ALLOC_NOBUSY);		VM_ALLOC_NOBUSY \| VM_ALLOC_IGN_SBUSY);
if ((allocflags & VM_ALLOC_NOWAIT) == 0)		if ((allocflags & VM_ALLOC_NOWAIT) == 0)
pflags \|= VM_ALLOC_WAITFAIL;		pflags \|= VM_ALLOC_WAITFAIL;
if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)		if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
pflags \|= VM_ALLOC_SBUSY;		pflags \|= VM_ALLOC_SBUSY;

return (pflags);		return (pflags);
}		}

▲ Show 20 Lines • Show All 156 Lines • ▼ Show 20 Lines	vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags)
KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|		KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|
(allocflags & VM_ALLOC_IGN_SBUSY) != 0,		(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));		("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));
KASSERT((allocflags &		KASSERT((allocflags &
(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL \| VM_ALLOC_ZERO)) == 0,		(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL \| VM_ALLOC_ZERO)) == 0,
("vm_page_grab_valid: Invalid flags 0x%X", allocflags));		("vm_page_grab_valid: Invalid flags 0x%X", allocflags));
VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);
pflags = allocflags & ~(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY \|		pflags = allocflags & ~(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY \|
VM_ALLOC_WIRED);		VM_ALLOC_WIRED \| VM_ALLOC_IGN_SBUSY);
pflags \|= VM_ALLOC_WAITFAIL;		pflags \|= VM_ALLOC_WAITFAIL;

retrylookup:		retrylookup:
if ((m = vm_page_lookup(object, pindex)) != NULL) {		if ((m = vm_page_lookup(object, pindex)) != NULL) {
/*		/*
* If the page is fully valid it can only become invalid		* If the page is fully valid it can only become invalid
* with the object lock held. If it is not valid it can		* with the object lock held. If it is not valid it can
* become valid with the busy lock held. Therefore, we		* become valid with the busy lock held. Therefore, we
▲ Show 20 Lines • Show All 926 Lines • Show Last 20 Lines

vm_page: Stop handling VM_ALLOC_NOOBJ in vm_page_alloc_domain_after()
ClosedPublic
Actions

Details

Diff Detail

Event Timeline

Revision Contents
Changeset List

Diff 97019

sys/vm/vm_page.h

sys/vm/vm_page.c

vm_page: Stop handling VM_ALLOC_NOOBJ in vm_page_alloc_domain_after()ClosedPublicActions

Details

Diff Detail

Event Timeline

Revision ContentsChangeset List

Diff 97019

sys/vm/vm_page.h

sys/vm/vm_page.c

vm_page: Stop handling VM_ALLOC_NOOBJ in vm_page_alloc_domain_after()
ClosedPublic
Actions

Revision Contents
Changeset List