Page MenuHomeFreeBSD
Differential D47036 Diff 145489 sys/vm/vm_page.c

Changeset View

Standalone View

View Options

sys/vm/vm_page.c

Show First 20 Lines • Show All 165 Lines • ▼ Show 20 Lines
static vm_page_t vm_page_alloc_nofree_domain(int domain, int req); static vm_page_t vm_page_alloc_nofree_domain(int domain, int req);
static bool _vm_page_busy_sleep(vm_object_t obj, vm_page_t m, static bool _vm_page_busy_sleep(vm_object_t obj, vm_page_t m,
vm_pindex_t pindex, const char *wmesg, int allocflags, bool locked); vm_pindex_t pindex, const char *wmesg, int allocflags, bool locked);
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
static void vm_page_enqueue(vm_page_t m, uint8_t queue); static void vm_page_enqueue(vm_page_t m, uint8_t queue);
static bool vm_page_free_prep(vm_page_t m); static bool vm_page_free_prep(vm_page_t m);
static void vm_page_free_toq(vm_page_t m); static void vm_page_free_toq(vm_page_t m);
static void vm_page_init(void *dummy); static void vm_page_init(void *dummy);
static int vm_page_insert_after(vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred);
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object, static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
vm_page_t mpred); vm_page_t mpred);
static void vm_page_mvqueue(vm_page_t m, const uint8_t queue, static void vm_page_mvqueue(vm_page_t m, const uint8_t queue,
const uint16_t nflag); const uint16_t nflag);
static int vm_page_reclaim_run(int req_class, int domain, u_long npages, static int vm_page_reclaim_run(int req_class, int domain, u_long npages,
vm_page_t m_run, vm_paddr_t high); vm_page_t m_run, vm_paddr_t high);
static void vm_page_release_toq(vm_page_t m, uint8_t nqueue, bool noreuse); static void vm_page_release_toq(vm_page_t m, uint8_t nqueue, bool noreuse);
static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,
▲ Show 20 Lines • Show All 1,286 Lines • ▼ Show 20 Linesvm_page_dirty_KBI(vm_page_t m)
/* Refer to this operation by its public name. */ /* Refer to this operation by its public name. */
KASSERT(vm_page_all_valid(m), ("vm_page_dirty: page is invalid!")); KASSERT(vm_page_all_valid(m), ("vm_page_dirty: page is invalid!"));
m->dirty = VM_PAGE_BITS_ALL; m->dirty = VM_PAGE_BITS_ALL;
} }
/* /*
* Insert the given page into the given object at the given pindex. mpred is * Insert the given page into the given object at the given pindex. mpred is
* used for memq linkage. From vm_page_insert, lookup is true, mpred is * used for memq linkage. From vm_page_insert, lookup is true, mpred is
* initially NULL, and this procedure looks it up. From vm_page_insert_after, * initially NULL, and this procedure looks it up.
* lookup is false and mpred is known to the caller to be valid, and may be
* NULL if this will be the page with the lowest pindex.
* *
* The procedure is marked __always_inline to suggest to the compiler to * The procedure is marked __always_inline to suggest to the compiler to
* eliminate the lookup parameter and the associated alternate branch. * eliminate the lookup parameter and the associated alternate branch.
*/ */
static __always_inline int static __always_inline int
vm_page_insert_lookup(vm_page_t m, vm_object_t object, vm_pindex_t pindex, vm_page_insert_lookup(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
vm_page_t mpred, bool lookup) vm_page_t mpred, bool lookup, struct pctrie_iter *pages, bool hasIter)
{ {
int error; int error;
VM_OBJECT_ASSERT_WLOCKED(object); VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(m->object == NULL, KASSERT(m->object == NULL,
("vm_page_insert: page %p already inserted", m)); ("vm_page_insert: page %p already inserted", m));
/* /*
* Record the object/offset pair in this page. * Record the object/offset pair in this page.
*/ */
m->object = object; m->object = object;
m->pindex = pindex; m->pindex = pindex;
m->ref_count |= VPRC_OBJREF; m->ref_count |= VPRC_OBJREF;
/* /*
* Add this page to the object's radix tree, and look up mpred if * Add this page to the object's radix tree, and look up mpred if
* needed. * needed.
*/ */
if (lookup) if (lookup)
error = vm_radix_insert_lookup_lt(&object->rtree, m, &mpred); error = vm_radix_insert_lookup_lt(&object->rtree, m, &mpred);
else if (hasIter)
error = vm_radix_iter_insert(pages, m);
else else
error = vm_radix_insert(&object->rtree, m); error = vm_radix_insert(&object->rtree, m);
if (__predict_false(error != 0)) { if (__predict_false(error != 0)) {
m->object = NULL; m->object = NULL;
m->pindex = 0; m->pindex = 0;
m->ref_count &= ~VPRC_OBJREF; m->ref_count &= ~VPRC_OBJREF;
return (1); return (1);
} }
/* /*
* Now link into the object's ordered list of backed pages. * Now link into the object's ordered list of backed pages.
*/ */
vm_page_insert_radixdone(m, object, mpred); vm_page_insert_radixdone(m, object, mpred);
vm_pager_page_inserted(object, m); vm_pager_page_inserted(object, m);
return (0); return (0);
} }
/* /*
* vm_page_insert: [ internal use only ] * Try to insert the given page at the given pindex, using an iterator.
*
* Inserts the given mem entry into the object and object list.
*
* The object must be locked.
*/ */
int static int
vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex) vm_page_iter_insert(struct pctrie_iter *pages, vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred)
alcUnsubmitted
Done
Inline Actions

With this version, I see a 5.3% reduction in alloc_contig cycles. If you restore passing mpred, rather than recomputing it, I see a 7.7% reduction.

alc: With this version, I see a 5.3% reduction in alloc_contig cycles. If you restore passing mpred…
{ {
return (vm_page_insert_lookup(m, object, pindex, NULL, true)); return (vm_page_insert_lookup(m, object, pindex,
mpred, false, pages, true));
} }
/* /*
* vm_page_insert_after: * vm_page_insert: [ internal use only ]
* *
* Inserts the page "m" into the specified object at offset "pindex". * Inserts the given mem entry into the object and object list.
* *
* The page "mpred" must immediately precede the offset "pindex" within
* the specified object.
*
* The object must be locked. * The object must be locked.
*/ */
static int int
vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex, vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
vm_page_t mpred)
{ {
return (vm_page_insert_lookup(m, object, pindex, mpred, false)); return (vm_page_insert_lookup(m, object, pindex, NULL, true,
NULL, false));
} }
/* /*
* vm_page_insert_radixdone: * vm_page_insert_radixdone:
* *
* Complete page "m" insertion into the specified object after the * Complete page "m" insertion into the specified object after the
* radix trie hooking. * radix trie hooking.
* *
▲ Show 20 Lines • Show All 267 Lines • ▼ Show 20 Linesvm_page_find_least(vm_object_t object, vm_pindex_t pindex)
VM_OBJECT_ASSERT_LOCKED(object); VM_OBJECT_ASSERT_LOCKED(object);
if ((m = TAILQ_FIRST(&object->memq)) != NULL && m->pindex < pindex) if ((m = TAILQ_FIRST(&object->memq)) != NULL && m->pindex < pindex)
m = vm_radix_lookup_ge(&object->rtree, pindex); m = vm_radix_lookup_ge(&object->rtree, pindex);
return (m); return (m);
} }
/* /*
* vm_page_iter_lookup_ge:
*
* Returns the page associated with the object with least pindex
* greater than or equal to the parameter pindex, or NULL. Initializes the
* iterator to point to that page.
*
* The iter pctrie must be locked.
*/
vm_page_t
vm_page_iter_lookup_ge(struct pctrie_iter *pages, vm_pindex_t pindex)
{
return (vm_radix_iter_lookup_ge(pages, pindex));
}
/*
* Returns the given page's successor (by pindex) within the object if it is * Returns the given page's successor (by pindex) within the object if it is
* resident; if none is found, NULL is returned. * resident; if none is found, NULL is returned.
* *
* The object must be locked. * The object must be locked.
*/ */
vm_page_t vm_page_t
vm_page_next(vm_page_t m) vm_page_next(vm_page_t m)
{ {
▲ Show 20 Lines • Show All 149 Lines • ▼ Show 20 Linesvm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
vm_page_insert_radixdone(m, new_object, mpred); vm_page_insert_radixdone(m, new_object, mpred);
vm_page_dirty(m); vm_page_dirty(m);
vm_pager_page_inserted(new_object, m); vm_pager_page_inserted(new_object, m);
return (0); return (0);
} }
/* /*
* vm_page_mpred: * Allocate a page in the specified object with the given page index. To
* * optimize insertion of the page into the object, the caller must also specify
* Return the greatest page of the object with index <= pindex, * the resident page in the object with largest index smaller than the given
* or NULL, if there is none. Assumes object lock is held. * page index, or NULL if no such page exists.
*/ */
vm_page_t static vm_page_t
vm_page_mpred(vm_object_t object, vm_pindex_t pindex) vm_page_alloc_after(struct pctrie_iter *pages, vm_object_t object,
vm_pindex_t pindex, int req, vm_page_t mpred)
{ {
return (vm_radix_lookup_le(&object->rtree, pindex)); struct vm_domainset_iter di;
vm_page_t m;
int domain;
vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
do {
m = vm_page_alloc_domain(pages, object, pindex, domain, req,
mpred);
if (m != NULL)
break;
} while (vm_domainset_iter_page(&di, object, &domain) == 0);
return (m);
} }
/* /*
* vm_page_alloc: * vm_page_alloc:
* *
* Allocate and return a page that is associated with the specified * Allocate and return a page that is associated with the specified
* object and offset pair. By default, this page is exclusive busied. * object and offset pair. By default, this page is exclusive busied.
* *
Show All 11 Lines
* VM_ALLOC_NODUMP do not include the page in a kernel core dump * VM_ALLOC_NODUMP do not include the page in a kernel core dump
* VM_ALLOC_SBUSY shared busy the allocated page * VM_ALLOC_SBUSY shared busy the allocated page
* VM_ALLOC_WIRED wire the allocated page * VM_ALLOC_WIRED wire the allocated page
* VM_ALLOC_ZERO prefer a zeroed page * VM_ALLOC_ZERO prefer a zeroed page
*/ */
vm_page_t vm_page_t
vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
{ {
struct pctrie_iter pages;
return (vm_page_alloc_after(object, pindex, req, vm_page_iter_init(&pages, object);
vm_page_mpred(object, pindex))); return (vm_page_alloc_after(&pages,object, pindex, req,
vm_radix_iter_lookup_le(&pages, pindex)));
} }
/* /*
* Allocate a page in the specified object with the given page index. To
* optimize insertion of the page into the object, the caller must also specify
* the resident page in the object with largest index smaller than the given
* page index, or NULL if no such page exists.
*/
vm_page_t
vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex,
int req, vm_page_t mpred)
{
struct vm_domainset_iter di;
vm_page_t m;
int domain;
vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
do {
m = vm_page_alloc_domain_after(object, pindex, domain, req,
mpred);
if (m != NULL)
break;
} while (vm_domainset_iter_page(&di, object, &domain) == 0);
return (m);
}
/*
* Returns true if the number of free pages exceeds the minimum * Returns true if the number of free pages exceeds the minimum
* for the request class and false otherwise. * for the request class and false otherwise.
*/ */
static int static int
_vm_domain_allocate(struct vm_domain *vmd, int req_class, int npages) _vm_domain_allocate(struct vm_domain *vmd, int req_class, int npages)
{ {
u_int limit, old, new; u_int limit, old, new;
Show All 37 Linesvm_domain_allocate(struct vm_domain *vmd, int req, int npages)
*/ */
req_class = req & VM_ALLOC_CLASS_MASK; req_class = req & VM_ALLOC_CLASS_MASK;
if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
req_class = VM_ALLOC_SYSTEM; req_class = VM_ALLOC_SYSTEM;
return (_vm_domain_allocate(vmd, req_class, npages)); return (_vm_domain_allocate(vmd, req_class, npages));
} }
vm_page_t vm_page_t
vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain, vm_page_alloc_domain(struct pctrie_iter *pages, vm_object_t object,
int req, vm_page_t mpred) vm_pindex_t pindex, int domain, int req, vm_page_t mpred)
{ {
struct vm_domain *vmd; struct vm_domain *vmd;
vm_page_t m; vm_page_t m;
int flags; int flags;
#define VPA_FLAGS (VM_ALLOC_CLASS_MASK | VM_ALLOC_WAITFAIL | \ #define VPA_FLAGS (VM_ALLOC_CLASS_MASK | VM_ALLOC_WAITFAIL | \
VM_ALLOC_NOWAIT | VM_ALLOC_NOBUSY | \ VM_ALLOC_NOWAIT | VM_ALLOC_NOBUSY | \
VM_ALLOC_SBUSY | VM_ALLOC_WIRED | \ VM_ALLOC_SBUSY | VM_ALLOC_WIRED | \
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Linesfound:
else else
m->busy_lock = VPB_UNBUSIED; m->busy_lock = VPB_UNBUSIED;
if (req & VM_ALLOC_WIRED) { if (req & VM_ALLOC_WIRED) {
vm_wire_add(1); vm_wire_add(1);
m->ref_count = 1; m->ref_count = 1;
} }
m->a.act_count = 0; m->a.act_count = 0;
if (vm_page_insert_after(m, object, pindex, mpred)) { if (vm_page_insert_lookup(m, object, pindex, mpred, false,
pages, true)) {
if (req & VM_ALLOC_WIRED) { if (req & VM_ALLOC_WIRED) {
vm_wire_sub(1); vm_wire_sub(1);
m->ref_count = 0; m->ref_count = 0;
} }
KASSERT(m->object == NULL, ("page %p has object", m)); KASSERT(m->object == NULL, ("page %p has object", m));
m->oflags = VPO_UNMANAGED; m->oflags = VPO_UNMANAGED;
m->busy_lock = VPB_UNBUSIED; m->busy_lock = VPB_UNBUSIED;
/* Don't change PG_ZERO. */ /* Don't change PG_ZERO. */
▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines#endif
return (NULL); return (NULL);
} }
vm_page_t vm_page_t
vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain, vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain,
int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary, vm_memattr_t memattr) vm_paddr_t boundary, vm_memattr_t memattr)
{ {
struct pctrie_iter pages;
vm_page_t m, m_ret, mpred; vm_page_t m, m_ret, mpred;
u_int busy_lock, flags, oflags; u_int busy_lock, flags, oflags;
#define VPAC_FLAGS (VPA_FLAGS | VM_ALLOC_NORECLAIM) #define VPAC_FLAGS (VPA_FLAGS | VM_ALLOC_NORECLAIM)
KASSERT((req & ~VPAC_FLAGS) == 0, KASSERT((req & ~VPAC_FLAGS) == 0,
("invalid request %#x", req)); ("invalid request %#x", req));
KASSERT(((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != KASSERT(((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) !=
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)),
("invalid request %#x", req)); ("invalid request %#x", req));
KASSERT((req & (VM_ALLOC_WAITOK | VM_ALLOC_NORECLAIM)) != KASSERT((req & (VM_ALLOC_WAITOK | VM_ALLOC_NORECLAIM)) !=
(VM_ALLOC_WAITOK | VM_ALLOC_NORECLAIM), (VM_ALLOC_WAITOK | VM_ALLOC_NORECLAIM),
("invalid request %#x", req)); ("invalid request %#x", req));
VM_OBJECT_ASSERT_WLOCKED(object); VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT((object->flags & OBJ_FICTITIOUS) == 0, KASSERT((object->flags & OBJ_FICTITIOUS) == 0,
("vm_page_alloc_contig: object %p has fictitious pages", ("vm_page_alloc_contig: object %p has fictitious pages",
object)); object));
KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero"));
mpred = vm_page_mpred(object, pindex); vm_page_iter_init(&pages, object);
mpred = vm_radix_iter_lookup_le(&pages, pindex);
KASSERT(mpred == NULL || mpred->pindex != pindex, KASSERT(mpred == NULL || mpred->pindex != pindex,
("vm_page_alloc_contig: pindex already allocated")); ("vm_page_alloc_contig: pindex already allocated"));
for (;;) { for (;;) {
#if VM_NRESERVLEVEL > 0 #if VM_NRESERVLEVEL > 0
/* /*
* Can we allocate the pages from a reservation? * Can we allocate the pages from a reservation?
*/ */
if (vm_object_reserv(object) && if (vm_object_reserv(object) &&
Show All 32 Lines for (m = m_ret; m < &m_ret[npages]; m++) {
vm_page_alloc_check(m); vm_page_alloc_check(m);
m->a.flags = 0; m->a.flags = 0;
m->flags = (m->flags | PG_NODUMP) & flags; m->flags = (m->flags | PG_NODUMP) & flags;
m->busy_lock = busy_lock; m->busy_lock = busy_lock;
if ((req & VM_ALLOC_WIRED) != 0) if ((req & VM_ALLOC_WIRED) != 0)
m->ref_count = 1; m->ref_count = 1;
m->a.act_count = 0; m->a.act_count = 0;
m->oflags = oflags; m->oflags = oflags;
if (vm_page_insert_after(m, object, pindex, mpred)) { if (vm_page_iter_insert(&pages, m, object, pindex, mpred)) {
if ((req & VM_ALLOC_WIRED) != 0) if ((req & VM_ALLOC_WIRED) != 0)
vm_wire_sub(npages); vm_wire_sub(npages);
KASSERT(m->object == NULL, KASSERT(m->object == NULL,
("page %p has object", m)); ("page %p has object", m));
mpred = m; mpred = m;
for (m = m_ret; m < &m_ret[npages]; m++) { for (m = m_ret; m < &m_ret[npages]; m++) {
if (m <= mpred && if (m <= mpred &&
(req & VM_ALLOC_WIRED) != 0) (req & VM_ALLOC_WIRED) != 0)
▲ Show 20 Lines • Show All 2,603 Lines • ▼ Show 20 Lines
* *
* If VM_ALLOC_NOWAIT is not specified, this routine may sleep. Otherwise, it * If VM_ALLOC_NOWAIT is not specified, this routine may sleep. Otherwise, it
* may return a partial prefix of the requested range. * may return a partial prefix of the requested range.
*/ */
int int
vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
vm_page_t *ma, int count) vm_page_t *ma, int count)
{ {
struct pctrie_iter pages;
vm_page_t m, mpred; vm_page_t m, mpred;
int pflags; int pflags;
int i; int i;
VM_OBJECT_ASSERT_WLOCKED(object); VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0, KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0,
("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed")); ("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed"));
KASSERT(count > 0, KASSERT(count > 0,
("vm_page_grab_pages: invalid page count %d", count)); ("vm_page_grab_pages: invalid page count %d", count));
vm_page_grab_check(allocflags); vm_page_grab_check(allocflags);
pflags = vm_page_grab_pflags(allocflags); pflags = vm_page_grab_pflags(allocflags);
i = 0; i = 0;
vm_page_iter_init(&pages, object);
retrylookup: retrylookup:
m = vm_page_mpred(object, pindex + i); mpred = vm_radix_iter_lookup_le(&pages, pindex + i);
if (m == NULL || m->pindex != pindex + i) {
mpred = m;
m = NULL;
} else
mpred = TAILQ_PREV(m, pglist, listq);
for (; i < count; i++) { for (; i < count; i++) {
m = vm_radix_iter_lookup(&pages, pindex + i);
if (m != NULL) { if (m != NULL) {
if (!vm_page_tryacquire(m, allocflags)) { if (!vm_page_tryacquire(m, allocflags)) {
if (vm_page_grab_sleep(object, m, pindex + i, if (vm_page_grab_sleep(object, m, pindex + i,
"grbmaw", allocflags, true)) "grbmaw", allocflags, true)) {
pctrie_iter_reset(&pages);
goto retrylookup; goto retrylookup;
}
break; break;
} }
} else { } else {
if ((allocflags & VM_ALLOC_NOCREAT) != 0) if ((allocflags & VM_ALLOC_NOCREAT) != 0)
break; break;
m = vm_page_alloc_after(object, pindex + i, m = vm_page_alloc_after(&pages, object, pindex + i,
pflags | VM_ALLOC_COUNT(count - i), mpred); pflags | VM_ALLOC_COUNT(count - i), mpred);
if (m == NULL) { if (m == NULL) {
if ((allocflags & (VM_ALLOC_NOWAIT | if ((allocflags & (VM_ALLOC_NOWAIT |
VM_ALLOC_WAITFAIL)) != 0) VM_ALLOC_WAITFAIL)) != 0)
break; break;
pctrie_iter_reset(&pages);
goto retrylookup; goto retrylookup;
} }
} }
if (vm_page_none_valid(m) && if (vm_page_none_valid(m) &&
(allocflags & VM_ALLOC_ZERO) != 0) { (allocflags & VM_ALLOC_ZERO) != 0) {
if ((m->flags & PG_ZERO) == 0) if ((m->flags & PG_ZERO) == 0)
pmap_zero_page(m); pmap_zero_page(m);
vm_page_valid(m); vm_page_valid(m);
} }
vm_page_grab_release(m, allocflags); vm_page_grab_release(m, allocflags);
ma[i] = mpred = m; ma[i] = mpred = m;
m = vm_page_next(m);
} }
return (i); return (i);
} }
/* /*
* Unlocked variant of vm_page_grab_pages(). This accepts the same flags * Unlocked variant of vm_page_grab_pages(). This accepts the same flags
* and will fall back to the locked variant to handle allocation. * and will fall back to the locked variant to handle allocation.
*/ */
▲ Show 20 Lines • Show All 733 Lines • Show Last 20 Lines