Page MenuHomeFreeBSD
Files F157514672

D14376.id39649.diff
No OneTemporary
Actions

D14376.id39649.diff
View Options

Index: sys/vm/swap_pager.c
===================================================================
--- sys/vm/swap_pager.c
+++ sys/vm/swap_pager.c
@@ -1097,10 +1097,10 @@
int *rahead)
{
struct buf *bp;
- vm_page_t mpred, msucc, p;
+ vm_page_t mpred, msucc;
vm_pindex_t pindex;
daddr_t blk;
- int i, j, maxahead, maxbehind, reqcount, shift;
+ int i, maxahead, maxbehind, reqcount, shift;
reqcount = count;
@@ -1136,39 +1136,27 @@
/*
* Allocate readahead and readbehind pages.
*/
- shift = rbehind != NULL ? *rbehind : 0;
- if (shift != 0) {
- for (i = 1; i <= shift; i++) {
- p = vm_page_alloc(object, ma[0]->pindex - i,
- VM_ALLOC_NORMAL);
- if (p == NULL) {
- /* Shift allocated pages to the left. */
- for (j = 0; j < i - 1; j++)
- bp->b_pages[j] =
- bp->b_pages[j + shift - i + 1];
- break;
- }
- bp->b_pages[shift - i] = p;
- }
- shift = i - 1;
- *rbehind = shift;
- }
+ if (rbehind != NULL && *rbehind > 0) {
+ shift = vm_page_alloc_pages_after(object,
+ ma[0]->pindex - *rbehind, VM_ALLOC_NORMAL, &bp->b_pages[0],
+ *rbehind, mpred);
+ if (shift != *rbehind) {
+ /* Drop a partially allocated run. */
+ for (i = 0; i < shift; i++)
+ vm_page_free(bp->b_pages[i]);
+ shift = 0;
+ } else
+ count += *rbehind;
+ } else
+ shift = 0;
for (i = 0; i < reqcount; i++)
bp->b_pages[i + shift] = ma[i];
- if (rahead != NULL) {
- for (i = 0; i < *rahead; i++) {
- p = vm_page_alloc(object,
- ma[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL);
- if (p == NULL)
- break;
- bp->b_pages[shift + reqcount + i] = p;
- }
- *rahead = i;
- }
- if (rbehind != NULL)
- count += *rbehind;
- if (rahead != NULL)
+ if (rahead != NULL && *rahead > 0) {
+ *rahead = vm_page_alloc_pages_after(object,
+ ma[reqcount - 1]->pindex + 1, VM_ALLOC_NORMAL,
+ &bp->b_pages[reqcount], *rahead, ma[reqcount - 1]);
count += *rahead;
+ }
vm_object_pip_add(object, count);
Index: sys/vm/vm_page.h
===================================================================
--- sys/vm/vm_page.h
+++ sys/vm/vm_page.h
@@ -467,6 +467,10 @@
vm_page_t vm_page_alloc_after(vm_object_t, vm_pindex_t, int, vm_page_t);
vm_page_t vm_page_alloc_domain_after(vm_object_t, vm_pindex_t, int, int,
vm_page_t);
+int vm_page_alloc_pages_after(vm_object_t, vm_pindex_t, int, vm_page_t *, int,
+ vm_page_t);
+int vm_page_alloc_pages_domain_after(vm_object_t, vm_pindex_t, int, int,
+ vm_page_t *, int, vm_page_t);
vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary, vm_memattr_t memattr);
Index: sys/vm/vm_page.c
===================================================================
--- sys/vm/vm_page.c
+++ sys/vm/vm_page.c
@@ -1639,6 +1639,41 @@
return (0);
}
+/*
+ * Determine the number of free pages that may be allocated according to
+ * the request class and request size.
+ */
+int
+vm_domain_available(struct vm_domain *vmd, int req, int npages)
+{
+ int avail;
+
+ vm_domain_free_assert_locked(vmd);
+
+ req &= VM_ALLOC_CLASS_MASK;
+
+ /*
+ * The page daemon is allowed to dig deeper into the free page list.
+ */
+ if (curproc == pageproc && req != VM_ALLOC_INTERRUPT)
+ req = VM_ALLOC_SYSTEM;
+
+ avail = vmd->vmd_free_count;
+ switch (req) {
+ case VM_ALLOC_NORMAL:
+ avail -= vmd->vmd_free_reserved;
+ break;
+ case VM_ALLOC_SYSTEM:
+ avail -= vmd->vmd_interrupt_free_min;
+ break;
+ default:
+ KASSERT(req == VM_ALLOC_INTERRUPT,
+ ("%s: invalid request type %d", __func__, req));
+ break;
+ }
+ return (imin(avail, npages));
+}
+
/*
* vm_page_alloc:
*
@@ -1706,41 +1741,13 @@
return (m);
}
-/*
- * Returns true if the number of free pages exceeds the minimum
- * for the request class and false otherwise.
- */
-int
-vm_domain_available(struct vm_domain *vmd, int req, int npages)
-{
-
- vm_domain_free_assert_locked(vmd);
- req = req & VM_ALLOC_CLASS_MASK;
-
- /*
- * The page daemon is allowed to dig deeper into the free page list.
- */
- if (curproc == pageproc && req != VM_ALLOC_INTERRUPT)
- req = VM_ALLOC_SYSTEM;
-
- if (vmd->vmd_free_count >= npages + vmd->vmd_free_reserved ||
- (req == VM_ALLOC_SYSTEM &&
- vmd->vmd_free_count >= npages + vmd->vmd_interrupt_free_min) ||
- (req == VM_ALLOC_INTERRUPT &&
- vmd->vmd_free_count >= npages))
- return (1);
-
- return (0);
-}
-
vm_page_t
vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain,
int req, vm_page_t mpred)
{
struct vm_domain *vmd;
vm_page_t m;
- int flags;
- u_int free_count;
+ u_int flags, free_count;
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&
@@ -1759,7 +1766,7 @@
m = NULL;
#if VM_NRESERVLEVEL > 0
if (vm_object_reserv(object) &&
- (m = vm_reserv_extend(req, object, pindex, domain, mpred))
+ (m = vm_reserv_extend(object, pindex, domain, req, mpred, NULL))
!= NULL) {
domain = vm_phys_domain(m);
vmd = VM_DOMAIN(domain);
@@ -1767,21 +1774,20 @@
}
#endif
vmd = VM_DOMAIN(domain);
- if (object != NULL && !vm_object_reserv(object) &&
- vmd->vmd_pgcache != NULL) {
+ if (!vm_object_reserv(object) && vmd->vmd_pgcache != NULL) {
m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT);
if (m != NULL)
goto found;
}
vm_domain_free_lock(vmd);
- if (vm_domain_available(vmd, req, 1)) {
+ if (vm_domain_available(vmd, req, 1) == 1) {
/*
* Can we allocate the page from a reservation?
*/
#if VM_NRESERVLEVEL > 0
if (!vm_object_reserv(object) ||
(m = vm_reserv_alloc_page(object, pindex,
- domain, mpred)) == NULL)
+ domain, mpred, NULL)) == NULL)
#endif
{
/*
@@ -1790,7 +1796,8 @@
m = vm_phys_alloc_pages(domain, object != NULL ?
VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
#if VM_NRESERVLEVEL > 0
- if (m == NULL && vm_reserv_reclaim_inactive(domain)) {
+ if (m == NULL &&
+ vm_reserv_reclaim_inactive(domain) != 0) {
m = vm_phys_alloc_pages(domain,
object != NULL ?
VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT,
@@ -1842,7 +1849,7 @@
m->busy_lock = VPB_SINGLE_EXCLUSIVER;
if ((req & VM_ALLOC_SBUSY) != 0)
m->busy_lock = VPB_SHARERS_WORD(1);
- if (req & VM_ALLOC_WIRED) {
+ if ((req & VM_ALLOC_WIRED) != 0) {
/*
* The page lock is not required for wiring a page until that
* page is inserted into the object.
@@ -1882,6 +1889,210 @@
return (m);
}
+int
+vm_page_alloc_pages_after(vm_object_t object, vm_pindex_t pindex, int req,
+ vm_page_t *ma, int nreq, vm_page_t mpred)
+{
+ struct vm_domainset_iter di;
+ int domain, n;
+
+ vm_domainset_iter_page_init(&di, object, &domain, &req);
+ do {
+ n = vm_page_alloc_pages_domain_after(object, pindex, domain,
+ req, ma, nreq, mpred);
+ if (n > 0)
+ break;
+ } while (vm_domainset_iter_page(&di, &domain, &req) == 0);
+
+ return (n);
+}
+
+/*
+ * vm_page_alloc_pages_after:
+ *
+ * Allocate a range of pages, contiguous in the pindex space. The
+ * number of pages actually allocated is returned and may be smaller
+ * than the number requested unless VM_ALLOC_WAITOK is specified.
+ * This function is otherwise identical to vm_page_alloc().
+ */
+int
+vm_page_alloc_pages_domain_after(vm_object_t object, vm_pindex_t pindex,
+ int domain, int req, vm_page_t *ma, int nreq, vm_page_t mpred)
+{
+ struct vm_domain *vmd;
+ vm_page_t m;
+ int avail, i, nalloc, pool;
+ u_int busy_lock, flags, free_count, oflags;
+
+ KASSERT(nreq > 0, ("invalid nreq %d", nreq));
+ KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
+ (object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&
+ ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) !=
+ (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)),
+ ("inconsistent object(%p)/req(%x)", object, req));
+ KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0,
+ ("Can't sleep and retry object insertion."));
+ KASSERT(mpred == NULL || mpred->pindex < pindex,
+ ("mpred %p doesn't precede pindex 0x%jx", mpred,
+ (uintmax_t)pindex));
+ if (object != NULL)
+ VM_OBJECT_ASSERT_WLOCKED(object);
+
+ nalloc = 0;
+ vmd = VM_DOMAIN(domain);
+#if VM_NRESERVLEVEL > 0
+ if (vm_object_reserv(object) &&
+ (m = vm_reserv_extend(object, pindex, domain, req, mpred,
+ &nalloc)) != NULL) {
+ domain = vm_phys_domain(m);
+ vmd = VM_DOMAIN(domain);
+ for (i = 0; i < nalloc; i++)
+ ma[i] = m++;
+ if (nreq == nalloc)
+ goto done;
+ }
+#endif
+
+again:
+ vm_domain_free_lock(vmd);
+ if ((avail = vm_domain_available(vmd, req, nreq)) > 0) {
+ /*
+ * Can we allocate the pages from a reservation?
+ */
+#if VM_NRESERVLEVEL > 0
+ if (!vm_object_reserv(object) ||
+ (m = vm_reserv_alloc_page(object, pindex,
+ domain, mpred, &avail)) == NULL)
+#endif
+ {
+ pool = object != NULL ? VM_FREEPOOL_DEFAULT :
+ VM_FREEPOOL_DIRECT;
+
+ /*
+ * If not, allocate them from the free page queues.
+ */
+ do {
+ i = vm_phys_alloc_npages(domain, pool, &m,
+ imin(avail, nreq - nalloc));
+ if (i == 0)
+ /* avail < nreq - nalloc */
+ break;
+ avail -= i;
+ for (; i > 0; i--)
+ ma[nalloc++] = m++;
+ } while (nalloc < nreq);
+#if VM_NRESERVLEVEL > 0
+ if ((nalloc == 0 ||
+ (nalloc < nreq && (req & VM_ALLOC_WAITOK) != 0)) &&
+ (avail = vm_reserv_reclaim_inactive(domain)) != 0) {
+ do {
+ i = vm_phys_alloc_npages(domain, pool,
+ &m, imin(avail, nreq - nalloc));
+ if (i == 0)
+ break;
+ avail -= i;
+ for (; i > 0; i--)
+ ma[nalloc++] = m++;
+ } while (nalloc < nreq);
+ }
+#endif
+ }
+#if VM_NRESERVLEVEL > 0
+ else {
+ for (i = 0; i < avail; i++)
+ ma[nalloc++] = m++;
+ }
+#endif
+ }
+ if (nalloc == 0 || (nalloc < nreq && (req & VM_ALLOC_WAITOK) != 0)) {
+ /*
+ * We failed to allocate a page, or the caller requested a
+ * blocking allocation and we weren't able to scrounge enough
+ * pages in the latest attempt.
+ */
+ if (vm_domain_alloc_fail(vmd, object, req))
+ goto again;
+ return (0);
+ }
+
+ free_count = vm_domain_freecnt_adj(vmd, -nalloc);
+ vm_domain_free_unlock(vmd);
+
+ /*
+ * Don't wakeup too often - wakeup the pageout daemon when
+ * we would be nearly out of memory.
+ */
+ if (vm_paging_needed(vmd, free_count))
+ pagedaemon_wakeup(vmd->vmd_domain);
+
+done:
+ /*
+ * Initialize the pages. Only the PG_ZERO flag is inherited.
+ */
+ flags = 0;
+ if ((req & VM_ALLOC_ZERO) != 0)
+ flags |= PG_ZERO;
+ if ((req & VM_ALLOC_NODUMP) != 0)
+ flags |= PG_NODUMP;
+ oflags = (object == NULL || (object->flags & OBJ_UNMANAGED) != 0) ?
+ VPO_UNMANAGED : 0;
+ busy_lock = VPB_UNBUSIED;
+ if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0)
+ busy_lock = VPB_SINGLE_EXCLUSIVER;
+ if ((req & VM_ALLOC_SBUSY) != 0)
+ busy_lock = VPB_SHARERS_WORD(1);
+
+ for (i = 0; i < nalloc; i++) {
+ m = ma[i];
+
+ m->flags = (m->flags | PG_NODUMP) & flags;
+ m->aflags = 0;
+ m->oflags = oflags;
+ m->busy_lock = busy_lock;
+ if ((req & VM_ALLOC_WIRED) != 0) {
+ /*
+ * The page lock is not required for wiring a page
+ * until that page is inserted into the object.
+ */
+ m->wire_count = 1;
+ }
+ m->act_count = 0;
+
+ if (object != NULL) {
+ if (vm_page_insert_after(m, object, pindex + i,
+ mpred)) {
+ m->wire_count = 0;
+ avail = i;
+ for (; i < nalloc; i++) {
+ m = ma[i];
+ KASSERT(m->object == NULL,
+ ("page %p has object", m));
+ /* Don't change PG_ZERO. */
+ vm_page_free_toq(m);
+ }
+ if ((req & VM_ALLOC_WAITFAIL) != 0) {
+ VM_OBJECT_WUNLOCK(object);
+ vm_radix_wait();
+ VM_OBJECT_WLOCK(object);
+ }
+ nalloc = avail;
+ break;
+ }
+
+ /* Ignore device objects; the pager sets "memattr" for them. */
+ if (object->memattr != VM_MEMATTR_DEFAULT &&
+ (object->flags & OBJ_FICTITIOUS) == 0)
+ pmap_page_set_memattr(m, object->memattr);
+ } else
+ m->pindex = pindex + i;
+ mpred = m;
+ }
+ if ((req & VM_ALLOC_WIRED) != 0)
+ VM_CNT_ADD(v_wire_count, nalloc);
+
+ return (nalloc);
+}
+
/*
* vm_page_alloc_contig:
*
@@ -1990,7 +2201,7 @@
m_ret = NULL;
vmd = VM_DOMAIN(domain);
vm_domain_free_lock(vmd);
- if (vm_domain_available(vmd, req, npages)) {
+ if (vm_domain_available(vmd, req, npages) == npages) {
/*
* Can we allocate the pages from a reservation?
*/
@@ -2158,7 +2369,7 @@
vmd = VM_DOMAIN(domain);
again:
vm_domain_free_lock(vmd);
- if (vm_domain_available(vmd, req, 1))
+ if (vm_domain_available(vmd, req, 1) == 1)
m = vm_phys_alloc_freelist_pages(domain, freelist,
VM_FREEPOOL_DIRECT, 0);
if (m == NULL) {
@@ -2205,7 +2416,7 @@
n = 64; /* Starting stride. */
vm_domain_free_lock(vmd);
for (i = 0; i < cnt; i+=n) {
- if (!vm_domain_available(vmd, VM_ALLOC_NORMAL, n))
+ if (vm_domain_available(vmd, VM_ALLOC_NORMAL, n) != n)
break;
n = vm_phys_alloc_npages(domain, VM_FREELIST_DEFAULT, &m,
MIN(n, cnt-i));
@@ -3249,14 +3460,14 @@
if ((m->oflags & VPO_UNMANAGED) == 0) {
vm_page_lock_assert(m, MA_OWNED);
KASSERT(!pmap_page_is_mapped(m),
- ("vm_page_free_toq: freeing mapped page %p", m));
+ ("vm_page_free_prep: freeing mapped page %p", m));
} else
KASSERT(m->queue == PQ_NONE,
- ("vm_page_free_toq: unmanaged page %p is queued", m));
+ ("vm_page_free_prep: unmanaged page %p is queued", m));
VM_CNT_INC(v_tfree);
if (vm_page_sbusied(m))
- panic("vm_page_free: freeing busy page %p", m);
+ panic("vm_page_free_prep: freeing busy page %p", m);
vm_page_remove(m);
@@ -3282,11 +3493,11 @@
vm_page_undirty(m);
if (m->wire_count != 0)
- panic("vm_page_free: freeing wired page %p", m);
+ panic("vm_page_free_prep: freeing wired page %p", m);
if (m->hold_count != 0) {
m->flags &= ~PG_ZERO;
KASSERT((m->flags & PG_UNHOLDFREE) == 0,
- ("vm_page_free: freeing PG_UNHOLDFREE page %p", m));
+ ("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m));
m->flags |= PG_UNHOLDFREE;
return (false);
}
@@ -3764,9 +3975,8 @@
vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
vm_page_t *ma, int count)
{
- vm_page_t m, mpred;
- int pflags;
- int i;
+ vm_page_t m, mpred, msucc;
+ int i, pflags, run;
bool sleep;
VM_OBJECT_ASSERT_WLOCKED(object);
@@ -3778,6 +3988,7 @@
KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 ||
(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch"));
+
if (count == 0)
return (0);
pflags = allocflags & ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK |
@@ -3789,10 +4000,14 @@
m = vm_radix_lookup_le(&object->rtree, pindex + i);
if (m == NULL || m->pindex != pindex + i) {
mpred = m;
+ msucc = mpred != NULL ? TAILQ_NEXT(mpred, listq) :
+ TAILQ_FIRST(&object->memq);
m = NULL;
- } else
+ } else {
mpred = TAILQ_PREV(m, pglist, listq);
- for (; i < count; i++) {
+ msucc = TAILQ_NEXT(m, listq);
+ }
+ while (i < count) {
if (m != NULL) {
sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ?
vm_page_xbusied(m) : vm_page_busied(m);
@@ -3822,21 +4037,41 @@
vm_page_xbusy(m);
if ((allocflags & VM_ALLOC_SBUSY) != 0)
vm_page_sbusy(m);
+ if (m->valid == 0 &&
+ (allocflags & VM_ALLOC_ZERO) != 0) {
+ if ((m->flags & PG_ZERO) == 0)
+ pmap_zero_page(m);
+ m->valid = VM_PAGE_BITS_ALL;
+ }
+ ma[i++] = m;
} else {
- m = vm_page_alloc_after(object, pindex + i,
- pflags | VM_ALLOC_COUNT(count - i), mpred);
- if (m == NULL) {
+ /*
+ * Try to allocate multiple consecutive pages. Use the
+ * succeeding page, if any, to bound the length of the
+ * requested run.
+ */
+ run = msucc == NULL || msucc->pindex >= pindex + count ?
+ count - i : msucc->pindex - (pindex + i);
+ run = vm_page_alloc_pages_after(object, pindex + i,
+ pflags | VM_ALLOC_COUNT(run), ma + i, run, mpred);
+ if (run == 0) {
if ((allocflags & VM_ALLOC_NOWAIT) != 0)
break;
goto retrylookup;
}
+ if ((allocflags & VM_ALLOC_ZERO) != 0) {
+ for (; run != 0; run--, i++) {
+ m = ma[i];
+ if ((m->flags & PG_ZERO) == 0)
+ pmap_zero_page(m);
+ m->valid = VM_PAGE_BITS_ALL;
+ }
+ } else
+ i += run;
+ m = ma[i - 1];
}
- if (m->valid == 0 && (allocflags & VM_ALLOC_ZERO) != 0) {
- if ((m->flags & PG_ZERO) == 0)
- pmap_zero_page(m);
- m->valid = VM_PAGE_BITS_ALL;
- }
- ma[i] = mpred = m;
+ mpred = m;
+ msucc = TAILQ_NEXT(m, listq);
m = vm_page_next(m);
}
return (i);
Index: sys/vm/vm_reserv.h
===================================================================
--- sys/vm/vm_reserv.h
+++ sys/vm/vm_reserv.h
@@ -55,9 +55,9 @@
vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary, vm_page_t mpred);
vm_page_t vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex,
- int domain, vm_page_t mpred);
-vm_page_t vm_reserv_extend(int req, vm_object_t object,
- vm_pindex_t pindex, int domain, vm_page_t mpred);
+ int domain, vm_page_t mpred, int *countp);
+vm_page_t vm_reserv_extend(vm_object_t object, vm_pindex_t pindex,
+ int domain, int req, vm_page_t mpred, int *countp);
void vm_reserv_break_all(vm_object_t object);
boolean_t vm_reserv_free_page(vm_page_t m);
void vm_reserv_init(void);
@@ -67,7 +67,7 @@
boolean_t vm_reserv_reclaim_contig(int domain, u_long npages,
vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary);
-boolean_t vm_reserv_reclaim_inactive(int domain);
+int vm_reserv_reclaim_inactive(int domain);
void vm_reserv_rename(vm_page_t m, vm_object_t new_object,
vm_object_t old_object, vm_pindex_t old_object_offset);
int vm_reserv_size(int level);
Index: sys/vm/vm_reserv.c
===================================================================
--- sys/vm/vm_reserv.c
+++ sys/vm/vm_reserv.c
@@ -578,7 +578,8 @@
domain = rv->domain;
vmd = VM_DOMAIN(domain);
vm_domain_free_lock(vmd);
- if (rv->object != object || !vm_domain_available(vmd, req, npages)) {
+ if (rv->object != object ||
+ vm_domain_available(vmd, req, npages) != npages) {
m = NULL;
goto out;
}
@@ -776,8 +777,10 @@
}
/*
- * Attempts to extend an existing reservation and allocate the page to the
- * object.
+ * Attempts to extend an existing reservation and allocate the request page to
+ * the object. Opportunistically returns up to "*countp" contiguous pages if
+ * the caller so requests. The number of pages allocated is returned in
+ * "*countp".
*
* The page "mpred" must immediately precede the offset "pindex" within the
* specified object.
@@ -785,13 +788,13 @@
* The object must be locked.
*/
vm_page_t
-vm_reserv_extend(int req, vm_object_t object, vm_pindex_t pindex, int domain,
- vm_page_t mpred)
+vm_reserv_extend(vm_object_t object, vm_pindex_t pindex, int domain, int req,
+ vm_page_t mpred, int *countp)
{
struct vm_domain *vmd;
vm_page_t m, msucc;
vm_reserv_t rv;
- int index, free_count;
+ int avail, free_count, index, nalloc;
VM_OBJECT_ASSERT_WLOCKED(object);
@@ -816,17 +819,28 @@
index = VM_RESERV_INDEX(object, pindex);
m = &rv->pages[index];
vm_domain_free_lock(vmd);
- if (vm_domain_available(vmd, req, 1) == 0 ||
+ if ((avail = vm_domain_available(vmd, req, VM_LEVEL_0_NPAGES)) == 0 ||
/* Handle reclaim race. */
rv->object != object ||
/* Handle vm_page_rename(m, new_object, ...). */
- popmap_is_set(rv->popmap, index))
+ popmap_is_set(rv->popmap, index)) {
m = NULL;
- if (m != NULL) {
- vm_reserv_populate(rv, index);
- free_count = vm_domain_freecnt_adj(vmd, -1);
- } else
free_count = vmd->vmd_free_count;
+ } else {
+ vm_reserv_populate(rv, index);
+ nalloc = 1;
+ if (countp != NULL) {
+ avail = imin(imin(avail, *countp),
+ VM_LEVEL_0_NPAGES - index);
+ for (; nalloc < avail; index++, nalloc++) {
+ if (popmap_is_set(rv->popmap, index))
+ break;
+ vm_reserv_populate(rv, index);
+ }
+ *countp = nalloc;
+ }
+ free_count = vm_domain_freecnt_adj(vmd, -nalloc);
+ }
vm_domain_free_unlock(vmd);
if (vm_paging_needed(vmd, free_count))
@@ -836,16 +850,19 @@
}
/*
- * Allocates a page from an existing reservation.
+ * Allocates a new reservation for the object, and returns a page from that
+ * reservation. Opportunistically returns up to *"countp" contiguous pages if
+ * the caller so requests. The number of pages allocated is returned in
+ * "*countp".
*
* The page "mpred" must immediately precede the offset "pindex" within the
* specified object.
*
- * The object and free page queue must be locked.
+ * The object and per-domain free page queues must be locked.
*/
vm_page_t
vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain,
- vm_page_t mpred)
+ vm_page_t mpred, int *countp)
{
vm_page_t m, msucc;
vm_pindex_t first, leftcap, rightcap;
@@ -930,7 +947,11 @@
vm_reserv_insert(rv, object, first);
index = VM_RESERV_INDEX(object, pindex);
vm_reserv_populate(rv, index);
- return (&rv->pages[index]);
+ m = &rv->pages[index];
+ if (countp != NULL)
+ for (; *countp > 1 && index < VM_LEVEL_0_NPAGES; (*countp)--)
+ vm_reserv_populate(rv, ++index);
+ return (m);
}
/*
@@ -1149,22 +1170,24 @@
/*
* Breaks the reservation at the head of the partially populated reservation
- * queue, releasing its free pages to the physical memory allocator. Returns
- * TRUE if a reservation is broken and FALSE otherwise.
+ * queue, releasing its free pages to the physical memory allocator, and
+ * returns the number of pages released.
*
* The free page queue lock must be held.
*/
-boolean_t
+int
vm_reserv_reclaim_inactive(int domain)
{
vm_reserv_t rv;
+ int freed;
vm_domain_free_assert_locked(VM_DOMAIN(domain));
if ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) {
+ freed = VM_LEVEL_0_NPAGES - rv->popcnt;
vm_reserv_reclaim(rv);
- return (TRUE);
+ return (freed);
}
- return (FALSE);
+ return (0);
}
/*
Index: sys/vm/vnode_pager.c
===================================================================
--- sys/vm/vnode_pager.c
+++ sys/vm/vnode_pager.c
@@ -897,35 +897,27 @@
/*
* Fill in the bp->b_pages[] array with requested and optional
- * read behind or read ahead pages. Read behind pages are looked
- * up in a backward direction, down to a first cached page. Same
- * for read ahead pages, but there is no need to shift the array
- * in case of encountering a cached page.
+ * read behind or read ahead pages.
*/
i = bp->b_npages = 0;
- if (rbehind) {
- vm_pindex_t startpindex, tpindex;
- vm_page_t p;
+ if (rbehind > 0) {
+ vm_pindex_t startpindex;
+ vm_page_t mpred;
VM_OBJECT_WLOCK(object);
startpindex = m[0]->pindex - rbehind;
- if ((p = TAILQ_PREV(m[0], pglist, listq)) != NULL &&
- p->pindex >= startpindex)
- startpindex = p->pindex + 1;
-
- /* tpindex is unsigned; beware of numeric underflow. */
- for (tpindex = m[0]->pindex - 1;
- tpindex >= startpindex && tpindex < m[0]->pindex;
- tpindex--, i++) {
- p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
- if (p == NULL) {
- /* Shift the array. */
- for (int j = 0; j < i; j++)
- bp->b_pages[j] = bp->b_pages[j +
- tpindex + 1 - startpindex];
- break;
- }
- bp->b_pages[tpindex - startpindex] = p;
+ if ((mpred = TAILQ_PREV(m[0], pglist, listq)) != NULL &&
+ mpred->pindex >= startpindex)
+ startpindex = mpred->pindex + 1;
+
+ i = vm_page_alloc_pages_after(object, startpindex,
+ VM_ALLOC_NORMAL, &bp->b_pages[0],
+ m[0]->pindex - startpindex, mpred);
+ if (i < m[0]->pindex - startpindex) {
+ /* We have to drop the partially allocated run. */
+ for (int j = 0; j < i; j++)
+ vm_page_free(bp->b_pages[j]);
+ i = 0;
}
bp->b_pgbefore = i;
@@ -939,29 +931,24 @@
bp->b_pages[i] = m[j];
bp->b_npages += count;
- if (rahead) {
- vm_pindex_t endpindex, tpindex;
- vm_page_t p;
+ if (rahead > 0) {
+ vm_pindex_t endpindex, startpindex;
+ vm_page_t msucc;
if (!VM_OBJECT_WOWNED(object))
VM_OBJECT_WLOCK(object);
- endpindex = m[count - 1]->pindex + rahead + 1;
- if ((p = TAILQ_NEXT(m[count - 1], listq)) != NULL &&
- p->pindex < endpindex)
- endpindex = p->pindex;
+ startpindex = m[count - 1]->pindex + 1;
+ endpindex = startpindex + rahead;
+ if ((msucc = TAILQ_NEXT(m[count - 1], listq)) != NULL &&
+ msucc->pindex < endpindex)
+ endpindex = msucc->pindex;
if (endpindex > object->size)
endpindex = object->size;
- for (tpindex = m[count - 1]->pindex + 1;
- tpindex < endpindex; i++, tpindex++) {
- p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
- if (p == NULL)
- break;
- bp->b_pages[i] = p;
- }
-
- bp->b_pgafter = i - bp->b_npages;
- bp->b_npages = i;
+ bp->b_pgafter = vm_page_alloc_pages_after(object, startpindex,
+ VM_ALLOC_NORMAL, &bp->b_pages[i], endpindex - startpindex,
+ m[count - 1]);
+ bp->b_npages += bp->b_pgafter;
} else
bp->b_pgafter = 0;

File Metadata

Mime Type
text/plain
Expires
Sat, May 23, 6:58 AM (15 h, 55 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
33442883
Default Alt Text
D14376.id39649.diff (24 KB)

Event Timeline