Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -1093,14 +1093,14 @@ * The pages in "m" must be busied and will remain busied upon return. */ static int -swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, +swap_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int *rbehind, int *rahead) { struct buf *bp; - vm_page_t mpred, msucc, p; + vm_page_t mpred, msucc; vm_pindex_t pindex; daddr_t blk; - int i, j, maxahead, maxbehind, reqcount, shift; + int i, maxahead, maxbehind, reqcount, shift; reqcount = count; @@ -1108,7 +1108,7 @@ bp = getpbuf(&nsw_rcount); VM_OBJECT_WLOCK(object); - if (!swap_pager_haspage(object, m[0]->pindex, &maxbehind, &maxahead)) { + if (!swap_pager_haspage(object, ma[0]->pindex, &maxbehind, &maxahead)) { relpbuf(bp, &nsw_rcount); return (VM_PAGER_FAIL); } @@ -1120,15 +1120,15 @@ KASSERT(reqcount - 1 <= maxahead, ("page count %d extends beyond swap block", reqcount)); *rahead = imin(*rahead, maxahead - (reqcount - 1)); - pindex = m[reqcount - 1]->pindex; - msucc = TAILQ_NEXT(m[reqcount - 1], listq); + pindex = ma[reqcount - 1]->pindex; + msucc = TAILQ_NEXT(ma[reqcount - 1], listq); if (msucc != NULL && msucc->pindex - pindex - 1 < *rahead) *rahead = msucc->pindex - pindex - 1; } if (rbehind != NULL) { *rbehind = imin(*rbehind, maxbehind); - pindex = m[0]->pindex; - mpred = TAILQ_PREV(m[0], pglist, listq); + pindex = ma[0]->pindex; + mpred = TAILQ_PREV(ma[0], pglist, listq); if (mpred != NULL && pindex - mpred->pindex - 1 < *rbehind) *rbehind = pindex - mpred->pindex - 1; } @@ -1136,39 +1136,26 @@ /* * Allocate readahead and readbehind pages. */ - shift = rbehind != NULL ? *rbehind : 0; - if (shift != 0) { - for (i = 1; i <= shift; i++) { - p = vm_page_alloc(object, m[0]->pindex - i, - VM_ALLOC_NORMAL); - if (p == NULL) { - /* Shift allocated pages to the left. */ - for (j = 0; j < i - 1; j++) - bp->b_pages[j] = - bp->b_pages[j + shift - i + 1]; - break; - } - bp->b_pages[shift - i] = p; - } - shift = i - 1; - *rbehind = shift; + if (rbehind != NULL) { + shift = vm_page_alloc_pages_after(object, + ma[0]->pindex - *rbehind, VM_ALLOC_NORMAL, &bp->b_pages[0], + *rbehind, mpred); + if (shift != *rbehind) { + /* Drop a partially allocated run. */ + for (i = 0; i < shift; i++) + vm_page_free(bp->b_pages[i]); + shift = 0; + } else + count += *rbehind; } for (i = 0; i < reqcount; i++) - bp->b_pages[i + shift] = m[i]; + bp->b_pages[i + shift] = ma[i]; if (rahead != NULL) { - for (i = 0; i < *rahead; i++) { - p = vm_page_alloc(object, - m[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL); - if (p == NULL) - break; - bp->b_pages[shift + reqcount + i] = p; - } - *rahead = i; - } - if (rbehind != NULL) - count += *rbehind; - if (rahead != NULL) + *rahead = vm_page_alloc_pages_after(object, + ma[reqcount - 1]->pindex + 1, VM_ALLOC_NORMAL, + &bp->b_pages[reqcount], *rahead, ma[reqcount - 1]); count += *rahead; + } vm_object_pip_add(object, count); @@ -1203,7 +1190,7 @@ * Instead, we look at the one page we are interested in which we * still hold a lock on even through the I/O completion. * - * The other pages in our m[] array are also released on completion, + * The other pages in our ma[] array are also released on completion, * so we cannot assume they are valid anymore either. * * NOTE: b_blkno is destroyed by the call to swapdev_strategy @@ -1217,8 +1204,8 @@ * is set in the metadata for each page in the request. */ VM_OBJECT_WLOCK(object); - while ((m[0]->oflags & VPO_SWAPINPROG) != 0) { - m[0]->oflags |= VPO_SWAPSLEEP; + while ((ma[0]->oflags & VPO_SWAPINPROG) != 0) { + ma[0]->oflags |= VPO_SWAPSLEEP; VM_CNT_INC(v_intrans); if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP, "swread", hz * 20)) { @@ -1232,7 +1219,7 @@ * If we had an unrecoverable read error pages will not be valid. */ for (i = 0; i < reqcount; i++) - if (m[i]->valid != VM_PAGE_BITS_ALL) + if (ma[i]->valid != VM_PAGE_BITS_ALL) return (VM_PAGER_ERROR); return (VM_PAGER_OK); Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -467,6 +467,10 @@ vm_page_t vm_page_alloc_after(vm_object_t, vm_pindex_t, int, vm_page_t); vm_page_t vm_page_alloc_domain_after(vm_object_t, vm_pindex_t, int, int, vm_page_t); +int vm_page_alloc_pages_after(vm_object_t, vm_pindex_t, int, vm_page_t *, int, + vm_page_t); +int vm_page_alloc_pages_domain_after(vm_object_t, vm_pindex_t, int, int, + vm_page_t *, int, vm_page_t); vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -1638,6 +1638,34 @@ return (0); } +/* + * Returns true if the number of free pages exceeds the minimum + * for the request class and false otherwise. + */ +int +vm_domain_available(struct vm_domain *vmd, int req, int npages) +{ + int d; + + vm_domain_free_assert_locked(vmd); + req = req & VM_ALLOC_CLASS_MASK; + + /* + * The page daemon is allowed to dig deeper into the free page list. + */ + if (curproc == pageproc && req != VM_ALLOC_INTERRUPT) + req = VM_ALLOC_SYSTEM; + + if ((d = vmd->vmd_free_count - npages - vmd->vmd_free_reserved) > 0 || + (req == VM_ALLOC_SYSTEM && + (d = vmd->vmd_free_count - npages - vmd->vmd_interrupt_free_min) > 0) || + (req == VM_ALLOC_INTERRUPT && + (d = vmd->vmd_free_count - npages) > 0)) + return (npages); + + return (d); +} + /* * vm_page_alloc: * @@ -1705,41 +1733,13 @@ return (m); } -/* - * Returns true if the number of free pages exceeds the minimum - * for the request class and false otherwise. - */ -int -vm_domain_available(struct vm_domain *vmd, int req, int npages) -{ - - vm_domain_free_assert_locked(vmd); - req = req & VM_ALLOC_CLASS_MASK; - - /* - * The page daemon is allowed to dig deeper into the free page list. - */ - if (curproc == pageproc && req != VM_ALLOC_INTERRUPT) - req = VM_ALLOC_SYSTEM; - - if (vmd->vmd_free_count >= npages + vmd->vmd_free_reserved || - (req == VM_ALLOC_SYSTEM && - vmd->vmd_free_count >= npages + vmd->vmd_interrupt_free_min) || - (req == VM_ALLOC_INTERRUPT && - vmd->vmd_free_count >= npages)) - return (1); - - return (0); -} - vm_page_t vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain, int req, vm_page_t mpred) { struct vm_domain *vmd; vm_page_t m; - int flags; - u_int free_count; + u_int flags, free_count; KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && @@ -1758,7 +1758,7 @@ m = NULL; #if VM_NRESERVLEVEL > 0 if (vm_object_reserv(object) && - (m = vm_reserv_extend(req, object, pindex, domain, mpred)) + (m = vm_reserv_extend(object, pindex, domain, req, mpred, NULL)) != NULL) { domain = vm_phys_domain(m); vmd = VM_DOMAIN(domain); @@ -1766,21 +1766,20 @@ } #endif vmd = VM_DOMAIN(domain); - if (object != NULL && !vm_object_reserv(object) && - vmd->vmd_pgcache != NULL) { + if (!vm_object_reserv(object) && vmd->vmd_pgcache != NULL) { m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT); if (m != NULL) goto found; } vm_domain_free_lock(vmd); - if (vm_domain_available(vmd, req, 1)) { + if (vm_domain_available(vmd, req, 1) == 1) { /* * Can we allocate the page from a reservation? */ #if VM_NRESERVLEVEL > 0 if (!vm_object_reserv(object) || (m = vm_reserv_alloc_page(object, pindex, - domain, mpred)) == NULL) + domain, mpred, NULL)) == NULL) #endif { /* @@ -1789,7 +1788,8 @@ m = vm_phys_alloc_pages(domain, object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); #if VM_NRESERVLEVEL > 0 - if (m == NULL && vm_reserv_reclaim_inactive(domain)) { + if (m == NULL && + vm_reserv_reclaim_inactive(domain) != 0) { m = vm_phys_alloc_pages(domain, object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, @@ -1841,7 +1841,7 @@ m->busy_lock = VPB_SINGLE_EXCLUSIVER; if ((req & VM_ALLOC_SBUSY) != 0) m->busy_lock = VPB_SHARERS_WORD(1); - if (req & VM_ALLOC_WIRED) { + if ((req & VM_ALLOC_WIRED) != 0) { /* * The page lock is not required for wiring a page until that * page is inserted into the object. @@ -1881,6 +1881,215 @@ return (m); } +int +vm_page_alloc_pages_after(vm_object_t object, vm_pindex_t pindex, int req, + vm_page_t *ma, int nreq, vm_page_t mpred) +{ + struct vm_domainset_iter di; + int domain, n; + + vm_domainset_iter_page_init(&di, object, &domain, &req); + do { + n = vm_page_alloc_pages_domain_after(object, pindex, domain, + req, ma, nreq, mpred); + if (n > 0) + break; + } while (vm_domainset_iter_page(&di, &domain, &req) == 0); + + return (n); +} + +/* + * vm_page_alloc_pages_after: + * + * Allocate a range of pages, contiguous in the pindex space. The + * number of pages actually allocated is returned and may be smaller + * than the number requested unless VM_ALLOC_WAITOK is specified. + * This function is otherwise identical to vm_page_alloc(). + */ +int +vm_page_alloc_pages_domain_after(vm_object_t object, vm_pindex_t pindex, + int domain, int req, vm_page_t *ma, int nreq, vm_page_t mpred) +{ + struct vm_domain *vmd; + vm_page_t m; + int avail, i, nalloc, pool; + u_int busy_lock, flags, free_count, oflags; + + KASSERT(nreq > 0, ("invalid nreq %d", nreq)); + KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && + (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && + ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != + (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), + ("inconsistent object(%p)/req(%x)", object, req)); + KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, + ("Can't sleep and retry object insertion.")); + KASSERT(mpred == NULL || mpred->pindex < pindex, + ("mpred %p doesn't precede pindex 0x%jx", mpred, + (uintmax_t)pindex)); + if (object != NULL) + VM_OBJECT_ASSERT_WLOCKED(object); + + nalloc = 0; + vmd = VM_DOMAIN(domain); +#if VM_NRESERVLEVEL > 0 + if (vm_object_reserv(object)) { + if ((m = vm_reserv_extend(object, pindex, domain, req, mpred, + &nalloc)) != NULL) { + domain = vm_phys_domain(m); + vmd = VM_DOMAIN(domain); + for (i = 0; i < nalloc; i++) + ma[i] = m++; + if (nreq == nalloc) + goto done; + } + } +#endif + +again: + vm_domain_free_lock(vmd); + if ((avail = vm_domain_available(vmd, req, nreq)) > 0) { + /* + * Can we allocate the pages from a reservation? + */ +#if VM_NRESERVLEVEL > 0 + if (!vm_object_reserv(object) || + (m = vm_reserv_alloc_page(object, pindex, + domain, mpred, &avail)) == NULL) +#endif + { + pool = object != NULL ? VM_FREEPOOL_DEFAULT : + VM_FREEPOOL_DIRECT; + + /* + * If not, allocate them from the free page queues. + */ + do { + i = vm_phys_alloc_npages(domain, pool, &m, + imin(avail, nreq - nalloc)); + if (i == 0) + /* avail < nreq - nalloc */ + break; + avail -= i; + for (; i > 0; i--) + ma[nalloc++] = m++; + } while (nalloc < nreq); +#if VM_NRESERVLEVEL > 0 + if ((nalloc == 0 || + (nalloc < nreq && (req & VM_ALLOC_WAITOK) != 0)) && + (avail = vm_reserv_reclaim_inactive(domain)) != 0) { + do { + i = vm_phys_alloc_npages(domain, pool, + &m, imin(avail, nreq - nalloc)); + if (i == 0) + break; + avail -= i; + for (; i > 0; i--) + ma[nalloc++] = m++; + } while (nalloc < nreq); + } +#endif + } +#if VM_NRESERVLEVEL > 0 + else { + for (i = 0; i < avail; i++) + ma[nalloc++] = m++; + } +#endif + } + if (nalloc == 0 || (nalloc < nreq && (req & VM_ALLOC_WAITOK) != 0)) { + /* + * We failed to allocate a page, or the caller requested a + * blocking allocation and we weren't able to scrounge enough + * pages in the latest attempt. + */ + if (vm_domain_alloc_fail(vmd, object, req)) + goto again; + return (0); + } + + free_count = vm_domain_freecnt_adj(vmd, -nalloc); + vm_domain_free_unlock(vmd); + + /* + * Don't wakeup too often - wakeup the pageout daemon when + * we would be nearly out of memory. + */ + if (vm_paging_needed(vmd, free_count)) + pagedaemon_wakeup(vmd->vmd_domain); + +done: + /* + * Initialize the pages. Only the PG_ZERO flag is inherited. + */ + flags = 0; + if ((req & VM_ALLOC_ZERO) != 0) + flags |= PG_ZERO; + if ((req & VM_ALLOC_NODUMP) != 0) + flags |= PG_NODUMP; + oflags = (object == NULL || (object->flags & OBJ_UNMANAGED) != 0) ? + VPO_UNMANAGED : 0; + busy_lock = VPB_UNBUSIED; + if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) + busy_lock = VPB_SINGLE_EXCLUSIVER; + if ((req & VM_ALLOC_SBUSY) != 0) + busy_lock = VPB_SHARERS_WORD(1); + + for (i = 0; i < nalloc; i++) { + m = ma[i]; + + m->flags = (m->flags | PG_NODUMP) & flags; + m->aflags = 0; + m->oflags = oflags; + m->busy_lock = busy_lock; + if ((req & VM_ALLOC_WIRED) != 0) { + /* + * The page lock is not required for wiring a page + * until that page is inserted into the object. + */ + VM_CNT_ADD(v_wire_count, 1); + m->wire_count = 1; + } + m->act_count = 0; + + if (object != NULL) { + if (vm_page_insert_after(m, object, pindex + i, + mpred)) { + avail = i; + for (; i < nalloc; i++) { + m = ma[i]; + if ((req & VM_ALLOC_WIRED) != 0) { + VM_CNT_ADD(v_wire_count, -1); + m->wire_count = 0; + } + KASSERT(m->object == NULL, + ("page %p has object", m)); + m->oflags = VPO_UNMANAGED; + m->busy_lock = VPB_UNBUSIED; + /* Don't change PG_ZERO. */ + vm_page_free_toq(m); + } + if ((req & VM_ALLOC_WAITFAIL) != 0) { + VM_OBJECT_WUNLOCK(object); + vm_radix_wait(); + VM_OBJECT_WLOCK(object); + } + nalloc = avail; + break; + } + + /* Ignore device objects; the pager sets "memattr" for them. */ + if (object->memattr != VM_MEMATTR_DEFAULT && + (object->flags & OBJ_FICTITIOUS) == 0) + pmap_page_set_memattr(m, object->memattr); + } else + m->pindex = pindex + i; + mpred = m; + } + + return (nalloc); +} + /* * vm_page_alloc_contig: * @@ -1989,7 +2198,7 @@ m_ret = NULL; vmd = VM_DOMAIN(domain); vm_domain_free_lock(vmd); - if (vm_domain_available(vmd, req, npages)) { + if (vm_domain_available(vmd, req, npages) == npages) { /* * Can we allocate the pages from a reservation? */ @@ -2157,7 +2366,7 @@ vmd = VM_DOMAIN(domain); again: vm_domain_free_lock(vmd); - if (vm_domain_available(vmd, req, 1)) + if (vm_domain_available(vmd, req, 1) == 1) m = vm_phys_alloc_freelist_pages(domain, freelist, VM_FREEPOOL_DIRECT, 0); if (m == NULL) { @@ -2204,7 +2413,7 @@ n = 64; /* Starting stride. */ vm_domain_free_lock(vmd); for (i = 0; i < cnt; i+=n) { - if (!vm_domain_available(vmd, VM_ALLOC_NORMAL, n)) + if (vm_domain_available(vmd, VM_ALLOC_NORMAL, n) != n) break; n = vm_phys_alloc_npages(domain, VM_FREELIST_DEFAULT, &m, MIN(n, cnt-i)); @@ -3248,14 +3457,14 @@ if ((m->oflags & VPO_UNMANAGED) == 0) { vm_page_lock_assert(m, MA_OWNED); KASSERT(!pmap_page_is_mapped(m), - ("vm_page_free_toq: freeing mapped page %p", m)); + ("vm_page_free_prep: freeing mapped page %p", m)); } else KASSERT(m->queue == PQ_NONE, - ("vm_page_free_toq: unmanaged page %p is queued", m)); + ("vm_page_free_prep: unmanaged page %p is queued", m)); VM_CNT_INC(v_tfree); if (vm_page_sbusied(m)) - panic("vm_page_free: freeing busy page %p", m); + panic("vm_page_free_prep: freeing busy page %p", m); vm_page_remove(m); @@ -3281,11 +3490,11 @@ vm_page_undirty(m); if (m->wire_count != 0) - panic("vm_page_free: freeing wired page %p", m); + panic("vm_page_free_prep: freeing wired page %p", m); if (m->hold_count != 0) { m->flags &= ~PG_ZERO; KASSERT((m->flags & PG_UNHOLDFREE) == 0, - ("vm_page_free: freeing PG_UNHOLDFREE page %p", m)); + ("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m)); m->flags |= PG_UNHOLDFREE; return (false); } @@ -3763,9 +3972,8 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, vm_page_t *ma, int count) { - vm_page_t m, mpred; - int pflags; - int i; + vm_page_t m, mpred, msucc; + int i, pflags, run; bool sleep; VM_OBJECT_ASSERT_WLOCKED(object); @@ -3777,6 +3985,7 @@ KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || (allocflags & VM_ALLOC_IGN_SBUSY) != 0, ("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch")); + if (count == 0) return (0); pflags = allocflags & ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | @@ -3788,10 +3997,14 @@ m = vm_radix_lookup_le(&object->rtree, pindex + i); if (m == NULL || m->pindex != pindex + i) { mpred = m; + msucc = mpred != NULL ? TAILQ_NEXT(mpred, listq) : + TAILQ_FIRST(&object->memq); m = NULL; - } else + } else { mpred = TAILQ_PREV(m, pglist, listq); - for (; i < count; i++) { + msucc = TAILQ_NEXT(m, listq); + } + while (i < count) { if (m != NULL) { sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ? vm_page_xbusied(m) : vm_page_busied(m); @@ -3821,21 +4034,41 @@ vm_page_xbusy(m); if ((allocflags & VM_ALLOC_SBUSY) != 0) vm_page_sbusy(m); + if (m->valid == 0 && + (allocflags & VM_ALLOC_ZERO) != 0) { + if ((m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + m->valid = VM_PAGE_BITS_ALL; + } + ma[i++] = m; } else { - m = vm_page_alloc_after(object, pindex + i, - pflags | VM_ALLOC_COUNT(count - i), mpred); - if (m == NULL) { + /* + * Try to allocate multiple consecutive pages. Use the + * succeeding page, if any, to bound the length of the + * requested run. + */ + run = msucc == NULL || msucc->pindex >= pindex + count ? + count - i : msucc->pindex - (pindex + i); + run = vm_page_alloc_pages_after(object, pindex + i, + pflags | VM_ALLOC_COUNT(run), ma + i, run, mpred); + if (run == 0) { if ((allocflags & VM_ALLOC_NOWAIT) != 0) break; goto retrylookup; } + if ((allocflags & VM_ALLOC_ZERO) != 0) { + for (; run != 0; run--, i++) { + m = ma[i]; + if ((m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + m->valid = VM_PAGE_BITS_ALL; + } + } else + i += run; + m = ma[i - 1]; } - if (m->valid == 0 && (allocflags & VM_ALLOC_ZERO) != 0) { - if ((m->flags & PG_ZERO) == 0) - pmap_zero_page(m); - m->valid = VM_PAGE_BITS_ALL; - } - ma[i] = mpred = m; + mpred = m; + msucc = TAILQ_NEXT(m, listq); m = vm_page_next(m); } return (i); Index: sys/vm/vm_reserv.h =================================================================== --- sys/vm/vm_reserv.h +++ sys/vm/vm_reserv.h @@ -55,9 +55,9 @@ vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_page_t mpred); vm_page_t vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, - int domain, vm_page_t mpred); -vm_page_t vm_reserv_extend(int req, vm_object_t object, - vm_pindex_t pindex, int domain, vm_page_t mpred); + int domain, vm_page_t mpred, int *countp); +vm_page_t vm_reserv_extend(vm_object_t object, vm_pindex_t pindex, + int domain, int req, vm_page_t mpred, int *countp); void vm_reserv_break_all(vm_object_t object); boolean_t vm_reserv_free_page(vm_page_t m); void vm_reserv_init(void); @@ -67,7 +67,7 @@ boolean_t vm_reserv_reclaim_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary); -boolean_t vm_reserv_reclaim_inactive(int domain); +int vm_reserv_reclaim_inactive(int domain); void vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, vm_pindex_t old_object_offset); int vm_reserv_size(int level); Index: sys/vm/vm_reserv.c =================================================================== --- sys/vm/vm_reserv.c +++ sys/vm/vm_reserv.c @@ -578,7 +578,8 @@ domain = rv->domain; vmd = VM_DOMAIN(domain); vm_domain_free_lock(vmd); - if (rv->object != object || !vm_domain_available(vmd, req, npages)) { + if (rv->object != object || + vm_domain_available(vmd, req, npages) != npages) { m = NULL; goto out; } @@ -776,8 +777,10 @@ } /* - * Attempts to extend an existing reservation and allocate the page to the - * object. + * Attempts to extend an existing reservation and allocate the request page to + * the object. Opportunistically returns up to "*countp" contiguous pages if + * the caller so requests. The number of pages allocated is returned in + * "*countp". * * The page "mpred" must immediately precede the offset "pindex" within the * specified object. @@ -785,13 +788,13 @@ * The object must be locked. */ vm_page_t -vm_reserv_extend(int req, vm_object_t object, vm_pindex_t pindex, int domain, - vm_page_t mpred) +vm_reserv_extend(vm_object_t object, vm_pindex_t pindex, int domain, int req, + vm_page_t mpred, int *countp) { struct vm_domain *vmd; vm_page_t m, msucc; vm_reserv_t rv; - int index, free_count; + int avail, free_count, index, nalloc; VM_OBJECT_ASSERT_WLOCKED(object); @@ -816,17 +819,28 @@ index = VM_RESERV_INDEX(object, pindex); m = &rv->pages[index]; vm_domain_free_lock(vmd); - if (vm_domain_available(vmd, req, 1) == 0 || + if ((avail = vm_domain_available(vmd, req, VM_LEVEL_0_NPAGES)) == 0 || /* Handle reclaim race. */ rv->object != object || /* Handle vm_page_rename(m, new_object, ...). */ - popmap_is_set(rv->popmap, index)) + popmap_is_set(rv->popmap, index)) { m = NULL; - if (m != NULL) { - vm_reserv_populate(rv, index); - free_count = vm_domain_freecnt_adj(vmd, -1); - } else free_count = vmd->vmd_free_count; + } else { + vm_reserv_populate(rv, index); + nalloc = 1; + if (countp != NULL) { + avail = imin(imin(avail, *countp), + VM_LEVEL_0_NPAGES - index); + for (; nalloc < avail; index++, nalloc++) { + if (popmap_is_set(rv->popmap, index)) + break; + vm_reserv_populate(rv, index); + } + *countp = nalloc; + } + free_count = vm_domain_freecnt_adj(vmd, -nalloc); + } vm_domain_free_unlock(vmd); if (vm_paging_needed(vmd, free_count)) @@ -836,16 +850,19 @@ } /* - * Allocates a page from an existing reservation. + * Allocates a new reservation for the object, and returns a page from that + * reservation. Opportunistically returns up to *"countp" contiguous pages if + * the caller so requests. The number of pages allocated is returned in + * "*countp". * * The page "mpred" must immediately precede the offset "pindex" within the * specified object. * - * The object and free page queue must be locked. + * The object and per-domain free page queues must be locked. */ vm_page_t vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain, - vm_page_t mpred) + vm_page_t mpred, int *countp) { vm_page_t m, msucc; vm_pindex_t first, leftcap, rightcap; @@ -930,7 +947,11 @@ vm_reserv_insert(rv, object, first); index = VM_RESERV_INDEX(object, pindex); vm_reserv_populate(rv, index); - return (&rv->pages[index]); + m = &rv->pages[index]; + if (countp != NULL) + for (; *countp > 1 && index < VM_LEVEL_0_NPAGES; (*countp)--) + vm_reserv_populate(rv, ++index); + return (m); } /* @@ -1161,22 +1182,24 @@ /* * Breaks the reservation at the head of the partially populated reservation - * queue, releasing its free pages to the physical memory allocator. Returns - * TRUE if a reservation is broken and FALSE otherwise. + * queue, releasing its free pages to the physical memory allocator, and + * returns the number of pages released. * * The free page queue lock must be held. */ -boolean_t +int vm_reserv_reclaim_inactive(int domain) { vm_reserv_t rv; + int freed; vm_domain_free_assert_locked(VM_DOMAIN(domain)); if ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) { + freed = VM_LEVEL_0_NPAGES - rv->popcnt; vm_reserv_reclaim(rv); - return (TRUE); + return (freed); } - return (FALSE); + return (0); } /* Index: sys/vm/vnode_pager.c =================================================================== --- sys/vm/vnode_pager.c +++ sys/vm/vnode_pager.c @@ -897,35 +897,27 @@ /* * Fill in the bp->b_pages[] array with requested and optional - * read behind or read ahead pages. Read behind pages are looked - * up in a backward direction, down to a first cached page. Same - * for read ahead pages, but there is no need to shift the array - * in case of encountering a cached page. + * read behind or read ahead pages. */ i = bp->b_npages = 0; if (rbehind) { - vm_pindex_t startpindex, tpindex; - vm_page_t p; + vm_pindex_t startpindex; + vm_page_t mpred; VM_OBJECT_WLOCK(object); startpindex = m[0]->pindex - rbehind; - if ((p = TAILQ_PREV(m[0], pglist, listq)) != NULL && - p->pindex >= startpindex) - startpindex = p->pindex + 1; - - /* tpindex is unsigned; beware of numeric underflow. */ - for (tpindex = m[0]->pindex - 1; - tpindex >= startpindex && tpindex < m[0]->pindex; - tpindex--, i++) { - p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL); - if (p == NULL) { - /* Shift the array. */ - for (int j = 0; j < i; j++) - bp->b_pages[j] = bp->b_pages[j + - tpindex + 1 - startpindex]; - break; - } - bp->b_pages[tpindex - startpindex] = p; + if ((mpred = TAILQ_PREV(m[0], pglist, listq)) != NULL && + mpred->pindex >= startpindex) + startpindex = mpred->pindex + 1; + + i = vm_page_alloc_pages_after(object, startpindex, + VM_ALLOC_NORMAL, &bp->b_pages[0], + m[0]->pindex - startpindex, mpred); + if (i < m[0]->pindex - startpindex) { + /* We have to drop the partially allocated run. */ + for (int j = 0; j < i; j++) + vm_page_free(bp->b_pages[j]); + i = 0; } bp->b_pgbefore = i; @@ -940,28 +932,23 @@ bp->b_npages += count; if (rahead) { - vm_pindex_t endpindex, tpindex; - vm_page_t p; + vm_pindex_t endpindex, startpindex; + vm_page_t msucc; if (!VM_OBJECT_WOWNED(object)) VM_OBJECT_WLOCK(object); - endpindex = m[count - 1]->pindex + rahead + 1; - if ((p = TAILQ_NEXT(m[count - 1], listq)) != NULL && - p->pindex < endpindex) - endpindex = p->pindex; + startpindex = m[count - 1]->pindex + 1; + endpindex = startpindex + rahead; + if ((msucc = TAILQ_NEXT(m[count - 1], listq)) != NULL && + msucc->pindex < endpindex) + endpindex = msucc->pindex; if (endpindex > object->size) endpindex = object->size; - for (tpindex = m[count - 1]->pindex + 1; - tpindex < endpindex; i++, tpindex++) { - p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL); - if (p == NULL) - break; - bp->b_pages[i] = p; - } - - bp->b_pgafter = i - bp->b_npages; - bp->b_npages = i; + bp->b_pgafter = vm_page_alloc_pages_after(object, startpindex, + VM_ALLOC_NORMAL, &bp->b_pages[i], endpindex - startpindex, + m[count - 1]); + bp->b_npages += bp->b_pgafter; } else bp->b_pgafter = 0;