Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -1349,34 +1349,37 @@ * Attempt to page in the pages in array "ma" of length "count". The * caller may optionally specify that additional pages preceding and * succeeding the specified range be paged in. The number of such pages - * is returned in the "rbehind" and "rahead" parameters, and they will + * is returned in the "a_rbehind" and "a_rahead" parameters, and they will * be in the inactive queue upon return. * * The pages in "ma" must be busied and will remain busied upon return. */ static int swap_pager_getpages_locked(struct pctrie_iter *blks, vm_object_t object, - vm_page_t *ma, int count, int *rbehind, int *rahead) + vm_page_t *ma, int count, int *a_rbehind, int *a_rahead) { + vm_page_t ahead[SWB_NPAGES - 1], behind[SWB_NPAGES - 1]; struct buf *bp; - vm_page_t bm, mpred, msucc, p; vm_pindex_t pindex; daddr_t blk; - int i, maxahead, maxbehind, reqcount; + int rahead, rbehind; VM_OBJECT_ASSERT_WLOCKED(object); - reqcount = count; KASSERT((object->flags & OBJ_SWAP) != 0, ("%s: object not swappable", __func__)); - if (!swp_pager_haspage_iter(blks, ma[0]->pindex, &maxbehind, - &maxahead)) { + pindex = ma[0]->pindex; + if (!swp_pager_haspage_iter(blks, pindex, &rbehind, &rahead)) { VM_OBJECT_WUNLOCK(object); return (VM_PAGER_FAIL); } - KASSERT(reqcount - 1 <= maxahead, - ("page count %d extends beyond swap block", reqcount)); + KASSERT(count - 1 <= rahead, + ("page count %d extends beyond swap block", count)); + KASSERT(rbehind < SWB_NPAGES, + ("rbehind %d too large for array", rbehind)); + KASSERT(rahead < SWB_NPAGES, + ("rahead %d too large for array", rahead)); /* * Do not transfer any pages other than those that are xbusied @@ -1385,69 +1388,31 @@ * moved into another object. */ if ((object->flags & (OBJ_SPLIT | OBJ_DEAD)) != 0) { - maxahead = reqcount - 1; - maxbehind = 0; + rahead = count - 1; + rbehind = 0; } /* - * Clip the readahead and readbehind ranges to exclude resident pages. + * Clip readbehind range to exclude resident pages. Allocate pages. */ - if (rahead != NULL) { - *rahead = imin(*rahead, maxahead - (reqcount - 1)); - pindex = ma[reqcount - 1]->pindex; - msucc = TAILQ_NEXT(ma[reqcount - 1], listq); - if (msucc != NULL && msucc->pindex - pindex - 1 < *rahead) - *rahead = msucc->pindex - pindex - 1; - } - if (rbehind != NULL) { - *rbehind = imin(*rbehind, maxbehind); - pindex = ma[0]->pindex; - mpred = TAILQ_PREV(ma[0], pglist, listq); - if (mpred != NULL && pindex - mpred->pindex - 1 < *rbehind) - *rbehind = pindex - mpred->pindex - 1; - } - - bm = ma[0]; - for (i = 0; i < count; i++) - ma[i]->oflags |= VPO_SWAPINPROG; + rbehind = a_rbehind != NULL ? imin(*a_rbehind, rbehind) : 0; + vm_object_getpages(object, behind, 0, &rbehind, NULL, ma[0], NULL); /* - * Allocate readahead and readbehind pages. + * Clip readahead range to exclude resident pages. Allocate pages. */ - if (rbehind != NULL) { - pindex = ma[0]->pindex; - /* Stepping backward from pindex, mpred doesn't change. */ - for (i = 1; i <= *rbehind; i++) { - p = vm_page_alloc_after(object, pindex - i, - VM_ALLOC_NORMAL, mpred); - if (p == NULL) - break; - p->oflags |= VPO_SWAPINPROG; - bm = p; - } - *rbehind = i - 1; - } - if (rahead != NULL) { - p = ma[reqcount - 1]; - pindex = p->pindex; - for (i = 0; i < *rahead; i++) { - p = vm_page_alloc_after(object, pindex + i + 1, - VM_ALLOC_NORMAL, p); - if (p == NULL) - break; - p->oflags |= VPO_SWAPINPROG; - } - *rahead = i; - } - if (rbehind != NULL) - count += *rbehind; - if (rahead != NULL) - count += *rahead; - - vm_object_pip_add(object, count); + rahead = a_rahead != NULL ? imin(*a_rahead, rahead - (count - 1)) : 0; + vm_object_getpages(object, ahead, 0, NULL, &rahead, NULL, ma[count - 1]); - pindex = bm->pindex; - blk = swp_pager_meta_lookup(blks, pindex); + vm_object_pip_add(object, rbehind + count + rahead); + for (int i = 0; i < rbehind; i++) + behind[i]->oflags |= VPO_SWAPINPROG; + for (int i = 0; i < count; i++) + ma[i]->oflags |= VPO_SWAPINPROG; + for (int i = 0; i < rahead; i++) + ahead[i]->oflags |= VPO_SWAPINPROG; + + blk = swp_pager_meta_lookup(blks, pindex - rbehind); KASSERT(blk != SWAPBLK_NONE, ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); @@ -1455,10 +1420,18 @@ bp = uma_zalloc(swrbuf_zone, M_WAITOK); MPASS((bp->b_flags & B_MAXPHYS) != 0); /* Pages cannot leave the object while busy. */ - for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) { - MPASS(p->pindex == bm->pindex + i); - bp->b_pages[i] = p; - } + for (int i = 0; i < rbehind; i++) + bp->b_pages[i] = behind[i]; + for (int i = 0; i < count; i++) + bp->b_pages[rbehind + i] = ma[i]; + for (int i = 0; i < rahead; i++) + bp->b_pages[rbehind + count + i] = ahead[i]; + + /* Report back actual behind/ahead read. */ + if (a_rbehind != NULL) + *a_rbehind = rbehind; + if (a_rahead != NULL) + *a_rahead = rahead; bp->b_flags |= B_PAGING; bp->b_iocmd = BIO_READ; @@ -1466,14 +1439,13 @@ bp->b_rcred = crhold(thread0.td_ucred); bp->b_wcred = crhold(thread0.td_ucred); bp->b_blkno = blk; - bp->b_bcount = PAGE_SIZE * count; - bp->b_bufsize = PAGE_SIZE * count; - bp->b_npages = count; - bp->b_pgbefore = rbehind != NULL ? *rbehind : 0; - bp->b_pgafter = rahead != NULL ? *rahead : 0; + bp->b_npages = rbehind + count + rahead; + bp->b_bufsize = bp->b_bcount = PAGE_SIZE * bp->b_npages; + bp->b_pgbefore = rbehind; + bp->b_pgafter = rahead; VM_CNT_INC(v_swapin); - VM_CNT_ADD(v_swappgsin, count); + VM_CNT_ADD(v_swappgsin, bp->b_npages); /* * perform the I/O. NOTE!!! bp cannot be considered valid after @@ -1511,7 +1483,7 @@ /* * If we had an unrecoverable read error pages will not be valid. */ - for (i = 0; i < reqcount; i++) + for (int i = 0; i < count; i++) if (ma[i]->valid != VM_PAGE_BITS_ALL) return (VM_PAGER_ERROR); Index: sys/vm/vm_object.h =================================================================== --- sys/vm/vm_object.h +++ sys/vm/vm_object.h @@ -376,6 +376,8 @@ vm_pindex_t end); void vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int options); +void vm_object_getpages(vm_object_t object, vm_page_t *ma, int count, + int *rbehind, int *rahead, vm_page_t m_beg, vm_page_t m_end); boolean_t vm_object_populate(vm_object_t, vm_pindex_t, vm_pindex_t); void vm_object_print(long addr, boolean_t have_addr, long count, char *modif); void vm_object_reference (vm_object_t); Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -2258,6 +2258,63 @@ return (TRUE); } +/* + * Fill in the ma array with up to *rbehind optional pages before m_beg and up + * to *rahead optional pages after m_end. In both cases, stop the + * filling-in short on encountering a cached page, an object boundary limit, or + * an allocation error. Update *rbehind and *rahead to indicate the number of + * pages allocated. Array elements from ma[*rbehind] to ma[*rbehind + count -1] + * are not affected. + */ +void +vm_object_getpages(vm_object_t object, vm_page_t *ma, int count, int *rbehind, + int *rahead, vm_page_t m_beg, vm_page_t m_end) +{ + vm_pindex_t pindex; + vm_page_t m, mpred, msucc; + + VM_OBJECT_ASSERT_LOCKED(object); + if (rbehind != NULL && *rbehind != 0) { + pindex = m_beg->pindex; + mpred = TAILQ_PREV(m_beg, pglist, listq); + + *rbehind = MIN(*rbehind, + pindex - (mpred != NULL ? mpred->pindex + 1 : 0)); + + /* Stepping backward from pindex, mpred doesn't change. */ + for (int i = 0; i < *rbehind; i++) { + m = vm_page_alloc_after(object, pindex - i - 1, + VM_ALLOC_NORMAL, mpred); + if (m == NULL) { + /* Shift the array. */ + for (int j = 0; j < i; j++) + ma[j] = ma[j + *rbehind - i]; + *rbehind = i; + break; + } + ma[*rbehind - i - 1] = m; + } + count += *rbehind; + } + if (rahead != NULL && *rahead != 0) { + pindex = m_end->pindex + 1; + msucc = TAILQ_NEXT(m_end, listq); + + *rahead = MIN(*rahead, + (msucc != NULL ? msucc->pindex : object->size) - pindex); + mpred = m_end; + for (int i = 0; i < *rahead; i++) { + m = vm_page_alloc_after(object, pindex + i, + VM_ALLOC_NORMAL, mpred); + if (m == NULL) { + *rahead = i; + break; + } + ma[count + i] = mpred = m; + } + } +} + void vm_object_set_writeable_dirty_(vm_object_t object) { Index: sys/vm/vnode_pager.c =================================================================== --- sys/vm/vnode_pager.c +++ sys/vm/vnode_pager.c @@ -1007,9 +1007,8 @@ rbehind = a_rbehind ? *a_rbehind : 0; rahead = a_rahead ? *a_rahead : 0; rbehind = min(rbehind, before); - rbehind = min(rbehind, m[0]->pindex); rahead = min(rahead, after); - rahead = min(rahead, object->size - m[count - 1]->pindex); + /* * Check that total amount of pages fit into buf. Trim rbehind and * rahead evenly if not. @@ -1039,72 +1038,20 @@ * for read ahead pages, but there is no need to shift the array * in case of encountering a cached page. */ - i = bp->b_npages = 0; - if (rbehind) { - vm_pindex_t startpindex, tpindex; - vm_page_t mpred, p; - + if (rbehind != 0 || rahead != 0) { VM_OBJECT_WLOCK(object); - tpindex = m[0]->pindex; - startpindex = MAX(tpindex, rbehind) - rbehind; - if ((mpred = TAILQ_PREV(m[0], pglist, listq)) != NULL) - startpindex = MAX(startpindex, mpred->pindex + 1); - - /* Stepping backward from pindex, mpred doesn't change. */ - for (; tpindex-- > startpindex; i++) { - p = vm_page_alloc_after(object, tpindex, - VM_ALLOC_NORMAL, mpred); - if (p == NULL) { - /* Shift the array. */ - for (int j = 0; j < i; j++) - bp->b_pages[j] = bp->b_pages[j + - tpindex + 1 - startpindex]; - break; - } - bp->b_pages[tpindex - startpindex] = p; - } - - bp->b_pgbefore = i; - bp->b_npages += i; - bp->b_blkno -= IDX_TO_OFF(i) / DEV_BSIZE; - } else - bp->b_pgbefore = 0; + vm_object_getpages(object, bp->b_pages, count, &rbehind, + &rahead, m[0], m[count - 1]); + VM_OBJECT_WUNLOCK(object); + } + bp->b_blkno -= IDX_TO_OFF(rbehind) / DEV_BSIZE; + bp->b_pgbefore = rbehind; + bp->b_pgafter = rahead; /* Requested pages. */ - for (int j = 0; j < count; j++, i++) - bp->b_pages[i] = m[j]; - bp->b_npages += count; - - if (rahead) { - vm_pindex_t endpindex, tpindex; - vm_page_t p; - - if (!VM_OBJECT_WOWNED(object)) - VM_OBJECT_WLOCK(object); - endpindex = m[count - 1]->pindex + rahead + 1; - if ((p = TAILQ_NEXT(m[count - 1], listq)) != NULL && - p->pindex < endpindex) - endpindex = p->pindex; - if (endpindex > object->size) - endpindex = object->size; - - p = m[count - 1]; - for (tpindex = p->pindex + 1; - tpindex < endpindex; i++, tpindex++) { - p = vm_page_alloc_after(object, tpindex, - VM_ALLOC_NORMAL, p); - if (p == NULL) - break; - bp->b_pages[i] = p; - } - - bp->b_pgafter = i - bp->b_npages; - bp->b_npages = i; - } else - bp->b_pgafter = 0; - - if (VM_OBJECT_WOWNED(object)) - VM_OBJECT_WUNLOCK(object); + for (int j = 0; j < count; j++) + bp->b_pages[rbehind + j] = m[j]; + bp->b_npages = rbehind + count + rahead; /* Report back actual behind/ahead read. */ if (a_rbehind)