Index: sys/kern/kern_sendfile.c =================================================================== --- sys/kern/kern_sendfile.c +++ sys/kern/kern_sendfile.c @@ -90,6 +90,7 @@ struct socket *so; struct mbuf *m; vm_object_t obj; + vm_pindex_t pindex0; #ifdef KERN_TLS struct ktls_session *tls; #endif @@ -261,13 +262,37 @@ { struct sf_io *sfio = arg; struct socket *so; + int i; - for (int i = 0; i < count; i++) - if (pg[i] != bogus_page) - vm_page_xunbusy_unchecked(pg[i]); - - if (error) + if (error != 0) { sfio->error = error; + /* + * Restore of the pg[] elements is done by + * sendfile_swapin(). + */ + } else if (pg != NULL) { + /* + * Restore the valid page pointers. They are already + * unbusied, but still wired. For error != 0 case, + * sendfile_swapin() handles unbusy. + * + * XXXKIB since pages are only wired, and we do not + * own the object lock, other users might invalidated + * them meantime. Similarly, after we unbusied the + * swapped-in pages, they can become invalid under us. + */ + for (i = 0; i < count; i++) { + if (pg[i] == bogus_page) { + pg[i] = vm_page_relookup(sfio->obj, + sfio->pindex0 + i + (sfio->pa - pg)); + KASSERT(pg[i] != NULL, + ("%s: page %p[%d] disappeared", + __func__, pg, i)); + } else { + vm_page_xunbusy_unchecked(pg[i]); + } + } + } if (!refcount_release(&sfio->nios)) return; @@ -353,6 +378,7 @@ *nios = 0; flags = (flags & SF_NODISKIO) ? VM_ALLOC_NOWAIT : 0; + sfio->pindex0 = OFF_TO_IDX(off); /* * First grab all the pages and wire them. Note that we grab @@ -409,18 +435,11 @@ count = min(a + 1, npages - i); /* - * We should not pagein into a valid page, thus we first trim - * any valid pages off the end of request, and substitute - * to bogus_page those, that are in the middle. + * We should not pagein into a valid page because + * there might be still unfinished write tracked by + * e.g. a buffer, thus we substitute any valid pages + * with the bogus one. */ - for (j = i + count - 1; j > i; j--) { - if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK, - xfsize(j, npages, off, len))) { - count--; - rhpages = 0; - } else - break; - } for (j = i + 1; j < i + count - 1; j++) if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK, xfsize(j, npages, off, len))) { @@ -454,7 +473,6 @@ __func__, pa, j)); vm_page_unwire(pa[j], PQ_INACTIVE); } - refcount_release(&sfio->nios); return (EIO); } @@ -463,18 +481,6 @@ if (i + count == npages) SFSTAT_ADD(sf_rhpages_read, rhpages); - /* - * Restore the valid page pointers. They are already - * unbusied, but still wired. - */ - for (j = i + 1; j < i + count - 1; j++) - if (pa[j] == bogus_page) { - pa[j] = vm_page_relookup(obj, - OFF_TO_IDX(vmoff(j, off))); - KASSERT(pa[j], ("%s: page %p[%d] disappeared", - __func__, pa, j)); - - } i += count; (*nios)++; } Index: sys/kern/vfs_bio.c =================================================================== --- sys/kern/vfs_bio.c +++ sys/kern/vfs_bio.c @@ -5154,12 +5154,16 @@ br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) ? GB_UNMAPPED : 0; again: - for (i = 0; i < count; i++) - vm_page_busy_downgrade(ma[i]); + for (i = 0; i < count; i++) { + if (ma[i] != bogus_page) + vm_page_busy_downgrade(ma[i]); + } lbnp = -1; for (i = 0; i < count; i++) { m = ma[i]; + if (m == bogus_page) + continue; /* * Pages are shared busy and the object lock is not @@ -5228,6 +5232,8 @@ redo = false; for (i = 0; i < count; i++) { + if (ma[i] == bogus_page) + continue; if (vm_page_busy_tryupgrade(ma[i]) == 0) { vm_page_sunbusy(ma[i]); ma[i] = vm_page_grab_unlocked(object, ma[i]->pindex, Index: sys/kern/vfs_default.c =================================================================== --- sys/kern/vfs_default.c +++ sys/kern/vfs_default.c @@ -765,7 +765,8 @@ error = VOP_GETPAGES(ap->a_vp, ap->a_m, ap->a_count, ap->a_rbehind, ap->a_rahead); - ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error); + if (ap->a_iodone != NULL) + ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error); return (error); } Index: sys/ufs/ffs/ffs_vnops.c =================================================================== --- sys/ufs/ffs/ffs_vnops.c +++ sys/ufs/ffs/ffs_vnops.c @@ -1780,18 +1780,25 @@ { struct vnode *vp; struct ufsmount *um; + bool do_iodone; int error; vp = ap->a_vp; um = VFSTOUFS(vp->v_mount); + do_iodone = true; - if (um->um_devvp->v_bufobj.bo_bsize <= PAGE_SIZE) - return (vnode_pager_generic_getpages(vp, ap->a_m, ap->a_count, - ap->a_rbehind, ap->a_rahead, ap->a_iodone, ap->a_arg)); - - error = vfs_bio_getpages(vp, ap->a_m, ap->a_count, ap->a_rbehind, - ap->a_rahead, ffs_gbp_getblkno, ffs_gbp_getblksz); - ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error); + if (um->um_devvp->v_bufobj.bo_bsize <= PAGE_SIZE) { + error = vnode_pager_generic_getpages(vp, ap->a_m, ap->a_count, + ap->a_rbehind, ap->a_rahead, ap->a_iodone, ap->a_arg); + if (error == 0) + do_iodone = false; + } else { + error = vfs_bio_getpages(vp, ap->a_m, ap->a_count, + ap->a_rbehind, ap->a_rahead, ffs_gbp_getblkno, + ffs_gbp_getblksz); + } + if (do_iodone && ap->a_iodone != NULL) + ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error); return (error); } Index: sys/vm/vnode_pager.c =================================================================== --- sys/vm/vnode_pager.c +++ sys/vm/vnode_pager.c @@ -776,9 +776,13 @@ int vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap) { + int error; - return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count, - ap->a_rbehind, ap->a_rahead, ap->a_iodone, ap->a_arg)); + error = vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count, + ap->a_rbehind, ap->a_rahead, ap->a_iodone, ap->a_arg); + if (error != 0 && ap->a_iodone != NULL) + ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error); + return (error); } /*