Index: sys/vm/vnode_pager.c =================================================================== --- sys/vm/vnode_pager.c +++ sys/vm/vnode_pager.c @@ -1179,6 +1179,23 @@ VM_OBJECT_WLOCK(object); } +static int +vn_off2bidx(vm_ooffset_t offset) +{ + + return ((offset & PAGE_MASK) / DEV_BSIZE); +} + +static bool +vn_dirty_blk(vm_page_t m, vm_ooffset_t offset) +{ + + KASSERT(IDX_TO_OFF(m->pindex) <= offset && + offset < IDX_TO_OFF(m->pindex + 1), + ("page %p pidx %ju offset %ju", m, (uintmax_t)m->pindex, + (uintmax_t)offset)); + return ((m->dirty & ((vm_page_bits_t)1 << vn_off2bidx(offset))) != 0); +} /* * This is now called from local media FS's to operate against their @@ -1195,10 +1212,12 @@ { vm_object_t object; vm_page_t m; - vm_ooffset_t poffset; + vm_ooffset_t maxblksz, next_offset, prev_offset, poffset; struct uio auio; struct iovec aiov; + off_t prev_resid, wrsz; int count, error, i, maxsize, ncount, pgoff, ppscheck; + bool in_hole; static struct timeval lastfail; static int curfail; @@ -1260,34 +1279,113 @@ for (i = ncount; i < count; i++) rtvals[i] = VM_PAGER_BAD; } - for (i = 0; i < ncount - ((btoc(maxsize) & PAGE_MASK) != 0); i++) - MPASS(ma[i]->dirty == VM_PAGE_BITS_ALL); - VM_OBJECT_WUNLOCK(object); - aiov.iov_base = NULL; - aiov.iov_len = maxsize; auio.uio_iov = &aiov; - auio.uio_iovcnt = 1; - auio.uio_offset = poffset; auio.uio_segflg = UIO_NOCOPY; auio.uio_rw = UIO_WRITE; - auio.uio_resid = maxsize; auio.uio_td = NULL; - error = VOP_WRITE(vp, &auio, vnode_pager_putpages_ioflags(flags), - curthread->td_ucred); + auio.uio_offset = poffset; + maxblksz = roundup2(poffset + maxsize, DEV_BSIZE); + + for (next_offset = prev_offset = poffset; prev_offset < maxblksz;) { + /* Skip clean blocks */ + for (in_hole = true; in_hole && prev_offset < maxblksz;) { + m = ma[OFF_TO_IDX(prev_offset - poffset)]; + for (i = vn_off2bidx(prev_offset); + i < sizeof(vm_page_bits_t) * NBBY && + prev_offset < maxblksz; i++) { + if (vn_dirty_blk(m, prev_offset)) { + in_hole = false; + break; + } + prev_offset += DEV_BSIZE; + auio.uio_offset += DEV_BSIZE; + } + } + if (in_hole) + goto write_done; + + /* Find longest run of dirty blocks */ + for (next_offset = prev_offset; next_offset < maxblksz;) { + m = ma[OFF_TO_IDX(next_offset - poffset)]; + for (i = vn_off2bidx(next_offset); + i < sizeof(vm_page_bits_t) * NBBY && + next_offset < maxblksz; i++) { + if (!vn_dirty_blk(m, next_offset)) + goto start_write; + next_offset += DEV_BSIZE; + } + } +start_write: + if (next_offset > poffset + maxsize) + next_offset = poffset + maxsize; + if (prev_offset >= next_offset) + break; + VM_OBJECT_WUNLOCK(object); + aiov.iov_base = NULL; + auio.uio_iovcnt = 1; + prev_resid = auio.uio_resid = aiov.iov_len = next_offset - + prev_offset; + error = VOP_WRITE(vp, &auio, + vnode_pager_putpages_ioflags(flags), curthread->td_ucred); + + wrsz = prev_resid - auio.uio_resid; + if (wrsz == 0 && (ppscheck != 0 || + ppsratecheck(&lastfail, &curfail, 1) != 0)) { + vn_printf(vp, "vnode_pager_putpages: zero-length write" + " at %ju resid %zd\n", auio.uio_offset, + auio.uio_resid); + VM_OBJECT_WLOCK(object); + break; + } + + /* + * Mark completely processed pages and adjust the + * starting offset for next iteration. + */ + for (; wrsz > 0; wrsz -= DEV_BSIZE) { + prev_offset += DEV_BSIZE; + if ((prev_offset & PAGE_MASK) == 0) { + rtvals[OFF_TO_IDX(prev_offset - poffset) - 1] = + VM_PAGER_OK; + VM_CNT_ADD(v_vnodepgsout, 1); + } + } + + if (wrsz != 0) { + /* + * Handle partially-written block. If it is + * marked as clean, skip it. For instance, + * this happens at EOF. + */ + wrsz += DEV_BSIZE; + m = ma[OFF_TO_IDX(prev_offset + wrsz - poffset)]; + if (!vn_dirty_blk(m, prev_offset + wrsz)) { + prev_offset += DEV_BSIZE; + if ((prev_offset & PAGE_MASK) == 0) { + rtvals[OFF_TO_IDX(prev_offset - + poffset) - 1] = VM_PAGER_OK; + VM_CNT_ADD(v_vnodepgsout, 1); + } + } + } + MPASS(auio.uio_offset == prev_offset); + + ppscheck = 0; + if (error != 0 && (ppscheck = ppsratecheck(&lastfail, + &curfail, 1)) != 0) + vn_printf(vp, "vnode_pager_putpages: I/O error %d\n", + error); + if (auio.uio_resid != 0 && (ppscheck != 0 || + ppsratecheck(&lastfail, &curfail, 1) != 0)) + vn_printf(vp, "vnode_pager_putpages: residual I/O %zd " + "at %ju\n", auio.uio_resid, + (uintmax_t)ma[0]->pindex); + VM_OBJECT_WLOCK(object); + } +write_done: + VM_OBJECT_WUNLOCK(object); VM_CNT_INC(v_vnodeout); - VM_CNT_ADD(v_vnodepgsout, ncount); - - ppscheck = 0; - if (error != 0 && (ppscheck = ppsratecheck(&lastfail, &curfail, 1)) - != 0) - printf("vnode_pager_putpages: I/O error %d\n", error); - if (auio.uio_resid != 0 && (ppscheck != 0 || - ppsratecheck(&lastfail, &curfail, 1) != 0)) - printf("vnode_pager_putpages: residual I/O %zd at %ju\n", - auio.uio_resid, (uintmax_t)ma[0]->pindex); - for (i = 0; i < ncount; i++) - rtvals[i] = VM_PAGER_OK; return (rtvals[0]); }