Index: sys/kern/vfs_bio.c =================================================================== --- sys/kern/vfs_bio.c +++ sys/kern/vfs_bio.c @@ -161,7 +161,7 @@ static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off, vm_page_t m); static void vfs_clean_pages_dirty_buf(struct buf *bp); -static void vfs_setdirty_locked_object(struct buf *bp); +static void vfs_setdirty_range(struct buf *bp); static void vfs_vmio_invalidate(struct buf *bp); static void vfs_vmio_truncate(struct buf *bp, int npages); static void vfs_vmio_extend(struct buf *bp, int npages, int size); @@ -954,8 +954,6 @@ vm_offset_t size, vm_page_t m) { - VM_OBJECT_ASSERT_LOCKED(m->object); - /* * This function and its results are protected by higher level * synchronization requiring vnode and buf locks to page in and @@ -2851,7 +2849,6 @@ bogus = false; iosize = bp->b_bcount - bp->b_resid; - VM_OBJECT_WLOCK(obj); for (i = 0; i < bp->b_npages; i++) { resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff; if (resid > iosize) @@ -2862,7 +2859,10 @@ */ m = bp->b_pages[i]; if (m == bogus_page) { - bogus = true; + if (bogus == false) { + bogus = true; + VM_OBJECT_RLOCK(obj); + } m = vm_page_lookup(obj, OFF_TO_IDX(foff)); if (m == NULL) panic("biodone: page disappeared!"); @@ -2886,8 +2886,9 @@ foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; iosize -= resid; } + if (bogus) + VM_OBJECT_RUNLOCK(obj); vm_object_pip_wakeupn(obj, bp->b_npages); - VM_OBJECT_WUNLOCK(obj); if (bogus && buf_mapped(bp)) { BUF_CHECK_MAPPED(bp); pmap_qenter(trunc_page((vm_offset_t)bp->b_data), @@ -3015,7 +3016,6 @@ * are not valid for the range covered by the buffer. */ obj = bp->b_bufobj->bo_object; - VM_OBJECT_WLOCK(obj); if (bp->b_npages < desiredpages) { /* * We must allocate system pages since blocking @@ -3027,11 +3027,13 @@ * deadlocks once allocbuf() is called after * pages are vfs_busy_pages(). */ + VM_OBJECT_WLOCK(obj); (void)vm_page_grab_pages(obj, OFF_TO_IDX(bp->b_offset) + bp->b_npages, VM_ALLOC_SYSTEM | VM_ALLOC_IGN_SBUSY | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED, &bp->b_pages[bp->b_npages], desiredpages - bp->b_npages); + VM_OBJECT_WUNLOCK(obj); bp->b_npages = desiredpages; } @@ -3062,7 +3064,6 @@ toff += tinc; tinc = PAGE_SIZE; } - VM_OBJECT_WUNLOCK(obj); /* * Step 3, fixup the KVA pmap. @@ -3641,9 +3642,8 @@ KASSERT(bp->b_offset != NOOFFSET, ("vfs_clean_pages_dirty_buf: no buffer offset")); - VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); vfs_busy_pages_acquire(bp); - vfs_setdirty_locked_object(bp); + vfs_setdirty_range(bp); for (i = 0; i < bp->b_npages; i++) { noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; eoff = noff; @@ -3655,69 +3655,57 @@ foff = noff; } vfs_busy_pages_release(bp); - VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); } static void -vfs_setdirty_locked_object(struct buf *bp) +vfs_setdirty_range(struct buf *bp) { - vm_object_t object; + vm_offset_t boffset; + vm_offset_t eoffset; int i; - object = bp->b_bufobj->bo_object; - VM_OBJECT_ASSERT_WLOCKED(object); - /* - * We qualify the scan for modified pages on whether the - * object has been flushed yet. + * test the pages to see if they have been modified directly + * by users through the VM system. */ - if ((object->flags & OBJ_MIGHTBEDIRTY) != 0) { - vm_offset_t boffset; - vm_offset_t eoffset; - - /* - * test the pages to see if they have been modified directly - * by users through the VM system. - */ - for (i = 0; i < bp->b_npages; i++) - vm_page_test_dirty(bp->b_pages[i]); + for (i = 0; i < bp->b_npages; i++) + vm_page_test_dirty(bp->b_pages[i]); - /* - * Calculate the encompassing dirty range, boffset and eoffset, - * (eoffset - boffset) bytes. - */ + /* + * Calculate the encompassing dirty range, boffset and eoffset, + * (eoffset - boffset) bytes. + */ - for (i = 0; i < bp->b_npages; i++) { - if (bp->b_pages[i]->dirty) - break; - } - boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK); + for (i = 0; i < bp->b_npages; i++) { + if (bp->b_pages[i]->dirty) + break; + } + boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK); - for (i = bp->b_npages - 1; i >= 0; --i) { - if (bp->b_pages[i]->dirty) { - break; - } + for (i = bp->b_npages - 1; i >= 0; --i) { + if (bp->b_pages[i]->dirty) { + break; } - eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK); + } + eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK); - /* - * Fit it to the buffer. - */ + /* + * Fit it to the buffer. + */ - if (eoffset > bp->b_bcount) - eoffset = bp->b_bcount; + if (eoffset > bp->b_bcount) + eoffset = bp->b_bcount; - /* - * If we have a good dirty range, merge with the existing - * dirty range. - */ + /* + * If we have a good dirty range, merge with the existing + * dirty range. + */ - if (boffset < eoffset) { - if (bp->b_dirtyoff > boffset) - bp->b_dirtyoff = boffset; - if (bp->b_dirtyend < eoffset) - bp->b_dirtyend = eoffset; - } + if (boffset < eoffset) { + if (bp->b_dirtyoff > boffset) + bp->b_dirtyoff = boffset; + if (bp->b_dirtyend < eoffset) + bp->b_dirtyend = eoffset; } } @@ -4457,16 +4445,21 @@ int i; vm_object_t obj; vm_page_t m; + bool bogus; runningbufwakeup(bp); if (!(bp->b_flags & B_VMIO)) return; obj = bp->b_bufobj->bo_object; - VM_OBJECT_WLOCK(obj); + bogus = false; for (i = 0; i < bp->b_npages; i++) { m = bp->b_pages[i]; if (m == bogus_page) { + if (bogus == false) { + bogus = true; + VM_OBJECT_RLOCK(obj); + } m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i); if (!m) panic("vfs_unbusy_pages: page missing\n"); @@ -4480,8 +4473,9 @@ } vm_page_sunbusy(m); } + if (bogus) + VM_OBJECT_RUNLOCK(obj); vm_object_pip_wakeupn(obj, bp->b_npages); - VM_OBJECT_WUNLOCK(obj); } /* @@ -4558,7 +4552,6 @@ { int i; - VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object); for (i = 0; i < bp->b_npages; i++) vm_page_busy_acquire(bp->b_pages[i], VM_ALLOC_SBUSY); } @@ -4568,7 +4561,6 @@ { int i; - VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object); for (i = 0; i < bp->b_npages; i++) vm_page_sunbusy(bp->b_pages[i]); } @@ -4601,13 +4593,12 @@ foff = bp->b_offset; KASSERT(bp->b_offset != NOOFFSET, ("vfs_busy_pages: no buffer offset")); - VM_OBJECT_WLOCK(obj); if ((bp->b_flags & B_CLUSTER) == 0) { vm_object_pip_add(obj, bp->b_npages); vfs_busy_pages_acquire(bp); } if (bp->b_bufsize != 0) - vfs_setdirty_locked_object(bp); + vfs_setdirty_range(bp); bogus = false; for (i = 0; i < bp->b_npages; i++) { m = bp->b_pages[i]; @@ -4638,7 +4629,6 @@ } foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; } - VM_OBJECT_WUNLOCK(obj); if (bogus && buf_mapped(bp)) { BUF_CHECK_MAPPED(bp); pmap_qenter(trunc_page((vm_offset_t)bp->b_data), @@ -4671,8 +4661,6 @@ base += (bp->b_offset & PAGE_MASK); n = PAGE_SIZE - (base & PAGE_MASK); - VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); - /* * Busy may not be strictly necessary here because the pages are * unlikely to be fully valid and the vnode lock will synchronize @@ -4690,7 +4678,6 @@ n = PAGE_SIZE; } vfs_busy_pages_release(bp); - VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); } /* @@ -4716,22 +4703,7 @@ } bp->b_flags &= ~B_INVAL; bp->b_ioflags &= ~BIO_ERROR; - VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); vfs_busy_pages_acquire(bp); - if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) && - (bp->b_offset & PAGE_MASK) == 0) { - if (bp->b_pages[0] == bogus_page) - goto unlock; - mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1; - VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[0]->object); - if ((bp->b_pages[0]->valid & mask) == mask) - goto unlock; - if ((bp->b_pages[0]->valid & mask) == 0) { - pmap_zero_page_area(bp->b_pages[0], 0, bp->b_bufsize); - bp->b_pages[0]->valid |= mask; - goto unlock; - } - } sa = bp->b_offset & PAGE_MASK; slide = 0; for (i = 0; i < bp->b_npages; i++, sa = 0) { @@ -4743,7 +4715,6 @@ continue; j = sa / DEV_BSIZE; mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j; - VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[i]->object); if ((bp->b_pages[i]->valid & mask) == mask) continue; if ((bp->b_pages[i]->valid & mask) == 0) @@ -4756,11 +4727,10 @@ } } } - bp->b_pages[i]->valid |= mask; + vm_page_set_valid_range(bp->b_pages[i], j * DEV_BSIZE, + roundup2(ea - sa, DEV_BSIZE)); } -unlock: vfs_busy_pages_release(bp); - VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); bp->b_resid = 0; } @@ -5171,11 +5141,9 @@ br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) ? GB_UNMAPPED : 0; - VM_OBJECT_WLOCK(object); again: for (i = 0; i < count; i++) vm_page_busy_downgrade(ma[i]); - VM_OBJECT_WUNLOCK(object); lbnp = -1; for (i = 0; i < count; i++) { @@ -5234,11 +5202,9 @@ vm_page_all_valid(m) || i == count - 1, ("buf %d %p invalid", i, m)); if (i == count - 1 && lpart) { - VM_OBJECT_WLOCK(object); if (!vm_page_none_valid(m) && !vm_page_all_valid(m)) vm_page_zero_invalid(m, TRUE); - VM_OBJECT_WUNLOCK(object); } next_page:; } @@ -5266,9 +5232,9 @@ if (!vm_page_all_valid(ma[i])) redo = true; } + VM_OBJECT_WUNLOCK(object); if (redo && error == 0) goto again; - VM_OBJECT_WUNLOCK(object); return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK); } Index: sys/kern/vfs_cluster.c =================================================================== --- sys/kern/vfs_cluster.c +++ sys/kern/vfs_cluster.c @@ -417,11 +417,9 @@ inc = btodb(size); for (bn = blkno, i = 0; i < run; ++i, bn += inc) { if (i == 0) { - VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); vm_object_pip_add(tbp->b_bufobj->bo_object, tbp->b_npages); vfs_busy_pages_acquire(tbp); - VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); } else { if ((bp->b_npages * PAGE_SIZE) + round_page(size) > vp->v_mount->mnt_iosize_max) { @@ -458,13 +456,11 @@ */ off = tbp->b_offset; tsize = size; - VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); for (j = 0; tsize > 0; j++) { toff = off & PAGE_MASK; tinc = tsize; if (toff + tinc > PAGE_SIZE) tinc = PAGE_SIZE - toff; - VM_OBJECT_ASSERT_WLOCKED(tbp->b_pages[j]->object); if (vm_page_trysbusy(tbp->b_pages[j]) == 0) break; if ((tbp->b_pages[j]->valid & @@ -482,11 +478,9 @@ j); for (k = 0; k < j; k++) vm_page_sunbusy(tbp->b_pages[k]); - VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); bqrelse(tbp); break; } - VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); /* * Set a read-ahead mark as appropriate @@ -506,7 +500,6 @@ if (tbp->b_blkno == tbp->b_lblkno) { tbp->b_blkno = bn; } else if (tbp->b_blkno != bn) { - VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); goto clean_sbusy; } } @@ -517,9 +510,9 @@ BUF_KERNPROC(tbp); TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head, tbp, b_cluster.cluster_entry); - VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); for (j = 0; j < tbp->b_npages; j += 1) { vm_page_t m; + m = tbp->b_pages[j]; if ((bp->b_npages == 0) || (bp->b_pages[bp->b_npages-1] != m)) { @@ -529,7 +522,7 @@ if (vm_page_all_valid(m)) tbp->b_pages[j] = bogus_page; } - VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); + /* * Don't inherit tbp->b_bufsize as it may be larger due to * a non-page-aligned size. Instead just aggregate using @@ -547,13 +540,10 @@ * Fully valid pages in the cluster are already good and do not need * to be re-read from disk. Replace the page with bogus_page */ - VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); for (j = 0; j < bp->b_npages; j++) { - VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[j]->object); if (vm_page_all_valid(bp->b_pages[j])) bp->b_pages[j] = bogus_page; } - VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); if (bp->b_bufsize > bp->b_kvasize) panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n", bp->b_bufsize, bp->b_kvasize); @@ -980,7 +970,6 @@ if (tbp->b_flags & B_VMIO) { vm_page_t m; - VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); if (i == 0) { vfs_busy_pages_acquire(tbp); } else { /* if not first buffer */ @@ -990,8 +979,6 @@ for (j--; j >= 0; j--) vm_page_sunbusy( tbp->b_pages[j]); - VM_OBJECT_WUNLOCK( - tbp->b_object); bqrelse(tbp); goto finishcluster; } @@ -1007,7 +994,6 @@ bp->b_npages++; } } - VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); } bp->b_bcount += size; bp->b_bufsize += size;