Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_bio.c
Show First 20 Lines • Show All 949 Lines • ▼ Show 20 Lines | |||||
* valid data. | * valid data. | ||||
*/ | */ | ||||
static __inline void | static __inline void | ||||
vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off, | vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off, | ||||
vm_offset_t size, vm_page_t m) | vm_offset_t size, vm_page_t m) | ||||
{ | { | ||||
VM_OBJECT_ASSERT_LOCKED(m->object); | VM_OBJECT_ASSERT_LOCKED(m->object); | ||||
/* | |||||
* This function and its results are protected by higher level | |||||
* synchronization requiring vnode and buf locks to page in and | |||||
* validate pages. | |||||
*/ | |||||
if (bp->b_flags & B_CACHE) { | if (bp->b_flags & B_CACHE) { | ||||
int base = (foff + off) & PAGE_MASK; | int base = (foff + off) & PAGE_MASK; | ||||
if (vm_page_is_valid(m, base, size) == 0) | if (vm_page_is_valid(m, base, size) == 0) | ||||
bp->b_flags &= ~B_CACHE; | bp->b_flags &= ~B_CACHE; | ||||
} | } | ||||
} | } | ||||
/* Wake up the buffer daemon if necessary */ | /* Wake up the buffer daemon if necessary */ | ||||
▲ Show 20 Lines • Show All 3,654 Lines • ▼ Show 20 Lines | for (i = 0; i < bp->b_npages; i++) { | ||||
* I/O from overwriting potentially dirty VM-backed | * I/O from overwriting potentially dirty VM-backed | ||||
* pages. XXX bogus page replacement is, uh, bogus. | * pages. XXX bogus page replacement is, uh, bogus. | ||||
* It may not work properly with small-block devices. | * It may not work properly with small-block devices. | ||||
* We need to find a better way. | * We need to find a better way. | ||||
*/ | */ | ||||
if (clear_modify) { | if (clear_modify) { | ||||
pmap_remove_write(m); | pmap_remove_write(m); | ||||
vfs_page_set_validclean(bp, foff, m); | vfs_page_set_validclean(bp, foff, m); | ||||
} else if (m->valid == VM_PAGE_BITS_ALL && | } else if (vm_page_all_valid(m) && | ||||
(bp->b_flags & B_CACHE) == 0) { | (bp->b_flags & B_CACHE) == 0) { | ||||
bp->b_pages[i] = bogus_page; | bp->b_pages[i] = bogus_page; | ||||
bogus = true; | bogus = true; | ||||
} | } | ||||
foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; | foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; | ||||
} | } | ||||
VM_OBJECT_WUNLOCK(obj); | VM_OBJECT_WUNLOCK(obj); | ||||
if (bogus && buf_mapped(bp)) { | if (bogus && buf_mapped(bp)) { | ||||
Show All 24 Lines | vfs_bio_set_valid(struct buf *bp, int base, int size) | ||||
* Fixup base to be relative to beginning of first page. | * Fixup base to be relative to beginning of first page. | ||||
* Set initial n to be the maximum number of bytes in the | * Set initial n to be the maximum number of bytes in the | ||||
* first page that can be validated. | * first page that can be validated. | ||||
*/ | */ | ||||
base += (bp->b_offset & PAGE_MASK); | base += (bp->b_offset & PAGE_MASK); | ||||
n = PAGE_SIZE - (base & PAGE_MASK); | n = PAGE_SIZE - (base & PAGE_MASK); | ||||
VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); | VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); | ||||
/* | |||||
* Busy may not be strictly necessary here because the pages are | |||||
* unlikely to be fully valid and the vnode lock will synchronize | |||||
* their access via getpages. It is grabbed for consistency with | |||||
* other page validation. | |||||
*/ | |||||
vfs_busy_pages_acquire(bp); | |||||
for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) { | for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) { | ||||
m = bp->b_pages[i]; | m = bp->b_pages[i]; | ||||
if (n > size) | if (n > size) | ||||
n = size; | n = size; | ||||
vm_page_set_valid_range(m, base & PAGE_MASK, n); | vm_page_set_valid_range(m, base & PAGE_MASK, n); | ||||
base += n; | base += n; | ||||
size -= n; | size -= n; | ||||
n = PAGE_SIZE; | n = PAGE_SIZE; | ||||
} | } | ||||
vfs_busy_pages_release(bp); | |||||
VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); | VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); | ||||
} | } | ||||
/* | /* | ||||
* vfs_bio_clrbuf: | * vfs_bio_clrbuf: | ||||
* | * | ||||
* If the specified buffer is a non-VMIO buffer, clear the entire | * If the specified buffer is a non-VMIO buffer, clear the entire | ||||
* buffer. If the specified buffer is a VMIO buffer, clear and | * buffer. If the specified buffer is a VMIO buffer, clear and | ||||
Show All 11 Lines | vfs_bio_clrbuf(struct buf *bp) | ||||
if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) { | if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) { | ||||
clrbuf(bp); | clrbuf(bp); | ||||
return; | return; | ||||
} | } | ||||
bp->b_flags &= ~B_INVAL; | bp->b_flags &= ~B_INVAL; | ||||
bp->b_ioflags &= ~BIO_ERROR; | bp->b_ioflags &= ~BIO_ERROR; | ||||
VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); | VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); | ||||
vfs_busy_pages_acquire(bp); | |||||
if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) && | if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) && | ||||
(bp->b_offset & PAGE_MASK) == 0) { | (bp->b_offset & PAGE_MASK) == 0) { | ||||
if (bp->b_pages[0] == bogus_page) | if (bp->b_pages[0] == bogus_page) | ||||
goto unlock; | goto unlock; | ||||
mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1; | mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1; | ||||
VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[0]->object); | VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[0]->object); | ||||
if ((bp->b_pages[0]->valid & mask) == mask) | if ((bp->b_pages[0]->valid & mask) == mask) | ||||
goto unlock; | goto unlock; | ||||
Show All 25 Lines | else { | ||||
pmap_zero_page_area(bp->b_pages[i], | pmap_zero_page_area(bp->b_pages[i], | ||||
sa, DEV_BSIZE); | sa, DEV_BSIZE); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
bp->b_pages[i]->valid |= mask; | bp->b_pages[i]->valid |= mask; | ||||
} | } | ||||
unlock: | unlock: | ||||
vfs_busy_pages_release(bp); | |||||
VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); | VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); | ||||
bp->b_resid = 0; | bp->b_resid = 0; | ||||
} | } | ||||
void | void | ||||
vfs_bio_bzero_buf(struct buf *bp, int base, int size) | vfs_bio_bzero_buf(struct buf *bp, int base, int size) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
▲ Show 20 Lines • Show All 415 Lines • ▼ Show 20 Lines | for (i = 0; i < count; i++) { | ||||
* owned, which together allow for the pages' | * owned, which together allow for the pages' | ||||
* invalidation. The racy test for validity avoids | * invalidation. The racy test for validity avoids | ||||
* useless creation of the buffer for the most typical | * useless creation of the buffer for the most typical | ||||
* case when invalidation is not used in redo or for | * case when invalidation is not used in redo or for | ||||
* parallel read. The shared->excl upgrade loop at | * parallel read. The shared->excl upgrade loop at | ||||
* the end of the function catches the race in a | * the end of the function catches the race in a | ||||
* reliable way (protected by the object lock). | * reliable way (protected by the object lock). | ||||
*/ | */ | ||||
if (m->valid == VM_PAGE_BITS_ALL) | if (vm_page_all_valid(m)) | ||||
continue; | continue; | ||||
poff = IDX_TO_OFF(m->pindex); | poff = IDX_TO_OFF(m->pindex); | ||||
poffe = MIN(poff + PAGE_SIZE, object->un_pager.vnp.vnp_size); | poffe = MIN(poff + PAGE_SIZE, object->un_pager.vnp.vnp_size); | ||||
for (; poff < poffe; poff += bsize) { | for (; poff < poffe; poff += bsize) { | ||||
lbn = get_lblkno(vp, poff); | lbn = get_lblkno(vp, poff); | ||||
if (lbn == lbnp) | if (lbn == lbnp) | ||||
goto next_page; | goto next_page; | ||||
Show All 13 Lines | for (; poff < poffe; poff += bsize) { | ||||
* buffer to do real read on next | * buffer to do real read on next | ||||
* bread() after redo. | * bread() after redo. | ||||
* | * | ||||
* Otherwise B_RELBUF is not strictly | * Otherwise B_RELBUF is not strictly | ||||
* necessary, enable to reduce buf | * necessary, enable to reduce buf | ||||
* cache pressure. | * cache pressure. | ||||
*/ | */ | ||||
if (buf_pager_relbuf || | if (buf_pager_relbuf || | ||||
m->valid != VM_PAGE_BITS_ALL) | !vm_page_all_valid(m)) | ||||
bp->b_flags |= B_RELBUF; | bp->b_flags |= B_RELBUF; | ||||
bp->b_flags &= ~B_NOCACHE; | bp->b_flags &= ~B_NOCACHE; | ||||
brelse(bp); | brelse(bp); | ||||
} else { | } else { | ||||
bqrelse(bp); | bqrelse(bp); | ||||
} | } | ||||
} | } | ||||
KASSERT(1 /* racy, enable for debugging */ || | KASSERT(1 /* racy, enable for debugging */ || | ||||
m->valid == VM_PAGE_BITS_ALL || i == count - 1, | vm_page_all_valid(m) || i == count - 1, | ||||
("buf %d %p invalid", i, m)); | ("buf %d %p invalid", i, m)); | ||||
if (i == count - 1 && lpart) { | if (i == count - 1 && lpart) { | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
if (m->valid != 0 && | if (!vm_page_none_valid(m) && | ||||
m->valid != VM_PAGE_BITS_ALL) | !vm_page_all_valid(m)) | ||||
vm_page_zero_invalid(m, TRUE); | vm_page_zero_invalid(m, TRUE); | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
} | } | ||||
next_page:; | next_page:; | ||||
} | } | ||||
end_pages: | end_pages: | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
Show All 10 Lines | for (i = 0; i < count; i++) { | ||||
* Recheck the valid bits and re-read as needed. | * Recheck the valid bits and re-read as needed. | ||||
* | * | ||||
* Note that the last page is made fully valid in the | * Note that the last page is made fully valid in the | ||||
* read loop, and partial validity for the page at | * read loop, and partial validity for the page at | ||||
* index count - 1 could mean that the page was | * index count - 1 could mean that the page was | ||||
* invalidated or removed, so we must restart for | * invalidated or removed, so we must restart for | ||||
* safety as well. | * safety as well. | ||||
*/ | */ | ||||
if (ma[i]->valid != VM_PAGE_BITS_ALL) | if (!vm_page_all_valid(ma[i])) | ||||
redo = true; | redo = true; | ||||
} | } | ||||
if (redo && error == 0) | if (redo && error == 0) | ||||
goto again; | goto again; | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK); | return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 191 Lines • Show Last 20 Lines |