Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_cluster.c
Show First 20 Lines • Show All 411 Lines • ▼ Show 20 Lines | cluster_rbuild(struct vnode *vp, u_quad_t filesize, daddr_t lbn, | ||||
bp->b_bcount = 0; | bp->b_bcount = 0; | ||||
bp->b_bufsize = 0; | bp->b_bufsize = 0; | ||||
bp->b_npages = 0; | bp->b_npages = 0; | ||||
inc = btodb(size); | inc = btodb(size); | ||||
for (bn = blkno, i = 0; i < run; ++i, bn += inc) { | for (bn = blkno, i = 0; i < run; ++i, bn += inc) { | ||||
if (i == 0) { | if (i == 0) { | ||||
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); | |||||
vm_object_pip_add(tbp->b_bufobj->bo_object, | vm_object_pip_add(tbp->b_bufobj->bo_object, | ||||
tbp->b_npages); | tbp->b_npages); | ||||
vfs_busy_pages_acquire(tbp); | vfs_busy_pages_acquire(tbp); | ||||
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); | |||||
} else { | } else { | ||||
if ((bp->b_npages * PAGE_SIZE) + | if ((bp->b_npages * PAGE_SIZE) + | ||||
round_page(size) > vp->v_mount->mnt_iosize_max) { | round_page(size) > vp->v_mount->mnt_iosize_max) { | ||||
break; | break; | ||||
} | } | ||||
tbp = getblk(vp, lbn + i, size, 0, 0, GB_LOCK_NOWAIT | | tbp = getblk(vp, lbn + i, size, 0, 0, GB_LOCK_NOWAIT | | ||||
(gbflags & GB_UNMAPPED)); | (gbflags & GB_UNMAPPED)); | ||||
Show All 20 Lines | if (i == 0) { | ||||
/* | /* | ||||
* The buffer must be completely invalid in order to | * The buffer must be completely invalid in order to | ||||
* take part in the cluster. If it is partially valid | * take part in the cluster. If it is partially valid | ||||
* then we stop. | * then we stop. | ||||
*/ | */ | ||||
off = tbp->b_offset; | off = tbp->b_offset; | ||||
tsize = size; | tsize = size; | ||||
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); | |||||
for (j = 0; tsize > 0; j++) { | for (j = 0; tsize > 0; j++) { | ||||
toff = off & PAGE_MASK; | toff = off & PAGE_MASK; | ||||
tinc = tsize; | tinc = tsize; | ||||
if (toff + tinc > PAGE_SIZE) | if (toff + tinc > PAGE_SIZE) | ||||
tinc = PAGE_SIZE - toff; | tinc = PAGE_SIZE - toff; | ||||
VM_OBJECT_ASSERT_WLOCKED(tbp->b_pages[j]->object); | |||||
if (vm_page_trysbusy(tbp->b_pages[j]) == 0) | if (vm_page_trysbusy(tbp->b_pages[j]) == 0) | ||||
break; | break; | ||||
if ((tbp->b_pages[j]->valid & | if ((tbp->b_pages[j]->valid & | ||||
vm_page_bits(toff, tinc)) != 0) { | vm_page_bits(toff, tinc)) != 0) { | ||||
vm_page_sunbusy(tbp->b_pages[j]); | vm_page_sunbusy(tbp->b_pages[j]); | ||||
break; | break; | ||||
} | } | ||||
vm_object_pip_add(tbp->b_bufobj->bo_object, 1); | vm_object_pip_add(tbp->b_bufobj->bo_object, 1); | ||||
off += tinc; | off += tinc; | ||||
tsize -= tinc; | tsize -= tinc; | ||||
} | } | ||||
if (tsize > 0) { | if (tsize > 0) { | ||||
clean_sbusy: | clean_sbusy: | ||||
vm_object_pip_wakeupn(tbp->b_bufobj->bo_object, | vm_object_pip_wakeupn(tbp->b_bufobj->bo_object, | ||||
j); | j); | ||||
for (k = 0; k < j; k++) | for (k = 0; k < j; k++) | ||||
vm_page_sunbusy(tbp->b_pages[k]); | vm_page_sunbusy(tbp->b_pages[k]); | ||||
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); | |||||
bqrelse(tbp); | bqrelse(tbp); | ||||
break; | break; | ||||
} | } | ||||
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); | |||||
/* | /* | ||||
* Set a read-ahead mark as appropriate | * Set a read-ahead mark as appropriate | ||||
*/ | */ | ||||
if ((fbp && (i == 1)) || (i == (run - 1))) | if ((fbp && (i == 1)) || (i == (run - 1))) | ||||
tbp->b_flags |= B_RAM; | tbp->b_flags |= B_RAM; | ||||
/* | /* | ||||
* Set the buffer up for an async read (XXX should | * Set the buffer up for an async read (XXX should | ||||
* we do this only if we do not wind up brelse()ing?). | * we do this only if we do not wind up brelse()ing?). | ||||
* Set the block number if it isn't set, otherwise | * Set the block number if it isn't set, otherwise | ||||
* if it is make sure it matches the block number we | * if it is make sure it matches the block number we | ||||
* expect. | * expect. | ||||
*/ | */ | ||||
tbp->b_flags |= B_ASYNC; | tbp->b_flags |= B_ASYNC; | ||||
tbp->b_iocmd = BIO_READ; | tbp->b_iocmd = BIO_READ; | ||||
if (tbp->b_blkno == tbp->b_lblkno) { | if (tbp->b_blkno == tbp->b_lblkno) { | ||||
tbp->b_blkno = bn; | tbp->b_blkno = bn; | ||||
} else if (tbp->b_blkno != bn) { | } else if (tbp->b_blkno != bn) { | ||||
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); | |||||
goto clean_sbusy; | goto clean_sbusy; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* XXX fbp from caller may not be B_ASYNC, but we are going | * XXX fbp from caller may not be B_ASYNC, but we are going | ||||
* to biodone() it in cluster_callback() anyway | * to biodone() it in cluster_callback() anyway | ||||
*/ | */ | ||||
BUF_KERNPROC(tbp); | BUF_KERNPROC(tbp); | ||||
TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head, | TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head, | ||||
tbp, b_cluster.cluster_entry); | tbp, b_cluster.cluster_entry); | ||||
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); | |||||
for (j = 0; j < tbp->b_npages; j += 1) { | for (j = 0; j < tbp->b_npages; j += 1) { | ||||
vm_page_t m; | vm_page_t m; | ||||
m = tbp->b_pages[j]; | m = tbp->b_pages[j]; | ||||
if ((bp->b_npages == 0) || | if ((bp->b_npages == 0) || | ||||
(bp->b_pages[bp->b_npages-1] != m)) { | (bp->b_pages[bp->b_npages-1] != m)) { | ||||
bp->b_pages[bp->b_npages] = m; | bp->b_pages[bp->b_npages] = m; | ||||
bp->b_npages++; | bp->b_npages++; | ||||
} | } | ||||
if (vm_page_all_valid(m)) | if (vm_page_all_valid(m)) | ||||
tbp->b_pages[j] = bogus_page; | tbp->b_pages[j] = bogus_page; | ||||
} | } | ||||
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); | |||||
/* | /* | ||||
* Don't inherit tbp->b_bufsize as it may be larger due to | * Don't inherit tbp->b_bufsize as it may be larger due to | ||||
* a non-page-aligned size. Instead just aggregate using | * a non-page-aligned size. Instead just aggregate using | ||||
* 'size'. | * 'size'. | ||||
*/ | */ | ||||
if (tbp->b_bcount != size) | if (tbp->b_bcount != size) | ||||
printf("warning: tbp->b_bcount wrong %ld vs %ld\n", tbp->b_bcount, size); | printf("warning: tbp->b_bcount wrong %ld vs %ld\n", tbp->b_bcount, size); | ||||
if (tbp->b_bufsize != size) | if (tbp->b_bufsize != size) | ||||
printf("warning: tbp->b_bufsize wrong %ld vs %ld\n", tbp->b_bufsize, size); | printf("warning: tbp->b_bufsize wrong %ld vs %ld\n", tbp->b_bufsize, size); | ||||
bp->b_bcount += size; | bp->b_bcount += size; | ||||
bp->b_bufsize += size; | bp->b_bufsize += size; | ||||
} | } | ||||
/* | /* | ||||
* Fully valid pages in the cluster are already good and do not need | * Fully valid pages in the cluster are already good and do not need | ||||
* to be re-read from disk. Replace the page with bogus_page | * to be re-read from disk. Replace the page with bogus_page | ||||
*/ | */ | ||||
VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); | |||||
for (j = 0; j < bp->b_npages; j++) { | for (j = 0; j < bp->b_npages; j++) { | ||||
VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[j]->object); | |||||
kib: I think this (and similar) asserts should be replaced by checks that bo_object == b_pages[i]… | |||||
if (vm_page_all_valid(bp->b_pages[j])) | if (vm_page_all_valid(bp->b_pages[j])) | ||||
bp->b_pages[j] = bogus_page; | bp->b_pages[j] = bogus_page; | ||||
} | } | ||||
VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); | |||||
if (bp->b_bufsize > bp->b_kvasize) | if (bp->b_bufsize > bp->b_kvasize) | ||||
panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n", | panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n", | ||||
bp->b_bufsize, bp->b_kvasize); | bp->b_bufsize, bp->b_kvasize); | ||||
if (buf_mapped(bp)) { | if (buf_mapped(bp)) { | ||||
pmap_qenter(trunc_page((vm_offset_t) bp->b_data), | pmap_qenter(trunc_page((vm_offset_t) bp->b_data), | ||||
(vm_page_t *)bp->b_pages, bp->b_npages); | (vm_page_t *)bp->b_pages, bp->b_npages); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 410 Lines • ▼ Show 20 Lines | for (i = 0; i < len; ++i, ++start_lbn) { | ||||
* block size may not be page-aligned it is possible | * block size may not be page-aligned it is possible | ||||
* for a page to be shared between two buffers. We | * for a page to be shared between two buffers. We | ||||
* have to get rid of the duplication when building | * have to get rid of the duplication when building | ||||
* the cluster. | * the cluster. | ||||
*/ | */ | ||||
if (tbp->b_flags & B_VMIO) { | if (tbp->b_flags & B_VMIO) { | ||||
vm_page_t m; | vm_page_t m; | ||||
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); | |||||
if (i == 0) { | if (i == 0) { | ||||
vfs_busy_pages_acquire(tbp); | vfs_busy_pages_acquire(tbp); | ||||
} else { /* if not first buffer */ | } else { /* if not first buffer */ | ||||
for (j = 0; j < tbp->b_npages; j += 1) { | for (j = 0; j < tbp->b_npages; j += 1) { | ||||
m = tbp->b_pages[j]; | m = tbp->b_pages[j]; | ||||
if (vm_page_trysbusy(m) == 0) { | if (vm_page_trysbusy(m) == 0) { | ||||
for (j--; j >= 0; j--) | for (j--; j >= 0; j--) | ||||
vm_page_sunbusy( | vm_page_sunbusy( | ||||
tbp->b_pages[j]); | tbp->b_pages[j]); | ||||
VM_OBJECT_WUNLOCK( | |||||
tbp->b_object); | |||||
bqrelse(tbp); | bqrelse(tbp); | ||||
goto finishcluster; | goto finishcluster; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
vm_object_pip_add(tbp->b_bufobj->bo_object, | vm_object_pip_add(tbp->b_bufobj->bo_object, | ||||
tbp->b_npages); | tbp->b_npages); | ||||
for (j = 0; j < tbp->b_npages; j += 1) { | for (j = 0; j < tbp->b_npages; j += 1) { | ||||
m = tbp->b_pages[j]; | m = tbp->b_pages[j]; | ||||
if ((bp->b_npages == 0) || | if ((bp->b_npages == 0) || | ||||
(bp->b_pages[bp->b_npages - 1] != m)) { | (bp->b_pages[bp->b_npages - 1] != m)) { | ||||
bp->b_pages[bp->b_npages] = m; | bp->b_pages[bp->b_npages] = m; | ||||
bp->b_npages++; | bp->b_npages++; | ||||
} | } | ||||
} | } | ||||
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); | |||||
} | } | ||||
bp->b_bcount += size; | bp->b_bcount += size; | ||||
bp->b_bufsize += size; | bp->b_bufsize += size; | ||||
/* | /* | ||||
* If any of the clustered buffers have their | * If any of the clustered buffers have their | ||||
* B_BARRIER flag set, transfer that request to | * B_BARRIER flag set, transfer that request to | ||||
* the cluster. | * the cluster. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 63 Lines • Show Last 20 Lines |
I think this (and similar) asserts should be replaced by checks that bo_object == b_pages[i]->object.