Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_bio.c
Show First 20 Lines • Show All 2,938 Lines • ▼ Show 20 Lines | vfs_vmio_iodone(struct buf *bp) | ||||
if (bogus && buf_mapped(bp)) { | if (bogus && buf_mapped(bp)) { | ||||
BUF_CHECK_MAPPED(bp); | BUF_CHECK_MAPPED(bp); | ||||
pmap_qenter(trunc_page((vm_offset_t)bp->b_data), | pmap_qenter(trunc_page((vm_offset_t)bp->b_data), | ||||
bp->b_pages, bp->b_npages); | bp->b_pages, bp->b_npages); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Unwire a page held by a buf and either free it or update the page queues to | |||||
* reflect its recent use. | |||||
*/ | |||||
static void | |||||
vfs_vmio_unwire(struct buf *bp, vm_page_t m) | |||||
{ | |||||
bool freed; | |||||
vm_page_lock(m); | |||||
if (vm_page_unwire_noq(m)) { | |||||
if ((bp->b_flags & B_DIRECT) != 0) | |||||
freed = vm_page_try_to_free(m); | |||||
else | |||||
freed = false; | |||||
if (!freed) { | |||||
/* | |||||
* Use a racy check of the valid bits to determine | |||||
* whether we can accelerate reclamation of the page. | |||||
* The valid bits will be stable unless the page is | |||||
* being mapped or is referenced by multiple buffers, | |||||
* and in those cases we expect races to be rare. At | |||||
* worst we will either accelerate reclamation of a | |||||
* valid page and violate LRU, or unnecessarily defer | |||||
* reclamation of an invalid page. | |||||
* | |||||
* The B_NOREUSE flag marks data that is not expected to | |||||
* be reused, so accelerate reclamation in that case | |||||
* too. Otherwise, maintain LRU. | |||||
*/ | |||||
if (m->valid == 0 || (bp->b_flags & B_NOREUSE) != 0) | |||||
vm_page_deactivate_noreuse(m); | |||||
else if (vm_page_active(m)) | |||||
vm_page_reference(m); | |||||
else | |||||
vm_page_deactivate(m); | |||||
} | |||||
} | |||||
vm_page_unlock(m); | |||||
} | |||||
/* | |||||
* Perform page invalidation when a buffer is released. The fully invalid | * Perform page invalidation when a buffer is released. The fully invalid | ||||
* pages will be reclaimed later in vfs_vmio_truncate(). | * pages will be reclaimed later in vfs_vmio_truncate(). | ||||
*/ | */ | ||||
static void | static void | ||||
vfs_vmio_invalidate(struct buf *bp) | vfs_vmio_invalidate(struct buf *bp) | ||||
{ | { | ||||
vm_object_t obj; | vm_object_t obj; | ||||
vm_page_t m; | vm_page_t m; | ||||
int i, resid, poffset, presid; | int flags, i, resid, poffset, presid; | ||||
if (buf_mapped(bp)) { | if (buf_mapped(bp)) { | ||||
BUF_CHECK_MAPPED(bp); | BUF_CHECK_MAPPED(bp); | ||||
pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages); | pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages); | ||||
} else | } else | ||||
BUF_CHECK_UNMAPPED(bp); | BUF_CHECK_UNMAPPED(bp); | ||||
/* | /* | ||||
* Get the base offset and length of the buffer. Note that | * Get the base offset and length of the buffer. Note that | ||||
* in the VMIO case if the buffer block size is not | * in the VMIO case if the buffer block size is not | ||||
* page-aligned then b_data pointer may not be page-aligned. | * page-aligned then b_data pointer may not be page-aligned. | ||||
* But our b_pages[] array *IS* page aligned. | * But our b_pages[] array *IS* page aligned. | ||||
* | * | ||||
* block sizes less then DEV_BSIZE (usually 512) are not | * block sizes less then DEV_BSIZE (usually 512) are not | ||||
* supported due to the page granularity bits (m->valid, | * supported due to the page granularity bits (m->valid, | ||||
* m->dirty, etc...). | * m->dirty, etc...). | ||||
* | * | ||||
* See man buf(9) for more information | * See man buf(9) for more information | ||||
*/ | */ | ||||
flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0; | |||||
obj = bp->b_bufobj->bo_object; | obj = bp->b_bufobj->bo_object; | ||||
resid = bp->b_bufsize; | resid = bp->b_bufsize; | ||||
poffset = bp->b_offset & PAGE_MASK; | poffset = bp->b_offset & PAGE_MASK; | ||||
VM_OBJECT_WLOCK(obj); | VM_OBJECT_WLOCK(obj); | ||||
for (i = 0; i < bp->b_npages; i++) { | for (i = 0; i < bp->b_npages; i++) { | ||||
m = bp->b_pages[i]; | m = bp->b_pages[i]; | ||||
if (m == bogus_page) | if (m == bogus_page) | ||||
panic("vfs_vmio_invalidate: Unexpected bogus page."); | panic("vfs_vmio_invalidate: Unexpected bogus page."); | ||||
bp->b_pages[i] = NULL; | bp->b_pages[i] = NULL; | ||||
presid = resid > (PAGE_SIZE - poffset) ? | presid = resid > (PAGE_SIZE - poffset) ? | ||||
(PAGE_SIZE - poffset) : resid; | (PAGE_SIZE - poffset) : resid; | ||||
KASSERT(presid >= 0, ("brelse: extra page")); | KASSERT(presid >= 0, ("brelse: extra page")); | ||||
while (vm_page_xbusied(m)) { | while (vm_page_xbusied(m)) { | ||||
vm_page_lock(m); | vm_page_lock(m); | ||||
VM_OBJECT_WUNLOCK(obj); | VM_OBJECT_WUNLOCK(obj); | ||||
vm_page_busy_sleep(m, "mbncsh", true); | vm_page_busy_sleep(m, "mbncsh", true); | ||||
VM_OBJECT_WLOCK(obj); | VM_OBJECT_WLOCK(obj); | ||||
} | } | ||||
if (pmap_page_wired_mappings(m) == 0) | if (pmap_page_wired_mappings(m) == 0) | ||||
vm_page_set_invalid(m, poffset, presid); | vm_page_set_invalid(m, poffset, presid); | ||||
vfs_vmio_unwire(bp, m); | vm_page_release_locked(m, flags); | ||||
resid -= presid; | resid -= presid; | ||||
poffset = 0; | poffset = 0; | ||||
} | } | ||||
VM_OBJECT_WUNLOCK(obj); | VM_OBJECT_WUNLOCK(obj); | ||||
bp->b_npages = 0; | bp->b_npages = 0; | ||||
} | } | ||||
/* | /* | ||||
* Page-granular truncation of an existing VMIO buffer. | * Page-granular truncation of an existing VMIO buffer. | ||||
*/ | */ | ||||
static void | static void | ||||
vfs_vmio_truncate(struct buf *bp, int desiredpages) | vfs_vmio_truncate(struct buf *bp, int desiredpages) | ||||
{ | { | ||||
vm_object_t obj; | vm_object_t obj; | ||||
vm_page_t m; | vm_page_t m; | ||||
int i; | int flags, i; | ||||
if (bp->b_npages == desiredpages) | if (bp->b_npages == desiredpages) | ||||
return; | return; | ||||
if (buf_mapped(bp)) { | if (buf_mapped(bp)) { | ||||
BUF_CHECK_MAPPED(bp); | BUF_CHECK_MAPPED(bp); | ||||
pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) + | pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) + | ||||
(desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages); | (desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages); | ||||
} else | } else | ||||
BUF_CHECK_UNMAPPED(bp); | BUF_CHECK_UNMAPPED(bp); | ||||
/* | /* | ||||
* The object lock is needed only if we will attempt to free pages. | * The object lock is needed only if we will attempt to free pages. | ||||
*/ | */ | ||||
obj = (bp->b_flags & B_DIRECT) != 0 ? bp->b_bufobj->bo_object : NULL; | flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0; | ||||
if (obj != NULL) | if ((bp->b_flags & B_DIRECT) != 0) { | ||||
flags |= VPR_TRYFREE; | |||||
obj = bp->b_bufobj->bo_object; | |||||
VM_OBJECT_WLOCK(obj); | VM_OBJECT_WLOCK(obj); | ||||
} else { | |||||
obj = NULL; | |||||
} | |||||
for (i = desiredpages; i < bp->b_npages; i++) { | for (i = desiredpages; i < bp->b_npages; i++) { | ||||
m = bp->b_pages[i]; | m = bp->b_pages[i]; | ||||
KASSERT(m != bogus_page, ("allocbuf: bogus page found")); | KASSERT(m != bogus_page, ("allocbuf: bogus page found")); | ||||
bp->b_pages[i] = NULL; | bp->b_pages[i] = NULL; | ||||
vfs_vmio_unwire(bp, m); | if (obj != NULL) | ||||
vm_page_release_locked(m, flags); | |||||
else | |||||
vm_page_release(m, flags); | |||||
} | } | ||||
if (obj != NULL) | if (obj != NULL) | ||||
VM_OBJECT_WUNLOCK(obj); | VM_OBJECT_WUNLOCK(obj); | ||||
bp->b_npages = desiredpages; | bp->b_npages = desiredpages; | ||||
} | } | ||||
/* | /* | ||||
* Byte granular extension of VMIO buffers. | * Byte granular extension of VMIO buffers. | ||||
▲ Show 20 Lines • Show All 2,467 Lines • Show Last 20 Lines |