Changeset View
Changeset View
Standalone View
Standalone View
head/sys/kern/vfs_bio.c
Show First 20 Lines • Show All 141 Lines • ▼ Show 20 Lines | |||||
#define BD_LOCK(bd) mtx_lock(BD_LOCKPTR((bd))) | #define BD_LOCK(bd) mtx_lock(BD_LOCKPTR((bd))) | ||||
#define BD_UNLOCK(bd) mtx_unlock(BD_LOCKPTR((bd))) | #define BD_UNLOCK(bd) mtx_unlock(BD_LOCKPTR((bd))) | ||||
#define BD_ASSERT_LOCKED(bd) mtx_assert(BD_LOCKPTR((bd)), MA_OWNED) | #define BD_ASSERT_LOCKED(bd) mtx_assert(BD_LOCKPTR((bd)), MA_OWNED) | ||||
#define BD_RUN_LOCKPTR(bd) (&(bd)->bd_run_lock) | #define BD_RUN_LOCKPTR(bd) (&(bd)->bd_run_lock) | ||||
#define BD_RUN_LOCK(bd) mtx_lock(BD_RUN_LOCKPTR((bd))) | #define BD_RUN_LOCK(bd) mtx_lock(BD_RUN_LOCKPTR((bd))) | ||||
#define BD_RUN_UNLOCK(bd) mtx_unlock(BD_RUN_LOCKPTR((bd))) | #define BD_RUN_UNLOCK(bd) mtx_unlock(BD_RUN_LOCKPTR((bd))) | ||||
#define BD_DOMAIN(bd) (bd - bdomain) | #define BD_DOMAIN(bd) (bd - bdomain) | ||||
static struct buf *buf; /* buffer header pool */ | static char *buf; /* buffer header pool */ | ||||
extern struct buf *swbuf; /* Swap buffer header pool. */ | static struct buf * | ||||
nbufp(unsigned i) | |||||
{ | |||||
return ((struct buf *)(buf + (sizeof(struct buf) + | |||||
sizeof(vm_page_t) * atop(maxbcachebuf)) * i)); | |||||
} | |||||
caddr_t __read_mostly unmapped_buf; | caddr_t __read_mostly unmapped_buf; | ||||
/* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */ | /* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */ | ||||
struct proc *bufdaemonproc; | struct proc *bufdaemonproc; | ||||
static void vm_hold_free_pages(struct buf *bp, int newbsize); | static void vm_hold_free_pages(struct buf *bp, int newbsize); | ||||
static void vm_hold_load_pages(struct buf *bp, vm_offset_t from, | static void vm_hold_load_pages(struct buf *bp, vm_offset_t from, | ||||
vm_offset_t to); | vm_offset_t to); | ||||
▲ Show 20 Lines • Show All 829 Lines • ▼ Show 20 Lines | maxbcachebuf_adjust(void) | ||||
* maxbcachebuf must be a power of 2 >= MAXBSIZE. | * maxbcachebuf must be a power of 2 >= MAXBSIZE. | ||||
*/ | */ | ||||
i = 2; | i = 2; | ||||
while (i * 2 <= maxbcachebuf) | while (i * 2 <= maxbcachebuf) | ||||
i *= 2; | i *= 2; | ||||
maxbcachebuf = i; | maxbcachebuf = i; | ||||
if (maxbcachebuf < MAXBSIZE) | if (maxbcachebuf < MAXBSIZE) | ||||
maxbcachebuf = MAXBSIZE; | maxbcachebuf = MAXBSIZE; | ||||
if (maxbcachebuf > MAXPHYS) | if (maxbcachebuf > maxphys) | ||||
maxbcachebuf = MAXPHYS; | maxbcachebuf = maxphys; | ||||
if (bootverbose != 0 && maxbcachebuf != MAXBCACHEBUF) | if (bootverbose != 0 && maxbcachebuf != MAXBCACHEBUF) | ||||
printf("maxbcachebuf=%d\n", maxbcachebuf); | printf("maxbcachebuf=%d\n", maxbcachebuf); | ||||
} | } | ||||
/* | /* | ||||
* bd_speedup - speedup the buffer cache flushing code | * bd_speedup - speedup the buffer cache flushing code | ||||
*/ | */ | ||||
void | void | ||||
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | if (buf_sz < maxbuf_sz / TRANSIENT_DENOM * | ||||
/* | /* | ||||
* Buffer map spans all KVA we could afford on | * Buffer map spans all KVA we could afford on | ||||
* this platform. Give 10% (20% on i386) of | * this platform. Give 10% (20% on i386) of | ||||
* the buffer map to the transient bio map. | * the buffer map to the transient bio map. | ||||
*/ | */ | ||||
biotmap_sz = buf_sz / TRANSIENT_DENOM; | biotmap_sz = buf_sz / TRANSIENT_DENOM; | ||||
buf_sz -= biotmap_sz; | buf_sz -= biotmap_sz; | ||||
} | } | ||||
if (biotmap_sz / INT_MAX > MAXPHYS) | if (biotmap_sz / INT_MAX > maxphys) | ||||
bio_transient_maxcnt = INT_MAX; | bio_transient_maxcnt = INT_MAX; | ||||
else | else | ||||
bio_transient_maxcnt = biotmap_sz / MAXPHYS; | bio_transient_maxcnt = biotmap_sz / maxphys; | ||||
/* | /* | ||||
* Artificially limit to 1024 simultaneous in-flight I/Os | * Artificially limit to 1024 simultaneous in-flight I/Os | ||||
* using the transient mapping. | * using the transient mapping. | ||||
*/ | */ | ||||
if (bio_transient_maxcnt > 1024) | if (bio_transient_maxcnt > 1024) | ||||
bio_transient_maxcnt = 1024; | bio_transient_maxcnt = 1024; | ||||
if (tuned_nbuf) | if (tuned_nbuf) | ||||
nbuf = buf_sz / BKVASIZE; | nbuf = buf_sz / BKVASIZE; | ||||
} | } | ||||
if (nswbuf == 0) { | if (nswbuf == 0) { | ||||
nswbuf = min(nbuf / 4, 256); | nswbuf = min(nbuf / 4, 256); | ||||
if (nswbuf < NSWBUF_MIN) | if (nswbuf < NSWBUF_MIN) | ||||
nswbuf = NSWBUF_MIN; | nswbuf = NSWBUF_MIN; | ||||
} | } | ||||
/* | /* | ||||
* Reserve space for the buffer cache buffers | * Reserve space for the buffer cache buffers | ||||
*/ | */ | ||||
buf = (void *)v; | buf = (char *)v; | ||||
v = (caddr_t)(buf + nbuf); | v = (caddr_t)buf + (sizeof(struct buf) + sizeof(vm_page_t) * | ||||
atop(maxbcachebuf)) * nbuf; | |||||
return(v); | return (v); | ||||
} | } | ||||
/* Initialize the buffer subsystem. Called before use of any buffers. */ | /* Initialize the buffer subsystem. Called before use of any buffers. */ | ||||
void | void | ||||
bufinit(void) | bufinit(void) | ||||
{ | { | ||||
struct buf *bp; | struct buf *bp; | ||||
int i; | int i; | ||||
KASSERT(maxbcachebuf >= MAXBSIZE, | KASSERT(maxbcachebuf >= MAXBSIZE, | ||||
("maxbcachebuf (%d) must be >= MAXBSIZE (%d)\n", maxbcachebuf, | ("maxbcachebuf (%d) must be >= MAXBSIZE (%d)\n", maxbcachebuf, | ||||
MAXBSIZE)); | MAXBSIZE)); | ||||
bq_init(&bqempty, QUEUE_EMPTY, -1, "bufq empty lock"); | bq_init(&bqempty, QUEUE_EMPTY, -1, "bufq empty lock"); | ||||
mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF); | mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF); | ||||
mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF); | mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF); | ||||
mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF); | mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF); | ||||
unmapped_buf = (caddr_t)kva_alloc(MAXPHYS); | unmapped_buf = (caddr_t)kva_alloc(maxphys); | ||||
/* finally, initialize each buffer header and stick on empty q */ | /* finally, initialize each buffer header and stick on empty q */ | ||||
for (i = 0; i < nbuf; i++) { | for (i = 0; i < nbuf; i++) { | ||||
bp = &buf[i]; | bp = nbufp(i); | ||||
bzero(bp, sizeof *bp); | bzero(bp, sizeof(*bp) + sizeof(vm_page_t) * atop(maxbcachebuf)); | ||||
bp->b_flags = B_INVAL; | bp->b_flags = B_INVAL; | ||||
bp->b_rcred = NOCRED; | bp->b_rcred = NOCRED; | ||||
bp->b_wcred = NOCRED; | bp->b_wcred = NOCRED; | ||||
bp->b_qindex = QUEUE_NONE; | bp->b_qindex = QUEUE_NONE; | ||||
bp->b_domain = -1; | bp->b_domain = -1; | ||||
bp->b_subqueue = mp_maxid + 1; | bp->b_subqueue = mp_maxid + 1; | ||||
bp->b_xflags = 0; | bp->b_xflags = 0; | ||||
bp->b_data = bp->b_kvabase = unmapped_buf; | bp->b_data = bp->b_kvabase = unmapped_buf; | ||||
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | bufinit(void) | ||||
* should be set appropriately to limit work per-iteration. | * should be set appropriately to limit work per-iteration. | ||||
*/ | */ | ||||
lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus); | lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus); | ||||
hifreebuffers = (3 * lofreebuffers) / 2; | hifreebuffers = (3 * lofreebuffers) / 2; | ||||
numfreebuffers = nbuf; | numfreebuffers = nbuf; | ||||
/* Setup the kva and free list allocators. */ | /* Setup the kva and free list allocators. */ | ||||
vmem_set_reclaim(buffer_arena, bufkva_reclaim); | vmem_set_reclaim(buffer_arena, bufkva_reclaim); | ||||
buf_zone = uma_zcache_create("buf free cache", sizeof(struct buf), | buf_zone = uma_zcache_create("buf free cache", | ||||
sizeof(struct buf) + sizeof(vm_page_t) * atop(maxbcachebuf), | |||||
NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0); | NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0); | ||||
/* | /* | ||||
* Size the clean queue according to the amount of buffer space. | * Size the clean queue according to the amount of buffer space. | ||||
* One queue per-256mb up to the max. More queues gives better | * One queue per-256mb up to the max. More queues gives better | ||||
* concurrency but less accurate LRU. | * concurrency but less accurate LRU. | ||||
*/ | */ | ||||
buf_domains = MIN(howmany(maxbufspace, 256*1024*1024), BUF_DOMAINS); | buf_domains = MIN(howmany(maxbufspace, 256*1024*1024), BUF_DOMAINS); | ||||
Show All 32 Lines | |||||
vfs_buf_check_mapped(struct buf *bp) | vfs_buf_check_mapped(struct buf *bp) | ||||
{ | { | ||||
KASSERT(bp->b_kvabase != unmapped_buf, | KASSERT(bp->b_kvabase != unmapped_buf, | ||||
("mapped buf: b_kvabase was not updated %p", bp)); | ("mapped buf: b_kvabase was not updated %p", bp)); | ||||
KASSERT(bp->b_data != unmapped_buf, | KASSERT(bp->b_data != unmapped_buf, | ||||
("mapped buf: b_data was not updated %p", bp)); | ("mapped buf: b_data was not updated %p", bp)); | ||||
KASSERT(bp->b_data < unmapped_buf || bp->b_data >= unmapped_buf + | KASSERT(bp->b_data < unmapped_buf || bp->b_data >= unmapped_buf + | ||||
MAXPHYS, ("b_data + b_offset unmapped %p", bp)); | maxphys, ("b_data + b_offset unmapped %p", bp)); | ||||
} | } | ||||
static inline void | static inline void | ||||
vfs_buf_check_unmapped(struct buf *bp) | vfs_buf_check_unmapped(struct buf *bp) | ||||
{ | { | ||||
KASSERT(bp->b_data == unmapped_buf, | KASSERT(bp->b_data == unmapped_buf, | ||||
("unmapped buf: corrupted b_data %p", bp)); | ("unmapped buf: corrupted b_data %p", bp)); | ||||
Show All 18 Lines | |||||
/* | /* | ||||
* Shutdown the system cleanly to prepare for reboot, halt, or power off. | * Shutdown the system cleanly to prepare for reboot, halt, or power off. | ||||
*/ | */ | ||||
void | void | ||||
bufshutdown(int show_busybufs) | bufshutdown(int show_busybufs) | ||||
{ | { | ||||
static int first_buf_printf = 1; | static int first_buf_printf = 1; | ||||
struct buf *bp; | struct buf *bp; | ||||
int iter, nbusy, pbusy; | int i, iter, nbusy, pbusy; | ||||
#ifndef PREEMPTION | #ifndef PREEMPTION | ||||
int subiter; | int subiter; | ||||
#endif | #endif | ||||
/* | /* | ||||
* Sync filesystems for shutdown | * Sync filesystems for shutdown | ||||
*/ | */ | ||||
wdog_kern_pat(WD_LASTVAL); | wdog_kern_pat(WD_LASTVAL); | ||||
kern_sync(curthread); | kern_sync(curthread); | ||||
/* | /* | ||||
* With soft updates, some buffers that are | * With soft updates, some buffers that are | ||||
* written will be remarked as dirty until other | * written will be remarked as dirty until other | ||||
* buffers are written. | * buffers are written. | ||||
*/ | */ | ||||
for (iter = pbusy = 0; iter < 20; iter++) { | for (iter = pbusy = 0; iter < 20; iter++) { | ||||
nbusy = 0; | nbusy = 0; | ||||
for (bp = &buf[nbuf]; --bp >= buf; ) | for (i = nbuf - 1; i >= 0; i--) { | ||||
bp = nbufp(i); | |||||
if (isbufbusy(bp)) | if (isbufbusy(bp)) | ||||
nbusy++; | nbusy++; | ||||
} | |||||
if (nbusy == 0) { | if (nbusy == 0) { | ||||
if (first_buf_printf) | if (first_buf_printf) | ||||
printf("All buffers synced."); | printf("All buffers synced."); | ||||
break; | break; | ||||
} | } | ||||
if (first_buf_printf) { | if (first_buf_printf) { | ||||
printf("Syncing disks, buffers remaining... "); | printf("Syncing disks, buffers remaining... "); | ||||
first_buf_printf = 0; | first_buf_printf = 0; | ||||
Show All 24 Lines | |||||
#endif | #endif | ||||
} | } | ||||
printf("\n"); | printf("\n"); | ||||
/* | /* | ||||
* Count only busy local buffers to prevent forcing | * Count only busy local buffers to prevent forcing | ||||
* a fsck if we're just a client of a wedged NFS server | * a fsck if we're just a client of a wedged NFS server | ||||
*/ | */ | ||||
nbusy = 0; | nbusy = 0; | ||||
for (bp = &buf[nbuf]; --bp >= buf; ) { | for (i = nbuf - 1; i >= 0; i--) { | ||||
bp = nbufp(i); | |||||
if (isbufbusy(bp)) { | if (isbufbusy(bp)) { | ||||
#if 0 | #if 0 | ||||
/* XXX: This is bogus. We should probably have a BO_REMOTE flag instead */ | /* XXX: This is bogus. We should probably have a BO_REMOTE flag instead */ | ||||
if (bp->b_dev == NULL) { | if (bp->b_dev == NULL) { | ||||
TAILQ_REMOVE(&mountlist, | TAILQ_REMOVE(&mountlist, | ||||
bp->b_vp->v_mount, mnt_list); | bp->b_vp->v_mount, mnt_list); | ||||
continue; | continue; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 163 Lines • ▼ Show 20 Lines | buf_free(struct buf *bp) | ||||
if (bp->b_wcred != NOCRED) { | if (bp->b_wcred != NOCRED) { | ||||
crfree(bp->b_wcred); | crfree(bp->b_wcred); | ||||
bp->b_wcred = NOCRED; | bp->b_wcred = NOCRED; | ||||
} | } | ||||
if (!LIST_EMPTY(&bp->b_dep)) | if (!LIST_EMPTY(&bp->b_dep)) | ||||
buf_deallocate(bp); | buf_deallocate(bp); | ||||
bufkva_free(bp); | bufkva_free(bp); | ||||
atomic_add_int(&bufdomain(bp)->bd_freebuffers, 1); | atomic_add_int(&bufdomain(bp)->bd_freebuffers, 1); | ||||
MPASS((bp->b_flags & B_MAXPHYS) == 0); | |||||
BUF_UNLOCK(bp); | BUF_UNLOCK(bp); | ||||
uma_zfree(buf_zone, bp); | uma_zfree(buf_zone, bp); | ||||
} | } | ||||
/* | /* | ||||
* buf_import: | * buf_import: | ||||
* | * | ||||
* Import bufs into the uma cache from the buf list. The system still | * Import bufs into the uma cache from the buf list. The system still | ||||
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines | buf_alloc(struct bufdomain *bd) | ||||
KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0, | KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0, | ||||
("invalid buffer %p flags %#x", bp, bp->b_flags)); | ("invalid buffer %p flags %#x", bp, bp->b_flags)); | ||||
KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0, | KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0, | ||||
("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags)); | ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags)); | ||||
KASSERT(bp->b_npages == 0, | KASSERT(bp->b_npages == 0, | ||||
("bp: %p still has %d vm pages\n", bp, bp->b_npages)); | ("bp: %p still has %d vm pages\n", bp, bp->b_npages)); | ||||
KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp)); | KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp)); | ||||
KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp)); | KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp)); | ||||
MPASS((bp->b_flags & B_MAXPHYS) == 0); | |||||
bp->b_domain = BD_DOMAIN(bd); | bp->b_domain = BD_DOMAIN(bd); | ||||
bp->b_flags = 0; | bp->b_flags = 0; | ||||
bp->b_ioflags = 0; | bp->b_ioflags = 0; | ||||
bp->b_xflags = 0; | bp->b_xflags = 0; | ||||
bp->b_vflags = 0; | bp->b_vflags = 0; | ||||
bp->b_vp = NULL; | bp->b_vp = NULL; | ||||
bp->b_blkno = bp->b_lblkno = 0; | bp->b_blkno = bp->b_lblkno = 0; | ||||
▲ Show 20 Lines • Show All 328 Lines • ▼ Show 20 Lines | |||||
static int | static int | ||||
bufkva_alloc(struct buf *bp, int maxsize, int gbflags) | bufkva_alloc(struct buf *bp, int maxsize, int gbflags) | ||||
{ | { | ||||
vm_offset_t addr; | vm_offset_t addr; | ||||
int error; | int error; | ||||
KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0, | KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0, | ||||
("Invalid gbflags 0x%x in %s", gbflags, __func__)); | ("Invalid gbflags 0x%x in %s", gbflags, __func__)); | ||||
MPASS((bp->b_flags & B_MAXPHYS) == 0); | |||||
KASSERT(maxsize <= maxbcachebuf, | |||||
("bufkva_alloc kva too large %d %u", maxsize, maxbcachebuf)); | |||||
bufkva_free(bp); | bufkva_free(bp); | ||||
addr = 0; | addr = 0; | ||||
error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr); | error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr); | ||||
if (error != 0) { | if (error != 0) { | ||||
/* | /* | ||||
* Buffer map is too fragmented. Request the caller | * Buffer map is too fragmented. Request the caller | ||||
▲ Show 20 Lines • Show All 1,002 Lines • ▼ Show 20 Lines | vfs_vmio_extend(struct buf *bp, int desiredpages, int size) | ||||
/* | /* | ||||
* Step 1, bring in the VM pages from the object, allocating | * Step 1, bring in the VM pages from the object, allocating | ||||
* them if necessary. We must clear B_CACHE if these pages | * them if necessary. We must clear B_CACHE if these pages | ||||
* are not valid for the range covered by the buffer. | * are not valid for the range covered by the buffer. | ||||
*/ | */ | ||||
obj = bp->b_bufobj->bo_object; | obj = bp->b_bufobj->bo_object; | ||||
if (bp->b_npages < desiredpages) { | if (bp->b_npages < desiredpages) { | ||||
KASSERT(desiredpages <= atop(maxbcachebuf), | |||||
("vfs_vmio_extend past maxbcachebuf %p %d %u", | |||||
bp, desiredpages, maxbcachebuf)); | |||||
/* | /* | ||||
* We must allocate system pages since blocking | * We must allocate system pages since blocking | ||||
* here could interfere with paging I/O, no | * here could interfere with paging I/O, no | ||||
* matter which process we are. | * matter which process we are. | ||||
* | * | ||||
* Only exclusive busy can be tested here. | * Only exclusive busy can be tested here. | ||||
* Blocking on shared busy might lead to | * Blocking on shared busy might lead to | ||||
* deadlocks once allocbuf() is called after | * deadlocks once allocbuf() is called after | ||||
▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Lines | vfs_bio_awrite(struct buf *bp) | ||||
* right now we support clustered writing only to regular files. If | * right now we support clustered writing only to regular files. If | ||||
* we find a clusterable block we could be in the middle of a cluster | * we find a clusterable block we could be in the middle of a cluster | ||||
* rather then at the beginning. | * rather then at the beginning. | ||||
*/ | */ | ||||
if ((vp->v_type == VREG) && | if ((vp->v_type == VREG) && | ||||
(vp->v_mount != 0) && /* Only on nodes that have the size info */ | (vp->v_mount != 0) && /* Only on nodes that have the size info */ | ||||
(bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) { | (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) { | ||||
size = vp->v_mount->mnt_stat.f_iosize; | size = vp->v_mount->mnt_stat.f_iosize; | ||||
maxcl = MAXPHYS / size; | maxcl = maxphys / size; | ||||
BO_RLOCK(bo); | BO_RLOCK(bo); | ||||
for (i = 1; i < maxcl; i++) | for (i = 1; i < maxcl; i++) | ||||
if (vfs_bio_clcheck(vp, size, lblkno + i, | if (vfs_bio_clcheck(vp, size, lblkno + i, | ||||
bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0) | bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0) | ||||
break; | break; | ||||
for (j = 1; i + j <= maxcl && j <= lblkno; j++) | for (j = 1; i + j <= maxcl && j <= lblkno; j++) | ||||
▲ Show 20 Lines • Show All 1,673 Lines • ▼ Show 20 Lines | vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to) | ||||
vm_page_t p; | vm_page_t p; | ||||
int index; | int index; | ||||
BUF_CHECK_MAPPED(bp); | BUF_CHECK_MAPPED(bp); | ||||
to = round_page(to); | to = round_page(to); | ||||
from = round_page(from); | from = round_page(from); | ||||
index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT; | index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT; | ||||
MPASS((bp->b_flags & B_MAXPHYS) == 0); | |||||
KASSERT(to - from <= maxbcachebuf, | |||||
("vm_hold_load_pages too large %p %#jx %#jx %u", | |||||
bp, (uintmax_t)from, (uintmax_t)to, maxbcachebuf)); | |||||
for (pg = from; pg < to; pg += PAGE_SIZE, index++) { | for (pg = from; pg < to; pg += PAGE_SIZE, index++) { | ||||
/* | /* | ||||
* note: must allocate system pages since blocking here | * note: must allocate system pages since blocking here | ||||
* could interfere with paging I/O, no matter which | * could interfere with paging I/O, no matter which | ||||
* process we are. | * process we are. | ||||
*/ | */ | ||||
p = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ | | p = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ | | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | |||||
* This function only works with pager buffers. | * This function only works with pager buffers. | ||||
*/ | */ | ||||
int | int | ||||
vmapbuf(struct buf *bp, void *uaddr, size_t len, int mapbuf) | vmapbuf(struct buf *bp, void *uaddr, size_t len, int mapbuf) | ||||
{ | { | ||||
vm_prot_t prot; | vm_prot_t prot; | ||||
int pidx; | int pidx; | ||||
MPASS((bp->b_flags & B_MAXPHYS) != 0); | |||||
prot = VM_PROT_READ; | prot = VM_PROT_READ; | ||||
if (bp->b_iocmd == BIO_READ) | if (bp->b_iocmd == BIO_READ) | ||||
prot |= VM_PROT_WRITE; /* Less backwards than it looks */ | prot |= VM_PROT_WRITE; /* Less backwards than it looks */ | ||||
if ((pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, | pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, | ||||
(vm_offset_t)uaddr, len, prot, bp->b_pages, | (vm_offset_t)uaddr, len, prot, bp->b_pages, PBUF_PAGES); | ||||
btoc(MAXPHYS))) < 0) | if (pidx < 0) | ||||
return (-1); | return (-1); | ||||
bp->b_bufsize = len; | bp->b_bufsize = len; | ||||
bp->b_npages = pidx; | bp->b_npages = pidx; | ||||
bp->b_offset = ((vm_offset_t)uaddr) & PAGE_MASK; | bp->b_offset = ((vm_offset_t)uaddr) & PAGE_MASK; | ||||
if (mapbuf || !unmapped_buf_allowed) { | if (mapbuf || !unmapped_buf_allowed) { | ||||
pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx); | pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx); | ||||
bp->b_data = bp->b_kvabase + bp->b_offset; | bp->b_data = bp->b_kvabase + bp->b_offset; | ||||
} else | } else | ||||
bp->b_data = unmapped_buf; | bp->b_data = unmapped_buf; | ||||
return(0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Free the io map PTEs associated with this IO operation. | * Free the io map PTEs associated with this IO operation. | ||||
* We also invalidate the TLB entries and restore the original b_addr. | * We also invalidate the TLB entries and restore the original b_addr. | ||||
* | * | ||||
* This function only works with pager buffers. | * This function only works with pager buffers. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 454 Lines • ▼ Show 20 Lines | for (i = 0; i < buf_domains; i++) { | ||||
db_printf("\twakeup\t\t%d\n", bd->bd_wanted); | db_printf("\twakeup\t\t%d\n", bd->bd_wanted); | ||||
db_printf("\tlim\t\t%d\n", bd->bd_lim); | db_printf("\tlim\t\t%d\n", bd->bd_lim); | ||||
db_printf("\tCPU "); | db_printf("\tCPU "); | ||||
for (j = 0; j <= mp_maxid; j++) | for (j = 0; j <= mp_maxid; j++) | ||||
db_printf("%d, ", bd->bd_subq[j].bq_len); | db_printf("%d, ", bd->bd_subq[j].bq_len); | ||||
db_printf("\n"); | db_printf("\n"); | ||||
cnt = 0; | cnt = 0; | ||||
total = 0; | total = 0; | ||||
for (j = 0; j < nbuf; j++) | for (j = 0; j < nbuf; j++) { | ||||
if (buf[j].b_domain == i && BUF_ISLOCKED(&buf[j])) { | bp = nbufp(j); | ||||
if (bp->b_domain == i && BUF_ISLOCKED(bp)) { | |||||
cnt++; | cnt++; | ||||
total += buf[j].b_bufsize; | total += bp->b_bufsize; | ||||
} | } | ||||
} | |||||
db_printf("\tLocked buffers: %d space %ld\n", cnt, total); | db_printf("\tLocked buffers: %d space %ld\n", cnt, total); | ||||
cnt = 0; | cnt = 0; | ||||
total = 0; | total = 0; | ||||
for (j = 0; j < nbuf; j++) | for (j = 0; j < nbuf; j++) { | ||||
if (buf[j].b_domain == i) { | bp = nbufp(j); | ||||
if (bp->b_domain == i) { | |||||
cnt++; | cnt++; | ||||
total += buf[j].b_bufsize; | total += bp->b_bufsize; | ||||
} | } | ||||
} | |||||
db_printf("\tTotal buffers: %d space %ld\n", cnt, total); | db_printf("\tTotal buffers: %d space %ld\n", cnt, total); | ||||
} | } | ||||
} | } | ||||
DB_SHOW_COMMAND(lockedbufs, lockedbufs) | DB_SHOW_COMMAND(lockedbufs, lockedbufs) | ||||
{ | { | ||||
struct buf *bp; | struct buf *bp; | ||||
int i; | int i; | ||||
for (i = 0; i < nbuf; i++) { | for (i = 0; i < nbuf; i++) { | ||||
bp = &buf[i]; | bp = nbufp(i); | ||||
if (BUF_ISLOCKED(bp)) { | if (BUF_ISLOCKED(bp)) { | ||||
db_show_buffer((uintptr_t)bp, 1, 0, NULL); | db_show_buffer((uintptr_t)bp, 1, 0, NULL); | ||||
db_printf("\n"); | db_printf("\n"); | ||||
if (db_pager_quit) | if (db_pager_quit) | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
Show All 26 Lines | DB_COMMAND(countfreebufs, db_coundfreebufs) | ||||
int i, used = 0, nfree = 0; | int i, used = 0, nfree = 0; | ||||
if (have_addr) { | if (have_addr) { | ||||
db_printf("usage: countfreebufs\n"); | db_printf("usage: countfreebufs\n"); | ||||
return; | return; | ||||
} | } | ||||
for (i = 0; i < nbuf; i++) { | for (i = 0; i < nbuf; i++) { | ||||
bp = &buf[i]; | bp = nbufp(i); | ||||
if (bp->b_qindex == QUEUE_EMPTY) | if (bp->b_qindex == QUEUE_EMPTY) | ||||
nfree++; | nfree++; | ||||
else | else | ||||
used++; | used++; | ||||
} | } | ||||
db_printf("Counted %d free, %d used (%d tot)\n", nfree, used, | db_printf("Counted %d free, %d used (%d tot)\n", nfree, used, | ||||
nfree + used); | nfree + used); | ||||
db_printf("numfreebuffers is %d\n", numfreebuffers); | db_printf("numfreebuffers is %d\n", numfreebuffers); | ||||
} | } | ||||
#endif /* DDB */ | #endif /* DDB */ |