Changeset View
Standalone View
sys/kern/vfs_bio.c
Show First 20 Lines • Show All 655 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
static void | static void | ||||
bufspace_adjust(struct buf *bp, int bufsize) | bufspace_adjust(struct buf *bp, int bufsize) | ||||
{ | { | ||||
struct bufdomain *bd; | struct bufdomain *bd; | ||||
long space; | long space; | ||||
int diff; | int diff; | ||||
KASSERT((bp->b_flags & B_MALLOC) == 0, | |||||
("bufspace_adjust: malloc buf %p", bp)); | |||||
bd = bufdomain(bp); | bd = bufdomain(bp); | ||||
diff = bufsize - bp->b_bufsize; | diff = bufsize - bp->b_bufsize; | ||||
if (diff < 0) { | if (diff < 0) { | ||||
atomic_subtract_long(&bd->bd_bufspace, -diff); | atomic_subtract_long(&bd->bd_bufspace, -diff); | ||||
} else if (diff > 0) { | } else if (diff > 0) { | ||||
space = atomic_fetchadd_long(&bd->bd_bufspace, diff); | space = atomic_fetchadd_long(&bd->bd_bufspace, diff); | ||||
/* Wake up the daemon on the transition. */ | /* Wake up the daemon on the transition. */ | ||||
if (space < bd->bd_bufspacethresh && | if (space < bd->bd_bufspacethresh && | ||||
▲ Show 20 Lines • Show All 172 Lines • ▼ Show 20 Lines | while (bd->bd_bufspace > bd->bd_lobufspace || | ||||
} | } | ||||
maybe_yield(); | maybe_yield(); | ||||
} | } | ||||
bufspace_daemon_wait(bd); | bufspace_daemon_wait(bd); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* bufmallocadjust: | |||||
* | |||||
* Adjust the reported bufspace for a malloc managed buffer, possibly | |||||
* waking any waiters. | |||||
*/ | |||||
static void | |||||
bufmallocadjust(struct buf *bp, int bufsize) | |||||
{ | |||||
int diff; | |||||
KASSERT((bp->b_flags & B_MALLOC) != 0, | |||||
("bufmallocadjust: non-malloc buf %p", bp)); | |||||
diff = bufsize - bp->b_bufsize; | |||||
if (diff < 0) | |||||
atomic_subtract_long(&bufmallocspace, -diff); | |||||
else | |||||
atomic_add_long(&bufmallocspace, diff); | |||||
bp->b_bufsize = bufsize; | |||||
} | |||||
/* | |||||
* runningwakeup: | * runningwakeup: | ||||
* | * | ||||
* Wake up processes that are waiting on asynchronous writes to fall | * Wake up processes that are waiting on asynchronous writes to fall | ||||
* below lorunningspace. | * below lorunningspace. | ||||
*/ | */ | ||||
static void | static void | ||||
runningwakeup(void) | runningwakeup(void) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 3,281 Lines • ▼ Show 20 Lines | geteblk(int size, int flags) | ||||
} | } | ||||
allocbuf(bp, size); | allocbuf(bp, size); | ||||
bufspace_release(bufdomain(bp), maxsize); | bufspace_release(bufdomain(bp), maxsize); | ||||
bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */ | bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */ | ||||
BUF_ASSERT_HELD(bp); | BUF_ASSERT_HELD(bp); | ||||
return (bp); | return (bp); | ||||
} | } | ||||
/* | /* | ||||
* Truncate the backing store for a non-vmio buffer. | |||||
*/ | |||||
static void | |||||
vfs_nonvmio_truncate(struct buf *bp, int newbsize) | |||||
{ | |||||
cem: This routine and nonvmio_extend can probably both be inlined now that they're so short (they're… | |||||
if (bp->b_flags & B_MALLOC) { | |||||
/* | |||||
* malloced buffers are not shrunk | |||||
*/ | |||||
if (newbsize == 0) { | |||||
bufmallocadjust(bp, 0); | |||||
free(bp->b_data, M_BIOBUF); | |||||
bp->b_data = bp->b_kvabase; | |||||
bp->b_flags &= ~B_MALLOC; | |||||
} | |||||
return; | |||||
} | |||||
vm_hold_free_pages(bp, newbsize); | |||||
bufspace_adjust(bp, newbsize); | |||||
} | |||||
/* | |||||
* Extend the backing for a non-VMIO buffer. | |||||
*/ | |||||
static void | |||||
vfs_nonvmio_extend(struct buf *bp, int newbsize) | |||||
{ | |||||
caddr_t origbuf; | |||||
int origbufsize; | |||||
/* | |||||
* We only use malloced memory on the first allocation. | |||||
* and revert to page-allocated memory when the buffer | |||||
* grows. | |||||
* | |||||
* There is a potential smp race here that could lead | |||||
* to bufmallocspace slightly passing the max. It | |||||
* is probably extremely rare and not worth worrying | |||||
* over. | |||||
*/ | |||||
if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 && | |||||
bufmallocspace < maxbufmallocspace) { | |||||
bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK); | |||||
bp->b_flags |= B_MALLOC; | |||||
bufmallocadjust(bp, newbsize); | |||||
return; | |||||
} | |||||
/* | |||||
* If the buffer is growing on its other-than-first | |||||
* allocation then we revert to the page-allocation | |||||
* scheme. | |||||
*/ | |||||
origbuf = NULL; | |||||
origbufsize = 0; | |||||
if (bp->b_flags & B_MALLOC) { | |||||
origbuf = bp->b_data; | |||||
origbufsize = bp->b_bufsize; | |||||
bp->b_data = bp->b_kvabase; | |||||
bufmallocadjust(bp, 0); | |||||
bp->b_flags &= ~B_MALLOC; | |||||
newbsize = round_page(newbsize); | |||||
} | |||||
vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize, | |||||
(vm_offset_t) bp->b_data + newbsize); | |||||
if (origbuf != NULL) { | |||||
bcopy(origbuf, bp->b_data, origbufsize); | |||||
free(origbuf, M_BIOBUF); | |||||
} | |||||
bufspace_adjust(bp, newbsize); | |||||
} | |||||
/* | |||||
* This code constitutes the buffer memory from either anonymous system | * This code constitutes the buffer memory from either anonymous system | ||||
Not Done Inline ActionsAh, by "inline," I had in mind literally moving the code inside the caller. (The "inline" annotation is usually discouraged since modern (C99+) compilers are free to more or less ignore it and use their own heuristics anyway.) cem: Ah, by "inline," I had in mind literally moving the code inside the caller.
(The "inline"… | |||||
Not Done Inline ActionsIf we fail the bufmallocspace check here, should we round newbsize up to page_size? Or remove the conditional roundup below? cem: If we fail the bufmallocspace check here, should we round newbsize up to page_size? Or remove… | |||||
Not Done Inline ActionsI'm fine with the roundup if that's the right thing to do. darrick.freebsd_gmail.com: I'm fine with the roundup if that's the right thing to do. | |||||
Not Done Inline Actionsiirc kib explicitly wanted the bufmalloc check removed from want_malloc_buf(). That would've caused the roundup to occur when bufmallocspace >= maxbufmallocspace. darrick.freebsd_gmail.com: iirc kib explicitly wanted the bufmalloc check removed from want_malloc_buf(). That would've… | |||||
Not Done Inline ActionsNo, I do not want to micro-manage. I only want malloc-backed buffer to start/continue to work. Code organization is up to you. kib: No, I do not want to micro-manage. I only want malloc-backed buffer to start/continue to work. | |||||
Not Done Inline ActionsNot a code organization question. Earlier the check in question was in want_malloc_buf(), so the check was done in two places: 1. before rounding up to PAGE_SIZE in the case where we don't want to set B_MALLOC, and 2. prior to setting B_MALLOC in the case where we do. But in an earlier comment you stated that "4291 darrick.freebsd_gmail.com: Not a code organization question. Earlier the check in question was in want_malloc_buf(), so… | |||||
Done Inline ActionsI do not quite understand what you are asking. Which check below ? If we do not allocate malloc buffer, there is no point (and it is even impossible) to allocate not in quantities of PAGE_SIZE. So if the maxbufmallocspace check fails and we decided to allocate page-backed buffer, we need to round. This is the only sense that I can see in the question. kib: I do not quite understand what you are asking. Which check below ?
If we do not allocate… | |||||
* memory (in the case of non-VMIO operations) or from an associated | * memory (in the case of non-VMIO operations) or from an associated | ||||
* VM object (in the case of VMIO operations). This code is able to | * VM object (in the case of VMIO operations). This code is able to | ||||
* resize a buffer up or down. | * resize a buffer up or down. | ||||
* | * | ||||
* Note that this code is tricky, and has many complications to resolve | * Note that this code is tricky, and has many complications to resolve | ||||
* deadlock or inconsistent data situations. Tread lightly!!! | * deadlock or inconsistent data situations. Tread lightly!!! | ||||
Not Done Inline ActionsBlank line is needed before the first statement if there is no local vars defined. kib: Blank line is needed before the first statement if there is no local vars defined. | |||||
* There are B_CACHE and B_DELWRI interactions that must be dealt with by | * There are B_CACHE and B_DELWRI interactions that must be dealt with by | ||||
Not Done Inline Actionsreturn (false); kib: return (false); | |||||
* the caller. Calling this code willy nilly can result in the loss of data. | * the caller. Calling this code willy nilly can result in the loss of data. | ||||
Not Done Inline ActionsNo need for this blank line. kib: No need for this blank line. | |||||
* | * | ||||
* allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with | * allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with | ||||
Not Done Inline Actionsreturn (false); kib: return (false); | |||||
* B_CACHE for the non-VMIO case. | * B_CACHE for the non-VMIO case. | ||||
Not Done Inline Actionsextra blank line, and so on for the whole function. kib: extra blank line, and so on for the whole function. | |||||
*/ | */ | ||||
int | int | ||||
allocbuf(struct buf *bp, int size) | allocbuf(struct buf *bp, int size) | ||||
{ | { | ||||
int newbsize; | int newbsize; | ||||
BUF_ASSERT_HELD(bp); | BUF_ASSERT_HELD(bp); | ||||
if (bp->b_bcount == size) | if (bp->b_bcount == size) | ||||
return (1); | return (1); | ||||
if (bp->b_kvasize != 0 && bp->b_kvasize < size) | if (bp->b_kvasize != 0 && bp->b_kvasize < size) | ||||
panic("allocbuf: buffer too small"); | panic("allocbuf: buffer too small"); | ||||
newbsize = roundup2(size, DEV_BSIZE); | newbsize = roundup2(size, DEV_BSIZE); | ||||
if ((bp->b_flags & B_VMIO) == 0) { | if ((bp->b_flags & B_VMIO) == 0) { | ||||
if ((bp->b_flags & B_MALLOC) == 0) | |||||
newbsize = round_page(newbsize); | newbsize = round_page(newbsize); | ||||
Not Done Inline ActionsI do not see why would check for bufmallocspace needed there. kib: I do not see why would check for bufmallocspace needed there. | |||||
Not Done Inline ActionsI suppose it's not. It just seemed clearer from a readability standpoint to call want_malloc_buf() here. If you feel strongly about it I can remove the function and just inline the if checks in both places where they're needed. darrick.freebsd_gmail.com: I suppose it's not. It just seemed clearer from a readability standpoint to call… | |||||
Not Done Inline ActionsI suggest to remove the space check from want_malloc_buf() and keep the check at the line 4231 inlined. kib: I suggest to remove the space check from want_malloc_buf() and keep the check at the line 4231… | |||||
/* | /* | ||||
* Just get anonymous memory from the kernel. Don't | * Just get anonymous memory from the kernel. Don't | ||||
* mess with B_CACHE. | * mess with B_CACHE. | ||||
*/ | */ | ||||
if (newbsize < bp->b_bufsize) | if (newbsize < bp->b_bufsize) { | ||||
vfs_nonvmio_truncate(bp, newbsize); | vm_hold_free_pages(bp, newbsize); | ||||
else if (newbsize > bp->b_bufsize) | bufspace_adjust(bp, newbsize); | ||||
vfs_nonvmio_extend(bp, newbsize); | } else if (newbsize > bp->b_bufsize) { | ||||
vm_hold_load_pages(bp, | |||||
(vm_offset_t)bp->b_data + bp->b_bufsize, | |||||
(vm_offset_t) bp->b_data + newbsize); | |||||
bufspace_adjust(bp, newbsize); | |||||
} | |||||
} else { | } else { | ||||
int desiredpages; | int desiredpages; | ||||
desiredpages = (size == 0) ? 0 : | desiredpages = (size == 0) ? 0 : | ||||
num_pages((bp->b_offset & PAGE_MASK) + newbsize); | num_pages((bp->b_offset & PAGE_MASK) + newbsize); | ||||
if (bp->b_flags & B_MALLOC) | |||||
panic("allocbuf: VMIO buffer can't be malloced"); | |||||
/* | /* | ||||
* Set B_CACHE initially if buffer is 0 length or will become | * Set B_CACHE initially if buffer is 0 length or will become | ||||
* 0-length. | * 0-length. | ||||
*/ | */ | ||||
if (size == 0 || bp->b_bufsize == 0) | if (size == 0 || bp->b_bufsize == 0) | ||||
bp->b_flags |= B_CACHE; | bp->b_flags |= B_CACHE; | ||||
if (newbsize < bp->b_bufsize) | if (newbsize < bp->b_bufsize) | ||||
▲ Show 20 Lines • Show All 454 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* Note that while we only theoretically need to clear through b_bcount, | * Note that while we only theoretically need to clear through b_bcount, | ||||
* we go ahead and clear through b_bufsize. | * we go ahead and clear through b_bufsize. | ||||
*/ | */ | ||||
void | void | ||||
vfs_bio_clrbuf(struct buf *bp) | vfs_bio_clrbuf(struct buf *bp) | ||||
{ | { | ||||
int i, j, mask, sa, ea, slide; | int i, j, mask, sa, ea, slide; | ||||
if ((bp->b_flags & B_VMIO) == 0) { | |||||
if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) { | |||||
clrbuf(bp); | clrbuf(bp); | ||||
return; | return; | ||||
} | } | ||||
bp->b_flags &= ~B_INVAL; | bp->b_flags &= ~B_INVAL; | ||||
bp->b_ioflags &= ~BIO_ERROR; | bp->b_ioflags &= ~BIO_ERROR; | ||||
VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); | VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); | ||||
if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) && | if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) && | ||||
(bp->b_offset & PAGE_MASK) == 0) { | (bp->b_offset & PAGE_MASK) == 0) { | ||||
▲ Show 20 Lines • Show All 738 Lines • Show Last 20 Lines |
This routine and nonvmio_extend can probably both be inlined now that they're so short (they're each only called in one place).