Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_aio.c
Show First 20 Lines • Show All 1,210 Lines • ▼ Show 20 Lines | aio_qbio(struct proc *p, struct kaiocb *job) | ||||
struct aiocb *cb; | struct aiocb *cb; | ||||
struct file *fp; | struct file *fp; | ||||
struct bio *bp; | struct bio *bp; | ||||
struct buf *pbuf; | struct buf *pbuf; | ||||
struct vnode *vp; | struct vnode *vp; | ||||
struct cdevsw *csw; | struct cdevsw *csw; | ||||
struct cdev *dev; | struct cdev *dev; | ||||
struct kaioinfo *ki; | struct kaioinfo *ki; | ||||
int error, ref, poff; | struct vm_page **pages; | ||||
int error, npages, poff, ref; | |||||
vm_prot_t prot; | vm_prot_t prot; | ||||
cb = &job->uaiocb; | cb = &job->uaiocb; | ||||
fp = job->fd_file; | fp = job->fd_file; | ||||
if (!(cb->aio_lio_opcode == LIO_WRITE || | if (!(cb->aio_lio_opcode == LIO_WRITE || | ||||
cb->aio_lio_opcode == LIO_READ)) | cb->aio_lio_opcode == LIO_READ)) | ||||
return (-1); | return (-1); | ||||
Show All 26 Lines | aio_qbio(struct proc *p, struct kaiocb *job) | ||||
poff = (vm_offset_t)cb->aio_buf & PAGE_MASK; | poff = (vm_offset_t)cb->aio_buf & PAGE_MASK; | ||||
if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) { | if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) { | ||||
if (cb->aio_nbytes > maxphys) { | if (cb->aio_nbytes > maxphys) { | ||||
error = -1; | error = -1; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
pbuf = NULL; | pbuf = NULL; | ||||
job->pages = malloc(sizeof(vm_page_t) * (atop(round_page( | pages = malloc(sizeof(vm_page_t) * (atop(round_page( | ||||
cb->aio_nbytes)) + 1), M_TEMP, M_WAITOK | M_ZERO); | cb->aio_nbytes)) + 1), M_TEMP, M_WAITOK | M_ZERO); | ||||
} else { | } else { | ||||
if (cb->aio_nbytes > maxphys) { | if (cb->aio_nbytes > maxphys) { | ||||
error = -1; | error = -1; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
if (ki->kaio_buffer_count >= max_buf_aio) { | if (ki->kaio_buffer_count >= max_buf_aio) { | ||||
error = EAGAIN; | error = EAGAIN; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
job->pbuf = pbuf = uma_zalloc(pbuf_zone, M_WAITOK); | pbuf = uma_zalloc(pbuf_zone, M_WAITOK); | ||||
BUF_KERNPROC(pbuf); | BUF_KERNPROC(pbuf); | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
ki->kaio_buffer_count++; | ki->kaio_buffer_count++; | ||||
AIO_UNLOCK(ki); | AIO_UNLOCK(ki); | ||||
job->pages = pbuf->b_pages; | pages = pbuf->b_pages; | ||||
} | } | ||||
job->bp = bp = g_alloc_bio(); | bp = g_alloc_bio(); | ||||
bp->bio_length = cb->aio_nbytes; | bp->bio_length = cb->aio_nbytes; | ||||
bp->bio_bcount = cb->aio_nbytes; | bp->bio_bcount = cb->aio_nbytes; | ||||
bp->bio_done = aio_biowakeup; | bp->bio_done = aio_biowakeup; | ||||
bp->bio_offset = cb->aio_offset; | bp->bio_offset = cb->aio_offset; | ||||
bp->bio_cmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ; | bp->bio_cmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ; | ||||
bp->bio_dev = dev; | bp->bio_dev = dev; | ||||
bp->bio_caller1 = (void *)job; | bp->bio_caller1 = job; | ||||
bp->bio_caller2 = pbuf; | |||||
prot = VM_PROT_READ; | prot = VM_PROT_READ; | ||||
if (cb->aio_lio_opcode == LIO_READ) | if (cb->aio_lio_opcode == LIO_READ) | ||||
prot |= VM_PROT_WRITE; /* Less backwards than it looks */ | prot |= VM_PROT_WRITE; /* Less backwards than it looks */ | ||||
job->npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, | npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, | ||||
(vm_offset_t)cb->aio_buf, bp->bio_length, prot, job->pages, | (vm_offset_t)cb->aio_buf, bp->bio_length, prot, pages, | ||||
atop(maxphys) + 1); | atop(maxphys) + 1); | ||||
if (job->npages < 0) { | if (npages < 0) { | ||||
error = EFAULT; | error = EFAULT; | ||||
goto doerror; | goto doerror; | ||||
} | } | ||||
if (pbuf != NULL) { | if (pbuf != NULL) { | ||||
pmap_qenter((vm_offset_t)pbuf->b_data, | pmap_qenter((vm_offset_t)pbuf->b_data, pages, npages); | ||||
job->pages, job->npages); | |||||
bp->bio_data = pbuf->b_data + poff; | bp->bio_data = pbuf->b_data + poff; | ||||
atomic_add_int(&num_buf_aio, 1); | atomic_add_int(&num_buf_aio, 1); | ||||
pbuf->b_npages = npages; | |||||
} else { | } else { | ||||
bp->bio_ma = job->pages; | bp->bio_ma = pages; | ||||
bp->bio_ma_n = job->npages; | bp->bio_ma_n = npages; | ||||
kib: I did a lot of code reading to convince myself that no other code does the similar trick.
But… | |||||
Done Inline ActionsOk, I can replace bp->bio_ma with pbuf->b_pages. But I still don't see where pbuf stores npages. Unless I duplicate a lot of logic from vm_fault_quick_hold_pages, I think I have to store npages somewhere, like in bp->bio_ma_n. Or could you enlighten me? asomers: Ok, I can replace `bp->bio_ma` with `pbuf->b_pages`. But I still don't see where `pbuf` stores… | |||||
Not Done Inline ActionsIt does not store npages, but there are a lot of places you can reuse safely. Most obvious and natural is b_npages that is not used by pbuf machinery at all. kib: It does not store npages, but there are a lot of places you can reuse safely. Most obvious and… | |||||
bp->bio_ma_offset = poff; | bp->bio_ma_offset = poff; | ||||
bp->bio_data = unmapped_buf; | bp->bio_data = unmapped_buf; | ||||
bp->bio_flags |= BIO_UNMAPPED; | bp->bio_flags |= BIO_UNMAPPED; | ||||
atomic_add_int(&num_unmapped_aio, 1); | atomic_add_int(&num_unmapped_aio, 1); | ||||
} | } | ||||
/* Perform transfer. */ | /* Perform transfer. */ | ||||
csw->d_strategy(bp); | csw->d_strategy(bp); | ||||
dev_relthread(dev, ref); | dev_relthread(dev, ref); | ||||
return (0); | return (0); | ||||
doerror: | doerror: | ||||
if (pbuf != NULL) { | if (pbuf != NULL) { | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
ki->kaio_buffer_count--; | ki->kaio_buffer_count--; | ||||
AIO_UNLOCK(ki); | AIO_UNLOCK(ki); | ||||
uma_zfree(pbuf_zone, pbuf); | uma_zfree(pbuf_zone, pbuf); | ||||
job->pbuf = NULL; | |||||
} else { | } else { | ||||
free(job->pages, M_TEMP); | free(pages, M_TEMP); | ||||
} | } | ||||
g_destroy_bio(bp); | g_destroy_bio(bp); | ||||
job->bp = NULL; | |||||
unref: | unref: | ||||
dev_relthread(dev, ref); | dev_relthread(dev, ref); | ||||
return (error); | return (error); | ||||
} | } | ||||
#ifdef COMPAT_FREEBSD6 | #ifdef COMPAT_FREEBSD6 | ||||
static int | static int | ||||
convert_old_sigevent(struct osigevent *osig, struct sigevent *nsig) | convert_old_sigevent(struct osigevent *osig, struct sigevent *nsig) | ||||
▲ Show 20 Lines • Show All 996 Lines • ▼ Show 20 Lines | sys_lio_listio(struct thread *td, struct lio_listio_args *uap) | ||||
return (error); | return (error); | ||||
} | } | ||||
static void | static void | ||||
aio_biowakeup(struct bio *bp) | aio_biowakeup(struct bio *bp) | ||||
{ | { | ||||
struct kaiocb *job = (struct kaiocb *)bp->bio_caller1; | struct kaiocb *job = (struct kaiocb *)bp->bio_caller1; | ||||
struct kaioinfo *ki; | struct kaioinfo *ki; | ||||
struct buf *pbuf = (struct buf*)bp->bio_caller2; | |||||
size_t nbytes; | size_t nbytes; | ||||
int error, nblks; | int error, nblks; | ||||
/* Release mapping into kernel space. */ | /* Release mapping into kernel space. */ | ||||
if (job->pbuf != NULL) { | if (pbuf != NULL) { | ||||
pmap_qremove((vm_offset_t)job->pbuf->b_data, job->npages); | MPASS(pbuf->b_npages <= atop(maxphys) + 1); | ||||
Done Inline ActionsPerhaps assert that bio_ma_n is <= atop(maxphys) + 1. kib: Perhaps assert that bio_ma_n is <= atop(maxphys) + 1. | |||||
vm_page_unhold_pages(job->pages, job->npages); | pmap_qremove((vm_offset_t)pbuf->b_data, pbuf->b_npages); | ||||
uma_zfree(pbuf_zone, job->pbuf); | vm_page_unhold_pages(pbuf->b_pages, pbuf->b_npages); | ||||
job->pbuf = NULL; | uma_zfree(pbuf_zone, pbuf); | ||||
atomic_subtract_int(&num_buf_aio, 1); | atomic_subtract_int(&num_buf_aio, 1); | ||||
ki = job->userproc->p_aioinfo; | ki = job->userproc->p_aioinfo; | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
ki->kaio_buffer_count--; | ki->kaio_buffer_count--; | ||||
AIO_UNLOCK(ki); | AIO_UNLOCK(ki); | ||||
} else { | } else { | ||||
vm_page_unhold_pages(job->pages, job->npages); | MPASS(bp->bio_ma_n <= atop(maxphys) + 1); | ||||
free(job->pages, M_TEMP); | vm_page_unhold_pages(bp->bio_ma, bp->bio_ma_n); | ||||
free(bp->bio_ma, M_TEMP); | |||||
atomic_subtract_int(&num_unmapped_aio, 1); | atomic_subtract_int(&num_unmapped_aio, 1); | ||||
} | } | ||||
bp = job->bp; | |||||
job->bp = NULL; | |||||
nbytes = job->uaiocb.aio_nbytes - bp->bio_resid; | nbytes = job->uaiocb.aio_nbytes - bp->bio_resid; | ||||
error = 0; | error = 0; | ||||
if (bp->bio_flags & BIO_ERROR) | if (bp->bio_flags & BIO_ERROR) | ||||
error = bp->bio_error; | error = bp->bio_error; | ||||
nblks = btodb(nbytes); | nblks = btodb(nbytes); | ||||
if (job->uaiocb.aio_lio_opcode == LIO_WRITE) | if (job->uaiocb.aio_lio_opcode == LIO_WRITE) | ||||
job->outblock += nblks; | job->outblock += nblks; | ||||
else | else | ||||
▲ Show 20 Lines • Show All 621 Lines • Show Last 20 Lines |
I did a lot of code reading to convince myself that no other code does the similar trick.
But then it is not clear to me why do you need to assign bio_ma and bio_ma_n. You have pbuf in bio_caller1, and this provides the needed info. aio_biowakeup still needs to distinguish unmapped vs mapped (pbuf used) cases.
If you use pbuf instead of using bio_ma* without BIO_UNMAPPED set, it fits with the current code assumptions.