Changeset View
Standalone View
sys/kern/vfs_aio.c
Show First 20 Lines • Show All 1,246 Lines • ▼ Show 20 Lines | aio_qbio(struct proc *p, struct kaiocb *job) | ||||
if (cb->aio_nbytes > dev->si_iosize_max) { | if (cb->aio_nbytes > dev->si_iosize_max) { | ||||
error = -1; | error = -1; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
ki = p->p_aioinfo; | ki = p->p_aioinfo; | ||||
poff = (vm_offset_t)cb->aio_buf & PAGE_MASK; | poff = (vm_offset_t)cb->aio_buf & PAGE_MASK; | ||||
if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) { | if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) { | ||||
if (cb->aio_nbytes > MAXPHYS) { | if (cb->aio_nbytes > maxphys) { | ||||
error = -1; | error = -1; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
pbuf = NULL; | pbuf = NULL; | ||||
job->pages = malloc(sizeof(vm_page_t) * (atop(round_page( | |||||
cb->aio_nbytes)) + 1), M_TEMP, M_WAITOK | M_ZERO); | |||||
markj: The `+ 1` looks like it's in the wrong place, but I do not really understand why it's there to… | |||||
Done Inline ActionsIndeed misplaced, while needed to represent misaligned maxphys buffers. mav: Indeed misplaced, while needed to represent misaligned maxphys buffers. | |||||
} else { | } else { | ||||
if (cb->aio_nbytes > MAXPHYS - poff) { | if (cb->aio_nbytes > maxphys) { | ||||
Done Inline ActionsThis condition can be relaxed with extra page in pbuf. mav: This condition can be relaxed with extra page in pbuf. | |||||
error = -1; | error = -1; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
if (ki->kaio_buffer_count >= max_buf_aio) { | if (ki->kaio_buffer_count >= max_buf_aio) { | ||||
error = EAGAIN; | error = EAGAIN; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
job->pbuf = pbuf = uma_zalloc(pbuf_zone, M_WAITOK); | job->pbuf = pbuf = uma_zalloc(pbuf_zone, M_WAITOK); | ||||
BUF_KERNPROC(pbuf); | BUF_KERNPROC(pbuf); | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
ki->kaio_buffer_count++; | ki->kaio_buffer_count++; | ||||
AIO_UNLOCK(ki); | AIO_UNLOCK(ki); | ||||
job->pages = pbuf->b_pages; | |||||
} | } | ||||
job->bp = bp = g_alloc_bio(); | job->bp = bp = g_alloc_bio(); | ||||
bp->bio_length = cb->aio_nbytes; | bp->bio_length = cb->aio_nbytes; | ||||
bp->bio_bcount = cb->aio_nbytes; | bp->bio_bcount = cb->aio_nbytes; | ||||
bp->bio_done = aio_biowakeup; | bp->bio_done = aio_biowakeup; | ||||
bp->bio_offset = cb->aio_offset; | bp->bio_offset = cb->aio_offset; | ||||
bp->bio_cmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ; | bp->bio_cmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ; | ||||
bp->bio_dev = dev; | bp->bio_dev = dev; | ||||
bp->bio_caller1 = (void *)job; | bp->bio_caller1 = (void *)job; | ||||
prot = VM_PROT_READ; | prot = VM_PROT_READ; | ||||
if (cb->aio_lio_opcode == LIO_READ) | if (cb->aio_lio_opcode == LIO_READ) | ||||
prot |= VM_PROT_WRITE; /* Less backwards than it looks */ | prot |= VM_PROT_WRITE; /* Less backwards than it looks */ | ||||
job->npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, | job->npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, | ||||
(vm_offset_t)cb->aio_buf, bp->bio_length, prot, job->pages, | (vm_offset_t)cb->aio_buf, bp->bio_length, prot, job->pages, | ||||
nitems(job->pages)); | atop(maxphys) + 1); | ||||
Done Inline Actionsnitems(job->pages) is wrong as well. kib: nitems(job->pages) is wrong as well. | |||||
if (job->npages < 0) { | if (job->npages < 0) { | ||||
error = EFAULT; | error = EFAULT; | ||||
goto doerror; | goto doerror; | ||||
} | } | ||||
if (pbuf != NULL) { | if (pbuf != NULL) { | ||||
pmap_qenter((vm_offset_t)pbuf->b_data, | pmap_qenter((vm_offset_t)pbuf->b_data, | ||||
job->pages, job->npages); | job->pages, job->npages); | ||||
bp->bio_data = pbuf->b_data + poff; | bp->bio_data = pbuf->b_data + poff; | ||||
Show All 14 Lines | |||||
doerror: | doerror: | ||||
if (pbuf != NULL) { | if (pbuf != NULL) { | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
ki->kaio_buffer_count--; | ki->kaio_buffer_count--; | ||||
AIO_UNLOCK(ki); | AIO_UNLOCK(ki); | ||||
uma_zfree(pbuf_zone, pbuf); | uma_zfree(pbuf_zone, pbuf); | ||||
job->pbuf = NULL; | job->pbuf = NULL; | ||||
} else { | |||||
free(job->pages, M_TEMP); | |||||
} | } | ||||
g_destroy_bio(bp); | g_destroy_bio(bp); | ||||
job->bp = NULL; | job->bp = NULL; | ||||
unref: | unref: | ||||
dev_relthread(dev, ref); | dev_relthread(dev, ref); | ||||
return (error); | return (error); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 1,006 Lines • ▼ Show 20 Lines | aio_biowakeup(struct bio *bp) | ||||
struct proc *userp; | struct proc *userp; | ||||
struct kaioinfo *ki; | struct kaioinfo *ki; | ||||
size_t nbytes; | size_t nbytes; | ||||
int error, nblks; | int error, nblks; | ||||
/* Release mapping into kernel space. */ | /* Release mapping into kernel space. */ | ||||
userp = job->userproc; | userp = job->userproc; | ||||
ki = userp->p_aioinfo; | ki = userp->p_aioinfo; | ||||
if (job->pbuf) { | vm_page_unhold_pages(job->pages, job->npages); | ||||
alcUnsubmitted Not Done Inline ActionsAre you should that reordering vm_page_unhold_pages() and pmap_qremove() is universally safe? Consider powerpc{,64}. The powerpc{,64} pmaps create PVO entries for the unmanaged mappings created by pmap_qenter(). Moreover, pmap_page_is_mapped() doesn't distinguish between PVO entries representing managed versus unmanaged mappings. So, if the page is freed by vm_page_unhold_pages(), can't the KASSERT(!pmap_page_is_mapped()) fail? alc: Are you should that reordering vm_page_unhold_pages() and pmap_qremove() is universally safe? | |||||
kibAuthorUnsubmitted Done Inline ActionsThis is, in fact, a surprise for me. I thought that qenter() mappings are never accounted for as mappings for pmap_page_is_mapped(). The reordering was needed because job->pages is really pbuf->b_pages, and code freed pbuf before doing unwire(). kib: This is, in fact, a surprise for me. I thought that qenter() mappings are never accounted for… | |||||
if (job->pbuf != NULL) { | |||||
pmap_qremove((vm_offset_t)job->pbuf->b_data, job->npages); | pmap_qremove((vm_offset_t)job->pbuf->b_data, job->npages); | ||||
uma_zfree(pbuf_zone, job->pbuf); | uma_zfree(pbuf_zone, job->pbuf); | ||||
job->pbuf = NULL; | job->pbuf = NULL; | ||||
atomic_subtract_int(&num_buf_aio, 1); | atomic_subtract_int(&num_buf_aio, 1); | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
ki->kaio_buffer_count--; | ki->kaio_buffer_count--; | ||||
AIO_UNLOCK(ki); | AIO_UNLOCK(ki); | ||||
} else | } else { | ||||
free(job->pages, M_TEMP); | |||||
atomic_subtract_int(&num_unmapped_aio, 1); | atomic_subtract_int(&num_unmapped_aio, 1); | ||||
vm_page_unhold_pages(job->pages, job->npages); | } | ||||
bp = job->bp; | bp = job->bp; | ||||
job->bp = NULL; | job->bp = NULL; | ||||
nbytes = job->uaiocb.aio_nbytes - bp->bio_resid; | nbytes = job->uaiocb.aio_nbytes - bp->bio_resid; | ||||
error = 0; | error = 0; | ||||
if (bp->bio_flags & BIO_ERROR) | if (bp->bio_flags & BIO_ERROR) | ||||
error = bp->bio_error; | error = bp->bio_error; | ||||
nblks = btodb(nbytes); | nblks = btodb(nbytes); | ||||
▲ Show 20 Lines • Show All 624 Lines • Show Last 20 Lines |
The + 1 looks like it's in the wrong place, but I do not really understand why it's there to begin with.