Changeset View
Standalone View
sys/kern/vfs_aio.c
Show First 20 Lines • Show All 286 Lines • ▼ Show 20 Lines | |||||
#define KAIO_RUNDOWN 0x1 /* process is being run down */ | #define KAIO_RUNDOWN 0x1 /* process is being run down */ | ||||
#define KAIO_WAKEUP 0x2 /* wakeup process when AIO completes */ | #define KAIO_WAKEUP 0x2 /* wakeup process when AIO completes */ | ||||
/* | /* | ||||
* Operations used to interact with userland aio control blocks. | * Operations used to interact with userland aio control blocks. | ||||
* Different ABIs provide their own operations. | * Different ABIs provide their own operations. | ||||
*/ | */ | ||||
struct aiocb_ops { | struct aiocb_ops { | ||||
int (*aio_copyin)(struct aiocb *ujob, struct aiocb *kjob); | int (*aio_copyin)(struct aiocb *ujob, struct kaiocb *kjob, int ty); | ||||
long (*fetch_status)(struct aiocb *ujob); | long (*fetch_status)(struct aiocb *ujob); | ||||
long (*fetch_error)(struct aiocb *ujob); | long (*fetch_error)(struct aiocb *ujob); | ||||
int (*store_status)(struct aiocb *ujob, long status); | int (*store_status)(struct aiocb *ujob, long status); | ||||
int (*store_error)(struct aiocb *ujob, long error); | int (*store_error)(struct aiocb *ujob, long error); | ||||
int (*store_kernelinfo)(struct aiocb *ujob, long jobref); | int (*store_kernelinfo)(struct aiocb *ujob, long jobref); | ||||
int (*store_aiocb)(struct aiocb **ujobp, struct aiocb *ujob); | int (*store_aiocb)(struct aiocb **ujobp, struct aiocb *ujob); | ||||
}; | }; | ||||
static TAILQ_HEAD(,aioproc) aio_freeproc; /* (c) Idle daemons */ | static TAILQ_HEAD(,aioproc) aio_freeproc; /* (c) Idle daemons */ | ||||
static struct sema aio_newproc_sem; | static struct sema aio_newproc_sem; | ||||
static struct mtx aio_job_mtx; | static struct mtx aio_job_mtx; | ||||
static TAILQ_HEAD(,kaiocb) aio_jobs; /* (c) Async job list */ | static TAILQ_HEAD(,kaiocb) aio_jobs; /* (c) Async job list */ | ||||
static struct unrhdr *aiod_unr; | static struct unrhdr *aiod_unr; | ||||
static void aio_biocleanup(struct bio *bp); | |||||
void aio_init_aioinfo(struct proc *p); | void aio_init_aioinfo(struct proc *p); | ||||
static int aio_onceonly(void); | static int aio_onceonly(void); | ||||
static int aio_free_entry(struct kaiocb *job); | static int aio_free_entry(struct kaiocb *job); | ||||
static void aio_process_rw(struct kaiocb *job); | static void aio_process_rw(struct kaiocb *job); | ||||
static void aio_process_sync(struct kaiocb *job); | static void aio_process_sync(struct kaiocb *job); | ||||
static void aio_process_mlock(struct kaiocb *job); | static void aio_process_mlock(struct kaiocb *job); | ||||
static void aio_schedule_fsync(void *context, int pending); | static void aio_schedule_fsync(void *context, int pending); | ||||
static int aio_newproc(int *); | static int aio_newproc(int *); | ||||
▲ Show 20 Lines • Show All 236 Lines • ▼ Show 20 Lines | aio_free_entry(struct kaiocb *job) | ||||
* knlist_delete(). This does mean that it is possible for the | * knlist_delete(). This does mean that it is possible for the | ||||
* thread pointer at close time to differ from the thread pointer | * thread pointer at close time to differ from the thread pointer | ||||
* at open time, but this is already true of file descriptors in | * at open time, but this is already true of file descriptors in | ||||
* a multithreaded process. | * a multithreaded process. | ||||
*/ | */ | ||||
if (job->fd_file) | if (job->fd_file) | ||||
fdrop(job->fd_file, curthread); | fdrop(job->fd_file, curthread); | ||||
crfree(job->cred); | crfree(job->cred); | ||||
if (job->uiop != &job->uio) | |||||
free(job->uiop, M_IOV); | |||||
uma_zfree(aiocb_zone, job); | uma_zfree(aiocb_zone, job); | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
aio_proc_rundown_exec(void *arg, struct proc *p, | aio_proc_rundown_exec(void *arg, struct proc *p, | ||||
▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
static void | static void | ||||
aio_process_rw(struct kaiocb *job) | aio_process_rw(struct kaiocb *job) | ||||
{ | { | ||||
struct ucred *td_savedcred; | struct ucred *td_savedcred; | ||||
struct thread *td; | struct thread *td; | ||||
struct aiocb *cb; | struct aiocb *cb; | ||||
struct file *fp; | struct file *fp; | ||||
struct uio auio; | |||||
struct iovec aiov; | |||||
ssize_t cnt; | ssize_t cnt; | ||||
long msgsnd_st, msgsnd_end; | long msgsnd_st, msgsnd_end; | ||||
long msgrcv_st, msgrcv_end; | long msgrcv_st, msgrcv_end; | ||||
long oublock_st, oublock_end; | long oublock_st, oublock_end; | ||||
long inblock_st, inblock_end; | long inblock_st, inblock_end; | ||||
int error; | int error, opcode; | ||||
KASSERT(job->uaiocb.aio_lio_opcode == LIO_READ || | KASSERT(job->uaiocb.aio_lio_opcode == LIO_READ || | ||||
job->uaiocb.aio_lio_opcode == LIO_WRITE, | job->uaiocb.aio_lio_opcode == LIO_READV || | ||||
job->uaiocb.aio_lio_opcode == LIO_WRITE || | |||||
job->uaiocb.aio_lio_opcode == LIO_WRITEV, | |||||
("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode)); | ("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode)); | ||||
aio_switch_vmspace(job); | aio_switch_vmspace(job); | ||||
td = curthread; | td = curthread; | ||||
td_savedcred = td->td_ucred; | td_savedcred = td->td_ucred; | ||||
td->td_ucred = job->cred; | td->td_ucred = job->cred; | ||||
job->uiop->uio_td = td; | |||||
cb = &job->uaiocb; | cb = &job->uaiocb; | ||||
fp = job->fd_file; | fp = job->fd_file; | ||||
aiov.iov_base = (void *)(uintptr_t)cb->aio_buf; | opcode = job->uaiocb.aio_lio_opcode; | ||||
aiov.iov_len = cb->aio_nbytes; | cnt = job->uiop->uio_resid; | ||||
auio.uio_iov = &aiov; | |||||
auio.uio_iovcnt = 1; | |||||
auio.uio_offset = cb->aio_offset; | |||||
auio.uio_resid = cb->aio_nbytes; | |||||
cnt = cb->aio_nbytes; | |||||
auio.uio_segflg = UIO_USERSPACE; | |||||
auio.uio_td = td; | |||||
msgrcv_st = td->td_ru.ru_msgrcv; | msgrcv_st = td->td_ru.ru_msgrcv; | ||||
msgsnd_st = td->td_ru.ru_msgsnd; | msgsnd_st = td->td_ru.ru_msgsnd; | ||||
inblock_st = td->td_ru.ru_inblock; | inblock_st = td->td_ru.ru_inblock; | ||||
oublock_st = td->td_ru.ru_oublock; | oublock_st = td->td_ru.ru_oublock; | ||||
/* | /* | ||||
* aio_aqueue() acquires a reference to the file that is | * aio_aqueue() acquires a reference to the file that is | ||||
* released in aio_free_entry(). | * released in aio_free_entry(). | ||||
*/ | */ | ||||
if (cb->aio_lio_opcode == LIO_READ) { | if (opcode == LIO_READ || opcode == LIO_READV) { | ||||
auio.uio_rw = UIO_READ; | if (job->uiop->uio_resid == 0) | ||||
if (auio.uio_resid == 0) | |||||
error = 0; | error = 0; | ||||
else | else | ||||
error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, td); | error = fo_read(fp, job->uiop, fp->f_cred, FOF_OFFSET, | ||||
td); | |||||
} else { | } else { | ||||
if (fp->f_type == DTYPE_VNODE) | if (fp->f_type == DTYPE_VNODE) | ||||
bwillwrite(); | bwillwrite(); | ||||
auio.uio_rw = UIO_WRITE; | error = fo_write(fp, job->uiop, fp->f_cred, FOF_OFFSET, td); | ||||
error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, td); | |||||
} | } | ||||
msgrcv_end = td->td_ru.ru_msgrcv; | msgrcv_end = td->td_ru.ru_msgrcv; | ||||
msgsnd_end = td->td_ru.ru_msgsnd; | msgsnd_end = td->td_ru.ru_msgsnd; | ||||
inblock_end = td->td_ru.ru_inblock; | inblock_end = td->td_ru.ru_inblock; | ||||
oublock_end = td->td_ru.ru_oublock; | oublock_end = td->td_ru.ru_oublock; | ||||
job->msgrcv = msgrcv_end - msgrcv_st; | job->msgrcv = msgrcv_end - msgrcv_st; | ||||
job->msgsnd = msgsnd_end - msgsnd_st; | job->msgsnd = msgsnd_end - msgsnd_st; | ||||
job->inblock = inblock_end - inblock_st; | job->inblock = inblock_end - inblock_st; | ||||
job->outblock = oublock_end - oublock_st; | job->outblock = oublock_end - oublock_st; | ||||
if ((error) && (auio.uio_resid != cnt)) { | if (error != 0 && job->uiop->uio_resid != cnt) { | ||||
kib: Extra () | |||||
if (error == ERESTART || error == EINTR || error == EWOULDBLOCK) | if (error == ERESTART || error == EINTR || error == EWOULDBLOCK) | ||||
error = 0; | error = 0; | ||||
if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) { | if (error == EPIPE && | ||||
(opcode == LIO_WRITE || opcode == LIO_WRITEV)) { | |||||
PROC_LOCK(job->userproc); | PROC_LOCK(job->userproc); | ||||
kern_psignal(job->userproc, SIGPIPE); | kern_psignal(job->userproc, SIGPIPE); | ||||
PROC_UNLOCK(job->userproc); | PROC_UNLOCK(job->userproc); | ||||
} | } | ||||
} | } | ||||
cnt -= auio.uio_resid; | cnt -= job->uiop->uio_resid; | ||||
td->td_ucred = td_savedcred; | td->td_ucred = td_savedcred; | ||||
if (error) | if (error) | ||||
aio_complete(job, -1, error); | aio_complete(job, -1, error); | ||||
else | else | ||||
aio_complete(job, cnt, 0); | aio_complete(job, cnt, 0); | ||||
} | } | ||||
static void | static void | ||||
▲ Show 20 Lines • Show All 367 Lines • ▼ Show 20 Lines | |||||
* structure's reference count, preventing its deallocation for the | * structure's reference count, preventing its deallocation for the | ||||
* duration of this call. | * duration of this call. | ||||
*/ | */ | ||||
static int | static int | ||||
aio_qbio(struct proc *p, struct kaiocb *job) | aio_qbio(struct proc *p, struct kaiocb *job) | ||||
{ | { | ||||
struct aiocb *cb; | struct aiocb *cb; | ||||
struct file *fp; | struct file *fp; | ||||
struct bio *bp; | |||||
struct buf *pbuf; | struct buf *pbuf; | ||||
struct vnode *vp; | struct vnode *vp; | ||||
struct cdevsw *csw; | struct cdevsw *csw; | ||||
struct cdev *dev; | struct cdev *dev; | ||||
struct kaioinfo *ki; | struct kaioinfo *ki; | ||||
struct vm_page **pages; | struct bio **bios = NULL; | ||||
int error, npages, poff, ref; | off_t offset; | ||||
int bio_cmd, error, i, iovcnt, opcode, poff, ref; | |||||
vm_prot_t prot; | vm_prot_t prot; | ||||
bool use_unmapped; | |||||
cb = &job->uaiocb; | cb = &job->uaiocb; | ||||
fp = job->fd_file; | fp = job->fd_file; | ||||
opcode = cb->aio_lio_opcode; | |||||
if (!(cb->aio_lio_opcode == LIO_WRITE || | if (!(opcode == LIO_WRITE || opcode == LIO_WRITEV || | ||||
cb->aio_lio_opcode == LIO_READ)) | opcode == LIO_READ || opcode == LIO_READV)) | ||||
return (-1); | return (-1); | ||||
if (fp == NULL || fp->f_type != DTYPE_VNODE) | if (fp == NULL || fp->f_type != DTYPE_VNODE) | ||||
return (-1); | return (-1); | ||||
vp = fp->f_vnode; | vp = fp->f_vnode; | ||||
if (vp->v_type != VCHR) | if (vp->v_type != VCHR) | ||||
return (-1); | return (-1); | ||||
if (vp->v_bufobj.bo_bsize == 0) | if (vp->v_bufobj.bo_bsize == 0) | ||||
return (-1); | return (-1); | ||||
if (cb->aio_nbytes % vp->v_bufobj.bo_bsize) | |||||
bio_cmd = opcode == LIO_WRITE || opcode == LIO_WRITEV ? BIO_WRITE : | |||||
BIO_READ; | |||||
iovcnt = job->uiop->uio_iovcnt; | |||||
if (iovcnt > max_buf_aio) | |||||
return (-1); | return (-1); | ||||
for (i = 0; i < iovcnt; i++) { | |||||
if (job->uiop->uio_iov[i].iov_len % vp->v_bufobj.bo_bsize != 0) | |||||
return (-1); | |||||
if (job->uiop->uio_iov[i].iov_len > maxphys) { | |||||
error = -1; | |||||
return (-1); | |||||
} | |||||
} | |||||
offset = cb->aio_offset; | |||||
ref = 0; | ref = 0; | ||||
csw = devvn_refthread(vp, &dev, &ref); | csw = devvn_refthread(vp, &dev, &ref); | ||||
if (csw == NULL) | if (csw == NULL) | ||||
return (ENXIO); | return (ENXIO); | ||||
if ((csw->d_flags & D_DISK) == 0) { | if ((csw->d_flags & D_DISK) == 0) { | ||||
error = -1; | error = -1; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
if (cb->aio_nbytes > dev->si_iosize_max) { | if (job->uiop->uio_resid > dev->si_iosize_max) { | ||||
error = -1; | error = -1; | ||||
goto unref; | goto unref; | ||||
} | } | ||||
ki = p->p_aioinfo; | ki = p->p_aioinfo; | ||||
poff = (vm_offset_t)cb->aio_buf & PAGE_MASK; | job->error = 0; | ||||
if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) { | |||||
if (cb->aio_nbytes > maxphys) { | use_unmapped = (dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed; | ||||
error = -1; | if (!use_unmapped) { | ||||
AIO_LOCK(ki); | |||||
if (ki->kaio_buffer_count + iovcnt > max_buf_aio) { | |||||
AIO_UNLOCK(ki); | |||||
error = EAGAIN; | |||||
goto unref; | goto unref; | ||||
} | } | ||||
ki->kaio_buffer_count += iovcnt; | |||||
Done Inline ActionsSo if an error occurs in the loop, after some iovs are already queued, you return -1 and aio_aqueue_file() re-queues them ? Am I missing something ? kib: So if an error occurs in the loop, after some iovs are already queued, you return -1 and… | |||||
Done Inline ActionsIf aio_qbio returns -1, then aio_aqueue_file will queue it using the slow path, aio_process_rw. asomers: If `aio_qbio` returns `-1`, then `aio_aqueue_file` will queue it using the slow path… | |||||
Done Inline ActionsSo you confirm my observation, and it is a valid bug. kib: So you confirm my observation, and it is a valid bug. | |||||
Done Inline ActionsNo, it's not a bug. aio_queue_file tries the fast path first, and if that doesn't work it tries the slow path. Perhaps you're thinking that aio_process_rw calls aio_qbio? It doesn't. Only aio_queue_file does. asomers: No, it's not a bug. `aio_queue_file` tries the fast path first, and if that doesn't work it… | |||||
Done Inline ActionsThe loop processes iovs one by one. If e.g. first iov was queued, but second causes an error, aio_queue_file() requeues everything. So first iov is queued twice. kib: The loop processes iovs one by one. If e.g. first iov was queued, but second causes an error… | |||||
Done Inline ActionsOk, I see what you're saying. Some of the iovs will be issued twice in that case. I think I can fix that. asomers: Ok, I see what you're saying. Some of the iovs will be issued twice in that case. I think I… | |||||
AIO_UNLOCK(ki); | |||||
} | |||||
bios = malloc(sizeof(struct bio *) * iovcnt, M_TEMP, M_WAITOK); | |||||
Done Inline Actionsstruct bio * (space before star) BTW might be mallocarray(9) is better kib: `struct bio *` (space before star)
BTW might be mallocarray(9) is better | |||||
atomic_store_int(&job->nbio, iovcnt); | |||||
for (i = 0; i < iovcnt; i++) { | |||||
struct vm_page** pages; | |||||
struct bio *bp; | |||||
void *buf; | |||||
size_t nbytes; | |||||
int npages; | |||||
buf = job->uiop->uio_iov[i].iov_base; | |||||
nbytes = job->uiop->uio_iov[i].iov_len; | |||||
bios[i] = g_alloc_bio(); | |||||
bp = bios[i]; | |||||
poff = (vm_offset_t)buf & PAGE_MASK; | |||||
if (use_unmapped) { | |||||
pbuf = NULL; | pbuf = NULL; | ||||
pages = malloc(sizeof(vm_page_t) * (atop(round_page( | pages = malloc(sizeof(vm_page_t) * (atop(round_page( | ||||
cb->aio_nbytes)) + 1), M_TEMP, M_WAITOK | M_ZERO); | nbytes)) + 1), M_TEMP, M_WAITOK | M_ZERO); | ||||
} else { | } else { | ||||
if (cb->aio_nbytes > maxphys) { | |||||
error = -1; | |||||
goto unref; | |||||
} | |||||
if (ki->kaio_buffer_count >= max_buf_aio) { | |||||
error = EAGAIN; | |||||
goto unref; | |||||
} | |||||
pbuf = uma_zalloc(pbuf_zone, M_WAITOK); | pbuf = uma_zalloc(pbuf_zone, M_WAITOK); | ||||
BUF_KERNPROC(pbuf); | BUF_KERNPROC(pbuf); | ||||
AIO_LOCK(ki); | |||||
ki->kaio_buffer_count++; | |||||
AIO_UNLOCK(ki); | |||||
pages = pbuf->b_pages; | pages = pbuf->b_pages; | ||||
} | } | ||||
bp = g_alloc_bio(); | |||||
bp->bio_length = cb->aio_nbytes; | bp->bio_length = nbytes; | ||||
bp->bio_bcount = cb->aio_nbytes; | bp->bio_bcount = nbytes; | ||||
bp->bio_done = aio_biowakeup; | bp->bio_done = aio_biowakeup; | ||||
bp->bio_offset = cb->aio_offset; | bp->bio_offset = offset; | ||||
bp->bio_cmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ; | bp->bio_cmd = bio_cmd; | ||||
bp->bio_dev = dev; | bp->bio_dev = dev; | ||||
bp->bio_caller1 = job; | bp->bio_caller1 = job; | ||||
bp->bio_caller2 = pbuf; | bp->bio_caller2 = pbuf; | ||||
prot = VM_PROT_READ; | prot = VM_PROT_READ; | ||||
if (cb->aio_lio_opcode == LIO_READ) | if (opcode == LIO_READ || opcode == LIO_READV) | ||||
prot |= VM_PROT_WRITE; /* Less backwards than it looks */ | prot |= VM_PROT_WRITE; /* Less backwards than it looks */ | ||||
npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, | npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, | ||||
(vm_offset_t)cb->aio_buf, bp->bio_length, prot, pages, | (vm_offset_t)buf, bp->bio_length, prot, pages, | ||||
atop(maxphys) + 1); | atop(maxphys) + 1); | ||||
if (npages < 0) { | if (npages < 0) { | ||||
if (pbuf != NULL) | |||||
uma_zfree(pbuf_zone, pbuf); | |||||
else | |||||
Done Inline ActionsWhy -i ? Also the cleanup for kaio_buffer_count might be more logical in destroy_bios. kib: Why `-i` ?
Also the cleanup for kaio_buffer_count might be more logical in destroy_bios. | |||||
Done Inline ActionsLeftover from an earlier revision. Should be just iovcnt. asomers: Leftover from an earlier revision. Should be just `iovcnt.` | |||||
free(pages, M_TEMP); | |||||
error = EFAULT; | error = EFAULT; | ||||
goto doerror; | g_destroy_bio(bp); | ||||
i--; | |||||
goto destroy_bios; | |||||
} | } | ||||
if (pbuf != NULL) { | if (pbuf != NULL) { | ||||
pmap_qenter((vm_offset_t)pbuf->b_data, pages, npages); | pmap_qenter((vm_offset_t)pbuf->b_data, pages, npages); | ||||
bp->bio_data = pbuf->b_data + poff; | bp->bio_data = pbuf->b_data + poff; | ||||
atomic_add_int(&num_buf_aio, 1); | |||||
pbuf->b_npages = npages; | pbuf->b_npages = npages; | ||||
atomic_add_int(&num_buf_aio, 1); | |||||
} else { | } else { | ||||
bp->bio_ma = pages; | bp->bio_ma = pages; | ||||
bp->bio_ma_n = npages; | bp->bio_ma_n = npages; | ||||
bp->bio_ma_offset = poff; | bp->bio_ma_offset = poff; | ||||
bp->bio_data = unmapped_buf; | bp->bio_data = unmapped_buf; | ||||
bp->bio_flags |= BIO_UNMAPPED; | bp->bio_flags |= BIO_UNMAPPED; | ||||
atomic_add_int(&num_unmapped_aio, 1); | atomic_add_int(&num_unmapped_aio, 1); | ||||
} | } | ||||
offset += nbytes; | |||||
} | |||||
/* Perform transfer. */ | /* Perform transfer. */ | ||||
csw->d_strategy(bp); | for (i = 0; i < iovcnt; i++) | ||||
csw->d_strategy(bios[i]); | |||||
free(bios, M_TEMP); | |||||
dev_relthread(dev, ref); | dev_relthread(dev, ref); | ||||
return (0); | return (0); | ||||
doerror: | destroy_bios: | ||||
if (pbuf != NULL) { | for (; i >= 0; i--) | ||||
AIO_LOCK(ki); | aio_biocleanup(bios[i]); | ||||
Done Inline ActionsYou need to qremove and unhold pages for already processed buffers. kib: You need to qremove and unhold pages for already processed buffers. | |||||
ki->kaio_buffer_count--; | free(bios, M_TEMP); | ||||
AIO_UNLOCK(ki); | |||||
uma_zfree(pbuf_zone, pbuf); | |||||
} else { | |||||
free(pages, M_TEMP); | |||||
} | |||||
g_destroy_bio(bp); | |||||
unref: | unref: | ||||
dev_relthread(dev, ref); | dev_relthread(dev, ref); | ||||
return (error); | return (error); | ||||
} | } | ||||
#ifdef COMPAT_FREEBSD6 | #ifdef COMPAT_FREEBSD6 | ||||
static int | static int | ||||
convert_old_sigevent(struct osigevent *osig, struct sigevent *nsig) | convert_old_sigevent(struct osigevent *osig, struct sigevent *nsig) | ||||
Show All 17 Lines | case SIGEV_KEVENT: | ||||
break; | break; | ||||
default: | default: | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
aiocb_copyin_old_sigevent(struct aiocb *ujob, struct aiocb *kjob) | aiocb_copyin_old_sigevent(struct aiocb *ujob, struct kaiocb *kjob, | ||||
int type __unused) | |||||
{ | { | ||||
struct oaiocb *ojob; | struct oaiocb *ojob; | ||||
struct aiocb *kcb = &kjob->uaiocb; | |||||
int error; | int error; | ||||
bzero(kjob, sizeof(struct aiocb)); | bzero(kcb, sizeof(struct aiocb)); | ||||
error = copyin(ujob, kjob, sizeof(struct oaiocb)); | error = copyin(ujob, kcb, sizeof(struct oaiocb)); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
ojob = (struct oaiocb *)kjob; | /* No need to copyin aio_iov, because it did not exist in FreeBSD 6 */ | ||||
return (convert_old_sigevent(&ojob->aio_sigevent, &kjob->aio_sigevent)); | ojob = (struct oaiocb *)kcb; | ||||
return (convert_old_sigevent(&ojob->aio_sigevent, &kcb->aio_sigevent)); | |||||
} | } | ||||
#endif | #endif | ||||
static int | static int | ||||
aiocb_copyin(struct aiocb *ujob, struct aiocb *kjob) | aiocb_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type) | ||||
{ | { | ||||
struct aiocb *kcb = &kjob->uaiocb; | |||||
int error; | |||||
return (copyin(ujob, kjob, sizeof(struct aiocb))); | error = copyin(ujob, kcb, sizeof(struct aiocb)); | ||||
if (error) | |||||
return (error); | |||||
if (type == LIO_READV || type == LIO_WRITEV) { | |||||
/* malloc a uio and copy in the iovec */ | |||||
error = copyinuio(kcb->aio_iov, kcb->aio_iovcnt, &kjob->uiop); | |||||
} | } | ||||
return (error); | |||||
} | |||||
static long | static long | ||||
aiocb_fetch_status(struct aiocb *ujob) | aiocb_fetch_status(struct aiocb *ujob) | ||||
{ | { | ||||
return (fuword(&ujob->_aiocb_private.status)); | return (fuword(&ujob->_aiocb_private.status)); | ||||
} | } | ||||
static long | static long | ||||
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | |||||
* Queue a new AIO request. Choosing either the threaded or direct bio VCHR | * Queue a new AIO request. Choosing either the threaded or direct bio VCHR | ||||
* technique is done in this code. | * technique is done in this code. | ||||
*/ | */ | ||||
int | int | ||||
aio_aqueue(struct thread *td, struct aiocb *ujob, struct aioliojob *lj, | aio_aqueue(struct thread *td, struct aiocb *ujob, struct aioliojob *lj, | ||||
int type, struct aiocb_ops *ops) | int type, struct aiocb_ops *ops) | ||||
{ | { | ||||
struct proc *p = td->td_proc; | struct proc *p = td->td_proc; | ||||
struct file *fp; | struct file *fp = NULL; | ||||
struct kaiocb *job; | struct kaiocb *job; | ||||
struct kaioinfo *ki; | struct kaioinfo *ki; | ||||
struct kevent kev; | struct kevent kev; | ||||
int opcode; | int opcode; | ||||
int error; | int error; | ||||
int fd, kqfd; | int fd, kqfd; | ||||
int jid; | int jid; | ||||
u_short evflags; | u_short evflags; | ||||
if (p->p_aioinfo == NULL) | if (p->p_aioinfo == NULL) | ||||
aio_init_aioinfo(p); | aio_init_aioinfo(p); | ||||
ki = p->p_aioinfo; | ki = p->p_aioinfo; | ||||
ops->store_status(ujob, -1); | ops->store_status(ujob, -1); | ||||
ops->store_error(ujob, 0); | ops->store_error(ujob, 0); | ||||
ops->store_kernelinfo(ujob, -1); | ops->store_kernelinfo(ujob, -1); | ||||
if (num_queue_count >= max_queue_count || | if (num_queue_count >= max_queue_count || | ||||
ki->kaio_count >= max_aio_queue_per_proc) { | ki->kaio_count >= max_aio_queue_per_proc) { | ||||
ops->store_error(ujob, EAGAIN); | error = EAGAIN; | ||||
return (EAGAIN); | goto err1; | ||||
} | } | ||||
job = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO); | job = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO); | ||||
knlist_init_mtx(&job->klist, AIO_MTX(ki)); | knlist_init_mtx(&job->klist, AIO_MTX(ki)); | ||||
error = ops->aio_copyin(ujob, &job->uaiocb); | error = ops->aio_copyin(ujob, job, type); | ||||
if (error) { | if (error) | ||||
ops->store_error(ujob, error); | goto err2; | ||||
uma_zfree(aiocb_zone, job); | |||||
return (error); | |||||
} | |||||
if (job->uaiocb.aio_nbytes > IOSIZE_MAX) { | if (job->uaiocb.aio_nbytes > IOSIZE_MAX) { | ||||
uma_zfree(aiocb_zone, job); | error = EINVAL; | ||||
return (EINVAL); | goto err2; | ||||
} | } | ||||
if (job->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT && | if (job->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT && | ||||
job->uaiocb.aio_sigevent.sigev_notify != SIGEV_SIGNAL && | job->uaiocb.aio_sigevent.sigev_notify != SIGEV_SIGNAL && | ||||
job->uaiocb.aio_sigevent.sigev_notify != SIGEV_THREAD_ID && | job->uaiocb.aio_sigevent.sigev_notify != SIGEV_THREAD_ID && | ||||
job->uaiocb.aio_sigevent.sigev_notify != SIGEV_NONE) { | job->uaiocb.aio_sigevent.sigev_notify != SIGEV_NONE) { | ||||
ops->store_error(ujob, EINVAL); | error = EINVAL; | ||||
uma_zfree(aiocb_zone, job); | goto err2; | ||||
return (EINVAL); | |||||
} | } | ||||
if ((job->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL || | if ((job->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL || | ||||
job->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) && | job->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) && | ||||
!_SIG_VALID(job->uaiocb.aio_sigevent.sigev_signo)) { | !_SIG_VALID(job->uaiocb.aio_sigevent.sigev_signo)) { | ||||
uma_zfree(aiocb_zone, job); | error = EINVAL; | ||||
return (EINVAL); | goto err2; | ||||
} | } | ||||
ksiginfo_init(&job->ksi); | ksiginfo_init(&job->ksi); | ||||
/* Save userspace address of the job info. */ | /* Save userspace address of the job info. */ | ||||
job->ujob = ujob; | job->ujob = ujob; | ||||
Done Inline ActionsThe blank line here in the old code is a style feature. :) jhb: The blank line here in the old code is a style feature. :) | |||||
/* Get the opcode. */ | /* Get the opcode. */ | ||||
if (type != LIO_NOP) | if (type != LIO_NOP) | ||||
job->uaiocb.aio_lio_opcode = type; | job->uaiocb.aio_lio_opcode = type; | ||||
opcode = job->uaiocb.aio_lio_opcode; | opcode = job->uaiocb.aio_lio_opcode; | ||||
/* | /* | ||||
* Validate the opcode and fetch the file object for the specified | * Validate the opcode and fetch the file object for the specified | ||||
* file descriptor. | * file descriptor. | ||||
* | * | ||||
* XXXRW: Moved the opcode validation up here so that we don't | * XXXRW: Moved the opcode validation up here so that we don't | ||||
* retrieve a file descriptor without knowing what the capabiltity | * retrieve a file descriptor without knowing what the capabiltity | ||||
* should be. | * should be. | ||||
*/ | */ | ||||
fd = job->uaiocb.aio_fildes; | fd = job->uaiocb.aio_fildes; | ||||
switch (opcode) { | switch (opcode) { | ||||
case LIO_WRITE: | case LIO_WRITE: | ||||
case LIO_WRITEV: | |||||
error = fget_write(td, fd, &cap_pwrite_rights, &fp); | error = fget_write(td, fd, &cap_pwrite_rights, &fp); | ||||
break; | break; | ||||
case LIO_READ: | case LIO_READ: | ||||
case LIO_READV: | |||||
error = fget_read(td, fd, &cap_pread_rights, &fp); | error = fget_read(td, fd, &cap_pread_rights, &fp); | ||||
break; | break; | ||||
case LIO_SYNC: | case LIO_SYNC: | ||||
error = fget(td, fd, &cap_fsync_rights, &fp); | error = fget(td, fd, &cap_fsync_rights, &fp); | ||||
break; | break; | ||||
case LIO_MLOCK: | case LIO_MLOCK: | ||||
fp = NULL; | |||||
break; | break; | ||||
case LIO_NOP: | case LIO_NOP: | ||||
error = fget(td, fd, &cap_no_rights, &fp); | error = fget(td, fd, &cap_no_rights, &fp); | ||||
break; | break; | ||||
default: | default: | ||||
error = EINVAL; | error = EINVAL; | ||||
} | } | ||||
if (error) { | if (error) | ||||
uma_zfree(aiocb_zone, job); | goto err3; | ||||
ops->store_error(ujob, error); | |||||
return (error); | |||||
} | |||||
if (opcode == LIO_SYNC && fp->f_vnode == NULL) { | if (opcode == LIO_SYNC && fp->f_vnode == NULL) { | ||||
error = EINVAL; | error = EINVAL; | ||||
goto aqueue_fail; | goto err3; | ||||
} | } | ||||
if ((opcode == LIO_READ || opcode == LIO_WRITE) && | if ((opcode == LIO_READ || opcode == LIO_READV || | ||||
opcode == LIO_WRITE || opcode == LIO_WRITEV) && | |||||
job->uaiocb.aio_offset < 0 && | job->uaiocb.aio_offset < 0 && | ||||
(fp->f_vnode == NULL || fp->f_vnode->v_type != VCHR)) { | (fp->f_vnode == NULL || fp->f_vnode->v_type != VCHR)) { | ||||
error = EINVAL; | error = EINVAL; | ||||
goto aqueue_fail; | goto err3; | ||||
} | } | ||||
job->fd_file = fp; | job->fd_file = fp; | ||||
mtx_lock(&aio_job_mtx); | mtx_lock(&aio_job_mtx); | ||||
jid = jobrefid++; | jid = jobrefid++; | ||||
job->seqno = jobseqno++; | job->seqno = jobseqno++; | ||||
mtx_unlock(&aio_job_mtx); | mtx_unlock(&aio_job_mtx); | ||||
error = ops->store_kernelinfo(ujob, jid); | error = ops->store_kernelinfo(ujob, jid); | ||||
if (error) { | if (error) { | ||||
error = EINVAL; | error = EINVAL; | ||||
goto aqueue_fail; | goto err3; | ||||
} | } | ||||
job->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jid; | job->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jid; | ||||
if (opcode == LIO_NOP) { | if (opcode == LIO_NOP) { | ||||
fdrop(fp, td); | fdrop(fp, td); | ||||
MPASS(job->uiop == &job->uio || job->uiop == NULL); | |||||
uma_zfree(aiocb_zone, job); | uma_zfree(aiocb_zone, job); | ||||
return (0); | return (0); | ||||
} | } | ||||
if (job->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT) | if (job->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT) | ||||
goto no_kqueue; | goto no_kqueue; | ||||
evflags = job->uaiocb.aio_sigevent.sigev_notify_kevent_flags; | evflags = job->uaiocb.aio_sigevent.sigev_notify_kevent_flags; | ||||
if ((evflags & ~(EV_CLEAR | EV_DISPATCH | EV_ONESHOT)) != 0) { | if ((evflags & ~(EV_CLEAR | EV_DISPATCH | EV_ONESHOT)) != 0) { | ||||
error = EINVAL; | error = EINVAL; | ||||
goto aqueue_fail; | goto err3; | ||||
} | } | ||||
kqfd = job->uaiocb.aio_sigevent.sigev_notify_kqueue; | kqfd = job->uaiocb.aio_sigevent.sigev_notify_kqueue; | ||||
memset(&kev, 0, sizeof(kev)); | memset(&kev, 0, sizeof(kev)); | ||||
kev.ident = (uintptr_t)job->ujob; | kev.ident = (uintptr_t)job->ujob; | ||||
kev.filter = EVFILT_AIO; | kev.filter = EVFILT_AIO; | ||||
kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1 | evflags; | kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1 | evflags; | ||||
kev.data = (intptr_t)job; | kev.data = (intptr_t)job; | ||||
kev.udata = job->uaiocb.aio_sigevent.sigev_value.sival_ptr; | kev.udata = job->uaiocb.aio_sigevent.sigev_value.sival_ptr; | ||||
error = kqfd_register(kqfd, &kev, td, M_WAITOK); | error = kqfd_register(kqfd, &kev, td, M_WAITOK); | ||||
if (error) | if (error) | ||||
goto aqueue_fail; | goto err3; | ||||
no_kqueue: | no_kqueue: | ||||
ops->store_error(ujob, EINPROGRESS); | ops->store_error(ujob, EINPROGRESS); | ||||
job->uaiocb._aiocb_private.error = EINPROGRESS; | job->uaiocb._aiocb_private.error = EINPROGRESS; | ||||
job->userproc = p; | job->userproc = p; | ||||
job->cred = crhold(td->td_ucred); | job->cred = crhold(td->td_ucred); | ||||
job->jobflags = KAIOCB_QUEUEING; | job->jobflags = KAIOCB_QUEUEING; | ||||
job->lio = lj; | job->lio = lj; | ||||
switch (opcode) { | |||||
case LIO_READV: | |||||
case LIO_WRITEV: | |||||
/* Use the uio copied in by aio_copyin */ | |||||
MPASS(job->uiop != &job->uio && job->uiop != NULL); | |||||
Done Inline ActionsThis needs to be done in the syscalls themselves as different ABIs need to populate the uio. For example, the freebsd32 versions need to use freebsd32_copyinuio rather than copyinuio. Either that or you have to make a new ops->copyinuio callback that matches the copyinuio signature. It can use copyinuio for the native ABI and freebsd32_copyinuio for freebsd32. jhb: This needs to be done in the syscalls themselves as different ABIs need to populate the uio. | |||||
Done Inline ActionsGood catch! asomers: Good catch! | |||||
break; | |||||
case LIO_READ: | |||||
case LIO_WRITE: | |||||
/* Setup the inline uio */ | |||||
job->iov[0].iov_base = (void *)(uintptr_t)job->uaiocb.aio_buf; | |||||
job->iov[0].iov_len = job->uaiocb.aio_nbytes; | |||||
job->uio.uio_iov = job->iov; | |||||
job->uio.uio_iovcnt = 1; | |||||
job->uio.uio_resid = job->uaiocb.aio_nbytes; | |||||
job->uio.uio_segflg = UIO_USERSPACE; | |||||
/* FALLTHROUGH */ | |||||
default: | |||||
job->uiop = &job->uio; | |||||
break; | |||||
} | |||||
switch (opcode) { | |||||
case LIO_READ: | |||||
case LIO_READV: | |||||
job->uiop->uio_rw = UIO_READ; | |||||
break; | |||||
case LIO_WRITE: | |||||
case LIO_WRITEV: | |||||
job->uiop->uio_rw = UIO_WRITE; | |||||
break; | |||||
} | |||||
job->uiop->uio_offset = job->uaiocb.aio_offset; | |||||
job->uiop->uio_td = td; | |||||
if (opcode == LIO_MLOCK) { | if (opcode == LIO_MLOCK) { | ||||
aio_schedule(job, aio_process_mlock); | aio_schedule(job, aio_process_mlock); | ||||
error = 0; | error = 0; | ||||
} else if (fp->f_ops->fo_aio_queue == NULL) | } else if (fp->f_ops->fo_aio_queue == NULL) | ||||
error = aio_queue_file(fp, job); | error = aio_queue_file(fp, job); | ||||
else | else | ||||
error = fo_aio_queue(fp, job); | error = fo_aio_queue(fp, job); | ||||
if (error) | if (error) | ||||
goto aqueue_fail; | goto err3; | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
job->jobflags &= ~KAIOCB_QUEUEING; | job->jobflags &= ~KAIOCB_QUEUEING; | ||||
TAILQ_INSERT_TAIL(&ki->kaio_all, job, allist); | TAILQ_INSERT_TAIL(&ki->kaio_all, job, allist); | ||||
ki->kaio_count++; | ki->kaio_count++; | ||||
if (lj) | if (lj) | ||||
lj->lioj_count++; | lj->lioj_count++; | ||||
atomic_add_int(&num_queue_count, 1); | atomic_add_int(&num_queue_count, 1); | ||||
if (job->jobflags & KAIOCB_FINISHED) { | if (job->jobflags & KAIOCB_FINISHED) { | ||||
/* | /* | ||||
* The queue callback completed the request synchronously. | * The queue callback completed the request synchronously. | ||||
* The bulk of the completion is deferred in that case | * The bulk of the completion is deferred in that case | ||||
* until this point. | * until this point. | ||||
*/ | */ | ||||
aio_bio_done_notify(p, job); | aio_bio_done_notify(p, job); | ||||
} else | } else | ||||
TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, job, plist); | TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, job, plist); | ||||
AIO_UNLOCK(ki); | AIO_UNLOCK(ki); | ||||
return (0); | return (0); | ||||
aqueue_fail: | err3: | ||||
knlist_delete(&job->klist, curthread, 0); | |||||
if (fp) | if (fp) | ||||
fdrop(fp, td); | fdrop(fp, td); | ||||
knlist_delete(&job->klist, curthread, 0); | |||||
err2: | |||||
if (job->uiop != &job->uio) | |||||
free(job->uiop, M_IOV); | |||||
uma_zfree(aiocb_zone, job); | uma_zfree(aiocb_zone, job); | ||||
err1: | |||||
ops->store_error(ujob, error); | ops->store_error(ujob, error); | ||||
return (error); | return (error); | ||||
} | } | ||||
static void | static void | ||||
aio_cancel_daemon_job(struct kaiocb *job) | aio_cancel_daemon_job(struct kaiocb *job) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines | aio_queue_file(struct file *fp, struct kaiocb *job) | ||||
if (!(safe || enable_aio_unsafe)) { | if (!(safe || enable_aio_unsafe)) { | ||||
counted_warning(&unsafe_warningcnt, | counted_warning(&unsafe_warningcnt, | ||||
"is attempting to use unsafe AIO requests"); | "is attempting to use unsafe AIO requests"); | ||||
return (EOPNOTSUPP); | return (EOPNOTSUPP); | ||||
} | } | ||||
switch (job->uaiocb.aio_lio_opcode) { | switch (job->uaiocb.aio_lio_opcode) { | ||||
case LIO_READ: | case LIO_READ: | ||||
case LIO_READV: | |||||
case LIO_WRITE: | case LIO_WRITE: | ||||
case LIO_WRITEV: | |||||
aio_schedule(job, aio_process_rw); | aio_schedule(job, aio_process_rw); | ||||
error = 0; | error = 0; | ||||
break; | break; | ||||
case LIO_SYNC: | case LIO_SYNC: | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
TAILQ_FOREACH(job2, &ki->kaio_jobqueue, plist) { | TAILQ_FOREACH(job2, &ki->kaio_jobqueue, plist) { | ||||
if (job2->fd_file == job->fd_file && | if (job2->fd_file == job->fd_file && | ||||
job2->uaiocb.aio_lio_opcode != LIO_SYNC && | job2->uaiocb.aio_lio_opcode != LIO_SYNC && | ||||
▲ Show 20 Lines • Show All 357 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
sys_aio_read(struct thread *td, struct aio_read_args *uap) | sys_aio_read(struct thread *td, struct aio_read_args *uap) | ||||
{ | { | ||||
return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READ, &aiocb_ops)); | return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READ, &aiocb_ops)); | ||||
} | } | ||||
int | |||||
sys_aio_readv(struct thread *td, struct aio_readv_args *uap) | |||||
{ | |||||
return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READV, &aiocb_ops)); | |||||
} | |||||
/* syscall - asynchronous write to a file (REALTIME) */ | /* syscall - asynchronous write to a file (REALTIME) */ | ||||
#ifdef COMPAT_FREEBSD6 | #ifdef COMPAT_FREEBSD6 | ||||
int | int | ||||
freebsd6_aio_write(struct thread *td, struct freebsd6_aio_write_args *uap) | freebsd6_aio_write(struct thread *td, struct freebsd6_aio_write_args *uap) | ||||
{ | { | ||||
return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE, | return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE, | ||||
&aiocb_ops_osigevent)); | &aiocb_ops_osigevent)); | ||||
} | } | ||||
#endif | #endif | ||||
int | int | ||||
sys_aio_write(struct thread *td, struct aio_write_args *uap) | sys_aio_write(struct thread *td, struct aio_write_args *uap) | ||||
{ | { | ||||
return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops)); | return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops)); | ||||
} | } | ||||
int | int | ||||
sys_aio_writev(struct thread *td, struct aio_writev_args *uap) | |||||
{ | |||||
return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITEV, &aiocb_ops)); | |||||
} | |||||
int | |||||
sys_aio_mlock(struct thread *td, struct aio_mlock_args *uap) | sys_aio_mlock(struct thread *td, struct aio_mlock_args *uap) | ||||
{ | { | ||||
return (aio_aqueue(td, uap->aiocbp, NULL, LIO_MLOCK, &aiocb_ops)); | return (aio_aqueue(td, uap->aiocbp, NULL, LIO_MLOCK, &aiocb_ops)); | ||||
} | } | ||||
static int | static int | ||||
kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list, | kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list, | ||||
▲ Show 20 Lines • Show All 205 Lines • ▼ Show 20 Lines | sys_lio_listio(struct thread *td, struct lio_listio_args *uap) | ||||
if (error == 0) | if (error == 0) | ||||
error = kern_lio_listio(td, uap->mode, uap->acb_list, acb_list, | error = kern_lio_listio(td, uap->mode, uap->acb_list, acb_list, | ||||
nent, sigp, &aiocb_ops); | nent, sigp, &aiocb_ops); | ||||
free(acb_list, M_LIO); | free(acb_list, M_LIO); | ||||
return (error); | return (error); | ||||
} | } | ||||
static void | static void | ||||
aio_biowakeup(struct bio *bp) | aio_biocleanup(struct bio *bp) | ||||
{ | { | ||||
struct kaiocb *job = (struct kaiocb *)bp->bio_caller1; | struct kaiocb *job = (struct kaiocb *)bp->bio_caller1; | ||||
struct kaioinfo *ki; | struct kaioinfo *ki; | ||||
struct buf *pbuf = (struct buf*)bp->bio_caller2; | struct buf *pbuf = (struct buf *)bp->bio_caller2; | ||||
Done Inline ActionsSpace before * in the cast type. kib: Space before * in the cast type. | |||||
size_t nbytes; | |||||
int error, nblks; | |||||
/* Release mapping into kernel space. */ | /* Release mapping into kernel space. */ | ||||
if (pbuf != NULL) { | if (pbuf != NULL) { | ||||
MPASS(pbuf->b_npages <= atop(maxphys) + 1); | MPASS(pbuf->b_npages <= atop(maxphys) + 1); | ||||
pmap_qremove((vm_offset_t)pbuf->b_data, pbuf->b_npages); | pmap_qremove((vm_offset_t)pbuf->b_data, pbuf->b_npages); | ||||
vm_page_unhold_pages(pbuf->b_pages, pbuf->b_npages); | vm_page_unhold_pages(pbuf->b_pages, pbuf->b_npages); | ||||
uma_zfree(pbuf_zone, pbuf); | uma_zfree(pbuf_zone, pbuf); | ||||
atomic_subtract_int(&num_buf_aio, 1); | atomic_subtract_int(&num_buf_aio, 1); | ||||
ki = job->userproc->p_aioinfo; | ki = job->userproc->p_aioinfo; | ||||
AIO_LOCK(ki); | AIO_LOCK(ki); | ||||
ki->kaio_buffer_count--; | ki->kaio_buffer_count--; | ||||
AIO_UNLOCK(ki); | AIO_UNLOCK(ki); | ||||
} else { | } else { | ||||
MPASS(bp->bio_ma_n <= atop(maxphys) + 1); | MPASS(bp->bio_ma_n <= atop(maxphys) + 1); | ||||
vm_page_unhold_pages(bp->bio_ma, bp->bio_ma_n); | vm_page_unhold_pages(bp->bio_ma, bp->bio_ma_n); | ||||
free(bp->bio_ma, M_TEMP); | free(bp->bio_ma, M_TEMP); | ||||
atomic_subtract_int(&num_unmapped_aio, 1); | atomic_subtract_int(&num_unmapped_aio, 1); | ||||
} | } | ||||
g_destroy_bio(bp); | |||||
} | |||||
nbytes = job->uaiocb.aio_nbytes - bp->bio_resid; | static void | ||||
error = 0; | aio_biowakeup(struct bio *bp) | ||||
if (bp->bio_flags & BIO_ERROR) | { | ||||
error = bp->bio_error; | struct kaiocb *job = (struct kaiocb *)bp->bio_caller1; | ||||
size_t nbytes; | |||||
long bcount = bp->bio_bcount; | |||||
long resid = bp->bio_resid; | |||||
int error, opcode, nblks; | |||||
int bio_error = bp->bio_error; | |||||
uint16_t flags = bp->bio_flags; | |||||
opcode = job->uaiocb.aio_lio_opcode; | |||||
aio_biocleanup(bp); | |||||
nbytes =bcount - resid; | |||||
atomic_add_acq_long(&job->nbytes, nbytes); | |||||
nblks = btodb(nbytes); | nblks = btodb(nbytes); | ||||
if (job->uaiocb.aio_lio_opcode == LIO_WRITE) | error = 0; | ||||
job->outblock += nblks; | /* | ||||
* If multiple bios experienced an error, the job will reflect the | |||||
* error of whichever failed bio completed last. | |||||
*/ | |||||
if (flags & BIO_ERROR) | |||||
atomic_set_int(&job->error, bio_error); | |||||
if (opcode == LIO_WRITE || opcode == LIO_WRITEV) | |||||
atomic_add_int(&job->outblock, nblks); | |||||
else | else | ||||
job->inblock += nblks; | atomic_add_int(&job->inblock, nblks); | ||||
atomic_subtract_int(&job->nbio, 1); | |||||
if (error) | |||||
aio_complete(job, -1, error); | |||||
else | |||||
aio_complete(job, nbytes, 0); | |||||
g_destroy_bio(bp); | if (atomic_load_int(&job->nbio) == 0) { | ||||
if (atomic_load_int(&job->error)) | |||||
aio_complete(job, -1, job->error); | |||||
else | |||||
aio_complete(job, atomic_load_long(&job->nbytes), 0); | |||||
} | } | ||||
} | |||||
/* syscall - wait for the next completion of an aio request */ | /* syscall - wait for the next completion of an aio request */ | ||||
static int | static int | ||||
kern_aio_waitcomplete(struct thread *td, struct aiocb **ujobp, | kern_aio_waitcomplete(struct thread *td, struct aiocb **ujobp, | ||||
struct timespec *ts, struct aiocb_ops *ops) | struct timespec *ts, struct aiocb_ops *ops) | ||||
{ | { | ||||
struct proc *p = td->td_proc; | struct proc *p = td->td_proc; | ||||
struct timeval atv; | struct timeval atv; | ||||
▲ Show 20 Lines • Show All 218 Lines • ▼ Show 20 Lines | typedef struct oaiocb32 { | ||||
int aio_reqprio; /* Request priority -- ignored */ | int aio_reqprio; /* Request priority -- ignored */ | ||||
struct __aiocb_private32 _aiocb_private; | struct __aiocb_private32 _aiocb_private; | ||||
} oaiocb32_t; | } oaiocb32_t; | ||||
#endif | #endif | ||||
typedef struct aiocb32 { | typedef struct aiocb32 { | ||||
int32_t aio_fildes; /* File descriptor */ | int32_t aio_fildes; /* File descriptor */ | ||||
uint64_t aio_offset __packed; /* File offset for I/O */ | uint64_t aio_offset __packed; /* File offset for I/O */ | ||||
union { | |||||
uint32_t aio_buf; /* I/O buffer in process space */ | uint32_t aio_buf; /* I/O buffer in process space */ | ||||
uint32_t aio_iov; /* I/O scatter/gather list */ | |||||
}; | |||||
union { | |||||
uint32_t aio_nbytes; /* Number of bytes for I/O */ | uint32_t aio_nbytes; /* Number of bytes for I/O */ | ||||
int32_t aio_iovcnt; /* Length of aio_iov */ | |||||
}; | |||||
int __spare__[2]; | int __spare__[2]; | ||||
uint32_t __spare2__; | uint32_t __spare2__; | ||||
int aio_lio_opcode; /* LIO opcode */ | int aio_lio_opcode; /* LIO opcode */ | ||||
int aio_reqprio; /* Request priority -- ignored */ | int aio_reqprio; /* Request priority -- ignored */ | ||||
struct __aiocb_private32 _aiocb_private; | struct __aiocb_private32 _aiocb_private; | ||||
struct sigevent32 aio_sigevent; /* Signal to deliver */ | struct sigevent32 aio_sigevent; /* Signal to deliver */ | ||||
} aiocb32_t; | } aiocb32_t; | ||||
Show All 20 Lines | case SIGEV_KEVENT: | ||||
break; | break; | ||||
default: | default: | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
aiocb32_copyin_old_sigevent(struct aiocb *ujob, struct aiocb *kjob) | aiocb32_copyin_old_sigevent(struct aiocb *ujob, struct kaiocb *kjob, | ||||
int type __unused) | |||||
{ | { | ||||
struct oaiocb32 job32; | struct oaiocb32 job32; | ||||
struct aiocb *kcb = &kjob->uaiocb; | |||||
int error; | int error; | ||||
bzero(kjob, sizeof(struct aiocb)); | bzero(kcb, sizeof(struct aiocb)); | ||||
error = copyin(ujob, &job32, sizeof(job32)); | error = copyin(ujob, &job32, sizeof(job32)); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
CP(job32, *kjob, aio_fildes); | /* No need to copyin aio_iov, because it did not exist in FreeBSD 6 */ | ||||
CP(job32, *kjob, aio_offset); | |||||
PTRIN_CP(job32, *kjob, aio_buf); | CP(job32, *kcb, aio_fildes); | ||||
CP(job32, *kjob, aio_nbytes); | CP(job32, *kcb, aio_offset); | ||||
CP(job32, *kjob, aio_lio_opcode); | PTRIN_CP(job32, *kcb, aio_buf); | ||||
CP(job32, *kjob, aio_reqprio); | CP(job32, *kcb, aio_nbytes); | ||||
CP(job32, *kjob, _aiocb_private.status); | CP(job32, *kcb, aio_lio_opcode); | ||||
CP(job32, *kjob, _aiocb_private.error); | CP(job32, *kcb, aio_reqprio); | ||||
PTRIN_CP(job32, *kjob, _aiocb_private.kernelinfo); | CP(job32, *kcb, _aiocb_private.status); | ||||
CP(job32, *kcb, _aiocb_private.error); | |||||
PTRIN_CP(job32, *kcb, _aiocb_private.kernelinfo); | |||||
Done Inline Actionsstyle nit: blank line before this comment. jhb: style nit: blank line before this comment. | |||||
return (convert_old_sigevent32(&job32.aio_sigevent, | return (convert_old_sigevent32(&job32.aio_sigevent, | ||||
&kjob->aio_sigevent)); | &kcb->aio_sigevent)); | ||||
} | } | ||||
#endif | #endif | ||||
static int | static int | ||||
aiocb32_copyin(struct aiocb *ujob, struct aiocb *kjob) | aiocb32_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type) | ||||
{ | { | ||||
struct aiocb32 job32; | struct aiocb32 job32; | ||||
struct aiocb *kcb = &kjob->uaiocb; | |||||
struct iovec32 *iov32; | |||||
int error; | int error; | ||||
error = copyin(ujob, &job32, sizeof(job32)); | error = copyin(ujob, &job32, sizeof(job32)); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
CP(job32, *kjob, aio_fildes); | CP(job32, *kcb, aio_fildes); | ||||
CP(job32, *kjob, aio_offset); | CP(job32, *kcb, aio_offset); | ||||
PTRIN_CP(job32, *kjob, aio_buf); | CP(job32, *kcb, aio_lio_opcode); | ||||
CP(job32, *kjob, aio_nbytes); | if (type == LIO_READV || type == LIO_WRITEV) { | ||||
CP(job32, *kjob, aio_lio_opcode); | iov32 = PTRIN(job32.aio_iov); | ||||
CP(job32, *kjob, aio_reqprio); | CP(job32, *kcb, aio_iovcnt); | ||||
CP(job32, *kjob, _aiocb_private.status); | /* malloc a uio and copy in the iovec */ | ||||
CP(job32, *kjob, _aiocb_private.error); | error = freebsd32_copyinuio(iov32, | ||||
PTRIN_CP(job32, *kjob, _aiocb_private.kernelinfo); | kcb->aio_iovcnt, &kjob->uiop); | ||||
return (convert_sigevent32(&job32.aio_sigevent, &kjob->aio_sigevent)); | if (error) | ||||
return (error); | |||||
} else { | |||||
PTRIN_CP(job32, *kcb, aio_buf); | |||||
CP(job32, *kcb, aio_nbytes); | |||||
} | } | ||||
CP(job32, *kcb, aio_reqprio); | |||||
CP(job32, *kcb, _aiocb_private.status); | |||||
CP(job32, *kcb, _aiocb_private.error); | |||||
PTRIN_CP(job32, *kcb, _aiocb_private.kernelinfo); | |||||
error = convert_sigevent32(&job32.aio_sigevent, &kcb->aio_sigevent); | |||||
return (error); | |||||
} | |||||
static long | static long | ||||
aiocb32_fetch_status(struct aiocb *ujob) | aiocb32_fetch_status(struct aiocb *ujob) | ||||
{ | { | ||||
struct aiocb32 *ujob32; | struct aiocb32 *ujob32; | ||||
ujob32 = (struct aiocb32 *)ujob; | ujob32 = (struct aiocb32 *)ujob; | ||||
return (fuword32(&ujob32->_aiocb_private.status)); | return (fuword32(&ujob32->_aiocb_private.status)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
freebsd32_aio_read(struct thread *td, struct freebsd32_aio_read_args *uap) | freebsd32_aio_read(struct thread *td, struct freebsd32_aio_read_args *uap) | ||||
{ | { | ||||
return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ, | return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ, | ||||
&aiocb32_ops)); | &aiocb32_ops)); | ||||
} | } | ||||
int | |||||
freebsd32_aio_readv(struct thread *td, struct freebsd32_aio_readv_args *uap) | |||||
{ | |||||
return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READV, | |||||
&aiocb32_ops)); | |||||
} | |||||
#ifdef COMPAT_FREEBSD6 | #ifdef COMPAT_FREEBSD6 | ||||
int | int | ||||
freebsd6_freebsd32_aio_write(struct thread *td, | freebsd6_freebsd32_aio_write(struct thread *td, | ||||
struct freebsd6_freebsd32_aio_write_args *uap) | struct freebsd6_freebsd32_aio_write_args *uap) | ||||
{ | { | ||||
return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE, | return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE, | ||||
&aiocb32_ops_osigevent)); | &aiocb32_ops_osigevent)); | ||||
} | } | ||||
#endif | #endif | ||||
int | int | ||||
freebsd32_aio_write(struct thread *td, struct freebsd32_aio_write_args *uap) | freebsd32_aio_write(struct thread *td, struct freebsd32_aio_write_args *uap) | ||||
{ | { | ||||
return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE, | return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE, | ||||
&aiocb32_ops)); | |||||
} | |||||
int | |||||
freebsd32_aio_writev(struct thread *td, struct freebsd32_aio_writev_args *uap) | |||||
{ | |||||
return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITEV, | |||||
&aiocb32_ops)); | &aiocb32_ops)); | ||||
} | } | ||||
int | int | ||||
freebsd32_aio_mlock(struct thread *td, struct freebsd32_aio_mlock_args *uap) | freebsd32_aio_mlock(struct thread *td, struct freebsd32_aio_mlock_args *uap) | ||||
{ | { | ||||
return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_MLOCK, | return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_MLOCK, | ||||
▲ Show 20 Lines • Show All 128 Lines • Show Last 20 Lines |
Extra ()