Changeset View
Changeset View
Standalone View
Standalone View
head/sys/kern/vfs_aio.c
Show First 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
static uint64_t jobseqno; | static uint64_t jobseqno; | ||||
#ifndef MAX_AIO_PER_PROC | #ifndef MAX_AIO_PER_PROC | ||||
#define MAX_AIO_PER_PROC 32 | #define MAX_AIO_PER_PROC 32 | ||||
#endif | #endif | ||||
#ifndef MAX_AIO_QUEUE_PER_PROC | #ifndef MAX_AIO_QUEUE_PER_PROC | ||||
#define MAX_AIO_QUEUE_PER_PROC 256 /* Bigger than AIO_LISTIO_MAX */ | #define MAX_AIO_QUEUE_PER_PROC 256 | ||||
#endif | #endif | ||||
#ifndef MAX_AIO_QUEUE | #ifndef MAX_AIO_QUEUE | ||||
#define MAX_AIO_QUEUE 1024 /* Bigger than AIO_LISTIO_MAX */ | #define MAX_AIO_QUEUE 1024 /* Bigger than MAX_AIO_QUEUE_PER_PROC */ | ||||
#endif | #endif | ||||
#ifndef MAX_BUF_AIO | #ifndef MAX_BUF_AIO | ||||
#define MAX_BUF_AIO 16 | #define MAX_BUF_AIO 16 | ||||
#endif | #endif | ||||
FEATURE(aio, "Asynchronous I/O"); | FEATURE(aio, "Asynchronous I/O"); | ||||
SYSCTL_DECL(_p1003_1b); | SYSCTL_DECL(_p1003_1b); | ||||
static MALLOC_DEFINE(M_LIO, "lio", "listio aio control block list"); | static MALLOC_DEFINE(M_LIO, "lio", "listio aio control block list"); | ||||
static MALLOC_DEFINE(M_AIOS, "aios", "aio_suspend aio control block list"); | |||||
static SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, | static SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, | ||||
"Async IO management"); | "Async IO management"); | ||||
static int enable_aio_unsafe = 0; | static int enable_aio_unsafe = 0; | ||||
SYSCTL_INT(_vfs_aio, OID_AUTO, enable_unsafe, CTLFLAG_RW, &enable_aio_unsafe, 0, | SYSCTL_INT(_vfs_aio, OID_AUTO, enable_unsafe, CTLFLAG_RW, &enable_aio_unsafe, 0, | ||||
"Permit asynchronous IO on all file types, not just known-safe types"); | "Permit asynchronous IO on all file types, not just known-safe types"); | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | |||||
SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW, | SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW, | ||||
&max_aio_queue_per_proc, 0, | &max_aio_queue_per_proc, 0, | ||||
"Maximum queued aio requests per process (stored in the process)"); | "Maximum queued aio requests per process (stored in the process)"); | ||||
static int max_buf_aio = MAX_BUF_AIO; | static int max_buf_aio = MAX_BUF_AIO; | ||||
SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0, | SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0, | ||||
"Maximum buf aio requests per process (stored in the process)"); | "Maximum buf aio requests per process (stored in the process)"); | ||||
static int aio_listio_max = AIO_LISTIO_MAX; | /* | ||||
* Though redundant with vfs.aio.max_aio_queue_per_proc, POSIX requires | |||||
* sysconf(3) to support AIO_LISTIO_MAX, and we implement that with | |||||
* vfs.aio.aio_listio_max. | |||||
*/ | |||||
SYSCTL_INT(_p1003_1b, CTL_P1003_1B_AIO_LISTIO_MAX, aio_listio_max, | SYSCTL_INT(_p1003_1b, CTL_P1003_1B_AIO_LISTIO_MAX, aio_listio_max, | ||||
CTLFLAG_RDTUN | CTLFLAG_CAPRD, &aio_listio_max, 0, | CTLFLAG_RD | CTLFLAG_CAPRD, &max_aio_queue_per_proc, | ||||
"Maximum aio requests for a single lio_listio call"); | 0, "Maximum aio requests for a single lio_listio call"); | ||||
#ifdef COMPAT_FREEBSD6 | #ifdef COMPAT_FREEBSD6 | ||||
typedef struct oaiocb { | typedef struct oaiocb { | ||||
int aio_fildes; /* File descriptor */ | int aio_fildes; /* File descriptor */ | ||||
off_t aio_offset; /* File offset for I/O */ | off_t aio_offset; /* File offset for I/O */ | ||||
volatile void *aio_buf; /* I/O buffer in process space */ | volatile void *aio_buf; /* I/O buffer in process space */ | ||||
size_t aio_nbytes; /* Number of bytes for I/O */ | size_t aio_nbytes; /* Number of bytes for I/O */ | ||||
struct osigevent aio_sigevent; /* Signal to deliver */ | struct osigevent aio_sigevent; /* Signal to deliver */ | ||||
▲ Show 20 Lines • Show All 145 Lines • ▼ Show 20 Lines | |||||
static void filt_liodetach(struct knote *kn); | static void filt_liodetach(struct knote *kn); | ||||
static int filt_lio(struct knote *kn, long hint); | static int filt_lio(struct knote *kn, long hint); | ||||
/* | /* | ||||
* Zones for: | * Zones for: | ||||
* kaio Per process async io info | * kaio Per process async io info | ||||
* aiop async io process data | * aiop async io process data | ||||
* aiocb async io jobs | * aiocb async io jobs | ||||
* aiol list io job pointer - internal to aio_suspend XXX | |||||
* aiolio list io jobs | * aiolio list io jobs | ||||
*/ | */ | ||||
static uma_zone_t kaio_zone, aiop_zone, aiocb_zone, aiol_zone, aiolio_zone; | static uma_zone_t kaio_zone, aiop_zone, aiocb_zone, aiolio_zone; | ||||
/* kqueue filters for aio */ | /* kqueue filters for aio */ | ||||
static struct filterops aio_filtops = { | static struct filterops aio_filtops = { | ||||
.f_isfd = 0, | .f_isfd = 0, | ||||
.f_attach = filt_aioattach, | .f_attach = filt_aioattach, | ||||
.f_detach = filt_aiodetach, | .f_detach = filt_aiodetach, | ||||
.f_event = filt_aio, | .f_event = filt_aio, | ||||
}; | }; | ||||
Show All 40 Lines | |||||
/* | /* | ||||
* Startup initialization | * Startup initialization | ||||
*/ | */ | ||||
static int | static int | ||||
aio_onceonly(void) | aio_onceonly(void) | ||||
{ | { | ||||
if (aio_listio_max < AIO_LISTIO_MAX) | |||||
aio_listio_max = AIO_LISTIO_MAX; | |||||
if (aio_listio_max > MIN(MAX_AIO_QUEUE_PER_PROC, max_queue_count)) | |||||
aio_listio_max = MIN(MAX_AIO_QUEUE_PER_PROC, max_queue_count); | |||||
exit_tag = EVENTHANDLER_REGISTER(process_exit, aio_proc_rundown, NULL, | exit_tag = EVENTHANDLER_REGISTER(process_exit, aio_proc_rundown, NULL, | ||||
EVENTHANDLER_PRI_ANY); | EVENTHANDLER_PRI_ANY); | ||||
exec_tag = EVENTHANDLER_REGISTER(process_exec, aio_proc_rundown_exec, | exec_tag = EVENTHANDLER_REGISTER(process_exec, aio_proc_rundown_exec, | ||||
NULL, EVENTHANDLER_PRI_ANY); | NULL, EVENTHANDLER_PRI_ANY); | ||||
kqueue_add_filteropts(EVFILT_AIO, &aio_filtops); | kqueue_add_filteropts(EVFILT_AIO, &aio_filtops); | ||||
kqueue_add_filteropts(EVFILT_LIO, &lio_filtops); | kqueue_add_filteropts(EVFILT_LIO, &lio_filtops); | ||||
TAILQ_INIT(&aio_freeproc); | TAILQ_INIT(&aio_freeproc); | ||||
sema_init(&aio_newproc_sem, 0, "aio_new_proc"); | sema_init(&aio_newproc_sem, 0, "aio_new_proc"); | ||||
mtx_init(&aio_job_mtx, "aio_job", NULL, MTX_DEF); | mtx_init(&aio_job_mtx, "aio_job", NULL, MTX_DEF); | ||||
TAILQ_INIT(&aio_jobs); | TAILQ_INIT(&aio_jobs); | ||||
aiod_unr = new_unrhdr(1, INT_MAX, NULL); | aiod_unr = new_unrhdr(1, INT_MAX, NULL); | ||||
kaio_zone = uma_zcreate("AIO", sizeof(struct kaioinfo), NULL, NULL, | kaio_zone = uma_zcreate("AIO", sizeof(struct kaioinfo), NULL, NULL, | ||||
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | ||||
aiop_zone = uma_zcreate("AIOP", sizeof(struct aioproc), NULL, | aiop_zone = uma_zcreate("AIOP", sizeof(struct aioproc), NULL, | ||||
NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | ||||
aiocb_zone = uma_zcreate("AIOCB", sizeof(struct kaiocb), NULL, NULL, | aiocb_zone = uma_zcreate("AIOCB", sizeof(struct kaiocb), NULL, NULL, | ||||
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | ||||
aiol_zone = uma_zcreate("AIOL", aio_listio_max * sizeof(intptr_t) , | |||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | |||||
aiolio_zone = uma_zcreate("AIOLIO", sizeof(struct aioliojob), NULL, | aiolio_zone = uma_zcreate("AIOLIO", sizeof(struct aioliojob), NULL, | ||||
NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); | ||||
aiod_lifetime = AIOD_LIFETIME_DEFAULT; | aiod_lifetime = AIOD_LIFETIME_DEFAULT; | ||||
jobrefid = 1; | jobrefid = 1; | ||||
p31b_setcfg(CTL_P1003_1B_ASYNCHRONOUS_IO, _POSIX_ASYNCHRONOUS_IO); | p31b_setcfg(CTL_P1003_1B_ASYNCHRONOUS_IO, _POSIX_ASYNCHRONOUS_IO); | ||||
p31b_setcfg(CTL_P1003_1B_AIO_MAX, MAX_AIO_QUEUE); | p31b_setcfg(CTL_P1003_1B_AIO_MAX, MAX_AIO_QUEUE); | ||||
p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, 0); | p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, 0); | ||||
▲ Show 20 Lines • Show All 1,519 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
sys_aio_suspend(struct thread *td, struct aio_suspend_args *uap) | sys_aio_suspend(struct thread *td, struct aio_suspend_args *uap) | ||||
{ | { | ||||
struct timespec ts, *tsp; | struct timespec ts, *tsp; | ||||
struct aiocb **ujoblist; | struct aiocb **ujoblist; | ||||
int error; | int error; | ||||
if (uap->nent < 0 || uap->nent > aio_listio_max) | if (uap->nent < 0 || uap->nent > max_aio_queue_per_proc) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (uap->timeout) { | if (uap->timeout) { | ||||
/* Get timespec struct. */ | /* Get timespec struct. */ | ||||
if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0) | if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0) | ||||
return (error); | return (error); | ||||
tsp = &ts; | tsp = &ts; | ||||
} else | } else | ||||
tsp = NULL; | tsp = NULL; | ||||
ujoblist = uma_zalloc(aiol_zone, M_WAITOK); | ujoblist = malloc(uap->nent * sizeof(ujoblist[0]), M_AIOS, M_WAITOK); | ||||
error = copyin(uap->aiocbp, ujoblist, uap->nent * sizeof(ujoblist[0])); | error = copyin(uap->aiocbp, ujoblist, uap->nent * sizeof(ujoblist[0])); | ||||
if (error == 0) | if (error == 0) | ||||
error = kern_aio_suspend(td, uap->nent, ujoblist, tsp); | error = kern_aio_suspend(td, uap->nent, ujoblist, tsp); | ||||
uma_zfree(aiol_zone, ujoblist); | free(ujoblist, M_AIOS); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* aio_cancel cancels any non-physio aio operations not currently in | * aio_cancel cancels any non-physio aio operations not currently in | ||||
* progress. | * progress. | ||||
*/ | */ | ||||
int | int | ||||
▲ Show 20 Lines • Show All 176 Lines • ▼ Show 20 Lines | kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list, | ||||
struct kevent kev; | struct kevent kev; | ||||
int error; | int error; | ||||
int nerror; | int nerror; | ||||
int i; | int i; | ||||
if ((mode != LIO_NOWAIT) && (mode != LIO_WAIT)) | if ((mode != LIO_NOWAIT) && (mode != LIO_WAIT)) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (nent < 0 || nent > aio_listio_max) | if (nent < 0 || nent > max_aio_queue_per_proc) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (p->p_aioinfo == NULL) | if (p->p_aioinfo == NULL) | ||||
aio_init_aioinfo(p); | aio_init_aioinfo(p); | ||||
ki = p->p_aioinfo; | ki = p->p_aioinfo; | ||||
lj = uma_zalloc(aiolio_zone, M_WAITOK); | lj = uma_zalloc(aiolio_zone, M_WAITOK); | ||||
▲ Show 20 Lines • Show All 115 Lines • ▼ Show 20 Lines | freebsd6_lio_listio(struct thread *td, struct freebsd6_lio_listio_args *uap) | ||||
struct sigevent *sigp, sig; | struct sigevent *sigp, sig; | ||||
struct osigevent osig; | struct osigevent osig; | ||||
int error, nent; | int error, nent; | ||||
if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) | if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) | ||||
return (EINVAL); | return (EINVAL); | ||||
nent = uap->nent; | nent = uap->nent; | ||||
if (nent < 0 || nent > aio_listio_max) | if (nent < 0 || nent > max_aio_queue_per_proc) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (uap->sig && (uap->mode == LIO_NOWAIT)) { | if (uap->sig && (uap->mode == LIO_NOWAIT)) { | ||||
error = copyin(uap->sig, &osig, sizeof(osig)); | error = copyin(uap->sig, &osig, sizeof(osig)); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
error = convert_old_sigevent(&osig, &sig); | error = convert_old_sigevent(&osig, &sig); | ||||
if (error) | if (error) | ||||
Show All 20 Lines | sys_lio_listio(struct thread *td, struct lio_listio_args *uap) | ||||
struct aiocb **acb_list; | struct aiocb **acb_list; | ||||
struct sigevent *sigp, sig; | struct sigevent *sigp, sig; | ||||
int error, nent; | int error, nent; | ||||
if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) | if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) | ||||
return (EINVAL); | return (EINVAL); | ||||
nent = uap->nent; | nent = uap->nent; | ||||
if (nent < 0 || nent > aio_listio_max) | if (nent < 0 || nent > max_aio_queue_per_proc) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (uap->sig && (uap->mode == LIO_NOWAIT)) { | if (uap->sig && (uap->mode == LIO_NOWAIT)) { | ||||
error = copyin(uap->sig, &sig, sizeof(sig)); | error = copyin(uap->sig, &sig, sizeof(sig)); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
sigp = &sig; | sigp = &sig; | ||||
} else | } else | ||||
▲ Show 20 Lines • Show All 452 Lines • ▼ Show 20 Lines | |||||
freebsd32_aio_suspend(struct thread *td, struct freebsd32_aio_suspend_args *uap) | freebsd32_aio_suspend(struct thread *td, struct freebsd32_aio_suspend_args *uap) | ||||
{ | { | ||||
struct timespec32 ts32; | struct timespec32 ts32; | ||||
struct timespec ts, *tsp; | struct timespec ts, *tsp; | ||||
struct aiocb **ujoblist; | struct aiocb **ujoblist; | ||||
uint32_t *ujoblist32; | uint32_t *ujoblist32; | ||||
int error, i; | int error, i; | ||||
if (uap->nent < 0 || uap->nent > aio_listio_max) | if (uap->nent < 0 || uap->nent > max_aio_queue_per_proc) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (uap->timeout) { | if (uap->timeout) { | ||||
/* Get timespec struct. */ | /* Get timespec struct. */ | ||||
if ((error = copyin(uap->timeout, &ts32, sizeof(ts32))) != 0) | if ((error = copyin(uap->timeout, &ts32, sizeof(ts32))) != 0) | ||||
return (error); | return (error); | ||||
CP(ts32, ts, tv_sec); | CP(ts32, ts, tv_sec); | ||||
CP(ts32, ts, tv_nsec); | CP(ts32, ts, tv_nsec); | ||||
tsp = &ts; | tsp = &ts; | ||||
} else | } else | ||||
tsp = NULL; | tsp = NULL; | ||||
ujoblist = uma_zalloc(aiol_zone, M_WAITOK); | ujoblist = malloc(uap->nent * sizeof(ujoblist[0]), M_AIOS, M_WAITOK); | ||||
ujoblist32 = (uint32_t *)ujoblist; | ujoblist32 = (uint32_t *)ujoblist; | ||||
error = copyin(uap->aiocbp, ujoblist32, uap->nent * | error = copyin(uap->aiocbp, ujoblist32, uap->nent * | ||||
sizeof(ujoblist32[0])); | sizeof(ujoblist32[0])); | ||||
if (error == 0) { | if (error == 0) { | ||||
for (i = uap->nent; i > 0; i--) | for (i = uap->nent; i > 0; i--) | ||||
ujoblist[i] = PTRIN(ujoblist32[i]); | ujoblist[i] = PTRIN(ujoblist32[i]); | ||||
error = kern_aio_suspend(td, uap->nent, ujoblist, tsp); | error = kern_aio_suspend(td, uap->nent, ujoblist, tsp); | ||||
} | } | ||||
uma_zfree(aiol_zone, ujoblist); | free(ujoblist, M_AIOS); | ||||
return (error); | return (error); | ||||
} | } | ||||
int | int | ||||
freebsd32_aio_error(struct thread *td, struct freebsd32_aio_error_args *uap) | freebsd32_aio_error(struct thread *td, struct freebsd32_aio_error_args *uap) | ||||
{ | { | ||||
return (kern_aio_error(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops)); | return (kern_aio_error(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops)); | ||||
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | freebsd6_freebsd32_lio_listio(struct thread *td, | ||||
struct osigevent32 osig; | struct osigevent32 osig; | ||||
uint32_t *acb_list32; | uint32_t *acb_list32; | ||||
int error, i, nent; | int error, i, nent; | ||||
if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) | if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) | ||||
return (EINVAL); | return (EINVAL); | ||||
nent = uap->nent; | nent = uap->nent; | ||||
if (nent < 0 || nent > aio_listio_max) | if (nent < 0 || nent > max_aio_queue_per_proc) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (uap->sig && (uap->mode == LIO_NOWAIT)) { | if (uap->sig && (uap->mode == LIO_NOWAIT)) { | ||||
error = copyin(uap->sig, &osig, sizeof(osig)); | error = copyin(uap->sig, &osig, sizeof(osig)); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
error = convert_old_sigevent32(&osig, &sig); | error = convert_old_sigevent32(&osig, &sig); | ||||
if (error) | if (error) | ||||
Show All 29 Lines | freebsd32_lio_listio(struct thread *td, struct freebsd32_lio_listio_args *uap) | ||||
struct sigevent32 sig32; | struct sigevent32 sig32; | ||||
uint32_t *acb_list32; | uint32_t *acb_list32; | ||||
int error, i, nent; | int error, i, nent; | ||||
if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) | if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) | ||||
return (EINVAL); | return (EINVAL); | ||||
nent = uap->nent; | nent = uap->nent; | ||||
if (nent < 0 || nent > aio_listio_max) | if (nent < 0 || nent > max_aio_queue_per_proc) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (uap->sig && (uap->mode == LIO_NOWAIT)) { | if (uap->sig && (uap->mode == LIO_NOWAIT)) { | ||||
error = copyin(uap->sig, &sig32, sizeof(sig32)); | error = copyin(uap->sig, &sig32, sizeof(sig32)); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
error = convert_sigevent32(&sig32, &sig); | error = convert_sigevent32(&sig32, &sig); | ||||
if (error) | if (error) | ||||
Show All 24 Lines |