Changeset View
Standalone View
opencrypto/crypto.c
Show First 20 Lines • Show All 128 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* There are two queues for crypto requests; one for symmetric (e.g. | * There are two queues for crypto requests; one for symmetric (e.g. | ||||
* cipher) operations and one for asymmetric (e.g. MOD)operations. | * cipher) operations and one for asymmetric (e.g. MOD)operations. | ||||
* A single mutex is used to lock access to both queues. We could | * A single mutex is used to lock access to both queues. We could | ||||
* have one per-queue but having one simplifies handling of block/unblock | * have one per-queue but having one simplifies handling of block/unblock | ||||
* operations. | * operations. | ||||
*/ | */ | ||||
static int crp_sleep = 0; | struct crypto_worker { | ||||
static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ | TAILQ_HEAD(,cryptop) crp_q; /* request queues */ | ||||
static TAILQ_HEAD(,cryptkop) crp_kq; | TAILQ_HEAD(,cryptkop) crp_kq; | ||||
static struct mtx crypto_q_mtx; | |||||
#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) | |||||
#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) | |||||
struct mtx crypto_q_mtx; | |||||
int crp_sleep; | |||||
struct proc *cryptoproc; | |||||
} __aligned(CACHE_LINE_SIZE); | |||||
static struct crypto_worker *crypto_workers = NULL; | |||||
#define CRYPTO_W(i) (&crypto_workers[i]) | |||||
#define CRYPTO_W_ID(w) ((int)(((uintptr_t)w - (uintptr_t)crypto_workers) / sizeof(struct crypto_worker))) | |||||
#define CRYPTO_W_FOREACH(w) \ | |||||
for (w = crypto_workers; w < crypto_workers + crypto_workers_num; ++w) | |||||
#define CRYPTO_W_LOCK(w) mtx_lock(&(w)->crypto_q_mtx) | |||||
#define CRYPTO_W_UNLOCK(w) mtx_unlock(&(w)->crypto_q_mtx) | |||||
#define CRYPTO_W_LOCKALL() do { \ | |||||
struct crypto_worker *_w; \ | |||||
CRYPTO_W_FOREACH(_w) \ | |||||
CRYPTO_W_LOCK(_w); \ | |||||
} while (0) | |||||
#define CRYPTO_W_UNLOCKALL() do { \ | |||||
struct crypto_worker *_w; \ | |||||
CRYPTO_W_FOREACH(_w) \ | |||||
CRYPTO_W_UNLOCK(_w); \ | |||||
} while (0) | |||||
struct crypto_ret_worker { | |||||
/* | /* | ||||
* There are two queues for processing completed crypto requests; one | * There are two queues for processing completed crypto requests; one | ||||
* for the symmetric and one for the asymmetric ops. We only need one | * for the symmetric and one for the asymmetric ops. We only need one | ||||
* but have two to avoid type futzing (cryptop vs. cryptkop). A single | * but have two to avoid type futzing (cryptop vs. cryptkop). A single | ||||
* mutex is used to lock access to both queues. Note that this lock | * mutex is used to lock access to both queues. Note that this lock | ||||
* must be separate from the lock on request queues to insure driver | * must be separate from the lock on request queues to insure driver | ||||
* callbacks don't generate lock order reversals. | * callbacks don't generate lock order reversals. | ||||
*/ | */ | ||||
static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ | TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ | ||||
static TAILQ_HEAD(,cryptkop) crp_ret_kq; | TAILQ_HEAD(,cryptkop) crp_ret_kq; | ||||
static struct mtx crypto_ret_q_mtx; | struct mtx crypto_ret_q_mtx; | ||||
#define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) | |||||
#define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) | |||||
#define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq)) | |||||
struct proc *cryptoretproc; | |||||
} __aligned(CACHE_LINE_SIZE); | |||||
static struct crypto_ret_worker *crypto_ret_workers = NULL; | |||||
#define CRYPTO_RETW_FOREACH(w) \ | |||||
for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w) | |||||
#define CRYPTO_RETW(i) (&crypto_ret_workers[i]) | |||||
#define CRYPTO_RETW_ID(w) ((int)(((uintptr_t)(w) - (uintptr_t)crypto_ret_workers) / sizeof(struct crypto_ret_worker))) | |||||
#define CRYPTO_RETQ_LOCK(w) mtx_lock(&w->crypto_ret_q_mtx) | |||||
#define CRYPTO_RETQ_UNLOCK(w) mtx_unlock(&w->crypto_ret_q_mtx) | |||||
#define CRYPTO_RETQ_EMPTY(w) (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq)) | |||||
static uma_zone_t cryptop_zone; | static uma_zone_t cryptop_zone; | ||||
static uma_zone_t cryptodesc_zone; | static uma_zone_t cryptodesc_zone; | ||||
static int crypto_workers_num = 1; | |||||
jmg: default should be 0, and then when zero, set to the number of cpu's so this gets used by… | |||||
TUNABLE_INT("kern.crypto_workers_num", &crypto_workers_num); | |||||
int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ | int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ | ||||
SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, | SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, | ||||
&crypto_userasymcrypto, 0, | &crypto_userasymcrypto, 0, | ||||
"Enable/disable user-mode access to asymmetric crypto support"); | "Enable/disable user-mode access to asymmetric crypto support"); | ||||
int crypto_devallowsoft = 0; /* only use hardware crypto */ | int crypto_devallowsoft = 0; /* only use hardware crypto */ | ||||
SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, | SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, | ||||
&crypto_devallowsoft, 0, | &crypto_devallowsoft, 0, | ||||
"Enable/disable use of software crypto by /dev/crypto"); | "Enable/disable use of software crypto by /dev/crypto"); | ||||
int crypto_max_requests = 256; | |||||
SYSCTL_INT(_kern, OID_AUTO, crypto_max_requests, CTLFLAG_RW, | |||||
&crypto_max_requests, 0, | |||||
"Max number of pending crypto requests"); | |||||
int crypto_nb_requests = 0; | |||||
SYSCTL_INT(_kern, OID_AUTO, crypto_nb_requests, CTLFLAG_RW, | |||||
jmgUnsubmitted Not Done Inline Actionswhy is this writable? jmg: why is this writable? | |||||
&crypto_nb_requests, 0, | |||||
"Current number of crypto requests"); | |||||
int crypto_dropped_requests = 0; | |||||
SYSCTL_INT(_kern, OID_AUTO, crypto_dropped_requests, CTLFLAG_RW, | |||||
jmgUnsubmitted Not Done Inline Actionswhy is this writable? jmg: why is this writable? | |||||
&crypto_dropped_requests, 0, | |||||
"Dropped requests due to limit reached"); | |||||
MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); | MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); | ||||
static void crypto_proc(void); | static void crypto_proc(struct crypto_worker *worker); | ||||
static struct proc *cryptoproc; | static void crypto_ret_proc(struct crypto_ret_worker *worker); | ||||
static void crypto_ret_proc(void); | |||||
static struct proc *cryptoretproc; | |||||
static void crypto_destroy(void); | static void crypto_destroy(void); | ||||
static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); | static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); | ||||
static int crypto_kinvoke(struct cryptkop *krp, int flags); | static int crypto_kinvoke(struct cryptkop *krp, int flags); | ||||
static struct cryptostats cryptostats; | static struct cryptostats cryptostats; | ||||
SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, | SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, | ||||
cryptostats, "Crypto system statistics"); | cryptostats, "Crypto system statistics"); | ||||
Show All 30 Lines | |||||
/* It'd be nice if we could store these in some kind of secure memory... */ | /* It'd be nice if we could store these in some kind of secure memory... */ | ||||
struct keybuf * get_keybuf(void) { | struct keybuf * get_keybuf(void) { | ||||
return (keybuf); | return (keybuf); | ||||
} | } | ||||
static int | static int | ||||
select_crypto_worker_id(const struct cryptop *crp) | |||||
{ | |||||
/* Each worker is assigned a session */ | |||||
if (crypto_workers_num > 1) | |||||
jmgUnsubmitted Not Done Inline Actionsjust remove the if, any unsigned num % 1 will always return 0, jmg: just remove the if, any unsigned num % 1 will always return 0, | |||||
return (crp->crp_sid % crypto_workers_num); | |||||
jmgUnsubmitted Not Done Inline Actionshave you profiled the session allocations to make sure that this makes sense and gives good distribution? what happens when heavily sessions collide? jmg: have you profiled the session allocations to make sure that this makes sense and gives good… | |||||
return 0; | |||||
} | |||||
static int | |||||
select_kcrypto_worker_id(struct cryptkop *krp) | |||||
{ | |||||
if (crypto_workers_num > 1) | |||||
return krp->krp_crid % crypto_workers_num; | |||||
gnnUnsubmitted Not Done Inline ActionsAdd parens to make this match the code above. gnn: Add parens to make this match the code above. | |||||
return 0; | |||||
} | |||||
static int | |||||
crypto_init(void) | crypto_init(void) | ||||
{ | { | ||||
struct crypto_ret_worker *ret_worker; | |||||
struct crypto_worker *worker; | |||||
int error; | int error; | ||||
crypto_nb_requests = 0; | |||||
mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", | mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", | ||||
MTX_DEF|MTX_QUIET); | MTX_DEF|MTX_QUIET); | ||||
TAILQ_INIT(&crp_q); | |||||
TAILQ_INIT(&crp_kq); | |||||
mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); | |||||
TAILQ_INIT(&crp_ret_q); | |||||
TAILQ_INIT(&crp_ret_kq); | |||||
mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF); | |||||
cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), | cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), | ||||
0, 0, 0, 0, | 0, 0, 0, 0, | ||||
UMA_ALIGN_PTR, UMA_ZONE_ZINIT); | UMA_ALIGN_PTR, UMA_ZONE_ZINIT); | ||||
cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), | cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), | ||||
0, 0, 0, 0, | 0, 0, 0, 0, | ||||
UMA_ALIGN_PTR, UMA_ZONE_ZINIT); | UMA_ALIGN_PTR, UMA_ZONE_ZINIT); | ||||
if (cryptodesc_zone == NULL || cryptop_zone == NULL) { | if (cryptodesc_zone == NULL || cryptop_zone == NULL) { | ||||
printf("crypto_init: cannot setup crypto zones\n"); | printf("crypto_init: cannot setup crypto zones\n"); | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto bad; | goto bad; | ||||
} | } | ||||
crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; | crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; | ||||
crypto_drivers = malloc(crypto_drivers_num * | crypto_drivers = malloc(crypto_drivers_num * | ||||
sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); | sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); | ||||
if (crypto_drivers == NULL) { | if (crypto_drivers == NULL) { | ||||
printf("crypto_init: cannot setup crypto drivers\n"); | printf("crypto_init: cannot setup crypto drivers\n"); | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto bad; | goto bad; | ||||
} | } | ||||
error = kproc_create((void (*)(void *)) crypto_proc, NULL, | if (crypto_workers_num < 1) | ||||
&cryptoproc, 0, 0, "crypto"); | crypto_workers_num = 1; | ||||
if (error) { | |||||
printf("crypto_init: cannot start crypto thread; error %d", | crypto_workers = malloc(crypto_workers_num * sizeof(struct crypto_worker), | ||||
error); | M_CRYPTO_DATA, M_NOWAIT|M_ZERO); | ||||
if (crypto_workers == NULL) { | |||||
error = ENOMEM; | |||||
printf("crypto_init: cannot allocate workers\n"); | |||||
goto bad; | goto bad; | ||||
} | } | ||||
error = kproc_create((void (*)(void *)) crypto_ret_proc, NULL, | if (bootverbose) | ||||
&cryptoretproc, 0, 0, "crypto returns"); | printf("crypto_init: instanciating %d worker(s)\n", crypto_workers_num); | ||||
CRYPTO_W_FOREACH(worker) { | |||||
TAILQ_INIT(&worker->crp_q); | |||||
TAILQ_INIT(&worker->crp_kq); | |||||
worker->crp_sleep = 0; | |||||
mtx_init(&worker->crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); | |||||
error = kproc_create((void (*)(void *)) crypto_proc, worker, | |||||
&worker->cryptoproc, 0, 0, "crypto %d", CRYPTO_W_ID(worker)); | |||||
if (error) { | if (error) { | ||||
printf("crypto_init: cannot start cryptoret thread; error %d", | printf("crypto_init: cannot start crypto thread %d; error %d", | ||||
error); | CRYPTO_W_ID(worker), error); | ||||
goto bad; | goto bad; | ||||
} | } | ||||
} | |||||
crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker), | |||||
M_CRYPTO_DATA, M_NOWAIT|M_ZERO); | |||||
if (crypto_ret_workers == NULL) { | |||||
error = ENOMEM; | |||||
printf("crypto_init: cannot allocate ret workers\n"); | |||||
goto bad; | |||||
} | |||||
keybuf_init(); | keybuf_init(); | ||||
CRYPTO_RETW_FOREACH(ret_worker) { | |||||
TAILQ_INIT(&ret_worker->crp_ret_q); | |||||
TAILQ_INIT(&ret_worker->crp_ret_kq); | |||||
mtx_init(&ret_worker->crypto_ret_q_mtx, "crypto", | |||||
"crypto return queues", MTX_DEF); | |||||
error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker, | |||||
&ret_worker->cryptoretproc, 0, 0, "crypto returns %d", | |||||
CRYPTO_RETW_ID(ret_worker)); | |||||
if (error) { | |||||
printf("crypto_init: cannot start crypto ret thread %d; error %d", | |||||
CRYPTO_RETW_ID(ret_worker), error); | |||||
goto bad; | |||||
} | |||||
} | |||||
return 0; | return 0; | ||||
bad: | bad: | ||||
crypto_destroy(); | crypto_destroy(); | ||||
return error; | return error; | ||||
} | } | ||||
/* | /* | ||||
* Signal a crypto thread to terminate. We use the driver | * Signal a crypto thread to terminate. We use the driver | ||||
* table lock to synchronize the sleep/wakeups so that we | * table lock to synchronize the sleep/wakeups so that we | ||||
* are sure the threads have terminated before we release | * are sure the threads have terminated before we release | ||||
* the data structures they use. See crypto_finis below | * the data structures they use. See crypto_finis below | ||||
* for the other half of this song-and-dance. | * for the other half of this song-and-dance. | ||||
*/ | */ | ||||
static void | static void | ||||
crypto_terminate(struct proc **pp, void *q) | crypto_terminate(struct proc **pp, void *q) | ||||
{ | { | ||||
struct proc *p; | struct proc *p; | ||||
mtx_assert(&crypto_drivers_mtx, MA_OWNED); | CRYPTO_DRIVER_ASSERT(); | ||||
jmgUnsubmitted Not Done Inline Actionsthis change is unrelated to this.. please break out jmg: this change is unrelated to this.. please break out | |||||
p = *pp; | p = *pp; | ||||
*pp = NULL; | *pp = NULL; | ||||
if (p) { | if (p) { | ||||
wakeup_one(q); | wakeup_one(q); | ||||
PROC_LOCK(p); /* NB: insure we don't miss wakeup */ | PROC_LOCK(p); /* NB: insure we don't miss wakeup */ | ||||
CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ | CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ | ||||
msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); | msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); | ||||
PROC_UNLOCK(p); | PROC_UNLOCK(p); | ||||
CRYPTO_DRIVER_LOCK(); | CRYPTO_DRIVER_LOCK(); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
crypto_destroy(void) | crypto_destroy(void) | ||||
{ | { | ||||
struct crypto_ret_worker *ret_worker; | |||||
struct crypto_worker *worker; | |||||
/* | /* | ||||
* Terminate any crypto threads. | * Terminate any crypto threads. | ||||
*/ | */ | ||||
CRYPTO_DRIVER_LOCK(); | CRYPTO_DRIVER_LOCK(); | ||||
crypto_terminate(&cryptoproc, &crp_q); | CRYPTO_W_FOREACH(worker) | ||||
crypto_terminate(&cryptoretproc, &crp_ret_q); | crypto_terminate(&worker->cryptoproc, &worker->crp_q); | ||||
CRYPTO_RETW_FOREACH(ret_worker) | |||||
crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q); | |||||
CRYPTO_DRIVER_UNLOCK(); | CRYPTO_DRIVER_UNLOCK(); | ||||
/* XXX flush queues??? */ | /* XXX flush queues??? */ | ||||
/* | /* | ||||
* Reclaim dynamically allocated resources. | * Reclaim dynamically allocated resources. | ||||
*/ | */ | ||||
if (crypto_drivers != NULL) | if (crypto_drivers != NULL) | ||||
free(crypto_drivers, M_CRYPTO_DATA); | free(crypto_drivers, M_CRYPTO_DATA); | ||||
if (cryptodesc_zone != NULL) | if (cryptodesc_zone != NULL) | ||||
uma_zdestroy(cryptodesc_zone); | uma_zdestroy(cryptodesc_zone); | ||||
if (cryptop_zone != NULL) | if (cryptop_zone != NULL) | ||||
uma_zdestroy(cryptop_zone); | uma_zdestroy(cryptop_zone); | ||||
mtx_destroy(&crypto_q_mtx); | |||||
mtx_destroy(&crypto_ret_q_mtx); | CRYPTO_W_FOREACH(worker) | ||||
mtx_destroy(&worker->crypto_q_mtx); | |||||
free(crypto_workers, M_CRYPTO_DATA); | |||||
CRYPTO_RETW_FOREACH(ret_worker) | |||||
mtx_destroy(&ret_worker->crypto_ret_q_mtx); | |||||
free(crypto_ret_workers, M_CRYPTO_DATA); | |||||
mtx_destroy(&crypto_drivers_mtx); | mtx_destroy(&crypto_drivers_mtx); | ||||
} | } | ||||
static struct cryptocap * | static struct cryptocap * | ||||
crypto_checkdriver(u_int32_t hid) | crypto_checkdriver(u_int32_t hid) | ||||
{ | { | ||||
if (crypto_drivers == NULL) | if (crypto_drivers == NULL) | ||||
return NULL; | return NULL; | ||||
▲ Show 20 Lines • Show All 446 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Clear blockage on a driver. The what parameter indicates whether | * Clear blockage on a driver. The what parameter indicates whether | ||||
* the driver is now ready for cryptop's and/or cryptokop's. | * the driver is now ready for cryptop's and/or cryptokop's. | ||||
*/ | */ | ||||
int | int | ||||
crypto_unblock(u_int32_t driverid, int what) | crypto_unblock(u_int32_t driverid, int what) | ||||
{ | { | ||||
struct cryptocap *cap; | struct cryptocap *cap; | ||||
struct crypto_worker *worker; | |||||
int err; | int err; | ||||
CRYPTO_Q_LOCK(); | CRYPTO_W_LOCKALL(); | ||||
cap = crypto_checkdriver(driverid); | cap = crypto_checkdriver(driverid); | ||||
if (cap != NULL) { | if (cap != NULL) { | ||||
if (what & CRYPTO_SYMQ) | if (what & CRYPTO_SYMQ) | ||||
cap->cc_qblocked = 0; | cap->cc_qblocked = 0; | ||||
if (what & CRYPTO_ASYMQ) | if (what & CRYPTO_ASYMQ) | ||||
cap->cc_kqblocked = 0; | cap->cc_kqblocked = 0; | ||||
if (crp_sleep) | CRYPTO_W_FOREACH(worker) { | ||||
wakeup_one(&crp_q); | if (worker->crp_sleep) | ||||
wakeup_one(&worker->crp_q); | |||||
} | |||||
err = 0; | err = 0; | ||||
} else | } else | ||||
err = EINVAL; | err = EINVAL; | ||||
CRYPTO_Q_UNLOCK(); | CRYPTO_W_UNLOCKALL(); | ||||
return err; | return err; | ||||
} | } | ||||
/* | /* | ||||
* Add a crypto request to a queue, to be processed by the kernel thread. | * Add a crypto request to a queue, to be processed by the kernel thread. | ||||
*/ | */ | ||||
int | int | ||||
crypto_dispatch(struct cryptop *crp) | crypto_dispatch(struct cryptop *crp) | ||||
{ | { | ||||
struct cryptocap *cap; | struct cryptocap *cap; | ||||
struct crypto_worker *worker; | |||||
u_int32_t hid; | u_int32_t hid; | ||||
int result; | int result; | ||||
cryptostats.cs_ops++; | atomic_add_acq_int(&cryptostats.cs_ops, 1); | ||||
jmgUnsubmitted Not Done Inline Actionsthis should be made into a PCPU option, and then aggregated as needed, if you have lots of threads working, this will bounce the stats line around cpus a lot.. jmg: this should be made into a PCPU option, and then aggregated as needed, if you have lots of… | |||||
#ifdef CRYPTO_TIMING | #ifdef CRYPTO_TIMING | ||||
if (crypto_timing) | if (crypto_timing) | ||||
binuptime(&crp->crp_tstamp); | binuptime(&crp->crp_tstamp); | ||||
#endif | #endif | ||||
hid = CRYPTO_SESID2HID(crp->crp_sid); | hid = CRYPTO_SESID2HID(crp->crp_sid); | ||||
Show All 11 Lines | if (!cap->cc_qblocked) { | ||||
if (result != ERESTART) | if (result != ERESTART) | ||||
return (result); | return (result); | ||||
/* | /* | ||||
* The driver ran out of resources, put the request on | * The driver ran out of resources, put the request on | ||||
* the queue. | * the queue. | ||||
*/ | */ | ||||
} | } | ||||
} | } | ||||
CRYPTO_Q_LOCK(); | |||||
TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); | worker = CRYPTO_W(select_crypto_worker_id(crp)); | ||||
if (crp_sleep) | CRYPTO_W_LOCK(worker); | ||||
wakeup_one(&crp_q); | TAILQ_INSERT_TAIL(&worker->crp_q, crp, crp_next); | ||||
CRYPTO_Q_UNLOCK(); | if (worker->crp_sleep) | ||||
wakeup_one(&worker->crp_q); | |||||
CRYPTO_W_UNLOCK(worker); | |||||
return 0; | return 0; | ||||
} | } | ||||
/* | /* | ||||
* Add an asymetric crypto request to a queue, | * Add an asymetric crypto request to a queue, | ||||
* to be processed by the kernel thread. | * to be processed by the kernel thread. | ||||
*/ | */ | ||||
int | int | ||||
crypto_kdispatch(struct cryptkop *krp) | crypto_kdispatch(struct cryptkop *krp) | ||||
{ | { | ||||
int error; | int error; | ||||
cryptostats.cs_kops++; | cryptostats.cs_kops++; | ||||
error = crypto_kinvoke(krp, krp->krp_crid); | error = crypto_kinvoke(krp, krp->krp_crid); | ||||
if (error == ERESTART) { | if (error == ERESTART) { | ||||
CRYPTO_Q_LOCK(); | struct crypto_worker *worker; | ||||
TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); | |||||
if (crp_sleep) | worker = CRYPTO_W(select_kcrypto_worker_id(krp)); | ||||
wakeup_one(&crp_q); | |||||
CRYPTO_Q_UNLOCK(); | CRYPTO_W_LOCK(worker); | ||||
TAILQ_INSERT_TAIL(&worker->crp_kq, krp, krp_next); | |||||
if (worker->crp_sleep) | |||||
wakeup_one(&worker->crp_q); | |||||
CRYPTO_W_UNLOCK(worker); | |||||
error = 0; | error = 0; | ||||
} | } | ||||
return error; | return error; | ||||
} | } | ||||
/* | /* | ||||
* Verify a driver is suitable for the specified operation. | * Verify a driver is suitable for the specified operation. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 203 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
crypto_freereq(struct cryptop *crp) | crypto_freereq(struct cryptop *crp) | ||||
{ | { | ||||
struct cryptodesc *crd; | struct cryptodesc *crd; | ||||
if (crp == NULL) | if (crp == NULL) | ||||
return; | return; | ||||
#ifdef DIAGNOSTIC | |||||
jmgUnsubmitted Not Done Inline Actionswhy is this code deleted as part of this change? jmg: why is this code deleted as part of this change? | |||||
{ | |||||
struct cryptop *crp2; | |||||
CRYPTO_Q_LOCK(); | |||||
TAILQ_FOREACH(crp2, &crp_q, crp_next) { | |||||
KASSERT(crp2 != crp, | |||||
("Freeing cryptop from the crypto queue (%p).", | |||||
crp)); | |||||
} | |||||
CRYPTO_Q_UNLOCK(); | |||||
CRYPTO_RETQ_LOCK(); | |||||
TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) { | |||||
KASSERT(crp2 != crp, | |||||
("Freeing cryptop from the return queue (%p).", | |||||
crp)); | |||||
} | |||||
CRYPTO_RETQ_UNLOCK(); | |||||
} | |||||
#endif | |||||
while ((crd = crp->crp_desc) != NULL) { | while ((crd = crp->crp_desc) != NULL) { | ||||
crp->crp_desc = crd->crd_next; | crp->crp_desc = crd->crd_next; | ||||
uma_zfree(cryptodesc_zone, crd); | uma_zfree(cryptodesc_zone, crd); | ||||
atomic_add_acq_int(&crypto_nb_requests, -1); | |||||
} | } | ||||
uma_zfree(cryptop_zone, crp); | uma_zfree(cryptop_zone, crp); | ||||
} | } | ||||
/* | /* | ||||
* Acquire a set of crypto descriptors. | * Acquire a set of crypto descriptors. | ||||
*/ | */ | ||||
struct cryptop * | struct cryptop * | ||||
crypto_getreq(int num) | crypto_getreq(int num) | ||||
{ | { | ||||
struct cryptodesc *crd; | struct cryptodesc *crd; | ||||
struct cryptop *crp; | struct cryptop *crp; | ||||
if (crypto_nb_requests > crypto_max_requests) { | |||||
jmgUnsubmitted Not Done Inline Actionswhy has this limit been introduced? we didn't have it before? and now it adds a new way for requests to fail. jmg: why has this limit been introduced? we didn't have it before? and now it adds a new way for… | |||||
crypto_dropped_requests++; | |||||
return NULL; | |||||
} | |||||
crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); | crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); | ||||
if (crp != NULL) { | if (crp != NULL) { | ||||
while (num--) { | while (num--) { | ||||
crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); | crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); | ||||
if (crd == NULL) { | if (crd == NULL) { | ||||
crypto_freereq(crp); | crypto_freereq(crp); | ||||
return NULL; | return NULL; | ||||
} | } | ||||
crd->crd_next = crp->crp_desc; | crd->crd_next = crp->crp_desc; | ||||
crp->crp_desc = crd; | crp->crp_desc = crd; | ||||
atomic_add_acq_int(&crypto_nb_requests, 1); | |||||
} | } | ||||
} | } | ||||
return crp; | return crp; | ||||
} | } | ||||
/* | /* | ||||
* Invoke the callback on behalf of the driver. | * Invoke the callback on behalf of the driver. | ||||
*/ | */ | ||||
Show All 34 Lines | if (crypto_timing) { | ||||
struct bintime t = crp->crp_tstamp; | struct bintime t = crp->crp_tstamp; | ||||
crypto_tstat(&cryptostats.cs_cb, &t); | crypto_tstat(&cryptostats.cs_cb, &t); | ||||
crp->crp_callback(crp); | crp->crp_callback(crp); | ||||
crypto_tstat(&cryptostats.cs_finis, &t); | crypto_tstat(&cryptostats.cs_finis, &t); | ||||
} else | } else | ||||
#endif | #endif | ||||
crp->crp_callback(crp); | crp->crp_callback(crp); | ||||
} else { | } else { | ||||
// struct cryptop *tmp; | |||||
gnnUnsubmitted Not Done Inline ActionsRemove commented out line before commit. gnn: Remove commented out line before commit. | |||||
struct crypto_ret_worker *ret_worker; | |||||
ret_worker = CRYPTO_RETW(select_crypto_worker_id(crp)); | |||||
/* | /* | ||||
* Normal case; queue the callback for the thread. | * Normal case; queue the callback for the thread. | ||||
*/ | */ | ||||
CRYPTO_RETQ_LOCK(); | CRYPTO_RETQ_LOCK(ret_worker); | ||||
if (CRYPTO_RETQ_EMPTY()) | if (CRYPTO_RETQ_EMPTY(ret_worker)) | ||||
wakeup_one(&crp_ret_q); /* shared wait channel */ | wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ | ||||
TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); | TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next); | ||||
CRYPTO_RETQ_UNLOCK(); | CRYPTO_RETQ_UNLOCK(ret_worker); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Invoke the callback on behalf of the driver. | * Invoke the callback on behalf of the driver. | ||||
*/ | */ | ||||
void | void | ||||
crypto_kdone(struct cryptkop *krp) | crypto_kdone(struct cryptkop *krp) | ||||
{ | { | ||||
struct crypto_ret_worker *ret_worker; | |||||
struct cryptocap *cap; | struct cryptocap *cap; | ||||
if (krp->krp_status != 0) | if (krp->krp_status != 0) | ||||
cryptostats.cs_kerrs++; | cryptostats.cs_kerrs++; | ||||
CRYPTO_DRIVER_LOCK(); | CRYPTO_DRIVER_LOCK(); | ||||
/* XXX: What if driver is loaded in the meantime? */ | /* XXX: What if driver is loaded in the meantime? */ | ||||
if (krp->krp_hid < crypto_drivers_num) { | if (krp->krp_hid < crypto_drivers_num) { | ||||
cap = &crypto_drivers[krp->krp_hid]; | cap = &crypto_drivers[krp->krp_hid]; | ||||
KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0")); | KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0")); | ||||
cap->cc_koperations--; | cap->cc_koperations--; | ||||
if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) | if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) | ||||
crypto_remove(cap); | crypto_remove(cap); | ||||
} | } | ||||
CRYPTO_DRIVER_UNLOCK(); | CRYPTO_DRIVER_UNLOCK(); | ||||
CRYPTO_RETQ_LOCK(); | |||||
if (CRYPTO_RETQ_EMPTY()) | ret_worker = CRYPTO_RETW(select_kcrypto_worker_id(krp)); | ||||
wakeup_one(&crp_ret_q); /* shared wait channel */ | |||||
TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); | CRYPTO_RETQ_LOCK(ret_worker); | ||||
CRYPTO_RETQ_UNLOCK(); | if (CRYPTO_RETQ_EMPTY(ret_worker)) | ||||
wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ | |||||
TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next); | |||||
CRYPTO_RETQ_UNLOCK(ret_worker); | |||||
} | } | ||||
int | int | ||||
crypto_getfeat(int *featp) | crypto_getfeat(int *featp) | ||||
{ | { | ||||
int hid, kalg, feat = 0; | int hid, kalg, feat = 0; | ||||
CRYPTO_DRIVER_LOCK(); | CRYPTO_DRIVER_LOCK(); | ||||
Show All 30 Lines | crypto_finis(void *chan) | ||||
CRYPTO_DRIVER_UNLOCK(); | CRYPTO_DRIVER_UNLOCK(); | ||||
kproc_exit(0); | kproc_exit(0); | ||||
} | } | ||||
/* | /* | ||||
* Crypto thread, dispatches crypto requests. | * Crypto thread, dispatches crypto requests. | ||||
*/ | */ | ||||
static void | static void | ||||
crypto_proc(void) | crypto_proc(struct crypto_worker *worker) | ||||
{ | { | ||||
struct cryptop *crp, *submit; | struct cryptop *crp, *submit; | ||||
struct cryptkop *krp; | struct cryptkop *krp; | ||||
struct cryptocap *cap; | struct cryptocap *cap; | ||||
u_int32_t hid; | u_int32_t hid; | ||||
int result, hint; | int result, hint; | ||||
#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) | #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) | ||||
fpu_kern_thread(FPU_KERN_NORMAL); | fpu_kern_thread(FPU_KERN_NORMAL); | ||||
#endif | #endif | ||||
CRYPTO_Q_LOCK(); | CRYPTO_W_LOCK(worker); | ||||
for (;;) { | for (;;) { | ||||
/* | /* | ||||
* Find the first element in the queue that can be | * Find the first element in the queue that can be | ||||
* processed and look-ahead to see if multiple ops | * processed and look-ahead to see if multiple ops | ||||
* are ready for the same driver. | * are ready for the same driver. | ||||
*/ | */ | ||||
submit = NULL; | submit = NULL; | ||||
hint = 0; | hint = 0; | ||||
TAILQ_FOREACH(crp, &crp_q, crp_next) { | TAILQ_FOREACH(crp, &worker->crp_q, crp_next) { | ||||
hid = CRYPTO_SESID2HID(crp->crp_sid); | hid = CRYPTO_SESID2HID(crp->crp_sid); | ||||
cap = crypto_checkdriver(hid); | cap = crypto_checkdriver(hid); | ||||
/* | /* | ||||
* Driver cannot disappeared when there is an active | * Driver cannot disappeared when there is an active | ||||
* session. | * session. | ||||
*/ | */ | ||||
KASSERT(cap != NULL, ("%s:%u Driver disappeared.", | KASSERT(cap != NULL, ("%s:%u Driver disappeared.", | ||||
__func__, __LINE__)); | __func__, __LINE__)); | ||||
Show All 20 Lines | TAILQ_FOREACH(crp, &worker->crp_q, crp_next) { | ||||
submit = crp; | submit = crp; | ||||
if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) | if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) | ||||
break; | break; | ||||
/* keep scanning for more are q'd */ | /* keep scanning for more are q'd */ | ||||
} | } | ||||
} | } | ||||
} | } | ||||
if (submit != NULL) { | if (submit != NULL) { | ||||
TAILQ_REMOVE(&crp_q, submit, crp_next); | TAILQ_REMOVE(&worker->crp_q, submit, crp_next); | ||||
hid = CRYPTO_SESID2HID(submit->crp_sid); | hid = CRYPTO_SESID2HID(submit->crp_sid); | ||||
cap = crypto_checkdriver(hid); | cap = crypto_checkdriver(hid); | ||||
KASSERT(cap != NULL, ("%s:%u Driver disappeared.", | KASSERT(cap != NULL, ("%s:%u Driver disappeared.", | ||||
__func__, __LINE__)); | __func__, __LINE__)); | ||||
result = crypto_invoke(cap, submit, hint); | result = crypto_invoke(cap, submit, hint); | ||||
if (result == ERESTART) { | if (result == ERESTART) { | ||||
/* | /* | ||||
* The driver ran out of resources, mark the | * The driver ran out of resources, mark the | ||||
* driver ``blocked'' for cryptop's and put | * driver ``blocked'' for cryptop's and put | ||||
* the request back in the queue. It would | * the request back in the queue. It would | ||||
* best to put the request back where we got | * best to put the request back where we got | ||||
* it but that's hard so for now we put it | * it but that's hard so for now we put it | ||||
* at the front. This should be ok; putting | * at the front. This should be ok; putting | ||||
* it at the end does not work. | * it at the end does not work. | ||||
*/ | */ | ||||
/* XXX validate sid again? */ | /* XXX validate sid again? */ | ||||
crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; | crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; | ||||
TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); | TAILQ_INSERT_HEAD(&worker->crp_q, submit, crp_next); | ||||
cryptostats.cs_blocks++; | atomic_add_acq_int(&cryptostats.cs_blocks, 1); | ||||
} | } | ||||
} | } | ||||
/* As above, but for key ops */ | /* As above, but for key ops */ | ||||
TAILQ_FOREACH(krp, &crp_kq, krp_next) { | TAILQ_FOREACH(krp, &worker->crp_kq, krp_next) { | ||||
cap = crypto_checkdriver(krp->krp_hid); | cap = crypto_checkdriver(krp->krp_hid); | ||||
if (cap == NULL || cap->cc_dev == NULL) { | if (cap == NULL || cap->cc_dev == NULL) { | ||||
/* | /* | ||||
* Operation needs to be migrated, invalidate | * Operation needs to be migrated, invalidate | ||||
* the assigned device so it will reselect a | * the assigned device so it will reselect a | ||||
* new one below. Propagate the original | * new one below. Propagate the original | ||||
* crid selection flags if supplied. | * crid selection flags if supplied. | ||||
*/ | */ | ||||
krp->krp_hid = krp->krp_crid & | krp->krp_hid = krp->krp_crid & | ||||
(CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE); | (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE); | ||||
if (krp->krp_hid == 0) | if (krp->krp_hid == 0) | ||||
krp->krp_hid = | krp->krp_hid = | ||||
CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE; | CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE; | ||||
break; | break; | ||||
} | } | ||||
if (!cap->cc_kqblocked) | if (!cap->cc_kqblocked) | ||||
break; | break; | ||||
} | } | ||||
if (krp != NULL) { | if (krp != NULL) { | ||||
TAILQ_REMOVE(&crp_kq, krp, krp_next); | TAILQ_REMOVE(&worker->crp_kq, krp, krp_next); | ||||
result = crypto_kinvoke(krp, krp->krp_hid); | result = crypto_kinvoke(krp, krp->krp_hid); | ||||
if (result == ERESTART) { | if (result == ERESTART) { | ||||
/* | /* | ||||
* The driver ran out of resources, mark the | * The driver ran out of resources, mark the | ||||
* driver ``blocked'' for cryptkop's and put | * driver ``blocked'' for cryptkop's and put | ||||
* the request back in the queue. It would | * the request back in the queue. It would | ||||
* best to put the request back where we got | * best to put the request back where we got | ||||
* it but that's hard so for now we put it | * it but that's hard so for now we put it | ||||
* at the front. This should be ok; putting | * at the front. This should be ok; putting | ||||
* it at the end does not work. | * it at the end does not work. | ||||
*/ | */ | ||||
/* XXX validate sid again? */ | /* XXX validate sid again? */ | ||||
crypto_drivers[krp->krp_hid].cc_kqblocked = 1; | crypto_drivers[krp->krp_hid].cc_kqblocked = 1; | ||||
TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); | TAILQ_INSERT_HEAD(&worker->crp_kq, krp, krp_next); | ||||
cryptostats.cs_kblocks++; | atomic_add_acq_int(&cryptostats.cs_kblocks, 1); | ||||
} | } | ||||
} | } | ||||
if (submit == NULL && krp == NULL) { | if (submit == NULL && krp == NULL) { | ||||
/* | /* | ||||
* Nothing more to be processed. Sleep until we're | * Nothing more to be processed. Sleep until we're | ||||
* woken because there are more ops to process. | * woken because there are more ops to process. | ||||
* This happens either by submission or by a driver | * This happens either by submission or by a driver | ||||
* becoming unblocked and notifying us through | * becoming unblocked and notifying us through | ||||
* crypto_unblock. Note that when we wakeup we | * crypto_unblock. Note that when we wakeup we | ||||
* start processing each queue again from the | * start processing each queue again from the | ||||
* front. It's not clear that it's important to | * front. It's not clear that it's important to | ||||
* preserve this ordering since ops may finish | * preserve this ordering since ops may finish | ||||
* out of order if dispatched to different devices | * out of order if dispatched to different devices | ||||
* and some become blocked while others do not. | * and some become blocked while others do not. | ||||
*/ | */ | ||||
crp_sleep = 1; | worker->crp_sleep = 1; | ||||
msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); | msleep(&worker->crp_q, &worker->crypto_q_mtx, PWAIT, "crypto_wait", 0); | ||||
crp_sleep = 0; | worker->crp_sleep = 0; | ||||
if (cryptoproc == NULL) | if (worker->cryptoproc == NULL) | ||||
break; | break; | ||||
cryptostats.cs_intrs++; | atomic_add_acq_int(&cryptostats.cs_intrs, 1); | ||||
jmgUnsubmitted Not Done Inline Actionsthese should be _32 instead. jmg: these should be _32 instead. | |||||
} | } | ||||
} | } | ||||
CRYPTO_Q_UNLOCK(); | CRYPTO_W_UNLOCK(worker); | ||||
crypto_finis(&crp_q); | crypto_finis(&worker->crp_q); | ||||
} | } | ||||
/* | /* | ||||
* Crypto returns thread, does callbacks for processed crypto requests. | * Crypto returns thread, does callbacks for processed crypto requests. | ||||
* Callbacks are done here, rather than in the crypto drivers, because | * Callbacks are done here, rather than in the crypto drivers, because | ||||
* callbacks typically are expensive and would slow interrupt handling. | * callbacks typically are expensive and would slow interrupt handling. | ||||
*/ | */ | ||||
static void | static void | ||||
crypto_ret_proc(void) | crypto_ret_proc(struct crypto_ret_worker *worker) | ||||
{ | { | ||||
struct cryptop *crpt; | struct cryptop *crpt; | ||||
struct cryptkop *krpt; | struct cryptkop *krpt; | ||||
CRYPTO_RETQ_LOCK(); | CRYPTO_RETQ_LOCK(worker); | ||||
for (;;) { | for (;;) { | ||||
/* Harvest return q's for completed ops */ | /* Harvest return q's for completed ops */ | ||||
crpt = TAILQ_FIRST(&crp_ret_q); | crpt = TAILQ_FIRST(&worker->crp_ret_q); | ||||
if (crpt != NULL) | if (crpt != NULL) | ||||
TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); | TAILQ_REMOVE(&worker->crp_ret_q, crpt, crp_next); | ||||
krpt = TAILQ_FIRST(&crp_ret_kq); | krpt = TAILQ_FIRST(&worker->crp_ret_kq); | ||||
if (krpt != NULL) | if (krpt != NULL) | ||||
TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); | TAILQ_REMOVE(&worker->crp_ret_kq, krpt, krp_next); | ||||
if (crpt != NULL || krpt != NULL) { | if (crpt != NULL || krpt != NULL) { | ||||
CRYPTO_RETQ_UNLOCK(); | CRYPTO_RETQ_UNLOCK(worker); | ||||
/* | /* | ||||
* Run callbacks unlocked. | * Run callbacks unlocked. | ||||
*/ | */ | ||||
if (crpt != NULL) { | if (crpt != NULL) { | ||||
#ifdef CRYPTO_TIMING | #ifdef CRYPTO_TIMING | ||||
if (crypto_timing) { | if (crypto_timing) { | ||||
/* | /* | ||||
* NB: We must copy the timestamp before | * NB: We must copy the timestamp before | ||||
* doing the callback as the cryptop is | * doing the callback as the cryptop is | ||||
* likely to be reclaimed. | * likely to be reclaimed. | ||||
*/ | */ | ||||
struct bintime t = crpt->crp_tstamp; | struct bintime t = crpt->crp_tstamp; | ||||
crypto_tstat(&cryptostats.cs_cb, &t); | crypto_tstat(&cryptostats.cs_cb, &t); | ||||
crpt->crp_callback(crpt); | crpt->crp_callback(crpt); | ||||
crypto_tstat(&cryptostats.cs_finis, &t); | crypto_tstat(&cryptostats.cs_finis, &t); | ||||
} else | } else | ||||
#endif | #endif | ||||
crpt->crp_callback(crpt); | crpt->crp_callback(crpt); | ||||
} | } | ||||
if (krpt != NULL) | if (krpt != NULL) | ||||
krpt->krp_callback(krpt); | krpt->krp_callback(krpt); | ||||
CRYPTO_RETQ_LOCK(); | CRYPTO_RETQ_LOCK(worker); | ||||
} else { | } else { | ||||
/* | /* | ||||
* Nothing more to be processed. Sleep until we're | * Nothing more to be processed. Sleep until we're | ||||
* woken because there are more returns to process. | * woken because there are more returns to process. | ||||
*/ | */ | ||||
msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, | msleep(&worker->crp_ret_q, &worker->crypto_ret_q_mtx, PWAIT, | ||||
"crypto_ret_wait", 0); | "crypto_ret_wait", 0); | ||||
if (cryptoretproc == NULL) | if (worker->cryptoretproc == NULL) | ||||
break; | break; | ||||
cryptostats.cs_rets++; | cryptostats.cs_rets++; | ||||
} | } | ||||
} | } | ||||
CRYPTO_RETQ_UNLOCK(); | CRYPTO_RETQ_UNLOCK(worker); | ||||
crypto_finis(&crp_ret_q); | crypto_finis(&worker->crp_ret_q); | ||||
} | } | ||||
#ifdef DDB | #ifdef DDB | ||||
static void | static void | ||||
db_show_drivers(void) | db_show_drivers(void) | ||||
{ | { | ||||
int hid; | int hid; | ||||
Show All 18 Lines | db_printf("%-12s %4u %4u %08x %2u %2u\n" | ||||
, cap->cc_kqblocked | , cap->cc_kqblocked | ||||
); | ); | ||||
} | } | ||||
} | } | ||||
DB_SHOW_COMMAND(crypto, db_show_crypto) | DB_SHOW_COMMAND(crypto, db_show_crypto) | ||||
{ | { | ||||
struct cryptop *crp; | struct cryptop *crp; | ||||
struct crypto_ret_worker *ret_worker; | |||||
struct crypto_worker *worker; | |||||
db_show_drivers(); | db_show_drivers(); | ||||
db_printf("\n"); | db_printf("\n"); | ||||
db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", | db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", | ||||
"HID", "Caps", "Ilen", "Olen", "Etype", "Flags", | "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", | ||||
"Desc", "Callback"); | "Desc", "Callback"); | ||||
TAILQ_FOREACH(crp, &crp_q, crp_next) { | CRYPTO_W_FOREACH(worker) { | ||||
db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n" | TAILQ_FOREACH(crp, &worker->crp_q, crp_next) { | ||||
db_printf("%4u %4u %08x %4u %4u %4u %04x %8p %8p\n" | |||||
, CRYPTO_W_ID(worker) | |||||
, (int) CRYPTO_SESID2HID(crp->crp_sid) | , (int) CRYPTO_SESID2HID(crp->crp_sid) | ||||
, (int) CRYPTO_SESID2CAPS(crp->crp_sid) | , (int) CRYPTO_SESID2CAPS(crp->crp_sid) | ||||
, crp->crp_ilen, crp->crp_olen | , crp->crp_ilen, crp->crp_olen | ||||
, crp->crp_etype | , crp->crp_etype | ||||
, crp->crp_flags | , crp->crp_flags | ||||
, crp->crp_desc | , crp->crp_desc | ||||
, crp->crp_callback | , crp->crp_callback | ||||
); | ); | ||||
} | } | ||||
if (!TAILQ_EMPTY(&crp_ret_q)) { | } | ||||
db_printf("\n%4s %4s %4s %8s\n", | CRYPTO_RETW_FOREACH(ret_worker) { | ||||
"HID", "Etype", "Flags", "Callback"); | if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) { | ||||
TAILQ_FOREACH(crp, &crp_ret_q, crp_next) { | db_printf("\n%7s %4s %4s %4s %8s\n", | ||||
jmgUnsubmitted Not Done Inline Actionswhy is this header printed for each worker? jmg: why is this header printed for each worker? | |||||
db_printf("%4u %4u %04x %8p\n" | "Worker", "HID", "Etype", "Flags", "Callback"); | ||||
TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) { | |||||
db_printf("%4u %4u %4u %04x %8p\n" | |||||
, CRYPTO_RETW_ID(ret_worker) | |||||
, (int) CRYPTO_SESID2HID(crp->crp_sid) | , (int) CRYPTO_SESID2HID(crp->crp_sid) | ||||
, crp->crp_etype | , crp->crp_etype | ||||
, crp->crp_flags | , crp->crp_flags | ||||
, crp->crp_callback | , crp->crp_callback | ||||
); | ); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
} | |||||
DB_SHOW_COMMAND(kcrypto, db_show_kcrypto) | DB_SHOW_COMMAND(kcrypto, db_show_kcrypto) | ||||
{ | { | ||||
struct cryptkop *krp; | struct cryptkop *krp; | ||||
struct crypto_ret_worker *ret_worker; | |||||
struct crypto_worker *worker; | |||||
db_show_drivers(); | db_show_drivers(); | ||||
db_printf("\n"); | db_printf("\n"); | ||||
db_printf("%4s %5s %4s %4s %8s %4s %8s\n", | db_printf("%7s %4s %5s %4s %4s %8s %4s %8s\n", | ||||
"Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback"); | "Worker", "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback"); | ||||
TAILQ_FOREACH(krp, &crp_kq, krp_next) { | CRYPTO_W_FOREACH(worker) { | ||||
db_printf("%4u %5u %4u %4u %08x %4u %8p\n" | TAILQ_FOREACH(krp, &worker->crp_kq, krp_next) { | ||||
db_printf("%4u %4u %5u %4u %4u %08x %4u %8p\n" | |||||
, CRYPTO_W_ID(worker) | |||||
, krp->krp_op | , krp->krp_op | ||||
, krp->krp_status | , krp->krp_status | ||||
, krp->krp_iparams, krp->krp_oparams | , krp->krp_iparams, krp->krp_oparams | ||||
, krp->krp_crid, krp->krp_hid | , krp->krp_crid, krp->krp_hid | ||||
, krp->krp_callback | , krp->krp_callback | ||||
); | ); | ||||
} | } | ||||
if (!TAILQ_EMPTY(&crp_ret_q)) { | } | ||||
db_printf("%4s %5s %8s %4s %8s\n", | CRYPTO_RETW_FOREACH(ret_worker) { | ||||
"Op", "Status", "CRID", "HID", "Callback"); | if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) { | ||||
TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) { | db_printf("%11s %4s %5s %8s %4s %8s\n", | ||||
jmgUnsubmitted Not Done Inline Actionsagain, why is this header printed for each worker? jmg: again, why is this header printed for each worker? | |||||
db_printf("%4u %5u %08x %4u %8p\n" | "Ret Worker", "Op", "Status", "CRID", "HID", "Callback"); | ||||
TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) { | |||||
db_printf("%4u %4u %5u %08x %4u %8p\n" | |||||
, CRYPTO_RETW_ID(ret_worker) | |||||
, krp->krp_op | , krp->krp_op | ||||
, krp->krp_status | , krp->krp_status | ||||
, krp->krp_crid, krp->krp_hid | , krp->krp_crid, krp->krp_hid | ||||
, krp->krp_callback | , krp->krp_callback | ||||
); | ); | ||||
} | |||||
} | } | ||||
} | } | ||||
} | } | ||||
#endif | #endif | ||||
int crypto_modevent(module_t mod, int type, void *unused); | int crypto_modevent(module_t mod, int type, void *unused); | ||||
/* | /* | ||||
Show All 27 Lines |
default should be 0, and then when zero, set to the number of cpu's so this gets used by default.