Changeset View
Changeset View
Standalone View
Standalone View
sys/opencrypto/crypto.c
Show First 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR | * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR | ||||
* IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY | * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY | ||||
* REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE | * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE | ||||
* MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR | * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR | ||||
* PURPOSE. | * PURPOSE. | ||||
*/ | */ | ||||
#define CRYPTO_TIMING /* enable timing support */ | |||||
#include "opt_compat.h" | #include "opt_compat.h" | ||||
#include "opt_ddb.h" | #include "opt_ddb.h" | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/eventhandler.h> | #include <sys/eventhandler.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/kthread.h> | #include <sys/kthread.h> | ||||
▲ Show 20 Lines • Show All 166 Lines • ▼ Show 20 Lines | |||||
static int crypto_kinvoke(struct cryptkop *krp); | static int crypto_kinvoke(struct cryptkop *krp); | ||||
static void crypto_task_invoke(void *ctx, int pending); | static void crypto_task_invoke(void *ctx, int pending); | ||||
static void crypto_batch_enqueue(struct cryptop *crp); | static void crypto_batch_enqueue(struct cryptop *crp); | ||||
static struct cryptostats cryptostats; | static struct cryptostats cryptostats; | ||||
SYSCTL_STRUCT(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, &cryptostats, | SYSCTL_STRUCT(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, &cryptostats, | ||||
cryptostats, "Crypto system statistics"); | cryptostats, "Crypto system statistics"); | ||||
#ifdef CRYPTO_TIMING | |||||
static int crypto_timing = 0; | |||||
SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, | |||||
&crypto_timing, 0, "Enable/disable crypto timing support"); | |||||
#endif | |||||
/* Try to avoid directly exposing the key buffer as a symbol */ | /* Try to avoid directly exposing the key buffer as a symbol */ | ||||
static struct keybuf *keybuf; | static struct keybuf *keybuf; | ||||
static struct keybuf empty_keybuf = { | static struct keybuf empty_keybuf = { | ||||
.kb_nents = 0 | .kb_nents = 0 | ||||
}; | }; | ||||
/* Obtain the key buffer from boot metadata */ | /* Obtain the key buffer from boot metadata */ | ||||
▲ Show 20 Lines • Show All 1,149 Lines • ▼ Show 20 Lines | crypto_dispatch(struct cryptop *crp) | ||||
int result; | int result; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
crp_sanity(crp); | crp_sanity(crp); | ||||
#endif | #endif | ||||
cryptostats.cs_ops++; | cryptostats.cs_ops++; | ||||
#ifdef CRYPTO_TIMING | |||||
if (crypto_timing) | |||||
binuptime(&crp->crp_tstamp); | |||||
#endif | |||||
crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num; | crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num; | ||||
if (CRYPTOP_ASYNC(crp)) { | if (CRYPTOP_ASYNC(crp)) { | ||||
if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) { | if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) { | ||||
struct crypto_ret_worker *ret_worker; | struct crypto_ret_worker *ret_worker; | ||||
ret_worker = CRYPTO_RETW(crp->crp_retw_id); | ret_worker = CRYPTO_RETW(crp->crp_retw_id); | ||||
▲ Show 20 Lines • Show All 217 Lines • ▼ Show 20 Lines | if (error == ERESTART) { | ||||
CRYPTO_DRIVER_UNLOCK(); | CRYPTO_DRIVER_UNLOCK(); | ||||
return (error); | return (error); | ||||
} | } | ||||
KASSERT(error == 0, ("error %d returned from crypto_kprocess", error)); | KASSERT(error == 0, ("error %d returned from crypto_kprocess", error)); | ||||
return (0); | return (0); | ||||
} | } | ||||
#ifdef CRYPTO_TIMING | |||||
static void | static void | ||||
crypto_tstat(struct cryptotstat *ts, struct bintime *bt) | |||||
{ | |||||
struct bintime now, delta; | |||||
struct timespec t; | |||||
uint64_t u; | |||||
binuptime(&now); | |||||
u = now.frac; | |||||
delta.frac = now.frac - bt->frac; | |||||
delta.sec = now.sec - bt->sec; | |||||
if (u < delta.frac) | |||||
delta.sec--; | |||||
bintime2timespec(&delta, &t); | |||||
timespecadd(&ts->acc, &t, &ts->acc); | |||||
if (timespeccmp(&t, &ts->min, <)) | |||||
ts->min = t; | |||||
if (timespeccmp(&t, &ts->max, >)) | |||||
ts->max = t; | |||||
ts->count++; | |||||
*bt = now; | |||||
} | |||||
#endif | |||||
static void | |||||
crypto_task_invoke(void *ctx, int pending) | crypto_task_invoke(void *ctx, int pending) | ||||
{ | { | ||||
struct cryptocap *cap; | struct cryptocap *cap; | ||||
struct cryptop *crp; | struct cryptop *crp; | ||||
int result; | int result; | ||||
crp = (struct cryptop *)ctx; | crp = (struct cryptop *)ctx; | ||||
cap = crp->crp_session->cap; | cap = crp->crp_session->cap; | ||||
Show All 10 Lines | |||||
{ | { | ||||
KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); | KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); | ||||
KASSERT(crp->crp_callback != NULL, | KASSERT(crp->crp_callback != NULL, | ||||
("%s: crp->crp_callback == NULL", __func__)); | ("%s: crp->crp_callback == NULL", __func__)); | ||||
KASSERT(crp->crp_session != NULL, | KASSERT(crp->crp_session != NULL, | ||||
("%s: crp->crp_session == NULL", __func__)); | ("%s: crp->crp_session == NULL", __func__)); | ||||
#ifdef CRYPTO_TIMING | |||||
if (crypto_timing) | |||||
crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); | |||||
#endif | |||||
if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { | if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { | ||||
struct crypto_session_params csp; | struct crypto_session_params csp; | ||||
crypto_session_t nses; | crypto_session_t nses; | ||||
/* | /* | ||||
* Driver has unregistered; migrate the session and return | * Driver has unregistered; migrate the session and return | ||||
* an error to the caller so they'll resubmit the op. | * an error to the caller so they'll resubmit the op. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
crypto_done(struct cryptop *crp) | crypto_done(struct cryptop *crp) | ||||
{ | { | ||||
KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, | KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, | ||||
("crypto_done: op already done, flags 0x%x", crp->crp_flags)); | ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); | ||||
crp->crp_flags |= CRYPTO_F_DONE; | crp->crp_flags |= CRYPTO_F_DONE; | ||||
if (crp->crp_etype != 0) | if (crp->crp_etype != 0) | ||||
cryptostats.cs_errs++; | cryptostats.cs_errs++; | ||||
#ifdef CRYPTO_TIMING | |||||
if (crypto_timing) | |||||
crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); | |||||
#endif | |||||
/* | /* | ||||
* CBIMM means unconditionally do the callback immediately; | * CBIMM means unconditionally do the callback immediately; | ||||
* CBIFSYNC means do the callback immediately only if the | * CBIFSYNC means do the callback immediately only if the | ||||
* operation was done synchronously. Both are used to avoid | * operation was done synchronously. Both are used to avoid | ||||
* doing extraneous context switches; the latter is mostly | * doing extraneous context switches; the latter is mostly | ||||
* used with the software crypto driver. | * used with the software crypto driver. | ||||
*/ | */ | ||||
if (!CRYPTOP_ASYNC_KEEPORDER(crp) && | if (!CRYPTOP_ASYNC_KEEPORDER(crp) && | ||||
((crp->crp_flags & CRYPTO_F_CBIMM) || | ((crp->crp_flags & CRYPTO_F_CBIMM) || | ||||
((crp->crp_flags & CRYPTO_F_CBIFSYNC) && | ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && | ||||
(crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) { | (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) { | ||||
/* | /* | ||||
* Do the callback directly. This is ok when the | * Do the callback directly. This is ok when the | ||||
* callback routine does very little (e.g. the | * callback routine does very little (e.g. the | ||||
* /dev/crypto callback method just does a wakeup). | * /dev/crypto callback method just does a wakeup). | ||||
*/ | */ | ||||
#ifdef CRYPTO_TIMING | |||||
if (crypto_timing) { | |||||
/* | |||||
* NB: We must copy the timestamp before | |||||
* doing the callback as the cryptop is | |||||
* likely to be reclaimed. | |||||
*/ | |||||
struct bintime t = crp->crp_tstamp; | |||||
crypto_tstat(&cryptostats.cs_cb, &t); | |||||
crp->crp_callback(crp); | crp->crp_callback(crp); | ||||
crypto_tstat(&cryptostats.cs_finis, &t); | |||||
} else | |||||
#endif | |||||
crp->crp_callback(crp); | |||||
} else { | } else { | ||||
struct crypto_ret_worker *ret_worker; | struct crypto_ret_worker *ret_worker; | ||||
bool wake; | bool wake; | ||||
ret_worker = CRYPTO_RETW(crp->crp_retw_id); | ret_worker = CRYPTO_RETW(crp->crp_retw_id); | ||||
wake = false; | wake = false; | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 283 Lines • ▼ Show 20 Lines | for (;;) { | ||||
if (krpt != NULL) | if (krpt != NULL) | ||||
TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next); | TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next); | ||||
if (crpt != NULL || krpt != NULL) { | if (crpt != NULL || krpt != NULL) { | ||||
CRYPTO_RETW_UNLOCK(ret_worker); | CRYPTO_RETW_UNLOCK(ret_worker); | ||||
/* | /* | ||||
* Run callbacks unlocked. | * Run callbacks unlocked. | ||||
*/ | */ | ||||
if (crpt != NULL) { | if (crpt != NULL) | ||||
#ifdef CRYPTO_TIMING | |||||
if (crypto_timing) { | |||||
/* | |||||
* NB: We must copy the timestamp before | |||||
* doing the callback as the cryptop is | |||||
* likely to be reclaimed. | |||||
*/ | |||||
struct bintime t = crpt->crp_tstamp; | |||||
crypto_tstat(&cryptostats.cs_cb, &t); | |||||
crpt->crp_callback(crpt); | crpt->crp_callback(crpt); | ||||
crypto_tstat(&cryptostats.cs_finis, &t); | |||||
} else | |||||
#endif | |||||
crpt->crp_callback(crpt); | |||||
} | |||||
if (krpt != NULL) | if (krpt != NULL) | ||||
krpt->krp_callback(krpt); | krpt->krp_callback(krpt); | ||||
CRYPTO_RETW_LOCK(ret_worker); | CRYPTO_RETW_LOCK(ret_worker); | ||||
} else { | } else { | ||||
/* | /* | ||||
* Nothing more to be processed. Sleep until we're | * Nothing more to be processed. Sleep until we're | ||||
* woken because there are more returns to process. | * woken because there are more returns to process. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 146 Lines • Show Last 20 Lines |