Changeset View
Changeset View
Standalone View
Standalone View
sys/opencrypto/crypto.c
Show First 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
#define CRYPTO_RETW(i) (&crypto_ret_workers[i]) | #define CRYPTO_RETW(i) (&crypto_ret_workers[i]) | ||||
#define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers) | #define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers) | ||||
#define FOREACH_CRYPTO_RETW(w) \ | #define FOREACH_CRYPTO_RETW(w) \ | ||||
for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w) | for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w) | ||||
#define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx) | #define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx) | ||||
#define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx) | #define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx) | ||||
#define CRYPTO_RETW_EMPTY(w) \ | |||||
(TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q)) | |||||
static int crypto_workers_num = 0; | static int crypto_workers_num = 0; | ||||
SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN, | SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN, | ||||
&crypto_workers_num, 0, | &crypto_workers_num, 0, | ||||
"Number of crypto workers used to dispatch crypto jobs"); | "Number of crypto workers used to dispatch crypto jobs"); | ||||
#ifdef COMPAT_FREEBSD12 | #ifdef COMPAT_FREEBSD12 | ||||
SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN, | SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN, | ||||
&crypto_workers_num, 0, | &crypto_workers_num, 0, | ||||
▲ Show 20 Lines • Show All 1,200 Lines • ▼ Show 20 Lines | KASSERT(csp->csp_cipher_key != NULL || | ||||
("cipher request without a key")); | ("cipher request without a key")); | ||||
if (csp->csp_auth_klen != 0) | if (csp->csp_auth_klen != 0) | ||||
KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL, | KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL, | ||||
("auth request without a key")); | ("auth request without a key")); | ||||
KASSERT(crp->crp_callback != NULL, ("incoming crp without callback")); | KASSERT(crp->crp_callback != NULL, ("incoming crp without callback")); | ||||
} | } | ||||
#endif | #endif | ||||
/* | static int | ||||
* Add a crypto request to a queue, to be processed by the kernel thread. | crypto_dispatch_one(struct cryptop *crp, int hint) | ||||
*/ | |||||
int | |||||
crypto_dispatch(struct cryptop *crp) | |||||
{ | { | ||||
struct cryptocap *cap; | struct cryptocap *cap; | ||||
int result; | int result; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
crp_sanity(crp); | crp_sanity(crp); | ||||
#endif | #endif | ||||
CRYPTOSTAT_INC(cs_ops); | CRYPTOSTAT_INC(cs_ops); | ||||
crp->crp_retw_id = crp->crp_session->id % crypto_workers_num; | crp->crp_retw_id = crp->crp_session->id % crypto_workers_num; | ||||
if (CRYPTOP_ASYNC(crp)) { | /* | ||||
if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) { | * Caller marked the request to be processed immediately; dispatch it | ||||
* directly to the driver unless the driver is currently blocked, in | |||||
* which case it is queued for deferred dispatch. | |||||
*/ | |||||
cap = crp->crp_session->cap; | |||||
if (!atomic_load_int(&cap->cc_qblocked)) { | |||||
result = crypto_invoke(cap, crp, hint); | |||||
if (result != ERESTART) | |||||
return (result); | |||||
/* | |||||
* The driver ran out of resources, put the request on the | |||||
* queue. | |||||
*/ | |||||
} | |||||
crypto_batch_enqueue(crp); | |||||
return (0); | |||||
} | |||||
int | |||||
crypto_dispatch(struct cryptop *crp) | |||||
{ | |||||
return (crypto_dispatch_one(crp, 0)); | |||||
} | |||||
int | |||||
crypto_dispatch_async(struct cryptop *crp, int flags) | |||||
{ | |||||
struct crypto_ret_worker *ret_worker; | struct crypto_ret_worker *ret_worker; | ||||
ret_worker = CRYPTO_RETW(crp->crp_retw_id); | if (!CRYPTO_SESS_SYNC(crp->crp_session)) { | ||||
/* | |||||
* The driver issues completions asynchonously, don't bother | |||||
* deferring dispatch to a worker thread. | |||||
*/ | |||||
return (crypto_dispatch(crp)); | |||||
} | |||||
#ifdef INVARIANTS | |||||
crp_sanity(crp); | |||||
#endif | |||||
CRYPTOSTAT_INC(cs_ops); | |||||
crp->crp_retw_id = crp->crp_session->id % crypto_workers_num; | |||||
if ((flags & CRYPTO_ASYNC_ORDERED) != 0) { | |||||
crp->crp_flags |= CRYPTO_F_ASYNC_ORDERED; | |||||
ret_worker = CRYPTO_RETW(crp->crp_retw_id); | |||||
CRYPTO_RETW_LOCK(ret_worker); | CRYPTO_RETW_LOCK(ret_worker); | ||||
crp->crp_seq = ret_worker->reorder_ops++; | crp->crp_seq = ret_worker->reorder_ops++; | ||||
CRYPTO_RETW_UNLOCK(ret_worker); | CRYPTO_RETW_UNLOCK(ret_worker); | ||||
} | } | ||||
TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp); | TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp); | ||||
taskqueue_enqueue(crypto_tq, &crp->crp_task); | taskqueue_enqueue(crypto_tq, &crp->crp_task); | ||||
return (0); | return (0); | ||||
} | } | ||||
if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { | void | ||||
/* | crypto_dispatch_batch(struct cryptopq *crpq, int flags) | ||||
* Caller marked the request to be processed | { | ||||
* immediately; dispatch it directly to the | struct cryptop *crp; | ||||
* driver unless the driver is currently blocked. | int hint; | ||||
*/ | |||||
cap = crp->crp_session->cap; | while ((crp = TAILQ_FIRST(crpq)) != NULL) { | ||||
if (!cap->cc_qblocked) { | hint = TAILQ_NEXT(crp, crp_next) != NULL ? CRYPTO_HINT_MORE : 0; | ||||
result = crypto_invoke(cap, crp, 0); | TAILQ_REMOVE(crpq, crp, crp_next); | ||||
if (result != ERESTART) | if (crypto_dispatch_one(crp, hint) != 0) | ||||
return (result); | |||||
/* | |||||
* The driver ran out of resources, put the request on | |||||
* the queue. | |||||
*/ | |||||
} | |||||
} | |||||
crypto_batch_enqueue(crp); | crypto_batch_enqueue(crp); | ||||
return 0; | |||||
} | } | ||||
} | |||||
void | static void | ||||
crypto_batch_enqueue(struct cryptop *crp) | crypto_batch_enqueue(struct cryptop *crp) | ||||
{ | { | ||||
CRYPTO_Q_LOCK(); | CRYPTO_Q_LOCK(); | ||||
TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); | TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); | ||||
if (crp_sleep) | if (crp_sleep) | ||||
wakeup_one(&crp_q); | wakeup_one(&crp_q); | ||||
CRYPTO_Q_UNLOCK(); | CRYPTO_Q_UNLOCK(); | ||||
▲ Show 20 Lines • Show All 337 Lines • ▼ Show 20 Lines | crypto_done(struct cryptop *crp) | ||||
/* | /* | ||||
* CBIMM means unconditionally do the callback immediately; | * CBIMM means unconditionally do the callback immediately; | ||||
* CBIFSYNC means do the callback immediately only if the | * CBIFSYNC means do the callback immediately only if the | ||||
* operation was done synchronously. Both are used to avoid | * operation was done synchronously. Both are used to avoid | ||||
* doing extraneous context switches; the latter is mostly | * doing extraneous context switches; the latter is mostly | ||||
* used with the software crypto driver. | * used with the software crypto driver. | ||||
*/ | */ | ||||
if (!CRYPTOP_ASYNC_KEEPORDER(crp) && | if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) == 0 && | ||||
((crp->crp_flags & CRYPTO_F_CBIMM) || | ((crp->crp_flags & CRYPTO_F_CBIMM) != 0 || | ||||
((crp->crp_flags & CRYPTO_F_CBIFSYNC) && | ((crp->crp_flags & CRYPTO_F_CBIFSYNC) != 0 && | ||||
(crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) { | CRYPTO_SESS_SYNC(crp->crp_session)))) { | ||||
/* | /* | ||||
* Do the callback directly. This is ok when the | * Do the callback directly. This is ok when the | ||||
* callback routine does very little (e.g. the | * callback routine does very little (e.g. the | ||||
* /dev/crypto callback method just does a wakeup). | * /dev/crypto callback method just does a wakeup). | ||||
*/ | */ | ||||
crp->crp_callback(crp); | crp->crp_callback(crp); | ||||
} else { | } else { | ||||
struct crypto_ret_worker *ret_worker; | struct crypto_ret_worker *ret_worker; | ||||
bool wake; | bool wake; | ||||
ret_worker = CRYPTO_RETW(crp->crp_retw_id); | ret_worker = CRYPTO_RETW(crp->crp_retw_id); | ||||
wake = false; | |||||
/* | /* | ||||
* Normal case; queue the callback for the thread. | * Normal case; queue the callback for the thread. | ||||
*/ | */ | ||||
CRYPTO_RETW_LOCK(ret_worker); | CRYPTO_RETW_LOCK(ret_worker); | ||||
if (CRYPTOP_ASYNC_KEEPORDER(crp)) { | if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) != 0) { | ||||
struct cryptop *tmp; | struct cryptop *tmp; | ||||
TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q, | TAILQ_FOREACH_REVERSE(tmp, | ||||
cryptop_q, crp_next) { | &ret_worker->crp_ordered_ret_q, cryptop_q, | ||||
crp_next) { | |||||
if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) { | if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) { | ||||
TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q, | TAILQ_INSERT_AFTER( | ||||
tmp, crp, crp_next); | &ret_worker->crp_ordered_ret_q, tmp, | ||||
crp, crp_next); | |||||
break; | break; | ||||
} | } | ||||
} | } | ||||
if (tmp == NULL) { | if (tmp == NULL) { | ||||
TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q, | TAILQ_INSERT_HEAD( | ||||
crp, crp_next); | &ret_worker->crp_ordered_ret_q, crp, | ||||
crp_next); | |||||
} | } | ||||
if (crp->crp_seq == ret_worker->reorder_cur_seq) | wake = crp->crp_seq == ret_worker->reorder_cur_seq; | ||||
wake = true; | } else { | ||||
wake = TAILQ_EMPTY(&ret_worker->crp_ret_q); | |||||
TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, | |||||
crp_next); | |||||
} | } | ||||
else { | |||||
if (CRYPTO_RETW_EMPTY(ret_worker)) | |||||
wake = true; | |||||
TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next); | |||||
} | |||||
if (wake) | if (wake) | ||||
wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ | wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ | ||||
CRYPTO_RETW_UNLOCK(ret_worker); | CRYPTO_RETW_UNLOCK(ret_worker); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Invoke the callback on behalf of the driver. | * Invoke the callback on behalf of the driver. | ||||
Show All 17 Lines | if (cap != NULL) { | ||||
CRYPTO_DRIVER_UNLOCK(); | CRYPTO_DRIVER_UNLOCK(); | ||||
krp->krp_cap = NULL; | krp->krp_cap = NULL; | ||||
cap_rele(cap); | cap_rele(cap); | ||||
} | } | ||||
ret_worker = CRYPTO_RETW(0); | ret_worker = CRYPTO_RETW(0); | ||||
CRYPTO_RETW_LOCK(ret_worker); | CRYPTO_RETW_LOCK(ret_worker); | ||||
if (CRYPTO_RETW_EMPTY(ret_worker)) | if (TAILQ_EMPTY(&ret_worker->crp_ret_kq)) | ||||
wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ | wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ | ||||
TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next); | TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next); | ||||
CRYPTO_RETW_UNLOCK(ret_worker); | CRYPTO_RETW_UNLOCK(ret_worker); | ||||
} | } | ||||
int | int | ||||
crypto_getfeat(int *featp) | crypto_getfeat(int *featp) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines | TAILQ_FOREACH(crp, &crp_q, crp_next) { | ||||
* regardless whether its for the same | * regardless whether its for the same | ||||
* driver or not. We could keep | * driver or not. We could keep | ||||
* searching the queue but it might be | * searching the queue but it might be | ||||
* better to just use a per-driver | * better to just use a per-driver | ||||
* queue instead. | * queue instead. | ||||
*/ | */ | ||||
if (submit->crp_session->cap == cap) | if (submit->crp_session->cap == cap) | ||||
hint = CRYPTO_HINT_MORE; | hint = CRYPTO_HINT_MORE; | ||||
break; | |||||
} else { | } else { | ||||
submit = crp; | submit = crp; | ||||
if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) | |||||
break; | |||||
/* keep scanning for more are q'd */ | |||||
} | } | ||||
break; | |||||
} | } | ||||
} | } | ||||
if (submit != NULL) { | if (submit != NULL) { | ||||
TAILQ_REMOVE(&crp_q, submit, crp_next); | TAILQ_REMOVE(&crp_q, submit, crp_next); | ||||
cap = submit->crp_session->cap; | cap = submit->crp_session->cap; | ||||
KASSERT(cap != NULL, ("%s:%u Driver disappeared.", | KASSERT(cap != NULL, ("%s:%u Driver disappeared.", | ||||
__func__, __LINE__)); | __func__, __LINE__)); | ||||
CRYPTO_Q_UNLOCK(); | CRYPTO_Q_UNLOCK(); | ||||
▲ Show 20 Lines • Show All 276 Lines • Show Last 20 Lines |