Index: sys/kgssapi/krb5/kcrypto_aes.c =================================================================== --- sys/kgssapi/krb5/kcrypto_aes.c +++ sys/kgssapi/krb5/kcrypto_aes.c @@ -122,7 +122,7 @@ int error; struct aes_state *as = (struct aes_state *) crp->crp_opaque; - if (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC) + if (CRYPTO_SESS_SYNC(crp->crp_session)) return (0); error = crp->crp_etype; @@ -165,7 +165,7 @@ error = crypto_dispatch(crp); - if ((crypto_ses2caps(as->as_session_aes) & CRYPTOCAP_F_SYNC) == 0) { + if (!CRYPTO_SESS_SYNC(as->as_session_aes)) { mtx_lock(&as->as_lock); if (!error && !(crp->crp_flags & CRYPTO_F_DONE)) error = msleep(crp, &as->as_lock, 0, "gssaes", 0); @@ -335,7 +335,7 @@ error = crypto_dispatch(crp); - if ((crypto_ses2caps(as->as_session_sha1) & CRYPTOCAP_F_SYNC) == 0) { + if (!CRYPTO_SESS_SYNC(as->as_session_sha1)) { mtx_lock(&as->as_lock); if (!error && !(crp->crp_flags & CRYPTO_F_DONE)) error = msleep(crp, &as->as_lock, 0, "gssaes", 0); Index: sys/netipsec/xform_ah.c =================================================================== --- sys/netipsec/xform_ah.c +++ sys/netipsec/xform_ah.c @@ -652,8 +652,6 @@ /* Crypto operation descriptor. */ crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; crp->crp_flags = CRYPTO_F_CBIFSYNC; - if (V_async_crypto) - crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER; crypto_use_mbuf(crp, m); crp->crp_callback = ah_input_cb; crp->crp_opaque = xd; @@ -671,7 +669,10 @@ xd->skip = skip; xd->cryptoid = cryptoid; xd->vnet = curvnet; - return (crypto_dispatch(crp)); + if (V_async_crypto) + return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); + else + return (crypto_dispatch(crp)); bad: m_freem(m); key_freesav(&sav); @@ -1036,8 +1037,6 @@ /* Crypto operation descriptor. */ crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; crp->crp_flags = CRYPTO_F_CBIFSYNC; - if (V_async_crypto) - crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER; crypto_use_mbuf(crp, m); crp->crp_callback = ah_output_cb; crp->crp_opaque = xd; @@ -1055,7 +1054,10 @@ xd->cryptoid = cryptoid; xd->vnet = curvnet; - return crypto_dispatch(crp); + if (V_async_crypto) + return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); + else + return (crypto_dispatch(crp)); bad: if (m) m_freem(m); Index: sys/netipsec/xform_esp.c =================================================================== --- sys/netipsec/xform_esp.c +++ sys/netipsec/xform_esp.c @@ -406,8 +406,6 @@ /* Crypto operation descriptor */ crp->crp_flags = CRYPTO_F_CBIFSYNC; - if (V_async_crypto) - crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER; crypto_use_mbuf(crp, m); crp->crp_callback = esp_input_cb; crp->crp_opaque = xd; @@ -460,7 +458,10 @@ } else if (sav->ivlen != 0) crp->crp_iv_start = skip + hlen - sav->ivlen; - return (crypto_dispatch(crp)); + if (V_async_crypto) + return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); + else + return (crypto_dispatch(crp)); crp_aad_fail: free(xd, M_XDATA); @@ -895,8 +896,6 @@ /* Crypto operation descriptor. */ crp->crp_flags |= CRYPTO_F_CBIFSYNC; - if (V_async_crypto) - crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER; crypto_use_mbuf(crp, m); crp->crp_callback = esp_output_cb; crp->crp_opaque = xd; @@ -944,7 +943,10 @@ crp->crp_digest_start = m->m_pkthdr.len - alen; } - return crypto_dispatch(crp); + if (V_async_crypto) + return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); + else + return (crypto_dispatch(crp)); crp_aad_fail: free(xd, M_XDATA); Index: sys/opencrypto/crypto.c =================================================================== --- sys/opencrypto/crypto.c +++ sys/opencrypto/crypto.c @@ -187,8 +187,6 @@ #define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx) #define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx) -#define CRYPTO_RETW_EMPTY(w) \ - (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q)) static int crypto_workers_num = 0; SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN, @@ -1416,27 +1414,9 @@ #ifdef INVARIANTS crp_sanity(crp); #endif - CRYPTOSTAT_INC(cs_ops); crp->crp_retw_id = crp->crp_session->id % crypto_workers_num; - - if (CRYPTOP_ASYNC(crp)) { - if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) { - struct crypto_ret_worker *ret_worker; - - ret_worker = CRYPTO_RETW(crp->crp_retw_id); - - CRYPTO_RETW_LOCK(ret_worker); - crp->crp_seq = ret_worker->reorder_ops++; - CRYPTO_RETW_UNLOCK(ret_worker); - } - - TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp); - taskqueue_enqueue(crypto_tq, &crp->crp_task); - return (0); - } - if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { /* * Caller marked the request to be processed @@ -1455,7 +1435,38 @@ } } crypto_batch_enqueue(crp); - return 0; + return (0); +} + +int +crypto_dispatch_async(struct cryptop *crp, int flags) +{ + struct crypto_ret_worker *ret_worker; + + if (!CRYPTO_SESS_SYNC(crp->crp_session)) { + /* + * The driver issues completions asynchonously, don't bother + * deferring dispatch to a worker thread. + */ + return (crypto_dispatch(crp)); + } + +#ifdef INVARIANTS + crp_sanity(crp); +#endif + CRYPTOSTAT_INC(cs_ops); + + crp->crp_retw_id = crp->crp_session->id % crypto_workers_num; + if ((flags & CRYPTO_ASYNC_ORDERED) != 0) { + crp->crp_flags |= CRYPTO_F_ASYNC_ORDERED; + ret_worker = CRYPTO_RETW(crp->crp_retw_id); + CRYPTO_RETW_LOCK(ret_worker); + crp->crp_seq = ret_worker->reorder_ops++; + CRYPTO_RETW_UNLOCK(ret_worker); + } + TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp); + taskqueue_enqueue(crypto_tq, &crp->crp_task); + return (0); } void @@ -1812,10 +1823,10 @@ * doing extraneous context switches; the latter is mostly * used with the software crypto driver. */ - if (!CRYPTOP_ASYNC_KEEPORDER(crp) && - ((crp->crp_flags & CRYPTO_F_CBIMM) || - ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && - (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) { + if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) == 0 && + ((crp->crp_flags & CRYPTO_F_CBIMM) != 0 || + ((crp->crp_flags & CRYPTO_F_CBIFSYNC) != 0 && + CRYPTO_SESS_SYNC(crp->crp_session)))) { /* * Do the callback directly. This is ok when the * callback routine does very little (e.g. the @@ -1827,36 +1838,35 @@ bool wake; ret_worker = CRYPTO_RETW(crp->crp_retw_id); - wake = false; /* * Normal case; queue the callback for the thread. */ CRYPTO_RETW_LOCK(ret_worker); - if (CRYPTOP_ASYNC_KEEPORDER(crp)) { + if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) != 0) { struct cryptop *tmp; - TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q, - cryptop_q, crp_next) { + TAILQ_FOREACH_REVERSE(tmp, + &ret_worker->crp_ordered_ret_q, cryptop_q, + crp_next) { if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) { - TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q, - tmp, crp, crp_next); + TAILQ_INSERT_AFTER( + &ret_worker->crp_ordered_ret_q, tmp, + crp, crp_next); break; } } if (tmp == NULL) { - TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q, - crp, crp_next); + TAILQ_INSERT_HEAD( + &ret_worker->crp_ordered_ret_q, crp, + crp_next); } - if (crp->crp_seq == ret_worker->reorder_cur_seq) - wake = true; - } - else { - if (CRYPTO_RETW_EMPTY(ret_worker)) - wake = true; - - TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next); + wake = crp->crp_seq == ret_worker->reorder_cur_seq; + } else { + wake = TAILQ_EMPTY(&ret_worker->crp_ret_q); + TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, + crp_next); } if (wake) @@ -1892,7 +1902,7 @@ ret_worker = CRYPTO_RETW(0); CRYPTO_RETW_LOCK(ret_worker); - if (CRYPTO_RETW_EMPTY(ret_worker)) + if (TAILQ_EMPTY(&ret_worker->crp_ret_kq)) wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next); CRYPTO_RETW_UNLOCK(ret_worker); Index: sys/opencrypto/cryptodev.h =================================================================== --- sys/opencrypto/cryptodev.h +++ sys/opencrypto/cryptodev.h @@ -459,14 +459,7 @@ #define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */ #define CRYPTO_F_DONE 0x0020 /* Operation completed */ #define CRYPTO_F_CBIFSYNC 0x0040 /* Do CBIMM if op is synchronous */ -#define CRYPTO_F_ASYNC 0x0080 /* Dispatch crypto jobs on several threads - * if op is synchronous - */ -#define CRYPTO_F_ASYNC_KEEPORDER 0x0100 /* - * Dispatch the crypto jobs in the same - * order there are submitted. Applied only - * if CRYPTO_F_ASYNC flags is set - */ +#define CRYPTO_F_ASYNC_ORDERED 0x0100 /* Completions must happen in order */ #define CRYPTO_F_IV_SEPARATE 0x0200 /* Use crp_iv[] as IV. */ int crp_op; @@ -587,12 +580,6 @@ _crypto_use_uio(&crp->crp_obuf, uio); } -#define CRYPTOP_ASYNC(crp) \ - (((crp)->crp_flags & CRYPTO_F_ASYNC) && \ - crypto_ses2caps((crp)->crp_session) & CRYPTOCAP_F_SYNC) -#define CRYPTOP_ASYNC_KEEPORDER(crp) \ - (CRYPTOP_ASYNC(crp) && \ - (crp)->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) #define CRYPTO_HAS_OUTPUT_BUFFER(crp) \ ((crp)->crp_obuf.cb_type != CRYPTO_BUF_NONE) @@ -642,6 +629,8 @@ #define CRYPTOCAP_F_SOFTWARE CRYPTO_FLAG_SOFTWARE #define CRYPTOCAP_F_SYNC 0x04000000 /* operates synchronously */ #define CRYPTOCAP_F_ACCEL_SOFTWARE 0x08000000 +#define CRYPTO_SESS_SYNC(sess) \ + ((crypto_ses2caps(sess) & CRYPTOCAP_F_SYNC) != 0) extern int32_t crypto_get_driverid(device_t dev, size_t session_size, int flags); extern int crypto_find_driver(const char *); @@ -650,6 +639,8 @@ extern int crypto_kregister(uint32_t, int, uint32_t); extern int crypto_unregister_all(uint32_t driverid); extern int crypto_dispatch(struct cryptop *crp); +#define CRYPTO_ASYNC_ORDERED 0x1 /* complete in order dispatched */ +extern int crypto_dispatch_async(struct cryptop *crp, int flags); extern int crypto_kdispatch(struct cryptkop *); #define CRYPTO_SYMQ 0x1 #define CRYPTO_ASYMQ 0x2