Index: sys/contrib/openzfs/module/os/freebsd/zfs/crypto_os.c =================================================================== --- sys/contrib/openzfs/module/os/freebsd/zfs/crypto_os.c +++ sys/contrib/openzfs/module/os/freebsd/zfs/crypto_os.c @@ -158,16 +158,17 @@ } static int -zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) +zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) { int error; + error = 0; crp->crp_opaque = session; crp->crp_callback = freebsd_zfs_crypt_done; for (;;) { - error = crypto_dispatch(crp); - if (error) - break; + crypto_dispatch(crp); + + /* XXX-MJ this is wasteful for synchronous sessions */ mtx_lock(&session->fs_lock); while (session->fs_done == false) msleep(crp, &session->fs_lock, 0, Index: sys/dev/cxgbe/crypto/t4_crypto.c =================================================================== --- sys/dev/cxgbe/crypto/t4_crypto.c +++ sys/dev/cxgbe/crypto/t4_crypto.c @@ -1775,7 +1775,6 @@ ccr_soft(struct ccr_session *s, struct cryptop *crp) { struct cryptop *new; - int error; new = crypto_clonereq(crp, s->sw_session, M_NOWAIT); if (new == NULL) { @@ -1792,11 +1791,7 @@ */ new->crp_opaque = crp; new->crp_callback = ccr_soft_done; - error = crypto_dispatch_async(new, CRYPTO_ASYNC_ORDERED); - if (error != 0) { - crp->crp_etype = error; - crypto_done(crp); - } + crypto_dispatch_async(new, CRYPTO_ASYNC_ORDERED); } static void Index: sys/geom/eli/g_eli.c =================================================================== --- sys/geom/eli/g_eli.c +++ sys/geom/eli/g_eli.c @@ -244,7 +244,6 @@ struct g_eli_softc *sc; struct g_eli_worker *wr; struct bio *bp; - int error; bp = (struct bio *)crp->crp_opaque; sc = bp->bio_to->geom->softc; @@ -259,12 +258,8 @@ crp->crp_session); wr->w_sid = crp->crp_session; crp->crp_etype = 0; - error = crypto_dispatch(crp); - if (error == 0) - return (0); - G_ELI_DEBUG(1, "%s: crypto_dispatch() returned %d.", __func__, error); - crp->crp_etype = error; - return (error); + crypto_dispatch(crp); + return (0); } static void Index: sys/geom/eli/g_eli_crypto.c =================================================================== --- sys/geom/eli/g_eli_crypto.c +++ sys/geom/eli/g_eli_crypto.c @@ -94,12 +94,9 @@ crp->crp_callback = g_eli_crypto_done; crypto_use_buf(crp, data, datasize); - error = crypto_dispatch(crp); - if (error == 0) { - while (crp->crp_opaque == NULL) - tsleep(crp, PRIBIO, "geli", hz / 5); - error = crp->crp_etype; - } + crypto_dispatch(crp); + while (crp->crp_opaque == NULL) + tsleep(crp, PRIBIO, "geli", hz / 5); crypto_freereq(crp); crypto_freesession(sid); Index: sys/geom/eli/g_eli_integrity.c =================================================================== --- sys/geom/eli/g_eli_integrity.c +++ sys/geom/eli/g_eli_integrity.c @@ -456,7 +456,6 @@ u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; off_t dstoff; u_char *p, *data, *authkey, *plaindata; - int error __diagused; bool batch; G_ELI_LOGREQ(3, bp, "%s", __func__); @@ -579,9 +578,7 @@ if (batch) { TAILQ_INSERT_TAIL(&crpq, crp, crp_next); } else { - error = crypto_dispatch(crp); - KASSERT(error == 0, - ("crypto_dispatch() failed (error=%d)", error)); + crypto_dispatch(crp); } } Index: sys/geom/eli/g_eli_privacy.c =================================================================== --- sys/geom/eli/g_eli_privacy.c +++ sys/geom/eli/g_eli_privacy.c @@ -263,7 +263,7 @@ u_int i, nsec, secsize; off_t dstoff; u_char *data = NULL; - int error __diagused, pages_offset; + int pages_offset; bool batch; G_ELI_LOGREQ(3, bp, "%s", __func__); @@ -351,9 +351,7 @@ if (batch) { TAILQ_INSERT_TAIL(&crpq, crp, crp_next); } else { - error = crypto_dispatch(crp); - KASSERT(error == 0, - ("crypto_dispatch() failed (error=%d)", error)); + crypto_dispatch(crp); } } Index: sys/kgssapi/krb5/kcrypto_aes.c =================================================================== --- sys/kgssapi/krb5/kcrypto_aes.c +++ sys/kgssapi/krb5/kcrypto_aes.c @@ -119,15 +119,17 @@ static int aes_crypto_cb(struct cryptop *crp) { - int error; struct aes_state *as = (struct aes_state *) crp->crp_opaque; + int error; if (CRYPTO_SESS_SYNC(crp->crp_session)) return (0); error = crp->crp_etype; - if (error == EAGAIN) - error = crypto_dispatch(crp); + if (error == EAGAIN) { + crypto_dispatch(crp); + error = 0; + } mtx_lock(&as->as_lock); if (error || (crp->crp_flags & CRYPTO_F_DONE)) wakeup(crp); @@ -142,7 +144,6 @@ { struct aes_state *as = ks->ks_priv; struct cryptop *crp; - int error; crp = crypto_getreq(as->as_session_aes, M_WAITOK); @@ -162,13 +163,12 @@ crypto_use_buf(crp, buf, skip + len); crp->crp_opaque = as; crp->crp_callback = aes_crypto_cb; - - error = crypto_dispatch(crp); + crypto_dispatch(crp); if (!CRYPTO_SESS_SYNC(as->as_session_aes)) { mtx_lock(&as->as_lock); - if (!error && !(crp->crp_flags & CRYPTO_F_DONE)) - error = msleep(crp, &as->as_lock, 0, "gssaes", 0); + if (!(crp->crp_flags & CRYPTO_F_DONE)) + (void)msleep(crp, &as->as_lock, 0, "gssaes", 0); mtx_unlock(&as->as_lock); } @@ -321,7 +321,6 @@ { struct aes_state *as = ks->ks_priv; struct cryptop *crp; - int error; crp = crypto_getreq(as->as_session_sha1, M_WAITOK); @@ -332,13 +331,12 @@ crypto_use_mbuf(crp, inout); crp->crp_opaque = as; crp->crp_callback = aes_crypto_cb; - - error = crypto_dispatch(crp); + crypto_dispatch(crp); if (!CRYPTO_SESS_SYNC(as->as_session_sha1)) { mtx_lock(&as->as_lock); - if (!error && !(crp->crp_flags & CRYPTO_F_DONE)) - error = msleep(crp, &as->as_lock, 0, "gssaes", 0); + if (!(crp->crp_flags & CRYPTO_F_DONE)) + (void)msleep(crp, &as->as_lock, 0, "gssaes", 0); mtx_unlock(&as->as_lock); } Index: sys/netipsec/xform_ah.c =================================================================== --- sys/netipsec/xform_ah.c +++ sys/netipsec/xform_ah.c @@ -673,9 +673,10 @@ xd->cryptoid = cryptoid; xd->vnet = curvnet; if (V_async_crypto) - return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); + crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); else - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); bad: m_freem(m); key_freesav(&sav); @@ -720,7 +721,8 @@ crypto_freesession(cryptoid); xd->cryptoid = crp->crp_session; CURVNET_RESTORE(); - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); } AHSTAT_INC(ahs_noxform); DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); @@ -1058,9 +1060,10 @@ xd->vnet = curvnet; if (V_async_crypto) - return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); + crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); else - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); bad: if (m) m_freem(m); @@ -1102,7 +1105,8 @@ crypto_freesession(cryptoid); xd->cryptoid = crp->crp_session; CURVNET_RESTORE(); - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); } AHSTAT_INC(ahs_noxform); DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); Index: sys/netipsec/xform_esp.c =================================================================== --- sys/netipsec/xform_esp.c +++ sys/netipsec/xform_esp.c @@ -462,9 +462,10 @@ crp->crp_iv_start = skip + hlen - sav->ivlen; if (V_async_crypto) - return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); + crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); else - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); crp_aad_fail: free(xd, M_ESP); @@ -512,7 +513,8 @@ crypto_freesession(cryptoid); xd->cryptoid = crp->crp_session; CURVNET_RESTORE(); - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); } /* EBADMSG indicates authentication failure. */ @@ -947,9 +949,10 @@ } if (V_async_crypto) - return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); + crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); else - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); crp_aad_fail: free(xd, M_ESP); @@ -994,7 +997,8 @@ crypto_freesession(cryptoid); xd->cryptoid = crp->crp_session; CURVNET_RESTORE(); - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); } ESPSTAT_INC(esps_noxform); DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); Index: sys/netipsec/xform_ipcomp.c =================================================================== --- sys/netipsec/xform_ipcomp.c +++ sys/netipsec/xform_ipcomp.c @@ -268,7 +268,8 @@ crp->crp_session = xd->cryptoid = sav->tdb_cryptoid; SECASVAR_UNLOCK(sav); - return crypto_dispatch(crp); + crypto_dispatch(crp); + return (0); bad: m_freem(m); key_freesav(&sav); @@ -312,7 +313,8 @@ crypto_freesession(cryptoid); xd->cryptoid = crp->crp_session; CURVNET_RESTORE(); - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); } IPCOMPSTAT_INC(ipcomps_noxform); DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); @@ -512,7 +514,8 @@ crp->crp_callback = ipcomp_output_cb; crp->crp_opaque = xd; - return crypto_dispatch(crp); + crypto_dispatch(crp); + return (0); bad: if (m) m_freem(m); @@ -554,7 +557,8 @@ crypto_freesession(cryptoid); xd->cryptoid = crp->crp_session; CURVNET_RESTORE(); - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return (0); } IPCOMPSTAT_INC(ipcomps_noxform); DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); Index: sys/opencrypto/crypto.c =================================================================== --- sys/opencrypto/crypto.c +++ sys/opencrypto/crypto.c @@ -1407,7 +1407,7 @@ } #endif -static int +static void crypto_dispatch_one(struct cryptop *crp, int hint) { struct cryptocap *cap; @@ -1428,8 +1428,8 @@ cap = crp->crp_session->cap; if (!atomic_load_int(&cap->cc_qblocked)) { result = crypto_invoke(cap, crp, hint); - if (result != ERESTART) - return (result); + if (result == 0) + return; /* * The driver ran out of resources, put the request on the @@ -1437,16 +1437,15 @@ */ } crypto_batch_enqueue(crp); - return (0); } -int +void crypto_dispatch(struct cryptop *crp) { - return (crypto_dispatch_one(crp, 0)); + crypto_dispatch_one(crp, 0); } -int +void crypto_dispatch_async(struct cryptop *crp, int flags) { struct crypto_ret_worker *ret_worker; @@ -1456,7 +1455,8 @@ * The driver issues completions asynchonously, don't bother * deferring dispatch to a worker thread. */ - return (crypto_dispatch(crp)); + crypto_dispatch(crp); + return; } #ifdef INVARIANTS @@ -1474,7 +1474,6 @@ } TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp); taskqueue_enqueue(crypto_tq, &crp->crp_task); - return (0); } void @@ -1486,8 +1485,7 @@ while ((crp = TAILQ_FIRST(crpq)) != NULL) { hint = TAILQ_NEXT(crp, crp_next) != NULL ? CRYPTO_HINT_MORE : 0; TAILQ_REMOVE(crpq, crp, crp_next); - if (crypto_dispatch_one(crp, hint) != 0) - crypto_batch_enqueue(crp); + crypto_dispatch_one(crp, hint); } } Index: sys/opencrypto/cryptodev.h =================================================================== --- sys/opencrypto/cryptodev.h +++ sys/opencrypto/cryptodev.h @@ -623,9 +623,9 @@ device_t crypto_find_device_byhid(int hid); int crypto_getcaps(int hid); int crypto_unregister_all(uint32_t driverid); -int crypto_dispatch(struct cryptop *crp); +void crypto_dispatch(struct cryptop *crp); #define CRYPTO_ASYNC_ORDERED 0x1 /* complete in order dispatched */ -int crypto_dispatch_async(struct cryptop *crp, int flags); +void crypto_dispatch_async(struct cryptop *crp, int flags); void crypto_dispatch_batch(struct cryptopq *crpq, int flags); #define CRYPTO_SYMQ 0x1 int crypto_unblock(uint32_t, int); Index: sys/opencrypto/cryptodev.c =================================================================== --- sys/opencrypto/cryptodev.c +++ sys/opencrypto/cryptodev.c @@ -813,11 +813,7 @@ * results in a lock order reversal between crypto_dispatch forced * entry and the crypto_done callback into us. */ - error = crypto_dispatch(crp); - if (error != 0) { - SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - goto bail; - } + crypto_dispatch(crp); mtx_lock(&cse->lock); while (!cod->done) @@ -1016,11 +1012,7 @@ * results in a lock order reversal between crypto_dispatch forced * entry and the crypto_done callback into us. */ - error = crypto_dispatch(crp); - if (error != 0) { - SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - goto bail; - } + crypto_dispatch(crp); mtx_lock(&cse->lock); while (!cod->done) Index: sys/opencrypto/ktls_ocf.c =================================================================== --- sys/opencrypto/ktls_ocf.c +++ sys/opencrypto/ktls_ocf.c @@ -200,15 +200,14 @@ oo.os = os; oo.done = false; + error = 0; crp->crp_opaque = &oo; for (;;) { async = !CRYPTO_SESS_SYNC(crp->crp_session); crp->crp_callback = async ? ktls_ocf_callback_async : ktls_ocf_callback_sync; - error = crypto_dispatch(crp); - if (error) - break; + crypto_dispatch(crp); if (async) { mtx_lock(&os->lock); while (!oo.done) @@ -240,11 +239,7 @@ crp->crp_etype = 0; crp->crp_flags &= ~CRYPTO_F_DONE; counter_u64_add(ocf_retries, 1); - error = crypto_dispatch(crp); - if (error != 0) { - crypto_destroyreq(crp); - ktls_encrypt_cb(state, error); - } + crypto_dispatch(crp); return (0); } @@ -258,14 +253,10 @@ ktls_ocf_dispatch_async(struct ktls_ocf_encrypt_state *state, struct cryptop *crp) { - int error; - crp->crp_opaque = state; crp->crp_callback = ktls_ocf_dispatch_async_cb; - error = crypto_dispatch(crp); - if (error != 0) - crypto_destroyreq(crp); - return (error); + crypto_dispatch(crp); + return (0); } static int