Changeset View
Changeset View
Standalone View
Standalone View
head/sys/crypto/aesni/aesni.c
Show First 20 Lines • Show All 126 Lines • ▼ Show 20 Lines | aesni_probe(device_t dev) | ||||
bool has_aes, has_sha; | bool has_aes, has_sha; | ||||
detect_cpu_features(&has_aes, &has_sha); | detect_cpu_features(&has_aes, &has_sha); | ||||
if (!has_aes && !has_sha) { | if (!has_aes && !has_sha) { | ||||
device_printf(dev, "No AES or SHA support.\n"); | device_printf(dev, "No AES or SHA support.\n"); | ||||
return (EINVAL); | return (EINVAL); | ||||
} else if (has_aes && has_sha) | } else if (has_aes && has_sha) | ||||
device_set_desc(dev, | device_set_desc(dev, | ||||
"AES-CBC,AES-XTS,AES-GCM,AES-ICM,SHA1,SHA256"); | "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS,SHA1,SHA256"); | ||||
else if (has_aes) | else if (has_aes) | ||||
device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM,AES-ICM"); | device_set_desc(dev, | ||||
"AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS"); | |||||
else | else | ||||
device_set_desc(dev, "SHA1,SHA256"); | device_set_desc(dev, "SHA1,SHA256"); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
aesni_cleanctx(void) | aesni_cleanctx(void) | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | aesni_attach(device_t dev) | ||||
if (sc->has_aes) { | if (sc->has_aes) { | ||||
crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); | crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0); | crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0); | crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); | crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); | crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); | crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); | crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_AES_CCM_16, 0, 0); | |||||
crypto_register(sc->cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0); | |||||
} | } | ||||
if (sc->has_sha) { | if (sc->has_sha) { | ||||
crypto_register(sc->cid, CRYPTO_SHA1, 0, 0); | crypto_register(sc->cid, CRYPTO_SHA1, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0); | crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_SHA2_224, 0, 0); | crypto_register(sc->cid, CRYPTO_SHA2_224, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_SHA2_224_HMAC, 0, 0); | crypto_register(sc->cid, CRYPTO_SHA2_224_HMAC, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_SHA2_256, 0, 0); | crypto_register(sc->cid, CRYPTO_SHA2_256, 0, 0); | ||||
crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0); | crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0); | ||||
Show All 17 Lines | |||||
static int | static int | ||||
aesni_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) | aesni_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) | ||||
{ | { | ||||
struct aesni_softc *sc; | struct aesni_softc *sc; | ||||
struct aesni_session *ses; | struct aesni_session *ses; | ||||
struct cryptoini *encini, *authini; | struct cryptoini *encini, *authini; | ||||
bool gcm_hash, gcm; | bool gcm_hash, gcm; | ||||
bool cbc_hash, ccm; | |||||
int error; | int error; | ||||
KASSERT(cses != NULL, ("EDOOFUS")); | KASSERT(cses != NULL, ("EDOOFUS")); | ||||
if (cri == NULL) { | if (cri == NULL) { | ||||
CRYPTDEB("no cri"); | CRYPTDEB("no cri"); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
sc = device_get_softc(dev); | sc = device_get_softc(dev); | ||||
ses = crypto_get_driver_session(cses); | ses = crypto_get_driver_session(cses); | ||||
authini = NULL; | authini = NULL; | ||||
encini = NULL; | encini = NULL; | ||||
gcm = false; | gcm = false; | ||||
gcm_hash = false; | gcm_hash = false; | ||||
ccm = cbc_hash = false; | |||||
for (; cri != NULL; cri = cri->cri_next) { | for (; cri != NULL; cri = cri->cri_next) { | ||||
switch (cri->cri_alg) { | switch (cri->cri_alg) { | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
case CRYPTO_AES_CCM_16: | |||||
if (cri->cri_alg == CRYPTO_AES_NIST_GCM_16) { | |||||
gcm = true; | gcm = true; | ||||
} else if (cri->cri_alg == CRYPTO_AES_CCM_16) { | |||||
ccm = true; | |||||
} | |||||
/* FALLTHROUGH */ | /* FALLTHROUGH */ | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
case CRYPTO_AES_ICM: | case CRYPTO_AES_ICM: | ||||
case CRYPTO_AES_XTS: | case CRYPTO_AES_XTS: | ||||
if (!sc->has_aes) | if (!sc->has_aes) | ||||
goto unhandled; | goto unhandled; | ||||
if (encini != NULL) { | if (encini != NULL) { | ||||
CRYPTDEB("encini already set"); | CRYPTDEB("encini already set"); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
encini = cri; | encini = cri; | ||||
break; | break; | ||||
case CRYPTO_AES_CCM_CBC_MAC: | |||||
cbc_hash = true; | |||||
authini = cri; | |||||
break; | |||||
case CRYPTO_AES_128_NIST_GMAC: | case CRYPTO_AES_128_NIST_GMAC: | ||||
case CRYPTO_AES_192_NIST_GMAC: | case CRYPTO_AES_192_NIST_GMAC: | ||||
case CRYPTO_AES_256_NIST_GMAC: | case CRYPTO_AES_256_NIST_GMAC: | ||||
/* | /* | ||||
* nothing to do here, maybe in the future cache some | * nothing to do here, maybe in the future cache some | ||||
* values for GHASH | * values for GHASH | ||||
*/ | */ | ||||
if (authini != NULL) { | |||||
CRYPTDEB("authini already set"); | |||||
return (EINVAL); | |||||
} | |||||
gcm_hash = true; | gcm_hash = true; | ||||
authini = cri; | |||||
break; | break; | ||||
case CRYPTO_SHA1: | case CRYPTO_SHA1: | ||||
case CRYPTO_SHA1_HMAC: | case CRYPTO_SHA1_HMAC: | ||||
case CRYPTO_SHA2_224: | case CRYPTO_SHA2_224: | ||||
case CRYPTO_SHA2_224_HMAC: | case CRYPTO_SHA2_224_HMAC: | ||||
case CRYPTO_SHA2_256: | case CRYPTO_SHA2_256: | ||||
case CRYPTO_SHA2_256_HMAC: | case CRYPTO_SHA2_256_HMAC: | ||||
if (!sc->has_sha) | if (!sc->has_sha) | ||||
Show All 13 Lines | unhandled: | ||||
if (encini == NULL && authini == NULL) { | if (encini == NULL && authini == NULL) { | ||||
CRYPTDEB("no cipher"); | CRYPTDEB("no cipher"); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
/* | /* | ||||
* GMAC algorithms are only supported with simultaneous GCM. Likewise | * GMAC algorithms are only supported with simultaneous GCM. Likewise | ||||
* GCM is not supported without GMAC. | * GCM is not supported without GMAC. | ||||
*/ | */ | ||||
if (gcm_hash != gcm) | if (gcm_hash != gcm) { | ||||
CRYPTDEB("gcm_hash != gcm"); | |||||
return (EINVAL); | return (EINVAL); | ||||
} | |||||
if (cbc_hash != ccm) { | |||||
CRYPTDEB("cbc_hash != ccm"); | |||||
return (EINVAL); | |||||
} | |||||
if (encini != NULL) | if (encini != NULL) | ||||
ses->algo = encini->cri_alg; | ses->algo = encini->cri_alg; | ||||
if (authini != NULL) | if (authini != NULL) | ||||
ses->auth_algo = authini->cri_alg; | ses->auth_algo = authini->cri_alg; | ||||
error = aesni_cipher_setup(ses, encini, authini); | error = aesni_cipher_setup(ses, encini, authini); | ||||
if (error != 0) { | if (error != 0) { | ||||
CRYPTDEB("setup failed"); | CRYPTDEB("setup failed"); | ||||
Show All 24 Lines | if (crp->crp_callback == NULL || crp->crp_desc == NULL || | ||||
crp->crp_session == NULL) { | crp->crp_session == NULL) { | ||||
error = EINVAL; | error = EINVAL; | ||||
goto out; | goto out; | ||||
} | } | ||||
for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { | for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { | ||||
switch (crd->crd_alg) { | switch (crd->crd_alg) { | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
case CRYPTO_AES_CCM_16: | |||||
needauth = 1; | needauth = 1; | ||||
/* FALLTHROUGH */ | /* FALLTHROUGH */ | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
case CRYPTO_AES_ICM: | case CRYPTO_AES_ICM: | ||||
case CRYPTO_AES_XTS: | case CRYPTO_AES_XTS: | ||||
if (enccrd != NULL) { | if (enccrd != NULL) { | ||||
error = EINVAL; | error = EINVAL; | ||||
goto out; | goto out; | ||||
} | } | ||||
enccrd = crd; | enccrd = crd; | ||||
break; | break; | ||||
case CRYPTO_AES_128_NIST_GMAC: | case CRYPTO_AES_128_NIST_GMAC: | ||||
case CRYPTO_AES_192_NIST_GMAC: | case CRYPTO_AES_192_NIST_GMAC: | ||||
case CRYPTO_AES_256_NIST_GMAC: | case CRYPTO_AES_256_NIST_GMAC: | ||||
case CRYPTO_AES_CCM_CBC_MAC: | |||||
case CRYPTO_SHA1: | case CRYPTO_SHA1: | ||||
case CRYPTO_SHA1_HMAC: | case CRYPTO_SHA1_HMAC: | ||||
case CRYPTO_SHA2_224: | case CRYPTO_SHA2_224: | ||||
case CRYPTO_SHA2_224_HMAC: | case CRYPTO_SHA2_224_HMAC: | ||||
case CRYPTO_SHA2_256: | case CRYPTO_SHA2_256: | ||||
case CRYPTO_SHA2_256_HMAC: | case CRYPTO_SHA2_256_HMAC: | ||||
if (authcrd != NULL) { | if (authcrd != NULL) { | ||||
error = EINVAL; | error = EINVAL; | ||||
▲ Show 20 Lines • Show All 278 Lines • ▼ Show 20 Lines | aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd, | ||||
struct cryptodesc *authcrd, struct cryptop *crp) | struct cryptodesc *authcrd, struct cryptop *crp) | ||||
{ | { | ||||
struct fpu_kern_ctx *ctx; | struct fpu_kern_ctx *ctx; | ||||
int error, ctxidx; | int error, ctxidx; | ||||
bool kt; | bool kt; | ||||
if (enccrd != NULL) { | if (enccrd != NULL) { | ||||
if ((enccrd->crd_alg == CRYPTO_AES_ICM || | if ((enccrd->crd_alg == CRYPTO_AES_ICM || | ||||
enccrd->crd_alg == CRYPTO_AES_CCM_16 || | |||||
enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) && | enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) && | ||||
(enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) | (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
ctx = NULL; | ctx = NULL; | ||||
ctxidx = 0; | ctxidx = 0; | ||||
error = 0; | error = 0; | ||||
Show All 37 Lines | |||||
static int | static int | ||||
aesni_cipher_crypt(struct aesni_session *ses, struct cryptodesc *enccrd, | aesni_cipher_crypt(struct aesni_session *ses, struct cryptodesc *enccrd, | ||||
struct cryptodesc *authcrd, struct cryptop *crp) | struct cryptodesc *authcrd, struct cryptop *crp) | ||||
{ | { | ||||
uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN], *buf, *authbuf; | uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN], *buf, *authbuf; | ||||
int error, ivlen; | int error, ivlen; | ||||
bool encflag, allocated, authallocated; | bool encflag, allocated, authallocated; | ||||
KASSERT(ses->algo != CRYPTO_AES_NIST_GCM_16 || authcrd != NULL, | KASSERT((ses->algo != CRYPTO_AES_NIST_GCM_16 && | ||||
("AES_NIST_GCM_16 must include MAC descriptor")); | ses->algo != CRYPTO_AES_CCM_16) || authcrd != NULL, | ||||
("AES_NIST_GCM_16/AES_CCM_16 must include MAC descriptor")); | |||||
ivlen = 0; | ivlen = 0; | ||||
authbuf = NULL; | authbuf = NULL; | ||||
buf = aesni_cipher_alloc(enccrd, crp, &allocated); | buf = aesni_cipher_alloc(enccrd, crp, &allocated); | ||||
if (buf == NULL) | if (buf == NULL) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
authallocated = false; | authallocated = false; | ||||
if (ses->algo == CRYPTO_AES_NIST_GCM_16) { | if (ses->algo == CRYPTO_AES_NIST_GCM_16 || | ||||
ses->algo == CRYPTO_AES_CCM_16) { | |||||
authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated); | authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated); | ||||
if (authbuf == NULL) { | if (authbuf == NULL) { | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto out; | goto out; | ||||
} | } | ||||
} | } | ||||
error = 0; | error = 0; | ||||
Show All 9 Lines | aesni_cipher_crypt(struct aesni_session *ses, struct cryptodesc *enccrd, | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
case CRYPTO_AES_ICM: | case CRYPTO_AES_ICM: | ||||
ivlen = AES_BLOCK_LEN; | ivlen = AES_BLOCK_LEN; | ||||
break; | break; | ||||
case CRYPTO_AES_XTS: | case CRYPTO_AES_XTS: | ||||
ivlen = 8; | ivlen = 8; | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
case CRYPTO_AES_CCM_16: | |||||
ivlen = 12; /* should support arbitarily larger */ | ivlen = 12; /* should support arbitarily larger */ | ||||
break; | break; | ||||
} | } | ||||
/* Setup iv */ | /* Setup iv */ | ||||
if (encflag) { | if (encflag) { | ||||
if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) | if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) | ||||
bcopy(enccrd->crd_iv, iv, ivlen); | bcopy(enccrd->crd_iv, iv, ivlen); | ||||
Show All 33 Lines | case CRYPTO_AES_XTS: | ||||
else | else | ||||
aesni_decrypt_xts(ses->rounds, ses->dec_schedule, | aesni_decrypt_xts(ses->rounds, ses->dec_schedule, | ||||
ses->xts_schedule, enccrd->crd_len, buf, buf, | ses->xts_schedule, enccrd->crd_len, buf, buf, | ||||
iv); | iv); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
if (!encflag) | if (!encflag) | ||||
crypto_copydata(crp->crp_flags, crp->crp_buf, | crypto_copydata(crp->crp_flags, crp->crp_buf, | ||||
authcrd->crd_inject, GMAC_DIGEST_LEN, tag); | authcrd->crd_inject, sizeof(tag), tag); | ||||
else | else | ||||
bzero(tag, sizeof tag); | bzero(tag, sizeof tag); | ||||
if (encflag) { | if (encflag) { | ||||
AES_GCM_encrypt(buf, buf, authbuf, iv, tag, | AES_GCM_encrypt(buf, buf, authbuf, iv, tag, | ||||
enccrd->crd_len, authcrd->crd_len, ivlen, | enccrd->crd_len, authcrd->crd_len, ivlen, | ||||
ses->enc_schedule, ses->rounds); | ses->enc_schedule, ses->rounds); | ||||
if (authcrd != NULL) | if (authcrd != NULL) | ||||
crypto_copyback(crp->crp_flags, crp->crp_buf, | crypto_copyback(crp->crp_flags, crp->crp_buf, | ||||
authcrd->crd_inject, GMAC_DIGEST_LEN, tag); | authcrd->crd_inject, sizeof(tag), tag); | ||||
} else { | } else { | ||||
if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag, | if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag, | ||||
enccrd->crd_len, authcrd->crd_len, ivlen, | enccrd->crd_len, authcrd->crd_len, ivlen, | ||||
ses->enc_schedule, ses->rounds)) | ses->enc_schedule, ses->rounds)) | ||||
error = EBADMSG; | error = EBADMSG; | ||||
} | } | ||||
break; | break; | ||||
case CRYPTO_AES_CCM_16: | |||||
if (!encflag) | |||||
crypto_copydata(crp->crp_flags, crp->crp_buf, | |||||
authcrd->crd_inject, sizeof(tag), tag); | |||||
else | |||||
bzero(tag, sizeof tag); | |||||
if (encflag) { | |||||
AES_CCM_encrypt(buf, buf, authbuf, iv, tag, | |||||
enccrd->crd_len, authcrd->crd_len, ivlen, | |||||
ses->enc_schedule, ses->rounds); | |||||
if (authcrd != NULL) | |||||
crypto_copyback(crp->crp_flags, crp->crp_buf, | |||||
authcrd->crd_inject, sizeof(tag), tag); | |||||
} else { | |||||
if (!AES_CCM_decrypt(buf, buf, authbuf, iv, tag, | |||||
enccrd->crd_len, authcrd->crd_len, ivlen, | |||||
ses->enc_schedule, ses->rounds)) | |||||
error = EBADMSG; | |||||
} | } | ||||
break; | |||||
if (allocated) | } | ||||
if (allocated && error == 0) | |||||
crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, | crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, | ||||
enccrd->crd_len, buf); | enccrd->crd_len, buf); | ||||
out: | out: | ||||
if (allocated) { | if (allocated) { | ||||
explicit_bzero(buf, enccrd->crd_len); | explicit_bzero(buf, enccrd->crd_len); | ||||
free(buf, M_AESNI); | free(buf, M_AESNI); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 102 Lines • Show Last 20 Lines |