Differential D19090 Diff 53949 head/sys/opencrypto/cryptosoft.c

Changeset View

Standalone View

head/sys/opencrypto/cryptosoft.c

Show First 20 Lines • Show All 56 Lines • ▼ Show 20 Lines
#include <opencrypto/cryptodev.h>		#include <opencrypto/cryptodev.h>
#include <opencrypto/cryptosoft.h>		#include <opencrypto/cryptosoft.h>
#include <opencrypto/xform.h>		#include <opencrypto/xform.h>

#include <sys/kobj.h>		#include <sys/kobj.h>
#include <sys/bus.h>		#include <sys/bus.h>
#include "cryptodev_if.h"		#include "cryptodev_if.h"

		_Static_assert(AES_CCM_IV_LEN == AES_GCM_IV_LEN,
		"AES_GCM_IV_LEN must currently be the same as AES_CCM_IV_LEN");

static int32_t swcr_id;		static int32_t swcr_id;

u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];		u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];		u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];

static int swcr_encdec(struct cryptodesc , struct swcr_data , caddr_t, int);		static int swcr_encdec(struct cryptodesc , struct swcr_data , caddr_t, int);
static int swcr_authcompute(struct cryptodesc , struct swcr_data , caddr_t, int);		static int swcr_authcompute(struct cryptodesc , struct swcr_data , caddr_t, int);
static int swcr_authenc(struct cryptop *crp);		static int swcr_authenc(struct cryptop *crp);
▲ Show 20 Lines • Show All 428 Lines • ▼ Show 20 Lines	swcr_authenc(struct cryptop *crp)
struct swcr_session *ses;		struct swcr_session *ses;
struct cryptodesc crd, crda = NULL, *crde = NULL;		struct cryptodesc crd, crda = NULL, *crde = NULL;
struct swcr_data sw, swa, *swe = NULL;		struct swcr_data sw, swa, *swe = NULL;
struct auth_hash *axf = NULL;		struct auth_hash *axf = NULL;
struct enc_xform *exf = NULL;		struct enc_xform *exf = NULL;
caddr_t buf = (caddr_t)crp->crp_buf;		caddr_t buf = (caddr_t)crp->crp_buf;
uint32_t *blkp;		uint32_t *blkp;
int aadlen, blksz, i, ivlen, len, iskip, oskip, r;		int aadlen, blksz, i, ivlen, len, iskip, oskip, r;
		int isccm = 0;

ivlen = blksz = iskip = oskip = 0;		ivlen = blksz = iskip = oskip = 0;

ses = crypto_get_driver_session(crp->crp_session);		ses = crypto_get_driver_session(crp->crp_session);

for (crd = crp->crp_desc; crd; crd = crd->crd_next) {		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
for (i = 0; i < nitems(ses->swcr_algorithms) &&		for (i = 0; i < nitems(ses->swcr_algorithms) &&
ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++)		ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++)
;		;
if (i == nitems(ses->swcr_algorithms))		if (i == nitems(ses->swcr_algorithms))
return (EINVAL);		return (EINVAL);

sw = &ses->swcr_algorithms[i];		sw = &ses->swcr_algorithms[i];
switch (sw->sw_alg) {		switch (sw->sw_alg) {
		case CRYPTO_AES_CCM_16:
case CRYPTO_AES_NIST_GCM_16:		case CRYPTO_AES_NIST_GCM_16:
case CRYPTO_AES_NIST_GMAC:		case CRYPTO_AES_NIST_GMAC:
swe = sw;		swe = sw;
crde = crd;		crde = crd;
exf = swe->sw_exf;		exf = swe->sw_exf;
ivlen = 12;		/* AES_CCM_IV_LEN and AES_GCM_IV_LEN are both 12 */
		ivlen = AES_CCM_IV_LEN;
break;		break;
		case CRYPTO_AES_CCM_CBC_MAC:
		isccm = 1;
		/* FALLTHROUGH */
case CRYPTO_AES_128_NIST_GMAC:		case CRYPTO_AES_128_NIST_GMAC:
case CRYPTO_AES_192_NIST_GMAC:		case CRYPTO_AES_192_NIST_GMAC:
case CRYPTO_AES_256_NIST_GMAC:		case CRYPTO_AES_256_NIST_GMAC:
swa = sw;		swa = sw;
crda = crd;		crda = crd;
axf = swa->sw_axf;		axf = swa->sw_axf;
if (swa->sw_ictx == 0)		if (swa->sw_ictx == 0)
return (EINVAL);		return (EINVAL);
bcopy(swa->sw_ictx, &ctx, axf->ctxsize);		bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
blksz = axf->blocksize;		blksz = axf->blocksize;
break;		break;
default:		default:
return (EINVAL);		return (EINVAL);
}		}
}		}
if (crde == NULL \|\| crda == NULL)		if (crde == NULL \|\| crda == NULL)
return (EINVAL);		return (EINVAL);
		/*
		* We need to make sure that the auth algorithm matches the
		* encr algorithm. Specifically, for AES-GCM must go with
		* AES NIST GMAC, and AES-CCM must go with CBC-MAC.
		*/
		if (crde->crd_alg == CRYPTO_AES_NIST_GCM_16) {
		switch (crda->crd_alg) {
		case CRYPTO_AES_128_NIST_GMAC:
		case CRYPTO_AES_192_NIST_GMAC:
		case CRYPTO_AES_256_NIST_GMAC:
		break; /* Good! */
		default:
		return (EINVAL); /* Not good! */
		}
		} else if (crde->crd_alg == CRYPTO_AES_CCM_16 &&
		crda->crd_alg != CRYPTO_AES_CCM_CBC_MAC)
		return (EINVAL);

if (crde->crd_alg == CRYPTO_AES_NIST_GCM_16 &&		if ((crde->crd_alg == CRYPTO_AES_NIST_GCM_16 \|\|
		crde->crd_alg == CRYPTO_AES_CCM_16) &&
(crde->crd_flags & CRD_F_IV_EXPLICIT) == 0)		(crde->crd_flags & CRD_F_IV_EXPLICIT) == 0)
return (EINVAL);		return (EINVAL);

if (crde->crd_klen != crda->crd_klen)		if (crde->crd_klen != crda->crd_klen)
return (EINVAL);		return (EINVAL);

/* Initialize the IV */		/* Initialize the IV */
if (crde->crd_flags & CRD_F_ENCRYPT) {		if (crde->crd_flags & CRD_F_ENCRYPT) {
Show All 14 Lines	if (crde->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(crde->crd_iv, iv, ivlen);		bcopy(crde->crd_iv, iv, ivlen);
else {		else {
/* Get IV off buf */		/* Get IV off buf */
crypto_copydata(crp->crp_flags, buf, crde->crd_inject,		crypto_copydata(crp->crp_flags, buf, crde->crd_inject,
ivlen, iv);		ivlen, iv);
}		}
}		}

		if (swa->sw_alg == CRYPTO_AES_CCM_CBC_MAC) {
		/*
		* AES CCM-CBC needs to know the length of
		* both the auth data, and payload data, before
		* doing the auth computation.
		*/
		ctx.aes_cbc_mac_ctx.authDataLength = crda->crd_len;
		ctx.aes_cbc_mac_ctx.cryptDataLength = crde->crd_len;
		}
/* Supply MAC with IV */		/* Supply MAC with IV */
if (axf->Reinit)		if (axf->Reinit)
axf->Reinit(&ctx, iv, ivlen);		axf->Reinit(&ctx, iv, ivlen);

/* Supply MAC with AAD */		/* Supply MAC with AAD */
aadlen = crda->crd_len;		aadlen = crda->crd_len;

for (i = iskip; i < crda->crd_len; i += blksz) {		for (i = iskip; i < crda->crd_len; i += blksz) {
Show All 18 Lines	if (exf->encrypt_multi != NULL) {
len = MIN(len, sizeof(blkbuf));		len = MIN(len, sizeof(blkbuf));
} else		} else
len = blksz;		len = blksz;
len = MIN(crde->crd_len - i, len);		len = MIN(crde->crd_len - i, len);
if (len < blksz)		if (len < blksz)
bzero(blk, blksz);		bzero(blk, blksz);
crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len,		crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len,
blk);		blk);
		/*
		* One of the problems with CCM+CBC is that the authentication
		* is done on the unecncrypted data. As a result, we have
		* to do the authentication update at different times,
		* depending on whether it's CCM or not.
		*/
if (crde->crd_flags & CRD_F_ENCRYPT) {		if (crde->crd_flags & CRD_F_ENCRYPT) {
		if (isccm)
		axf->Update(&ctx, blk, len);
if (exf->encrypt_multi != NULL)		if (exf->encrypt_multi != NULL)
exf->encrypt_multi(swe->sw_kschedule, blk,		exf->encrypt_multi(swe->sw_kschedule, blk,
len);		len);
else		else
exf->encrypt(swe->sw_kschedule, blk);		exf->encrypt(swe->sw_kschedule, blk);
		if (!isccm)
axf->Update(&ctx, blk, len);		axf->Update(&ctx, blk, len);
crypto_copyback(crp->crp_flags, buf,		crypto_copyback(crp->crp_flags, buf,
crde->crd_skip + i, len, blk);		crde->crd_skip + i, len, blk);
} else {		} else {
		if (isccm) {
		KASSERT(exf->encrypt_multi == NULL,
		("assume CCM is single-block only"));
		exf->decrypt(swe->sw_kschedule, blk);
		}
axf->Update(&ctx, blk, len);		axf->Update(&ctx, blk, len);
}		}
}		}

/* Do any required special finalization */		/* Do any required special finalization */
switch (crda->crd_alg) {		switch (crda->crd_alg) {
case CRYPTO_AES_128_NIST_GMAC:		case CRYPTO_AES_128_NIST_GMAC:
case CRYPTO_AES_192_NIST_GMAC:		case CRYPTO_AES_192_NIST_GMAC:
Show All 14 Lines	swcr_authenc(struct cryptop *crp)
/* Validate tag */		/* Validate tag */
if (!(crde->crd_flags & CRD_F_ENCRYPT)) {		if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
crypto_copydata(crp->crp_flags, buf, crda->crd_inject,		crypto_copydata(crp->crp_flags, buf, crda->crd_inject,
axf->hashsize, uaalg);		axf->hashsize, uaalg);

r = timingsafe_bcmp(aalg, uaalg, axf->hashsize);		r = timingsafe_bcmp(aalg, uaalg, axf->hashsize);
if (r == 0) {		if (r == 0) {
/* tag matches, decrypt data */		/* tag matches, decrypt data */
		if (isccm) {
		KASSERT(exf->reinit != NULL,
		("AES-CCM reinit function must be set"));
		exf->reinit(swe->sw_kschedule, iv);
		}
for (i = 0; i < crde->crd_len; i += blksz) {		for (i = 0; i < crde->crd_len; i += blksz) {
len = MIN(crde->crd_len - i, blksz);		len = MIN(crde->crd_len - i, blksz);
if (len < blksz)		if (len < blksz)
bzero(blk, blksz);		bzero(blk, blksz);
crypto_copydata(crp->crp_flags, buf,		crypto_copydata(crp->crp_flags, buf,
crde->crd_skip + i, len, blk);		crde->crd_skip + i, len, blk);
exf->decrypt(swe->sw_kschedule, blk);		exf->decrypt(swe->sw_kschedule, blk);
crypto_copyback(crp->crp_flags, buf,		crypto_copyback(crp->crp_flags, buf,
▲ Show 20 Lines • Show All 133 Lines • ▼ Show 20 Lines	case CRYPTO_AES_XTS:
txf = &enc_xform_aes_xts;		txf = &enc_xform_aes_xts;
goto enccommon;		goto enccommon;
case CRYPTO_AES_ICM:		case CRYPTO_AES_ICM:
txf = &enc_xform_aes_icm;		txf = &enc_xform_aes_icm;
goto enccommon;		goto enccommon;
case CRYPTO_AES_NIST_GCM_16:		case CRYPTO_AES_NIST_GCM_16:
txf = &enc_xform_aes_nist_gcm;		txf = &enc_xform_aes_nist_gcm;
goto enccommon;		goto enccommon;
		case CRYPTO_AES_CCM_16:
		txf = &enc_xform_ccm;
		goto enccommon;
case CRYPTO_AES_NIST_GMAC:		case CRYPTO_AES_NIST_GMAC:
txf = &enc_xform_aes_nist_gmac;		txf = &enc_xform_aes_nist_gmac;
swd->sw_exf = txf;		swd->sw_exf = txf;
break;		break;
case CRYPTO_CAMELLIA_CBC:		case CRYPTO_CAMELLIA_CBC:
txf = &enc_xform_camellia;		txf = &enc_xform_camellia;
goto enccommon;		goto enccommon;
case CRYPTO_NULL_CBC:		case CRYPTO_NULL_CBC:
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines	auth3common:
return ENOBUFS;		return ENOBUFS;
}		}

axf->Init(swd->sw_ictx);		axf->Init(swd->sw_ictx);
swd->sw_mlen = cri->cri_mlen;		swd->sw_mlen = cri->cri_mlen;
swd->sw_axf = axf;		swd->sw_axf = axf;
break;		break;

		case CRYPTO_AES_CCM_CBC_MAC:
		switch (cri->cri_klen) {
		case 128:
		axf = &auth_hash_ccm_cbc_mac_128;
		break;
		case 192:
		axf = &auth_hash_ccm_cbc_mac_192;
		break;
		case 256:
		axf = &auth_hash_ccm_cbc_mac_256;
		break;
		default:
		swcr_freesession(dev, cses);
		return EINVAL;
		}
		goto auth4common;
case CRYPTO_AES_128_NIST_GMAC:		case CRYPTO_AES_128_NIST_GMAC:
axf = &auth_hash_nist_gmac_aes_128;		axf = &auth_hash_nist_gmac_aes_128;
goto auth4common;		goto auth4common;

case CRYPTO_AES_192_NIST_GMAC:		case CRYPTO_AES_192_NIST_GMAC:
axf = &auth_hash_nist_gmac_aes_192;		axf = &auth_hash_nist_gmac_aes_192;
goto auth4common;		goto auth4common;

▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines	for (i = 0; i < nitems(ses->swcr_algorithms); i++) {
case CRYPTO_RIJNDAEL128_CBC:		case CRYPTO_RIJNDAEL128_CBC:
case CRYPTO_AES_XTS:		case CRYPTO_AES_XTS:
case CRYPTO_AES_ICM:		case CRYPTO_AES_ICM:
case CRYPTO_AES_NIST_GCM_16:		case CRYPTO_AES_NIST_GCM_16:
case CRYPTO_AES_NIST_GMAC:		case CRYPTO_AES_NIST_GMAC:
case CRYPTO_CAMELLIA_CBC:		case CRYPTO_CAMELLIA_CBC:
case CRYPTO_NULL_CBC:		case CRYPTO_NULL_CBC:
case CRYPTO_CHACHA20:		case CRYPTO_CHACHA20:
		case CRYPTO_AES_CCM_16:
txf = swd->sw_exf;		txf = swd->sw_exf;

if (swd->sw_kschedule)		if (swd->sw_kschedule)
txf->zerokey(&(swd->sw_kschedule));		txf->zerokey(&(swd->sw_kschedule));
break;		break;

case CRYPTO_MD5_HMAC:		case CRYPTO_MD5_HMAC:
case CRYPTO_SHA1_HMAC:		case CRYPTO_SHA1_HMAC:
case CRYPTO_SHA2_224_HMAC:		case CRYPTO_SHA2_224_HMAC:
case CRYPTO_SHA2_256_HMAC:		case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:		case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:		case CRYPTO_SHA2_512_HMAC:
case CRYPTO_RIPEMD160_HMAC:		case CRYPTO_RIPEMD160_HMAC:
case CRYPTO_NULL_HMAC:		case CRYPTO_NULL_HMAC:
		case CRYPTO_AES_CCM_CBC_MAC:
axf = swd->sw_axf;		axf = swd->sw_axf;

if (swd->sw_ictx) {		if (swd->sw_ictx) {
bzero(swd->sw_ictx, axf->ctxsize);		bzero(swd->sw_ictx, axf->ctxsize);
free(swd->sw_ictx, M_CRYPTO_DATA);		free(swd->sw_ictx, M_CRYPTO_DATA);
}		}
if (swd->sw_octx) {		if (swd->sw_octx) {
bzero(swd->sw_octx, axf->ctxsize);		bzero(swd->sw_octx, axf->ctxsize);
▲ Show 20 Lines • Show All 129 Lines • ▼ Show 20 Lines	case CRYPTO_POLY1305:
goto done;		goto done;
break;		break;

case CRYPTO_AES_NIST_GCM_16:		case CRYPTO_AES_NIST_GCM_16:
case CRYPTO_AES_NIST_GMAC:		case CRYPTO_AES_NIST_GMAC:
case CRYPTO_AES_128_NIST_GMAC:		case CRYPTO_AES_128_NIST_GMAC:
case CRYPTO_AES_192_NIST_GMAC:		case CRYPTO_AES_192_NIST_GMAC:
case CRYPTO_AES_256_NIST_GMAC:		case CRYPTO_AES_256_NIST_GMAC:
		case CRYPTO_AES_CCM_16:
		case CRYPTO_AES_CCM_CBC_MAC:
crp->crp_etype = swcr_authenc(crp);		crp->crp_etype = swcr_authenc(crp);
goto done;		goto done;

case CRYPTO_DEFLATE_COMP:		case CRYPTO_DEFLATE_COMP:
if ((crp->crp_etype = swcr_compdec(crd, sw,		if ((crp->crp_etype = swcr_compdec(crd, sw,
crp->crp_buf, crp->crp_flags)) != 0)		crp->crp_buf, crp->crp_flags)) != 0)
goto done;		goto done;
else		else
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines	#define REGISTER(alg) \
REGISTER(CRYPTO_AES_128_NIST_GMAC);		REGISTER(CRYPTO_AES_128_NIST_GMAC);
REGISTER(CRYPTO_AES_192_NIST_GMAC);		REGISTER(CRYPTO_AES_192_NIST_GMAC);
REGISTER(CRYPTO_AES_256_NIST_GMAC);		REGISTER(CRYPTO_AES_256_NIST_GMAC);
REGISTER(CRYPTO_CAMELLIA_CBC);		REGISTER(CRYPTO_CAMELLIA_CBC);
REGISTER(CRYPTO_DEFLATE_COMP);		REGISTER(CRYPTO_DEFLATE_COMP);
REGISTER(CRYPTO_BLAKE2B);		REGISTER(CRYPTO_BLAKE2B);
REGISTER(CRYPTO_BLAKE2S);		REGISTER(CRYPTO_BLAKE2S);
REGISTER(CRYPTO_CHACHA20);		REGISTER(CRYPTO_CHACHA20);
		REGISTER(CRYPTO_AES_CCM_16);
		REGISTER(CRYPTO_AES_CCM_CBC_MAC);
REGISTER(CRYPTO_POLY1305);		REGISTER(CRYPTO_POLY1305);
#undef REGISTER		#undef REGISTER

return 0;		return 0;
}		}

static int		static int
swcr_detach(device_t dev)		swcr_detach(device_t dev)
Show All 37 Lines