Differential D27841 Diff 84151 sys/opencrypto/ktls_ocf.c

Changeset View

Standalone View

sys/opencrypto/ktls_ocf.c

Show First 20 Lines • Show All 81 Lines • ▼ Show 20 Lines	SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls11_cbc_crypts,
CTLFLAG_RD, &ocf_tls11_cbc_crypts,		CTLFLAG_RD, &ocf_tls11_cbc_crypts,
"Total number of OCF TLS 1.1/1.2 CBC encryption operations");		"Total number of OCF TLS 1.1/1.2 CBC encryption operations");

static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_crypts);		static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_crypts);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,		SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,
CTLFLAG_RD, &ocf_tls12_gcm_crypts,		CTLFLAG_RD, &ocf_tls12_gcm_crypts,
"Total number of OCF TLS 1.2 GCM encryption operations");		"Total number of OCF TLS 1.2 GCM encryption operations");

		static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_crypts);
		SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_crypts,
		CTLFLAG_RD, &ocf_tls12_chacha20_crypts,
		"Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations");

static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_crypts);		static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_crypts);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,		SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,
CTLFLAG_RD, &ocf_tls13_gcm_crypts,		CTLFLAG_RD, &ocf_tls13_gcm_crypts,
"Total number of OCF TLS 1.3 GCM encryption operations");		"Total number of OCF TLS 1.3 GCM encryption operations");

		static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_crypts);
		SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_crypts,
		CTLFLAG_RD, &ocf_tls13_chacha20_crypts,
		"Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations");

static COUNTER_U64_DEFINE_EARLY(ocf_inplace);		static COUNTER_U64_DEFINE_EARLY(ocf_inplace);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace,		SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace,
CTLFLAG_RD, &ocf_inplace,		CTLFLAG_RD, &ocf_inplace,
"Total number of OCF in-place operations");		"Total number of OCF in-place operations");

static COUNTER_U64_DEFINE_EARLY(ocf_separate_output);		static COUNTER_U64_DEFINE_EARLY(ocf_separate_output);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output,		SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output,
CTLFLAG_RD, &ocf_separate_output,		CTLFLAG_RD, &ocf_separate_output,
▲ Show 20 Lines • Show All 218 Lines • ▼ Show 20 Lines	#ifdef INVARIANTS
os->in_progress = false;		os->in_progress = false;
mtx_unlock(&os->lock);		mtx_unlock(&os->lock);
#endif		#endif
}		}
return (error);		return (error);
}		}

static int		static int
ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,		ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
const struct tls_record_layer hdr, uint8_t trailer, struct iovec *iniov,		const struct tls_record_layer hdr, uint8_t trailer, struct iovec *iniov,
struct iovec *outiov, int iovcnt, uint64_t seqno,		struct iovec *outiov, int iovcnt, uint64_t seqno,
uint8_t record_type __unused)		uint8_t record_type __unused)
{		{
struct uio uio, out_uio, *tag_uio;		struct uio uio, out_uio, *tag_uio;
struct tls_aead_data ad;		struct tls_aead_data ad;
struct cryptop crp;		struct cryptop crp;
struct ocf_session *os;		struct ocf_session *os;
Show All 14 Lines	ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
out_uio.uio_iovcnt = iovcnt;		out_uio.uio_iovcnt = iovcnt;
out_uio.uio_offset = 0;		out_uio.uio_offset = 0;
out_uio.uio_segflg = UIO_SYSSPACE;		out_uio.uio_segflg = UIO_SYSSPACE;
out_uio.uio_td = curthread;		out_uio.uio_td = curthread;

crypto_initreq(&crp, os->sid);		crypto_initreq(&crp, os->sid);

/* Setup the IV. */		/* Setup the IV. */
		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
		gallatinUnsubmitted Done Inline Actions Maybe a switch, or at least a comment for the else cases that they are chacha? gallatin: Maybe a switch, or at least a comment for the else cases that they are chacha?
		jhbAuthorUnsubmitted Done Inline Actions I'll add a comment about Chacha using the TLS 1.3 IV construction in TLS 1.2. jhb: I'll add a comment about Chacha using the TLS 1.3 IV construction in TLS 1.2.
memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);		memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));		memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1,
		sizeof(uint64_t));
		} else {
		memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
		(uint64_t )(crp.crp_iv + 4) ^= htobe64(seqno);
		}

/* Setup the AAD. */		/* Setup the AAD. */
		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
tls_comp_len = ntohs(hdr->tls_length) -		tls_comp_len = ntohs(hdr->tls_length) -
(AES_GMAC_HASH_LEN + sizeof(uint64_t));		(AES_GMAC_HASH_LEN + sizeof(uint64_t));
		else
		tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN;
ad.seq = htobe64(seqno);		ad.seq = htobe64(seqno);
ad.type = hdr->tls_type;		ad.type = hdr->tls_type;
ad.tls_vmajor = hdr->tls_vmajor;		ad.tls_vmajor = hdr->tls_vmajor;
ad.tls_vminor = hdr->tls_vminor;		ad.tls_vminor = hdr->tls_vminor;
ad.tls_length = htons(tls_comp_len);		ad.tls_length = htons(tls_comp_len);
crp.crp_aad = &ad;		crp.crp_aad = &ad;
crp.crp_aad_length = sizeof(ad);		crp.crp_aad_length = sizeof(ad);

Show All 23 Lines	ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
tag_uio->uio_resid += AES_GMAC_HASH_LEN;		tag_uio->uio_resid += AES_GMAC_HASH_LEN;

crp.crp_op = CRYPTO_OP_ENCRYPT \| CRYPTO_OP_COMPUTE_DIGEST;		crp.crp_op = CRYPTO_OP_ENCRYPT \| CRYPTO_OP_COMPUTE_DIGEST;
crp.crp_flags = CRYPTO_F_CBIMM \| CRYPTO_F_IV_SEPARATE;		crp.crp_flags = CRYPTO_F_CBIMM \| CRYPTO_F_IV_SEPARATE;
crypto_use_uio(&crp, &uio);		crypto_use_uio(&crp, &uio);
if (!inplace)		if (!inplace)
crypto_use_output_uio(&crp, &out_uio);		crypto_use_output_uio(&crp, &out_uio);

		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
counter_u64_add(ocf_tls12_gcm_crypts, 1);		counter_u64_add(ocf_tls12_gcm_crypts, 1);
		else
		counter_u64_add(ocf_tls12_chacha20_crypts, 1);
if (inplace)		if (inplace)
counter_u64_add(ocf_inplace, 1);		counter_u64_add(ocf_inplace, 1);
else		else
counter_u64_add(ocf_separate_output, 1);		counter_u64_add(ocf_separate_output, 1);
error = ktls_ocf_dispatch(os, &crp);		error = ktls_ocf_dispatch(os, &crp);

crypto_destroyreq(&crp);		crypto_destroyreq(&crp);
return (error);		return (error);
}		}

static int		static int
ktls_ocf_tls12_gcm_decrypt(struct ktls_session *tls,		ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls,
const struct tls_record_layer hdr, struct mbuf m, uint64_t seqno,		const struct tls_record_layer hdr, struct mbuf m, uint64_t seqno,
int *trailer_len)		int *trailer_len)
{		{
struct tls_aead_data ad;		struct tls_aead_data ad;
struct cryptop crp;		struct cryptop crp;
struct ocf_session *os;		struct ocf_session *os;
struct ocf_operation oo;		struct ocf_operation oo;
int error;		int error;
uint16_t tls_comp_len;		uint16_t tls_comp_len;

os = tls->cipher;		os = tls->cipher;

oo.os = os;		oo.os = os;
oo.done = false;		oo.done = false;

crypto_initreq(&crp, os->sid);		crypto_initreq(&crp, os->sid);

/* Setup the IV. */		/* Setup the IV. */
		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);		memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));		memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1,
		sizeof(uint64_t));
		} else {
		memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
		(uint64_t )(crp.crp_iv + 4) ^= htobe64(seqno);
		}

/* Setup the AAD. */		/* Setup the AAD. */
		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
tls_comp_len = ntohs(hdr->tls_length) -		tls_comp_len = ntohs(hdr->tls_length) -
(AES_GMAC_HASH_LEN + sizeof(uint64_t));		(AES_GMAC_HASH_LEN + sizeof(uint64_t));
		else
		tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN;
ad.seq = htobe64(seqno);		ad.seq = htobe64(seqno);
ad.type = hdr->tls_type;		ad.type = hdr->tls_type;
ad.tls_vmajor = hdr->tls_vmajor;		ad.tls_vmajor = hdr->tls_vmajor;
ad.tls_vminor = hdr->tls_vminor;		ad.tls_vminor = hdr->tls_vminor;
ad.tls_length = htons(tls_comp_len);		ad.tls_length = htons(tls_comp_len);
crp.crp_aad = &ad;		crp.crp_aad = &ad;
crp.crp_aad_length = sizeof(ad);		crp.crp_aad_length = sizeof(ad);

crp.crp_payload_start = tls->params.tls_hlen;		crp.crp_payload_start = tls->params.tls_hlen;
crp.crp_payload_length = tls_comp_len;		crp.crp_payload_length = tls_comp_len;
crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;		crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;

crp.crp_op = CRYPTO_OP_DECRYPT \| CRYPTO_OP_VERIFY_DIGEST;		crp.crp_op = CRYPTO_OP_DECRYPT \| CRYPTO_OP_VERIFY_DIGEST;
crp.crp_flags = CRYPTO_F_CBIMM \| CRYPTO_F_IV_SEPARATE;		crp.crp_flags = CRYPTO_F_CBIMM \| CRYPTO_F_IV_SEPARATE;
crypto_use_mbuf(&crp, m);		crypto_use_mbuf(&crp, m);

		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
counter_u64_add(ocf_tls12_gcm_crypts, 1);		counter_u64_add(ocf_tls12_gcm_crypts, 1);
		else
		counter_u64_add(ocf_tls12_chacha20_crypts, 1);
error = ktls_ocf_dispatch(os, &crp);		error = ktls_ocf_dispatch(os, &crp);

crypto_destroyreq(&crp);		crypto_destroyreq(&crp);
*trailer_len = AES_GMAC_HASH_LEN;		*trailer_len = AES_GMAC_HASH_LEN;
return (error);		return (error);
}		}

static int		static int
ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,		ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
const struct tls_record_layer hdr, uint8_t trailer, struct iovec *iniov,		const struct tls_record_layer hdr, uint8_t trailer, struct iovec *iniov,
struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)		struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
{		{
struct uio uio, out_uio;		struct uio uio, out_uio;
struct tls_aead_data_13 ad;		struct tls_aead_data_13 ad;
char nonce[12];		char nonce[12];
struct cryptop crp;		struct cryptop crp;
struct ocf_session *os;		struct ocf_session *os;
Show All 33 Lines	ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,

/*		/*
* Duplicate the input iov to append the trailer. Always		* Duplicate the input iov to append the trailer. Always
* include the full trailer as input to get the record_type		* include the full trailer as input to get the record_type
* even if only the first byte is used.		* even if only the first byte is used.
*/		*/
memcpy(iov, iniov, iovcnt * sizeof(*iov));		memcpy(iov, iniov, iovcnt * sizeof(*iov));
iov[iovcnt].iov_base = trailer;		iov[iovcnt].iov_base = trailer;
iov[iovcnt].iov_len = AES_GMAC_HASH_LEN + 1;		iov[iovcnt].iov_len = tls->params.tls_tlen;
uio.uio_iov = iov;		uio.uio_iov = iov;
uio.uio_iovcnt = iovcnt + 1;		uio.uio_iovcnt = iovcnt + 1;
uio.uio_offset = 0;		uio.uio_offset = 0;
uio.uio_resid = crp.crp_payload_length + AES_GMAC_HASH_LEN;		uio.uio_resid = crp.crp_payload_length + tls->params.tls_tlen - 1;
uio.uio_segflg = UIO_SYSSPACE;		uio.uio_segflg = UIO_SYSSPACE;
uio.uio_td = curthread;		uio.uio_td = curthread;
crypto_use_uio(&crp, &uio);		crypto_use_uio(&crp, &uio);

if (!inplace) {		if (!inplace) {
/* Duplicate the output iov to append the trailer. */		/* Duplicate the output iov to append the trailer. */
memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov));		memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov));
out_iov[iovcnt] = iov[iovcnt];		out_iov[iovcnt] = iov[iovcnt];

out_uio.uio_iov = out_iov;		out_uio.uio_iov = out_iov;
out_uio.uio_iovcnt = iovcnt + 1;		out_uio.uio_iovcnt = iovcnt + 1;
out_uio.uio_offset = 0;		out_uio.uio_offset = 0;
out_uio.uio_resid = crp.crp_payload_length +		out_uio.uio_resid = crp.crp_payload_length +
AES_GMAC_HASH_LEN;		tls->params.tls_tlen - 1;
out_uio.uio_segflg = UIO_SYSSPACE;		out_uio.uio_segflg = UIO_SYSSPACE;
out_uio.uio_td = curthread;		out_uio.uio_td = curthread;
crypto_use_output_uio(&crp, &out_uio);		crypto_use_output_uio(&crp, &out_uio);
}		}

crp.crp_op = CRYPTO_OP_ENCRYPT \| CRYPTO_OP_COMPUTE_DIGEST;		crp.crp_op = CRYPTO_OP_ENCRYPT \| CRYPTO_OP_COMPUTE_DIGEST;
crp.crp_flags = CRYPTO_F_CBIMM \| CRYPTO_F_IV_SEPARATE;		crp.crp_flags = CRYPTO_F_CBIMM \| CRYPTO_F_IV_SEPARATE;

memcpy(crp.crp_iv, nonce, sizeof(nonce));		memcpy(crp.crp_iv, nonce, sizeof(nonce));

		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
counter_u64_add(ocf_tls13_gcm_crypts, 1);		counter_u64_add(ocf_tls13_gcm_crypts, 1);
		else
		counter_u64_add(ocf_tls13_chacha20_crypts, 1);
if (inplace)		if (inplace)
counter_u64_add(ocf_inplace, 1);		counter_u64_add(ocf_inplace, 1);
else		else
counter_u64_add(ocf_separate_output, 1);		counter_u64_add(ocf_separate_output, 1);
error = ktls_ocf_dispatch(os, &crp);		error = ktls_ocf_dispatch(os, &crp);

crypto_destroyreq(&crp);		crypto_destroyreq(&crp);
return (error);		return (error);
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines	case CRYPTO_AES_CBC:
csp.csp_ivlen = AES_BLOCK_LEN;		csp.csp_ivlen = AES_BLOCK_LEN;

mac_csp.csp_flags \|= CSP_F_SEPARATE_OUTPUT;		mac_csp.csp_flags \|= CSP_F_SEPARATE_OUTPUT;
mac_csp.csp_mode = CSP_MODE_DIGEST;		mac_csp.csp_mode = CSP_MODE_DIGEST;
mac_csp.csp_auth_alg = tls->params.auth_algorithm;		mac_csp.csp_auth_alg = tls->params.auth_algorithm;
mac_csp.csp_auth_key = tls->params.auth_key;		mac_csp.csp_auth_key = tls->params.auth_key;
mac_csp.csp_auth_klen = tls->params.auth_key_len;		mac_csp.csp_auth_klen = tls->params.auth_key_len;
break;		break;
		case CRYPTO_CHACHA20_POLY1305:
		switch (tls->params.cipher_key_len) {
		case 256 / 8:
		break;
default:		default:
		return (EINVAL);
		}

		/* Only TLS 1.2 and 1.3 are supported. */
		if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE \|\|
		tls->params.tls_vminor < TLS_MINOR_VER_TWO \|\|
		tls->params.tls_vminor > TLS_MINOR_VER_THREE)
return (EPROTONOSUPPORT);		return (EPROTONOSUPPORT);

		/* TLS 1.3 is not yet supported for receive. */
		if (direction == KTLS_RX &&
		tls->params.tls_vminor == TLS_MINOR_VER_THREE)
		return (EPROTONOSUPPORT);

		csp.csp_flags \|= CSP_F_SEPARATE_OUTPUT \| CSP_F_SEPARATE_AAD;
		csp.csp_mode = CSP_MODE_AEAD;
		csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
		csp.csp_cipher_key = tls->params.cipher_key;
		csp.csp_cipher_klen = tls->params.cipher_key_len;
		csp.csp_ivlen = CHACHA20_POLY1305_IV_LEN;
		break;
		default:
		return (EPROTONOSUPPORT);
}		}

os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT \| M_ZERO);		os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT \| M_ZERO);
if (os == NULL)		if (os == NULL)
return (ENOMEM);		return (ENOMEM);

error = crypto_newsession(&os->sid, &csp,		error = crypto_newsession(&os->sid, &csp,
CRYPTO_FLAG_HARDWARE \| CRYPTO_FLAG_SOFTWARE);		CRYPTO_FLAG_HARDWARE \| CRYPTO_FLAG_SOFTWARE);
Show All 10 Lines	if (error) {
free(os, M_KTLS_OCF);		free(os, M_KTLS_OCF);
return (error);		return (error);
}		}
os->mac_len = mac_len;		os->mac_len = mac_len;
}		}

mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);		mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
tls->cipher = os;		tls->cipher = os;
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 \|\|
		tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) {
if (direction == KTLS_TX) {		if (direction == KTLS_TX) {
if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)		if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt;		tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt;
else		else
tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt;		tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt;
} else {		} else {
tls->sw_decrypt = ktls_ocf_tls12_gcm_decrypt;		tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt;
}		}
} else {		} else {
tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt;		tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt;
if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) {		if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) {
os->implicit_iv = true;		os->implicit_iv = true;
memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN);		memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN);
}		}
}		}
Show All 31 Lines