Page MenuHomeFreeBSD
Differential D27841 Diff 84217 sys/opencrypto/ktls_ocf.c

Changeset View

Standalone View

View Options

sys/opencrypto/ktls_ocf.c

Show First 20 Lines • Show All 81 Lines • ▼ Show 20 LinesSYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls11_cbc_crypts,
CTLFLAG_RD, &ocf_tls11_cbc_crypts, CTLFLAG_RD, &ocf_tls11_cbc_crypts,
"Total number of OCF TLS 1.1/1.2 CBC encryption operations"); "Total number of OCF TLS 1.1/1.2 CBC encryption operations");
static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_crypts); static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_crypts);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,
CTLFLAG_RD, &ocf_tls12_gcm_crypts, CTLFLAG_RD, &ocf_tls12_gcm_crypts,
"Total number of OCF TLS 1.2 GCM encryption operations"); "Total number of OCF TLS 1.2 GCM encryption operations");
static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_crypts);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_crypts,
CTLFLAG_RD, &ocf_tls12_chacha20_crypts,
"Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations");
static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_crypts); static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_crypts);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,
CTLFLAG_RD, &ocf_tls13_gcm_crypts, CTLFLAG_RD, &ocf_tls13_gcm_crypts,
"Total number of OCF TLS 1.3 GCM encryption operations"); "Total number of OCF TLS 1.3 GCM encryption operations");
static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_crypts);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_crypts,
CTLFLAG_RD, &ocf_tls13_chacha20_crypts,
"Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations");
static COUNTER_U64_DEFINE_EARLY(ocf_inplace); static COUNTER_U64_DEFINE_EARLY(ocf_inplace);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace, SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace,
CTLFLAG_RD, &ocf_inplace, CTLFLAG_RD, &ocf_inplace,
"Total number of OCF in-place operations"); "Total number of OCF in-place operations");
static COUNTER_U64_DEFINE_EARLY(ocf_separate_output); static COUNTER_U64_DEFINE_EARLY(ocf_separate_output);
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output, SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output,
CTLFLAG_RD, &ocf_separate_output, CTLFLAG_RD, &ocf_separate_output,
▲ Show 20 Lines • Show All 218 Lines • ▼ Show 20 Lines#ifdef INVARIANTS
os->in_progress = false; os->in_progress = false;
mtx_unlock(&os->lock); mtx_unlock(&os->lock);
#endif #endif
} }
return (error); return (error);
} }
static int static int
ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
struct iovec *outiov, int iovcnt, uint64_t seqno, struct iovec *outiov, int iovcnt, uint64_t seqno,
uint8_t record_type __unused) uint8_t record_type __unused)
{ {
struct uio uio, out_uio, *tag_uio; struct uio uio, out_uio, *tag_uio;
struct tls_aead_data ad; struct tls_aead_data ad;
struct cryptop crp; struct cryptop crp;
struct ocf_session *os; struct ocf_session *os;
Show All 14 Linesktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
out_uio.uio_iovcnt = iovcnt; out_uio.uio_iovcnt = iovcnt;
out_uio.uio_offset = 0; out_uio.uio_offset = 0;
out_uio.uio_segflg = UIO_SYSSPACE; out_uio.uio_segflg = UIO_SYSSPACE;
out_uio.uio_td = curthread; out_uio.uio_td = curthread;
crypto_initreq(&crp, os->sid); crypto_initreq(&crp, os->sid);
/* Setup the IV. */ /* Setup the IV. */
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1,
sizeof(uint64_t));
} else {
/*
* Chacha20-Poly1305 constructs the IV for TLS 1.2
* identically to constructing the IV for AEAD in TLS
* 1.3.
*/
memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
*(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno);
}
gallatinUnsubmitted
Done
Inline Actions

Maybe a switch, or at least a comment for the else cases that they are chacha?

gallatin: Maybe a switch, or at least a comment for the else cases that they are chacha?
jhbAuthorUnsubmitted
Done
Inline Actions

I'll add a comment about Chacha using the TLS 1.3 IV construction in TLS 1.2.

jhb: I'll add a comment about Chacha using the TLS 1.3 IV construction in TLS 1.2.
/* Setup the AAD. */ /* Setup the AAD. */
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
tls_comp_len = ntohs(hdr->tls_length) - tls_comp_len = ntohs(hdr->tls_length) -
(AES_GMAC_HASH_LEN + sizeof(uint64_t)); (AES_GMAC_HASH_LEN + sizeof(uint64_t));
else
tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN;
ad.seq = htobe64(seqno); ad.seq = htobe64(seqno);
ad.type = hdr->tls_type; ad.type = hdr->tls_type;
ad.tls_vmajor = hdr->tls_vmajor; ad.tls_vmajor = hdr->tls_vmajor;
ad.tls_vminor = hdr->tls_vminor; ad.tls_vminor = hdr->tls_vminor;
ad.tls_length = htons(tls_comp_len); ad.tls_length = htons(tls_comp_len);
crp.crp_aad = &ad; crp.crp_aad = &ad;
crp.crp_aad_length = sizeof(ad); crp.crp_aad_length = sizeof(ad);
Show All 23 Linesktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
tag_uio->uio_resid += AES_GMAC_HASH_LEN; tag_uio->uio_resid += AES_GMAC_HASH_LEN;
crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
crypto_use_uio(&crp, &uio); crypto_use_uio(&crp, &uio);
if (!inplace) if (!inplace)
crypto_use_output_uio(&crp, &out_uio); crypto_use_output_uio(&crp, &out_uio);
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
counter_u64_add(ocf_tls12_gcm_crypts, 1); counter_u64_add(ocf_tls12_gcm_crypts, 1);
else
counter_u64_add(ocf_tls12_chacha20_crypts, 1);
if (inplace) if (inplace)
counter_u64_add(ocf_inplace, 1); counter_u64_add(ocf_inplace, 1);
else else
counter_u64_add(ocf_separate_output, 1); counter_u64_add(ocf_separate_output, 1);
error = ktls_ocf_dispatch(os, &crp); error = ktls_ocf_dispatch(os, &crp);
crypto_destroyreq(&crp); crypto_destroyreq(&crp);
return (error); return (error);
} }
static int static int
ktls_ocf_tls12_gcm_decrypt(struct ktls_session *tls, ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls,
const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno,
int *trailer_len) int *trailer_len)
{ {
struct tls_aead_data ad; struct tls_aead_data ad;
struct cryptop crp; struct cryptop crp;
struct ocf_session *os; struct ocf_session *os;
struct ocf_operation oo; struct ocf_operation oo;
int error; int error;
uint16_t tls_comp_len; uint16_t tls_comp_len;
os = tls->cipher; os = tls->cipher;
oo.os = os; oo.os = os;
oo.done = false; oo.done = false;
crypto_initreq(&crp, os->sid); crypto_initreq(&crp, os->sid);
/* Setup the IV. */ /* Setup the IV. */
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1,
sizeof(uint64_t));
} else {
/*
* Chacha20-Poly1305 constructs the IV for TLS 1.2
* identically to constructing the IV for AEAD in TLS
* 1.3.
*/
memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
*(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno);
}
/* Setup the AAD. */ /* Setup the AAD. */
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
tls_comp_len = ntohs(hdr->tls_length) - tls_comp_len = ntohs(hdr->tls_length) -
(AES_GMAC_HASH_LEN + sizeof(uint64_t)); (AES_GMAC_HASH_LEN + sizeof(uint64_t));
else
tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN;
ad.seq = htobe64(seqno); ad.seq = htobe64(seqno);
ad.type = hdr->tls_type; ad.type = hdr->tls_type;
ad.tls_vmajor = hdr->tls_vmajor; ad.tls_vmajor = hdr->tls_vmajor;
ad.tls_vminor = hdr->tls_vminor; ad.tls_vminor = hdr->tls_vminor;
ad.tls_length = htons(tls_comp_len); ad.tls_length = htons(tls_comp_len);
crp.crp_aad = &ad; crp.crp_aad = &ad;
crp.crp_aad_length = sizeof(ad); crp.crp_aad_length = sizeof(ad);
crp.crp_payload_start = tls->params.tls_hlen; crp.crp_payload_start = tls->params.tls_hlen;
crp.crp_payload_length = tls_comp_len; crp.crp_payload_length = tls_comp_len;
crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;
crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST;
crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
crypto_use_mbuf(&crp, m); crypto_use_mbuf(&crp, m);
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
counter_u64_add(ocf_tls12_gcm_crypts, 1); counter_u64_add(ocf_tls12_gcm_crypts, 1);
else
counter_u64_add(ocf_tls12_chacha20_crypts, 1);
error = ktls_ocf_dispatch(os, &crp); error = ktls_ocf_dispatch(os, &crp);
crypto_destroyreq(&crp); crypto_destroyreq(&crp);
*trailer_len = AES_GMAC_HASH_LEN; *trailer_len = AES_GMAC_HASH_LEN;
return (error); return (error);
} }
static int static int
ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
{ {
struct uio uio, out_uio; struct uio uio, out_uio;
struct tls_aead_data_13 ad; struct tls_aead_data_13 ad;
char nonce[12]; char nonce[12];
struct cryptop crp; struct cryptop crp;
struct ocf_session *os; struct ocf_session *os;
Show All 33 Linesktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
/* /*
* Duplicate the input iov to append the trailer. Always * Duplicate the input iov to append the trailer. Always
* include the full trailer as input to get the record_type * include the full trailer as input to get the record_type
* even if only the first byte is used. * even if only the first byte is used.
*/ */
memcpy(iov, iniov, iovcnt * sizeof(*iov)); memcpy(iov, iniov, iovcnt * sizeof(*iov));
iov[iovcnt].iov_base = trailer; iov[iovcnt].iov_base = trailer;
iov[iovcnt].iov_len = AES_GMAC_HASH_LEN + 1; iov[iovcnt].iov_len = tls->params.tls_tlen;
uio.uio_iov = iov; uio.uio_iov = iov;
uio.uio_iovcnt = iovcnt + 1; uio.uio_iovcnt = iovcnt + 1;
uio.uio_offset = 0; uio.uio_offset = 0;
uio.uio_resid = crp.crp_payload_length + AES_GMAC_HASH_LEN; uio.uio_resid = crp.crp_payload_length + tls->params.tls_tlen - 1;
uio.uio_segflg = UIO_SYSSPACE; uio.uio_segflg = UIO_SYSSPACE;
uio.uio_td = curthread; uio.uio_td = curthread;
crypto_use_uio(&crp, &uio); crypto_use_uio(&crp, &uio);
if (!inplace) { if (!inplace) {
/* Duplicate the output iov to append the trailer. */ /* Duplicate the output iov to append the trailer. */
memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov)); memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov));
out_iov[iovcnt] = iov[iovcnt]; out_iov[iovcnt] = iov[iovcnt];
out_uio.uio_iov = out_iov; out_uio.uio_iov = out_iov;
out_uio.uio_iovcnt = iovcnt + 1; out_uio.uio_iovcnt = iovcnt + 1;
out_uio.uio_offset = 0; out_uio.uio_offset = 0;
out_uio.uio_resid = crp.crp_payload_length + out_uio.uio_resid = crp.crp_payload_length +
AES_GMAC_HASH_LEN; tls->params.tls_tlen - 1;
out_uio.uio_segflg = UIO_SYSSPACE; out_uio.uio_segflg = UIO_SYSSPACE;
out_uio.uio_td = curthread; out_uio.uio_td = curthread;
crypto_use_output_uio(&crp, &out_uio); crypto_use_output_uio(&crp, &out_uio);
} }
crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
memcpy(crp.crp_iv, nonce, sizeof(nonce)); memcpy(crp.crp_iv, nonce, sizeof(nonce));
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
counter_u64_add(ocf_tls13_gcm_crypts, 1); counter_u64_add(ocf_tls13_gcm_crypts, 1);
else
counter_u64_add(ocf_tls13_chacha20_crypts, 1);
if (inplace) if (inplace)
counter_u64_add(ocf_inplace, 1); counter_u64_add(ocf_inplace, 1);
else else
counter_u64_add(ocf_separate_output, 1); counter_u64_add(ocf_separate_output, 1);
error = ktls_ocf_dispatch(os, &crp); error = ktls_ocf_dispatch(os, &crp);
crypto_destroyreq(&crp); crypto_destroyreq(&crp);
return (error); return (error);
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines case CRYPTO_AES_CBC:
csp.csp_ivlen = AES_BLOCK_LEN; csp.csp_ivlen = AES_BLOCK_LEN;
mac_csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; mac_csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
mac_csp.csp_mode = CSP_MODE_DIGEST; mac_csp.csp_mode = CSP_MODE_DIGEST;
mac_csp.csp_auth_alg = tls->params.auth_algorithm; mac_csp.csp_auth_alg = tls->params.auth_algorithm;
mac_csp.csp_auth_key = tls->params.auth_key; mac_csp.csp_auth_key = tls->params.auth_key;
mac_csp.csp_auth_klen = tls->params.auth_key_len; mac_csp.csp_auth_klen = tls->params.auth_key_len;
break; break;
case CRYPTO_CHACHA20_POLY1305:
switch (tls->params.cipher_key_len) {
case 256 / 8:
break;
default: default:
return (EINVAL);
}
/* Only TLS 1.2 and 1.3 are supported. */
if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
tls->params.tls_vminor > TLS_MINOR_VER_THREE)
return (EPROTONOSUPPORT); return (EPROTONOSUPPORT);
/* TLS 1.3 is not yet supported for receive. */
if (direction == KTLS_RX &&
tls->params.tls_vminor == TLS_MINOR_VER_THREE)
return (EPROTONOSUPPORT);
csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
csp.csp_mode = CSP_MODE_AEAD;
csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
csp.csp_cipher_key = tls->params.cipher_key;
csp.csp_cipher_klen = tls->params.cipher_key_len;
csp.csp_ivlen = CHACHA20_POLY1305_IV_LEN;
break;
default:
return (EPROTONOSUPPORT);
} }
os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
if (os == NULL) if (os == NULL)
return (ENOMEM); return (ENOMEM);
error = crypto_newsession(&os->sid, &csp, error = crypto_newsession(&os->sid, &csp,
CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
Show All 10 Lines if (error) {
free(os, M_KTLS_OCF); free(os, M_KTLS_OCF);
return (error); return (error);
} }
os->mac_len = mac_len; os->mac_len = mac_len;
} }
mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
tls->cipher = os; tls->cipher = os;
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 ||
tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) {
if (direction == KTLS_TX) { if (direction == KTLS_TX) {
if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt;
else else
tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt;
} else { } else {
tls->sw_decrypt = ktls_ocf_tls12_gcm_decrypt; tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt;
} }
} else { } else {
tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt; tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt;
if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) { if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) {
os->implicit_iv = true; os->implicit_iv = true;
memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN); memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN);
} }
} }
Show All 31 Lines