Changeset View
Standalone View
sys/opencrypto/ktls_ocf.c
Show First 20 Lines • Show All 100 Lines • ▼ Show 20 Lines | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_decrypts, | ||||
CTLFLAG_RD, &ocf_tls12_chacha20_decrypts, | CTLFLAG_RD, &ocf_tls12_chacha20_decrypts, | ||||
"Total number of OCF TLS 1.2 Chacha20-Poly1305 decryption operations"); | "Total number of OCF TLS 1.2 Chacha20-Poly1305 decryption operations"); | ||||
static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_encrypts); | static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_encrypts); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_encrypts, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_encrypts, | ||||
CTLFLAG_RD, &ocf_tls12_chacha20_encrypts, | CTLFLAG_RD, &ocf_tls12_chacha20_encrypts, | ||||
"Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations"); | "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations"); | ||||
static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_encrypts); | static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_encrypts); | ||||
jhb: Does the Mellanox NIC support hardware NIC TLS for ChaCha20-Poly1305? | |||||
Done Inline ActionsNo. hselasky: No. | |||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_encrypts, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_encrypts, | ||||
CTLFLAG_RD, &ocf_tls13_gcm_encrypts, | CTLFLAG_RD, &ocf_tls13_gcm_encrypts, | ||||
"Total number of OCF TLS 1.3 GCM encryption operations"); | "Total number of OCF TLS 1.3 GCM encryption operations"); | ||||
static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_encrypts); | static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_encrypts); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_encrypts, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_encrypts, | ||||
CTLFLAG_RD, &ocf_tls13_chacha20_encrypts, | CTLFLAG_RD, &ocf_tls13_chacha20_encrypts, | ||||
"Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations"); | "Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations"); | ||||
▲ Show 20 Lines • Show All 400 Lines • ▼ Show 20 Lines | ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls, | ||||
error = ktls_ocf_dispatch(os, &crp); | error = ktls_ocf_dispatch(os, &crp); | ||||
crypto_destroyreq(&crp); | crypto_destroyreq(&crp); | ||||
*trailer_len = tls->params.tls_tlen; | *trailer_len = tls->params.tls_tlen; | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
ktls_ocf_tls12_aead_recrypt(struct ktls_session *tls, | |||||
const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno) | |||||
{ | |||||
char tag[MAX(AES_GMAC_HASH_LEN, POLY1305_HASH_LEN)] __aligned(8); | |||||
struct tls_aead_data ad; | |||||
struct cryptop crp; | |||||
struct ktls_ocf_session *os; | |||||
int error; | |||||
uint16_t tls_comp_len; | |||||
if (tls->params.tls_tlen > sizeof(tag)) | |||||
return (ENOMEM); | |||||
os = tls->ocf_session; | |||||
crypto_initreq(&crp, os->sid); | |||||
/* Setup the IV. */ | |||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { | |||||
memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); | |||||
memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, | |||||
sizeof(uint64_t)); | |||||
} else { | |||||
/* | |||||
* Chacha20-Poly1305 constructs the IV for TLS 1.2 | |||||
* identically to constructing the IV for AEAD in TLS | |||||
* 1.3. | |||||
*/ | |||||
memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len); | |||||
*(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno); | |||||
} | |||||
/* Setup the AAD. */ | |||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) | |||||
tls_comp_len = ntohs(hdr->tls_length) - | |||||
(AES_GMAC_HASH_LEN + sizeof(uint64_t)); | |||||
else | |||||
tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN; | |||||
ad.seq = htobe64(seqno); | |||||
ad.type = hdr->tls_type; | |||||
ad.tls_vmajor = hdr->tls_vmajor; | |||||
ad.tls_vminor = hdr->tls_vminor; | |||||
ad.tls_length = htons(tls_comp_len); | |||||
crp.crp_aad = &ad; | |||||
crp.crp_aad_length = sizeof(ad); | |||||
crp.crp_payload_start = tls->params.tls_hlen; | |||||
crp.crp_payload_length = tls_comp_len; | |||||
crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; | |||||
crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; | |||||
crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; | |||||
crypto_use_mbuf(&crp, m); | |||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) | |||||
counter_u64_add(ocf_tls12_gcm_crypts, 1); | |||||
else | |||||
counter_u64_add(ocf_tls12_chacha20_crypts, 1); | |||||
counter_u64_add(ocf_inplace, 1); | |||||
m_copydata(m, tls->params.tls_hlen + tls_comp_len, tls->params.tls_tlen, tag); | |||||
error = ktls_ocf_dispatch(os, &crp); | |||||
m_copyback(m, tls->params.tls_hlen + tls_comp_len, tls->params.tls_tlen, tag); | |||||
crypto_destroyreq(&crp); | |||||
Not Done Inline ActionsAs I said earlier, this is way more complicated than you need. I would make each kthread pre-allocate a buffer sized to one TLS record and just memset it to 0 and use AES-CTR to encrypt the zeroed buffer, then xor the relevant portion of the mbuf chain with the resulting buffer. I would probably not even bother with OCF for this but just use the 'enc_xform_aes_ctr' directly to do this (especially once the multi-block hooks review I have lands). You might even be able to re-encrypt the mbufs in place in some cases without needing the temporary buffer. As it is you are doing all the GMAC computations just to throw them away. jhb: As I said earlier, this is way more complicated than you need. I would make each kthread pre… | |||||
Done Inline ActionsOK, I'll try to re-do this part. hselasky: OK, I'll try to re-do this part. | |||||
Done Inline Actions@jhb : From what I can see the current encrypt functions are tightly bound to single / EXT PG mbufs. Can you help decouple the current decrypt functions from EXT PG mbufs only? hselasky: @jhb : From what I can see the current encrypt functions are tightly bound to single / EXT PG… | |||||
Not Done Inline ActionsIf you use enc_xform they are not. Also, I still think you should have each ktls worker thread just malloc() a single 16KB block during startup and pass a pointer down to the decrypt worker routine instead having to do malloc/frees. If you use enc_xform you can bypass OCF entirely and generally not have to touch anything at all in ktls_ocf.c. If you do use OCF you need to allocate a completely different session since it is doing AES-CTR, not AES-GCM. However, you could also just create a single session for this in uipc_ktls.c and use per-op keys instead of a session key. But the idea is that back in ktls_decrypt in the MIXED case you would simply do the encrypt zeros + XOR and then fall through to the "ENCRYPTED" case that calls the normal callback here. But in particular, you shouldn't be trying to reuse the existing routines here to encrypt, you should be writing completely new code to encrypt the block of zeroes. You want to just do the encryption (AES-CTR for AES-GCM) without the MAC. If it's a single span that is block aligned at the start (but doesn't have to be at the end) you can even construct the request (if you use OCF) to just encrypt (or decrypt, in AES-CTR it's all the same) that one span of the mbuf chain directly without even using the 16KB block. (This optimization is probably only worth doing if you know it's always a single span at the end, which I don't see how it would be otherwise unless the NIC decrypts the packet _after_ a drop that is still within the known TLS header length which would be awfully weird since you know in that case you have to undo the work anyway). In fact, I still don't understand how the case can ever be anything but that the you have decrypted the start of a TLS record and had to stop somewhere in the middle due to a drop, and the rest of the TLS record is then still encrypted (as presumably the NIC just stops decrypting everything else in the stream once a drop happens (TCP sequence mismatch) until you get back in sync again after fixing up a record). In that case you can use an AES-CTR request on the first part of the TLS record (it will always start at offset 0 in terms of the encryption) and a length just of the decrypted bits without the need for the block of zeroes. jhb: If you use enc_xform they are not. Also, I still think you should have each ktls worker thread… | |||||
return (error); | |||||
} | |||||
static int | |||||
ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state, | ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state, | ||||
struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, | struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, | ||||
int outiovcnt) | int outiovcnt) | ||||
{ | { | ||||
const struct tls_record_layer *hdr; | const struct tls_record_layer *hdr; | ||||
struct uio *uio; | struct uio *uio; | ||||
struct tls_aead_data_13 *ad; | struct tls_aead_data_13 *ad; | ||||
struct cryptop *crp; | struct cryptop *crp; | ||||
▲ Show 20 Lines • Show All 221 Lines • ▼ Show 20 Lines | if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 || | ||||
tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) { | tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) { | ||||
if (direction == KTLS_TX) { | if (direction == KTLS_TX) { | ||||
if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) | if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) | ||||
tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt; | tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt; | ||||
else | else | ||||
tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt; | tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt; | ||||
} else { | } else { | ||||
tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt; | tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt; | ||||
tls->sw_recrypt = ktls_ocf_tls12_aead_recrypt; | |||||
} | } | ||||
} else { | } else { | ||||
tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt; | tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt; | ||||
if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) { | if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) { | ||||
os->implicit_iv = true; | os->implicit_iv = true; | ||||
memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN); | memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
os->next_seqno = tls->next_seqno; | os->next_seqno = tls->next_seqno; | ||||
Show All 13 Lines |
Does the Mellanox NIC support hardware NIC TLS for ChaCha20-Poly1305?