Index: sys/kern/uipc_ktls.c =================================================================== --- sys/kern/uipc_ktls.c +++ sys/kern/uipc_ktls.c @@ -1193,11 +1193,6 @@ if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) return (ENOTSUP); - /* TLS 1.3 is not yet supported. */ - if (en->tls_vmajor == TLS_MAJOR_VER_ONE && - en->tls_vminor == TLS_MINOR_VER_THREE) - return (ENOTSUP); - error = ktls_create_session(so, en, &tls); if (error) return (error); @@ -1911,6 +1906,53 @@ return (top); } +/* + * Determine the length of the trailing zero padding and find the real + * record type in the byte before the padding. + * + * Walking the mbuf chain backwards is clumsy, so another option would + * be to scan forwards remembering the last non-zero byte before the + * trailer. However, it would be expensive to scan the entire record. + * Instead, find the last non-zero byte of each mbuf in the chain + * keeping track of the relative offset of that nonzero byte. + * + * trail_len is the size of the MAC/tag on input and is set to the + * size of the full trailer including padding and the record type on + * return. + */ +static int +tls13_find_record_type(struct ktls_session *tls, struct mbuf *m, int tls_len, + int *trailer_len, uint8_t *record_typep) +{ + char *cp; + u_int digest_start, last_offset, m_len, offset; + uint8_t record_type; + + digest_start = tls_len - *trailer_len; + last_offset = 0; + offset = 0; + for (; m != NULL && offset < digest_start; + offset += m->m_len, m = m->m_next) { + /* Don't look for padding in the tag. */ + m_len = min(digest_start - offset, m->m_len); + cp = mtod(m, char *); + + /* Find last non-zero byte in this mbuf. */ + while (m_len > 0 && cp[m_len - 1] == 0) + m_len--; + if (m_len > 0) { + record_type = cp[m_len - 1]; + last_offset = offset + m_len; + } + } + if (last_offset < tls->params.tls_hlen) + return (EBADMSG); + + *record_typep = record_type; + *trailer_len = tls_len - last_offset + 1; + return (0); +} + static void ktls_decrypt(struct socket *so) { @@ -1922,6 +1964,8 @@ struct mbuf *control, *data, *m; uint64_t seqno; int error, remain, tls_len, trail_len; + bool tls13; + uint8_t vminor, record_type; hdr = (struct tls_record_layer *)tls_header; sb = &so->so_rcv; @@ -1932,6 +1976,11 @@ tls = sb->sb_tls_info; MPASS(tls != NULL); + tls13 = (tls->params.tls_vminor == TLS_MINOR_VER_THREE); + if (tls13) + vminor = TLS_MINOR_VER_TWO; + else + vminor = tls->params.tls_vminor; for (;;) { /* Is there enough queued for a TLS header? */ if (sb->sb_tlscc < tls->params.tls_hlen) @@ -1941,7 +1990,9 @@ tls_len = sizeof(*hdr) + ntohs(hdr->tls_length); if (hdr->tls_vmajor != tls->params.tls_vmajor || - hdr->tls_vminor != tls->params.tls_vminor) + hdr->tls_vminor != vminor) + error = EINVAL; + else if (tls13 && hdr->tls_type != TLS_RLTYPE_APP) error = EINVAL; else if (tls_len < tls->params.tls_hlen || tls_len > tls->params.tls_hlen + TLS_MAX_MSG_SIZE_V10_2 + @@ -1984,6 +2035,13 @@ SOCKBUF_UNLOCK(sb); error = tls->sw_decrypt(tls, hdr, data, seqno, &trail_len); + if (error == 0) { + if (tls13) + error = tls13_find_record_type(tls, data, + tls_len, &trail_len, &record_type); + else + record_type = hdr->tls_type; + } if (error) { counter_u64_add(ktls_offload_failed_crypto, 1); @@ -2016,7 +2074,7 @@ } /* Allocate the control mbuf. */ - tgr.tls_type = hdr->tls_type; + tgr.tls_type = record_type; tgr.tls_vmajor = hdr->tls_vmajor; tgr.tls_vminor = hdr->tls_vminor; tgr.tls_length = htobe16(tls_len - tls->params.tls_hlen - Index: sys/opencrypto/ktls_ocf.c =================================================================== --- sys/opencrypto/ktls_ocf.c +++ sys/opencrypto/ktls_ocf.c @@ -106,11 +106,21 @@ CTLFLAG_RD, &ocf_tls12_chacha20_encrypts, "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations"); +static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_decrypts); +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_decrypts, + CTLFLAG_RD, &ocf_tls13_gcm_decrypts, + "Total number of OCF TLS 1.3 GCM decryption operations"); + static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_encrypts); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_encrypts, CTLFLAG_RD, &ocf_tls13_gcm_encrypts, "Total number of OCF TLS 1.3 GCM encryption operations"); +static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_decrypts); +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_decrypts, + CTLFLAG_RD, &ocf_tls13_chacha20_decrypts, + "Total number of OCF TLS 1.3 Chacha20-Poly1305 decryption operations"); + static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_encrypts); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_encrypts, CTLFLAG_RD, &ocf_tls13_chacha20_encrypts, @@ -600,6 +610,58 @@ return (error); } +static int +ktls_ocf_tls13_aead_decrypt(struct ktls_session *tls, + const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, + int *trailer_len) +{ + struct tls_aead_data_13 ad; + struct cryptop crp; + struct ktls_ocf_session *os; + int error; + u_int tag_len; + + os = tls->ocf_session; + + tag_len = tls->params.tls_tlen - 1; + + /* Payload must contain at least one byte for the record type. */ + if (ntohs(hdr->tls_length) < tag_len + 1) + return (EBADMSG); + + crypto_initreq(&crp, os->sid); + + /* Setup the nonce. */ + memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len); + *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno); + + /* Setup the AAD. */ + ad.type = hdr->tls_type; + ad.tls_vmajor = hdr->tls_vmajor; + ad.tls_vminor = hdr->tls_vminor; + ad.tls_length = hdr->tls_length; + crp.crp_aad = &ad; + crp.crp_aad_length = sizeof(ad); + + crp.crp_payload_start = tls->params.tls_hlen; + crp.crp_payload_length = ntohs(hdr->tls_length) - tag_len; + crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; + + crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; + crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; + crypto_use_mbuf(&crp, m); + + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) + counter_u64_add(ocf_tls13_gcm_decrypts, 1); + else + counter_u64_add(ocf_tls13_chacha20_decrypts, 1); + error = ktls_ocf_dispatch(os, &crp); + + crypto_destroyreq(&crp); + *trailer_len = tag_len; + return (error); +} + void ktls_ocf_free(struct ktls_session *tls) { @@ -639,11 +701,6 @@ tls->params.tls_vminor > TLS_MINOR_VER_THREE) return (EPROTONOSUPPORT); - /* TLS 1.3 is not yet supported for receive. */ - if (direction == KTLS_RX && - tls->params.tls_vminor == TLS_MINOR_VER_THREE) - return (EPROTONOSUPPORT); - csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; csp.csp_mode = CSP_MODE_AEAD; csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; @@ -711,11 +768,6 @@ tls->params.tls_vminor > TLS_MINOR_VER_THREE) return (EPROTONOSUPPORT); - /* TLS 1.3 is not yet supported for receive. */ - if (direction == KTLS_RX && - tls->params.tls_vminor == TLS_MINOR_VER_THREE) - return (EPROTONOSUPPORT); - csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; csp.csp_mode = CSP_MODE_AEAD; csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; @@ -759,7 +811,10 @@ else tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt; } else { - tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt; + if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) + tls->sw_decrypt = ktls_ocf_tls13_aead_decrypt; + else + tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt; } } else { tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt; Index: tests/sys/kern/ktls_test.c =================================================================== --- tests/sys/kern/ktls_test.c +++ tests/sys/kern/ktls_test.c @@ -816,26 +816,74 @@ return (record_len); } +static size_t +encrypt_tls_13_aead(struct tls_enable *en, uint8_t record_type, uint64_t seqno, + const void *src, size_t len, void *dst, size_t padding) +{ + struct tls_record_layer *hdr; + struct tls_aead_data_13 aad; + char nonce[12]; + char *buf; + size_t hdr_len, mac_len, record_len; + + hdr = dst; + + hdr_len = tls_header_len(en); + mac_len = tls_mac_len(en); + record_len = hdr_len + len + 1 + padding + mac_len; + + hdr->tls_type = TLS_RLTYPE_APP; + hdr->tls_vmajor = TLS_MAJOR_VER_ONE; + hdr->tls_vminor = TLS_MINOR_VER_TWO; + hdr->tls_length = htons(record_len - sizeof(*hdr)); + + tls_13_aad(en, hdr, seqno, &aad); + tls_13_nonce(en, seqno, nonce); + + /* + * Have to use a temporary buffer for the input so that the record + * type can be appended. + */ + buf = malloc(len + 1 + padding); + memcpy(buf, src, len); + buf[len] = record_type; + memset(buf + len + 1, 0, padding); + + ATF_REQUIRE(aead_encrypt(tls_EVP_CIPHER(en), en->cipher_key, nonce, + &aad, sizeof(aad), buf, (char *)dst + hdr_len, len + 1 + padding, + (char *)dst + hdr_len + len + 1 + padding, mac_len)); + + free(buf); + + return (record_len); +} + static size_t encrypt_tls_aead(struct tls_enable *en, uint8_t record_type, uint64_t seqno, - const void *src, size_t len, void *dst, size_t avail) + const void *src, size_t len, void *dst, size_t avail, size_t padding) { size_t record_len; - record_len = tls_header_len(en) + len + tls_trailer_len(en); + record_len = tls_header_len(en) + len + padding + tls_trailer_len(en); ATF_REQUIRE(record_len <= avail); - ATF_REQUIRE(encrypt_tls_12_aead(en, record_type, seqno, src, len, - dst) == record_len); + if (en->tls_vminor == TLS_MINOR_VER_TWO) { + ATF_REQUIRE(padding == 0); + ATF_REQUIRE(encrypt_tls_12_aead(en, record_type, seqno, src, + len, dst) == record_len); + } else + ATF_REQUIRE(encrypt_tls_13_aead(en, record_type, seqno, src, + len, dst, padding) == record_len); return (record_len); } static size_t encrypt_tls_record(struct tls_enable *en, uint8_t record_type, uint64_t seqno, - const void *src, size_t len, void *dst, size_t avail) + const void *src, size_t len, void *dst, size_t avail, size_t padding) { - return (encrypt_tls_aead(en, record_type, seqno, src, len, dst, avail)); + return (encrypt_tls_aead(en, record_type, seqno, src, len, dst, avail, + padding)); } static void @@ -1121,14 +1169,19 @@ tgr = (struct tls_get_record *)CMSG_DATA(cmsg); ATF_REQUIRE(tgr->tls_type == record_type); ATF_REQUIRE(tgr->tls_vmajor == en->tls_vmajor); - ATF_REQUIRE(tgr->tls_vminor == en->tls_vminor); + /* XXX: Not sure if this is what OpenSSL expects? */ + if (en->tls_vminor == TLS_MINOR_VER_THREE) + ATF_REQUIRE(tgr->tls_vminor == TLS_MINOR_VER_TWO); + else + ATF_REQUIRE(tgr->tls_vminor == en->tls_vminor); ATF_REQUIRE(tgr->tls_length == htons(rv)); return (rv); } static void -test_ktls_receive_app_data(struct tls_enable *en, uint64_t seqno, size_t len) +test_ktls_receive_app_data(struct tls_enable *en, uint64_t seqno, size_t len, + size_t padding) { struct kevent ev; char *plaintext, *received, *outbuf; @@ -1169,11 +1222,11 @@ if (outbuf_len == 0) { ATF_REQUIRE(written < len); todo = len - written; - if (todo > TLS_MAX_MSG_SIZE_V10_2) - todo = TLS_MAX_MSG_SIZE_V10_2; + if (todo > TLS_MAX_MSG_SIZE_V10_2 - padding) + todo = TLS_MAX_MSG_SIZE_V10_2 - padding; outbuf_len = encrypt_tls_record(en, TLS_RLTYPE_APP, seqno, plaintext + written, - todo, outbuf, outbuf_cap); + todo, outbuf, outbuf_cap, padding); outbuf_sent = 0; written += todo; seqno++; @@ -1229,14 +1282,6 @@ M(aes256_cbc_1_0_sha1, CRYPTO_AES_CBC, 256 / 8, \ CRYPTO_SHA1_HMAC) -#define TLS_12_TESTS(M) \ - M(aes128_gcm_1_2, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ - TLS_MINOR_VER_TWO) \ - M(aes256_gcm_1_2, CRYPTO_AES_NIST_GCM_16, 256 / 8, 0, \ - TLS_MINOR_VER_TWO) \ - M(chacha20_poly1305_1_2, CRYPTO_CHACHA20_POLY1305, 256 / 8, 0, \ - TLS_MINOR_VER_TWO) - #define TLS_13_TESTS(M) \ M(aes128_gcm_1_3, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ TLS_MINOR_VER_THREE) \ @@ -1528,7 +1573,7 @@ INVALID_CIPHER_SUITES(GEN_INVALID_TRANSMIT_TEST); #define GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ - auth_alg, minor, name, len) \ + auth_alg, minor, name, len, padding) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_##name); \ ATF_TC_BODY(ktls_receive_##cipher_name##_##name, tc) \ { \ @@ -1539,7 +1584,7 @@ seqno = random(); \ build_tls_enable(cipher_alg, key_size, auth_alg, minor, seqno, \ &en); \ - test_ktls_receive_app_data(&en, seqno, len); \ + test_ktls_receive_app_data(&en, seqno, len, padding); \ free_tls_enable(&en); \ } @@ -1550,9 +1595,9 @@ #define GEN_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, auth_alg, \ minor) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ - auth_alg, minor, short, 64) \ + auth_alg, minor, short, 64, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ - auth_alg, minor, long, 64 * 1024) + auth_alg, minor, long, 64 * 1024, 0) #define ADD_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, auth_alg, \ minor) \ @@ -1573,7 +1618,28 @@ * Note that receive is currently only supported for TLS 1.2 AEAD * cipher suites. */ -TLS_12_TESTS(GEN_RECEIVE_TESTS); +AES_GCM_TESTS(GEN_RECEIVE_TESTS); +CHACHA20_TESTS(GEN_RECEIVE_TESTS); + +#define GEN_PADDING_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, \ + auth_alg, minor) \ + GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ + auth_alg, minor, short_padded, 64, 16) \ + GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ + auth_alg, minor, long_padded, 64 * 1024, 15) + +#define ADD_PADDING_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, \ + auth_alg, minor) \ + ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ + auth_alg, minor, short_padded) \ + ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ + auth_alg, minor, long_padded) + +/* + * For TLS 1.3 cipher suites, run two additional receive tests which + * use add padding to each record. + */ +TLS_13_TESTS(GEN_PADDING_RECEIVE_TESTS); static void test_ktls_invalid_receive_cipher_suite(struct tls_enable *en) @@ -1584,12 +1650,7 @@ ATF_REQUIRE(setsockopt(sockets[1], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == -1); - - /* - * XXX: TLS 1.3 fails with ENOTSUP before checking for invalid - * ciphers. - */ - ATF_REQUIRE(errno == EINVAL || errno == ENOTSUP); + ATF_REQUIRE(errno == EINVAL); ATF_REQUIRE(close(sockets[1]) == 0); ATF_REQUIRE(close(sockets[0]) == 0); @@ -1629,7 +1690,7 @@ ATF_REQUIRE(setsockopt(sockets[1], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == -1); - ATF_REQUIRE(errno == EPROTONOSUPPORT || errno == ENOTSUP); + ATF_REQUIRE(errno == EPROTONOSUPPORT); ATF_REQUIRE(close(sockets[1]) == 0); ATF_REQUIRE(close(sockets[0]) == 0); @@ -1660,7 +1721,6 @@ * rejected. */ AES_CBC_TESTS(GEN_UNSUPPORTED_RECEIVE_TEST); -TLS_13_TESTS(GEN_UNSUPPORTED_RECEIVE_TEST); ATF_TP_ADD_TCS(tp) { @@ -1674,8 +1734,9 @@ /* Receive tests */ AES_CBC_TESTS(ADD_UNSUPPORTED_RECEIVE_TEST); - TLS_12_TESTS(ADD_RECEIVE_TESTS); - TLS_13_TESTS(ADD_UNSUPPORTED_RECEIVE_TEST); + AES_GCM_TESTS(ADD_RECEIVE_TESTS); + CHACHA20_TESTS(ADD_RECEIVE_TESTS); + TLS_13_TESTS(ADD_PADDING_RECEIVE_TESTS); INVALID_CIPHER_SUITES(ADD_INVALID_RECEIVE_TEST); return (atf_no_error());