Index: crypto/openssl/crypto/evp/digest.c =================================================================== --- crypto/openssl/crypto/evp/digest.c +++ crypto/openssl/crypto/evp/digest.c @@ -185,7 +185,11 @@ } else /* Ask if an ENGINE is reserved for this job */ impl = ENGINE_get_digest_engine(type->type); - if (impl) { + if (impl +#ifdef CHSSL_OFFLOAD + && !ctx->is_chssl +#endif + ) { /* There's an ENGINE for this job ... (apparently) */ const EVP_MD *d = ENGINE_get_digest(impl, type->type); if (!d) { @@ -364,6 +368,26 @@ return ret; } +#ifdef CHSSL_OFFLOAD +int CHSSL_EVP_Digest(const void *data, size_t count, + unsigned char *md, unsigned int *size, + const EVP_MD *type, ENGINE *impl) +{ + EVP_MD_CTX ctx; + int ret; + + EVP_MD_CTX_init(&ctx); + ctx.is_chssl = 1; + EVP_MD_CTX_set_flags(&ctx,EVP_MD_CTX_FLAG_ONESHOT); + ret=EVP_DigestInit_ex(&ctx, type, impl) + && EVP_DigestUpdate(&ctx, data, count) + && EVP_DigestFinal_ex(&ctx, md, size); + EVP_MD_CTX_cleanup(&ctx); + + return ret; +} +#endif + void EVP_MD_CTX_destroy(EVP_MD_CTX *ctx) { if (ctx) { Index: crypto/openssl/crypto/evp/evp.h =================================================================== --- crypto/openssl/crypto/evp/evp.h +++ crypto/openssl/crypto/evp/evp.h @@ -270,6 +270,9 @@ ENGINE *engine; /* functional reference if 'digest' is * ENGINE-provided */ unsigned long flags; +#ifdef CHSSL_OFFLOAD + unsigned short is_chssl; +#endif void *md_data; /* Public key context for sign/verify */ EVP_PKEY_CTX *pctx; @@ -597,6 +600,11 @@ int EVP_Digest(const void *data, size_t count, unsigned char *md, unsigned int *size, const EVP_MD *type, ENGINE *impl); +#ifdef CHSSL_OFFLOAD +int CHSSL_EVP_Digest(const void *data, size_t count, + unsigned char *md, unsigned int *size, const EVP_MD *type, + ENGINE *impl); +#endif int EVP_MD_CTX_copy(EVP_MD_CTX *out, const EVP_MD_CTX *in); int EVP_DigestInit(EVP_MD_CTX *ctx, const EVP_MD *type); Index: crypto/openssl/crypto/evp/m_sha1.c =================================================================== --- crypto/openssl/crypto/evp/m_sha1.c +++ crypto/openssl/crypto/evp/m_sha1.c @@ -68,6 +68,19 @@ # include # endif +#ifdef CHSSL_OFFLOAD +#define uint64_t unsigned long long +#define htonll(data) ( \ + (((uint64_t)(data) >> 56) & 0x00000000000000FF) | \ + (((uint64_t)(data) >> 40) & 0x000000000000FF00) | \ + (((uint64_t)(data) >> 24) & 0x0000000000FF0000) | \ + (((uint64_t)(data) >> 8 ) & 0x00000000FF000000) | \ + (((uint64_t)(data) << 8) & 0x000000FF00000000) | \ + (((uint64_t)(data) << 24) & 0x0000FF0000000000) | \ + (((uint64_t)(data) << 40) & 0x00FF000000000000) | \ + (((uint64_t)(data) << 56) & 0xFF00000000000000)) +#endif + static int init(EVP_MD_CTX *ctx) { return SHA1_Init(ctx->md_data); @@ -80,6 +93,20 @@ static int final(EVP_MD_CTX *ctx, unsigned char *md) { +#ifdef CHSSL_OFFLOAD + if (ctx->is_chssl) { + unsigned int *temp; + unsigned char *iopad; + int i; + temp = ctx->md_data; + for (i=0; i< 5; i++) + temp[i] = htonl(temp[i]); + iopad = (unsigned char *)temp; + memcpy(md, iopad, 20); + return 1; + } +#endif + return SHA1_Final(md, ctx->md_data); } @@ -127,6 +154,20 @@ static int final256(EVP_MD_CTX *ctx, unsigned char *md) { +#ifdef CHSSL_OFFLOAD + if (ctx->is_chssl) { + unsigned int *temp; + unsigned char *iopad; + int i; + + temp = ctx->md_data; + for (i=0; i< 8; i++) + temp[i] = htonl(temp[i]); + iopad = (unsigned char *)temp; + memcpy(md, iopad, 32); + return 1; + } +#endif return SHA256_Final(md, ctx->md_data); } @@ -190,6 +231,21 @@ static int final512(EVP_MD_CTX *ctx, unsigned char *md) { +#ifdef CHSSL_OFFLOAD + if (ctx->is_chssl) { + unsigned long long *temp; + unsigned char *iopad; + int i; + + temp = ctx->md_data; + for (i = 0; i < 8; i++) + temp[i] = htonll(temp[i]); + iopad = (unsigned char *)temp; + memcpy(md, iopad, 64); + return 1; + } +#endif + return SHA512_Final(md, ctx->md_data); } Index: crypto/openssl/ssl/Makefile =================================================================== --- crypto/openssl/ssl/Makefile +++ crypto/openssl/ssl/Makefile @@ -31,7 +31,8 @@ ssl_lib.c ssl_err2.c ssl_cert.c ssl_sess.c \ ssl_ciph.c ssl_stat.c ssl_rsa.c \ ssl_asn1.c ssl_txt.c ssl_algs.c ssl_conf.c \ - bio_ssl.c ssl_err.c kssl.c t1_reneg.c tls_srp.c t1_trce.c ssl_utst.c + bio_ssl.c ssl_err.c kssl.c t1_reneg.c tls_srp.c t1_trce.c ssl_utst.c \ + ssl_tom.c LIBOBJ= \ s2_meth.o s2_srvr.o s2_clnt.o s2_lib.o s2_enc.o s2_pkt.o \ s3_meth.o s3_srvr.o s3_clnt.o s3_lib.o s3_enc.o s3_pkt.o s3_both.o s3_cbc.o \ @@ -42,7 +43,8 @@ ssl_lib.o ssl_err2.o ssl_cert.o ssl_sess.o \ ssl_ciph.o ssl_stat.o ssl_rsa.o \ ssl_asn1.o ssl_txt.o ssl_algs.o ssl_conf.o \ - bio_ssl.o ssl_err.o kssl.o t1_reneg.o tls_srp.o t1_trce.o ssl_utst.o + bio_ssl.o ssl_err.o kssl.o t1_reneg.o tls_srp.o t1_trce.o ssl_utst.o \ + ssl_tom.o SRC= $(LIBSRC) @@ -1122,3 +1124,4 @@ tls_srp.o: ../include/openssl/stack.h ../include/openssl/symhacks.h tls_srp.o: ../include/openssl/tls1.h ../include/openssl/x509.h tls_srp.o: ../include/openssl/x509_vfy.h ssl_locl.h tls_srp.c +ssl_tom.o: ../include/openssl/x509_vfy.h ssl_tom.c Index: crypto/openssl/ssl/d1_clnt.c =================================================================== --- crypto/openssl/ssl/d1_clnt.c +++ crypto/openssl/ssl/d1_clnt.c @@ -127,6 +127,9 @@ #ifndef OPENSSL_NO_DH # include #endif +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) +#include "ssl_tom.h" +#endif static const SSL_METHOD *dtls1_get_client_method(int ver); static int dtls1_get_hello_verify(SSL *s); @@ -615,6 +618,10 @@ case SSL3_ST_CW_FINISHED_A: case SSL3_ST_CW_FINISHED_B: +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + if (SSL_ofld(s)) + chssl_program_hwkey_context(s, KEY_WRITE_TX, SSL_ST_CONNECT); +#endif if (!s->hit) dtls1_start_timer(s); ret = ssl3_send_finished(s, @@ -700,6 +707,10 @@ SSL3_ST_CR_FINISHED_B); if (ret <= 0) goto end; +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + if (SSL_ofld_rx(s)) + chssl_program_hwkey_context(s, KEY_WRITE_RX,SSL_ST_CONNECT); +#endif dtls1_stop_timer(s); if (s->hit) Index: crypto/openssl/ssl/d1_pkt.c =================================================================== --- crypto/openssl/ssl/d1_pkt.c +++ crypto/openssl/ssl/d1_pkt.c @@ -121,6 +121,9 @@ #include #include #include +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) +#include "ssl_tom.h" +#endif /* mod 128 saturating subtract of two 64-bit values in big-endian order */ static int satsub64be(const unsigned char *v1, const unsigned char *v2) @@ -503,6 +506,10 @@ /* decrypt in place in 'rr->input' */ rr->data = rr->input; +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + if (!(SSL_enc_offload(s) && SSL_ofld_vers(s) && SSL_ofld_rx(s))) + { +#endif enc_err = s->method->ssl3_enc->enc(s, 0); /*- * enc_err is: @@ -516,6 +523,9 @@ s->packet_length = 0; goto err; } +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + } +#endif #ifdef TLS_DEBUG printf("dec %d\n", rr->length); { @@ -528,7 +538,11 @@ /* r->length is now the compressed data plus mac */ if ((sess != NULL) && - (s->enc_read_ctx != NULL) && (EVP_MD_CTX_md(s->read_hash) != NULL)) { + (s->enc_read_ctx != NULL) && (EVP_MD_CTX_md(s->read_hash) != NULL) +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + && !(SSL_mac_offload(s) && SSL_ofld_vers(s) && SSL_ofld_rx(s)) +#endif + ) { /* s->read_hash != NULL => mac_size != -1 */ unsigned char *mac = NULL; unsigned char mac_tmp[EVP_MAX_MD_SIZE]; @@ -648,7 +662,7 @@ int dtls1_get_record(SSL *s) { int ssl_major, ssl_minor; - int i, n; + int i, n, al; SSL3_RECORD *rr; unsigned char *p = NULL; unsigned short version; @@ -684,6 +698,13 @@ goto again; } +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + #define SSL3_RT_CHERROR 127 + if (SSL_ofld_vers(s) && SSL_ofld_rx(s) && (*(p) == SSL3_RT_CHERROR)) { + s->rstate = SSL_ST_READ_ERROR; + } + else +#endif s->rstate = SSL_ST_READ_BODY; p = s->packet; @@ -733,11 +754,27 @@ /* now s->rstate == SSL_ST_READ_BODY */ } +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + if (SSL_Chelsio_ofld(s) && SSL_ofld_rx(s)) { + if (s->rstate == SSL_ST_READ_ERROR) { + i = rr->length; + n = ssl3_read_n(s, i, i, 1); + s->rstate = SSL_ST_READ_HEADER; + if (n<=0) return n; + al = chssl_process_cherror(s); + printf("%s rstate SSL_ST_READ_ERROR \n",__func__); + goto again; + } + } + /* this may not be required len in Hdr = '13bytes + decrypted payload' */ +#endif + /* s->rstate == SSL_ST_READ_BODY, get and decode the data */ if (rr->length > s->packet_length - DTLS1_RT_HEADER_LENGTH) { /* now s->packet_length == DTLS1_RT_HEADER_LENGTH */ i = rr->length; + printf("%s i:%d rrl:%d pl:%d\n",__func__,i,rr->length,s->packet_length); n = ssl3_read_n(s, i, i, 1); /* this packet contained a partial record, dump it */ if (n != i) { @@ -1680,7 +1717,11 @@ p += 10; /* Explicit IV length, block ciphers appropriate version flag */ - if (s->enc_write_ctx) { + if (s->enc_write_ctx +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + && !(SSL_enc_offload(s) && SSL_ofld_vers(s) && SSL_Tx_keys(s)) +#endif + ) { int mode = EVP_CIPHER_CTX_mode(s->enc_write_ctx); if (mode == EVP_CIPH_CBC_MODE) { eivlen = EVP_CIPHER_CTX_iv_length(s->enc_write_ctx); @@ -1721,7 +1762,11 @@ * wb->buf */ - if (mac_size != 0) { + if (mac_size != 0 +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + && !(SSL_enc_offload(s) && SSL_ofld_vers(s) && SSL_Tx_keys(s)) +#endif + ) { if (s->method->ssl3_enc->mac(s, &(p[wr->length + eivlen]), 1) < 0) goto err; wr->length += mac_size; @@ -1734,8 +1779,14 @@ if (eivlen) wr->length += eivlen; +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + if (!(SSL_enc_offload(s) && SSL_ofld_vers(s) && SSL_Tx_keys(s))) { +#endif if (s->method->ssl3_enc->enc(s, 1) < 1) goto err; +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + } +#endif /* record length after mac and block padding */ /* Index: crypto/openssl/ssl/d1_srvr.c =================================================================== --- crypto/openssl/ssl/d1_srvr.c +++ crypto/openssl/ssl/d1_srvr.c @@ -125,6 +125,9 @@ #ifndef OPENSSL_NO_DH # include #endif +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) +#include "ssl_tom.h" +#endif static const SSL_METHOD *dtls1_get_server_method(int ver); static int dtls1_send_hello_verify_request(SSL *s); @@ -752,6 +755,10 @@ SSL3_ST_SR_FINISHED_B); if (ret <= 0) goto end; +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + if (SSL_ofld_rx(s)) + chssl_program_hwkey_context(s, KEY_WRITE_RX, SSL_ST_ACCEPT); +#endif dtls1_stop_timer(s); if (s->hit) s->state = SSL_ST_OK; @@ -812,8 +819,14 @@ 0, NULL); } #endif - +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + if (SSL_ofld(s)) { + s->s3->tmp.next_state = SSL3_ST_SW_FINISHED_A; + s->state = SSL3_ST_SW_FLUSH; + } else { s->state = SSL3_ST_SW_FINISHED_A; + } +#endif s->init_num = 0; if (!s->method->ssl3_enc->change_cipher_state(s, @@ -829,6 +842,10 @@ case SSL3_ST_SW_FINISHED_A: case SSL3_ST_SW_FINISHED_B: +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_DTLS) + if (SSL_ofld(s)) + chssl_program_hwkey_context(s, KEY_WRITE_TX, SSL_ST_ACCEPT); +#endif ret = ssl3_send_finished(s, SSL3_ST_SW_FINISHED_A, SSL3_ST_SW_FINISHED_B, Index: crypto/openssl/ssl/s23_clnt.c =================================================================== --- crypto/openssl/ssl/s23_clnt.c +++ crypto/openssl/ssl/s23_clnt.c @@ -115,6 +115,9 @@ #include #include #include +#ifdef CHSSL_OFFLOAD +#include "ssl_tom.h" +#endif static const SSL_METHOD *ssl23_get_client_method(int ver); static int ssl23_client_hello(SSL *s); @@ -221,6 +224,10 @@ ret = ssl23_client_hello(s); if (ret <= 0) goto end; +#ifdef CHSSL_OFFLOAD + if (chssl_new(s)) + ssl_tls_offload(s); +#endif s->state = SSL23_ST_CR_SRVR_HELLO_A; s->init_num = 0; Index: crypto/openssl/ssl/s23_srvr.c =================================================================== --- crypto/openssl/ssl/s23_srvr.c +++ crypto/openssl/ssl/s23_srvr.c @@ -118,6 +118,9 @@ #ifdef OPENSSL_FIPS # include #endif +#ifdef CHSSL_OFFLOAD +#include "ssl_tom.h" +#endif static const SSL_METHOD *ssl23_get_server_method(int ver); int ssl23_get_client_hello(SSL *s); @@ -181,6 +184,10 @@ /* s->version=SSL3_VERSION; */ s->type = SSL_ST_ACCEPT; +#ifdef CHSSL_OFFLOAD + if (chssl_new(s)) + ssl_tls_offload(s); +#endif if (s->init_buf == NULL) { if ((buf = BUF_MEM_new()) == NULL) { Index: crypto/openssl/ssl/s3_both.c =================================================================== --- crypto/openssl/ssl/s3_both.c +++ crypto/openssl/ssl/s3_both.c @@ -123,6 +123,9 @@ #include #include #include +#ifdef CHSSL_OFFLOAD +#include "ssl_tom.h" +#endif /* * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or @@ -131,6 +134,9 @@ int ssl3_do_write(SSL *s, int type) { int ret; +#ifdef TLS_DEBUG + printf("%s type:%d len:%d \n",__func__,type,s->init_num); +#endif ret = ssl3_write_bytes(s, type, &s->init_buf->data[s->init_off], s->init_num); @@ -161,6 +167,9 @@ unsigned char *p; int i; unsigned long l; +#ifdef TLS_DEBUG + printf("ssl3_send_finished: state:%x a:%x b%x slen:%d M:%x s:%p\n",s->state,a,b,slen,s->method->version,s); +#endif if (s->state == a) { p = ssl_handshake_start(s); @@ -168,6 +177,9 @@ i = s->method->ssl3_enc->final_finish_mac(s, sender, slen, s->s3->tmp.finish_md); +#ifdef TLS_DEBUG + printf("ssl3_send_finished: i:%d slen:%d ver:%x s:%p\n",i,slen,s->method->version,s); +#endif if (i <= 0) return 0; s->s3->tmp.finish_md_len = i; @@ -244,10 +256,22 @@ * spec message and is in s->s3->tmp.peer_finish_md */ #endif +#ifdef TLS_DEBUG + printf("Enter %s \n",__func__); +#endif /* 64 argument should actually be 36+4 :-) */ +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + /* XXX: Why a smaller maximum message size? */ + if (SSL_ofld_rx(s)) + n = s->method->ssl_get_message(s, a, b, SSL3_MT_FINISHED, 16, &ok); + else +#endif n = s->method->ssl_get_message(s, a, b, SSL3_MT_FINISHED, 64, &ok); +#ifdef TLS_DEBUG + printf("%s ok:%d n:%d \n",__func__,ok,n); +#endif if (!ok) return ((int)n); @@ -346,6 +370,9 @@ unsigned long l; long n; int i, al; +#ifdef TLS_DEBUG + printf("ssl3_get_message st1:%d stn:%d mt:%d max:%d\n",st1,stn,mt,max); +#endif if (s->s3->tmp.reuse_message) { s->s3->tmp.reuse_message = 0; Index: crypto/openssl/ssl/s3_clnt.c =================================================================== --- crypto/openssl/ssl/s3_clnt.c +++ crypto/openssl/ssl/s3_clnt.c @@ -166,6 +166,9 @@ #ifndef OPENSSL_NO_ENGINE # include #endif +#ifdef CHSSL_OFFLOAD +#include "ssl_tom.h" +#endif static int ca_dn_cmp(const X509_NAME *const *a, const X509_NAME *const *b); #ifndef OPENSSL_NO_TLSEXT @@ -299,12 +302,22 @@ ret = ssl3_client_hello(s); if (ret <= 0) goto end; +#ifdef CHSSL_OFFLOAD + if (chssl_new(s) || SSL_ofld_vers(s)) + ssl_tls_offload(s); +#endif s->state = SSL3_ST_CR_SRVR_HELLO_A; s->init_num = 0; /* turn on buffering for the next lot of output */ +#ifdef CHSSL_OFFLOAD + if (!(SSL_ofld(s) && s->renegotiate)) { +#endif if (s->bbio != s->wbio) s->wbio = BIO_push(s->bbio, s->wbio); +#ifdef CHSSL_OFFLOAD + } +#endif break; @@ -485,6 +498,14 @@ else s->state = SSL3_ST_CW_FINISHED_A; #endif + +#ifdef CHSSL_OFFLOAD + //if (SSL_ofld(s)) { + s->s3->tmp.next_state = s->state; + s->state = SSL3_ST_CW_FLUSH; + //} +#endif + s->init_num = 0; s->session->cipher = s->s3->tmp.new_cipher; @@ -509,7 +530,6 @@ s->state = SSL_ST_ERR; goto end; } - break; #if !defined(OPENSSL_NO_TLSEXT) && !defined(OPENSSL_NO_NEXTPROTONEG) @@ -524,6 +544,10 @@ case SSL3_ST_CW_FINISHED_A: case SSL3_ST_CW_FINISHED_B: +#ifdef CHSSL_OFFLOAD + if (SSL_ofld(s)) + chssl_program_hwkey_context(s, KEY_WRITE_TX, SSL_ST_CONNECT); +#endif ret = ssl3_send_finished(s, SSL3_ST_CW_FINISHED_A, SSL3_ST_CW_FINISHED_B, @@ -553,7 +577,6 @@ s->s3->tmp.next_state = SSL3_ST_CR_SESSION_TICKET_A; else #endif - s->s3->tmp.next_state = SSL3_ST_CR_FINISHED_A; } s->init_num = 0; Index: crypto/openssl/ssl/s3_pkt.c =================================================================== --- crypto/openssl/ssl/s3_pkt.c +++ crypto/openssl/ssl/s3_pkt.c @@ -112,11 +112,15 @@ #include #include #include + #include #define USE_SOCKETS #include "ssl_locl.h" #include #include #include +#ifdef CHSSL_OFFLOAD +#include "ssl_tom.h" +#endif #ifndef EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK # define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0 @@ -152,6 +156,7 @@ long align = 0; unsigned char *pkt; SSL3_BUFFER *rb; + //printf("%s n:%d \n",__func__,n); if (n <= 0) return n; @@ -161,6 +166,7 @@ if (!ssl3_setup_read_buffer(s)) return -1; + //printf("%s left:%d \n",__func__,left); left = rb->left; #if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0 align = (long)rb->buf + SSL3_RT_HEADER_LENGTH; @@ -212,6 +218,7 @@ s->packet_length += n; rb->left = left - n; rb->offset += n; + //printf("%s n:%d off:%d\n",__func__,n,rb->offset); return (n); } @@ -228,6 +235,7 @@ s->packet = pkt; rb->offset = len + align; } + //printf("%s n:%d rb->len:%d off:%d\n",__func__,n,rb->len,rb->offset); if (n > (int)(rb->len - rb->offset)) { /* does not happen */ SSLerr(SSL_F_SSL3_READ_N, ERR_R_INTERNAL_ERROR); @@ -244,6 +252,7 @@ if (max > (int)(rb->len - rb->offset)) max = rb->len - rb->offset; } + //printf("max:%d \n",max); while (left < n) { /* @@ -260,6 +269,7 @@ SSLerr(SSL_F_SSL3_READ_N, SSL_R_READ_BIO_NOT_SET); i = -1; } + //printf("bio read:%d sf:%d\n",i,s->rbio->num); if (i <= 0) { rb->left = left; @@ -285,6 +295,7 @@ rb->left = left - n; s->packet_length += n; s->rwstate = SSL_NOTHING; + //printf("%s Exit n:%d \n",__func__,n); return (n); } @@ -309,7 +320,7 @@ static int ssl3_get_record(SSL *s) { int ssl_major, ssl_minor, al; - int enc_err, n, i, ret = -1; + int enc_err = 0, n, i, ret = -1; SSL3_RECORD *rr; SSL_SESSION *sess; unsigned char *p; @@ -340,12 +351,25 @@ if ((s->rstate != SSL_ST_READ_BODY) || (s->packet_length < SSL3_RT_HEADER_LENGTH)) { n = ssl3_read_n(s, SSL3_RT_HEADER_LENGTH, s->s3->rbuf.len, 0); - if (n <= 0) + if (n <= 0) { return (n); /* error or non-blocking */ - s->rstate = SSL_ST_READ_BODY; - + } p = s->packet; - if (s->msg_callback) + +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + #define SSL3_RT_CHERROR 127 + if (SSL_ofld_vers(s) && SSL_ofld_rx(s) && (*(p) == SSL3_RT_CHERROR)) { + s->rstate = SSL_ST_READ_ERROR; + } else +#endif + s->rstate = SSL_ST_READ_BODY; + + if (s->msg_callback +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + && (SSL_ofld_vers(s) && SSL_ofld_rx(s) + && (s->rstate != SSL_ST_READ_ERROR)) +#endif + ) s->msg_callback(0, 0, SSL3_RT_HEADER, p, 5, s, s->msg_callback_arg); @@ -399,18 +423,40 @@ /* now s->rstate == SSL_ST_READ_BODY */ } +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + if (SSL_Chelsio_ofld(s) && SSL_ofld_rx(s)) { + if (s->rstate == SSL_ST_READ_ERROR) { + i = rr->length; + n = ssl3_read_n(s, i, i, 1); + s->rstate = SSL_ST_READ_HEADER; + if (n<=0) + return n; + al = chssl_process_cherror(s); + printf("%s rstate SSL_ST_READ_ERROR \n",__func__); + goto f_err; + } + } + + /* this may not be required len in Hdr = '5bytes + decrypted payload' */ +#endif + /* s->rstate == SSL_ST_READ_BODY, get and decode the data */ - if (rr->length > s->packet_length - SSL3_RT_HEADER_LENGTH) { /* now s->packet_length == SSL3_RT_HEADER_LENGTH */ i = rr->length; n = ssl3_read_n(s, i, i, 1); - if (n <= 0) + if (n <= 0) { + //printf("%s n:%d \n",__func__,n); return (n); /* error or non-blocking io */ + } /* * now n == rr->length, and s->packet_length == SSL3_RT_HEADER_LENGTH * + rr->length */ +#if 0 + for(i=0;i<16;i++) + printf("%x ",s->packet[i]); +#endif } s->rstate = SSL_ST_READ_HEADER; /* set state for later operations */ @@ -437,12 +483,17 @@ if (rr->length > SSL3_RT_MAX_ENCRYPTED_LENGTH + extra) { al = SSL_AD_RECORD_OVERFLOW; SSLerr(SSL_F_SSL3_GET_RECORD, SSL_R_ENCRYPTED_LENGTH_TOO_LONG); + printf("%s SSL_R_ENCRYPTED_LENGTH_TOO_LONG \n",__func__); goto f_err; } /* decrypt in place in 'rr->input' */ rr->data = rr->input; +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + if (!(SSL_enc_offload(s) && SSL_ofld_vers(s) && SSL_ofld_rx(s))) +#endif + { enc_err = s->method->ssl3_enc->enc(s, 0); /*- * enc_err is: @@ -453,10 +504,17 @@ if (enc_err == 0) { al = SSL_AD_DECRYPTION_FAILED; SSLerr(SSL_F_SSL3_GET_RECORD, SSL_R_BLOCK_CIPHER_PAD_IS_WRONG); + printf("%s SSL_R_BLOCK_CIPHER_PAD_IS_WRONG \n",__func__); goto f_err; } + } #ifdef TLS_DEBUG - printf("dec %d\n", rr->length); +#ifdef CHSSL_OFFLOAD + printf("\n dec %d host enc:%d mac:%d enc_err:%d\n", + rr->length,SSL_enc_host(s),SSL_mac_host(s),enc_err); +#else + printf("\n dec %d\n", rr->length); +#endif { unsigned int z; for (z = 0; z < rr->length; z++) @@ -467,7 +525,13 @@ /* r->length is now the compressed data plus mac */ if ((sess != NULL) && - (s->enc_read_ctx != NULL) && (EVP_MD_CTX_md(s->read_hash) != NULL)) { + (s->enc_read_ctx != NULL) && + (EVP_MD_CTX_md(s->read_hash) != NULL) +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + && + !(SSL_mac_offload(s) && SSL_ofld_vers(s) && SSL_ofld_rx(s)) +#endif + ) { /* s->read_hash != NULL => mac_size != -1 */ unsigned char *mac = NULL; unsigned char mac_tmp[EVP_MAX_MD_SIZE]; @@ -491,6 +555,7 @@ orig_len < mac_size + 1)) { al = SSL_AD_DECODE_ERROR; SSLerr(SSL_F_SSL3_GET_RECORD, SSL_R_LENGTH_TOO_SHORT); + printf("%s SSL_R_LENGTH_TOO_SHORT \n",__func__); goto f_err; } @@ -533,6 +598,7 @@ al = SSL_AD_BAD_RECORD_MAC; SSLerr(SSL_F_SSL3_GET_RECORD, SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); + printf("%s SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC \n",__func__); goto f_err; } @@ -541,6 +607,7 @@ if (rr->length > SSL3_RT_MAX_COMPRESSED_LENGTH + extra) { al = SSL_AD_RECORD_OVERFLOW; SSLerr(SSL_F_SSL3_GET_RECORD, SSL_R_COMPRESSED_LENGTH_TOO_LONG); + printf("%s SSL_R_COMPRESSED_LENGTH_TOO_LONG \n",__func__); goto f_err; } if (!ssl3_do_uncompress(s)) { @@ -553,6 +620,7 @@ if (rr->length > SSL3_RT_MAX_PLAIN_LENGTH + extra) { al = SSL_AD_RECORD_OVERFLOW; SSLerr(SSL_F_SSL3_GET_RECORD, SSL_R_DATA_LENGTH_TOO_LONG); + printf("%s rr->length :%d extra:%zu\n",__func__,rr->length,extra); goto f_err; } @@ -575,11 +643,13 @@ if (empty_record_count > MAX_EMPTY_RECORDS) { al = SSL_AD_UNEXPECTED_MESSAGE; SSLerr(SSL_F_SSL3_GET_RECORD, SSL_R_RECORD_TOO_SMALL); + printf("rr->length == 0 empty_record_count:%d\n",empty_record_count); + printf("%s SSL_F_SSL3_GET_RECORD \n",__func__); goto f_err; } goto again; } -#if 0 +#ifdef TLS_DEBUG fprintf(stderr, "Ultimate Record type=%d, Length=%d\n", rr->type, rr->length); #endif @@ -701,7 +771,11 @@ SSL_USE_EXPLICIT_IV(s) && s->enc_write_ctx != NULL && EVP_CIPHER_flags(s->enc_write_ctx->cipher) & - EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK) { + EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK +#ifdef CHSSL_OFFLOAD + && !SSL_ofld(s) +#endif + ) { unsigned char aad[13]; EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM mb_param; int packlen; @@ -986,7 +1060,11 @@ plen = p; p += 2; /* Explicit IV length, block ciphers appropriate version flag */ - if (s->enc_write_ctx && SSL_USE_EXPLICIT_IV(s)) { + if (s->enc_write_ctx && SSL_USE_EXPLICIT_IV(s) +#ifdef CHSSL_OFFLOAD + && !(SSL_enc_offload(s) && SSL_ofld_vers(s)) +#endif + ) { int mode = EVP_CIPHER_CTX_mode(s->enc_write_ctx); if (mode == EVP_CIPH_CBC_MODE) { eivlen = EVP_CIPHER_CTX_iv_length(s->enc_write_ctx); @@ -1027,7 +1105,11 @@ * wb->buf */ - if (mac_size != 0) { + if (mac_size != 0 +#ifdef CHSSL_OFFLOAD + && !(SSL_enc_offload(s) && SSL_ofld_vers(s)) +#endif + ) { if (s->method->ssl3_enc->mac(s, &(p[wr->length + eivlen]), 1) < 0) goto err; wr->length += mac_size; @@ -1043,8 +1125,14 @@ wr->length += eivlen; } +#ifdef CHSSL_OFFLOAD + if (!(SSL_enc_offload(s) && SSL_ofld_vers(s))) { +#endif if (s->method->ssl3_enc->enc(s, 1) < 1) goto err; +#ifdef CHSSL_OFFLOAD + } +#endif /* record length after mac and block padding */ s2n(wr->length, plen); @@ -1172,8 +1260,10 @@ void (*cb) (const SSL *ssl, int type2, int val) = NULL; if (s->s3->rbuf.buf == NULL) /* Not initialized yet */ - if (!ssl3_setup_read_buffer(s)) + if (!ssl3_setup_read_buffer(s)) { + printf("%s ssl3_setup_read_buffer fail \n",__func__); return (-1); + } if ((type && (type != SSL3_RT_APPLICATION_DATA) && (type != SSL3_RT_HANDSHAKE)) || (peek @@ -1232,9 +1322,10 @@ /* get new packet if necessary */ if ((rr->length == 0) || (s->rstate == SSL_ST_READ_BODY)) { ret = ssl3_get_record(s); - if (ret <= 0) + if (ret <= 0) { return (ret); } + } /* * Reset the count of consecutive warning alerts if we've got a non-empty @@ -1276,8 +1367,10 @@ goto f_err; } - if (len <= 0) + if (len <= 0) { + printf("%s len:%d \n",__func__,len); return (len); + } if ((unsigned int)len > rr->length) n = rr->length; @@ -1296,6 +1389,9 @@ ssl3_release_read_buffer(s); } } +#ifdef TLS_DEBUG + printf("%s %d n:%d \n",__func__,__LINE__,n); +#endif return (n); } @@ -1337,6 +1433,9 @@ s->rwstate = SSL_READING; BIO_clear_retry_flags(SSL_get_rbio(s)); BIO_set_retry_read(SSL_get_rbio(s)); +#ifdef TLS_DEBUG + printf("%s TLS1_RT_HEARTBEAT \n",__func__); +#endif return (-1); } #endif @@ -1666,7 +1765,6 @@ int i; const char *sender; int slen; - if (s->state & SSL_ST_ACCEPT) i = SSL3_CHANGE_CIPHER_SERVER_READ; else @@ -1677,25 +1775,39 @@ /* might happen if dtls1_read_bytes() calls this */ SSLerr(SSL_F_SSL3_DO_CHANGE_CIPHER_SPEC, SSL_R_CCS_RECEIVED_EARLY); - return (0); + goto err; } s->session->cipher = s->s3->tmp.new_cipher; - if (!s->method->ssl3_enc->setup_key_block(s)) - return (0); + if (!s->method->ssl3_enc->setup_key_block(s)) { + goto err; + } } - if (!s->method->ssl3_enc->change_cipher_state(s, i)) - return (0); + if (!s->method->ssl3_enc->change_cipher_state(s, i)){ + printf("change_cipher_state error \n"); + goto err; + } /* * we have to record the message digest at this point so we can get it * before we read the finished message */ if (s->state & SSL_ST_CONNECT) { +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + if (SSL_ofld_rx(s)) { + chssl_program_hwkey_context(s, KEY_WRITE_RX, SSL_ST_CONNECT); + if (SSL_clr_quiesce(s)) + chssl_clear_tom(s); + } +#endif sender = s->method->ssl3_enc->server_finished_label; slen = s->method->ssl3_enc->server_finished_label_len; } else { +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + if (SSL_ofld_rx(s)) + chssl_program_hwkey_context(s, KEY_WRITE_RX, SSL_ST_ACCEPT); +#endif sender = s->method->ssl3_enc->client_finished_label; slen = s->method->ssl3_enc->client_finished_label_len; } @@ -1707,9 +1819,16 @@ SSLerr(SSL_F_SSL3_DO_CHANGE_CIPHER_SPEC, ERR_R_INTERNAL_ERROR); return 0; } + s->s3->tmp.peer_finish_md_len = i; return (1); +err: +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + if (SSL_ofld_rx(s)) + chssl_clear_quies(s); +#endif + return (0); } int ssl3_send_alert(SSL *s, int level, int desc) Index: crypto/openssl/ssl/s3_srvr.c =================================================================== --- crypto/openssl/ssl/s3_srvr.c +++ crypto/openssl/ssl/s3_srvr.c @@ -169,6 +169,9 @@ # include #endif #include +#ifdef CHSSL_OFFLOAD +#include "ssl_tom.h" +#endif #ifndef OPENSSL_NO_SSL3_METHOD static const SSL_METHOD *ssl3_get_server_method(int ver); @@ -250,6 +253,9 @@ for (;;) { state = s->state; +#ifdef TLS_DEBUG + fprintf(stderr, "%s: state %d\n", __func__, state); +#endif switch (s->state) { case SSL_ST_RENEGOTIATE: s->renegotiate = 1; @@ -270,6 +276,10 @@ return -1; } s->type = SSL_ST_ACCEPT; +#ifdef CHSSL_OFFLOAD + if (chssl_new(s)) + ssl_tls_offload(s); +#endif if (s->init_buf == NULL) { if ((buf = BUF_MEM_new()) == NULL) { @@ -423,6 +433,12 @@ else s->state = SSL3_ST_SW_CERT_A; s->init_num = 0; +#ifdef CHSSL_OFFLOAD + if (SSL_ofld(s) && s->new_session) { + s->s3->tmp.next_state = s->state; + s->state = SSL3_ST_SW_FLUSH; + } +#endif break; case SSL3_ST_SW_CERT_A: @@ -453,6 +469,12 @@ s->state = SSL3_ST_SW_KEY_EXCH_A; #endif s->init_num = 0; +#ifdef CHSSL_OFFLOAD + if (SSL_ofld(s) && s->new_session) { + s->s3->tmp.next_state = s->state; + s->state = SSL3_ST_SW_FLUSH; + } +#endif break; case SSL3_ST_SW_KEY_EXCH_A: @@ -505,6 +527,12 @@ s->state = SSL3_ST_SW_CERT_REQ_A; s->init_num = 0; +#ifdef CHSSL_OFFLOAD + if (SSL_ofld(s) && s->new_session) { + s->s3->tmp.next_state = s->state; + s->state = SSL3_ST_SW_FLUSH; + } +#endif break; case SSL3_ST_SW_CERT_REQ_A: @@ -765,6 +793,12 @@ goto end; s->state = SSL3_ST_SW_CHANGE_A; s->init_num = 0; +#ifdef CHSSL_OFFLOAD + if (SSL_ofld(s) && s->new_session) { + s->s3->tmp.next_state = s->state; + s->state = SSL3_ST_SW_FLUSH; + } +#endif break; case SSL3_ST_SW_CERT_STATUS_A: @@ -774,6 +808,12 @@ goto end; s->state = SSL3_ST_SW_KEY_EXCH_A; s->init_num = 0; +#ifdef CHSSL_OFFLOAD + if (SSL_ofld(s) && s->new_session) { + s->s3->tmp.next_state = s->state; + s->state = SSL3_ST_SW_FLUSH; + } +#endif break; #endif @@ -794,7 +834,17 @@ if (ret <= 0) goto end; + +#ifdef CHSSL_OFFLOAD + if (SSL_ofld(s)) { + s->s3->tmp.next_state = SSL3_ST_SW_FINISHED_A; + s->state = SSL3_ST_SW_FLUSH; + } else { s->state = SSL3_ST_SW_FINISHED_A; + } +#else + s->state = SSL3_ST_SW_FINISHED_A; +#endif s->init_num = 0; if (!s->method->ssl3_enc->change_cipher_state(s, @@ -804,11 +854,14 @@ s->state = SSL_ST_ERR; goto end; } - break; case SSL3_ST_SW_FINISHED_A: case SSL3_ST_SW_FINISHED_B: +#ifdef CHSSL_OFFLOAD + if (SSL_ofld(s)) + chssl_program_hwkey_context(s, KEY_WRITE_TX, SSL_ST_ACCEPT); +#endif ret = ssl3_send_finished(s, SSL3_ST_SW_FINISHED_A, SSL3_ST_SW_FINISHED_B, @@ -831,6 +884,7 @@ } else s->s3->tmp.next_state = SSL_ST_OK; s->init_num = 0; + break; case SSL_ST_OK: Index: crypto/openssl/ssl/ssl.h =================================================================== --- crypto/openssl/ssl/ssl.h +++ crypto/openssl/ssl/ssl.h @@ -1408,11 +1408,47 @@ # define SSL_READING 3 # define SSL_X509_LOOKUP 4 +#ifdef CHSSL_OFFLOAD +enum { + SSL_ENC_HOST = 0, + SSL_ENC_OFLD, +}; + +enum { + SSL_MAC_HOST = 0, + SSL_MAC_OFLD, +}; + +enum { + KEY_SPACE_AVAIL = 0, + KEY_SPACE_NOTAVL, +}; + +struct ch_ssl_st { + int nl_fd; + int sock_fd; + char ofld_tx_enable; + char ofld_rx_enable; + char ofld_enc; + char ofld_mac; + char key_state; + char rx_keys_copied; + char tx_keys_copied; + void *key_context; +}; +#endif + /* These will only be used when doing non-blocking IO */ # define SSL_want_nothing(s) (SSL_want(s) == SSL_NOTHING) # define SSL_want_read(s) (SSL_want(s) == SSL_READING) # define SSL_want_write(s) (SSL_want(s) == SSL_WRITING) # define SSL_want_x509_lookup(s) (SSL_want(s) == SSL_X509_LOOKUP) +#ifdef CHSSL_OFFLOAD +# define SSL_enc_offload(s) (SSL_enc(s) == SSL_ENC_OFLD) +# define SSL_enc_host(s) (SSL_enc(s) == SSL_ENC_HOST) +# define SSL_mac_offload(s) (SSL_mac(s) == SSL_MAC_OFLD) +# define SSL_mac_host(s) (SSL_mac(s) == SSL_MAC_HOST) +#endif # define SSL_MAC_FLAG_READ_MAC_STREAM 1 # define SSL_MAC_FLAG_WRITE_MAC_STREAM 2 @@ -1447,6 +1483,7 @@ char *wbio; char *bbio; # endif + /* * This holds a variable that indicates what we were doing when a 0 or -1 * is returned. This is needed for non-blocking IO so we know what @@ -1568,6 +1605,9 @@ unsigned int max_psk_len); # endif SSL_CTX *ctx; +#ifdef CHSSL_OFFLOAD + struct ch_ssl_st *chssl; +#endif /* * set this flag to 1 and a sleep(1) is put into all SSL_read() and * SSL_write() calls, good for nbio debuging :-) @@ -1758,6 +1798,9 @@ # define SSL_ST_READ_HEADER 0xF0 # define SSL_ST_READ_BODY 0xF1 # define SSL_ST_READ_DONE 0xF2 +#ifdef CHSSL_OFFLOAD +# define SSL_ST_READ_ERROR 0xF3 +#endif /*- * Obtain latest Finished message @@ -2136,6 +2179,23 @@ void SSL_CTX_set_cert_store(SSL_CTX *, X509_STORE *); int SSL_want(const SSL *s); int SSL_clear(SSL *s); +#ifdef CHSSL_OFFLOAD +int SSL_enc(const SSL *s); +int SSL_mac(const SSL *s); +#ifdef CHSSL_TLS_RX +int SSL_Rx_keys(const SSL *s); +#endif +int SSL_Tx_keys(const SSL *s); +int SSL_ofld(const SSL *s); +int SSL_compress(const SSL *s); +int SSL_Chelsio_ofld(const SSL *s); +#ifdef CHSSL_TLS_RX +int chssl_clear_quies(const SSL *s); +#endif +int chssl_clear_tom(const SSL *s); +int chssl_process_cherror(SSL *s); +void chssl_program_hwkey_context(SSL *s, int rw, int state); +#endif void SSL_CTX_flush_sessions(SSL_CTX *ctx, long tm); Index: crypto/openssl/ssl/ssl_lib.c =================================================================== --- crypto/openssl/ssl/ssl_lib.c +++ crypto/openssl/ssl/ssl_lib.c @@ -159,6 +159,9 @@ #ifndef OPENSSL_NO_ENGINE # include #endif +#ifdef CHSSL_OFFLOAD +#include "ssl_tom.h" +#endif const char *SSL_version_str = OPENSSL_VERSION_TEXT; @@ -742,6 +745,7 @@ } BIO_set_fd(bio, fd, BIO_NOCLOSE); SSL_set_bio(s, bio, bio); + //printf("SSL_set_fd: fd:%d bio_fd:%d BIO:%p:%p:%p\n",fd,bio->num,bio,s->wbio,s->rbio); ret = 1; err: return (ret); @@ -1068,6 +1072,11 @@ SSLerr(SSL_F_SSL_SHUTDOWN, SSL_R_UNINITIALIZED); return -1; } +#ifdef CHSSL_OFFLOAD + if (s->chssl) { + chssl_free(s); + } +#endif if (!SSL_in_init(s)) { return s->method->ssl_shutdown(s); Index: crypto/openssl/ssl/ssl_tom.h =================================================================== --- /dev/null +++ crypto/openssl/ssl/ssl_tom.h @@ -0,0 +1,181 @@ +/* + * Common Header file TLSOM functionalities + * + * Copyright (C) 2011-2015 Chelsio Communications. All rights reserved. + * + * Written by Atul Gupta + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this + * release for licensing terms and conditions. + */ +#ifdef __linux__ +#include +#endif +#ifndef __TLSOM_CMN_H +#define __TLSOM_CMN_H + +#define TLSOM_SUCCESS 0 +#define TLSOM_FAILURE -1 + +#ifdef CHSSL_DEBUG + #define chssl_print(f,c) fprintf(f,c) +#else + #define chssl_print(f,c) ; +#endif + +#define MAX_MAC_KSZ 64 /*512 bits */ +#define SHA512_BLOCK 128 /* Block size for 512 */ +#define MAX_CIPHER_KSZ 32 /* 256 bits */ +#define CIPHER_BLOCK_SZ 16 +#define IV_SIZE (4+8) /*reserved 8 bytes */ +#define SALT_SIZE 4 +#define TLS_TX_HDR_SZ 16 +#define TLS_RX_HDR_SZ 16 +#define GHASH_SIZE 16 +#define MAX_TLS_KSZ (2*MAX_MAC_KSZ + MAX_CIPHER_KSZ) + +#ifdef __FreeBSD__ +#include +#endif +#ifdef __linux__ +/* XXX: TCP_TLSKEY is not used */ +#define TCP_TLSKEY 291 /* Program Key Context on HW */ +#define IOCTL_TLSOM_SET_TLS_CONTEXT 201 /* Program Key Context on HW */ +#define IOCTL_TLSOM_GET_TLS_TOM 202 /* Query the TLS offload mode */ +#define IOCTL_TLSOM_CLR_TLS_TOM 203 /* Clear the Key */ +#define IOCTL_TLSOM_CLR_QUIES 204 /* Clear the Quiesec */ +#endif + +enum { + TLS_OFLD_FALSE = 0, + TLS_OFLD_TRUE, +}; + +#ifdef __linux__ +/* Can accomodate 16, 11-15 are reserved */ +enum { + CHSSL_SHA_NOP, + CHSSL_SHA1, + CHSSL_SHA224, + CHSSL_SHA256, + CHSSL_GHASH, + CHSSL_SHA512_224, + CHSSL_SHA512_256, + CHSSL_SHA512_384, + CHSSL_SHA512_512, + CHSSL_CBCMAC, + CHSSL_CMAC, +}; + +/* Can accomodate 16, 8-15 are reserved */ +enum { + CHSSL_CIPH_NOP, + CHSSL_AES_CBC, + CHSSL_AES_GCM, + CHSSL_AES_CTR, + CHSSL_AES_GEN, + CHSSL_IPSEC_ESP, + CHSSL_AES_XTS, + CHSSL_AES_CCM, +}; + +#define KEY_WRITE_RX 0x1 /* Program Receive Key */ +#define KEY_WRITE_TX 0x2 /* Program Transmit Key */ +#define KEY_DELETE_RX 0x4 /* Delete Receive Key */ +#define KEY_DELETE_TX 0x8 /* Delete Transmit Key */ + +#define S_KEY_CLR_LOC 4 +#define M_KEY_CLR_LOC 0xf +#define V_KEY_CLR_LOC(x) ((x) << S_KEY_CLR_LOC) +#define G_FW_WR_EQUIQ(s) (((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC) +#define F_KEY_CLR_LOC V_KEY_CLR_LOC(1U) + +/* XXX: Not used. */ +#define S_KEY_GET_LOC 0 +#define M_KEY_GET_LOC 0xf +#define V_KEY_GET_LOC(x) ((x) << S_KEY_GET_LOC) +#define G_KEY_GET_LOC(s) (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC) + +/* XXX: Not used. */ +enum { + CHSSL_OFLDMODE_NONE, + CHSSL_OFLDMODE_ALLIN_HOST, + CHSSL_OFLDMODE_ALLIN_CARD, + CHSSL_OFLDMODE_MACIN_HOST, + CHSSL_OFLDMODE_FRAGMENTATIONIN_HOST, +}; + +/* XXX: Not used. */ +enum { + CHSSL_HOST_FRAGMENT, + CHSSL_CARD_FRAGMENT, +}; + +struct tls_ofld_state { + unsigned char enc_mode; + unsigned char mac_mode; + unsigned char key_loc; + unsigned char ofld_mode; + unsigned char auth_mode; + unsigned char resv[3]; +}; + +struct tls_tx_ctxt { + unsigned char salt[SALT_SIZE]; + unsigned char key[MAX_CIPHER_KSZ]; + unsigned char ipad[MAX_MAC_KSZ]; + unsigned char opad[MAX_MAC_KSZ]; +}; + +struct tls_rx_ctxt { + unsigned char salt[SALT_SIZE]; + unsigned char key[MAX_CIPHER_KSZ]; + unsigned char ipad[MAX_MAC_KSZ]; + unsigned char opad[MAX_MAC_KSZ]; +}; + +struct tls_key_context { + struct tls_tx_ctxt tx; + struct tls_rx_ctxt rx; + + unsigned char l_p_key; + unsigned char hmac_ctrl; + unsigned char mac_first; + unsigned char iv_size; + unsigned char iv_ctrl; + unsigned char iv_algo; + unsigned char tx_seq_no; + unsigned char rx_seq_no; + + struct tls_ofld_state state; + + unsigned int tx_key_info_size; + unsigned int rx_key_info_size; + unsigned int frag_size; + unsigned int mac_secret_size; + unsigned int cipher_secret_size; + int proto_ver; + unsigned int sock_fd; + unsigned short dtls_epoch; + unsigned short rsv; +}; +#endif + +/* XXX: Not used. */ +struct tls_key_location { + unsigned int sock_fd; + unsigned int key_location; +}; + +struct ch_ssl_st *chssl_new(SSL *s); +struct ch_ssl_st *chssl_free(SSL *s); +int ssl_tls_offload(SSL *s); +int SSL_ofld_vers(const SSL *s); +#ifdef CHSSL_TLS_RX +int SSL_ofld_rx(const SSL *s); +int SSL_clr_quiesce(const SSL *s); +#endif +#endif + Index: crypto/openssl/ssl/ssl_tom.c =================================================================== --- /dev/null +++ crypto/openssl/ssl/ssl_tom.c @@ -0,0 +1,725 @@ +/* ssl/ssl_tom.c */ +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * w + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ +/* ==================================================================== + * Copyright (c) 1998-2007 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * Chelsio Software written by Atul Gupta (atul.gupta@chelsio.com) + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#endif +#ifdef __FreeBSD__ +#include +#endif +#include "ssl_locl.h" +#include +#include +#include "ssl_tom.h" +#include + +#ifdef CHSSL_OFFLOAD +int ssl_tls_offload(SSL *s) +{ +#ifdef __FreeBSD__ + socklen_t optlen; +#endif + int ret, mode = 0; + + if ((s->wbio == NULL) || (s->rbio == NULL)) { + SSLerr(SSL_R_BAD_STATE, SSL_R_BIO_NOT_SET); + return mode; + } + s->chssl->sock_fd = s->wbio->num = s->rbio->num; + +#ifdef __linux__ + ret = ioctl(s->chssl->sock_fd, IOCTL_TLSOM_GET_TLS_TOM, &mode); + if (!ret && mode) + s->chssl->ofld_rx_enable = s->chssl->ofld_tx_enable = TLS_OFLD_TRUE; + else + s->chssl->ofld_rx_enable = s->chssl->ofld_tx_enable = TLS_OFLD_FALSE; +#else + optlen = sizeof(mode); + ret = getsockopt(s->chssl->sock_fd, IPPROTO_TCP, TCP_TLSOM_GET_TLS_TOM, + &mode, &optlen); + s->chssl->ofld_rx_enable = s->chssl->ofld_tx_enable = TLS_OFLD_FALSE; + if (ret == 0) { + switch (mode) { + case TLS_TOM_BOTH: + s->chssl->ofld_rx_enable = TLS_OFLD_TRUE; + /* FALLTHROUGH */ + case TLS_TOM_TXONLY: + s->chssl->ofld_tx_enable = TLS_OFLD_TRUE; + break; + } + } +#endif + + return mode; +} + +int SSL_ofld_vers(const SSL *s) +{ + return (s->version == TLS1_1_VERSION || s->version == TLS1_2_VERSION + || s->version == DTLS1_2_VERSION); +} + +int SSL_ofld(const SSL *s) +{ + return(SSL_ofld_vers(s) && s->chssl && s->chssl->ofld_tx_enable); +} + +#ifdef CHSSL_TLS_RX +int SSL_ofld_rx(const SSL *s) +{ + return(s->chssl && s->chssl->ofld_rx_enable); +} +#endif + +int SSL_compress(const SSL *s) +{ + return(s->compress != NULL); +} + +#ifdef CHSSL_TLS_RX +int SSL_Rx_keys(const SSL *s) +{ + return(s->chssl && s->chssl->rx_keys_copied); +} +#endif + +int SSL_Tx_keys(const SSL *s) +{ + return(s->chssl && s->chssl->tx_keys_copied); +} + +int SSL_enc(const SSL *s) +{ + return(s->chssl && s->chssl->ofld_tx_enable && s->chssl->ofld_enc); +} + +int SSL_mac(const SSL *s) +{ + return(s->chssl && s->chssl->ofld_tx_enable && s->chssl->ofld_mac); +} + +int SSL_Chelsio_ofld(const SSL *s) +{ + return (SSL_ofld(s) && SSL_ofld_rx(s) && SSL_enc(s) && SSL_mac(s)); +} + +#ifdef CHSSL_TLS_RX +int SSL_clr_quiesce(const SSL *s) +{ + return (SSL_ofld_rx(s) && (s->chssl->key_state == KEY_SPACE_NOTAVL)); +} +#endif + +/* + * Determine HW capability for Digest and Cipher Offload + */ +static int tls_ofld_enc_mac(SSL *s) +{ + const EVP_CIPHER *p; + const SSL_CIPHER *c; + + c = s->s3->tmp.new_cipher; + p = s->s3->tmp.new_sym_enc; + + switch(c->algorithm_enc) { + case SSL_AES128GCM: + case SSL_AES256GCM: + s->chssl->ofld_enc = SSL_ENC_OFLD; + s->chssl->ofld_mac = SSL_MAC_OFLD; + return TLS_OFLD_TRUE; + + case SSL_AES128 : + case SSL_AES256 : + switch(EVP_CIPHER_mode(p)) { + case EVP_CIPH_CTR_MODE: + case EVP_CIPH_CBC_MODE: + s->chssl->ofld_enc = SSL_ENC_OFLD; + break; + default: + chssl_print(cl,"No HW support\n"); + return TLS_OFLD_FALSE; + } + break; + + case SSL_eNULL: + s->chssl->ofld_enc = SSL_ENC_OFLD; + break; + + default: + chssl_print(cl,"No HW support for ENC\n"); + s->chssl->ofld_enc = SSL_ENC_HOST; + return TLS_OFLD_FALSE; + } + + switch(c->algorithm_mac) { + case SSL_SHA1: + case SSL_SHA256: + case SSL_SHA384: + s->chssl->ofld_mac= SSL_MAC_OFLD; + break; + + default: + s->chssl->ofld_mac= SSL_MAC_HOST; + /* Revert enc mode to non-offload */ + s->chssl->ofld_enc = SSL_ENC_HOST; + chssl_print(cl,"No HW support for MAC\n"); + return TLS_OFLD_FALSE; + } + return TLS_OFLD_TRUE; +} + +/* + * Authentication Mode expected by HW + */ +static unsigned char get_auth_mode(SSL *s) +{ + const SSL_CIPHER *c = s->s3->tmp.new_cipher; + + if(c==NULL) return CHSSL_SHA_NOP; + + switch(c->algorithm_mac) { + case SSL_SHA1: + return CHSSL_SHA1; + case SSL_SHA256: + return CHSSL_SHA256; + case SSL_SHA384: + return CHSSL_SHA512_384; + case SSL_AEAD: + return CHSSL_GHASH; + default: + return CHSSL_SHA_NOP; +} +} + +/* + * Cipher Mode expected by HW + */ +static unsigned char get_cipher_mode(SSL *s) +{ + const EVP_CIPHER *c = s->s3->tmp.new_sym_enc; + + switch(EVP_CIPHER_mode(c)) { + case EVP_CIPH_CBC_MODE: + return CHSSL_AES_CBC; + case EVP_CIPH_GCM_MODE: + return CHSSL_AES_GCM; + case EVP_CIPH_CTR_MODE: + return CHSSL_AES_CTR; + case EVP_CIPH_STREAM_CIPHER: + return CHSSL_CIPH_NOP; + default: + chssl_print(cl,"invalid cipher mode\n"); + return CHSSL_CIPH_NOP; + } +} + +static void chssl_compute_ipad_opad(unsigned char *key, + unsigned char *ipad, + unsigned char *opad, + int k) +{ + int i; + + if(k < MAX_MAC_KSZ) { + for(i=k; i < MAX_MAC_KSZ; i++) { + ipad[i] = 0x0; + opad[i] = 0x0; + } + } + for(i=0; i < MAX_MAC_KSZ; i++) { + ipad[i] = key[i] ^ 0x36; + opad[i] = key[i] ^ 0x5c; + } + + if(k == SHA_DIGEST_LENGTH) { + CHSSL_EVP_Digest(ipad, MAX_MAC_KSZ, ipad, NULL, EVP_sha1(), NULL); + CHSSL_EVP_Digest(opad, MAX_MAC_KSZ, opad, NULL, EVP_sha1(), NULL); + } else if (k == SHA256_DIGEST_LENGTH) { + CHSSL_EVP_Digest(ipad, MAX_MAC_KSZ, ipad, NULL, EVP_sha256(), NULL); + CHSSL_EVP_Digest(opad, MAX_MAC_KSZ, opad, NULL, EVP_sha256(), NULL); + } +} + +static void chssl_compute_ipad_opad_512(unsigned char *key, + unsigned char *ipad, + unsigned char *opad, + int k) +{ + int i; + char ipad_512[SHA512_BLOCK] = {0}; + char opad_512[SHA512_BLOCK] = {0}; + + for(i = 0; i < SHA512_BLOCK; i++) { + if (i < k) { + ipad_512[i] = key[i] ^ 0x36; + opad_512[i] = key[i] ^ 0x5c; + } else { + ipad_512[i] = 0x36; + opad_512[i] = 0x5c; + } + } + + if(k == SHA384_DIGEST_LENGTH) { + CHSSL_EVP_Digest(ipad_512, SHA512_BLOCK, ipad_512, NULL, + EVP_sha384(), NULL); + CHSSL_EVP_Digest(opad_512, SHA512_BLOCK, opad_512, NULL, + EVP_sha384(), NULL); + } else if (k == SHA512_DIGEST_LENGTH) { + CHSSL_EVP_Digest(ipad_512, SHA512_BLOCK, ipad_512, NULL, + EVP_sha512(), NULL); + CHSSL_EVP_Digest(opad_512, SHA512_BLOCK, opad_512, NULL, + EVP_sha512(), NULL); + } + memcpy(ipad, ipad_512, MAX_MAC_KSZ); + memcpy(opad, opad_512, MAX_MAC_KSZ); +} + +static void chssl_compute_cipher_key(unsigned char *key, + int key_len, + unsigned char *ghash) +{ + int len,len1; + EVP_CIPHER_CTX ctx; + unsigned char plaintext[GHASH_SIZE] = {0}; + + EVP_CIPHER_CTX_init(&ctx); + if(key_len == 16) + EVP_EncryptInit_ex(&ctx, EVP_aes_128_cbc(), NULL, key, NULL); + else + EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, key, NULL); + EVP_CIPHER_CTX_set_padding(&ctx, 0); + EVP_EncryptUpdate(&ctx, ghash, &len, plaintext, 16); + EVP_EncryptFinal_ex(&ctx, ghash+len, &len1); + EVP_CIPHER_CTX_cleanup(&ctx); +} + +/* + * Create key Context for receive/transmit and program on HW + */ +static int ssl_key_context(SSL *s, int rw, int state) +{ + const EVP_CIPHER *c; + const EVP_MD *m; + const SSL_CIPHER *sc; + unsigned int mac_key_size = 0, cipher_key_size, iv_size; + unsigned char *key; + unsigned char s_ipad_hash[MAX_MAC_KSZ]= {0x0}; /* blk sz for hashing */ + unsigned char s_opad_hash[MAX_MAC_KSZ]= {0x0}; /* blk sz for hashing */ + unsigned char c_ipad_hash[MAX_MAC_KSZ]= {0x0}; /* blk sz for hashing */ + unsigned char c_opad_hash[MAX_MAC_KSZ]= {0x0}; /* blk sz for hashing */ + + unsigned char s_mac_key[MAX_MAC_KSZ] = {0x0}; + unsigned char c_mac_key[MAX_MAC_KSZ] = {0x0}; + unsigned char s_key[MAX_CIPHER_KSZ] = {0x0}; + unsigned char c_key[MAX_CIPHER_KSZ] = {0x0}; + unsigned char s_iv[MAX_CIPHER_KSZ] = {0x0}; + unsigned char c_iv[MAX_CIPHER_KSZ] = {0x0}; + unsigned char ghash[GHASH_SIZE] = {0x0}; + int pad = 12; + int index = 0; + int ret = 0; + struct tls_key_context *kctx = s->chssl->key_context; + + if (!tls_ofld_enc_mac(s)) { + return ret; + } + + c = s->s3->tmp.new_sym_enc; + m = s->s3->tmp.new_hash; + sc = s->s3->tmp.new_cipher; + kctx->l_p_key = rw; + + if (s->new_session) + kctx->l_p_key |= F_KEY_CLR_LOC; + key = s->s3->tmp.key_block; + + if (s->version >= TLS1_VERSION) + mac_key_size = s->s3->tmp.new_mac_secret_size; + else + if (m) mac_key_size = m->md_size; + kctx->mac_secret_size = mac_key_size; + + cipher_key_size = EVP_CIPHER_key_length(c); + kctx->cipher_secret_size = cipher_key_size; + + iv_size = (EVP_CIPHER_mode(c) == EVP_CIPH_GCM_MODE) ? + EVP_GCM_TLS_FIXED_IV_LEN: + EVP_CIPHER_iv_length(c); + kctx->iv_size = iv_size; + kctx->iv_ctrl = 1; + kctx->iv_algo = 0; + + if ((mac_key_size == SHA256_DIGEST_LENGTH) || + (mac_key_size == SHA384_DIGEST_LENGTH)) + pad = 0; + + if (mac_key_size) { + memcpy(c_mac_key, key, mac_key_size); + key += mac_key_size; + memcpy(s_mac_key, key, mac_key_size); + key += mac_key_size; + } + memcpy(c_key, key, cipher_key_size); + key += cipher_key_size; + memcpy(s_key, key, cipher_key_size); + key += cipher_key_size; + + memcpy(c_iv, key, iv_size); + key += iv_size; + memcpy(s_iv, key, iv_size); + + if (s->chssl->ofld_mac) { + /* IPAD/OPAD for SHA384/512 calculated over 128B block */ + if (mac_key_size >= SHA384_DIGEST_LENGTH) { + chssl_compute_ipad_opad_512(c_mac_key, c_ipad_hash, + c_opad_hash, mac_key_size); + chssl_compute_ipad_opad_512(s_mac_key, s_ipad_hash, + s_opad_hash, mac_key_size); + } else { + chssl_compute_ipad_opad(c_mac_key, c_ipad_hash, + c_opad_hash, mac_key_size); + chssl_compute_ipad_opad(s_mac_key, s_ipad_hash, + s_opad_hash, mac_key_size); + } + } + + if (state == SSL_ST_ACCEPT) { + memcpy(kctx->tx.key, s_key, cipher_key_size); + memcpy(kctx->rx.key, c_key, cipher_key_size); + } else { + memcpy(kctx->tx.key, c_key, cipher_key_size); + memcpy(kctx->rx.key, s_key, cipher_key_size); + } + + if (mac_key_size == SHA384_DIGEST_LENGTH) mac_key_size = MAX_MAC_KSZ; + index = cipher_key_size; + if (s->chssl->ofld_mac) { + if (mac_key_size) { + if (state == SSL_ST_ACCEPT) + memcpy(kctx->tx.key+index, s_ipad_hash, mac_key_size); + else + memcpy(kctx->tx.key+index, c_ipad_hash, mac_key_size); + + index += (mac_key_size + pad); + if (state == SSL_ST_ACCEPT) + memcpy(kctx->tx.key+index, s_opad_hash, mac_key_size); + else + memcpy(kctx->tx.key+index, c_opad_hash, mac_key_size); + + index += (mac_key_size + pad); + } else { + if (state == SSL_ST_ACCEPT) { + chssl_compute_cipher_key(s_key, cipher_key_size, ghash); + memcpy(kctx->tx.key+index, ghash, GHASH_SIZE); + } else { + chssl_compute_cipher_key(c_key, cipher_key_size, ghash); + memcpy(kctx->tx.key+index, ghash, GHASH_SIZE); + } + index += GHASH_SIZE; + } + } + kctx->tx_key_info_size = TLS_TX_HDR_SZ + index; + + index = cipher_key_size; + if (s->chssl->ofld_mac) { + if (mac_key_size) { + if (state == SSL_ST_ACCEPT) + memcpy(kctx->rx.key+index, c_ipad_hash, mac_key_size); + else + memcpy(kctx->rx.key+index, s_ipad_hash, mac_key_size); + + index += (mac_key_size + pad); + if (state == SSL_ST_ACCEPT) + memcpy(kctx->rx.key+index, c_opad_hash, mac_key_size); + else + memcpy(kctx->rx.key+index, s_opad_hash, mac_key_size); + + index += (mac_key_size + pad); + } else { + if (state == SSL_ST_ACCEPT) { + chssl_compute_cipher_key(c_key, cipher_key_size, ghash); + memcpy(kctx->rx.key+index, ghash, GHASH_SIZE); + } else { + chssl_compute_cipher_key(s_key, cipher_key_size, ghash); + memcpy(kctx->rx.key+index, ghash, GHASH_SIZE); + } + index += GHASH_SIZE; + } + } + + kctx->tx_key_info_size = TLS_RX_HDR_SZ + index; + + if (EVP_CIPHER_mode(c) == EVP_CIPH_GCM_MODE) { + if (state == SSL_ST_ACCEPT) { + memcpy(kctx->tx.salt, s_iv, SALT_SIZE); + memcpy(kctx->rx.salt, c_iv, SALT_SIZE); + } else { + memcpy(kctx->tx.salt, c_iv, SALT_SIZE); + memcpy(kctx->rx.salt, s_iv, SALT_SIZE); + } + } + + kctx->proto_ver = s->version; + kctx->state.auth_mode = get_auth_mode(s); + kctx->state.enc_mode = get_cipher_mode(s); + if (s->version == DTLS1_2_VERSION) + kctx->dtls_epoch = s->d1->r_epoch; + + if (s->max_send_fragment) + kctx->frag_size = s->max_send_fragment; + else + kctx->frag_size = SSL3_RT_MAX_PLAIN_LENGTH; + + /* handle renegotiation here */ + if(!s->chssl->tx_keys_copied) + kctx->tx_seq_no = 0; + else + kctx->tx_seq_no = 1; + + if(!s->chssl->rx_keys_copied) + kctx->rx_seq_no = 0; + else + kctx->rx_seq_no = 1; + + if(EVP_CIPHER_mode(c) != EVP_CIPH_GCM_MODE) { + if (!SSL_mac(s) && SSL_enc(s)) + kctx->hmac_ctrl = 0; + else + kctx->hmac_ctrl = 1; + } + kctx->sock_fd = s->chssl->sock_fd; + + return 1; +} + +void chssl_program_hwkey_context(SSL *s, int rw, int state) +{ + int ret = 0; + + if (!s->chssl->key_context) { + s->chssl->key_context = (struct tls_key_context *) + OPENSSL_malloc(sizeof(struct tls_key_context)); + if (s->chssl->key_context == NULL) + return; + } + + memset(s->chssl->key_context, 0, sizeof(struct tls_key_context)); + if((ret = ssl_key_context(s, rw, state)) <=0) { +#ifdef CHSSL_TLS_RX + /* Clear quiesce after CCS receive */ + if (rw == KEY_WRITE_RX) { + ret = chssl_clear_tom(s); + s->chssl->ofld_rx_enable = TLS_OFLD_FALSE; + s->chssl->ofld_tx_enable = TLS_OFLD_FALSE; + } +#endif + goto end; + } + +#ifdef __linux__ + ret = ioctl(s->chssl->sock_fd, IOCTL_TLSOM_SET_TLS_CONTEXT, + s->chssl->key_context); +#else + ret = setsockopt(s->chssl->sock_fd, IPPROTO_TCP, TCP_TLSOM_SET_TLS_CONTEXT, + s->chssl->key_context, sizeof(struct tls_key_context)); +#endif + if (!ret) { + if (rw & KEY_WRITE_TX) + s->chssl->tx_keys_copied = 1; + else + s->chssl->rx_keys_copied = 1; + } else { + s->chssl->ofld_rx_enable = TLS_OFLD_FALSE; + s->chssl->ofld_tx_enable = TLS_OFLD_FALSE; + s->chssl->key_state = KEY_SPACE_NOTAVL; + } + +end: + free(s->chssl->key_context); + s->chssl->key_context = NULL; + return; +} + +#ifdef CHSSL_TLS_RX +int chssl_clear_quies(const SSL *s) +{ +#ifdef __linux__ + return ioctl(s->chssl->sock_fd, IOCTL_TLSOM_CLR_QUIES); +#else + return setsockopt(s->chssl->sock_fd, IPPROTO_TCP, TCP_TLSOM_CLR_QUIES, + NULL, 0); +#endif +} + +int chssl_clear_tom(const SSL *s) +{ +#ifdef __linux__ + return ioctl(s->chssl->sock_fd, IOCTL_TLSOM_CLR_TLS_TOM); +#else + return setsockopt(s->chssl->sock_fd, IPPROTO_TCP, TCP_TLSOM_CLR_TLS_TOM, + NULL, 0); +#endif +} +#endif + +int chssl_process_cherror(SSL *s) +{ + const char *buf = &(s->s3->rbuf.buf[0]); + unsigned int err = atoi(buf); + + switch (err) { + case 0: + return SSL_AD_BAD_RECORD_MAC; + case 1: + case 2: + return SSL_AD_ILLEGAL_PARAMETER; + case 3: + return SSL_AD_RECORD_OVERFLOW; + case 4: + return SSL_AD_INTERNAL_ERROR; + } + return 0; +} + +struct ch_ssl_st *chssl_new(SSL *s) +{ + if (s->chssl) + return s->chssl; + s->chssl = OPENSSL_malloc(sizeof(struct ch_ssl_st)); + if (s->chssl) + memset(s->chssl, 0, sizeof(struct ch_ssl_st)); + return s->chssl; +} + +struct ch_ssl_st *chssl_free(SSL *s) +{ + if (!s->chssl) + return NULL; + + free(s->chssl); + s->chssl = NULL; + return NULL; +} +#endif Index: crypto/openssl/ssl/t1_lib.c =================================================================== --- crypto/openssl/ssl/t1_lib.c +++ crypto/openssl/ssl/t1_lib.c @@ -121,6 +121,9 @@ #include #include #include "ssl_locl.h" +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) +#include "ssl_tom.h" +#endif const char tls1_version_str[] = "TLSv1" OPENSSL_VERSION_PTEXT; @@ -4051,6 +4054,10 @@ s->tlsext_hb_pending = 0; } } +#if defined(CHSSL_OFFLOAD) && defined(CHSSL_TLS_RX) + if (SSL_ofld_rx(s)) + chssl_clear_quies(s); +#endif return 0; } Index: secure/lib/libcrypto/Makefile =================================================================== --- secure/lib/libcrypto/Makefile +++ secure/lib/libcrypto/Makefile @@ -406,6 +406,10 @@ CFLAGS+= -I${LCRYPTO_SRC}/crypto/evp CFLAGS+= -I${LCRYPTO_SRC}/crypto/modes +.if defined(CHSSL_OFFLOAD) +CFLAGS+= -DCHSSL_OFFLOAD +.endif + .if !empty(SRCS:M*.S) ACFLAGS+= -Wa,--noexecstack .endif Index: secure/lib/libssl/Makefile =================================================================== --- secure/lib/libssl/Makefile +++ secure/lib/libssl/Makefile @@ -25,6 +25,15 @@ CFLAGS+= -I${LCRYPTO_SRC}/crypto +.if defined(CHSSL_OFFLOAD) +SRCS+= ssl_tom.c + +CFLAGS+= -DCHSSL_OFFLOAD -I${SRCTOP}/sys +.if defined(CHSSL_TLS_RX) +CFLAGS+= -DCHSSL_TLS_RX +.endif +.endif + .include .PATH: ${LCRYPTO_SRC}/ssl \ Index: sys/dev/cxgbe/adapter.h =================================================================== --- sys/dev/cxgbe/adapter.h +++ sys/dev/cxgbe/adapter.h @@ -297,6 +297,10 @@ struct port_stats stats; u_int tnl_cong_drops; u_int tx_parse_error; + u_long tx_tls_records; + u_long tx_tls_octets; + u_long rx_tls_records; + u_long rx_tls_octets; struct callout tick; }; Index: sys/dev/cxgbe/firmware/t6fw_cfg.txt =================================================================== --- sys/dev/cxgbe/firmware/t6fw_cfg.txt +++ sys/dev/cxgbe/firmware/t6fw_cfg.txt @@ -163,10 +163,12 @@ nserver = 512 nhpfilter = 0 nhash = 16384 - protocol = ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu, iscsi_t10dif, crypto_lookaside + protocol = ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu, iscsi_t10dif, tlskeys, crypto_lookaside tp_l2t = 4096 tp_ddp = 2 tp_ddp_iscsi = 2 + tp_tls_key = 3 + tp_tls_mxrxsize = 17408 # 16384 + 1024, governs max rx data, pm max xfer len, rx coalesce sizes tp_stag = 2 tp_pbl = 5 tp_rq = 7 @@ -273,7 +275,7 @@ [fini] version = 0x1 - checksum = 0x7191019f + checksum = 0x9e8952d2 # # $FreeBSD$ # Index: sys/dev/cxgbe/offload.h =================================================================== --- sys/dev/cxgbe/offload.h +++ sys/dev/cxgbe/offload.h @@ -151,6 +151,9 @@ int sndbuf; int ddp; int rx_coalesce; + int tls; + int *tls_rx_ports; + int num_tls_rx_ports; int tx_align; int tx_zcopy; }; Index: sys/dev/cxgbe/t4_main.c =================================================================== --- sys/dev/cxgbe/t4_main.c +++ sys/dev/cxgbe/t4_main.c @@ -591,6 +591,7 @@ static int sysctl_tc_params(SYSCTL_HANDLER_ARGS); #endif #ifdef TCP_OFFLOAD +static int sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS); static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS); static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS); static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS); @@ -1390,6 +1391,7 @@ free(sc->sge.iqmap, M_CXGBE); free(sc->sge.eqmap, M_CXGBE); free(sc->tids.ftid_tab, M_CXGBE); + free(sc->tt.tls_rx_ports, M_CXGBE); t4_destroy_dma_tag(sc); if (mtx_initialized(&sc->sc_lock)) { sx_xlock(&t4_list_lock); @@ -5433,6 +5435,14 @@ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); + sc->tt.tls = 0; + SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tls", CTLFLAG_RW, + &sc->tt.tls, 0, "Inline TLS allowed"); + + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls_rx_ports", + CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tls_rx_ports, + "I", "TCP ports that use inline TLS+TOE RX"); + sc->tt.tx_align = 1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); @@ -5836,6 +5846,19 @@ "# of buffer-group 3 truncated packets"); #undef SYSCTL_ADD_T4_PORTSTAT + + SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_records", + CTLFLAG_RD, &pi->tx_tls_records, + "# of TLS records transmitted"); + SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_octets", + CTLFLAG_RD, &pi->tx_tls_octets, + "# of payload octets in transmitted TLS records"); + SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_records", + CTLFLAG_RD, &pi->rx_tls_records, + "# of TLS records received"); + SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_octets", + CTLFLAG_RD, &pi->rx_tls_octets, + "# of payload octets in received TLS records"); } static int @@ -8257,6 +8280,68 @@ #endif #ifdef TCP_OFFLOAD +static int +sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS) +{ + struct adapter *sc = arg1; + int *old_ports, *new_ports; + int i, new_count, rc; + + if (req->newptr == NULL && req->oldptr == NULL) + return (SYSCTL_OUT(req, NULL, imax(sc->tt.num_tls_rx_ports, 1) * + sizeof(sc->tt.tls_rx_ports[0]))); + + rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4tlsrx"); + if (rc) + return (rc); + + if (sc->tt.num_tls_rx_ports == 0) { + i = -1; + rc = SYSCTL_OUT(req, &i, sizeof(i)); + } else + rc = SYSCTL_OUT(req, sc->tt.tls_rx_ports, + sc->tt.num_tls_rx_ports * sizeof(sc->tt.tls_rx_ports[0])); + if (rc == 0 && req->newptr != NULL) { + new_count = req->newlen / sizeof(new_ports[0]); + new_ports = malloc(new_count * sizeof(new_ports[0]), M_CXGBE, + M_WAITOK); + rc = SYSCTL_IN(req, new_ports, new_count * + sizeof(new_ports[0])); + if (rc) + goto err; + + /* Allow setting to a single '-1' to clear the list. */ + if (new_count == 1 && new_ports[0] == -1) { + ADAPTER_LOCK(sc); + old_ports = sc->tt.tls_rx_ports; + sc->tt.tls_rx_ports = NULL; + sc->tt.num_tls_rx_ports = 0; + ADAPTER_UNLOCK(sc); + free(old_ports, M_CXGBE); + } else { + for (i = 0; i < new_count; i++) { + if (new_ports[i] < 1 || + new_ports[i] > IPPORT_MAX) { + rc = EINVAL; + goto err; + } + } + + ADAPTER_LOCK(sc); + old_ports = sc->tt.tls_rx_ports; + sc->tt.tls_rx_ports = new_ports; + sc->tt.num_tls_rx_ports = new_count; + ADAPTER_UNLOCK(sc); + free(old_ports, M_CXGBE); + new_ports = NULL; + } + err: + free(new_ports, M_CXGBE); + } + end_synchronized_op(sc, 0); + return (rc); +} + static void unit_conv(char *buf, size_t len, u_int val, u_int factor) { Index: sys/dev/cxgbe/tom/t4_connect.c =================================================================== --- sys/dev/cxgbe/tom/t4_connect.c +++ sys/dev/cxgbe/tom/t4_connect.c @@ -142,6 +142,10 @@ } make_established(toep, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt); + + if (toep->ulp_mode == ULP_MODE_TLS) + tls_establish(toep); + done: INP_WUNLOCK(inp); CURVNET_RESTORE(); @@ -268,6 +272,11 @@ if (toep->ulp_mode == ULP_MODE_TCPDDP) opt2 |= F_RX_FC_VALID | F_RX_FC_DDP; #endif + if (toep->ulp_mode == ULP_MODE_TLS) { + opt2 |= F_RX_FC_VALID; + opt2 &= ~V_RX_COALESCE(M_RX_COALESCE); + opt2 |= F_RX_FC_DISABLE; + } return (htobe32(opt2)); } @@ -378,10 +387,7 @@ DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM); toep->vnet = so->so_vnet; - if (sc->tt.ddp && (so->so_options & SO_NO_DDP) == 0) - set_tcpddp_ulp_mode(toep); - else - toep->ulp_mode = ULP_MODE_NONE; + set_ulp_mode(toep, select_ulp_mode(so, sc)); SOCKBUF_LOCK(&so->so_rcv); /* opt0 rcv_bufsiz initially, assumes its normal meaning later */ toep->rx_credits = min(select_rcv_wnd(so) >> 10, M_RCV_BUFSIZ); Index: sys/dev/cxgbe/tom/t4_cpl_io.c =================================================================== --- sys/dev/cxgbe/tom/t4_cpl_io.c +++ sys/dev/cxgbe/tom/t4_cpl_io.c @@ -73,9 +73,6 @@ #include "tom/t4_tom_l2t.h" #include "tom/t4_tom.h" -#define IS_AIOTX_MBUF(m) \ - ((m)->m_flags & M_EXT && (m)->m_ext.ext_flags & EXT_FLAG_AIOTX) - static void t4_aiotx_cancel(struct kaiocb *job); static void t4_aiotx_queue_toep(struct toepcb *toep); @@ -106,7 +103,7 @@ { struct wrqe *wr; struct fw_flowc_wr *flowc; - unsigned int nparams = ftxp ? 8 : 6, flowclen; + unsigned int nparams, flowclen, paramidx; struct vi_info *vi = toep->vi; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; @@ -116,6 +113,15 @@ KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT), ("%s: flowc for tid %u sent already", __func__, toep->tid)); + if (ftxp != NULL) + nparams = 8; + else + nparams = 6; + if (toep->ulp_mode == ULP_MODE_TLS) + nparams++; + if (toep->tls.fcplenmax != 0) + nparams++; + flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq); @@ -131,38 +137,44 @@ flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | V_FW_WR_FLOWID(toep->tid)); - flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; - flowc->mnemval[0].val = htobe32(pfvf); - flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; - flowc->mnemval[1].val = htobe32(pi->tx_chan); - flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; - flowc->mnemval[2].val = htobe32(pi->tx_chan); - flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; - flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id); +#define FLOWC_PARAM(__m, __v) \ + do { \ + flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \ + flowc->mnemval[paramidx].val = htobe32(__v); \ + paramidx++; \ + } while (0) + + paramidx = 0; + + FLOWC_PARAM(PFNVFN, pfvf); + FLOWC_PARAM(CH, pi->tx_chan); + FLOWC_PARAM(PORT, pi->tx_chan); + FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id); if (ftxp) { uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf); - flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; - flowc->mnemval[4].val = htobe32(ftxp->snd_nxt); - flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; - flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt); - flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; - flowc->mnemval[6].val = htobe32(sndbuf); - flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; - flowc->mnemval[7].val = htobe32(ftxp->mss); + FLOWC_PARAM(SNDNXT, ftxp->snd_nxt); + FLOWC_PARAM(RCVNXT, ftxp->rcv_nxt); + FLOWC_PARAM(SNDBUF, sndbuf); + FLOWC_PARAM(MSS, ftxp->mss); CTR6(KTR_CXGBE, "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x", __func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt, ftxp->rcv_nxt); } else { - flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF; - flowc->mnemval[4].val = htobe32(512); - flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS; - flowc->mnemval[5].val = htobe32(512); + FLOWC_PARAM(SNDBUF, 512); + FLOWC_PARAM(MSS, 512); CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid); } + if (toep->ulp_mode == ULP_MODE_TLS) + FLOWC_PARAM(ULP_MODE, toep->ulp_mode); + if (toep->tls.fcplenmax != 0) + FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax); +#undef FLOWC_PARAM + + KASSERT(paramidx == nparams, ("nparams mismatch")); txsd->tx_credits = howmany(flowclen, 16); txsd->plen = 0; @@ -421,7 +433,7 @@ soisconnected(so); } -static int +int send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits) { struct wrqe *wr; @@ -442,6 +454,23 @@ return (credits); } +void +send_rx_modulate(struct adapter *sc, struct toepcb *toep) +{ + struct wrqe *wr; + struct cpl_rx_data_ack *req; + + wr = alloc_wrqe(sizeof(*req), toep->ctrlq); + if (wr == NULL) + return; + req = wrtod(wr); + + INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); + req->credit_dack = htobe32(F_RX_MODULATE_RX); + + t4_wrq_tx(sc, wr); +} + void t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp) { @@ -459,8 +488,18 @@ ("%s: sb %p has more data (%d) than last time (%d).", __func__, sb, sbused(sb), toep->sb_cc)); - toep->rx_credits += toep->sb_cc - sbused(sb); + credits = toep->sb_cc - sbused(sb); toep->sb_cc = sbused(sb); + if (toep->ulp_mode == ULP_MODE_TLS) { + if (toep->tls.rcv_over >= credits) { + toep->tls.rcv_over -= credits; + credits = 0; + } else { + credits -= toep->tls.rcv_over; + toep->tls.rcv_over = 0; + } + } + toep->rx_credits += credits; if (toep->rx_credits > 0 && (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 || @@ -471,7 +510,8 @@ toep->rx_credits -= credits; tp->rcv_wnd += credits; tp->rcv_adv += credits; - } + } else if (toep->flags & TPF_FORCE_CREDITS) + send_rx_modulate(sc, toep); } void @@ -489,8 +529,8 @@ /* * Close a connection by sending a CPL_CLOSE_CON_REQ message. */ -static int -close_conn(struct adapter *sc, struct toepcb *toep) +int +t4_close_conn(struct adapter *sc, struct toepcb *toep) { struct wrqe *wr; struct cpl_close_con_req *req; @@ -691,6 +731,7 @@ KASSERT(toep->ulp_mode == ULP_MODE_NONE || toep->ulp_mode == ULP_MODE_TCPDDP || + toep->ulp_mode == ULP_MODE_TLS || toep->ulp_mode == ULP_MODE_RDMA, ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); @@ -905,7 +946,7 @@ /* Send a FIN if requested, but only if there's no more data to send */ if (m == NULL && toep->flags & TPF_SEND_FIN) - close_conn(sc, toep); + t4_close_conn(sc, toep); } static inline void @@ -1097,7 +1138,7 @@ /* Send a FIN if requested, but only if there are no more PDUs to send */ if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) - close_conn(sc, toep); + t4_close_conn(sc, toep); } int @@ -1116,6 +1157,8 @@ if (toep->ulp_mode == ULP_MODE_ISCSI) t4_push_pdus(sc, toep, 0); + else if (tls_tx_key(toep)) + t4_push_tls_records(sc, toep, 0); else t4_push_frames(sc, toep, 0); @@ -1140,6 +1183,8 @@ if (tp->t_state >= TCPS_ESTABLISHED) { if (toep->ulp_mode == ULP_MODE_ISCSI) t4_push_pdus(sc, toep, 0); + else if (tls_tx_key(toep)) + t4_push_tls_records(sc, toep, 0); else t4_push_frames(sc, toep, 0); } @@ -1772,6 +1817,10 @@ credits -= txsd->tx_credits; toep->tx_credits += txsd->tx_credits; plen += txsd->plen; + if (txsd->iv_buffer) { + free(txsd->iv_buffer, M_CXGBE); + txsd->iv_buffer = NULL; + } txsd++; toep->txsd_avail++; KASSERT(toep->txsd_avail <= toep->txsd_total, @@ -1797,6 +1846,8 @@ CURVNET_SET(toep->vnet); if (toep->ulp_mode == ULP_MODE_ISCSI) t4_push_pdus(sc, toep, plen); + else if (tls_tx_key(toep)) + t4_push_tls_records(sc, toep, plen); else t4_push_frames(sc, toep, plen); CURVNET_RESTORE(); @@ -1826,6 +1877,12 @@ tid, plen); #endif sbdrop_locked(sb, plen); + if (tls_tx_key(toep)) { + struct tls_ofld_info *tls_ofld = &toep->tls; + + MPASS(tls_ofld->sb_off >= plen); + tls_ofld->sb_off -= plen; + } if (!TAILQ_EMPTY(&toep->aiotx_jobq)) t4_aiotx_queue_toep(toep); sowwakeup_locked(so); /* unlocks so_snd */ @@ -2300,6 +2357,9 @@ if (!sc->tt.tx_zcopy) return (EOPNOTSUPP); + if (is_tls_offload(toep) || tls_tx_key(toep)) + return (EOPNOTSUPP); + SOCKBUF_LOCK(&so->so_snd); #ifdef VERBOSE_TRACES CTR2(KTR_CXGBE, "%s: queueing %p", __func__, job); Index: sys/dev/cxgbe/tom/t4_ddp.c =================================================================== --- sys/dev/cxgbe/tom/t4_ddp.c +++ sys/dev/cxgbe/tom/t4_ddp.c @@ -1939,7 +1939,7 @@ return (0); } -int +void t4_ddp_mod_load(void) { @@ -1948,7 +1948,6 @@ TAILQ_INIT(&ddp_orphan_pagesets); mtx_init(&ddp_orphan_pagesets_lock, "ddp orphans", NULL, MTX_DEF); TASK_INIT(&ddp_orphan_task, 0, ddp_free_orphan_pagesets, NULL); - return (0); } void Index: sys/dev/cxgbe/tom/t4_listen.c =================================================================== --- sys/dev/cxgbe/tom/t4_listen.c +++ sys/dev/cxgbe/tom/t4_listen.c @@ -1056,6 +1056,11 @@ if (ulp_mode == ULP_MODE_TCPDDP) opt2 |= F_RX_FC_VALID | F_RX_FC_DDP; #endif + if (ulp_mode == ULP_MODE_TLS) { + opt2 |= F_RX_FC_VALID; + opt2 &= ~V_RX_COALESCE(M_RX_COALESCE); + opt2 |= F_RX_FC_DISABLE; + } return htobe32(opt2); } @@ -1347,11 +1352,15 @@ INIT_TP_WR_MIT_CPL(rpl5, CPL_PASS_ACCEPT_RPL, tid); } - if (sc->tt.ddp && (so->so_options & SO_NO_DDP) == 0) { - ulp_mode = ULP_MODE_TCPDDP; + ulp_mode = select_ulp_mode(so, sc); + switch (ulp_mode) { + case ULP_MODE_TCPDDP: synqe->flags |= TPF_SYNQE_TCPDDP; - } else - ulp_mode = ULP_MODE_NONE; + break; + case ULP_MODE_TLS: + synqe->flags |= TPF_SYNQE_TLS; + break; + } rpl->opt0 = calc_opt0(so, vi, e, mtu_idx, rscale, rx_credits, ulp_mode); rpl->opt2 = calc_opt2p(sc, pi, rxqid, &cpl->tcpopt, &th, ulp_mode); @@ -1407,8 +1416,8 @@ REJECT_PASS_ACCEPT(); } - CTR5(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p, synqe %p, SYNACK", - __func__, stid, tid, lctx, synqe); + CTR6(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p, synqe %p, SYNACK mode %d", + __func__, stid, tid, lctx, synqe, ulp_mode); INP_WLOCK(inp); synqe->flags |= TPF_SYNQE_HAS_L2TE; @@ -1557,9 +1566,11 @@ toep->tid = tid; toep->l2te = &sc->l2t->l2tab[synqe->l2e_idx]; if (synqe->flags & TPF_SYNQE_TCPDDP) - set_tcpddp_ulp_mode(toep); + set_ulp_mode(toep, ULP_MODE_TCPDDP); + else if (synqe->flags & TPF_SYNQE_TLS) + set_ulp_mode(toep, ULP_MODE_TLS); else - toep->ulp_mode = ULP_MODE_NONE; + set_ulp_mode(toep, ULP_MODE_NONE); /* opt0 rcv_bufsiz initially, assumes its normal meaning later */ toep->rx_credits = synqe->rcv_bufsize; Index: sys/dev/cxgbe/tom/t4_tls.h =================================================================== --- /dev/null +++ sys/dev/cxgbe/tom/t4_tls.h @@ -0,0 +1,591 @@ +/*- + * Copyright (c) 2017 Chelsio Communications, Inc. + * All rights reserved. + * Written by: John Baldwin , Atul Gupta + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef __T4_TLS_H__ +#define __T4_TLS_H__ + +#define TLS1_VERSION 0x0301 +#define TLS1_1_VERSION 0x0302 +#define TLS1_2_VERSION 0x0303 +#define TLS_MAX_VERSION TLS1_2_VERSION + +#define DTLS1_VERSION 0xFEFF +#define DTLS1_2_VERSION 0xFEFD +#define DTLS_MAX_VERSION DTLS1_2_VERSION +#define DTLS1_VERSION_MAJOR 0xFE + +/* Custom socket options for TLS+TOE. */ + +#define MAX_MAC_KSZ 64 /*512 bits */ +#define MAX_CIPHER_KSZ 32 /* 256 bits */ +#define CIPHER_BLOCK_SZ 16 +#define SALT_SIZE 4 + +/* Can accomodate 16, 11-15 are reserved */ +enum { + CHSSL_SHA_NOP, + CHSSL_SHA1, + CHSSL_SHA224, + CHSSL_SHA256, + CHSSL_GHASH, + CHSSL_SHA512_224, + CHSSL_SHA512_256, + CHSSL_SHA512_384, + CHSSL_SHA512_512, + CHSSL_CBCMAC, + CHSSL_CMAC, +}; + +/* Can accomodate 16, 8-15 are reserved */ +enum { + CHSSL_CIPH_NOP, + CHSSL_AES_CBC, + CHSSL_AES_GCM, + CHSSL_AES_CTR, + CHSSL_AES_GEN, + CHSSL_IPSEC_ESP, + CHSSL_AES_XTS, + CHSSL_AES_CCM, +}; + +/* Key Context Programming Operation type */ +#define KEY_WRITE_RX 0x1 +#define KEY_WRITE_TX 0x2 +#define KEY_DELETE_RX 0x4 +#define KEY_DELETE_TX 0x8 + +#define S_KEY_CLR_LOC 4 +#define M_KEY_CLR_LOC 0xf +#define V_KEY_CLR_LOC(x) ((x) << S_KEY_CLR_LOC) +#define G_KEY_CLR_LOC(x) (((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC) +#define F_KEY_CLR_LOC V_KEY_CLR_LOC(1U) + +#define S_KEY_GET_LOC 0 +#define M_KEY_GET_LOC 0xf +#define V_KEY_GET_LOC(x) ((x) << S_KEY_GET_LOC) +#define G_KEY_GET_LOC(x) (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC) + +struct tls_ofld_state { + unsigned char enc_mode; + unsigned char mac_mode; + unsigned char key_loc; + unsigned char ofld_mode; + unsigned char auth_mode; + unsigned char resv[3]; +}; + +struct tls_tx_ctxt { + unsigned char salt[SALT_SIZE]; + unsigned char key[MAX_CIPHER_KSZ]; + unsigned char ipad[MAX_MAC_KSZ]; + unsigned char opad[MAX_MAC_KSZ]; +}; + +struct tls_rx_ctxt { + unsigned char salt[SALT_SIZE]; + unsigned char key[MAX_CIPHER_KSZ]; + unsigned char ipad[MAX_MAC_KSZ]; + unsigned char opad[MAX_MAC_KSZ]; +}; + +struct tls_key_context { + struct tls_tx_ctxt tx; + struct tls_rx_ctxt rx; + + unsigned char l_p_key; + unsigned char hmac_ctrl; + unsigned char mac_first; + unsigned char iv_size; + unsigned char iv_ctrl; + unsigned char iv_algo; + unsigned char tx_seq_no; + unsigned char rx_seq_no; + + struct tls_ofld_state state; + + unsigned int tx_key_info_size; + unsigned int rx_key_info_size; + unsigned int frag_size; + unsigned int mac_secret_size; + unsigned int cipher_secret_size; + int proto_ver; + unsigned int sock_fd; + unsigned short dtls_epoch; + unsigned short rsv; +}; + +/* Set with 'struct tls_key_context'. */ +#define TCP_TLSOM_SET_TLS_CONTEXT (TCP_VENDOR) + +/* Get returns int of enabled (1) / disabled (0). */ +#define TCP_TLSOM_GET_TLS_TOM (TCP_VENDOR + 1) + +enum { + TLS_TOM_NONE = 0, + TLS_TOM_TXONLY, + TLS_TOM_BOTH +}; + +/* Set with no value. */ +#define TCP_TLSOM_CLR_TLS_TOM (TCP_VENDOR + 2) + +/* Set with no value. */ +#define TCP_TLSOM_CLR_QUIES (TCP_VENDOR + 3) + +#ifdef _KERNEL +/* Timeouts for handshake timer in seconds. */ +#define TLS_SRV_HELLO_DONE 9 +#define TLS_SRV_HELLO_RD_TM 5 +#define TLS_SRV_HELLO_BKOFF_TM 15 + +#define CONTENT_TYPE_CCS 20 +#define CONTENT_TYPE_ALERT 21 +#define CONTENT_TYPE_HANDSHAKE 22 +#define CONTENT_TYPE_APP_DATA 23 +#define CONTENT_TYPE_HEARTBEAT 24 +#define CONTENT_TYPE_KEY_CONTEXT 32 +#define CONTENT_TYPE_ERROR 127 + +#define GCM_TAG_SIZE 16 +#define AEAD_EXPLICIT_DATA_SIZE 8 +#define TLS_HEADER_LENGTH 5 +#define TP_TX_PG_SZ 65536 +#define FC_TP_PLEN_MAX 17408 + +#define IPAD_SIZE 64 +#define OPAD_SIZE 64 +#define KEY_SIZE 32 +#define CIPHER_BLOCK_SIZE 16 +#define HDR_KCTX_SIZE (IPAD_SIZE + OPAD_SIZE + KEY_SIZE) + +#define KEY_IN_DDR_SIZE 16 +#define TLS_KEY_CONTEXT_SZ roundup2(sizeof(struct tls_tx_ctxt), 32) + +/* MAC KEY SIZE */ +#define SHA_NOP 0 +#define SHA_GHASH 16 +#define SHA_224 28 +#define SHA_256 32 +#define SHA_384 48 +#define SHA_512 64 +#define SHA1 20 + +/* CIPHER KEY SIZE */ +#define AES_NOP 0 +#define AES_128 16 +#define AES_192 24 +#define AES_256 32 + +enum { + TLS_1_2_VERSION, + TLS_1_1_VERSION, + DTLS_1_2_VERSION, + TLS_VERSION_MAX, +}; + +enum { + CH_EVP_CIPH_STREAM_CIPHER, + CH_EVP_CIPH_CBC_MODE, + CH_EVP_CIPH_GCM_MODE, + CH_EVP_CIPH_CTR_MODE, +}; + +enum { + TLS_SFO_WR_CONTEXTLOC_DSGL, + TLS_SFO_WR_CONTEXTLOC_IMMEDIATE, + TLS_SFO_WR_CONTEXTLOC_DDR, +}; + +enum { + CPL_TX_TLS_SFO_TYPE_CCS, + CPL_TX_TLS_SFO_TYPE_ALERT, + CPL_TX_TLS_SFO_TYPE_HANDSHAKE, + CPL_TX_TLS_SFO_TYPE_DATA, + CPL_TX_TLS_SFO_TYPE_HEARTBEAT, /* XXX: Shouldn't this be "CUSTOM"? */ +}; + +enum { + CH_CK_SIZE_128, + CH_CK_SIZE_192, + CH_CK_SIZE_256, + CH_CK_SIZE_NOP, +}; + +enum { + CH_MK_SIZE_128, + CH_MK_SIZE_160, + CH_MK_SIZE_192, + CH_MK_SIZE_256, + CH_MK_SIZE_512, + CH_MK_SIZE_NOP, +}; + +#define SCMD_ENCDECCTRL_ENCRYPT 0 +#define SCMD_ENCDECCTRL_DECRYPT 1 + +#define SCMD_CIPH_MODE_NOP 0 +#define SCMD_CIPH_MODE_AES_CBC 1 +#define SCMD_CIPH_MODE_AES_GCM 2 +#define SCMD_CIPH_MODE_AES_CTR 3 +#define SCMD_CIPH_MODE_AES_GEN 4 +#define SCMD_CIPH_MODE_AES_CCM 7 + +struct tls_scmd { + __be32 seqno_numivs; + __be32 ivgen_hdrlen; +}; + +struct tls_ofld_info { + struct tls_key_context k_ctx; + int key_location; + int mac_length; + int rx_key_addr; + int tx_key_addr; + uint64_t tx_seq_no; + unsigned short fcplenmax; + unsigned short adjusted_plen; + unsigned short expn_per_ulp; + unsigned short pdus_per_ulp; + struct tls_scmd scmd0; + u_int sb_off; + struct callout handshake_timer; + u_int rcv_over; +}; + +struct tls_key_req { + __be32 wr_hi; + __be32 wr_mid; + __be32 ftid; + __u8 reneg_to_write_rx; + __u8 protocol; + __be16 mfs; + /* master command */ + __be32 cmd; + __be32 len16; /* command length */ + __be32 dlen; /* data length in 32-byte units */ + __be32 kaddr; + /* sub-command */ + __be32 sc_more; + __be32 sc_len; +}__packed; + +struct tls_keyctx { + union key_ctx { + struct tx_keyctx_hdr { + __u8 ctxlen; + __u8 r2; + __be16 dualck_to_txvalid; + __u8 txsalt[4]; + __be64 r5; + } txhdr; + struct rx_keyctx_hdr { + __u8 flitcnt_hmacctrl; + __u8 protover_ciphmode; + __u8 authmode_to_rxvalid; + __u8 ivpresent_to_rxmk_size; + __u8 rxsalt[4]; + __be64 ivinsert_to_authinsrt; + } rxhdr; + } u; + struct keys { + __u8 edkey[32]; + __u8 ipad[64]; + __u8 opad[64]; + } keys; +}; + +#define S_TLS_KEYCTX_TX_WR_DUALCK 12 +#define M_TLS_KEYCTX_TX_WR_DUALCK 0x1 +#define V_TLS_KEYCTX_TX_WR_DUALCK(x) ((x) << S_TLS_KEYCTX_TX_WR_DUALCK) +#define G_TLS_KEYCTX_TX_WR_DUALCK(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_DUALCK) & M_TLS_KEYCTX_TX_WR_DUALCK) +#define F_TLS_KEYCTX_TX_WR_DUALCK V_TLS_KEYCTX_TX_WR_DUALCK(1U) + +#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11 +#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1 +#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) +#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \ + M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) +#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \ + V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U) + +#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10 +#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1 +#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT) +#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \ + M_TLS_KEYCTX_TX_WR_SALT_PRESENT) +#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \ + V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U) + +#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6 +#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf +#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE) +#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \ + M_TLS_KEYCTX_TX_WR_TXCK_SIZE) + +#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2 +#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf +#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE) +#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \ + M_TLS_KEYCTX_TX_WR_TXMK_SIZE) + +#define S_TLS_KEYCTX_TX_WR_TXVALID 0 +#define M_TLS_KEYCTX_TX_WR_TXVALID 0x1 +#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_TXVALID) +#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID) +#define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U) + +#define S_TLS_KEYCTX_TX_WR_FLITCNT 3 +#define M_TLS_KEYCTX_TX_WR_FLITCNT 0x1f +#define V_TLS_KEYCTX_TX_WR_FLITCNT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_FLITCNT) +#define G_TLS_KEYCTX_TX_WR_FLITCNT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_FLITCNT) & M_TLS_KEYCTX_TX_WR_FLITCNT) + +#define S_TLS_KEYCTX_TX_WR_HMACCTRL 0 +#define M_TLS_KEYCTX_TX_WR_HMACCTRL 0x7 +#define V_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_HMACCTRL) +#define G_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_HMACCTRL) & M_TLS_KEYCTX_TX_WR_HMACCTRL) + +#define S_TLS_KEYCTX_TX_WR_PROTOVER 4 +#define M_TLS_KEYCTX_TX_WR_PROTOVER 0xf +#define V_TLS_KEYCTX_TX_WR_PROTOVER(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_PROTOVER) +#define G_TLS_KEYCTX_TX_WR_PROTOVER(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_PROTOVER) & M_TLS_KEYCTX_TX_WR_PROTOVER) + +#define S_TLS_KEYCTX_TX_WR_CIPHMODE 0 +#define M_TLS_KEYCTX_TX_WR_CIPHMODE 0xf +#define V_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_CIPHMODE) +#define G_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_CIPHMODE) & M_TLS_KEYCTX_TX_WR_CIPHMODE) + +#define S_TLS_KEYCTX_TX_WR_AUTHMODE 4 +#define M_TLS_KEYCTX_TX_WR_AUTHMODE 0xf +#define V_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AUTHMODE) +#define G_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AUTHMODE) & M_TLS_KEYCTX_TX_WR_AUTHMODE) + +#define S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 3 +#define M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 0x1 +#define V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) +#define G_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) & \ + M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) +#define F_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL \ + V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1U) + +#define S_TLS_KEYCTX_TX_WR_SEQNUMCTRL 1 +#define M_TLS_KEYCTX_TX_WR_SEQNUMCTRL 0x3 +#define V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) +#define G_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) & \ + M_TLS_KEYCTX_TX_WR_SEQNUMCTRL) + +#define S_TLS_KEYCTX_TX_WR_RXVALID 0 +#define M_TLS_KEYCTX_TX_WR_RXVALID 0x1 +#define V_TLS_KEYCTX_TX_WR_RXVALID(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_RXVALID) +#define G_TLS_KEYCTX_TX_WR_RXVALID(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_RXVALID) & M_TLS_KEYCTX_TX_WR_RXVALID) +#define F_TLS_KEYCTX_TX_WR_RXVALID V_TLS_KEYCTX_TX_WR_RXVALID(1U) + +#define S_TLS_KEYCTX_TX_WR_IVPRESENT 7 +#define M_TLS_KEYCTX_TX_WR_IVPRESENT 0x1 +#define V_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_IVPRESENT) +#define G_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_IVPRESENT) & \ + M_TLS_KEYCTX_TX_WR_IVPRESENT) +#define F_TLS_KEYCTX_TX_WR_IVPRESENT V_TLS_KEYCTX_TX_WR_IVPRESENT(1U) + +#define S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 6 +#define M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 0x1 +#define V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) +#define G_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) & \ + M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) +#define F_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT \ + V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1U) + +#define S_TLS_KEYCTX_TX_WR_RXCK_SIZE 3 +#define M_TLS_KEYCTX_TX_WR_RXCK_SIZE 0x7 +#define V_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_RXCK_SIZE) +#define G_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_RXCK_SIZE) & \ + M_TLS_KEYCTX_TX_WR_RXCK_SIZE) + +#define S_TLS_KEYCTX_TX_WR_RXMK_SIZE 0 +#define M_TLS_KEYCTX_TX_WR_RXMK_SIZE 0x7 +#define V_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_RXMK_SIZE) +#define G_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_RXMK_SIZE) & \ + M_TLS_KEYCTX_TX_WR_RXMK_SIZE) + +#define S_TLS_KEYCTX_TX_WR_IVINSERT 55 +#define M_TLS_KEYCTX_TX_WR_IVINSERT 0x1ffULL +#define V_TLS_KEYCTX_TX_WR_IVINSERT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_IVINSERT) +#define G_TLS_KEYCTX_TX_WR_IVINSERT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_IVINSERT) & M_TLS_KEYCTX_TX_WR_IVINSERT) + +#define S_TLS_KEYCTX_TX_WR_AADSTRTOFST 47 +#define M_TLS_KEYCTX_TX_WR_AADSTRTOFST 0xffULL +#define V_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AADSTRTOFST) +#define G_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AADSTRTOFST) & \ + M_TLS_KEYCTX_TX_WR_AADSTRTOFST) + +#define S_TLS_KEYCTX_TX_WR_AADSTOPOFST 39 +#define M_TLS_KEYCTX_TX_WR_AADSTOPOFST 0xffULL +#define V_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AADSTOPOFST) +#define G_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AADSTOPOFST) & \ + M_TLS_KEYCTX_TX_WR_AADSTOPOFST) + +#define S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 30 +#define M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 0x1ffULL +#define V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) +#define G_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) & \ + M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) + +#define S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 23 +#define M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 0x7f +#define V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) +#define G_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) & \ + M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) + +#define S_TLS_KEYCTX_TX_WR_AUTHSRTOFST 14 +#define M_TLS_KEYCTX_TX_WR_AUTHSRTOFST 0x1ff +#define V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) +#define G_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) & \ + M_TLS_KEYCTX_TX_WR_AUTHSRTOFST) + +#define S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 7 +#define M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 0x7f +#define V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) +#define G_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) & \ + M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) + +#define S_TLS_KEYCTX_TX_WR_AUTHINSRT 0 +#define M_TLS_KEYCTX_TX_WR_AUTHINSRT 0x7f +#define V_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AUTHINSRT) +#define G_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AUTHINSRT) & \ + M_TLS_KEYCTX_TX_WR_AUTHINSRT) + +struct tls_hdr { + __u8 type; + __be16 version; + __be16 length; +} __packed; + +struct tlsrx_hdr_pkt { + __u8 type; + __be16 version; + __be16 length; + + __be64 tls_seq; + __be16 reserved1; + __u8 res_to_mac_error; +} __packed; + +/* res_to_mac_error fields */ +#define S_TLSRX_HDR_PKT_INTERNAL_ERROR 4 +#define M_TLSRX_HDR_PKT_INTERNAL_ERROR 0x1 +#define V_TLSRX_HDR_PKT_INTERNAL_ERROR(x) \ + ((x) << S_TLSRX_HDR_PKT_INTERNAL_ERROR) +#define G_TLSRX_HDR_PKT_INTERNAL_ERROR(x) \ +(((x) >> S_TLSRX_HDR_PKT_INTERNAL_ERROR) & M_TLSRX_HDR_PKT_INTERNAL_ERROR) +#define F_TLSRX_HDR_PKT_INTERNAL_ERROR V_TLSRX_HDR_PKT_INTERNAL_ERROR(1U) + +#define S_TLSRX_HDR_PKT_SPP_ERROR 3 +#define M_TLSRX_HDR_PKT_SPP_ERROR 0x1 +#define V_TLSRX_HDR_PKT_SPP_ERROR(x) ((x) << S_TLSRX_HDR_PKT_SPP_ERROR) +#define G_TLSRX_HDR_PKT_SPP_ERROR(x) \ +(((x) >> S_TLSRX_HDR_PKT_SPP_ERROR) & M_TLSRX_HDR_PKT_SPP_ERROR) +#define F_TLSRX_HDR_PKT_SPP_ERROR V_TLSRX_HDR_PKT_SPP_ERROR(1U) + +#define S_TLSRX_HDR_PKT_CCDX_ERROR 2 +#define M_TLSRX_HDR_PKT_CCDX_ERROR 0x1 +#define V_TLSRX_HDR_PKT_CCDX_ERROR(x) ((x) << S_TLSRX_HDR_PKT_CCDX_ERROR) +#define G_TLSRX_HDR_PKT_CCDX_ERROR(x) \ +(((x) >> S_TLSRX_HDR_PKT_CCDX_ERROR) & M_TLSRX_HDR_PKT_CCDX_ERROR) +#define F_TLSRX_HDR_PKT_CCDX_ERROR V_TLSRX_HDR_PKT_CCDX_ERROR(1U) + +#define S_TLSRX_HDR_PKT_PAD_ERROR 1 +#define M_TLSRX_HDR_PKT_PAD_ERROR 0x1 +#define V_TLSRX_HDR_PKT_PAD_ERROR(x) ((x) << S_TLSRX_HDR_PKT_PAD_ERROR) +#define G_TLSRX_HDR_PKT_PAD_ERROR(x) \ +(((x) >> S_TLSRX_HDR_PKT_PAD_ERROR) & M_TLSRX_HDR_PKT_PAD_ERROR) +#define F_TLSRX_HDR_PKT_PAD_ERROR V_TLSRX_HDR_PKT_PAD_ERROR(1U) + +#define S_TLSRX_HDR_PKT_MAC_ERROR 0 +#define M_TLSRX_HDR_PKT_MAC_ERROR 0x1 +#define V_TLSRX_HDR_PKT_MAC_ERROR(x) ((x) << S_TLSRX_HDR_PKT_MAC_ERROR) +#define G_TLSRX_HDR_PKT_MAC_ERROR(x) \ +(((x) >> S_TLSRX_HDR_PKT_MAC_ERROR) & M_TLSRX_HDR_PKT_MAC_ERROR) +#define F_TLSRX_HDR_PKT_MAC_ERROR V_TLSRX_HDR_PKT_MAC_ERROR(1U) + +#define M_TLSRX_HDR_PKT_ERROR 0x1F + +#endif /* _KERNEL */ + +#endif /* !__T4_TLS_H__ */ Index: sys/dev/cxgbe/tom/t4_tls.c =================================================================== --- /dev/null +++ sys/dev/cxgbe/tom/t4_tls.c @@ -0,0 +1,1683 @@ +/*- + * Copyright (c) 2017 Chelsio Communications, Inc. + * All rights reserved. + * Written by: John Baldwin + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "opt_inet.h" + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef TCP_OFFLOAD +#include "common/common.h" +#include "common/t4_tcb.h" +#include "tom/t4_tom_l2t.h" +#include "tom/t4_tom.h" + +/* + * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while + * the mbuf is in the ulp_pdu_reclaimq. + */ +#define tls_tcp_seq PH_loc.thirtytwo[0] + +/* + * Handshake lock used for the handshake timer. Having a global lock + * is perhaps not ideal, but it avoids having to use callout_drain() + * in tls_uninit_toep() which can't block. Also, the timer shouldn't + * actually fire for most connections. + */ +static struct mtx tls_handshake_lock; + +static void +t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, + uint64_t val) +{ + struct adapter *sc = td_adapter(toep->td); + + t4_set_tcb_field(sc, toep->ctrlq, toep->tid, word, mask, val, 0, 0, + toep->ofld_rxq->iq.abs_id); +} + +/* TLS and DTLS common routines */ +int +tls_tx_key(struct toepcb *toep) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + + return (tls_ofld->tx_key_addr >= 0); +} + +int +tls_rx_key(struct toepcb *toep) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + + return (tls_ofld->rx_key_addr >= 0); +} + +static int +key_size(struct toepcb *toep) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + + return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ? + tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE); +} + +/* Set TLS Key-Id in TCB */ +static void +t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) +{ + + t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, + V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), + V_TCB_RX_TLS_KEY_TAG(key_id)); +} + +/* Clear TF_RX_QUIESCE to re-enable receive. */ +static void +t4_clear_rx_quiesce(struct toepcb *toep) +{ + + t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); +} + +static void +tls_clr_ofld_mode(struct toepcb *toep) +{ + + tls_stop_handshake_timer(toep); + + /* Operate in PDU extraction mode only. */ + t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, + V_TCB_ULP_RAW(M_TCB_ULP_RAW), + V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); + t4_clear_rx_quiesce(toep); +} + +static void +tls_clr_quiesce(struct toepcb *toep) +{ + + tls_stop_handshake_timer(toep); + t4_clear_rx_quiesce(toep); +} + +/* + * Calculate the TLS data expansion size + */ +static int +tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only, + unsigned short *pdus_per_ulp) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + struct tls_scmd *scmd = &tls_ofld->scmd0; + int expn_size = 0, frag_count = 0, pad_per_pdu = 0, + pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0; + int exp_per_pdu = 0; + int hdr_len = TLS_HEADER_LENGTH; + + do { + max_frag_size = tls_ofld->k_ctx.frag_size; + if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) == + SCMD_CIPH_MODE_AES_GCM) { + frag_count = (data_len / max_frag_size); + exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE + + hdr_len; + expn_size = frag_count * exp_per_pdu; + if (full_pdus_only) { + *pdus_per_ulp = data_len / (exp_per_pdu + + max_frag_size); + if (*pdus_per_ulp > 32) + *pdus_per_ulp = 32; + else if(!*pdus_per_ulp) + *pdus_per_ulp = 1; + expn_size = (*pdus_per_ulp) * exp_per_pdu; + break; + } + if ((last_frag_size = data_len % max_frag_size) > 0) { + frag_count += 1; + expn_size += exp_per_pdu; + } + break; + } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) != + SCMD_CIPH_MODE_NOP) { + /* Calculate the number of fragments we can make */ + frag_count = (data_len / max_frag_size); + if (frag_count > 0) { + pad_per_pdu = (((howmany((max_frag_size + + tls_ofld->mac_length), + CIPHER_BLOCK_SIZE)) * + CIPHER_BLOCK_SIZE) - + (max_frag_size + + tls_ofld->mac_length)); + if (!pad_per_pdu) + pad_per_pdu = CIPHER_BLOCK_SIZE; + exp_per_pdu = pad_per_pdu + + tls_ofld->mac_length + + hdr_len + CIPHER_BLOCK_SIZE; + expn_size = frag_count * exp_per_pdu; + } + if (full_pdus_only) { + *pdus_per_ulp = data_len / (exp_per_pdu + + max_frag_size); + if (*pdus_per_ulp > 32) + *pdus_per_ulp = 32; + else if (!*pdus_per_ulp) + *pdus_per_ulp = 1; + expn_size = (*pdus_per_ulp) * exp_per_pdu; + break; + } + /* Consider the last fragment */ + if ((last_frag_size = data_len % max_frag_size) > 0) { + pad_last_pdu = (((howmany((last_frag_size + + tls_ofld->mac_length), + CIPHER_BLOCK_SIZE)) * + CIPHER_BLOCK_SIZE) - + (last_frag_size + + tls_ofld->mac_length)); + if (!pad_last_pdu) + pad_last_pdu = CIPHER_BLOCK_SIZE; + expn_size += (pad_last_pdu + + tls_ofld->mac_length + hdr_len + + CIPHER_BLOCK_SIZE); + } + } + } while (0); + + return (expn_size); +} + +/* Copy Key to WR */ +static void +tls_copy_tx_key(struct toepcb *toep, void *dst) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + struct ulptx_sc_memrd *sc_memrd; + struct ulptx_idata *sc; + + if (tls_ofld->k_ctx.tx_key_info_size <= 0) + return; + + if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) { + sc = dst; + sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); + sc->len = htobe32(0); + sc_memrd = (struct ulptx_sc_memrd *)(sc + 1); + sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | + V_ULP_TX_SC_MORE(1) | + V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4)); + sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5); + } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) { + memcpy(dst, &tls_ofld->k_ctx.tx, + tls_ofld->k_ctx.tx_key_info_size); + } +} + +/* TLS/DTLS content type for CPL SFO */ +static inline unsigned char +tls_content_type(unsigned char content_type) +{ + /* + * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and + * default to "CUSTOM" for all other types including + * heartbeat? + */ + switch (content_type) { + case CONTENT_TYPE_CCS: + return CPL_TX_TLS_SFO_TYPE_CCS; + case CONTENT_TYPE_ALERT: + return CPL_TX_TLS_SFO_TYPE_ALERT; + case CONTENT_TYPE_HANDSHAKE: + return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; + case CONTENT_TYPE_HEARTBEAT: + return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; + } + return CPL_TX_TLS_SFO_TYPE_DATA; +} + +static unsigned char +get_cipher_key_size(unsigned int ck_size) +{ + switch (ck_size) { + case AES_NOP: /* NOP */ + return 15; + case AES_128: /* AES128 */ + return CH_CK_SIZE_128; + case AES_192: /* AES192 */ + return CH_CK_SIZE_192; + case AES_256: /* AES256 */ + return CH_CK_SIZE_256; + default: + return CH_CK_SIZE_256; + } +} + +static unsigned char +get_mac_key_size(unsigned int mk_size) +{ + switch (mk_size) { + case SHA_NOP: /* NOP */ + return CH_MK_SIZE_128; + case SHA_GHASH: /* GHASH */ + case SHA_512: /* SHA512 */ + return CH_MK_SIZE_512; + case SHA_224: /* SHA2-224 */ + return CH_MK_SIZE_192; + case SHA_256: /* SHA2-256*/ + return CH_MK_SIZE_256; + case SHA_384: /* SHA384 */ + return CH_MK_SIZE_512; + case SHA1: /* SHA1 */ + default: + return CH_MK_SIZE_160; + } +} + +static unsigned int +get_proto_ver(int proto_ver) +{ + switch (proto_ver) { + case TLS1_2_VERSION: + return TLS_1_2_VERSION; + case TLS1_1_VERSION: + return TLS_1_1_VERSION; + case DTLS1_2_VERSION: + return DTLS_1_2_VERSION; + default: + return TLS_VERSION_MAX; + } +} + +static void +tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx) +{ + + if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { + kwr->u.rxhdr.ivinsert_to_authinsrt = + htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | + V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | + V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | + V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | + V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | + V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | + V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | + V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); + kwr->u.rxhdr.ivpresent_to_rxmk_size &= + ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1)); + kwr->u.rxhdr.authmode_to_rxvalid &= + ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1)); + } else { + kwr->u.rxhdr.ivinsert_to_authinsrt = + htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | + V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | + V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | + V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | + V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | + V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | + V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | + V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); + } +} + +/* Rx key */ +static void +prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) +{ + unsigned int ck_size = kctx->cipher_secret_size; + unsigned int mk_size = kctx->mac_secret_size; + int proto_ver = kctx->proto_ver; + + kwr->u.rxhdr.flitcnt_hmacctrl = + ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; + + kwr->u.rxhdr.protover_ciphmode = + V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) | + V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode); + + kwr->u.rxhdr.authmode_to_rxvalid = + V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) | + V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) | + V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | + V_TLS_KEYCTX_TX_WR_RXVALID(1); + + kwr->u.rxhdr.ivpresent_to_rxmk_size = + V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | + V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) | + V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) | + V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size)); + + tls_rxkey_flit1(kwr, kctx); + + /* No key reversal for GCM */ + if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) { + t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key, + (kctx->cipher_secret_size << 3)); + memcpy(kwr->keys.edkey + kctx->cipher_secret_size, + kctx->rx.key + kctx->cipher_secret_size, + (IPAD_SIZE + OPAD_SIZE)); + } else { + memcpy(kwr->keys.edkey, kctx->rx.key, + (kctx->tx_key_info_size - SALT_SIZE)); + memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE); + } +} + +/* Tx key */ +static void +prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) +{ + unsigned int ck_size = kctx->cipher_secret_size; + unsigned int mk_size = kctx->mac_secret_size; + + kwr->u.txhdr.ctxlen = + (kctx->tx_key_info_size >> 4); + kwr->u.txhdr.dualck_to_txvalid = + V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) | + V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | + V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) | + V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) | + V_TLS_KEYCTX_TX_WR_TXVALID(1); + + memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE); + if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { + memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE); + kwr->u.txhdr.dualck_to_txvalid &= + ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1)); + } + kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid); +} + +/* TLS Key memory management */ +int +tls_init_kmap(struct adapter *sc, struct tom_data *td) +{ + + td->key_map = vmem_create("T4TLS key map", sc->vres.key.start, + sc->vres.key.size, 8, 0, M_FIRSTFIT | M_NOWAIT); + if (td->key_map == NULL) + return (ENOMEM); + return (0); +} + +void +tls_free_kmap(struct tom_data *td) +{ + + if (td->key_map != NULL) + vmem_destroy(td->key_map); +} + +static int +get_new_keyid(struct toepcb *toep, struct tls_key_context *k_ctx) +{ + struct tom_data *td = toep->td; + vmem_addr_t addr; + + if (vmem_alloc(td->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, + &addr) != 0) + return (-1); + + return (addr); +} + +static void +free_keyid(struct toepcb *toep, int keyid) +{ + struct tom_data *td = toep->td; + + vmem_free(td->key_map, keyid, TLS_KEY_CONTEXT_SZ); +} + +static void +clear_tls_keyid(struct toepcb *toep) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + + if (tls_ofld->rx_key_addr >= 0) { + free_keyid(toep, tls_ofld->rx_key_addr); + tls_ofld->rx_key_addr = -1; + } + if (tls_ofld->tx_key_addr >= 0) { + free_keyid(toep, tls_ofld->tx_key_addr); + tls_ofld->tx_key_addr = -1; + } +} + +static int +get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops) +{ + return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr : + ((ops & KEY_WRITE_TX) ? tls_ofld->rx_key_addr : -1)); +} + +static int +get_tp_plen_max(struct tls_ofld_info *tls_ofld) +{ + int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; + + return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX); +} + +/* Send request to get the key-id */ +static int +tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + struct adapter *sc = td_adapter(toep->td); + struct ofld_tx_sdesc *txsd; + int kwrlen, kctxlen, keyid, len; + struct wrqe *wr; + struct tls_key_req *kwr; + struct tls_keyctx *kctx; + + kwrlen = roundup2(sizeof(*kwr), 16); + kctxlen = roundup2(sizeof(*kctx), 32); + len = kwrlen + kctxlen; + + if (toep->txsd_avail == 0) + return (EAGAIN); + + /* Dont initialize key for re-neg */ + if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) { + if ((keyid = get_new_keyid(toep, k_ctx)) < 0) { + return (ENOSPC); + } + } else { + keyid = get_keyid(tls_ofld, k_ctx->l_p_key); + } + + wr = alloc_wrqe(len, toep->ofld_txq); + if (wr == NULL) { + free_keyid(toep, keyid); + return (ENOMEM); + } + kwr = wrtod(wr); + memset(kwr, 0, kwrlen); + + kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL | + F_FW_WR_ATOMIC); + kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) | + V_FW_WR_FLOWID(toep->tid)); + kwr->protocol = get_proto_ver(k_ctx->proto_ver); + kwr->mfs = htons(k_ctx->frag_size); + kwr->reneg_to_write_rx = k_ctx->l_p_key; + + /* master command */ + kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | + V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); + kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); + kwr->len16 = htobe32((toep->tid << 8) | + DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); + kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); + + /* sub command */ + kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); + kwr->sc_len = htobe32(kctxlen); + + /* XXX: This assumes that kwrlen == sizeof(*kwr). */ + kctx = (struct tls_keyctx *)(kwr + 1); + memset(kctx, 0, kctxlen); + + if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { + tls_ofld->tx_key_addr = keyid; + prepare_txkey_wr(kctx, k_ctx); + } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { + tls_ofld->rx_key_addr = keyid; + prepare_rxkey_wr(kctx, k_ctx); + } + + txsd = &toep->txsd[toep->txsd_pidx]; + txsd->tx_credits = DIV_ROUND_UP(len, 16); + txsd->plen = 0; + toep->tx_credits -= txsd->tx_credits; + if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) + toep->txsd_pidx = 0; + toep->txsd_avail--; + +#if 0 + device_printf(sc->dev, "submitting TLS key for addr %#x\n", keyid); + hexdump(kwr, len, NULL, HD_OMIT_CHARS | HD_OMIT_COUNT); +#endif + t4_wrq_tx(sc, wr); + + return (0); +} + +/* Store a key received from SSL in DDR. */ +static int +program_key_context(struct tcpcb *tp, struct toepcb *toep, + struct tls_key_context *uk_ctx) +{ + struct adapter *sc = td_adapter(toep->td); + struct tls_ofld_info *tls_ofld = &toep->tls; + struct tls_key_context *k_ctx; + int error, key_offset; + + if (tp->t_state != TCPS_ESTABLISHED) { + /* + * XXX: Matches Linux driver, but not sure this is a + * very appropriate error. + */ + return (ENOENT); + } + + /* Stop timer on handshake completion */ + tls_stop_handshake_timer(toep); + + toep->flags &= ~TPF_FORCE_CREDITS; + + CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid, + G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" : + "KEY_WRITE_TX", uk_ctx->proto_ver); + + if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX && + toep->ulp_mode != ULP_MODE_TLS) + return (EOPNOTSUPP); + + /* Don't copy the 'tx' and 'rx' fields. */ + k_ctx = &tls_ofld->k_ctx; + memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key, + sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key)); + + /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */ + if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) { + if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { + tls_ofld->rx_key_addr = -1; + t4_clear_rx_quiesce(toep); + } else { + tls_ofld->tx_key_addr = -1; + } + return (0); + } + + if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { + k_ctx->iv_size = 4; + k_ctx->mac_first = 0; + k_ctx->hmac_ctrl = 0; + } else { + k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ + k_ctx->mac_first = 1; + } + + tls_ofld->scmd0.seqno_numivs = + (V_SCMD_SEQ_NO_CTRL(3) | + V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | + V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | + V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | + V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | + V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | + V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | + V_SCMD_IV_SIZE(k_ctx->iv_size)); + + tls_ofld->scmd0.ivgen_hdrlen = + (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | + V_SCMD_KEY_CTX_INLINE(0) | + V_SCMD_TLS_FRAG_ENABLE(1)); + + tls_ofld->mac_length = k_ctx->mac_secret_size; + + if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { + k_ctx->rx = uk_ctx->rx; + /* Dont initialize key for re-neg */ + if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) + tls_ofld->rx_key_addr = -1; + } else { + k_ctx->tx = uk_ctx->tx; + /* Dont initialize key for re-neg */ + if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) + tls_ofld->tx_key_addr = -1; + } + + /* Flush pending data before new Tx key becomes active */ + if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { + struct sockbuf *sb; + + /* XXX: This might not drain everything. */ + t4_push_frames(sc, toep, 0); + sb = &toep->inp->inp_socket->so_snd; + SOCKBUF_LOCK(sb); + + /* XXX: This asserts that everything has been pushed. */ + MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL); + sb->sb_sndptr = NULL; + tls_ofld->sb_off = sbavail(sb); + SOCKBUF_UNLOCK(sb); + tls_ofld->tx_seq_no = 0; + } + + if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) || + (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) { + error = tls_program_key_id(toep, k_ctx); + if (error) { + /* XXX: Only clear quiesce for KEY_WRITE_RX? */ + t4_clear_rx_quiesce(toep); + return (error); + } + } + + if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { + /* + * RX key tags are an index into the key portion of MA + * memory stored as an offset from the base address in + * units of 64 bytes. + */ + key_offset = tls_ofld->rx_key_addr - sc->vres.key.start; + t4_set_tls_keyid(toep, key_offset / 64); + t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, + V_TCB_ULP_RAW(M_TCB_ULP_RAW), + V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | + V_TF_TLS_CONTROL(1) | + V_TF_TLS_ACTIVE(1) | + V_TF_TLS_ENABLE(1)))); + t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, + V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), + V_TCB_TLS_SEQ(0)); + t4_clear_rx_quiesce(toep); + } else { + unsigned short pdus_per_ulp; + + if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) + tls_ofld->tx_key_addr = 1; + + tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld); + tls_ofld->expn_per_ulp = tls_expansion_size(toep, + tls_ofld->fcplenmax, 1, &pdus_per_ulp); + tls_ofld->pdus_per_ulp = pdus_per_ulp; + tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp * + ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) + + tls_ofld->k_ctx.frag_size); + } + + return (0); +} + +/* + * In some cases a client connection can hang without sending the + * ServerHelloDone message from the NIC to the host. Send a dummy + * RX_DATA_ACK with RX_MODULATE to unstick the connection. + */ +static void +tls_send_handshake_ack(void *arg) +{ + struct toepcb *toep = arg; + struct tls_ofld_info *tls_ofld = &toep->tls; + struct adapter *sc = td_adapter(toep->td); + + /* + * XXX: Does not have the t4_get_tcb() checks to refine the + * workaround. + */ + callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); + + CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); + send_rx_modulate(sc, toep); +} + +static void +tls_start_handshake_timer(struct toepcb *toep) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + + mtx_lock(&tls_handshake_lock); + callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, + tls_send_handshake_ack, toep); + mtx_unlock(&tls_handshake_lock); +} + +void +tls_stop_handshake_timer(struct toepcb *toep) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + + mtx_lock(&tls_handshake_lock); + callout_stop(&tls_ofld->handshake_timer); + mtx_unlock(&tls_handshake_lock); +} + +int +t4_ctloutput_tls(struct socket *so, struct sockopt *sopt) +{ + struct tls_key_context uk_ctx; + struct inpcb *inp; + struct tcpcb *tp; + struct toepcb *toep; + int error, optval; + + error = 0; + if (sopt->sopt_dir == SOPT_SET && + sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) { + error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx), + sizeof(uk_ctx)); + if (error) + return (error); + } + + inp = sotoinpcb(so); + KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL")); + INP_WLOCK(inp); + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + tp = intotcpcb(inp); + toep = tp->t_toe; + switch (sopt->sopt_dir) { + case SOPT_SET: + switch (sopt->sopt_name) { + case TCP_TLSOM_SET_TLS_CONTEXT: + error = program_key_context(tp, toep, &uk_ctx); + INP_WUNLOCK(inp); + break; + case TCP_TLSOM_CLR_TLS_TOM: + if (toep->ulp_mode == ULP_MODE_TLS) { + CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM", + __func__, toep->tid); + tls_clr_ofld_mode(toep); + } else + error = EOPNOTSUPP; + INP_WUNLOCK(inp); + break; + case TCP_TLSOM_CLR_QUIES: + if (toep->ulp_mode == ULP_MODE_TLS) { + CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES", + __func__, toep->tid); + tls_clr_quiesce(toep); + } else + error = EOPNOTSUPP; + INP_WUNLOCK(inp); + break; + default: + INP_WUNLOCK(inp); + error = EOPNOTSUPP; + break; + } + break; + case SOPT_GET: + switch (sopt->sopt_name) { + case TCP_TLSOM_GET_TLS_TOM: + /* + * TLS TX is permitted on any TOE socket, but + * TLS RX requires a TLS ULP mode. + */ + optval = TLS_TOM_NONE; + if (can_tls_offload(td_adapter(toep->td))) { + switch (toep->ulp_mode) { + case ULP_MODE_NONE: + case ULP_MODE_TCPDDP: + optval = TLS_TOM_TXONLY; + break; + case ULP_MODE_TLS: + optval = TLS_TOM_BOTH; + break; + } + } + CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d", + __func__, toep->tid, optval); + INP_WUNLOCK(inp); + error = sooptcopyout(sopt, &optval, sizeof(optval)); + break; + default: + INP_WUNLOCK(inp); + error = EOPNOTSUPP; + break; + } + break; + } + return (error); +} + +void +tls_init_toep(struct toepcb *toep) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + + tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR; + tls_ofld->rx_key_addr = -1; + tls_ofld->tx_key_addr = -1; + if (toep->ulp_mode == ULP_MODE_TLS) + callout_init_mtx(&tls_ofld->handshake_timer, + &tls_handshake_lock, 0); +} + +void +tls_establish(struct toepcb *toep) +{ + + /* + * Enable PDU extraction. + * + * XXX: Supposedly this should be done by the firmware when + * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but + * in practice this seems to be required. + */ + CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); + t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), + V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); + + toep->flags |= TPF_FORCE_CREDITS; + + tls_start_handshake_timer(toep); +} + +void +tls_uninit_toep(struct toepcb *toep) +{ + + if (toep->ulp_mode == ULP_MODE_TLS) + tls_stop_handshake_timer(toep); + clear_tls_keyid(toep); +} + +#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) +#define MIN_OFLD_TLSTX_CREDITS(toep) \ + (howmany(sizeof(struct fw_tlstx_data_wr) + \ + sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \ + CIPHER_BLOCK_SIZE + 1, 16)) + +static inline u_int +max_imm_tls_space(int tx_credits) +{ + const int n = 2; /* Use only up to 2 desc for imm. data WR */ + int space; + + KASSERT(tx_credits >= 0 && + tx_credits <= MAX_OFLD_TX_CREDITS, + ("%s: %d credits", __func__, tx_credits)); + + if (tx_credits >= (n * EQ_ESIZE) / 16) + space = (n * EQ_ESIZE); + else + space = tx_credits * 16; + return (space); +} + +static int +count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp) +{ + int max_nsegs_1mbuf, n, nsegs; + + while (skip >= m->m_len) { + skip -= m->m_len; + m = m->m_next; + } + + nsegs = 0; + max_nsegs_1mbuf = 0; + while (len > 0) { + n = sglist_count(mtod(m, char *) + skip, m->m_len - skip); + if (n > max_nsegs_1mbuf) + max_nsegs_1mbuf = n; + nsegs += n; + len -= m->m_len - skip; + skip = 0; + m = m->m_next; + } + *max_nsegs_1mbufp = max_nsegs_1mbuf; + return (nsegs); +} + +static void +write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, + unsigned int immdlen, unsigned int plen, unsigned int expn, + unsigned int pdus, uint8_t credits, int shove, int imm_ivs) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + unsigned int len = plen + expn; + + txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | + V_FW_TLSTX_DATA_WR_COMPL(1) | + V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen)); + txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | + V_FW_TLSTX_DATA_WR_LEN16(credits)); + txwr->plen = htobe32(len); + txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | + V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); + txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) | + V_FW_TLSTX_DATA_WR_EXP(expn) | + V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) | + V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) | + V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4)); + txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size); + txwr->adjustedplen_pkd = htobe16( + V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); + txwr->expinplenmax_pkd = htobe16( + V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); + txwr->pdusinplenmax_pkd = htobe16( + V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp)); +} + +static void +write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, + struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus) +{ + struct tls_ofld_info *tls_ofld = &toep->tls; + int data_type, seglen; + + if (plen < tls_ofld->k_ctx.frag_size) + seglen = plen; + else + seglen = tls_ofld->k_ctx.frag_size; + data_type = tls_content_type(tls_hdr->type); + cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | + V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | + V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); + cpl->pld_len = htobe32(plen); + if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) + cpl->type_protover = htobe32( + V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); + cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | + V_SCMD_NUM_IVS(pdus)); + cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); + cpl->scmd1 = htobe64(tls_ofld->tx_seq_no); + tls_ofld->tx_seq_no += pdus; +} + +/* + * Similar to write_tx_sgl() except that it accepts an optional + * trailer buffer for IVs. + */ +static void +write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen, + void *iv_buffer, int iv_len, int nsegs, int n) +{ + struct mbuf *m; + struct ulptx_sgl *usgl = dst; + int i, j, rc; + struct sglist sg; + struct sglist_seg segs[n]; + + KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); + + sglist_init(&sg, n, segs); + usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | + V_ULPTX_NSGE(nsegs)); + + for (m = start; skip >= m->m_len; m = m->m_next) + skip -= m->m_len; + + i = -1; + for (m = start; plen > 0; m = m->m_next) { + rc = sglist_append(&sg, mtod(m, char *) + skip, + m->m_len - skip); + if (__predict_false(rc != 0)) + panic("%s: sglist_append %d", __func__, rc); + plen -= m->m_len - skip; + skip = 0; + + for (j = 0; j < sg.sg_nseg; i++, j++) { + if (i < 0) { + usgl->len0 = htobe32(segs[j].ss_len); + usgl->addr0 = htobe64(segs[j].ss_paddr); + } else { + usgl->sge[i / 2].len[i & 1] = + htobe32(segs[j].ss_len); + usgl->sge[i / 2].addr[i & 1] = + htobe64(segs[j].ss_paddr); + } +#ifdef INVARIANTS + nsegs--; +#endif + } + sglist_reset(&sg); + } + if (iv_buffer != NULL) { + rc = sglist_append(&sg, iv_buffer, iv_len); + if (__predict_false(rc != 0)) + panic("%s: sglist_append %d", __func__, rc); + + for (j = 0; j < sg.sg_nseg; i++, j++) { + if (i < 0) { + usgl->len0 = htobe32(segs[j].ss_len); + usgl->addr0 = htobe64(segs[j].ss_paddr); + } else { + usgl->sge[i / 2].len[i & 1] = + htobe32(segs[j].ss_len); + usgl->sge[i / 2].addr[i & 1] = + htobe64(segs[j].ss_paddr); + } +#ifdef INVARIANTS + nsegs--; +#endif + } + } + if (i & 1) + usgl->sge[i / 2].len[1] = htobe32(0); + KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p", + __func__, nsegs, start, iv_buffer)); +} + +#if 0 +#include +#include +#include + +static void +dump_payload(struct adapter *sc, const void *dst, int sgl_nsegs) +{ + const struct ulptx_sgl *usgl; + uint64_t addr; + uint32_t len; + int i; + + usgl = dst; + device_printf(sc->dev, "payload:\n"); + for (i = 0; i < sgl_nsegs; i++) { + if (i == 0) { + addr = usgl->addr0; + len = usgl->len0; + } else { + addr = usgl->sge[(i - 1) / 2].addr[(i - 1) & 1]; + len = usgl->sge[(i - 1) / 2].len[(i - 1) & 1]; + } + addr = be64toh(addr); + len = be32toh(len); + printf("SGL[%d]: (%#lx:%#x)\n", i, addr, len); + hexdump((void *)PHYS_TO_DMAP(addr), len, NULL, HD_OMIT_CHARS | + HD_OMIT_COUNT); + } +} +#endif + +/* + * Similar to t4_push_frames() but handles TLS sockets when TLS offload + * is enabled. Rather than transmitting bulk data, the socket buffer + * contains TLS records. The work request requires a full TLS record, + * so batch mbufs up until a full TLS record is seen. This requires + * reading the TLS header out of the start of each record to determine + * its length. + */ +void +t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop) +{ + struct tls_hdr thdr; + struct mbuf *sndptr; + struct fw_tlstx_data_wr *txwr; + struct cpl_tx_tls_sfo *cpl; + struct wrqe *wr; + u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len; + u_int expn_size, iv_len, pdus, sndptroff; + struct tls_ofld_info *tls_ofld = &toep->tls; + struct inpcb *inp = toep->inp; + struct tcpcb *tp = intotcpcb(inp); + struct socket *so = inp->inp_socket; + struct sockbuf *sb = &so->so_snd; + int tls_size, tx_credits, shove, /* compl,*/ sowwakeup; + struct ofld_tx_sdesc *txsd; + bool imm_ivs, imm_payload; + void *iv_buffer, *iv_dst, *buf; + + INP_WLOCK_ASSERT(inp); + KASSERT(toep->flags & TPF_FLOWC_WR_SENT, + ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); + + KASSERT(toep->ulp_mode == ULP_MODE_NONE || + toep->ulp_mode == ULP_MODE_TCPDDP || toep->ulp_mode == ULP_MODE_TLS, + ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); + KASSERT(tls_tx_key(toep), + ("%s: TX key not set for toep %p", __func__, toep)); + +#ifdef VERBOSE_TRACES + CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", + __func__, toep->tid, toep->flags, tp->t_flags); +#endif + if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) + return; + +#ifdef RATELIMIT + if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && + (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { + inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; + } +#endif + + /* + * This function doesn't resume by itself. Someone else must clear the + * flag and call this function. + */ + if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { + KASSERT(drop == 0, + ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); + return; + } + + txsd = &toep->txsd[toep->txsd_pidx]; + for (;;) { + tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); + space = max_imm_tls_space(tx_credits); + wr_len = sizeof(struct fw_tlstx_data_wr) + + sizeof(struct cpl_tx_tls_sfo) + key_size(toep); + if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) { +#ifdef VERBOSE_TRACES + CTR5(KTR_CXGBE, + "%s: tid %d tx_credits %d min_wr %d space %d", + __func__, toep->tid, tx_credits, wr_len + + CIPHER_BLOCK_SIZE + 1, space); +#endif + return; + } + + SOCKBUF_LOCK(sb); + sowwakeup = drop; + if (drop) { + sbdrop_locked(sb, drop); + MPASS(tls_ofld->sb_off >= drop); + tls_ofld->sb_off -= drop; + drop = 0; + } + + /* + * Send a FIN if requested, but only if there's no + * more data to send. + */ + if (sbavail(sb) == 0 && toep->flags & TPF_SEND_FIN) { + if (sowwakeup) + sowwakeup_locked(so); + else + SOCKBUF_UNLOCK(sb); + SOCKBUF_UNLOCK_ASSERT(sb); + t4_close_conn(sc, toep); + return; + } + + if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) { + /* + * A full TLS header is not yet queued, stop + * for now until more data is added to the + * socket buffer. + */ +#ifdef VERBOSE_TRACES + CTR4(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d", + __func__, toep->tid, sbavail(sb), tls_ofld->sb_off); +#endif + if (sowwakeup) + sowwakeup_locked(so); + else + SOCKBUF_UNLOCK(sb); + SOCKBUF_UNLOCK_ASSERT(sb); + return; + } + + /* Read the header of the next TLS record. */ + sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff); + MPASS(!IS_AIOTX_MBUF(sndptr)); + m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr); + tls_size = htons(thdr.length); + plen = TLS_HEADER_LENGTH + tls_size; + pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size); + iv_len = pdus * CIPHER_BLOCK_SIZE; + + if (sbavail(sb) < tls_ofld->sb_off + plen) { + /* + * The full TLS record is not yet queued, stop + * for now until more data is added to the + * socket buffer. + */ +#ifdef VERBOSE_TRACES + CTR5(KTR_CXGBE, + "%s: tid %d sbavail %d sb_off %d plen %d", + __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, + plen); +#endif + if (sowwakeup) + sowwakeup_locked(so); + else + SOCKBUF_UNLOCK(sb); + SOCKBUF_UNLOCK_ASSERT(sb); + return; + } + + /* Shove if there is no additional data pending. */ + shove = (sbavail(sb) == tls_ofld->sb_off + plen) && + !(tp->t_flags & TF_MORETOCOME); + + if (sb->sb_flags & SB_AUTOSIZE && + V_tcp_do_autosndbuf && + sb->sb_hiwat < V_tcp_autosndbuf_max && + sbused(sb) >= sb->sb_hiwat * 7 / 8) { + int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, + V_tcp_autosndbuf_max); + + if (!sbreserve_locked(sb, newsize, so, NULL)) + sb->sb_flags &= ~SB_AUTOSIZE; + else + sowwakeup = 1; /* room available */ + } + if (sowwakeup) + sowwakeup_locked(so); + else + SOCKBUF_UNLOCK(sb); + SOCKBUF_UNLOCK_ASSERT(sb); + + if (__predict_false(toep->flags & TPF_FIN_SENT)) + panic("%s: excess tx.", __func__); + + /* Determine whether to use immediate vs SGL. */ + imm_payload = false; + imm_ivs = false; + if (wr_len + iv_len <= space) { + imm_ivs = true; + wr_len += iv_len; + if (wr_len + tls_size <= space) { + wr_len += tls_size; + imm_payload = true; + } + } + + /* Allocate space for IVs if needed. */ + if (!imm_ivs) { + iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT); + if (iv_buffer == NULL) { + /* + * XXX: How to restart this? + */ + if (sowwakeup) + sowwakeup_locked(so); + else + SOCKBUF_UNLOCK(sb); + SOCKBUF_UNLOCK_ASSERT(sb); + CTR3(KTR_CXGBE, + "%s: tid %d failed to alloc IV space len %d", + __func__, toep->tid, iv_len); + return; + } + } else + iv_buffer = NULL; + + /* Determine size of SGL. */ + nsegs = 0; + max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ + if (!imm_payload) { + nsegs = count_mbuf_segs(sndptr, sndptroff + + TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf); + if (!imm_ivs) { + int n = sglist_count(iv_buffer, iv_len); + nsegs += n; + if (n > max_nsegs_1mbuf) + max_nsegs_1mbuf = n; + } + + /* Account for SGL in work request length. */ + wr_len += sizeof(struct ulptx_sgl) + + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; + } + + wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); + if (wr == NULL) { + /* XXX: how will we recover from this? */ + toep->flags |= TPF_TX_SUSPENDED; + return; + } + +#ifdef VERBOSE_TRACES + CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d", + __func__, toep->tid, thdr.type, tls_size, pdus); +#endif + txwr = wrtod(wr); + cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); + memset(txwr, 0, roundup2(wr_len, 16)); + credits = howmany(wr_len, 16); + expn_size = tls_expansion_size(toep, tls_size, 0, NULL); + write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0, + tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0); + write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus); + tls_copy_tx_key(toep, cpl + 1); + + /* Generate random IVs */ + buf = (char *)(cpl + 1) + key_size(toep); + if (imm_ivs) { + MPASS(iv_buffer == NULL); + iv_dst = buf; + buf = (char *)iv_dst + iv_len; + } else + iv_dst = iv_buffer; + arc4rand(iv_dst, iv_len, 0); + + if (imm_payload) { + m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH, + tls_size, buf); + } else { + write_tlstx_sgl(buf, sndptr, + sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer, + iv_len, nsegs, max_nsegs_1mbuf); + } + + KASSERT(toep->tx_credits >= credits, + ("%s: not enough credits", __func__)); + + toep->tx_credits -= credits; + + tp->snd_nxt += plen; + tp->snd_max += plen; + + SOCKBUF_LOCK(sb); + sbsndptr(sb, tls_ofld->sb_off, plen, &sndptroff); + tls_ofld->sb_off += plen; + SOCKBUF_UNLOCK(sb); + + toep->flags |= TPF_TX_DATA_SENT; + if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) + toep->flags |= TPF_TX_SUSPENDED; + + KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); + txsd->plen = plen; + txsd->tx_credits = credits; + txsd->iv_buffer = iv_buffer; + txsd++; + if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { + toep->txsd_pidx = 0; + txsd = &toep->txsd[0]; + } + toep->txsd_avail--; + + atomic_add_long(&toep->vi->pi->tx_tls_records, 1); + atomic_add_long(&toep->vi->pi->tx_tls_octets, plen); + +#if 0 + device_printf(sc->dev, "submitting TLS record %d:%#x\n", + thdr.type, tls_size); + hexdump(txwr, wr_len, NULL, HD_OMIT_CHARS | HD_OMIT_COUNT); + if (!imm_payload) + dump_payload(sc, buf, nsegs); +#endif + t4_l2t_send(sc, wr, toep->l2te); + } +} + +/* + * For TLS data we place received mbufs received via CPL_TLS_DATA into + * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is + * received, the completed PDUs are placed into the socket receive + * buffer. + * + * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. + */ +static int +do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) +{ + struct adapter *sc = iq->adapter; + const struct cpl_tls_data *cpl = mtod(m, const void *); + unsigned int tid = GET_TID(cpl); + struct toepcb *toep = lookup_tid(sc, tid); + struct inpcb *inp = toep->inp; + struct tcpcb *tp; + int len; + + /* XXX: Should this match do_rx_data instead? */ + KASSERT(!(toep->flags & TPF_SYNQE), + ("%s: toep %p claims to be a synq entry", __func__, toep)); + + KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); + + /* strip off CPL header */ + m_adj(m, sizeof(*cpl)); + len = m->m_pkthdr.len; + + atomic_add_long(&toep->vi->pi->rx_tls_octets, len); + + KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), + ("%s: payload length mismatch", __func__)); + + INP_WLOCK(inp); + if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { + CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", + __func__, tid, len, inp->inp_flags); + INP_WUNLOCK(inp); + m_freem(m); + return (0); + } + + /* Save TCP sequence number. */ + m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); + + if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { +#ifdef INVARIANTS + panic("Failed to queue TLS data packet"); +#else + printf("%s: Failed to queue TLS data packet\n", __func__); + INP_WUNLOCK(inp); + m_freem(m); + return (0); +#endif + } + + tp = intotcpcb(inp); + tp->t_rcvtime = ticks; + +#ifdef VERBOSE_TRACES + CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, + be32toh(cpl->seq)); +#endif + + INP_WUNLOCK(inp); + return (0); +} + +static int +do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) +{ + struct adapter *sc = iq->adapter; + const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); + struct tlsrx_hdr_pkt *tls_hdr_pkt; + unsigned int tid = GET_TID(cpl); + struct toepcb *toep = lookup_tid(sc, tid); + struct inpcb *inp = toep->inp; + struct tcpcb *tp; + struct socket *so; + struct sockbuf *sb; + struct mbuf *tls_data; + int len, pdu_length, pdu_overhead, sb_length; + + KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); + KASSERT(!(toep->flags & TPF_SYNQE), + ("%s: toep %p claims to be a synq entry", __func__, toep)); + + /* strip off CPL header */ + m_adj(m, sizeof(*cpl)); + len = m->m_pkthdr.len; + + atomic_add_long(&toep->vi->pi->rx_tls_records, 1); + + KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), + ("%s: payload length mismatch", __func__)); + + INP_WLOCK(inp); + if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { + CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", + __func__, tid, len, inp->inp_flags); + INP_WUNLOCK(inp); + m_freem(m); + return (0); + } + + pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); + + tp = intotcpcb(inp); + +#ifdef VERBOSE_TRACES + CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", + __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); +#endif + + tp->rcv_nxt += pdu_length; + if (tp->rcv_wnd < pdu_length) { + toep->tls.rcv_over += pdu_length - tp->rcv_wnd; + tp->rcv_wnd = 0; + } else + tp->rcv_wnd -= pdu_length; + + /* XXX: Not sure what to do about urgent data. */ + + /* + * The payload of this CPL is the TLS header followed by + * additional fields. + */ + KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), + ("%s: payload too small", __func__)); + tls_hdr_pkt = mtod(m, void *); + + /* + * Only the TLS header is sent to OpenSSL, so report errors by + * altering the record type. + */ + if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) + tls_hdr_pkt->type = CONTENT_TYPE_ERROR; + + /* Trim this CPL's mbuf to only include the TLS header. */ + KASSERT(m->m_len == len && m->m_next == NULL, + ("%s: CPL spans multiple mbufs", __func__)); + m->m_len = TLS_HEADER_LENGTH; + m->m_pkthdr.len = TLS_HEADER_LENGTH; + + tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); + if (tls_data != NULL) { + KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, + ("%s: sequence mismatch", __func__)); + + /* + * Update the TLS header length to be the length of + * the payload data. + */ + tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); + + m->m_next = tls_data; + m->m_pkthdr.len += tls_data->m_len; + } + + so = inp_inpcbtosocket(inp); + sb = &so->so_rcv; + SOCKBUF_LOCK(sb); + + if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { + CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", + __func__, tid, pdu_length); + m_freem(m); + SOCKBUF_UNLOCK(sb); + INP_WUNLOCK(inp); + + CURVNET_SET(toep->vnet); + INP_INFO_RLOCK(&V_tcbinfo); + INP_WLOCK(inp); + tp = tcp_drop(tp, ECONNRESET); + if (tp) + INP_WUNLOCK(inp); + INP_INFO_RUNLOCK(&V_tcbinfo); + CURVNET_RESTORE(); + + return (0); + } + + /* + * Not all of the bytes on the wire are included in the socket + * buffer (e.g. the MAC of the TLS record). However, those + * bytes are included in the TCP sequence space. To handle + * this, compute the delta for this TLS record in + * 'pdu_overhead' and treat those bytes as having already been + * "read" by the application for the purposes of expanding the + * window. The meat of the TLS record passed to the + * application ('sb_length') will still not be counted as + * "read" until userland actually reads the bytes. + * + * XXX: Some of the calculations below are probably still not + * really correct. + */ + sb_length = m->m_pkthdr.len; + pdu_overhead = pdu_length - sb_length; + toep->rx_credits += pdu_overhead; + tp->rcv_wnd += pdu_overhead; + tp->rcv_adv += pdu_overhead; + + /* receive buffer autosize */ + MPASS(toep->vnet == so->so_vnet); + CURVNET_SET(toep->vnet); + if (sb->sb_flags & SB_AUTOSIZE && + V_tcp_do_autorcvbuf && + sb->sb_hiwat < V_tcp_autorcvbuf_max && + sb_length > (sbspace(sb) / 8 * 7)) { + unsigned int hiwat = sb->sb_hiwat; + unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, + V_tcp_autorcvbuf_max); + + if (!sbreserve_locked(sb, newsize, so, NULL)) + sb->sb_flags &= ~SB_AUTOSIZE; + else + toep->rx_credits += newsize - hiwat; + } + + KASSERT(toep->sb_cc >= sbused(sb), + ("%s: sb %p has more data (%d) than last time (%d).", + __func__, sb, sbused(sb), toep->sb_cc)); + toep->rx_credits += toep->sb_cc - sbused(sb); + sbappendstream_locked(sb, m, 0); + toep->sb_cc = sbused(sb); +#ifdef VERBOSE_TRACES + CTR5(KTR_CXGBE, "%s: tid %u PDU overhead %d rx_credits %u rcv_wnd %u", + __func__, tid, pdu_overhead, toep->rx_credits, tp->rcv_wnd); +#endif + if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) { + int credits; + + credits = send_rx_credits(sc, toep, toep->rx_credits); + toep->rx_credits -= credits; + tp->rcv_wnd += credits; + tp->rcv_adv += credits; + } + + sorwakeup_locked(so); + SOCKBUF_UNLOCK_ASSERT(sb); + + INP_WUNLOCK(inp); + CURVNET_RESTORE(); + return (0); +} + +void +t4_tls_mod_load(void) +{ + + mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF); + t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); + t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); +} + +void +t4_tls_mod_unload(void) +{ + + t4_register_cpl_handler(CPL_TLS_DATA, NULL); + t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); + mtx_destroy(&tls_handshake_lock); +} +#endif /* TCP_OFFLOAD */ Index: sys/dev/cxgbe/tom/t4_tom.h =================================================================== --- sys/dev/cxgbe/tom/t4_tom.h +++ sys/dev/cxgbe/tom/t4_tom.h @@ -33,6 +33,7 @@ #ifndef __T4_TOM_H__ #define __T4_TOM_H__ #include +#include "tom/t4_tls.h" #define LISTEN_HASH_SIZE 32 @@ -71,6 +72,8 @@ TPF_SYNQE_TCPDDP = (1 << 10), /* ulp_mode TCPDDP in toepcb */ TPF_SYNQE_EXPANDED = (1 << 11), /* toepcb ready, tid context updated */ TPF_SYNQE_HAS_L2TE = (1 << 12), /* we've replied to PASS_ACCEPT_REQ */ + TPF_SYNQE_TLS = (1 << 13), /* ulp_mode TLS in toepcb */ + TPF_FORCE_CREDITS = (1 << 14), /* always send credits */ }; enum { @@ -83,9 +86,12 @@ DDP_DEAD = (1 << 6), /* toepcb is shutting down */ }; +struct sockopt; + struct ofld_tx_sdesc { uint32_t plen; /* payload length */ uint8_t tx_credits; /* firmware tx credits (unit is 16B) */ + void *iv_buffer; /* optional buffer holding IVs for TLS */ }; struct ppod_region { @@ -125,6 +131,9 @@ #define EXT_FLAG_AIOTX EXT_FLAG_VENDOR1 +#define IS_AIOTX_MBUF(m) \ + ((m)->m_flags & M_EXT && (m)->m_ext.ext_flags & EXT_FLAG_AIOTX) + struct ddp_buffer { struct pageset *ps; @@ -185,6 +194,7 @@ struct mbufq ulp_pdu_reclaimq; struct ddp_pcb ddp; + struct tls_ofld_info tls; TAILQ_HEAD(, kaiocb) aiotx_jobq; struct task aiotx_task; @@ -269,6 +279,8 @@ struct ppod_region pr; + vmem_t *key_map; + struct mtx clip_table_lock; struct clip_head clip_table; int clip_gen; @@ -309,6 +321,18 @@ return (m->m_pkthdr.PH_per.eight[0]); } +static inline int +is_tls_offload(struct toepcb *toep) +{ + return (toep->ulp_mode == ULP_MODE_TLS); +} + +static inline int +can_tls_offload(struct adapter *sc) +{ + return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); +} + /* t4_tom.c */ struct toepcb *alloc_toepcb(struct vi_info *, int, int, int); struct toepcb *hold_toepcb(struct toepcb *); @@ -327,7 +351,8 @@ uint64_t calc_opt0(struct socket *, struct vi_info *, struct l2t_entry *, int, int, int, int); uint64_t select_ntuple(struct vi_info *, struct l2t_entry *); -void set_tcpddp_ulp_mode(struct toepcb *); +int select_ulp_mode(struct socket *, struct adapter *); +void set_ulp_mode(struct toepcb *, int); int negative_advice(int); struct clip_entry *hold_lip(struct tom_data *, struct in6_addr *, struct clip_entry *); @@ -362,7 +387,10 @@ void send_abort_rpl(struct adapter *, struct sge_wrq *, int , int); void send_flowc_wr(struct toepcb *, struct flowc_tx_params *); void send_reset(struct adapter *, struct toepcb *, uint32_t); +int send_rx_credits(struct adapter *, struct toepcb *, int); +void send_rx_modulate(struct adapter *, struct toepcb *); void make_established(struct toepcb *, uint32_t, uint32_t, uint16_t); +int t4_close_conn(struct adapter *, struct toepcb *); void t4_rcvd(struct toedev *, struct tcpcb *); void t4_rcvd_locked(struct toedev *, struct tcpcb *); int t4_tod_output(struct toedev *, struct tcpcb *); @@ -389,7 +417,7 @@ int t4_soreceive_ddp(struct socket *, struct sockaddr **, struct uio *, struct mbuf **, struct mbuf **, int *); int t4_aio_queue_ddp(struct socket *, struct kaiocb *); -int t4_ddp_mod_load(void); +void t4_ddp_mod_load(void); void t4_ddp_mod_unload(void); void ddp_assert_empty(struct toepcb *); void ddp_init_toep(struct toepcb *); @@ -401,4 +429,18 @@ void handle_ddp_tcb_rpl(struct toepcb *, const struct cpl_set_tcb_rpl *); void insert_ddp_data(struct toepcb *, uint32_t); +/* t4_tls.c */ +int t4_ctloutput_tls(struct socket *, struct sockopt *); +void t4_push_tls_records(struct adapter *, struct toepcb *, int); +void t4_tls_mod_load(void); +void t4_tls_mod_unload(void); +void tls_establish(struct toepcb *); +void tls_free_kmap(struct tom_data *); +int tls_init_kmap(struct adapter *, struct tom_data *); +void tls_init_toep(struct toepcb *); +int tls_rx_key(struct toepcb *); +void tls_stop_handshake_timer(struct toepcb *); +int tls_tx_key(struct toepcb *); +void tls_uninit_toep(struct toepcb *); + #endif Index: sys/dev/cxgbe/tom/t4_tom.c =================================================================== --- sys/dev/cxgbe/tom/t4_tom.c +++ sys/dev/cxgbe/tom/t4_tom.c @@ -71,6 +71,7 @@ #include "common/t4_tcb.h" #include "tom/t4_tom_l2t.h" #include "tom/t4_tom.h" +#include "tom/t4_tls.h" static struct protosw toe_protosw; static struct pr_usrreqs toe_usrreqs; @@ -199,6 +200,7 @@ if (toep->ulp_mode == ULP_MODE_TCPDDP) ddp_uninit_toep(toep); + tls_uninit_toep(toep); free(toep, M_CXGBE); } @@ -619,12 +621,48 @@ return (htobe64(V_FILTER_TUPLE(ntuple))); } +static int +is_tls_sock(struct socket *so, struct adapter *sc) +{ + struct inpcb *inp = sotoinpcb(so); + int i, rc; + + /* XXX: Eventually add a SO_WANT_TLS socket option perhaps? */ + rc = 0; + ADAPTER_LOCK(sc); + for (i = 0; i < sc->tt.num_tls_rx_ports; i++) { + if (inp->inp_lport == htons(sc->tt.tls_rx_ports[i]) || + inp->inp_fport == htons(sc->tt.tls_rx_ports[i])) { + rc = 1; + break; + } + } + ADAPTER_UNLOCK(sc); + return (rc); +} + +int +select_ulp_mode(struct socket *so, struct adapter *sc) +{ + + if (can_tls_offload(sc) && is_tls_sock(so, sc)) + return (ULP_MODE_TLS); + else if (sc->tt.ddp && (so->so_options & SO_NO_DDP) == 0) + return (ULP_MODE_TCPDDP); + else + return (ULP_MODE_NONE); +} + void -set_tcpddp_ulp_mode(struct toepcb *toep) +set_ulp_mode(struct toepcb *toep, int ulp_mode) { - toep->ulp_mode = ULP_MODE_TCPDDP; - ddp_init_toep(toep); + CTR4(KTR_CXGBE, "%s: toep %p (tid %d) ulp_mode %d", + __func__, toep, toep->tid, ulp_mode); + toep->ulp_mode = ulp_mode; + tls_init_toep(toep); + if (toep->ulp_mode == ULP_MODE_TCPDDP) + ddp_init_toep(toep); } int @@ -959,6 +997,7 @@ KASSERT(td->lctx_count == 0, ("%s: lctx hash table is not empty.", __func__)); + tls_free_kmap(td); t4_free_ppod_region(&td->pr); destroy_clip_table(sc, td); @@ -1063,6 +1102,12 @@ /* CLIP table for IPv6 offload */ init_clip_table(sc, td); + if (sc->vres.key.size != 0) { + rc = tls_init_kmap(sc, td); + if (rc != 0) + goto done; + } + /* toedev ops */ tod = &td->tod; init_toedev(tod); @@ -1167,10 +1212,27 @@ return (t4_aio_queue_aiotx(so, job)); } +static int +t4_ctloutput_tom(struct socket *so, struct sockopt *sopt) +{ + + if (sopt->sopt_level != IPPROTO_TCP) + return (tcp_ctloutput(so, sopt)); + + switch (sopt->sopt_name) { + case TCP_TLSOM_SET_TLS_CONTEXT: + case TCP_TLSOM_GET_TLS_TOM: + case TCP_TLSOM_CLR_TLS_TOM: + case TCP_TLSOM_CLR_QUIES: + return (t4_ctloutput_tls(so, sopt)); + default: + return (tcp_ctloutput(so, sopt)); + } +} + static int t4_tom_mod_load(void) { - int rc; struct protosw *tcp_protosw, *tcp6_protosw; /* CPL handlers */ @@ -1178,9 +1240,8 @@ t4_init_listen_cpl_handlers(); t4_init_cpl_io_handlers(); - rc = t4_ddp_mod_load(); - if (rc != 0) - return (rc); + t4_ddp_mod_load(); + t4_tls_mod_load(); tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM); if (tcp_protosw == NULL) @@ -1188,6 +1249,7 @@ bcopy(tcp_protosw, &toe_protosw, sizeof(toe_protosw)); bcopy(tcp_protosw->pr_usrreqs, &toe_usrreqs, sizeof(toe_usrreqs)); toe_usrreqs.pru_aio_queue = t4_aio_queue_tom; + toe_protosw.pr_ctloutput = t4_ctloutput_tom; toe_protosw.pr_usrreqs = &toe_usrreqs; tcp6_protosw = pffindproto(PF_INET6, IPPROTO_TCP, SOCK_STREAM); @@ -1196,17 +1258,14 @@ bcopy(tcp6_protosw, &toe6_protosw, sizeof(toe6_protosw)); bcopy(tcp6_protosw->pr_usrreqs, &toe6_usrreqs, sizeof(toe6_usrreqs)); toe6_usrreqs.pru_aio_queue = t4_aio_queue_tom; + toe6_protosw.pr_ctloutput = t4_ctloutput_tom; toe6_protosw.pr_usrreqs = &toe6_usrreqs; TIMEOUT_TASK_INIT(taskqueue_thread, &clip_task, 0, t4_clip_task, NULL); ifaddr_evhandler = EVENTHANDLER_REGISTER(ifaddr_event, t4_tom_ifaddr_event, NULL, EVENTHANDLER_PRI_ANY); - rc = t4_register_uld(&tom_uld_info); - if (rc != 0) - t4_tom_mod_unload(); - - return (rc); + return (t4_register_uld(&tom_uld_info)); } static void @@ -1235,6 +1294,7 @@ taskqueue_cancel_timeout(taskqueue_thread, &clip_task, NULL); } + t4_tls_mod_unload(); t4_ddp_mod_unload(); t4_uninit_connect_cpl_handlers(); Index: sys/modules/cxgbe/tom/Makefile =================================================================== --- sys/modules/cxgbe/tom/Makefile +++ sys/modules/cxgbe/tom/Makefile @@ -16,6 +16,7 @@ SRCS+= t4_cpl_io.c SRCS+= t4_ddp.c SRCS+= t4_listen.c +SRCS+= t4_tls.c SRCS+= t4_tom.c SRCS+= t4_tom_l2t.c