Index: share/man/man4/tcp.4 =================================================================== --- share/man/man4/tcp.4 +++ share/man/man4/tcp.4 @@ -319,14 +319,11 @@ The payload of this control message is a single byte holding the desired TLS record type. .Pp -Data read from this socket will still be encrypted and must be parsed by -a TLS-aware consumer. -.Pp -At present, only a single key may be set on a socket. +At present, only a single transmit key may be set on a socket. As such, users of this option must disable rekeying. .It Dv TCP_TXTLS_MODE -The integer argument can be used to get or set the current TLS mode of a -socket. +The integer argument can be used to get or set the current TLS transmit mode +of a socket. Setting the mode can only used to toggle between software and NIC TLS after TLS has been initially enabled via the .Dv TCP_TXTLS_ENABLE @@ -342,6 +339,33 @@ .It Dv TCP_TLS_MODE_IFNET TLS records are encrypted by the network interface card (NIC). .El +.It Dv TCP_RXTLS_ENABLE +Enable in-kernel TLS for data read from this socket. +The +.Vt struct tls_so_enable +argument defines the encryption and authentication algorithms and keys +used to decrypt the socket data. +.Pp +Each received TLS record must be read from the socket using +.Xr recvmsg 2 . +Each received TLS record will contain a +.Dv TLS_GET_RECORD +control message along with the decrypted payload. +The control message contains a +.Vt struct tls_get_record +which includes fields from the TLS record header. +If a corrupted TLS record is received, +recvmsg 2 +will fail with +.Dv EBADMSG . +.Pp +At present, only a single receive key may be set on a socket. +As such, users of this option must disable rekeying. +.It Dv TCP_RXTLS_MODE +The integer argument can be used to get the current TLS receive mode +of a socket. +The available modes are the same as for +.Dv TCP_TXTLS_MODE . .El .Pp The option level for the Index: sys/dev/cxgbe/tom/t4_tls.c =================================================================== --- sys/dev/cxgbe/tom/t4_tls.c +++ sys/dev/cxgbe/tom/t4_tls.c @@ -379,7 +379,7 @@ int proto_ver = kctx->proto_ver; kwr->u.rxhdr.flitcnt_hmacctrl = - ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; + ((kctx->rx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; kwr->u.rxhdr.protover_ciphmode = V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) | @@ -408,7 +408,7 @@ (IPAD_SIZE + OPAD_SIZE)); } else { memcpy(kwr->keys.edkey, kctx->rx.key, - (kctx->tx_key_info_size - SALT_SIZE)); + (kctx->rx_key_info_size - SALT_SIZE)); memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE); } } @@ -674,6 +674,13 @@ if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) || (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) { + + /* + * XXX: The userland library sets tx_key_info_size, not + * rx_key_info_size. + */ + k_ctx->rx_key_info_size = k_ctx->tx_key_info_size; + error = tls_program_key_id(toep, k_ctx); if (error) { /* XXX: Only clear quiesce for KEY_WRITE_RX? */ @@ -866,31 +873,36 @@ #ifdef KERN_TLS static void -init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx) +init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx, + bool transmit) { struct auth_hash *axf; - u_int mac_key_size; - char *hash; + u_int key_info_size, mac_key_size; + char *hash, *key; - k_ctx->l_p_key = V_KEY_GET_LOC(KEY_WRITE_TX); - if (tls->params.tls_vminor == TLS_MINOR_VER_ONE) - k_ctx->proto_ver = SCMD_PROTO_VERSION_TLS_1_1; - else - k_ctx->proto_ver = SCMD_PROTO_VERSION_TLS_1_2; + k_ctx->l_p_key = V_KEY_GET_LOC(transmit ? KEY_WRITE_TX : KEY_WRITE_RX); + k_ctx->proto_ver = tls->params.tls_vmajor << 8 | tls->params.tls_vminor; k_ctx->cipher_secret_size = tls->params.cipher_key_len; - k_ctx->tx_key_info_size = sizeof(struct tx_keyctx_hdr) + + key_info_size = sizeof(struct tx_keyctx_hdr) + k_ctx->cipher_secret_size; - memcpy(k_ctx->tx.key, tls->params.cipher_key, - tls->params.cipher_key_len); - hash = k_ctx->tx.key + tls->params.cipher_key_len; + if (transmit) + key = k_ctx->tx.key; + else + key = k_ctx->rx.key; + memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len); + hash = key + tls->params.cipher_key_len; if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { k_ctx->state.auth_mode = SCMD_AUTH_MODE_GHASH; k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_GCM; k_ctx->iv_size = 4; k_ctx->mac_first = 0; k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NOP; - k_ctx->tx_key_info_size += GMAC_BLOCK_LEN; - memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE); + key_info_size += GMAC_BLOCK_LEN; + k_ctx->mac_secret_size = 0; + if (transmit) + memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE); + else + memcpy(k_ctx->rx.salt, tls->params.iv, SALT_SIZE); t4_init_gmac_hash(tls->params.cipher_key, tls->params.cipher_key_len * 8, hash); } else { @@ -917,29 +929,38 @@ k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ k_ctx->mac_first = 1; k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; - k_ctx->tx_key_info_size += roundup2(mac_key_size, 16) * 2; + key_info_size += roundup2(mac_key_size, 16) * 2; k_ctx->mac_secret_size = mac_key_size; t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, tls->params.auth_key_len * 8, hash); } + if (transmit) + k_ctx->tx_key_info_size = key_info_size; + else + k_ctx->rx_key_info_size = key_info_size; k_ctx->frag_size = tls->params.max_frame_len; k_ctx->iv_ctrl = 1; } int -tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls) +tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, bool transmit) { + struct adapter *sc = td_adapter(toep->td); struct tls_key_context *k_ctx; - int error; + int error, key_offset; if (toep->tls.mode == TLS_MODE_TLSOM) return (EINVAL); if (!can_tls_offload(td_adapter(toep->td))) return (EINVAL); switch (ulp_mode(toep)) { + case ULP_MODE_TLS: + break; case ULP_MODE_NONE: case ULP_MODE_TCPDDP: + if (!transmit) + return (EINVAL); break; default: return (EINVAL); @@ -987,47 +1008,80 @@ tls->params.tls_vminor > TLS_MINOR_VER_TWO) return (EPROTONOSUPPORT); + /* Bail if we already have a key. */ + if (transmit) { + if (toep->tls.tx_key_addr != -1) + return (EOPNOTSUPP); + } else { + if (toep->tls.rx_key_addr != -1) + return (EOPNOTSUPP); + } + /* * XXX: This assumes no key renegotation. If KTLS ever supports * that we will want to allocate TLS sessions dynamically rather * than as a static member of toep. */ k_ctx = &toep->tls.k_ctx; - init_ktls_key_context(tls, k_ctx); - - toep->tls.scmd0.seqno_numivs = - (V_SCMD_SEQ_NO_CTRL(3) | - V_SCMD_PROTO_VERSION(k_ctx->proto_ver) | - V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | - V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | - V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | - V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | - V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | - V_SCMD_IV_SIZE(k_ctx->iv_size)); - - toep->tls.scmd0.ivgen_hdrlen = - (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | - V_SCMD_KEY_CTX_INLINE(0) | - V_SCMD_TLS_FRAG_ENABLE(1)); - - if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) - toep->tls.iv_len = 8; - else - toep->tls.iv_len = AES_BLOCK_LEN; - - toep->tls.mac_length = k_ctx->mac_secret_size; - - toep->tls.tx_key_addr = -1; + init_ktls_key_context(tls, k_ctx, transmit); error = tls_program_key_id(toep, k_ctx); if (error) return (error); - toep->tls.fcplenmax = get_tp_plen_max(&toep->tls); - toep->tls.expn_per_ulp = tls->params.tls_hlen + tls->params.tls_tlen; - toep->tls.pdus_per_ulp = 1; - toep->tls.adjusted_plen = toep->tls.expn_per_ulp + - toep->tls.k_ctx.frag_size; + if (transmit) { + toep->tls.scmd0.seqno_numivs = + (V_SCMD_SEQ_NO_CTRL(3) | + V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | + V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | + V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | + V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | + V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | + V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | + V_SCMD_IV_SIZE(k_ctx->iv_size)); + + toep->tls.scmd0.ivgen_hdrlen = + (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | + V_SCMD_KEY_CTX_INLINE(0) | + V_SCMD_TLS_FRAG_ENABLE(1)); + + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) + toep->tls.iv_len = 8; + else + toep->tls.iv_len = AES_BLOCK_LEN; + + toep->tls.mac_length = k_ctx->mac_secret_size; + + toep->tls.fcplenmax = get_tp_plen_max(&toep->tls); + toep->tls.expn_per_ulp = tls->params.tls_hlen + + tls->params.tls_tlen; + toep->tls.pdus_per_ulp = 1; + toep->tls.adjusted_plen = toep->tls.expn_per_ulp + + toep->tls.k_ctx.frag_size; + } else { + /* Stop timer on handshake completion */ + tls_stop_handshake_timer(toep); + + toep->flags &= ~TPF_FORCE_CREDITS; + + /* + * RX key tags are an index into the key portion of MA + * memory stored as an offset from the base address in + * units of 64 bytes. + */ + key_offset = toep->tls.rx_key_addr - sc->vres.key.start; + t4_set_tls_keyid(toep, key_offset / 64); + t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, + V_TCB_ULP_RAW(M_TCB_ULP_RAW), + V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | + V_TF_TLS_CONTROL(1) | + V_TF_TLS_ACTIVE(1) | + V_TF_TLS_ENABLE(1)))); + t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, + V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), + V_TCB_TLS_SEQ(0)); + t4_clear_rx_quiesce(toep); + } toep->tls.mode = TLS_MODE_KTLS; @@ -1669,7 +1723,7 @@ ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); KASSERT(ulp_mode(toep) == ULP_MODE_NONE || - ulp_mode(toep) == ULP_MODE_TCPDDP, + ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); KASSERT(tls_tx_key(toep), ("%s: TX key not set for toep %p", __func__, toep)); @@ -1952,6 +2006,10 @@ struct socket *so; struct sockbuf *sb; struct mbuf *tls_data; +#ifdef KERN_TLS + struct tls_get_record *tgr; + struct mbuf *control; +#endif int len, pdu_length, rx_credits; KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); @@ -1978,6 +2036,7 @@ pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); + so = inp_inpcbtosocket(inp); tp = intotcpcb(inp); #ifdef VERBOSE_TRACES @@ -2002,35 +2061,94 @@ ("%s: payload too small", __func__)); tls_hdr_pkt = mtod(m, void *); - /* - * Only the TLS header is sent to OpenSSL, so report errors by - * altering the record type. - */ - if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) - tls_hdr_pkt->type = CONTENT_TYPE_ERROR; - - /* Trim this CPL's mbuf to only include the TLS header. */ - KASSERT(m->m_len == len && m->m_next == NULL, - ("%s: CPL spans multiple mbufs", __func__)); - m->m_len = TLS_HEADER_LENGTH; - m->m_pkthdr.len = TLS_HEADER_LENGTH; - tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); if (tls_data != NULL) { KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, ("%s: sequence mismatch", __func__)); + } +#ifdef KERN_TLS + if (toep->tls.mode == TLS_MODE_KTLS) { + /* Report decryption errors as EBADMSG. */ + if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != + 0) { + m_freem(m); + m_freem(tls_data); + + CURVNET_SET(toep->vnet); + so->so_error = EBADMSG; + sorwakeup(so); + + INP_WUNLOCK(inp); + CURVNET_RESTORE(); + + return (0); + } + + /* Allocate the control message mbuf. */ + control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD, + IPPROTO_TCP); + if (control == NULL) { + m_freem(m); + m_freem(tls_data); + + CURVNET_SET(toep->vnet); + so->so_error = ENOBUFS; + sorwakeup(so); + + INP_WUNLOCK(inp); + CURVNET_RESTORE(); + + return (0); + } + + tgr = (struct tls_get_record *) + CMSG_DATA(mtod(control, struct cmsghdr *)); + tgr->tls_type = tls_hdr_pkt->type; + tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8; + tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff; + + m_freem(m); + + if (tls_data != NULL) { + m_last(tls_data)->m_flags |= M_EOR; + tgr->tls_length = htobe16(tls_data->m_pkthdr.len); + } else + tgr->tls_length = 0; + m = tls_data; + } else +#endif + { /* - * Update the TLS header length to be the length of - * the payload data. + * Only the TLS header is sent to OpenSSL, so report + * errors by altering the record type. */ - tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); + if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != + 0) + tls_hdr_pkt->type = CONTENT_TYPE_ERROR; - m->m_next = tls_data; - m->m_pkthdr.len += tls_data->m_len; + /* Trim this CPL's mbuf to only include the TLS header. */ + KASSERT(m->m_len == len && m->m_next == NULL, + ("%s: CPL spans multiple mbufs", __func__)); + m->m_len = TLS_HEADER_LENGTH; + m->m_pkthdr.len = TLS_HEADER_LENGTH; + + if (tls_data != NULL) { + /* + * Update the TLS header length to be the length of + * the payload data. + */ + tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); + + m->m_next = tls_data; + m->m_pkthdr.len += tls_data->m_len; + } + +#ifdef KERN_TLS + control = NULL; +#endif } - so = inp_inpcbtosocket(inp); sb = &so->so_rcv; SOCKBUF_LOCK(sb); @@ -2040,6 +2158,9 @@ CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", __func__, tid, pdu_length); m_freem(m); +#ifdef KERN_TLS + m_freem(control); +#endif SOCKBUF_UNLOCK(sb); INP_WUNLOCK(inp); @@ -2076,7 +2197,12 @@ sb->sb_flags &= ~SB_AUTOSIZE; } - sbappendstream_locked(sb, m, 0); +#ifdef KERN_TLS + if (control != NULL) + sbappendcontrol_locked(sb, m, control); + else +#endif + sbappendstream_locked(sb, m, 0); rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", Index: sys/dev/cxgbe/tom/t4_tom.h =================================================================== --- sys/dev/cxgbe/tom/t4_tom.h +++ sys/dev/cxgbe/tom/t4_tom.h @@ -450,6 +450,6 @@ void tls_stop_handshake_timer(struct toepcb *); int tls_tx_key(struct toepcb *); void tls_uninit_toep(struct toepcb *); -int tls_alloc_ktls(struct toepcb *, struct ktls_session *); +int tls_alloc_ktls(struct toepcb *, struct ktls_session *, bool); #endif Index: sys/dev/cxgbe/tom/t4_tom.c =================================================================== --- sys/dev/cxgbe/tom/t4_tom.c +++ sys/dev/cxgbe/tom/t4_tom.c @@ -810,14 +810,14 @@ #ifdef KERN_TLS static int t4_alloc_tls_session(struct toedev *tod, struct tcpcb *tp, - struct ktls_session *tls) + struct ktls_session *tls, bool transmit) { struct toepcb *toep = tp->t_toe; INP_WLOCK_ASSERT(tp->t_inpcb); MPASS(tls != NULL); - return (tls_alloc_ktls(toep, tls)); + return (tls_alloc_ktls(toep, tls, transmit)); } #endif @@ -1076,7 +1076,9 @@ struct inpcb *inp = sotoinpcb(so); int i, rc; - /* XXX: Eventually add a SO_WANT_TLS socket option perhaps? */ + if (so_options_get(so) & SO_WANT_KTLS) + return (1); + rc = 0; ADAPTER_LOCK(sc); for (i = 0; i < sc->tt.num_tls_rx_ports; i++) { Index: sys/kern/uipc_ktls.c =================================================================== --- sys/kern/uipc_ktls.c +++ sys/kern/uipc_ktls.c @@ -694,7 +694,7 @@ #ifdef TCP_OFFLOAD static int -ktls_try_toe(struct socket *so, struct ktls_session *tls) +ktls_try_toe(struct socket *so, struct ktls_session *tls, bool transmit) { struct inpcb *inp; struct tcpcb *tp; @@ -720,7 +720,7 @@ return (EOPNOTSUPP); } - error = tcp_offload_alloc_tls_session(tp, tls); + error = tcp_offload_alloc_tls_session(tp, tls, transmit); INP_WUNLOCK(inp); if (error == 0) { tls->mode = TCP_TLS_MODE_TOE; @@ -891,6 +891,60 @@ return (0); } +int +ktls_enable_rx(struct socket *so, struct tls_enable *en) +{ + struct ktls_session *tls; + int error; + + if (!ktls_offload_enable) + return (ENOTSUP); + + counter_u64_add(ktls_offload_enable_calls, 1); + + /* + * This should always be true since only the TCP socket option + * invokes this function. + */ + if (so->so_proto->pr_protocol != IPPROTO_TCP) + return (EINVAL); + + /* + * XXX: Don't overwrite existing sessions. We should permit + * this to support rekeying in the future. + */ + if (so->so_rcv.sb_tls_info != NULL) + return (EALREADY); + + if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) + return (ENOTSUP); + + error = ktls_create_session(so, en, &tls); + if (error) + return (error); + + /* TLS RX offload is only supported on TOE currently. */ +#ifdef TCP_OFFLOAD + error = ktls_try_toe(so, tls, false); +#else + error = EOPNOTSUPP; +#endif + + if (error) { + ktls_cleanup(tls); + return (error); + } + + /* Mark the socket as using TLS offload. */ + SOCKBUF_LOCK(&so->so_rcv); + so->so_rcv.sb_tls_info = tls; + SOCKBUF_UNLOCK(&so->so_rcv); + + counter_u64_add(ktls_offload_total, 1); + + return (0); +} + int ktls_enable_tx(struct socket *so, struct tls_enable *en) { @@ -929,7 +983,7 @@ /* Prefer TOE -> ifnet TLS -> software TLS. */ #ifdef TCP_OFFLOAD - error = ktls_try_toe(so, tls); + error = ktls_try_toe(so, tls, true); if (error) #endif error = ktls_try_ifnet(so, tls, false); @@ -959,6 +1013,25 @@ return (0); } +int +ktls_get_rx_mode(struct socket *so) +{ + struct ktls_session *tls; + struct inpcb *inp; + int mode; + + inp = so->so_pcb; + INP_WLOCK_ASSERT(inp); + SOCKBUF_LOCK(&so->so_rcv); + tls = so->so_rcv.sb_tls_info; + if (tls == NULL) + mode = TCP_TLS_MODE_NONE; + else + mode = tls->mode; + SOCKBUF_UNLOCK(&so->so_rcv); + return (mode); +} + int ktls_get_tx_mode(struct socket *so) { Index: sys/kern/uipc_socket.c =================================================================== --- sys/kern/uipc_socket.c +++ sys/kern/uipc_socket.c @@ -2275,12 +2275,34 @@ sb = &so->so_rcv; +#ifdef KERN_TLS + /* + * KTLS store TLS records as records with a control message to + * describe the framing. + * + * We check once here before acquiring locks to optimize the + * common case. + */ + if (sb->sb_tls_info != NULL) + return (soreceive_generic(so, psa, uio, mp0, controlp, + flagsp)); +#endif + /* Prevent other readers from entering the socket. */ error = sblock(sb, SBLOCKWAIT(flags)); if (error) return (error); SOCKBUF_LOCK(sb); +#ifdef KERN_TLS + if (sb->sb_tls_info != NULL) { + SOCKBUF_UNLOCK(sb); + sbunlock(sb); + return (soreceive_generic(so, psa, uio, mp0, controlp, + flagsp)); + } +#endif + /* Easy one, no space to copyout anything. */ if (uio->uio_resid == 0) { error = EINVAL; @@ -2883,6 +2905,7 @@ case SO_NOSIGPIPE: case SO_NO_DDP: case SO_NO_OFFLOAD: + case SO_WANT_KTLS: error = sooptcopyin(sopt, &optval, sizeof optval, sizeof optval); if (error) @@ -3102,6 +3125,9 @@ case SO_TIMESTAMP: case SO_BINTIME: case SO_NOSIGPIPE: + case SO_NO_DDP: + case SO_NO_OFFLOAD: + case SO_WANT_KTLS: optval = so->so_options & sopt->sopt_name; integer: error = sooptcopyout(sopt, &optval, sizeof optval); Index: sys/netinet/tcp.h =================================================================== --- sys/netinet/tcp.h +++ sys/netinet/tcp.h @@ -177,6 +177,8 @@ device */ #define TCP_TXTLS_ENABLE 39 /* TLS framing and encryption for transmit */ #define TCP_TXTLS_MODE 40 /* Transmit TLS mode */ +#define TCP_RXTLS_ENABLE 41 /* TLS framing and encryption for receive */ +#define TCP_RXTLS_MODE 42 /* Receive TLS mode */ #define TCP_CONGESTION 64 /* get/set congestion control algorithm */ #define TCP_CCALGOOPT 65 /* get/set cc algorithm specific options */ #define TCP_DELACK 72 /* socket option for delayed ack */ @@ -364,6 +366,7 @@ * TCP Control message types */ #define TLS_SET_RECORD_TYPE 1 +#define TLS_GET_RECORD 2 /* * TCP specific variables of interest for tp->t_stats stats(9) accounting. Index: sys/netinet/tcp_offload.h =================================================================== --- sys/netinet/tcp_offload.h +++ sys/netinet/tcp_offload.h @@ -46,7 +46,7 @@ void tcp_offload_rcvd(struct tcpcb *); void tcp_offload_ctloutput(struct tcpcb *, int, int); void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *); -int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *); +int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *, bool); void tcp_offload_detach(struct tcpcb *); #endif Index: sys/netinet/tcp_offload.c =================================================================== --- sys/netinet/tcp_offload.c +++ sys/netinet/tcp_offload.c @@ -179,14 +179,15 @@ } int -tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls) +tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls, + bool transmit) { struct toedev *tod = tp->tod; KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); INP_WLOCK_ASSERT(tp->t_inpcb); - return (tod->tod_alloc_tls_session(tod, tp, tls)); + return (tod->tod_alloc_tls_session(tod, tp, tls, transmit)); } void Index: sys/netinet/tcp_usrreq.c =================================================================== --- sys/netinet/tcp_usrreq.c +++ sys/netinet/tcp_usrreq.c @@ -2037,6 +2037,14 @@ error = ktls_set_tx_mode(so, ui); INP_WUNLOCK(inp); break; + case TCP_RXTLS_ENABLE: + INP_WUNLOCK(inp); + error = sooptcopyin(sopt, &tls, sizeof(tls), + sizeof(tls)); + if (error) + break; + error = ktls_enable_rx(so, &tls); + break; #endif case TCP_KEEPIDLE: @@ -2375,6 +2383,11 @@ INP_WUNLOCK(inp); error = sooptcopyout(sopt, &optval, sizeof(optval)); break; + case TCP_RXTLS_MODE: + optval = ktls_get_rx_mode(so); + INP_WUNLOCK(inp); + error = sooptcopyout(sopt, &optval, sizeof(optval)); + break; #endif default: INP_WUNLOCK(inp); Index: sys/netinet/toecore.h =================================================================== --- sys/netinet/toecore.h +++ sys/netinet/toecore.h @@ -112,7 +112,7 @@ /* Create a TLS session */ int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *, - struct ktls_session *); + struct ktls_session *, bool); }; typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *); Index: sys/netinet/toecore.c =================================================================== --- sys/netinet/toecore.c +++ sys/netinet/toecore.c @@ -193,7 +193,7 @@ static int toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused, - struct ktls_session *tls __unused) + struct ktls_session *tls __unused, bool transmit __unused) { return (EINVAL); Index: sys/sys/ktls.h =================================================================== --- sys/sys/ktls.h +++ sys/sys/ktls.h @@ -98,7 +98,7 @@ #define TLS_MINOR_VER_TWO 3 /* 3, 3 */ #define TLS_MINOR_VER_THREE 4 /* 3, 4 */ -/* For TCP_TXTLS_ENABLE */ +/* For TCP_TXTLS_ENABLE and TCP_RXTLS_ENABLE. */ struct tls_enable { const uint8_t *cipher_key; const uint8_t *iv; /* Implicit IV. */ @@ -113,6 +113,17 @@ uint8_t tls_vminor; }; +/* Structure for TLS_GET_RECORD. */ +struct tls_get_record { + /* TLS record header. */ + uint8_t tls_type; + uint8_t tls_vmajor; + uint8_t tls_vminor; + uint16_t tls_length; +}; + +#ifdef _KERNEL + struct tls_session_params { uint8_t *cipher_key; uint8_t *auth_key; @@ -131,8 +142,6 @@ uint8_t flags; }; -#ifdef _KERNEL - #define KTLS_API_VERSION 6 struct iovec; @@ -175,6 +184,7 @@ int ktls_crypto_backend_register(struct ktls_crypto_backend *be); int ktls_crypto_backend_deregister(struct ktls_crypto_backend *be); +int ktls_enable_rx(struct socket *so, struct tls_enable *en); int ktls_enable_tx(struct socket *so, struct tls_enable *en); void ktls_destroy(struct ktls_session *tls); int ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt, @@ -182,6 +192,7 @@ void ktls_seq(struct sockbuf *sb, struct mbuf *m); void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count); void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs); +int ktls_get_rx_mode(struct socket *so); int ktls_set_tx_mode(struct socket *so, int mode); int ktls_get_tx_mode(struct socket *so); int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls); Index: sys/sys/socket.h =================================================================== --- sys/sys/socket.h +++ sys/sys/socket.h @@ -147,6 +147,9 @@ #define SO_NO_OFFLOAD 0x00004000 /* socket cannot be offloaded */ #define SO_NO_DDP 0x00008000 /* disable direct data placement */ #define SO_REUSEPORT_LB 0x00010000 /* reuse with load balancing */ +#if __BSD_VISIBLE +#define SO_WANT_KTLS 0x00020000 /* kernel TLS might be requested */ +#endif /* * Additional options, not kept in so_options.