Index: share/man/man4/tcp.4 =================================================================== --- share/man/man4/tcp.4 +++ share/man/man4/tcp.4 @@ -319,14 +319,11 @@ The payload of this control message is a single byte holding the desired TLS record type. .Pp -Data read from this socket will still be encrypted and must be parsed by -a TLS-aware consumer. -.Pp -At present, only a single key may be set on a socket. +At present, only a single transmit key may be set on a socket. As such, users of this option must disable rekeying. .It Dv TCP_TXTLS_MODE -The integer argument can be used to get or set the current TLS mode of a -socket. +The integer argument can be used to get or set the current TLS transmit mode +of a socket. Setting the mode can only used to toggle between software and NIC TLS after TLS has been initially enabled via the .Dv TCP_TXTLS_ENABLE @@ -342,6 +339,33 @@ .It Dv TCP_TLS_MODE_IFNET TLS records are encrypted by the network interface card (NIC). .El +.It Dv TCP_RXTLS_ENABLE +Enable in-kernel TLS for data read from this socket. +The +.Vt struct tls_so_enable +argument defines the encryption and authentication algorithms and keys +used to decrypt the socket data. +.Pp +Each received TLS record must be read from the socket using +.Xr recvmsg 2 . +Each received TLS record will contain a +.Dv TLS_GET_RECORD +control message along with the decrypted payload. +The control message contains a +.Vt struct tls_get_record +which includes fields from the TLS record header. +If a corrupted TLS record is received, +recvmsg 2 +will fail with +.Dv EBADMSG . +.Pp +At present, only a single receive key may be set on a socket. +As such, users of this option must disable rekeying. +.It Dv TCP_RXTLS_MODE +The integer argument can be used to get the current TLS receive mode +of a socket. +The available modes are the same as for +.Dv TCP_TXTLS_MODE . .El .Pp The option level for the Index: sys/dev/cxgbe/tom/t4_tom.c =================================================================== --- sys/dev/cxgbe/tom/t4_tom.c +++ sys/dev/cxgbe/tom/t4_tom.c @@ -40,6 +40,9 @@ #include #include #include +#ifdef KERN_TLS +#include +#endif #include #include #include @@ -814,13 +817,16 @@ #ifdef KERN_TLS static int t4_alloc_tls_session(struct toedev *tod, struct tcpcb *tp, - struct ktls_session *tls) + struct ktls_session *tls, int direction) { struct toepcb *toep = tp->t_toe; INP_WLOCK_ASSERT(tp->t_inpcb); MPASS(tls != NULL); + if (direction != KTLS_TX) + return (EOPNOTSUPP); + return (tls_alloc_ktls(toep, tls)); } #endif Index: sys/kern/uipc_ktls.c =================================================================== --- sys/kern/uipc_ktls.c +++ sys/kern/uipc_ktls.c @@ -699,7 +699,7 @@ #ifdef TCP_OFFLOAD static int -ktls_try_toe(struct socket *so, struct ktls_session *tls) +ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction) { struct inpcb *inp; struct tcpcb *tp; @@ -725,7 +725,7 @@ return (EOPNOTSUPP); } - error = tcp_offload_alloc_tls_session(tp, tls); + error = tcp_offload_alloc_tls_session(tp, tls, direction); INP_WUNLOCK(inp); if (error == 0) { tls->mode = TCP_TLS_MODE_TOE; @@ -897,6 +897,60 @@ return (0); } +int +ktls_enable_rx(struct socket *so, struct tls_enable *en) +{ + struct ktls_session *tls; + int error; + + if (!ktls_offload_enable) + return (ENOTSUP); + + counter_u64_add(ktls_offload_enable_calls, 1); + + /* + * This should always be true since only the TCP socket option + * invokes this function. + */ + if (so->so_proto->pr_protocol != IPPROTO_TCP) + return (EINVAL); + + /* + * XXX: Don't overwrite existing sessions. We should permit + * this to support rekeying in the future. + */ + if (so->so_rcv.sb_tls_info != NULL) + return (EALREADY); + + if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) + return (ENOTSUP); + + error = ktls_create_session(so, en, &tls); + if (error) + return (error); + + /* TLS RX offload is only supported on TOE currently. */ +#ifdef TCP_OFFLOAD + error = ktls_try_toe(so, tls, KTLS_RX); +#else + error = EOPNOTSUPP; +#endif + + if (error) { + ktls_cleanup(tls); + return (error); + } + + /* Mark the socket as using TLS offload. */ + SOCKBUF_LOCK(&so->so_rcv); + so->so_rcv.sb_tls_info = tls; + SOCKBUF_UNLOCK(&so->so_rcv); + + counter_u64_add(ktls_offload_total, 1); + + return (0); +} + int ktls_enable_tx(struct socket *so, struct tls_enable *en) { @@ -935,7 +989,7 @@ /* Prefer TOE -> ifnet TLS -> software TLS. */ #ifdef TCP_OFFLOAD - error = ktls_try_toe(so, tls); + error = ktls_try_toe(so, tls, KTLS_TX); if (error) #endif error = ktls_try_ifnet(so, tls, false); @@ -966,6 +1020,25 @@ return (0); } +int +ktls_get_rx_mode(struct socket *so) +{ + struct ktls_session *tls; + struct inpcb *inp; + int mode; + + inp = so->so_pcb; + INP_WLOCK_ASSERT(inp); + SOCKBUF_LOCK(&so->so_rcv); + tls = so->so_rcv.sb_tls_info; + if (tls == NULL) + mode = TCP_TLS_MODE_NONE; + else + mode = tls->mode; + SOCKBUF_UNLOCK(&so->so_rcv); + return (mode); +} + int ktls_get_tx_mode(struct socket *so) { Index: sys/kern/uipc_socket.c =================================================================== --- sys/kern/uipc_socket.c +++ sys/kern/uipc_socket.c @@ -2372,12 +2372,34 @@ sb = &so->so_rcv; +#ifdef KERN_TLS + /* + * KTLS store TLS records as records with a control message to + * describe the framing. + * + * We check once here before acquiring locks to optimize the + * common case. + */ + if (sb->sb_tls_info != NULL) + return (soreceive_generic(so, psa, uio, mp0, controlp, + flagsp)); +#endif + /* Prevent other readers from entering the socket. */ error = sblock(sb, SBLOCKWAIT(flags)); if (error) return (error); SOCKBUF_LOCK(sb); +#ifdef KERN_TLS + if (sb->sb_tls_info != NULL) { + SOCKBUF_UNLOCK(sb); + sbunlock(sb); + return (soreceive_generic(so, psa, uio, mp0, controlp, + flagsp)); + } +#endif + /* Easy one, no space to copyout anything. */ if (uio->uio_resid == 0) { error = EINVAL; Index: sys/netinet/tcp.h =================================================================== --- sys/netinet/tcp.h +++ sys/netinet/tcp.h @@ -178,6 +178,8 @@ device */ #define TCP_TXTLS_ENABLE 39 /* TLS framing and encryption for transmit */ #define TCP_TXTLS_MODE 40 /* Transmit TLS mode */ +#define TCP_RXTLS_ENABLE 41 /* TLS framing and encryption for receive */ +#define TCP_RXTLS_MODE 42 /* Receive TLS mode */ #define TCP_CONGESTION 64 /* get/set congestion control algorithm */ #define TCP_CCALGOOPT 65 /* get/set cc algorithm specific options */ #define TCP_DELACK 72 /* socket option for delayed ack */ @@ -365,6 +367,7 @@ * TCP Control message types */ #define TLS_SET_RECORD_TYPE 1 +#define TLS_GET_RECORD 2 /* * TCP specific variables of interest for tp->t_stats stats(9) accounting. Index: sys/netinet/tcp_offload.h =================================================================== --- sys/netinet/tcp_offload.h +++ sys/netinet/tcp_offload.h @@ -46,7 +46,7 @@ void tcp_offload_rcvd(struct tcpcb *); void tcp_offload_ctloutput(struct tcpcb *, int, int); void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *); -int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *); +int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *, int); void tcp_offload_detach(struct tcpcb *); #endif Index: sys/netinet/tcp_offload.c =================================================================== --- sys/netinet/tcp_offload.c +++ sys/netinet/tcp_offload.c @@ -179,14 +179,15 @@ } int -tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls) +tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls, + int direction) { struct toedev *tod = tp->tod; KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); INP_WLOCK_ASSERT(tp->t_inpcb); - return (tod->tod_alloc_tls_session(tod, tp, tls)); + return (tod->tod_alloc_tls_session(tod, tp, tls, direction)); } void Index: sys/netinet/tcp_usrreq.c =================================================================== --- sys/netinet/tcp_usrreq.c +++ sys/netinet/tcp_usrreq.c @@ -2080,6 +2080,14 @@ error = ktls_set_tx_mode(so, ui); INP_WUNLOCK(inp); break; + case TCP_RXTLS_ENABLE: + INP_WUNLOCK(inp); + error = sooptcopyin(sopt, &tls, sizeof(tls), + sizeof(tls)); + if (error) + break; + error = ktls_enable_rx(so, &tls); + break; #endif case TCP_KEEPIDLE: @@ -2418,6 +2426,11 @@ INP_WUNLOCK(inp); error = sooptcopyout(sopt, &optval, sizeof(optval)); break; + case TCP_RXTLS_MODE: + optval = ktls_get_rx_mode(so); + INP_WUNLOCK(inp); + error = sooptcopyout(sopt, &optval, sizeof(optval)); + break; #endif default: INP_WUNLOCK(inp); Index: sys/netinet/toecore.h =================================================================== --- sys/netinet/toecore.h +++ sys/netinet/toecore.h @@ -112,7 +112,7 @@ /* Create a TLS session */ int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *, - struct ktls_session *); + struct ktls_session *, int); }; typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *); Index: sys/netinet/toecore.c =================================================================== --- sys/netinet/toecore.c +++ sys/netinet/toecore.c @@ -193,7 +193,7 @@ static int toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused, - struct ktls_session *tls __unused) + struct ktls_session *tls __unused, int direction __unused) { return (EINVAL); Index: sys/sys/ktls.h =================================================================== --- sys/sys/ktls.h +++ sys/sys/ktls.h @@ -98,7 +98,7 @@ #define TLS_MINOR_VER_TWO 3 /* 3, 3 */ #define TLS_MINOR_VER_THREE 4 /* 3, 4 */ -/* For TCP_TXTLS_ENABLE */ +/* For TCP_TXTLS_ENABLE and TCP_RXTLS_ENABLE. */ #ifdef _KERNEL struct tls_enable_v0 { const uint8_t *cipher_key; @@ -130,6 +130,17 @@ uint8_t rec_seq[8]; }; +/* Structure for TLS_GET_RECORD. */ +struct tls_get_record { + /* TLS record header. */ + uint8_t tls_type; + uint8_t tls_vmajor; + uint8_t tls_vminor; + uint16_t tls_length; +}; + +#ifdef _KERNEL + struct tls_session_params { uint8_t *cipher_key; uint8_t *auth_key; @@ -148,7 +159,9 @@ uint8_t flags; }; -#ifdef _KERNEL +/* Used in APIs to request RX vs TX sessions. */ +#define KTLS_TX 1 +#define KTLS_RX 2 #define KTLS_API_VERSION 6 @@ -192,6 +205,7 @@ int ktls_crypto_backend_register(struct ktls_crypto_backend *be); int ktls_crypto_backend_deregister(struct ktls_crypto_backend *be); +int ktls_enable_rx(struct socket *so, struct tls_enable *en); int ktls_enable_tx(struct socket *so, struct tls_enable *en); void ktls_destroy(struct ktls_session *tls); void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt, @@ -199,6 +213,7 @@ void ktls_seq(struct sockbuf *sb, struct mbuf *m); void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count); void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs); +int ktls_get_rx_mode(struct socket *so); int ktls_set_tx_mode(struct socket *so, int mode); int ktls_get_tx_mode(struct socket *so); int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls);