Differential D33007 Diff 100017 sys/kern/uipc_ktls.c

Changeset View

Standalone View

sys/kern/uipc_ktls.c

Show First 20 Lines • Show All 1,179 Lines • ▼ Show 20 Lines	ktls_enable_rx(struct socket so, struct tls_enable en)
* this to support rekeying in the future.		* this to support rekeying in the future.
*/		*/
if (so->so_rcv.sb_tls_info != NULL)		if (so->so_rcv.sb_tls_info != NULL)
return (EALREADY);		return (EALREADY);

if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable)		if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable)
return (ENOTSUP);		return (ENOTSUP);

/* TLS 1.3 is not yet supported. */
if (en->tls_vmajor == TLS_MAJOR_VER_ONE &&
en->tls_vminor == TLS_MINOR_VER_THREE)
return (ENOTSUP);

error = ktls_create_session(so, en, &tls);		error = ktls_create_session(so, en, &tls);
if (error)		if (error)
return (error);		return (error);

error = ktls_ocf_try(so, tls, KTLS_RX);		error = ktls_ocf_try(so, tls, KTLS_RX);
if (error) {		if (error) {
ktls_cleanup(tls);		ktls_cleanup(tls);
return (error);		return (error);
▲ Show 20 Lines • Show All 697 Lines • ▼ Show 20 Lines	out:
for (m = top; m != NULL; m = m->m_next)		for (m = top; m != NULL; m = m->m_next)
sbfree_ktls_rx(sb, m);		sbfree_ktls_rx(sb, m);
sb->sb_tlsdcc = len;		sb->sb_tlsdcc = len;
sb->sb_ccc += len;		sb->sb_ccc += len;
SBCHECK(sb);		SBCHECK(sb);
return (top);		return (top);
}		}

		/*
		* Determine the length of the trailing zero padding and find the real
		* record type in the byte before the padding.
		*
		* Walking the mbuf chain backwards is clumsy, so another option would
		* be to scan forwards remembering the last non-zero byte before the
		* trailer. However, it would be expensive to scan the entire record.
		* Instead, find the last non-zero byte of each mbuf in the chain
		* keeping track of the relative offset of that nonzero byte.
		*
		* trail_len is the size of the MAC/tag on input and is set to the
		* size of the full trailer including padding and the record type on
		* return.
		*/
		static int
		tls13_find_record_type(struct ktls_session tls, struct mbuf m, int tls_len,
		int trailer_len, uint8_t record_typep)
		{
		char *cp;
		u_int digest_start, last_offset, m_len, offset;
		uint8_t record_type;

		digest_start = tls_len - *trailer_len;
		last_offset = 0;
		offset = 0;
		for (; m != NULL && offset < digest_start;
		offset += m->m_len, m = m->m_next) {
		/* Don't look for padding in the tag. */
		m_len = min(digest_start - offset, m->m_len);
		cp = mtod(m, char *);

		/* Find last non-zero byte in this mbuf. */
		while (m_len > 0 && cp[m_len - 1] == 0)
		m_len--;
		if (m_len > 0) {
		record_type = cp[m_len - 1];
		last_offset = offset + m_len;
		}
		}
		if (last_offset < tls->params.tls_hlen)
		return (EBADMSG);

		*record_typep = record_type;
		*trailer_len = tls_len - last_offset + 1;
		return (0);
		}

static void		static void
ktls_decrypt(struct socket *so)		ktls_decrypt(struct socket *so)
{		{
char tls_header[MBUF_PEXT_HDR_LEN];		char tls_header[MBUF_PEXT_HDR_LEN];
struct ktls_session *tls;		struct ktls_session *tls;
struct sockbuf *sb;		struct sockbuf *sb;
struct tls_record_layer *hdr;		struct tls_record_layer *hdr;
struct tls_get_record tgr;		struct tls_get_record tgr;
struct mbuf control, data, *m;		struct mbuf control, data, *m;
uint64_t seqno;		uint64_t seqno;
int error, remain, tls_len, trail_len;		int error, remain, tls_len, trail_len;
		bool tls13;
		uint8_t vminor, record_type;

hdr = (struct tls_record_layer *)tls_header;		hdr = (struct tls_record_layer *)tls_header;
sb = &so->so_rcv;		sb = &so->so_rcv;
SOCKBUF_LOCK(sb);		SOCKBUF_LOCK(sb);
KASSERT(sb->sb_flags & SB_TLS_RX_RUNNING,		KASSERT(sb->sb_flags & SB_TLS_RX_RUNNING,
("%s: socket %p not running", __func__, so));		("%s: socket %p not running", __func__, so));

tls = sb->sb_tls_info;		tls = sb->sb_tls_info;
MPASS(tls != NULL);		MPASS(tls != NULL);

		tls13 = (tls->params.tls_vminor == TLS_MINOR_VER_THREE);
		if (tls13)
		vminor = TLS_MINOR_VER_TWO;
		else
		vminor = tls->params.tls_vminor;
for (;;) {		for (;;) {
/* Is there enough queued for a TLS header? */		/* Is there enough queued for a TLS header? */
if (sb->sb_tlscc < tls->params.tls_hlen)		if (sb->sb_tlscc < tls->params.tls_hlen)
break;		break;

m_copydata(sb->sb_mtls, 0, tls->params.tls_hlen, tls_header);		m_copydata(sb->sb_mtls, 0, tls->params.tls_hlen, tls_header);
tls_len = sizeof(*hdr) + ntohs(hdr->tls_length);		tls_len = sizeof(*hdr) + ntohs(hdr->tls_length);

if (hdr->tls_vmajor != tls->params.tls_vmajor \|\|		if (hdr->tls_vmajor != tls->params.tls_vmajor \|\|
hdr->tls_vminor != tls->params.tls_vminor)		hdr->tls_vminor != vminor)
error = EINVAL;		error = EINVAL;
		else if (tls13 && hdr->tls_type != TLS_RLTYPE_APP)
		error = EINVAL;
else if (tls_len < tls->params.tls_hlen \|\| tls_len >		else if (tls_len < tls->params.tls_hlen \|\| tls_len >
tls->params.tls_hlen + TLS_MAX_MSG_SIZE_V10_2 +		tls->params.tls_hlen + TLS_MAX_MSG_SIZE_V10_2 +
tls->params.tls_tlen)		tls->params.tls_tlen)
error = EMSGSIZE;		error = EMSGSIZE;
else		else
error = 0;		error = 0;
if (__predict_false(error != 0)) {		if (__predict_false(error != 0)) {
/*		/*
Show All 25 Lines	for (;;) {
MPASS(sb->sb_tlsdcc == tls_len);		MPASS(sb->sb_tlsdcc == tls_len);

seqno = sb->sb_tls_seqno;		seqno = sb->sb_tls_seqno;
sb->sb_tls_seqno++;		sb->sb_tls_seqno++;
SBCHECK(sb);		SBCHECK(sb);
SOCKBUF_UNLOCK(sb);		SOCKBUF_UNLOCK(sb);

error = tls->sw_decrypt(tls, hdr, data, seqno, &trail_len);		error = tls->sw_decrypt(tls, hdr, data, seqno, &trail_len);
		if (error == 0) {
		hselaskyUnsubmitted Done Inline Actions Could the record_type be extracted outside this function? We will need this for the hardware decrypted traffic. hselasky: Could the record_type be extracted outside this function? We will need this for the hardware…
		jhbAuthorUnsubmitted Done Inline Actions I think this version should work for you for NIC TLS as you can fall through to the code below with the decrypted record. jhb: I think this version should work for you for NIC TLS as you can fall through to the code below…
		if (tls13)
		error = tls13_find_record_type(tls, data,
		tls_len, &trail_len, &record_type);
		else
		record_type = hdr->tls_type;
		}
if (error) {		if (error) {
counter_u64_add(ktls_offload_failed_crypto, 1);		counter_u64_add(ktls_offload_failed_crypto, 1);

SOCKBUF_LOCK(sb);		SOCKBUF_LOCK(sb);
if (sb->sb_tlsdcc == 0) {		if (sb->sb_tlsdcc == 0) {
/*		/*
* sbcut/drop/flush discarded these		* sbcut/drop/flush discarded these
* mbufs.		* mbufs.
Show All 16 Lines	if (error) {

m_freem(data);		m_freem(data);

SOCKBUF_LOCK(sb);		SOCKBUF_LOCK(sb);
continue;		continue;
}		}

/* Allocate the control mbuf. */		/* Allocate the control mbuf. */
tgr.tls_type = hdr->tls_type;		tgr.tls_type = record_type;
tgr.tls_vmajor = hdr->tls_vmajor;		tgr.tls_vmajor = hdr->tls_vmajor;
tgr.tls_vminor = hdr->tls_vminor;		tgr.tls_vminor = hdr->tls_vminor;
tgr.tls_length = htobe16(tls_len - tls->params.tls_hlen -		tgr.tls_length = htobe16(tls_len - tls->params.tls_hlen -
trail_len);		trail_len);
control = sbcreatecontrol_how(&tgr, sizeof(tgr),		control = sbcreatecontrol_how(&tgr, sizeof(tgr),
TLS_GET_RECORD, IPPROTO_TCP, M_WAITOK);		TLS_GET_RECORD, IPPROTO_TCP, M_WAITOK);

SOCKBUF_LOCK(sb);		SOCKBUF_LOCK(sb);
▲ Show 20 Lines • Show All 765 Lines • Show Last 20 Lines