Differential D32356 Diff 97105 sys/dev/cxgbe/tom/t4_tls.c

Changeset View

Standalone View

sys/dev/cxgbe/tom/t4_tls.c

Show First 20 Lines • Show All 98 Lines • ▼ Show 20 Lines
/* Clear TF_RX_QUIESCE to re-enable receive. */		/* Clear TF_RX_QUIESCE to re-enable receive. */
static void		static void
t4_clear_rx_quiesce(struct toepcb *toep)		t4_clear_rx_quiesce(struct toepcb *toep)
{		{

t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0);		t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0);
}		}

static void
tls_clr_ofld_mode(struct toepcb *toep)
{

tls_stop_handshake_timer(toep);

KASSERT(toep->tls.rx_key_addr == -1,
("%s: tid %d has RX key", __func__, toep->tid));

/* Switch to plain TOE mode. */
t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)),
V_TCB_ULP_RAW(V_TF_TLS_ENABLE(0)));
t4_set_tls_tcb_field(toep, W_TCB_ULP_TYPE,
V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_NONE));
t4_clear_rx_quiesce(toep);

toep->flags &= ~(TPF_FORCE_CREDITS \| TPF_TLS_ESTABLISHED);
toep->params.ulp_mode = ULP_MODE_NONE;
}

/* TLS/DTLS content type for CPL SFO */		/* TLS/DTLS content type for CPL SFO */
static inline unsigned char		static inline unsigned char
tls_content_type(unsigned char content_type)		tls_content_type(unsigned char content_type)
{		{
/*		/*
* XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and		* XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and
* default to "CUSTOM" for all other types including		* default to "CUSTOM" for all other types including
* heartbeat?		* heartbeat?
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines	if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
toep->txsd_pidx = 0;		toep->txsd_pidx = 0;
toep->txsd_avail--;		toep->txsd_avail--;

t4_wrq_tx(sc, wr);		t4_wrq_tx(sc, wr);

return (0);		return (0);
}		}

/*
* In some cases a client connection can hang without sending the
* ServerHelloDone message from the NIC to the host. Send a dummy
* RX_DATA_ACK with RX_MODULATE to unstick the connection.
*/
static void
tls_send_handshake_ack(void *arg)
{
struct toepcb *toep = arg;
struct tls_ofld_info *tls_ofld = &toep->tls;
struct adapter *sc = td_adapter(toep->td);

/* Bail without rescheduling if the connection has closed. */
if ((toep->flags & (TPF_FIN_SENT \| TPF_ABORT_SHUTDOWN)) != 0)
return;

/*
* If this connection has timed out without receiving more
* data, downgrade to plain TOE mode and don't re-arm the
* timer.
*/
if (sc->tt.tls_rx_timeout != 0) {
struct inpcb *inp;
struct tcpcb *tp;

inp = toep->inp;
tp = intotcpcb(inp);
if ((ticks - tp->t_rcvtime) >= sc->tt.tls_rx_timeout) {
CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__,
toep->tid);
tls_clr_ofld_mode(toep);
return;
}
}

/*
* XXX: Does not have the t4_get_tcb() checks to refine the
* workaround.
*/
callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz);

CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid);
send_rx_modulate(sc, toep);
}

static void
tls_start_handshake_timer(struct toepcb *toep)
{
struct tls_ofld_info *tls_ofld = &toep->tls;

INP_WLOCK_ASSERT(toep->inp);
callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz,
tls_send_handshake_ack, toep);
}

void
tls_stop_handshake_timer(struct toepcb *toep)
{
struct tls_ofld_info *tls_ofld = &toep->tls;

INP_WLOCK_ASSERT(toep->inp);
callout_stop(&tls_ofld->handshake_timer);
}

int		int
tls_alloc_ktls(struct toepcb toep, struct ktls_session tls, int direction)		tls_alloc_ktls(struct toepcb toep, struct ktls_session tls, int direction)
{		{
struct adapter *sc = td_adapter(toep->td);		struct adapter *sc = td_adapter(toep->td);
int error, explicit_iv_size, key_offset, mac_first;		int error, explicit_iv_size, mac_first;

if (!can_tls_offload(td_adapter(toep->td)))		if (!can_tls_offload(sc))
return (EINVAL);		return (EINVAL);

		if (direction == KTLS_RX) {
		if (ulp_mode(toep) != ULP_MODE_NONE)
		return (EINVAL);
		if ((toep->flags & TPF_TLS_STARTING) != 0)
		return (EINVAL);
		} else {
switch (ulp_mode(toep)) {		switch (ulp_mode(toep)) {
case ULP_MODE_TLS:
break;
case ULP_MODE_NONE:		case ULP_MODE_NONE:
		case ULP_MODE_TLS:
case ULP_MODE_TCPDDP:		case ULP_MODE_TCPDDP:
if (direction != KTLS_TX)
return (EINVAL);
break;		break;
default:		default:
return (EINVAL);		return (EINVAL);
}		}
		}

switch (tls->params.cipher_algorithm) {		switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:		case CRYPTO_AES_CBC:
/* XXX: Explicitly ignore any provided IV. */		/* XXX: Explicitly ignore any provided IV. */
switch (tls->params.cipher_key_len) {		switch (tls->params.cipher_key_len) {
case 128 / 8:		case 128 / 8:
case 192 / 8:		case 192 / 8:
case 256 / 8:		case 256 / 8:
break;		break;
default:		default:
error = EINVAL;		return (EINVAL);
goto clr_ofld;
}		}
switch (tls->params.auth_algorithm) {		switch (tls->params.auth_algorithm) {
case CRYPTO_SHA1_HMAC:		case CRYPTO_SHA1_HMAC:
case CRYPTO_SHA2_256_HMAC:		case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:		case CRYPTO_SHA2_384_HMAC:
break;		break;
default:		default:
error = EPROTONOSUPPORT;		return (EPROTONOSUPPORT);
goto clr_ofld;
}		}
explicit_iv_size = AES_BLOCK_LEN;		explicit_iv_size = AES_BLOCK_LEN;
mac_first = 1;		mac_first = 1;
break;		break;
case CRYPTO_AES_NIST_GCM_16:		case CRYPTO_AES_NIST_GCM_16:
if (tls->params.iv_len != SALT_SIZE) {		if (tls->params.iv_len != SALT_SIZE) {
error = EINVAL;		return (EINVAL);
goto clr_ofld;
}		}
switch (tls->params.cipher_key_len) {		switch (tls->params.cipher_key_len) {
case 128 / 8:		case 128 / 8:
case 192 / 8:		case 192 / 8:
case 256 / 8:		case 256 / 8:
break;		break;
default:		default:
error = EINVAL;		return (EINVAL);
goto clr_ofld;
}		}
explicit_iv_size = 8;		explicit_iv_size = 8;
mac_first = 0;		mac_first = 0;
break;		break;
default:		default:
error = EPROTONOSUPPORT;		return (EPROTONOSUPPORT);
goto clr_ofld;
}		}

/* Only TLS 1.1 and TLS 1.2 are currently supported. */		/* Only TLS 1.1 and TLS 1.2 are currently supported. */
if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE \|\|		if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE \|\|
tls->params.tls_vminor < TLS_MINOR_VER_ONE \|\|		tls->params.tls_vminor < TLS_MINOR_VER_ONE \|\|
tls->params.tls_vminor > TLS_MINOR_VER_TWO) {		tls->params.tls_vminor > TLS_MINOR_VER_TWO) {
error = EPROTONOSUPPORT;		return (EPROTONOSUPPORT);
goto clr_ofld;
}		}

/* Bail if we already have a key. */		/* Bail if we already have a key. */
if (direction == KTLS_TX) {		if (direction == KTLS_TX) {
if (toep->tls.tx_key_addr != -1)		if (toep->tls.tx_key_addr != -1)
return (EOPNOTSUPP);		return (EOPNOTSUPP);
} else {		} else {
if (toep->tls.rx_key_addr != -1)		if (toep->tls.rx_key_addr != -1)
return (EOPNOTSUPP);		return (EOPNOTSUPP);
}		}

error = tls_program_key_id(toep, tls, direction);		error = tls_program_key_id(toep, tls, direction);
if (error) {		if (error)
if (direction == KTLS_RX)
goto clr_ofld;
return (error);		return (error);
}

if (direction == KTLS_TX) {		if (direction == KTLS_TX) {
toep->tls.scmd0.seqno_numivs =		toep->tls.scmd0.seqno_numivs =
(V_SCMD_SEQ_NO_CTRL(3) \|		(V_SCMD_SEQ_NO_CTRL(3) \|
V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) \|		V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) \|
V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) \|		V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) \|
V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) \|		V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) \|
V_SCMD_CIPH_MODE(t4_tls_cipher_mode(tls)) \|		V_SCMD_CIPH_MODE(t4_tls_cipher_mode(tls)) \|
Show All 15 Lines	if (direction == KTLS_TX) {
toep->tls.fcplenmax = get_tp_plen_max(tls);		toep->tls.fcplenmax = get_tp_plen_max(tls);
toep->tls.expn_per_ulp = tls->params.tls_hlen +		toep->tls.expn_per_ulp = tls->params.tls_hlen +
tls->params.tls_tlen;		tls->params.tls_tlen;
toep->tls.pdus_per_ulp = 1;		toep->tls.pdus_per_ulp = 1;
toep->tls.adjusted_plen = toep->tls.expn_per_ulp +		toep->tls.adjusted_plen = toep->tls.expn_per_ulp +
tls->params.max_frame_len;		tls->params.max_frame_len;
toep->tls.tx_key_info_size = t4_tls_key_info_size(tls);		toep->tls.tx_key_info_size = t4_tls_key_info_size(tls);
} else {		} else {
/* Stop timer on handshake completion */		toep->flags \|= TPF_TLS_STARTING;
tls_stop_handshake_timer(toep);

toep->flags &= ~TPF_FORCE_CREDITS;
toep->flags \|= TPF_TLS_RECEIVE;
toep->tls.rx_version = tls->params.tls_vmajor << 8 \|		toep->tls.rx_version = tls->params.tls_vmajor << 8 \|
tls->params.tls_vminor;		tls->params.tls_vminor;

/*		CTR2(KTR_CXGBE, "%s: tid %d setting RX_QUIESCE", __func__,
* RX key tags are an index into the key portion of MA		toep->tid);
* memory stored as an offset from the base address in		t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, W_TCB_T_FLAGS,
* units of 64 bytes.		V_TF_RX_QUIESCE(1), V_TF_RX_QUIESCE(1), 1, CPL_COOKIE_TOM);
*/
key_offset = toep->tls.rx_key_addr - sc->vres.key.start;
t4_set_tls_keyid(toep, key_offset / 64);
t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
V_TCB_ULP_RAW(M_TCB_ULP_RAW),
V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) \|
V_TF_TLS_CONTROL(1) \|
V_TF_TLS_ACTIVE(1) \|
V_TF_TLS_ENABLE(1))));
t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ,
V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
V_TCB_TLS_SEQ(0));
t4_clear_rx_quiesce(toep);
}		}

return (0);		return (0);

clr_ofld:
if (ulp_mode(toep) == ULP_MODE_TLS) {
CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__,
toep->tid);
tls_clr_ofld_mode(toep);
}		}
return (error);
}

void		void
tls_init_toep(struct toepcb *toep)		tls_init_toep(struct toepcb *toep)
{		{
struct tls_ofld_info *tls_ofld = &toep->tls;		struct tls_ofld_info *tls_ofld = &toep->tls;

tls_ofld->rx_key_addr = -1;		tls_ofld->rx_key_addr = -1;
tls_ofld->tx_key_addr = -1;		tls_ofld->tx_key_addr = -1;
}		}

void		void
tls_establish(struct toepcb *toep)
{

/*
* Enable PDU extraction.
*
* XXX: Supposedly this should be done by the firmware when
* the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but
* in practice this seems to be required.
*/
CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid);
t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW),
V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));

toep->flags \|= TPF_FORCE_CREDITS \| TPF_TLS_ESTABLISHED;

callout_init_rw(&toep->tls.handshake_timer, &toep->inp->inp_lock, 0);
tls_start_handshake_timer(toep);
}

void
tls_detach(struct toepcb *toep)
{

if (toep->flags & TPF_TLS_ESTABLISHED) {
tls_stop_handshake_timer(toep);
toep->flags &= ~TPF_TLS_ESTABLISHED;
}
}

void
tls_uninit_toep(struct toepcb *toep)		tls_uninit_toep(struct toepcb *toep)
{		{

MPASS((toep->flags & TPF_TLS_ESTABLISHED) == 0);
clear_tls_keyid(toep);		clear_tls_keyid(toep);
}		}

#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)		#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
#define MIN_OFLD_TLSTX_CREDITS(toep) \		#define MIN_OFLD_TLSTX_CREDITS(toep) \
(howmany(sizeof(struct fw_tlstx_data_wr) + \		(howmany(sizeof(struct fw_tlstx_data_wr) + \
sizeof(struct cpl_tx_tls_sfo) + sizeof(struct ulptx_idata) + \		sizeof(struct cpl_tx_tls_sfo) + sizeof(struct ulptx_idata) + \
sizeof(struct ulptx_sc_memrd) + \		sizeof(struct ulptx_sc_memrd) + \
▲ Show 20 Lines • Show All 502 Lines • ▼ Show 20 Lines	#endif
tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);		tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);
if (tls_data != NULL) {		if (tls_data != NULL) {
KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq,		KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq,
("%s: sequence mismatch", __func__));		("%s: sequence mismatch", __func__));
}		}

/* Report decryption errors as EBADMSG. */		/* Report decryption errors as EBADMSG. */
if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) {		if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) {
		CTR4(KTR_CXGBE, "%s: tid %u TLS error %#x ddp_vld %#x",
		__func__, toep->tid, tls_hdr_pkt->res_to_mac_error,
		be32toh(cpl->ddp_valid));
m_freem(m);		m_freem(m);
m_freem(tls_data);		m_freem(tls_data);

CURVNET_SET(toep->vnet);		CURVNET_SET(toep->vnet);
so->so_error = EBADMSG;		so->so_error = EBADMSG;
sorwakeup(so);		sorwakeup(so);

INP_WUNLOCK(inp);		INP_WUNLOCK(inp);
▲ Show 20 Lines • Show All 201 Lines • ▼ Show 20 Lines	#endif
SOCKBUF_UNLOCK_ASSERT(sb);		SOCKBUF_UNLOCK_ASSERT(sb);

INP_WUNLOCK(inp);		INP_WUNLOCK(inp);
CURVNET_RESTORE();		CURVNET_RESTORE();

m_freem(m);		m_freem(m);
}		}

		/*
		* Finish converting a connection to ULP_MODE_TLS and enable TLS
		* decryption.
		*/
		static void
		finish_tls_rx_enable(struct adapter sc, struct toepcb toep, uint64_t seqno)
		{
		int key_offset;

		CTR3(KTR_CXGBE, "%s: tid %d seqno %lu", __func__, toep->tid, seqno);

		/*
		* RX key tags are an index into the key portion of MA memory
		* stored as an offset from the base address in units of 64
		* bytes.
		*/
		key_offset = toep->tls.rx_key_addr - sc->vres.key.start;
		t4_set_tls_keyid(toep, key_offset / 64);
		t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
		V_TCB_TLS_SEQ(seqno));

		/* XXX: Just set ACTIVE? */
		t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW),
		V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) \| V_TF_TLS_CONTROL(1) \|
		V_TF_TLS_ACTIVE(1) \| V_TF_TLS_ENABLE(1))));
		t4_clear_rx_quiesce(toep);

		toep->flags &= ~TPF_TLS_STARTING;
		toep->flags \|= TPF_TLS_RECEIVE;
		}

		/*
		* Examine the pending data in the socket buffer and either finish
		* enabling TLS RX or request more unencrypted data.
		*/
		static void
		tls_check_rx_sockbuf(struct adapter sc, struct toepcb toep,
		struct sockbuf *sb)
		{
		uint64_t seqno;
		size_t resid;
		bool have_header;

		SOCKBUF_LOCK_ASSERT(sb);
		MPASS(toep->tls.rx_resid == 0);

		have_header = ktls_pending_rx_info(sb, &seqno, &resid);
		CTR5(KTR_CXGBE, "%s: tid %d have_header %d seqno %lu resid %zu",
		__func__, toep->tid, have_header, seqno, resid);

		/*
		* If socket buffer is empty or only contains complete TLS
		* records, enable TLS decryption.
		*/
		if (resid == 0) {
		finish_tls_rx_enable(sc, toep, seqno);
		return;
		}

		/*
		* If we have a partial header at the end of the socket
		* buffer, just ask for a complete TLS header (the smallest
		* number of bytes possible) and check again after that data
		* is received.
		*
		* If we need fewer bytes than the size of a TLS header to
		* complete the last record in the socket buffer, punt a bit.
		* We can't ask for fewer bytes, so instead ask for the
		* remaining bytes plus the next full TLS header. This will
		* let us ask for the exact payload needed on the next check
		* and get back in sync after that.
		*/
		if (!have_header) {
		printf(
		"%s: tid %u requesting %zu bytes for missing %zu header bytes\n",
		__func__, toep->tid, sizeof(struct tls_hdr), resid);
		resid = sizeof(struct tls_hdr);
		} else if (resid < sizeof(struct tls_hdr)) {
		resid += sizeof(struct tls_hdr);
		printf("%s: tid %u requesting %zu bytes for trailer + header\n",
		__func__, toep->tid, resid);
		} else
		printf("%s: tid %u requesting %zu bytes for remainder\n",
		__func__, toep->tid, resid);

		/*
		* Set PDU length. This is treating the 'resid' bytes as a
		* TLS PDU, so the first 5 bytes are a fake header and the
		* rest are the PDU length.
		*/
		t4_set_tls_tcb_field(toep, W_TCB_TX_PDU_LEN,
		V_TCB_TX_PDU_LEN(M_TCB_TX_PDU_LEN),
		V_TCB_TX_PDU_LEN(resid - sizeof(struct tls_hdr)));
		CTR3(KTR_CXGBE, "%s: tid %d setting TX_PDU_LEN to %zu", __func__,
		toep->tid, resid - sizeof(struct tls_hdr));

		/* Allow the 'resid' bytes to be delivered as CPL_RX_DATA. */
		toep->tls.rx_resid = resid;
		t4_clear_rx_quiesce(toep);
		}

void		void
		tls_received_starting_data(struct adapter sc, struct toepcb toep,
		struct sockbuf *sb, int len)
		{
		MPASS(toep->flags & TPF_TLS_STARTING);
		KASSERT(len <= toep->tls.rx_resid,
		("%s: received excess bytes %d (waiting for %zu)", __func__, len,
		toep->tls.rx_resid));
		toep->tls.rx_resid -= len;
		if (toep->tls.rx_resid != 0)
		return;

		tls_check_rx_sockbuf(sc, toep, sb);
		}

		static int
		do_tls_tcb_rpl(struct sge_iq iq, const struct rss_header rss, struct mbuf *m)
		{
		struct adapter *sc = iq->adapter;
		const struct cpl_set_tcb_rpl cpl = (const void )(rss + 1);
		unsigned int tid = GET_TID(cpl);
		struct toepcb *toep;
		struct inpcb *inp;
		struct socket *so;
		struct sockbuf *sb;

		if (cpl->status != CPL_ERR_NONE)
		panic("XXX: tcp_rpl failed: %d", cpl->status);

		toep = lookup_tid(sc, tid);
		inp = toep->inp;
		switch (cpl->cookie) {
		case V_WORD(W_TCB_T_FLAGS) \| V_COOKIE(CPL_COOKIE_TOM):
		INP_WLOCK(inp);
		if ((toep->flags & TPF_TLS_STARTING) == 0)
		panic("%s: connection is not starting TLS RX\n",
		__func__);

		/* Set the ULP mode to ULP_MODE_TLS. */
		toep->params.ulp_mode = ULP_MODE_TLS;
		t4_set_tls_tcb_field(toep, W_TCB_ULP_TYPE,
		V_TCB_ULP_TYPE(M_TCB_ULP_TYPE),
		V_TCB_ULP_TYPE(ULP_MODE_TLS));

		/* Clear all bits in ULP_RAW except for ENABLE. */
		t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
		V_TCB_ULP_RAW(M_TCB_ULP_RAW),
		V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));

		/* Clear the entire TLS overlay region: 1023:832. */
		for (u_int word = 832 / 32; word < 1024 / 32; word += 2)
		t4_set_tls_tcb_field(toep, word, 0xffffffffffffffff, 0);

		so = inp->inp_socket;
		sb = &so->so_rcv;
		SOCKBUF_LOCK(sb);
		tls_check_rx_sockbuf(sc, toep, sb);
		SOCKBUF_UNLOCK(sb);
		INP_WUNLOCK(inp);
		break;
		default:
		panic("XXX: unknown tcb_rpl offset %#x, cookie %#x",
		G_WORD(cpl->cookie), G_COOKIE(cpl->cookie));
		}

		return (0);
		}

		void
t4_tls_mod_load(void)		t4_tls_mod_load(void)
{		{

t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data);		t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data);
t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp);		t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp);
		t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_tls_tcb_rpl,
		CPL_COOKIE_TOM);
}		}

void		void
t4_tls_mod_unload(void)		t4_tls_mod_unload(void)
{		{

t4_register_cpl_handler(CPL_TLS_DATA, NULL);		t4_register_cpl_handler(CPL_TLS_DATA, NULL);
t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL);		t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL);
		t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, NULL, CPL_COOKIE_TOM);
}		}
#endif /* TCP_OFFLOAD */		#endif /* TCP_OFFLOAD */
#endif /* KERN_TLS */		#endif /* KERN_TLS */