diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
--- a/sys/kern/uipc_ktls.c
+++ b/sys/kern/uipc_ktls.c
@@ -784,7 +784,6 @@
 			counter_u64_add(ktls_sw_chacha20, -1);
 			break;
 		}
-		ktls_ocf_free(tls);
 		break;
 	case TCP_TLS_MODE_IFNET:
 		switch (tls->params.cipher_algorithm) {
@@ -817,6 +816,8 @@
 		break;
 #endif
 	}
+	if (tls->ocf_session != NULL)
+		ktls_ocf_free(tls);
 	if (tls->params.auth_key != NULL) {
 		zfree(tls->params.auth_key, M_KTLS);
 		tls->params.auth_key = NULL;
@@ -1004,14 +1005,9 @@
 	return (error);
 }
 
-static int
-ktls_try_sw(struct socket *so, struct ktls_session *tls, int direction)
+static void
+ktls_use_sw(struct ktls_session *tls)
 {
-	int error;
-
-	error = ktls_ocf_try(so, tls, direction);
-	if (error)
-		return (error);
 	tls->mode = TCP_TLS_MODE_SW;
 	switch (tls->params.cipher_algorithm) {
 	case CRYPTO_AES_CBC:
@@ -1024,6 +1020,17 @@
 		counter_u64_add(ktls_sw_chacha20, 1);
 		break;
 	}
+}
+
+static int
+ktls_try_sw(struct socket *so, struct ktls_session *tls, int direction)
+{
+	int error;
+
+	error = ktls_ocf_try(so, tls, direction);
+	if (error)
+		return (error);
+	ktls_use_sw(tls);
 	return (0);
 }
 
@@ -1184,17 +1191,18 @@
 	if (error)
 		return (error);
 
-#ifdef TCP_OFFLOAD
-	error = ktls_try_toe(so, tls, KTLS_RX);
-	if (error)
-#endif
-		error = ktls_try_sw(so, tls, KTLS_RX);
-
+	error = ktls_ocf_try(so, tls, KTLS_RX);
 	if (error) {
 		ktls_cleanup(tls);
 		return (error);
 	}
 
+#ifdef TCP_OFFLOAD
+	error = ktls_try_toe(so, tls, KTLS_RX);
+	if (error)
+#endif
+		ktls_use_sw(tls);
+
 	/* Mark the socket as using TLS offload. */
 	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_tls_seqno = be64dec(en->rec_seq);
diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h
--- a/sys/sys/ktls.h
+++ b/sys/sys/ktls.h
@@ -184,10 +184,8 @@
 		    const struct tls_record_layer *hdr, struct mbuf *m,
 		    uint64_t seqno, int *trailer_len);
 	};
-	union {
-		struct ktls_ocf_session *ocf_session;
-		struct m_snd_tag *snd_tag;
-	};
+	struct ktls_ocf_session *ocf_session;
+	struct m_snd_tag *snd_tag;
 	struct tls_session_params params;
 	u_int	wq_index;
 	volatile u_int refcount;