D58606.1794279800.diff
No OneTemporary
Actions

Authored By

Unknown

Size

100 KB

Referenced Files

None

Subscribers

None

D58606.1794279800.diff
View Options

	Index: sys/conf/files
	===================================================================
	--- sys/conf/files
	+++ sys/conf/files
	@@ -1334,6 +1334,7 @@
	dev/bnxt/bnxt_en/bnxt_coredump.c optional bnxt iflib pci compile-with "${BNXT_C}"
	dev/bnxt/bnxt_en/bnxt_ptp.c optional bnxt iflib pci compile-with "${BNXT_C}"
	dev/bnxt/bnxt_en/bnxt_mpc.c optional bnxt iflib pci compile-with "${BNXT_C}"
	+dev/bnxt/bnxt_en/bnxt_ktls.c optional bnxt iflib pci compile-with "${BNXT_C}"
	dev/bnxt/bnxt_en/if_bnxt.c optional bnxt iflib pci compile-with "${BNXT_C}"
	dev/bwi/bwimac.c optional bwi
	dev/bwi/bwiphy.c optional bwi
	Index: sys/dev/bnxt/bnxt_en/bnxt.h
	===================================================================
	--- sys/dev/bnxt/bnxt_en/bnxt.h
	+++ sys/dev/bnxt/bnxt_en/bnxt.h
	@@ -238,6 +238,11 @@

	#define BNXT_MAX_NUM_QUEUES (BNXT_MAX_L2_QUEUES + BNXT_ROCE_IRQ_COUNT)

	+#define BNXT_DEBUG(dev, fmt, ...) do { \
	+ if (bootverbose) \
	+ device_printf(dev, "" fmt, ##__VA_ARGS__); \
	+} while (0)
	+
	/* Completion related defines */
	#define CMP_VALID(cmp, v_bit) \
	((!!(((struct cmpl_base *)(cmp))->info3_v & htole32(CMPL_BASE_V))) == !!(v_bit) )
	@@ -322,6 +327,7 @@
	(offsetof(struct tx_port_stats_ext, counter) / 8)

	extern const char bnxt_driver_version[];
	+extern const uint16_t bnxt_tx_lhint[];
	typedef void (bnxt_doorbell_tx)(void , uint16_t idx);
	typedef void (bnxt_doorbell_rx)(void , uint16_t idx);
	typedef void (bnxt_doorbell_rx_cq)(void , bool);
	@@ -609,11 +615,47 @@

	#define TX_RING_MASK(txr) ((txr)->ring_size - 1)

	+#define SET_TX_OPAQUE_KTLS(opq, txr, replay, prod, segs) \
	+ do { \
	+ opq->ktls_replay = (replay); \
	+ opq->idx = (prod) & TX_RING_MASK(txr); \
	+ opq->bds = (segs); \
	+ } while(0);
	+
	+#define KID_HIGH(kid) ((kid & 0x000fff80) >> 7) /Upper 13 bits of KID/
	+#define KID_LOW(kid) (kid & 0x0000007f) /Lower 7 bits of KID/
	+#define TX_BD_FLAGS_CRYPTO_EN (1 << 15)
	+
	+/* Hand-rolled generic BD views used only for byte-addressable copies of
	+ * the kTLS presync command (see bnxt_ktls_pre_xmit()); distinct from the
	+ * full HSI tx_bd_short/tx_bd_long types used for normal TX BDs. */
	+struct tx_bd {
	+ uint32_t tx_bd_len_flags_type;
	+#define TX_BD_TYPE (0x3f << 0)
	+#define TX_BD_FLAGS_BD_CNT_SHIFT 8
	+#define TX_BD_LEN_SHIFT 16
	+ uint32_t tx_bd_opaque;
	+ uint64_t tx_bd_haddr;
	+} __packed;
	+
	+struct tx_bd_presync {
	+ uint32_t tx_bd_len_flags_type;
	+#define TX_BD_TYPE_PRESYNC_TX_BD (0x09 << 0)
	+ uint32_t tx_bd_opaque;
	+ uint32_t tx_bd_kid;
	+ uint32_t tx_bd_unused;
	+} __packed;
	+
	struct bnxt_sw_mpc_tx_bd {
	uint8_t inline_bds;
	unsigned long handle;
	};

	+struct bnxt_sw_tx_bd {
	+ uint8_t is_replay;
	+ uint8_t inline_bds;
	+};
	+
	struct bnxt_ring {
	uint64_t paddr;
	vm_offset_t doorbell;
	@@ -632,7 +674,11 @@
	uint16_t prod; /* MPC TX ring producer index */
	uint16_t cons; /* MPC TX ring consumer index */
	struct mtx tx_lock; /* MPC TX ring lock */
	- struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring;
	+ union {
	+ struct bnxt_sw_tx_bd *tx_buf_ring;
	+ struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring;
	+ };
	+ struct bnxt_replay_pkt *replay_pkt;
	struct bnxt_full_tpa_start *tpa_start;
	union {
	u64 db_key64;
	@@ -861,6 +907,7 @@
	#define BNXT_CTX_FTQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_FP_TQM_RING
	#define BNXT_CTX_MRAV HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_MRAV
	#define BNXT_CTX_TIM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TIM
	+#define BNXT_CTX_TCK HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TX_CK
	#define BNXT_CTX_TKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TKC
	#define BNXT_CTX_RKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_RKC
	#define BNXT_CTX_MTQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_MP_TQM_RING
	@@ -910,6 +957,20 @@
	struct bnxt_ctx_mem_type ctx_arr[BNXT_CTX_V2_MAX];
	};

	+enum bnxt_tls_rtpe {
	+ BNXT_CRYPTO_TYPE_KTLS = 0,
	+ BNXT_CRYPTO_TYPE_QUIC,
	+};
	+
	+struct bnxt_hw_tls_resc {
	+ uint32_t min_tx_key_ctxs;
	+ uint32_t max_tx_key_ctxs;
	+ uint32_t resv_tx_key_ctxs;
	+ uint32_t min_rx_key_ctxs;
	+ uint32_t max_rx_key_ctxs;
	+ uint32_t resv_rx_key_ctxs;
	+};
	+
	struct bnxt_hw_resc {
	uint16_t min_rsscos_ctxs;
	uint16_t max_rsscos_ctxs;
	@@ -937,6 +998,7 @@
	uint16_t max_nqs;
	uint16_t max_irqs;
	uint16_t resv_irqs;
	+ struct bnxt_hw_tls_resc tls_resc[2];
	};

	enum bnxt_type_ets {
	@@ -1350,6 +1412,7 @@
	#define BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO BIT_ULL(47)
	#define BNXT_FW_CAP_ENABLE_RDMA_SRIOV BIT_ULL(48)
	#define BNXT_FW_CAP_RSS_TCAM BIT_ULL(49)
	+ #define BNXT_FW_CAP_KTLS_SUPPORTED BIT_ULL(50)
	#define BNXT_FLAG_TX_COAL_CMPL BIT_ULL(51)

	#define BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS BIT_ULL(61)
	@@ -1455,6 +1518,12 @@
	bool rx_ts_enabled;

	struct bnxt_mpc_info *mpc_info;
	+ struct bnxt_tls_info *ktls_info;
	+ struct sysctl_ctx_list ktls_stats;
	+ struct sysctl_oid *ktls_stats_oid;
	+ struct sysctl_ctx_list mpc_cmp_time_stats;
	+ struct sysctl_oid *mpc_cmp_time_stats_oid;
	+ uint32_t max_ktls_entries;

	#define BNXT_CAGR_CQCOAL_OFFSET 0xc00
	#define BNXT_CAGR_NQAGG_MAXTIMER 0x5930000
	@@ -1608,4 +1677,10 @@
	bit_clear(bmap, nr);
	return (1);
	}
	+
	+int bnxt_tls_snd_tag_alloc(if_t ifp,
	+ union if_snd_tag_alloc_params *params,
	+ struct m_snd_tag **ppmt);
	+int bnxt_ktls_xmit(struct bnxt_softc* softc, struct bnxt_ring* txr,
	+ struct mbuf** ppmb, uint16_t* lflags, uint32_t* kid, if_pkt_info_t pi);
	#endif /* _BNXT_H */
	Index: sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
	===================================================================
	--- sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
	+++ sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
	@@ -30,9 +30,12 @@
	#include <sys/endian.h>
	#include <linux/pci.h>

	+#include "opt_kern_tls.h"
	+
	#include "bnxt.h"
	#include "bnxt_hwrm.h"
	#include "bnxt_mpc.h"
	+#include "bnxt_ktls.h"
	#include "hsi_struct_def.h"
	#include "bnxt_coredump.h"

	@@ -789,11 +792,21 @@
	hw_resc->min_stat_ctxs = le16toh(resp->min_stat_ctx);
	hw_resc->max_stat_ctxs = le16toh(resp->max_stat_ctx);

	+ {
	+ struct bnxt_hw_tls_resc *tls_resc =
	+ &hw_resc->tls_resc[BNXT_CRYPTO_TYPE_KTLS];
	+
	+ tls_resc->min_tx_key_ctxs = le32toh(resp->min_ktls_tx_key_ctxs);
	+ tls_resc->max_tx_key_ctxs = le32toh(resp->max_ktls_tx_key_ctxs);
	+ tls_resc->min_rx_key_ctxs = le32toh(resp->min_ktls_rx_key_ctxs);
	+ tls_resc->max_rx_key_ctxs = le32toh(resp->max_ktls_rx_key_ctxs);
	+ }
	+
	if (BNXT_CHIP_P5_PLUS(softc)) {
	hw_resc->max_nqs = hw_resc->max_irqs = le16toh(resp->max_msix);
	hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
	}
	-
	+
	if (BNXT_PF(softc)) {
	struct bnxt_pf_info *pf = &softc->pf;

	@@ -1206,6 +1219,11 @@
	(void *)softc->hwrm_cmd_resp.idi_vaddr;
	struct bnxt_func_info *func = &softc->func;
	uint32_t flags, flags_ext, flags_ext2;
	+#if defined(KERN_TLS) && defined(KTLS_IFLIB_SUPPORT)
	+ bool ktls_fw_supported = false;
	+ uint16_t ktls_max_keys = 0, ktls_ctxs_per_partition = 0;
	+ bool ktls_partition_cap = false;
	+#endif

	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS);
	req.fid = htole16(0xffff);
	@@ -1258,6 +1276,21 @@
	if (BNXT_PF(softc) && (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_ECN_STATS_SUPPORTED))
	softc->fw_cap \|= BNXT_FW_CAP_ECN_STATS;

	+#if defined(KERN_TLS) && defined(KTLS_IFLIB_SUPPORT)
	+ if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_KTLS_SUPPORTED) {
	+ softc->fw_cap \|= BNXT_FW_CAP_KTLS_SUPPORTED;
	+ /*
	+ * Only extract the fields needed here; the actual alloc/free
	+ * work happens after BNXT_HWRM_UNLOCK() below, since
	+ * bnxt_free_ktls_info() can sleep (taskqueue_quiesce(),
	+ * uma_zdestroy()) and must not run with hwrm_lock held.
	+ */
	+ ktls_fw_supported = true;
	+ ktls_max_keys = le16toh(resp->max_key_ctxs_alloc);
	+ ktls_partition_cap = BNXT_PARTITION_CAP(resp);
	+ ktls_ctxs_per_partition = le16toh(resp->ctxs_per_partition);
	+ }
	+#endif
	if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PTP_PPS_SUPPORTED)
	softc->fw_cap \|= BNXT_FW_CAP_PTP_PPS;
	if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PTP_PTM_SUPPORTED)
	@@ -1358,6 +1391,23 @@
	if (BNXT_PF(softc))
	bnxt_alloc_mpc_info(softc, resp->mpc_chnls_cap);

	+ BNXT_HWRM_UNLOCK(softc);
	+#if defined(KERN_TLS) && defined(KTLS_IFLIB_SUPPORT)
	+ if (ktls_fw_supported) {
	+ bnxt_alloc_ktls_info(softc, ktls_max_keys, ktls_partition_cap,
	+ ktls_ctxs_per_partition);
	+ /*
	+ * HW Tx completion coalescing is not used with HW KTLS; the
	+ * driver expects ordinary per-packet TX completions on that path.
	+ */
	+ if (softc->ktls_info != NULL)
	+ softc->fw_cap &= ~BNXT_FLAG_TX_COAL_CMPL;
	+ } else {
	+ bnxt_free_ktls_info(softc);
	+ }
	+#endif
	+ return rc;
	+
	fail:
	BNXT_HWRM_UNLOCK(softc);
	return rc;
	@@ -1453,6 +1503,8 @@
	struct hwrm_func_qcfg_output *resp =
	(void *)softc->hwrm_cmd_resp.idi_vaddr;
	struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg;
	+ struct bnxt_hw_resc *hw_resc = &softc->hw_resc;
	+ struct bnxt_hw_tls_resc *tls_resc;

	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCFG);
	req.fid = htole16(0xffff);
	@@ -1519,6 +1571,10 @@
	if (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_ENABLE_RDMA_SRIOV)
	softc->fw_cap \|= BNXT_FW_CAP_ENABLE_RDMA_SRIOV;

	+ tls_resc = &hw_resc->tls_resc[BNXT_CRYPTO_TYPE_KTLS];
	+ tls_resc->resv_tx_key_ctxs = le32toh(resp->num_ktls_tx_key_ctxs);
	+ tls_resc->resv_rx_key_ctxs = le32toh(resp->num_ktls_rx_key_ctxs);
	+
	if (softc->db_size)
	goto end;

	@@ -2470,6 +2526,7 @@
	req.num_rx_rings = htole16(BNXT_MAX_NUM_QUEUES);
	req.num_vnics = htole16(BNXT_MAX_NUM_QUEUES);
	req.num_stat_ctxs = htole16(BNXT_MAX_NUM_QUEUES * 2);
	+ bnxt_hwrm_reserve_pf_key_ctxs(softc, &req, BNXT_CRYPTO_TYPE_KTLS);

	return hwrm_send_message(softc, &req, sizeof(req));
	}
	Index: sys/dev/bnxt/bnxt_en/bnxt_ktls.h
	===================================================================
	--- /dev/null
	+++ sys/dev/bnxt/bnxt_en/bnxt_ktls.h
	@@ -0,0 +1,414 @@
	+/*
	+ * Broadcom NetXtreme-C/E network driver.
	+ *
	+ * Copyright (c) 2026 Broadcom, All Rights Reserved.
	+ * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
	+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
	+ * THE POSSIBILITY OF SUCH DAMAGE.
	+ */
	+
	+#ifndef BNXT_KTLS_H
	+#define BNXT_KTLS_H
	+
	+#include <sys/ktls.h>
	+#include <sys/queue.h>
	+#include <sys/bitstring.h>
	+#include <sys/condvar.h>
	+#include <sys/taskqueue.h>
	+#include <machine/atomic.h>
	+
	+#include "bnxt.h"
	+#include "bnxt_mpc.h"
	+
	+#define BNXT_TX_CRYPTO_KEY_TYPE HWRM_FUNC_KEY_CTX_ALLOC_INPUT_KEY_CTX_TYPE_TX
	+#define BNXT_RX_CRYPTO_KEY_TYPE HWRM_FUNC_KEY_CTX_ALLOC_INPUT_KEY_CTX_TYPE_RX
	+
	+#define BNXT_PARTITION_CAP_BITS \
	+ (HWRM_FUNC_QCAPS_OUTPUT_XID_PARTITION_CAP_TX_CK \| \
	+ HWRM_FUNC_QCAPS_OUTPUT_XID_PARTITION_CAP_RX_CK)
	+
	+#define BNXT_PARTITION_CAP(resp) \
	+ ((le32toh((resp)->flags_ext2) & \
	+ HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_KEY_XID_PARTITION_SUPPORTED) && \
	+ ((le16toh((resp)->xid_partition_cap) & \
	+ BNXT_PARTITION_CAP_BITS) == BNXT_PARTITION_CAP_BITS))
	+
	+#define BNXT_KID_BATCH_SIZE 128
	+
	+
	+#define bnxt_u64_set(counter, val) \
	+ do { \
	+ counter_u64_zero(counter); \
	+ counter_u64_add(counter, val); \
	+ } while (0)
	+
	+#define BNXT_KTLS_ENTRIES_DEFAULT 102400
	+#define BNXT_KTLS_MAX_ENTRIES 512000
	+
	+struct bnxt_kid_info {
	+ TAILQ_ENTRY(bnxt_kid_info) list;
	+ uint32_t start_id;
	+ uint32_t count;
	+ bitstr_t *ids;
	+};
	+
	+struct bnxt_kctx {
	+ TAILQ_HEAD(, bnxt_kid_info) list;
	+ /* to serialize update to the linked list and total_alloc */
	+ struct mtx lock;
	+ struct cv alloc_pending_cv;
	+ uint8_t type;
	+ uint32_t total_alloc;
	+ uint32_t max_ctx;
	+ volatile uint32_t alloc_pending;
	+#define BNXT_KCTX_ALLOC_PENDING_MAX 8
	+ bitstr_t *partition_bmap;
	+ int partition_bmap_nbits; /* number of bits in partition_bmap */
	+ unsigned int next;
	+};
	+
	+#define BNXT_KCTX_ALLOC_OK(kctx) \
	+ (atomic_load_32(&((kctx)->alloc_pending)) < BNXT_KCTX_ALLOC_PENDING_MAX)
	+
	+#define BNXT_MAX_CRYPTO_KEY_TYPE (BNXT_RX_CRYPTO_KEY_TYPE + 1)
	+
	+enum bnxt_ktls_counters {
	+ BNXT_KTLS_TX_ADD = 0,
	+ BNXT_KTLS_TX_DEL,
	+ BNXT_KTLS_TX_HW_PKT,
	+ BNXT_KTLS_TX_FAILED,
	+ BNXT_KTLS_TX_OOO,
	+ BNXT_KTLS_TX_RETRANS,
	+ BNXT_KTLS_TX_REPLAY,
	+ BNXT_KTLS_TX_SEQ_FWD,
	+ BNXT_KTLS_TX_SEQ_FWD_TLS_HDR,
	+ BNXT_KTLS_TX_SEQ_FWD_SILENT_DROPS,
	+ BNXT_KTLS_TX_SEQ_FWD_REPLAY,
	+ BNXT_KTLS_TX_MBUF_ALLOCS,
	+ BNXT_KTLS_TX_MBUF_FREES,
	+ BNXT_KTLS_TX_MBUF_ERRORS,
	+ BNXT_KTLS_RX_ADD,
	+ BNXT_KTLS_RX_DEL,
	+ BNXT_KTLS_RX_HW_PKT,
	+ BNXT_KTLS_RX_SW_PKT,
	+ BNXT_KTLS_RX_RESYNC_REQ,
	+ BNXT_KTLS_RX_RESYNC_ACK,
	+ BNXT_KTLS_RX_RESYNC_DISCARD,
	+ BNXT_KTLS_RX_RESYNC_NAK,
	+ BNXT_KTLS_MAX_COUNTERS,
	+};
	+
	+enum bnxt_mpc_cmp_time_counters {
	+ BNXT_MPC_CMP_TIME_1US = 0,
	+ BNXT_MPC_CMP_TIME_1_5US,
	+ BNXT_MPC_CMP_TIME_5_10US,
	+ BNXT_MPC_CMP_TIME_10_15US,
	+ BNXT_MPC_CMP_TIME_15_20US,
	+ BNXT_MPC_CMP_TIME_20_25US,
	+ BNXT_MPC_CMP_TIME_25_50US,
	+ BNXT_MPC_CMP_TIME_50_100US,
	+ BNXT_MPC_CMP_TIME_100_200US,
	+ BNXT_MPC_CMP_TIME_200_500US,
	+ BNXT_MPC_CMP_TIME_500_1000US,
	+ BNXT_MPC_CMP_TIME_1000_1500US,
	+ BNXT_MPC_CMP_TIME_1500_2000US,
	+ BNXT_MPC_CMP_TIME_2000_2500US,
	+ BNXT_MPC_CMP_TIME_2500_3000US,
	+ BNXT_MPC_CMP_TIME_3000_3500US,
	+ BNXT_MPC_CMP_TIME_3500_4000US,
	+ BNXT_MPC_CMP_TIME_4000_4500US,
	+ BNXT_MPC_CMP_TIME_4500_5000US,
	+ BNXT_MPC_CMP_TIME_5000_PLUS_US,
	+ BNXT_MPC_CMP_TIME_MAX_COUNTERS,
	+};
	+
	+enum bnxt_ktls_err_counters {
	+ BNXT_KTLS_TX_DEV_ADD_FAILED = 0,
	+ BNXT_KTLS_TX_MPC_TIMEOUT,
	+ BNXT_KTLS_TX_UNINITIALIZED,
	+ BNXT_KTLS_TX_CTX_ALLOC_FAILED,
	+ BNXT_KTLS_TX_MPC_FAILED,
	+ BNXT_KTLS_TX_TCE_NOT_READY,
	+ BNXT_KTLS_TX_TCE_READY_DELAYED,
	+ BNXT_KTLS_TX_TCE_BAD,
	+ BNXT_KTLS_TX_UNSUPPORTED_TLS_HDR,
	+ BNXT_KTLS_TX_INVALID_TAG,
	+ BNXT_KTLS_TX_ZERO_TCP_PAYLEN,
	+ BNXT_KTLS_TX_INVALID_REC_SN,
	+ BNXT_KTLS_TX_TCE_FREE,
	+ BNXT_KTLS_ERR_MAX_COUNTERS,
	+};
	+
	+struct bnxt_tls_info
	+{
	+ uint16_t max_key_ctxs_alloc;
	+ uint16_t ctxs_per_partition;
	+ uint8_t partition_mode : 1;
	+
	+ struct bnxt_kctx kctx[BNXT_MAX_CRYPTO_KEY_TYPE];
	+
	+ uma_zone_t mpc_zone;
	+
	+ volatile uint32_t pending;
	+ volatile uint32_t snd_tag_alloc_ref; /* threads inside bnxt_tls_snd_tag_alloc() */
	+
	+#define BNXT_MAX_TLS_FILTER 460
	+
	+ bus_dma_tag_t dma_tag;
	+
	+ struct taskqueue *tq;
	+ uma_zone_t zone;
	+ uint32_t max_resources; /* max number of resources */
	+ volatile uint32_t num_resources; /* current number of resources */
	+ int init; /* set when ready */
	+ char zname[32];
	+
	+ counter_u64_t counters[BNXT_KTLS_MAX_COUNTERS];
	+ counter_u64_t err_counters[BNXT_KTLS_ERR_MAX_COUNTERS];
	+
	+ counter_u64_t mpc_cmp_time[BNXT_MPC_CMP_TIME_MAX_COUNTERS];
	+};
	+
	+#define tck kctx[BNXT_TX_CRYPTO_KEY_TYPE]
	+#define rck kctx[BNXT_RX_CRYPTO_KEY_TYPE]
	+
	+#define BNXT_TLS_TAG_LOCK(tag) mtx_lock(&(tag)->lock)
	+#define BNXT_TLS_TAG_UNLOCK(tag) mtx_unlock(&(tag)->lock)
	+
	+struct bnxt_replay_pkt {
	+ bus_dmamap_t dma_map;
	+ struct mbuf *mbuf;
	+};
	+
	+struct bnxt_ktls_offload_ctx_tx
	+{
	+ struct m_snd_tag tag;
	+ uint32_t tcp_seq_no;
	+ uint64_t rec_seq;
	+ uint32_t kid;
	+ struct mtx lock;
	+ uint8_t state;
	+#define BNXT_TLS_STATE_INIT 0
	+#define BNXT_TLS_STATE_READY 1
	+#define BNXT_TLS_STATE_BAD 2
	+#define BNXT_TLS_STATE_FREE 3
	+#define BNXT_TLS_STATE_FREED 4
	+ struct bnxt_tls_info *ktls;
	+ uint64_t inorder_rec_sn;
	+ uint32_t inorder_paylen;
	+ uint32_t inorder_seq;
	+ struct tls_session_params crypto_info;
	+ int enobufs;
	+#define MAX_MPC_RETRY 3
	+ int mpc_retry;
	+ struct timeval start;
	+ struct timeval end;
	+ struct task work_task;
	+};
	+
	+/* Resync timeout: 2500 ms in ticks */
	+#define BNXT_KTLS_RESYNC_TMO ((2500 * hz) / 1000)
	+#define BNXT_KTLS_MAX_RESYNC_BYTES 32768
	+
	+#define BNXT_KTLS_MAX_REPLAY_MSS 9000
	+
	+struct ce_add_cmd
	+{
	+ uint32_t ver_algo_kid_opcode;
	+#define CE_ADD_CMD_OPCODE_MASK 0xfUL
	+#define CE_ADD_CMD_OPCODE_SFT 0
	+#define CE_ADD_CMD_OPCODE_ADD 0x1UL
	+#define CE_ADD_CMD_KID_MASK 0xfffff0UL
	+#define CE_ADD_CMD_KID_SFT 4
	+#define CE_ADD_CMD_ALGORITHM_MASK 0xf000000UL
	+#define CE_ADD_CMD_ALGORITHM_SFT 24
	+#define CE_ADD_CMD_ALGORITHM_AES_GCM_128 0x1000000UL
	+#define CE_ADD_CMD_ALGORITHM_AES_GCM_256 0x2000000UL
	+#define CE_ADD_CMD_VERSION_MASK 0xf0000000UL
	+#define CE_ADD_CMD_VERSION_SFT 28
	+#define CE_ADD_CMD_VERSION_TLS1_2 (0x0UL << 28)
	+#define CE_ADD_CMD_VERSION_TLS1_3 (0x1UL << 28)
	+ uint8_t ctx_kind;
	+#define CE_ADD_CMD_CTX_KIND_MASK 0x1fUL
	+#define CE_ADD_CMD_CTX_KIND_SFT 0
	+#define CE_ADD_CMD_CTX_KIND_CK_TX 0x11UL
	+#define CE_ADD_CMD_CTX_KIND_CK_RX 0x12UL
	+ uint8_t unused0[3];
	+ uint8_t salt[4];
	+ uint8_t unused1[4];
	+ uint32_t pkt_tcp_seq_num;
	+ uint32_t tls_header_tcp_seq_num;
	+ uint8_t record_seq_num[8];
	+ uint8_t session_key[32];
	+ uint8_t addl_iv[8];
	+};
	+
	+struct ce_delete_cmd
	+{
	+ uint32_t ctx_kind_kid_opcode;
	+#define CE_DELETE_CMD_OPCODE_MASK 0xfUL
	+#define CE_DELETE_CMD_OPCODE_SFT 0
	+#define CE_DELETE_CMD_OPCODE_DEL 0x2UL
	+#define CE_DELETE_CMD_KID_MASK 0xfffff0UL
	+#define CE_DELETE_CMD_KID_SFT 4
	+#define CE_DELETE_CMD_CTX_KIND_MASK 0x1f000000UL
	+#define CE_DELETE_CMD_CTX_KIND_SFT 24
	+#define CE_DELETE_CMD_CTX_KIND_CK_TX (0x11UL << 24)
	+#define CE_DELETE_CMD_CTX_KIND_CK_RX (0x12UL << 24)
	+};
	+
	+struct ce_resync_resp_ack_cmd
	+{
	+ uint32_t resync_status_kid_opcode;
	+#define CE_RESYNC_RESP_ACK_CMD_OPCODE_MASK 0xfUL
	+#define CE_RESYNC_RESP_ACK_CMD_OPCODE_SFT 0
	+#define CE_RESYNC_RESP_ACK_CMD_OPCODE_RESYNC 0x3UL
	+#define CE_RESYNC_RESP_ACK_CMD_KID_MASK 0xfffff0UL
	+#define CE_RESYNC_RESP_ACK_CMD_KID_SFT 4
	+#define CE_RESYNC_RESP_ACK_CMD_RESYNC_STATUS 0x1000000UL
	+#define CE_RESYNC_RESP_ACK_CMD_RESYNC_STATUS_ACK (0x0UL << 24)
	+#define CE_RESYNC_RESP_ACK_CMD_RESYNC_STATUS_NAK (0x1UL << 24)
	+ uint32_t resync_record_tcp_seq_num;
	+ uint8_t resync_record_seq_num[8];
	+};
	+
	+#define resync_record_seq_num_end resync_record_seq_num[7]
	+
	+#define CE_CMD_KID_MASK 0xfffff0UL
	+#define CE_CMD_KID_SFT 4
	+
	+#define CE_CMD_KID(cmd_p) \
	+ (((uint32_t)(cmd_p)&CE_CMD_KID_MASK) >> CE_CMD_KID_SFT)
	+
	+#define BNXT_KMPC_OPAQUE(client, kid) (((client) << 24) \| (kid))
	+
	+#define BNXT_INV_KMPC_OPAQUE 0xffffffff
	+
	+struct ce_cmpl
	+{
	+ uint16_t client_subtype_type;
	+#define CE_CMPL_TYPE_MASK 0x3fUL
	+#define CE_CMPL_TYPE_SFT 0
	+#define CE_CMPL_TYPE_MID_PATH_SHORT 0x1eUL
	+#define CE_CMPL_SUBTYPE_MASK 0xf00UL
	+#define CE_CMPL_SUBTYPE_SFT 8
	+#define CE_CMPL_SUBTYPE_SOLICITED (0x0UL << 8)
	+#define CE_CMPL_SUBTYPE_ERR (0x1UL << 8)
	+#define CE_CMPL_SUBTYPE_RESYNC (0x2UL << 8)
	+#define CE_CMPL_MP_CLIENT_MASK 0xf000UL
	+#define CE_CMPL_MP_CLIENT_SFT 12
	+#define CE_CMPL_MP_CLIENT_TCE (0x0UL << 12)
	+#define CE_CMPL_MP_CLIENT_RCE (0x1UL << 12)
	+ uint16_t status;
	+#define CE_CMPL_STATUS_MASK 0xfUL
	+#define CE_CMPL_STATUS_SFT 0
	+#define CE_CMPL_STATUS_OK 0x0UL
	+#define CE_CMPL_STATUS_CTX_LD_ERR 0x1UL
	+#define CE_CMPL_STATUS_FID_CHK_ERR 0x2UL
	+#define CE_CMPL_STATUS_CTX_VER_ERR 0x3UL
	+#define CE_CMPL_STATUS_DST_ID_ERR 0x4UL
	+#define CE_CMPL_STATUS_MP_CMD_ERR 0x5UL
	+ uint32_t opaque;
	+ uint32_t v;
	+#define CE_CMPL_V 0x1UL
	+ uint32_t kid;
	+#define CE_CMPL_KID_MASK 0xfffffUL
	+#define CE_CMPL_KID_SFT 0
	+};
	+
	+#define CE_CMPL_STATUS(ce_cmpl) \
	+ (le16toh((ce_cmpl)->status) & CE_CMPL_STATUS_MASK)
	+
	+#define CE_CMPL_KID(ce_cmpl) (le32toh((ce_cmpl)->kid) & CE_CMPL_KID_MASK)
	+
	+#define BNXT_SALT_LEN 4
	+#define BNXT_ADDL_IV_LEN_TLS1_3 8
	+#define BNXT_EXPLICIT_NONCE_LEN 8
	+
	+struct crypto_prefix_cmd
	+{
	+ uint32_t flags;
	+#define CRYPTO_PREFIX_CMD_FLAGS_UPDATE_IN_ORDER_VAR 0x1UL
	+#define CRYPTO_PREFIX_CMD_FLAGS_FULL_REPLAY_RETRAN 0x2UL
	+ uint32_t header_tcp_seq_num;
	+ uint32_t start_tcp_seq_num;
	+ uint32_t end_tcp_seq_num;
	+ uint8_t explicit_nonce[8];
	+ uint8_t record_seq_num[8];
	+};
	+
	+#define CRYPTO_PREFIX_CMD_SIZE ((uint32_t)sizeof(struct crypto_prefix_cmd))
	+#define CRYPTO_PREFIX_CMD_BDS (CRYPTO_PREFIX_CMD_SIZE / sizeof(struct tx_bd))
	+#define CRYPTO_PRESYNC_BDS (CRYPTO_PREFIX_CMD_BDS + 1)
	+
	+#define CRYPTO_PRESYNC_BD_CMD \
	+ (htole32((CRYPTO_PREFIX_CMD_SIZE << TX_BD_LEN_SHIFT) \| \
	+ (CRYPTO_PRESYNC_BDS << TX_BD_FLAGS_BD_CNT_SHIFT) \| \
	+ TX_BD_TYPE_PRESYNC_TX_BD))
	+
	+#define BNXT_METADATA_OFF(len) ALIGN(len, 32)
	+
	+struct bnxt_crypto_cmd_ctx
	+{
	+ struct ce_cmpl ce_cmp;
	+ struct bnxt_ktls_offload_ctx_tx *kctx_tx;
	+};
	+
	+static inline bool before(uint32_t seq1, uint32_t seq2)
	+{
	+ return (int32_t)(seq1 - seq2) < 0;
	+}
	+#define after(seq2, seq1) before(seq1, seq2)
	+
	+static inline bool bnxt_ktls_busy(struct bnxt_softc* bp)
	+{
	+ return bp->ktls_info && atomic_load_32(&bp->ktls_info->pending) > 0;
	+}
	+
	+void bnxt_alloc_ktls_info(struct bnxt_softc* bp, uint16_t max_keys,
	+ bool partition_cap, uint16_t ctxs_per_partition);
	+void bnxt_free_ktls_info(struct bnxt_softc* bp);
	+void bnxt_free_ktls_counters(struct bnxt_softc* softc);
	+void bnxt_alloc_ktls_counters(struct bnxt_softc* softc);
	+void bnxt_hwrm_reserve_pf_key_ctxs(struct bnxt_softc* bp,
	+ struct hwrm_func_cfg_input* req,
	+ uint8_t type);
	+int bnxt_ktls_init(struct bnxt_softc* bp);
	+int bnxt_xmit_crypto_cmd(struct bnxt_softc* softc,
	+ void* cmd, uint len,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx);
	+int bnxt_key_ctx_alloc_one(struct bnxt_softc* bp, struct bnxt_kctx* kctx,
	+ uint32_t* id, uint8_t type);
	+void bnxt_free_one_kctx(struct bnxt_kctx* kctx, uint32_t id, device_t dev);
	+int bnxt_set_partition_mode(struct bnxt_softc* bp);
	+int bnxt_hwrm_key_ctx_alloc(struct bnxt_softc* bp, struct bnxt_kctx* kctx,
	+ uint16_t num, uint32_t* id, uint8_t type);
	+void bnxt_ktls_mpc_cmp(struct bnxt_softc* bp, uint32_t client,
	+ unsigned long handle, struct bnxt_cmpl_entry cmpl[],
	+ uint32_t entries);
	+void bnxt_ktls_del_all(struct bnxt_softc *bp);
	+void bnxt_clear_ktls(struct bnxt_softc *bp);
	+int
	+bnxt_alloc_ktls_rexmit_mbuf(struct bnxt_softc softc, struct bnxt_ring txr);
	+void
	+bnxt_free_ktls_rexmit_mbuf(struct bnxt_ring *txr);
	+#endif
	Index: sys/dev/bnxt/bnxt_en/bnxt_ktls.c
	===================================================================
	--- /dev/null
	+++ sys/dev/bnxt/bnxt_en/bnxt_ktls.c
	@@ -0,0 +1,2066 @@
	+/*
	+ * Broadcom NetXtreme-C/E network driver.
	+ *
	+ * Copyright (c) 2026 Broadcom, All Rights Reserved.
	+ * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
	+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
	+ * THE POSSIBILITY OF SUCH DAMAGE.
	+ */
	+
	+#include <sys/cdefs.h>
	+#include <sys/param.h>
	+#include <sys/stdatomic.h>
	+#include <sys/ktls.h>
	+#include <sys/malloc.h>
	+#include <sys/socket.h>
	+#include <sys/socketvar.h>
	+#include <sys/kernel.h>
	+#include <sys/counter.h>
	+#include <sys/kthread.h>
	+#include <sys/bitstring.h>
	+#include <sys/malloc.h>
	+#include <sys/queue.h>
	+#include <machine/atomic.h>
	+#include <vm/uma.h>
	+
	+#include <net/if.h>
	+#include <net/ethernet.h>
	+#include <net/pfil.h>
	+
	+#include <netinet/in.h>
	+#include <netinet/ip.h>
	+#include <netinet/udp.h>
	+#include <netinet/ip6.h>
	+#include <netinet/tcp.h>
	+#include <netinet/tcp_lro.h>
	+#include <netinet/in_pcb.h>
	+#include <netinet/tcp_var.h>
	+
	+#include "opt_kern_tls.h"
	+#include "opt_rss.h"
	+#include "opt_ratelimit.h"
	+
	+#include "bnxt.h"
	+#include "bnxt_hwrm.h"
	+#include "bnxt_ktls.h"
	+#include "bnxt_mpc.h"
	+
	+#include <opencrypto/cryptodev.h>
	+
	+#ifdef KERN_TLS
	+
	+static if_snd_tag_free_t bnxt_tls_snd_tag_free;
	+
	+static const struct if_snd_tag_sw bnxt_tls_snd_tag_sw = {
	+ .snd_tag_free = bnxt_tls_snd_tag_free,
	+ .type = IF_SND_TAG_TYPE_TLS
	+};
	+static int bnxt_ktls_dev_add(struct bnxt_softc* softc,
	+ const struct tls_session_params* en,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx);
	+static void bnxt_ktls_dev_del(struct bnxt_softc* softc,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx);
	+static int bnxt_crypto_del(struct bnxt_softc* softc,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx,
	+ uint8_t type);
	+static void bnxt_tls_work(void *context, int pending);
	+static inline void
	+bnxt_calc_mpc_cmp_time(struct bnxt_softc* softc, unsigned long rtt);
	+
	+static int
	+bnxt_tls_tag_import(void arg, void *store, int cnt, int domain, int flags)
	+{
	+ struct bnxt_ktls_offload_ctx_tx *kctx_tx;
	+ int i;
	+
	+ for (i = 0; i != cnt; i++) {
	+ kctx_tx = malloc(sizeof(*kctx_tx), M_DEVBUF, flags \| M_ZERO);
	+ if (kctx_tx == NULL)
	+ return (i);
	+ TASK_INIT(&kctx_tx->work_task, 0, bnxt_tls_work, kctx_tx);
	+ store[i] = kctx_tx;
	+ }
	+ return (i);
	+}
	+
	+static void
	+bnxt_tls_tag_release(void arg, void *store, int cnt)
	+{
	+ struct bnxt_ktls_offload_ctx_tx *kctx_tx;
	+ if_ctx_t ctx;
	+ struct bnxt_softc* priv;
	+ int i;
	+
	+ for (i = 0; i != cnt; i++) {
	+ kctx_tx = store[i];
	+ if (kctx_tx->tag.ifp) {
	+ ctx = if_getsoftc(kctx_tx->tag.ifp);
	+ priv = iflib_get_softc(ctx);
	+ if (priv->ktls_info->tq)
	+ taskqueue_drain(priv->ktls_info->tq, &kctx_tx->work_task);
	+ }
	+ free(kctx_tx, M_DEVBUF);
	+ }
	+}
	+
	+static void
	+bnxt_tls_tag_zfree(struct bnxt_ktls_offload_ctx_tx *kctx_tx)
	+{
	+ /* make sure any unhandled taskqueue events are ignored */
	+ kctx_tx->state = BNXT_TLS_STATE_FREED;
	+
	+ /* avoid leaking keys */
	+ memset(&kctx_tx->crypto_info, 0, sizeof(kctx_tx->crypto_info));
	+
	+ /* return tag to UMA */
	+ uma_zfree(kctx_tx->ktls->zone, kctx_tx);
	+}
	+
	+static void bnxt_tls_snd_tag_free(struct m_snd_tag* pmt)
	+{
	+ if_ctx_t ctx;
	+ struct bnxt_softc* priv;
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx =
	+ __containerof(pmt, struct bnxt_ktls_offload_ctx_tx, tag);
	+
	+ kctx_tx->state = BNXT_TLS_STATE_FREE;
	+
	+ ctx = if_getsoftc(kctx_tx->tag.ifp);
	+ priv = iflib_get_softc(ctx);
	+ bnxt_ktls_dev_del(priv, kctx_tx);
	+
	+ taskqueue_enqueue(priv->ktls_info->tq, &kctx_tx->work_task);
	+}
	+
	+static void
	+bnxt_tls_work(void *context, int pending)
	+{
	+ struct bnxt_ktls_offload_ctx_tx *kctx_tx = context;
	+
	+ if (kctx_tx->state == BNXT_TLS_STATE_FREE) {
	+ /* free tag */
	+ bnxt_tls_tag_zfree(kctx_tx);
	+ }
	+}
	+
	+int bnxt_tls_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params* params,
	+ struct m_snd_tag** ppmt)
	+{
	+ union if_snd_tag_alloc_params tag_params;
	+ const struct if_snd_tag_sw* snd_tag_sw;
	+ struct bnxt_softc* priv;
	+ struct bnxt_tls_info* ktls;
	+ if_ctx_t ctx;
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx = NULL;
	+ const struct tls_session_params* en;
	+ int error;
	+
	+ ctx = if_getsoftc(ifp);
	+ priv = iflib_get_softc(ctx);
	+ ktls = priv->ktls_info;
	+
	+ if (!test_bit(BNXT_STATE_OPEN, &priv->state))
	+ return (EPROTONOSUPPORT);
	+
	+ atomic_add_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
	+
	+ if (priv->ktls_info->init == 0) {
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_UNINITIALIZED], 1);
	+ atomic_subtract_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
	+ BNXT_DEBUG(priv->dev, "%s:%d: ifp:(%pa) ktls is not initialized\n",
	+ __func__, __LINE__, ifp);
	+ return (EPROTONOSUPPORT);
	+ }
	+
	+ /* allocate new tag from zone, if any */
	+ kctx_tx = uma_zalloc(priv->ktls_info->zone, M_NOWAIT);
	+ if (kctx_tx == NULL) {
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_CTX_ALLOC_FAILED], 1);
	+ atomic_subtract_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
	+ BNXT_DEBUG(priv->dev, "%s:%d: ifp:(%pa) kctx_tx uma_zalloc failed\n",
	+ __func__, __LINE__, ifp);
	+ return (ENOMEM);
	+ }
	+
	+ en = &params->tls.tls->params;
	+
	+ /* only TLS v1.2 and v1.3 is currently supported */
	+ if (en->tls_vmajor != TLS_MAJOR_VER_ONE \|\|
	+ (en->tls_vminor != TLS_MINOR_VER_TWO
	+#ifdef TLS_MINOR_VER_THREE
	+ && en->tls_vminor != TLS_MINOR_VER_THREE
	+#endif
	+ )) {
	+ BNXT_DEBUG(priv->dev, "kTLS snd_tag_alloc: unsupported TLS version %u.%u\n",
	+ en->tls_vmajor, en->tls_vminor);
	+ error = EPROTONOSUPPORT;
	+ goto failure;
	+ }
	+
	+ switch (en->cipher_algorithm) {
	+ case CRYPTO_AES_NIST_GCM_16:
	+ if ((en->cipher_key_len != 128 / 8) &&
	+ (en->cipher_key_len != 256 / 8)) {
	+ BNXT_DEBUG(priv->dev, "kTLS snd_tag_alloc: unsupported cipher key len %u\n",
	+ en->cipher_key_len * 8);
	+ error = EPROTONOSUPPORT;
	+ goto failure;
	+ }
	+ break;
	+
	+ default:
	+ BNXT_DEBUG(priv->dev, "kTLS snd_tag_alloc: unsupported cipher algorithm %u\n",
	+ en->cipher_algorithm);
	+ error = EPROTONOSUPPORT;
	+ goto failure;
	+ }
	+
	+ /* Fetch TCP seq number from struct tcpcb */
	+ kctx_tx->tcp_seq_no = intotcpcb(params->tls.tls->inp)->snd_nxt;
	+
	+ /*
	+ * Fetch record seq number from struct ktls_session.
	+ * So far 'next_seqno' provided TLS seq number for TLS v1.0 only.
	+ * It requires changes in the kernel to use 'next_seqno' for all
	+ * TLS versions as the driver needs TLS record sequence number
	+ * to setup a crypto connection in HW.
	+ */
	+#ifdef KTLS_REC_SEQ_NUMBER
	+ kctx_tx->rec_seq = params->tls.tls->next_seqno;
	+#endif
	+
	+ kctx_tx->state = BNXT_TLS_STATE_INIT;
	+
	+ error = bnxt_ktls_dev_add(priv, en, kctx_tx);
	+ if (error) {
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_DEV_ADD_FAILED], 1);
	+ error = EINVAL;
	+ goto failure;
	+ }
	+
	+ memset(&tag_params, 0, sizeof(tag_params));
	+ tag_params.hdr = params->hdr;
	+
	+ if (params->hdr.type == IF_SND_TAG_TYPE_TLS) {
	+ tag_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED;
	+ snd_tag_sw = &bnxt_tls_snd_tag_sw;
	+ } else {
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_UNSUPPORTED_TLS_HDR], 1);
	+ BNXT_DEBUG(priv->dev, "kTLS snd_tag_alloc: unsupported snd_tag type %u\n",
	+ params->hdr.type);
	+ bnxt_ktls_dev_del(priv, kctx_tx);
	+ error = EOPNOTSUPP;
	+ goto failure;
	+ }
	+
	+ MPASS(kctx_tx->tag.refcount == 0);
	+ m_snd_tag_init(&kctx_tx->tag, ifp, snd_tag_sw);
	+ *ppmt = &kctx_tx->tag;
	+ kctx_tx->ktls = priv->ktls_info;
	+ atomic_subtract_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
	+ BNXT_DEBUG(priv->dev, "[KID:%u]: TAG_ALLOC: tcp_seqno: %u"
	+ " record_no: %lu\n", kctx_tx->kid, kctx_tx->tcp_seq_no, kctx_tx->rec_seq);
	+
	+ return (0);
	+
	+failure:
	+ kctx_tx->state = BNXT_TLS_STATE_FREE;
	+ uma_zfree(priv->ktls_info->zone, kctx_tx);
	+ atomic_subtract_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
	+ return (error);
	+}
	+
	+void bnxt_alloc_ktls_info(struct bnxt_softc* softc, uint16_t max_keys,
	+ bool partition_cap, uint16_t ctxs_per_partition)
	+{
	+ struct bnxt_tls_info* ktls = softc->ktls_info;
	+
	+ if (BNXT_VF(softc))
	+ return;
	+
	+ if (!ktls) {
	+ bool partition_mode = false;
	+ struct bnxt_kctx* kctx;
	+ uint16_t batch_sz = 0;
	+ int i;
	+
	+ ktls = malloc(sizeof(*ktls), M_DEVBUF, M_NOWAIT \| M_ZERO);
	+
	+ if (ktls == NULL) {
	+ device_printf(softc->dev, "kTLS info alloc failed\n");
	+ return;
	+ }
	+
	+ if (partition_cap) {
	+ batch_sz = ctxs_per_partition;
	+ if (batch_sz && batch_sz <= BNXT_KID_BATCH_SIZE)
	+ partition_mode = true;
	+ }
	+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
	+ kctx = &ktls->kctx[i];
	+ kctx->type = i;
	+ kctx->max_ctx = softc->max_ktls_entries;
	+ TAILQ_INIT(&kctx->list);
	+ mtx_init(&kctx->lock, "BNXT KCTL LOCK", NULL,
	+ MTX_DEF \| MTX_NOWITNESS);
	+ cv_init(&kctx->alloc_pending_cv, "bnxt_ktls_alloc");
	+ atomic_store_32(&kctx->alloc_pending, 0);
	+ if (partition_mode) {
	+ int bmap_sz;
	+
	+ bmap_sz = howmany(kctx->max_ctx, batch_sz);
	+ kctx->partition_bmap_nbits = bmap_sz;
	+ kctx->partition_bmap =
	+ bit_alloc(bmap_sz, M_DEVBUF, M_NOWAIT);
	+ if (kctx->partition_bmap == NULL) {
	+ BNXT_DEBUG(softc->dev,
	+ "kTLS partition_bmap alloc failed, disabling partition mode\n");
	+ partition_mode = false;
	+ }
	+ }
	+ }
	+ atomic_store_32(&ktls->snd_tag_alloc_ref, 0);
	+ ktls->partition_mode = partition_mode;
	+ ktls->ctxs_per_partition = batch_sz;
	+
	+ atomic_store_32(&ktls->pending, 0);
	+
	+ softc->ktls_info = ktls;
	+
	+ /* Create DMA descriptor TAG */
	+ if ((bus_dma_tag_create(
	+ bus_get_dma_tag(softc->dev),
	+ 1, /* any alignment */
	+ 0, /* no boundary */
	+ BUS_SPACE_MAXADDR, /* lowaddr */
	+ BUS_SPACE_MAXADDR, /* highaddr */
	+ NULL, NULL, /* filter, filterarg */
	+ 65536,
	+ BNXT_MAX_NUM_SEGS,
	+ 16384,
	+ 0, /* flags */
	+ NULL, NULL, /* lockfunc, lockfuncarg */
	+ &ktls->dma_tag)))
	+ device_printf(softc->dev, "kTLS dma tag alloc failed\n");
	+ }
	+ ktls->max_key_ctxs_alloc = max_keys;
	+}
	+
	+void bnxt_ktls_del_all(struct bnxt_softc *softc)
	+{
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ struct bnxt_kid_info *kid;
	+ struct bnxt_kctx *kctx;
	+ int i, j;
	+
	+ if (!ktls)
	+ return;
	+
	+ /*
	+ * Called during FW reset (with BNXT_STATE_IN_FW_RESET already set)
	+ * before bnxt_clear_ktls() tears down the software state. The FW
	+ * reinitialises its key-context table during recovery, and
	+ * FreeBSD's async MPC path cannot safely wait for completions here
	+ * (pci_disable_busmaster() follows shortly), so there is no crypto
	+ * delete to send: every in-use KID is simply marked free in
	+ * software. bnxt_crypto_del() would no-op here anyway, since
	+ * BNXT_STATE_IN_FW_RESET is always set on this path, but skip
	+ * calling it rather than build a stack-local dummy kctx_tx just to
	+ * feed a no-op.
	+ */
	+ /* Shutting down; no need to hold kctx locks. */
	+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
	+ kctx = &ktls->kctx[i];
	+ TAILQ_FOREACH(kid, &kctx->list, list) {
	+ for (j = 0; j < (int)kid->count; j++)
	+ bit_set(kid->ids, j);
	+ }
	+ }
	+}
	+
	+void bnxt_clear_ktls(struct bnxt_softc *softc)
	+{
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ struct bnxt_kid_info kid, tmp;
	+ struct bnxt_kctx *kctx;
	+ int i;
	+
	+ if (!ktls)
	+ return;
	+
	+ /* Shutting down or FW reset, no need to protect the lists. */
	+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
	+ kctx = &ktls->kctx[i];
	+ TAILQ_FOREACH_SAFE(kid, &kctx->list, list, tmp) {
	+ TAILQ_REMOVE(&kctx->list, kid, list);
	+ if (kid->ids != NULL)
	+ free(kid->ids, M_DEVBUF);
	+ free(kid, M_DEVBUF);
	+ }
	+ if (kctx->partition_bmap != NULL) {
	+ free(kctx->partition_bmap, M_DEVBUF);
	+ kctx->partition_bmap = NULL;
	+ }
	+ kctx->total_alloc = 0;
	+ }
	+}
	+
	+void bnxt_free_ktls_counters(struct bnxt_softc* softc)
	+{
	+ uint8_t i;
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+
	+ if (!ktls)
	+ return;
	+
	+ for (i = 0; i < BNXT_MPC_CMP_TIME_MAX_COUNTERS; i++) {
	+ if (ktls->mpc_cmp_time[i]) {
	+ counter_u64_free(ktls->mpc_cmp_time[i]);
	+ ktls->mpc_cmp_time[i] = NULL;
	+ }
	+ }
	+
	+ for (i = 0; i < BNXT_KTLS_MAX_COUNTERS; i++) {
	+ if (ktls->counters[i]) {
	+ counter_u64_free(ktls->counters[i]);
	+ ktls->counters[i] = NULL;
	+ }
	+ }
	+
	+ for (i = 0; i < BNXT_KTLS_ERR_MAX_COUNTERS; i++) {
	+ if (ktls->err_counters[i]) {
	+ counter_u64_free(ktls->err_counters[i]);
	+ ktls->err_counters[i] = NULL;
	+ }
	+ }
	+}
	+
	+void bnxt_alloc_ktls_counters(struct bnxt_softc* softc)
	+{
	+ uint8_t i;
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+
	+ if (!ktls)
	+ return;
	+
	+ for (i = 0; i < BNXT_MPC_CMP_TIME_MAX_COUNTERS; i++) {
	+ if (!ktls->mpc_cmp_time[i])
	+ ktls->mpc_cmp_time[i] = counter_u64_alloc(M_WAITOK);
	+ }
	+
	+ for (i = 0; i < BNXT_KTLS_MAX_COUNTERS; i++) {
	+ if (!ktls->counters[i])
	+ ktls->counters[i] = counter_u64_alloc(M_WAITOK);
	+ }
	+
	+ for (i = 0; i < BNXT_KTLS_ERR_MAX_COUNTERS; i++) {
	+ if (!ktls->err_counters[i])
	+ ktls->err_counters[i] = counter_u64_alloc(M_WAITOK);
	+ }
	+
	+}
	+
	+void bnxt_free_ktls_info(struct bnxt_softc* softc)
	+{
	+ struct bnxt_tls_info* ktls = softc->ktls_info;
	+ int i;
	+
	+ if (!ktls)
	+ return;
	+
	+ bnxt_clear_ktls(softc);
	+ bus_dma_tag_destroy(ktls->dma_tag);
	+ if (ktls->mpc_zone != NULL)
	+ uma_zdestroy(ktls->mpc_zone);
	+
	+ if (ktls->init) {
	+ ktls->init = 0;
	+ if (ktls->tq != NULL) {
	+ taskqueue_quiesce(ktls->tq);
	+ taskqueue_free(ktls->tq);
	+ ktls->tq = NULL;
	+ }
	+ uma_zdestroy(ktls->zone);
	+ }
	+
	+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
	+ cv_destroy(&ktls->kctx[i].alloc_pending_cv);
	+ if (mtx_initialized(&ktls->kctx[i].lock))
	+ mtx_destroy(&ktls->kctx[i].lock);
	+ }
	+
	+ free(ktls, M_DEVBUF);
	+ softc->ktls_info = NULL;
	+}
	+
	+void bnxt_hwrm_reserve_pf_key_ctxs(struct bnxt_softc* softc,
	+ struct hwrm_func_cfg_input* req,
	+ uint8_t type)
	+{
	+ struct bnxt_tls_info* tls = softc->ktls_info;
	+ struct bnxt_hw_resc* hw_resc = &softc->hw_resc;
	+ struct bnxt_hw_tls_resc* tls_resc;
	+ uint32_t tx, rx;
	+
	+ if (!tls)
	+ return;
	+
	+ tls_resc = &hw_resc->tls_resc[type];
	+ tx = min(tls->tck.max_ctx, tls_resc->max_tx_key_ctxs);
	+ rx = min(tls->rck.max_ctx, tls_resc->max_rx_key_ctxs);
	+ if (type == BNXT_CRYPTO_TYPE_KTLS) {
	+ req->num_ktls_tx_key_ctxs = htole32(tx);
	+ req->num_ktls_rx_key_ctxs = htole32(rx);
	+ req->enables \|= htole32(
	+ HWRM_FUNC_CFG_INPUT_ENABLES_KTLS_TX_KEY_CTXS \|
	+ HWRM_FUNC_CFG_INPUT_ENABLES_KTLS_RX_KEY_CTXS);
	+ }
	+}
	+
	+static int __bnxt_partition_alloc(struct bnxt_kctx* kctx, uint32_t* id)
	+{
	+ int max = kctx->partition_bmap_nbits;
	+ ssize_t next;
	+
	+ bit_ffc_at(kctx->partition_bmap, kctx->next, max, &next);
	+ if (next == -1)
	+ bit_ffc(kctx->partition_bmap, max, &next);
	+ if (next == -1)
	+ return ENOSPC;
	+ *id = next;
	+ kctx->next = next;
	+ return 0;
	+}
	+
	+static int bnxt_partition_alloc(struct bnxt_kctx* kctx, uint32_t* id)
	+{
	+ int rc;
	+
	+ mtx_lock(&kctx->lock);
	+ do {
	+ rc = __bnxt_partition_alloc(kctx, id);
	+ if (rc) {
	+ mtx_unlock(&kctx->lock);
	+ return rc;
	+ }
	+ } while (bnxt_test_and_set_bit_nonatomic(*id, kctx->partition_bmap));
	+ mtx_unlock(&kctx->lock);
	+ return 0;
	+}
	+
	+static int bnxt_key_ctx_store(struct bnxt_softc* softc, __le32* key_buf,
	+ uint32_t num, bool contig, struct bnxt_kctx* kctx,
	+ uint32_t* id)
	+{
	+ struct bnxt_kid_info* kid;
	+ uint32_t i;
	+ int j;
	+
	+ for (i = 0; i < num;) {
	+ kid = malloc(sizeof(*kid), M_DEVBUF, M_NOWAIT \| M_ZERO);
	+ if (kid == NULL) {
	+ device_printf(softc->dev, "kTLS kid alloc failed\n");
	+ return ENOMEM;
	+ }
	+ kid->ids = bit_alloc(BNXT_KID_BATCH_SIZE, M_DEVBUF, M_NOWAIT);
	+ if (kid->ids == NULL) {
	+ free(kid, M_DEVBUF);
	+ device_printf(softc->dev, "kTLS kid ids alloc failed\n");
	+ return ENOMEM;
	+ }
	+ kid->start_id = le32toh(key_buf[i]);
	+ if (contig)
	+ kid->count = (num > BNXT_KID_BATCH_SIZE) ? BNXT_KID_BATCH_SIZE : num;
	+ else
	+ kid->count = 1;
	+ for (j = 0; j < kid->count; j++)
	+ bit_set(kid->ids, j);
	+ if (id != NULL && i == 0) {
	+ bit_clear(kid->ids, 0);
	+ *id = kid->start_id;
	+ }
	+ mtx_lock(&kctx->lock);
	+ TAILQ_INSERT_TAIL(&kctx->list, kid, list);
	+ kctx->total_alloc += kid->count;
	+ mtx_unlock(&kctx->lock);
	+ i += kid->count;
	+ }
	+ return 0;
	+}
	+
	+int bnxt_hwrm_key_ctx_alloc(struct bnxt_softc* softc, struct bnxt_kctx* kctx,
	+ uint16_t num, uint32_t* id, uint8_t type)
	+{
	+ struct bnxt_tls_info* tls = softc->ktls_info;
	+ struct hwrm_func_key_ctx_alloc_output* resp =
	+ (void*)softc->hwrm_cmd_resp.idi_vaddr;
	+ struct hwrm_func_key_ctx_alloc_input req = {0};
	+ int pending_count;
	+ __le32* key_buf;
	+ bool contig;
	+ int rc;
	+ struct iflib_dma_info dma_data = {};
	+ bool dma_valid = false;
	+ uint32_t partition_id = 0;
	+ bool partition_allocated = false;
	+
	+ num = min(num, tls->max_key_ctxs_alloc);
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_KEY_CTX_ALLOC);
	+
	+ if (tls->partition_mode) {
	+ num = tls->ctxs_per_partition;
	+ rc = bnxt_partition_alloc(kctx, &partition_id);
	+ if (rc)
	+ return rc;
	+ partition_allocated = true;
	+ req.partition_start_xid = htole32(partition_id * num);
	+ } else {
	+ rc = iflib_dma_alloc(softc->ctx, num * 4, &dma_data,
	+ BUS_DMA_NOWAIT);
	+ if (rc) {
	+ device_printf(softc->dev, "kTLS key ctx DMA alloc failed, rc=%d\n", rc);
	+ return rc;
	+ }
	+ dma_valid = true;
	+ bus_dmamap_sync(dma_data.idi_tag, dma_data.idi_map,
	+ BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	+ key_buf = (__le32*)dma_data.idi_vaddr;
	+ if (!key_buf) {
	+ device_printf(softc->dev, "kTLS key ctx DMA vaddr NULL after alloc\n");
	+ iflib_dma_free(&dma_data);
	+ dma_valid = false;
	+ return ENOMEM;
	+ }
	+ req.dma_bufr_size_bytes = htole32(num * 4);
	+ req.host_dma_addr = htole64(dma_data.idi_paddr);
	+ }
	+
	+ req.key_ctx_type = kctx->type;
	+ req.num_key_ctxs = htole16(num);
	+
	+ pending_count = atomic_fetchadd_32(&kctx->alloc_pending, 1) + 1;
	+ BNXT_HWRM_LOCK(softc);
	+ rc = _hwrm_send_message(softc, &req, sizeof(req));
	+ atomic_subtract_32(&kctx->alloc_pending, 1);
	+ if (rc) {
	+ device_printf(softc->dev, "kTLS HWRM key ctx alloc failed, rc=%d\n", rc);
	+ goto key_alloc_exit_wake;
	+ }
	+
	+ num = le16toh(resp->num_key_ctxs_allocated);
	+ contig = resp->flags &
	+ HWRM_FUNC_KEY_CTX_ALLOC_OUTPUT_FLAGS_KEY_CTXS_CONTIGUOUS;
	+ if (tls->partition_mode)
	+ key_buf = &resp->partition_start_xid;
	+ rc = bnxt_key_ctx_store(softc, key_buf, num, contig, kctx, id);
	+ if (rc)
	+ BNXT_DEBUG(softc->dev, "kTLS key_ctx_store failed, rc=%d\n", rc);
	+
	+key_alloc_exit_wake:
	+ /*
	+ * Lock ordering: BNXT_HWRM_LOCK (outer) -> kctx->lock (inner).
	+ * No code path may acquire kctx->lock before calling a HWRM function,
	+ * as that would invert this ordering and risk deadlock.
	+ */
	+ if (rc && partition_allocated) {
	+ mtx_lock(&kctx->lock);
	+ bit_clear(kctx->partition_bmap, partition_id);
	+ mtx_unlock(&kctx->lock);
	+ }
	+
	+ if (pending_count >= BNXT_KCTX_ALLOC_PENDING_MAX) {
	+ mtx_lock(&kctx->lock);
	+ cv_broadcast(&kctx->alloc_pending_cv);
	+ mtx_unlock(&kctx->lock);
	+ }
	+
	+ BNXT_HWRM_UNLOCK(softc);
	+ if (dma_valid)
	+ iflib_dma_free(&dma_data);
	+ return rc;
	+}
	+
	+static int bnxt_alloc_one_kctx(struct bnxt_kctx* kctx, uint32_t* id)
	+{
	+ struct bnxt_kid_info* kid;
	+ int rc = ENOMEM;
	+
	+ mtx_lock(&kctx->lock);
	+ TAILQ_FOREACH(kid, &kctx->list, list) {
	+ ssize_t idx = 0;
	+
	+ do {
	+ bit_ffs_at(kid->ids, idx, kid->count, &idx);
	+ if (idx == -1)
	+ break;
	+ if (bnxt_test_and_clear_bit_nonatomic(idx, kid->ids)) {
	+ *id = kid->start_id + idx;
	+ rc = 0;
	+ goto alloc_done;
	+ }
	+ } while (1);
	+ }
	+
	+alloc_done:
	+ mtx_unlock(&kctx->lock);
	+ return (rc);
	+}
	+
	+void bnxt_free_one_kctx(struct bnxt_kctx* kctx, uint32_t id, device_t dev)
	+{
	+ struct bnxt_kid_info* kid;
	+
	+ mtx_lock(&kctx->lock);
	+ TAILQ_FOREACH(kid, &kctx->list, list) {
	+ if (id >= kid->start_id && id < kid->start_id + kid->count) {
	+ bit_set(kid->ids, id - kid->start_id);
	+ break;
	+ }
	+ }
	+ mtx_unlock(&kctx->lock);
	+}
	+
	+#define BNXT_KCTX_ALLOC_RETRY_MAX 3
	+
	+int bnxt_key_ctx_alloc_one(struct bnxt_softc* softc, struct bnxt_kctx* kctx,
	+ uint32_t* id, uint8_t type)
	+{
	+ int rc, retry = 0;
	+
	+ while (retry++ < BNXT_KCTX_ALLOC_RETRY_MAX) {
	+ rc = bnxt_alloc_one_kctx(kctx, id);
	+ if (!rc)
	+ return 0;
	+
	+ if ((kctx->total_alloc + BNXT_KID_BATCH_SIZE) > kctx->max_ctx) {
	+ BNXT_DEBUG(softc->dev, "kTLS key_ctx_alloc_one: max_ctx reached\n");
	+ return ENOSPC;
	+ }
	+
	+ if (!BNXT_KCTX_ALLOC_OK(kctx)) {
	+ mtx_lock(&kctx->lock);
	+ while (!BNXT_KCTX_ALLOC_OK(kctx))
	+ cv_wait(&kctx->alloc_pending_cv, &kctx->lock);
	+ mtx_unlock(&kctx->lock);
	+ continue;
	+ }
	+ rc = bnxt_hwrm_key_ctx_alloc(softc, kctx, BNXT_KID_BATCH_SIZE,
	+ id, type);
	+ if (!rc)
	+ return 0;
	+ }
	+ BNXT_DEBUG(softc->dev, "kTLS key_ctx_alloc_one: retries exhausted\n");
	+ return EAGAIN;
	+}
	+
	+static inline void
	+bnxt_calc_mpc_cmp_time(struct bnxt_softc* softc, unsigned long rtt)
	+{
	+ struct bnxt_tls_info* ktls = softc->ktls_info;
	+
	+ if (rtt <= 1)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_1US], 1);
	+ else if (rtt <= 5)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_1_5US], 1);
	+ else if (rtt <= 10)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_5_10US], 1);
	+ else if (rtt <= 15)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_10_15US], 1);
	+ else if (rtt <= 20)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_15_20US], 1);
	+ else if (rtt <= 25)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_20_25US], 1);
	+ else if (rtt <= 50)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_25_50US], 1);
	+ else if (rtt <= 100)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_50_100US], 1);
	+ else if (rtt <= 200)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_100_200US], 1);
	+ else if (rtt <= 500)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_200_500US], 1);
	+ else if (rtt <= 1000)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_500_1000US], 1);
	+ else if (rtt <= 1500)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_1000_1500US], 1);
	+ else if (rtt <= 2000)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_1500_2000US], 1);
	+ else if (rtt <= 2500)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_2000_2500US], 1);
	+ else if (rtt <= 3000)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_2500_3000US], 1);
	+ else if (rtt <= 3500)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_3000_3500US], 1);
	+ else if (rtt <= 4000)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_3500_4000US], 1);
	+ else if (rtt <= 4500)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_4000_4500US], 1);
	+ else if (rtt <= 5000)
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_4500_5000US], 1);
	+ else
	+ counter_u64_add(ktls->mpc_cmp_time[BNXT_MPC_CMP_TIME_5000_PLUS_US], 1);
	+}
	+
	+
	+int bnxt_xmit_crypto_cmd(struct bnxt_softc* softc,
	+ void* cmd, uint len,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx)
	+{
	+ struct bnxt_mpc_info* mpc = softc->mpc_info;
	+ struct bnxt_tls_info* ktls = softc->ktls_info;
	+ struct bnxt_crypto_cmd_ctx* ctx = NULL;
	+ unsigned long handle = 0;
	+ struct bnxt_ring* txr;
	+ int rc;
	+
	+ txr = bnxt_select_mpc_ring(mpc);
	+
	+ uint32_t kid = CE_CMD_KID(cmd);
	+
	+ ctx = uma_zalloc(ktls->mpc_zone, M_NOWAIT);
	+ if (ctx == NULL) {
	+ device_printf(softc->dev, "kTLS MPC ctx alloc failed\n");
	+ return ENOMEM;
	+ }
	+
	+ handle = (unsigned long)ctx;
	+ ctx->ce_cmp.opaque = BNXT_KMPC_OPAQUE(
	+ txr->mpc_chnl_type, kid);
	+
	+ ctx->kctx_tx = kctx_tx;
	+
	+ getmicrotime(&kctx_tx->start);
	+
	+ mtx_lock_spin(&txr->tx_lock);
	+ rc = bnxt_start_xmit_mpc(softc, txr, cmd, len, handle);
	+ mtx_unlock_spin(&txr->tx_lock);
	+ if (rc) {
	+ BNXT_DEBUG(softc->dev, "kTLS MPC xmit failed, kid=%u rc=%d\n", kid, rc);
	+ uma_zfree(ktls->mpc_zone, ctx);
	+ }
	+
	+ return rc;
	+}
	+
	+static int bnxt_crypto_add(struct bnxt_softc* softc,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx,
	+ uint8_t type)
	+{
	+ struct tls_session_params* crypto_info = &kctx_tx->crypto_info;
	+ uint32_t tcp_seq_no = kctx_tx->tcp_seq_no;
	+ uint64_t rec_seq = kctx_tx->rec_seq;
	+ uint32_t kid = kctx_tx->kid;
	+ struct ce_add_cmd cmd = {0};
	+ uint32_t data;
	+ int rc;
	+
	+ cmd.ctx_kind = CE_ADD_CMD_CTX_KIND_CK_TX;
	+
	+ data = CE_ADD_CMD_OPCODE_ADD \| (kid << CE_ADD_CMD_KID_SFT);
	+ switch (crypto_info->cipher_key_len) {
	+ case 128 / 8:
	+ data \|= CE_ADD_CMD_ALGORITHM_AES_GCM_128;
	+#ifdef TLS_MINOR_VER_THREE
	+ if (crypto_info->tls_vminor == TLS_MINOR_VER_THREE) {
	+ data \|= CE_ADD_CMD_VERSION_TLS1_3;
	+ memcpy(&cmd.addl_iv, crypto_info->iv + BNXT_SALT_LEN,
	+ BNXT_ADDL_IV_LEN_TLS1_3);
	+ }
	+#endif
	+ memcpy(&cmd.session_key, crypto_info->cipher_key,
	+ crypto_info->cipher_key_len);
	+ memcpy(&cmd.salt, crypto_info->iv, BNXT_SALT_LEN);
	+ memcpy(&cmd.record_seq_num[0], &rec_seq, sizeof(rec_seq));
	+ break;
	+ case 256 / 8:
	+ data \|= CE_ADD_CMD_ALGORITHM_AES_GCM_256;
	+#ifdef TLS_MINOR_VER_THREE
	+ if (crypto_info->tls_vminor == TLS_MINOR_VER_THREE) {
	+ data \|= CE_ADD_CMD_VERSION_TLS1_3;
	+ memcpy(&cmd.addl_iv, crypto_info->iv + BNXT_SALT_LEN,
	+ BNXT_ADDL_IV_LEN_TLS1_3);
	+ }
	+#endif
	+ memcpy(&cmd.session_key, crypto_info->cipher_key,
	+ crypto_info->cipher_key_len);
	+ memcpy(&cmd.salt, crypto_info->iv, BNXT_SALT_LEN);
	+ memcpy(&cmd.record_seq_num[0], &rec_seq, sizeof(rec_seq));
	+ break;
	+ }
	+ cmd.ver_algo_kid_opcode = htole32(data);
	+ cmd.pkt_tcp_seq_num = htole32(tcp_seq_no);
	+ cmd.tls_header_tcp_seq_num = cmd.pkt_tcp_seq_num;
	+ rc = bnxt_xmit_crypto_cmd(softc, &cmd, sizeof(cmd), kctx_tx);
	+ explicit_bzero(&cmd, sizeof(cmd));
	+ return rc;
	+}
	+
	+static int bnxt_crypto_del(struct bnxt_softc* softc,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx, uint8_t type)
	+{
	+ struct ce_delete_cmd cmd = {0};
	+ uint32_t data;
	+ uint32_t kid;
	+
	+ if (test_bit(BNXT_STATE_IN_FW_RESET, &softc->state) &&
	+ test_bit(BNXT_STATE_FW_FATAL_COND, &softc->state))
	+ return 0;
	+
	+ kid = kctx_tx->kid;
	+ data = CE_DELETE_CMD_CTX_KIND_CK_TX;
	+
	+ data \|= CE_DELETE_CMD_OPCODE_DEL \| (kid << CE_DELETE_CMD_KID_SFT);
	+
	+ cmd.ctx_kind_kid_opcode = htole32(data);
	+ return bnxt_xmit_crypto_cmd(softc, &cmd, sizeof(cmd),
	+ kctx_tx);
	+}
	+
	+static int bnxt_ktls_dev_add(struct bnxt_softc* softc,
	+ const struct tls_session_params* crypto_info,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx)
	+{
	+ struct bnxt_tls_info* ktls;
	+ struct bnxt_kctx* kctx;
	+ uint32_t kid;
	+ int rc = 0;
	+
	+ ktls = softc->ktls_info;
	+ atomic_add_32(&ktls->pending, 1);
	+
	+ atomic_thread_fence_seq_cst();
	+ if (!test_bit(BNXT_STATE_OPEN, &softc->state)) {
	+ BNXT_DEBUG(softc->dev, "kTLS dev_add: device not open\n");
	+ rc = ENODEV;
	+ goto exit;
	+ }
	+
	+ memcpy(&kctx_tx->crypto_info, crypto_info, sizeof(*crypto_info));
	+
	+ kctx = &ktls->tck;
	+ rc = bnxt_key_ctx_alloc_one(softc, kctx, &kid, BNXT_CRYPTO_TYPE_KTLS);
	+ if (rc) {
	+ BNXT_DEBUG(softc->dev, "kTLS dev_add: key_ctx_alloc_one failed, rc=%d\n", rc);
	+ goto exit;
	+ }
	+
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_ADD], 1);
	+
	+ BNXT_DEBUG(softc->dev, "[KID:%d] KTLS device add is successful\n", kid);
	+
	+ kctx_tx->kid = kid;
	+ kctx_tx->enobufs = 0;
	+
	+ rc = bnxt_crypto_add(softc, kctx_tx, BNXT_CRYPTO_TYPE_KTLS);
	+
	+ if (rc) {
	+ device_printf(softc->dev, "kTLS crypto add failed for KID %u, rc=%d\n", kid, rc);
	+ bnxt_free_one_kctx(kctx, kid, softc->dev);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_DEL], 1);
	+ goto exit;
	+ }
	+
	+exit:
	+ atomic_subtract_32(&ktls->pending, 1);
	+ return rc;
	+}
	+
	+#define BNXT_RETRY_MAX 20
	+
	+static void bnxt_ktls_dev_del(struct bnxt_softc* softc,
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx)
	+{
	+ struct bnxt_tls_info* ktls;
	+ struct bnxt_kctx* kctx;
	+ int retry_cnt = 0;
	+ uint32_t kid;
	+ int rc;
	+
	+ ktls = softc->ktls_info;
	+ kid = kctx_tx->kid;
	+ kctx = &ktls->tck;
	+
	+ /* BNXT_STATE_OPEN bit is cleared in bnxt_stop() and where
	+ * all the rings (including MPC rings) are freed with fw.
	+ * So, return from here when BNXT_STATE_OPEN is cleared
	+ * otherwise MPC command timeouts will be observed while
	+ * unloading the driver.
	+ */
	+ if (!test_bit(BNXT_STATE_OPEN, &softc->state)) {
	+ BNXT_DEBUG(softc->dev, "[KID:%u]: KTLS dev_del: device not up, freeing KID\n", kid);
	+ bnxt_free_one_kctx(kctx, kid, softc->dev);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_DEL], 1);
	+ return;
	+ }
	+
	+retry:
	+ atomic_add_32(&ktls->pending, 1);
	+ /* Make sure bnxt_close_nic() sees pending before we check the
	+ * BNXT_STATE_OPEN flag.
	+ */
	+ atomic_thread_fence_seq_cst();
	+ while (!test_bit(BNXT_STATE_OPEN, &softc->state)) {
	+ atomic_subtract_32(&ktls->pending, 1);
	+ if (retry_cnt > BNXT_RETRY_MAX) {
	+ device_printf(softc->dev,
	+ "%s retry max %d exceeded, state %lx\n",
	+ __func__, retry_cnt, softc->state);
	+ bnxt_free_one_kctx(kctx, kid, softc->dev);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_DEL], 1);
	+ return;
	+ }
	+ retry_cnt++;
	+ DELAY(1000 * 100);
	+ goto retry;
	+ }
	+
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_DEL], 1);
	+ rc = bnxt_crypto_del(softc, kctx_tx, BNXT_CRYPTO_TYPE_KTLS);
	+ if (!rc) {
	+ bnxt_free_one_kctx(kctx, kid, softc->dev);
	+ } else {
	+ device_printf(softc->dev, "kTLS crypto delete failed for KID %u, rc=%d\n", kid, rc);
	+ }
	+
	+ atomic_subtract_32(&ktls->pending, 1);
	+ BNXT_DEBUG(softc->dev, "[KID:%d]: KTLS device delete is successful\n", kid);
	+}
	+
	+int bnxt_set_partition_mode(struct bnxt_softc* softc)
	+{
	+ struct hwrm_func_cfg_input req = {0};
	+ int rc;
	+
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG);
	+ req.fid = htole16(0xffff);
	+ req.enables2 =
	+ htole32(HWRM_FUNC_CFG_INPUT_ENABLES2_XID_PARTITION_CFG);
	+ req.xid_partition_cfg =
	+ htole16(HWRM_FUNC_CFG_INPUT_XID_PARTITION_CFG_TX_CK \|
	+ HWRM_FUNC_CFG_INPUT_XID_PARTITION_CFG_RX_CK);
	+ rc = hwrm_send_message(softc, &req, sizeof(req));
	+ if (rc)
	+ device_printf(softc->dev, "kTLS set partition mode failed, rc=%d\n", rc);
	+ return rc;
	+}
	+
	+static int bnxt_pre_alloc_tx_crypto_keys(struct bnxt_softc* softc)
	+{
	+ struct bnxt_tls_info* ktls = softc->ktls_info;
	+ uint32_t alloc_keys = softc->max_ktls_entries;
	+ uint32_t allocated_keys_count = 0;
	+ uint16_t batch_sz;
	+ int rc = 0;
	+ unsigned long start_ticks, elapsed_ticks;
	+
	+ start_ticks = ticks;
	+ while (alloc_keys > 0) {
	+ batch_sz = (uint16_t)min(alloc_keys, (uint32_t)BNXT_KID_BATCH_SIZE);
	+
	+ rc = bnxt_hwrm_key_ctx_alloc(softc, &ktls->tck, batch_sz,
	+ NULL, BNXT_CRYPTO_TYPE_KTLS);
	+
	+
	+ if (rc) {
	+ if (!allocated_keys_count) {
	+ device_printf(softc->dev,
	+ "kTLS Tx keys allocation is failed, disable the HW kTLS tx\n");
	+ return rc;
	+ } else {
	+ rc = 0;
	+ device_printf(softc->dev,
	+ "HWRM key ctx allocation is failed\n");
	+ break;
	+ }
	+ }
	+
	+ allocated_keys_count += batch_sz;
	+ alloc_keys -= batch_sz;
	+ }
	+
	+ elapsed_ticks = ticks - start_ticks;
	+
	+ device_printf(softc->dev,
	+ "allocated kTLS Tx crypto keys count: %d time to alloc: (%d msecs)\n",
	+ allocated_keys_count, (int)((elapsed_ticks * 1000) / hz));
	+
	+ return rc;
	+}
	+
	+int bnxt_ktls_init(struct bnxt_softc* softc)
	+{
	+ struct bnxt_tls_info* ktls = softc->ktls_info;
	+ struct bnxt_hw_resc* hw_resc = &softc->hw_resc;
	+ struct bnxt_hw_tls_resc* tls_resc;
	+ int rc = 0;
	+
	+ if (!ktls)
	+ return 0;
	+
	+ tls_resc = &hw_resc->tls_resc[BNXT_CRYPTO_TYPE_KTLS];
	+ ktls->tck.max_ctx = tls_resc->resv_tx_key_ctxs;
	+
	+ if (!ktls->tck.max_ctx) {
	+ device_printf(softc->dev,
	+ "max Tx ctx count is zero, disable HW kTLS\n");
	+ return EINVAL;
	+ }
	+
	+ if (ktls->partition_mode) {
	+ rc = bnxt_set_partition_mode(softc);
	+ if (rc) {
	+ BNXT_DEBUG(softc->dev, "kTLS set partition mode failed, disabling partition mode\n");
	+ ktls->partition_mode = false;
	+ }
	+ }
	+
	+ rc = bnxt_pre_alloc_tx_crypto_keys(softc);
	+
	+ if (rc)
	+ return rc;
	+
	+ ktls->mpc_zone = uma_zcreate("bnxt_ktls_mpc",
	+ sizeof(struct bnxt_crypto_cmd_ctx), NULL, NULL, NULL, NULL,
	+ UMA_ALIGN_PTR, 0);
	+ if (ktls->mpc_zone == NULL) {
	+ device_printf(softc->dev, "kTLS mpc_zone create failed\n");
	+ return ENOMEM;
	+ }
	+
	+ ktls->tq = taskqueue_create("bnxt_tls", M_NOWAIT,
	+ taskqueue_thread_enqueue, &ktls->tq);
	+ if (ktls->tq == NULL) {
	+ device_printf(softc->dev, "kTLS taskqueue create failed\n");
	+ uma_zdestroy(ktls->mpc_zone);
	+ ktls->mpc_zone = NULL;
	+ return (ENOMEM);
	+ }
	+ rc = taskqueue_start_threads(&ktls->tq, 1, PWAIT, "bnxt_tls");
	+ if (rc != 0) {
	+ device_printf(softc->dev, "kTLS taskqueue thread start failed, rc=%d\n", rc);
	+ taskqueue_free(ktls->tq);
	+ ktls->tq = NULL;
	+ uma_zdestroy(ktls->mpc_zone);
	+ ktls->mpc_zone = NULL;
	+ return (rc);
	+ }
	+
	+ snprintf(ktls->zname, sizeof(ktls->zname),
	+ "bnxt_%u_tls", device_get_unit(softc->dev));
	+
	+ ktls->zone = uma_zcache_create(ktls->zname,
	+ sizeof(struct bnxt_ktls_offload_ctx_tx), NULL, NULL, NULL, NULL,
	+ bnxt_tls_tag_import, bnxt_tls_tag_release, softc,
	+ UMA_ZONE_UNMANAGED);
	+ if (ktls->zone == NULL) {
	+ device_printf(softc->dev, "kTLS UMA zone create failed\n");
	+ taskqueue_free(ktls->tq);
	+ ktls->tq = NULL;
	+ uma_zdestroy(ktls->mpc_zone);
	+ ktls->mpc_zone = NULL;
	+ return (ENOMEM);
	+ }
	+
	+ ktls->init = 1;
	+
	+ return 0;
	+}
	+
	+void bnxt_ktls_mpc_cmp(struct bnxt_softc* softc, uint32_t client,
	+ unsigned long handle, struct bnxt_cmpl_entry cmpl[],
	+ uint32_t entries)
	+{
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx = NULL;
	+ struct bnxt_tls_info* tls = softc->ktls_info;
	+ struct bnxt_crypto_cmd_ctx* ctx;
	+ unsigned long rtt_us;
	+ struct ce_cmpl* cmp;
	+ uint32_t len, kid;
	+
	+ cmp = cmpl[0].cmpl;
	+ if (!handle \|\| entries != 1) {
	+ if (!handle)
	+ device_printf(softc->dev,
	+ "kTLS MPC cmp: invalid handle 0\n");
	+ else if (entries != 1) {
	+ device_printf(softc->dev,
	+ "Invalid entries %d with handle %lx cmpl "
	+ "%08x in %s()\n",
	+ entries, handle, (uint32_t)cmp,
	+ __func__);
	+ }
	+ return;
	+ }
	+ ctx = (void*)handle;
	+ kid = CE_CMPL_KID(cmp);
	+ if (ctx->ce_cmp.opaque != BNXT_KMPC_OPAQUE(client, kid)) {
	+ device_printf(softc->dev,
	+ "Invalid CE cmpl software opaque %08x, cmpl "
	+ "%08x, kid %x\n",
	+ ctx->ce_cmp.opaque, (uint32_t)cmp, kid);
	+
	+ kctx_tx = ctx->kctx_tx;
	+ uma_zfree(tls->mpc_zone, ctx);
	+
	+ switch (kctx_tx->state) {
	+ case BNXT_TLS_STATE_INIT:
	+ if (kctx_tx->mpc_retry < MAX_MPC_RETRY) {
	+ kctx_tx->mpc_retry++;
	+ BNXT_DEBUG(softc->dev, "[KID: %u]: retry(%u) crypto add\n",
	+ kctx_tx->kid, kctx_tx->mpc_retry);
	+ if (bnxt_crypto_add(softc, kctx_tx, BNXT_CRYPTO_TYPE_KTLS))
	+ kctx_tx->state = BNXT_TLS_STATE_BAD;
	+ } else
	+ kctx_tx->state = BNXT_TLS_STATE_BAD;
	+ break;
	+ case BNXT_TLS_STATE_FREE:
	+ if (kctx_tx->mpc_retry < MAX_MPC_RETRY) {
	+ kctx_tx->mpc_retry++;
	+ BNXT_DEBUG(softc->dev, "[KID: %u]: retry(%u) crypto delete\n",
	+ kctx_tx->kid, kctx_tx->mpc_retry);
	+ if (bnxt_crypto_del(softc, kctx_tx, BNXT_CRYPTO_TYPE_KTLS))
	+ kctx_tx->state = BNXT_TLS_STATE_BAD;
	+ } else
	+ kctx_tx->state = BNXT_TLS_STATE_BAD;
	+ break;
	+ }
	+
	+ return;
	+ }
	+
	+ len = min(cmpl[0].len, (uint32_t)sizeof(ctx->ce_cmp));
	+ memcpy(&ctx->ce_cmp, cmpl[0].cmpl, len);
	+ kctx_tx = ctx->kctx_tx;
	+
	+ if (CE_CMPL_STATUS(&ctx->ce_cmp) == CE_CMPL_STATUS_OK) {
	+ if (kctx_tx->state == BNXT_TLS_STATE_INIT)
	+ kctx_tx->state = BNXT_TLS_STATE_READY;
	+ } else {
	+ counter_u64_add(softc->ktls_info->err_counters[BNXT_KTLS_TX_MPC_FAILED], 1);
	+ BNXT_DEBUG(softc->dev, "[KID: %u]: MPC failed: current state: %u cmpl_status: %lu\n",
	+ kctx_tx->kid, kctx_tx->state, CE_CMPL_STATUS(&ctx->ce_cmp));
	+ kctx_tx->state = BNXT_TLS_STATE_BAD;
	+ }
	+
	+ getmicrotime(&kctx_tx->end);
	+ timevalsub(&kctx_tx->end, &kctx_tx->start);
	+ rtt_us = kctx_tx->end.tv_sec * 1000000 + kctx_tx->end.tv_usec;
	+ bnxt_calc_mpc_cmp_time(softc, rtt_us);
	+
	+ kctx_tx->start.tv_sec = 0;
	+ kctx_tx->start.tv_usec = 0;
	+ kctx_tx->end.tv_sec = 0;
	+ kctx_tx->end.tv_usec = 0;
	+
	+ /*
	+ * Reset MPC retry counter for a KID once one successful
	+ * MPC completion is received for a KID.
	+ */
	+ kctx_tx->mpc_retry = 0;
	+ uma_zfree(tls->mpc_zone, ctx);
	+}
	+
	+static int bnxt_get_tls_seq(struct mbuf* mb, uint64_t* seq)
	+{
	+
	+ for (; mb != NULL; mb = mb->m_next) {
	+ if (!(mb->m_flags & M_EXTPG))
	+ continue;
	+ *seq = mb->m_epg_seqno;
	+ return (1);
	+ }
	+ return (0);
	+}
	+
	+int
	+bnxt_alloc_ktls_rexmit_mbuf(struct bnxt_softc softc, struct bnxt_ring txr)
	+{
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ int ring_sz = softc->scctx->isc_ntxd[1];
	+ struct bnxt_replay_pkt *pkt;
	+ int i, err;
	+
	+ txr->replay_pkt = malloc(sizeof(struct bnxt_replay_pkt) * ring_sz,
	+ M_DEVBUF, M_NOWAIT \| M_ZERO);
	+
	+ if (!txr->replay_pkt) {
	+ BNXT_DEBUG(softc->dev, "kTLS replay_pkt malloc failed\n");
	+ return ENOMEM;
	+ }
	+
	+ for (i = 0; i < ring_sz; i++) {
	+ pkt = &txr->replay_pkt[i];
	+
	+ err = bus_dmamap_create(ktls->dma_tag, 0, &pkt->dma_map);
	+ if (err != 0) {
	+ device_printf(softc->dev,
	+ "kTLS replay_pkt dmamap create failed for entry %d\n", i);
	+ goto err;
	+ }
	+ }
	+
	+ return (0);
	+err:
	+ for (i = i - 1; i >= 0; i--) {
	+ pkt = &txr->replay_pkt[i];
	+ bus_dmamap_destroy(ktls->dma_tag, pkt->dma_map);
	+ }
	+
	+ free(txr->replay_pkt, M_DEVBUF);
	+ txr->replay_pkt = NULL;
	+
	+ return ENOMEM;
	+}
	+
	+void inline
	+bnxt_free_ktls_rexmit_mbuf(struct bnxt_ring *txr)
	+{
	+ struct bnxt_softc *softc = txr->softc;
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ int ring_sz = softc->scctx->isc_ntxd[1];
	+ struct bnxt_replay_pkt *pkt;
	+ int i;
	+
	+ for (i = 0; i < ring_sz; i++) {
	+ pkt = &txr->replay_pkt[i];
	+ if (pkt->mbuf) {
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+
	+ bus_dmamap_sync(ktls->dma_tag, pkt->dma_map,
	+ BUS_DMASYNC_POSTWRITE);
	+ bus_dmamap_unload(ktls->dma_tag, pkt->dma_map);
	+ m_freem(pkt->mbuf);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_FREES], 1);
	+ pkt->mbuf = NULL;
	+ }
	+ bus_dmamap_destroy(ktls->dma_tag, pkt->dma_map);
	+ }
	+
	+
	+ free(txr->replay_pkt, M_DEVBUF);
	+}
	+
	+#define BNXT_REPLAY_PACKET_MSS 9000
	+
	+static int bnxt_ktls_submit_mbuf(struct bnxt_softc softc, struct bnxt_ring txr,
	+ struct mbuf *mbp, struct bnxt_ktls_offload_ctx_tx kctx_tx, int length, uint16_t hsz)
	+{
	+ bus_dma_segment_t segs[BNXT_MAX_NUM_SEGS];
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ struct bnxt_sw_tx_bd *tx_buf;
	+ struct tx_bd_long_hi *tbdh;
	+ struct tx_bd_opaque *opq;
	+ struct tx_bd_long *tbd;
	+ struct bnxt_replay_pkt *pkt;
	+ uint32_t kid = kctx_tx->kid;
	+ uint16_t flags_type;
	+ uint16_t lflags = 0;
	+ uint16_t pkt_size, i;
	+ uint32_t cfa_meta;
	+ struct mbuf *mb;
	+ uint32_t prod;
	+ int seg = 0;
	+ int nsegs;
	+ int xsegs;
	+ int err;
	+
	+ pkt_size = length + hsz;
	+ prod = txr->prod;
	+ mb = *mbp;
	+
	+ if (!txr->replay_pkt) {
	+ BNXT_DEBUG(softc->dev, "kTLS submit_mbuf: no replay_pkt\n");
	+ m_freem(mb);
	+ return ENOMEM;
	+ }
	+
	+ pkt = &txr->replay_pkt[prod];
	+ tx_buf = &txr->tx_buf_ring[prod];
	+
	+ if (__predict_false(tx_buf->is_replay == 1)) {
	+ BNXT_DEBUG(softc->dev, "kTLS submit_mbuf: replay slot busy KID %u\n", kid);
	+ m_freem(mb);
	+ return EINVAL;
	+ }
	+
	+ pkt->mbuf = mb;
	+ /* check number of segments in mbuf */
	+ err = bus_dmamap_load_mbuf_sg(ktls->dma_tag, pkt->dma_map,
	+ mb, segs, &nsegs, BUS_DMA_NOWAIT);
	+
	+ if (err == EFBIG) {
	+ /* too many mbuf fragments */
	+ mb = m_defrag(*mbp, M_NOWAIT);
	+ if (mb == NULL) {
	+ mb = *mbp;
	+ pkt->mbuf = NULL;
	+ m_freem(mb);
	+ return err;
	+ }
	+
	+ pkt->mbuf = mb;
	+ *mbp = mb;
	+ /* try again */
	+ err = bus_dmamap_load_mbuf_sg(ktls->dma_tag, pkt->dma_map,
	+ mb, segs, &nsegs, BUS_DMA_NOWAIT);
	+ }
	+
	+ if (err != 0) {
	+ BNXT_DEBUG(softc->dev, "kTLS submit_mbuf: dmamap load failed, err=%d KID %u\n",
	+ err, kid);
	+ m_freem(mb);
	+ pkt->mbuf = NULL;
	+ return err;
	+ }
	+
	+ bus_dmamap_sync(ktls->dma_tag, pkt->dma_map,
	+ BUS_DMASYNC_PREWRITE);
	+
	+ /* Calculate segments to be re-played */
	+ int rep_len = pkt_size;
	+ int max_seg = 0;
	+ for (i = xsegs = 0; i != nsegs; i++) {
	+ if (rep_len <= 0)
	+ break;
	+
	+ /* Prevent underflow - adjust segment length if needed */
	+ if (segs[i].ds_len > rep_len) {
	+ /* Use only remaining length, then break */
	+ break;
	+ }
	+ rep_len = rep_len - (uint32_t)segs[i].ds_len;
	+
	+ if (max_seg < segs[i].ds_len)
	+ max_seg = segs[i].ds_len;
	+ xsegs++;
	+ }
	+
	+ /* check if there are no segments */
	+ if (__predict_false(xsegs == 0)) {
	+ BNXT_DEBUG(softc->dev, "kTLS submit_mbuf: no segments after len adjustment KID %u\n",
	+ kid);
	+ bus_dmamap_unload(ktls->dma_tag, pkt->dma_map);
	+ m_freem(mb);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_FREES], 1);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_ERRORS], 1);
	+ pkt->mbuf = NULL;
	+ mbp = NULL; / safety clear */
	+ return (0);
	+ }
	+
	+ tx_buf->is_replay = 1;
	+ lflags = htole16(TX_BD_FLAGS_CRYPTO_EN);
	+
	+ /* First(header) BD of the packet*/
	+ tbd = &((struct tx_bd_long *)txr->vaddr)[prod];
	+
	+ opq = (struct tx_bd_opaque *)&tbd->opaque;
	+ /* No need to byte-swap the opaque value */
	+ SET_TX_OPAQUE_KTLS(opq, txr, 1, prod, txr->running_bds + xsegs + 1);
	+ tbd->len = htole16(segs[seg].ds_len);
	+ tbd->addr = htole64(segs[seg++].ds_addr);
	+ flags_type = ((xsegs + 1) << TX_BD_SHORT_FLAGS_BD_CNT_SFT) & TX_BD_SHORT_FLAGS_BD_CNT_MASK;
	+ if (length >= 2048)
	+ flags_type \|= TX_BD_SHORT_FLAGS_LHINT_GTE2K;
	+ else
	+ flags_type \|= bnxt_tx_lhint[length >> 9];
	+
	+ /Meta BD of the packet/
	+ flags_type \|= TX_BD_LONG_TYPE_TX_BD_LONG;
	+
	+ if (softc->tx_host_coal_enable) {
	+ /*
	+ * Must generate Tx completion for replay packet, so do not set
	+ * TX_BD_LONG_FLAGS_NO_CMPL flag for replay packets.
	+ */
	+ flags_type \|= TX_BD_LONG_FLAGS_COAL_NOW;
	+ txr->running_bds = 0;
	+ }
	+
	+ prod = RING_NEXT(txr, prod);
	+
	+ if (length >= BNXT_REPLAY_PACKET_MSS) {
	+ mb->m_pkthdr.tso_segsz = BNXT_REPLAY_PACKET_MSS;
	+ mb->m_pkthdr.csum_flags \|= CSUM_TSO;
	+ } else if (xsegs > 1) {
	+ mb->m_pkthdr.tso_segsz = max_seg;
	+ mb->m_pkthdr.csum_flags \|= CSUM_TSO;
	+ }
	+
	+ tbdh = &((struct tx_bd_long_hi *)txr->vaddr)[prod];
	+ tbdh->kid_or_ts_high_mss = htole32((mb->m_pkthdr.tso_segsz & TX_BD_LONG_MSS_MASK) \|
	+ ((KID_HIGH(kid) << TX_BD_LONG_KID_OR_TS_HIGH_SFT) &
	+ TX_BD_LONG_KID_OR_TS_HIGH_MASK));
	+
	+ tbdh->kid_or_ts_low_hdr_size = htole16(((hsz >> 1) & TX_BD_LONG_HDR_SIZE_MASK) \|
	+ ((KID_LOW(kid) << TX_BD_LONG_KID_OR_TS_LOW_SFT) &
	+ TX_BD_LONG_KID_OR_TS_LOW_MASK));
	+ tbdh->cfa_action = 0;
	+
	+ cfa_meta = 0;
	+
	+ if (mb->m_flags & M_VLANTAG) {
	+ /* TODO: Do we need to byte-swap the vtag here? */
	+ cfa_meta = TX_BD_LONG_CFA_META_KEY_VLAN_TAG \|
	+ mb->m_pkthdr.ether_vtag;
	+ cfa_meta \|= TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100;
	+ }
	+ tbdh->cfa_meta = htole32(cfa_meta);
	+
	+ if (mb->m_pkthdr.csum_flags & CSUM_TSO) {
	+ lflags \|= TX_BD_LONG_LFLAGS_LSO \|
	+ TX_BD_LONG_LFLAGS_T_IPID;
	+ }
	+ else if(mb->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
	+ lflags \|= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM \|
	+ TX_BD_LONG_LFLAGS_IP_CHKSUM;
	+ }
	+ else if(mb->m_pkthdr.csum_flags & CSUM_IP) {
	+ lflags \|= TX_BD_LONG_LFLAGS_IP_CHKSUM;
	+ }
	+
	+ tbdh->lflags = htole16(lflags);
	+
	+ /* Payload BD*/
	+ for (; seg < xsegs; seg++) {
	+ tbd->flags_type = htole16(flags_type);
	+
	+ prod = RING_NEXT(txr, prod);
	+
	+ tbd = &((struct tx_bd_long *)txr->vaddr)[prod];
	+ tbd->len = htole16(segs[seg].ds_len);
	+ tbd->addr = htole64(segs[seg].ds_addr);
	+ flags_type = TX_BD_SHORT_TYPE_TX_BD_SHORT;
	+ }
	+
	+ flags_type \|= TX_BD_SHORT_FLAGS_PACKET_END;
	+ tbd->flags_type = htole16(flags_type);
	+
	+ prod = RING_NEXT(txr, prod);
	+
	+ txr->prod = prod;
	+
	+ softc->db_ops.bnxt_db_tx(txr, prod);
	+
	+ return 0;
	+}
	+
	+static void bnxt_ktls_pre_xmit(struct bnxt_softc bp, struct bnxt_ring txr,
	+ uint32_t kid, struct crypto_prefix_cmd *pre_cmd)
	+{
	+ struct bnxt_sw_tx_bd *tx_buf;
	+ uint32_t bd_space, space;
	+ uint8_t *pcmd;
	+ struct tx_bd *pcmd1;
	+ uint16_t prod;
	+ struct tx_bd_presync *psbd;
	+ struct tx_bd_opaque *opq;
	+ bool need_cpl = false;
	+
	+ prod = txr->prod;
	+
	+ tx_buf = &txr->tx_buf_ring[prod];
	+
	+ if (bp->tx_host_coal_enable) {
	+ /* BD field of opaque is only 15 bits; respect tx_host_coal_bds threshold */
	+ if ((txr->running_bds + CRYPTO_PREFIX_CMD_BDS + 1) >
	+ (bp->tx_host_coal_bds - (BNXT_MAX_NUM_SEGS + 1)))
	+ need_cpl = true;
	+ }
	+
	+ /* presync TX BD prep */
	+ psbd = &((struct tx_bd_presync *)txr->vaddr)[prod];
	+ psbd->tx_bd_len_flags_type = CRYPTO_PRESYNC_BD_CMD;
	+ psbd->tx_bd_kid = htole32(kid);
	+
	+ opq = (struct tx_bd_opaque *)&psbd->tx_bd_opaque;
	+ SET_TX_OPAQUE_KTLS(opq, txr, 0, prod,
	+ txr->running_bds + CRYPTO_PREFIX_CMD_BDS + 1);
	+
	+ if (bp->tx_host_coal_enable) {
	+ psbd->tx_bd_len_flags_type \|= TX_BD_LONG_FLAGS_COAL_NOW;
	+ if (need_cpl) {
	+ txr->running_bds = 0;
	+ } else {
	+ txr->running_bds += (CRYPTO_PREFIX_CMD_BDS + 1);
	+ psbd->tx_bd_len_flags_type \|= TX_BD_LONG_FLAGS_NO_CMPL;
	+ }
	+ }
	+
	+ prod = RING_NEXT(txr, prod);
	+
	+ /* Copy pre-cmd to TX desc ring*/
	+ pcmd1 = &((struct tx_bd *)txr->vaddr)[prod];
	+ pcmd = (uint8_t *)pcmd1;
	+ bd_space = txr->ring_size - prod;
	+ space = bd_space * sizeof(struct tx_bd);
	+ if (space >= CRYPTO_PREFIX_CMD_SIZE) {
	+ memcpy(pcmd, pre_cmd, CRYPTO_PREFIX_CMD_SIZE);
	+ prod += CRYPTO_PREFIX_CMD_BDS - 1;
	+ } else {
	+ memcpy(pcmd, pre_cmd, space);
	+ prod = 0;
	+ pcmd1 = &((struct tx_bd *)txr->vaddr)[prod];
	+ pcmd = (uint8_t *)pcmd1;
	+ memcpy((uint8_t )pcmd, (uint8_t )pre_cmd + space,
	+ CRYPTO_PREFIX_CMD_SIZE - space);
	+ prod += CRYPTO_PREFIX_CMD_BDS - bd_space - 1;
	+ }
	+
	+ prod = RING_NEXT(txr, prod);
	+ tx_buf->inline_bds = CRYPTO_PREFIX_CMD_BDS - 1;
	+ txr->prod = prod;
	+}
	+
	+static int bnxt_get_tls_record_len(struct mbuf *m)
	+{
	+ uint32_t rec_len = 0;
	+
	+ /*
	+ * A single-page record has no "middle" or trailing page: the whole
	+ * payload is m_epg_last_len bytes starting at m_epg_1st_off, per
	+ * m_epg_pagelen()'s treatment of pidx == m_epg_npgs - 1 == 0.
	+ */
	+ if (m->m_epg_npgs == 1)
	+ return m->m_epg_last_len;
	+
	+ rec_len += PAGE_SIZE - m->m_epg_1st_off;
	+ if (m->m_epg_npgs > 2)
	+ rec_len += PAGE_SIZE * (m->m_epg_npgs - 2);
	+
	+ rec_len += m->m_epg_last_len;
	+
	+ return rec_len;
	+}
	+
	+static struct mbuf bnxt_get_tls_record(struct mbuf m0, device_t dev)
	+{
	+ struct mbuf *m = m0;
	+
	+ /* find TLS record */
	+ for (; m != NULL; m = m->m_next) {
	+ if (!(m->m_flags & M_EXTPG))
	+ continue;
	+ else
	+ break;
	+ }
	+
	+ if (__predict_false((m == NULL))) {
	+ /* should never happen; means we got a non-TLS mbuf */
	+ device_printf(dev, "kTLS: non-TLS packet in %s\n", __func__);
	+ return NULL;
	+ }
	+ return m;
	+}
	+
	+static struct tcphdr bnxt_get_tcp_seq(if_pkt_info_t pi, struct mbuf m, struct mbuf **mp)
	+{
	+ struct tcphdr *th = NULL;
	+ uint32_t hlen;
	+
	+ hlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
	+ if (m->m_len < hlen) {
	+ m = m_pullup(m, hlen);
	+ if (m == NULL)
	+ return NULL;
	+ *mp = m;
	+ }
	+
	+ th = mtodo(m, pi->ipi_ehdrlen + pi->ipi_ip_hlen);
	+
	+ return th;
	+}
	+
	+
	+static int bnxt_ktls_tx_replay(struct bnxt_softc bp, struct bnxt_ring txr,
	+ struct mbuf mb, struct bnxt_ktls_offload_ctx_tx kctx_tx, uint32_t seq,
	+ uint32_t hdr_tcp_seq, if_pkt_info_t pi)
	+{
	+ struct bnxt_tls_info *ktls;
	+ struct mbuf r_mb, mp;
	+ struct tcphdr *th;
	+ int hsz;
	+ uint32_t replay_len;
	+ uint32_t tcp_seq = hdr_tcp_seq;
	+
	+ replay_len = seq - hdr_tcp_seq;
	+
	+ ktls = bp->ktls_info;
	+
	+ r_mb = m_copypacket(mb, M_NOWAIT);
	+
	+ if (r_mb == NULL) {
	+ BNXT_DEBUG(bp->dev, "[KID:%u]: kTLS replay m_copypacket failed\n", kctx_tx->kid);
	+ return EINVAL;
	+ }
	+
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_ALLOCS], 1);
	+ hsz = (pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen);
	+
	+ mp = r_mb;
	+
	+ th = bnxt_get_tcp_seq(pi, r_mb, &mp);
	+
	+ r_mb = mp;
	+
	+ if (th == NULL) {
	+ /* m_pullup() already called m_freem on r_mb on failure.
	+ * so no m_freem(r_mb) is needed again.
	+ */
	+ BNXT_DEBUG(bp->dev, "[KID:%u]: kTLS replay get_tcp_seq failed\n", kctx_tx->kid);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_FREES], 1);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_ERRORS], 1);
	+ return EINVAL;
	+ }
	+
	+ th->th_seq = htonl(tcp_seq);
	+
	+ if (r_mb->m_next == NULL) {
	+ BNXT_DEBUG(bp->dev, "[KID:%u]: kTLS replay missing payload mbuf\n", kctx_tx->kid);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_ERRORS], 1);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_FREES], 1);
	+ m_freem(r_mb);
	+ return EINVAL;
	+ }
	+
	+ r_mb->m_next->m_data -= replay_len;
	+ r_mb->m_next->m_len = replay_len;
	+
	+ r_mb->m_pkthdr.len = replay_len + hsz;
	+
	+ if (bnxt_ktls_submit_mbuf(bp, txr, &r_mb, kctx_tx, replay_len, hsz) != 0) {
	+ BNXT_DEBUG(bp->dev, "[KID:%u]: kTLS replay submit_mbuf failed\n", kctx_tx->kid);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_FREES], 1);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_ERRORS], 1);
	+ return ENOSPC;
	+ }
	+
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_REPLAY], 1);
	+
	+ return 0;
	+}
	+
	+static int bnxt_ktls_tx_ooo(struct bnxt_softc bp, struct bnxt_ring txr,
	+ struct mbuf *mbuf, uint32_t payload_len, uint32_t seq,
	+ struct bnxt_ktls_offload_ctx_tx *kctx_tx, if_pkt_info_t pi)
	+{
	+ struct tls_record_layer *tlshdr;
	+ struct crypto_prefix_cmd pcmd = {0};
	+ struct bnxt_tls_info *ktls;
	+ uint32_t presync_flags = 0;
	+ struct mbuf *m;
	+ uint32_t hdr_tcp_seq;
	+ uint8_t *hdr = NULL;
	+ bool fwd = false;
	+ int rc = 0;
	+ uint64_t rec_sn = 0;
	+ uint32_t rec_len;
	+ uint32_t record_end_seq;
	+ uint32_t offset;
	+ uint32_t nonce_bytes;
	+ uint32_t end_seq;
	+ bool is_replay = false;
	+ bool update_inorder = false;
	+ uint32_t tx_prod;
	+ uint16_t running_bds;
	+
	+ ktls = bp->ktls_info;
	+
	+ m = bnxt_get_tls_record(mbuf, bp->dev);
	+ if (m == NULL) {
	+ device_printf(bp->dev, "kTLS OOO: no TLS record in mbuf\n");
	+ return EINVAL;
	+ }
	+
	+ bnxt_get_tls_seq(m, &rec_sn);
	+ rec_len = bnxt_get_tls_record_len(m);
	+
	+ offset = (uintptr_t)m->m_data;
	+ tlshdr = (void *)m->m_epg_hdr;
	+ hdr_tcp_seq = seq - offset;
	+ hdr = (uint8_t*)tlshdr;
	+
	+ record_end_seq =
	+ (hdr_tcp_seq + rec_len + m->m_epg_hdrlen + m->m_epg_trllen) - 1;
	+
	+ end_seq = seq + payload_len - 1;
	+
	+ if (__predict_false(after(seq, kctx_tx->tcp_seq_no) \|\|
	+ after(end_seq, kctx_tx->tcp_seq_no))) {
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_SEQ_FWD], 1);
	+ fwd = true;
	+
	+ /* Check if the seq forward packet has a TLS header*/
	+ if (seq <= hdr_tcp_seq && end_seq >= hdr_tcp_seq) {
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_SEQ_FWD_TLS_HDR], 1);
	+ presync_flags = CRYPTO_PREFIX_CMD_FLAGS_UPDATE_IN_ORDER_VAR;
	+ update_inorder = true;
	+ } else {
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_SEQ_FWD_SILENT_DROPS], 1);
	+ }
	+ } else
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_RETRANS], 1);
	+
	+ pcmd.header_tcp_seq_num = htole32(hdr_tcp_seq);
	+ pcmd.start_tcp_seq_num = htole32(seq);
	+ pcmd.end_tcp_seq_num = htole32(end_seq);
	+
	+ if ((kctx_tx->crypto_info.tls_vmajor ==
	+ TLS_MAJOR_VER_ONE) &&
	+ (kctx_tx->crypto_info.tls_vminor ==
	+ TLS_MINOR_VER_TWO)) {
	+ nonce_bytes = BNXT_EXPLICIT_NONCE_LEN;
	+
	+ if (offset > 5 && (offset < (5 + nonce_bytes)))
	+ nonce_bytes = offset - 5;
	+
	+ /* Verify we don't exceed TLS header bounds */
	+ if ((hdr + 5 + nonce_bytes) > ((uint8_t*)tlshdr + m->m_epg_hdrlen)) {
	+ BNXT_DEBUG(bp->dev, "kTLS OOO: TLS header bounds exceeded\n");
	+ return EINVAL;
	+ }
	+
	+ memcpy(pcmd.explicit_nonce, hdr + 5, nonce_bytes);
	+ }
	+
	+ /* Out-of-sequence packet includes tag(trailer) */
	+ if (offset && (record_end_seq - m->m_epg_trllen) <
	+ (seq + payload_len)) {
	+ is_replay = true;
	+ }
	+
	+ memcpy(&pcmd.record_seq_num[0], &rec_sn, sizeof(rec_sn));
	+
	+ if (update_inorder)
	+ presync_flags = CRYPTO_PREFIX_CMD_FLAGS_UPDATE_IN_ORDER_VAR;
	+ else if (is_replay)
	+ presync_flags = CRYPTO_PREFIX_CMD_FLAGS_FULL_REPLAY_RETRAN;
	+ else
	+ presync_flags = 0;
	+
	+ pcmd.flags = htole32(presync_flags);
	+
	+ /*
	+ * Save tx ring producer index and running_bds accumulator before
	+ * writing presync command, it will be used to move the producer
	+ * index backwards to discard presync command and restore running_bds
	+ * to the state of before sending presync command, if replay packet fails.
	+ */
	+ tx_prod = txr->prod;
	+ running_bds = txr->running_bds;
	+
	+ bnxt_ktls_pre_xmit(bp, txr, kctx_tx->kid, &pcmd);
	+
	+ if (is_replay) {
	+ rc = bnxt_ktls_tx_replay(bp, txr, mbuf, kctx_tx, seq, hdr_tcp_seq, pi);
	+
	+ if (fwd)
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_SEQ_FWD_REPLAY], 1);
	+ /*
	+ * If replay failed, discard presync BD by rolling back both the
	+ * producer index and restore running_bds accumulator.
	+ */
	+ if (rc) {
	+ txr->prod = tx_prod;
	+ txr->running_bds = running_bds;
	+ }
	+ }
	+
	+ if (update_inorder && !rc)
	+ kctx_tx->tcp_seq_no = end_seq + 1;
	+
	+ return rc;
	+}
	+
	+int bnxt_ktls_xmit(struct bnxt_softc* softc, struct bnxt_ring* txr,
	+ struct mbuf** ppmb, uint16_t* lflags, uint32_t* kid, if_pkt_info_t pi)
	+{
	+ struct bnxt_ktls_offload_ctx_tx* kctx_tx;
	+ struct bnxt_tls_info* ktls = softc->ktls_info;
	+ struct m_snd_tag* ptag;
	+ struct mbuf* mb = *ppmb;
	+ uint64_t rec_sn;
	+ uint32_t header_size;
	+ uint32_t mb_seq;
	+ int rc;
	+
	+ if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0)
	+ return (0);
	+
	+ ptag = mb->m_pkthdr.snd_tag;
	+
	+ if (ptag == NULL \|\| ptag->sw == NULL \|\|
	+ ptag->sw->type != IF_SND_TAG_TYPE_TLS) {
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_INVALID_TAG], 1);
	+ return (0);
	+ }
	+
	+ kctx_tx = __containerof(ptag, struct bnxt_ktls_offload_ctx_tx, tag);
	+
	+ header_size = (pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen);
	+
	+
	+ if (__predict_false(header_size == 0)) {
	+ BNXT_DEBUG(softc->dev, "kTLS xmit: zero header size\n");
	+ return EINVAL;
	+ }
	+
	+ if (header_size == mb->m_pkthdr.len) {
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_ZERO_TCP_PAYLEN], 1);
	+ BNXT_DEBUG(softc->dev, "kTLS xmit: zero TCP payload\n");
	+ return (0);
	+ }
	+
	+ if (bnxt_get_tls_seq(mb, &rec_sn) == 0) {
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_INVALID_REC_SN], 1);
	+ BNXT_DEBUG(softc->dev, "kTLS xmit: invalid record seq\n");
	+ return (0);
	+ }
	+
	+ mb_seq = ntohl(pi->ipi_tcp_seq);
	+
	+ switch(kctx_tx->state) {
	+ /* Add crypto MPC is not yet completed*/
	+ case BNXT_TLS_STATE_INIT:
	+ if (!kctx_tx->enobufs)
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_TCE_NOT_READY], 1);
	+
	+ kctx_tx->enobufs++;
	+
	+ BNXT_DEBUG(softc->dev, "[KID:%u]: TCE INIT:"
	+ " tcp_seqno: %u record_no: %lu enobufs: %u\n",
	+ kctx_tx->kid, mb_seq, rec_sn, kctx_tx->enobufs);
	+
	+ return ENOBUFS;
	+ /* Crypto connection is setup and ready to use */
	+ case BNXT_TLS_STATE_READY:
	+ if (kctx_tx->enobufs) {
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_TCE_READY_DELAYED], 1);
	+ BNXT_DEBUG(softc->dev, "[KID:%u]: TCE READY:"
	+ " tcp_seqno: %u record_no: %lu enobufs: %u\n",
	+ kctx_tx->kid, mb_seq, rec_sn, kctx_tx->enobufs);
	+ kctx_tx->enobufs = 0;
	+ }
	+ break;
	+ /* Crypto connection is in bad state */
	+ case BNXT_TLS_STATE_BAD:
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_TCE_BAD], 1);
	+ BNXT_DEBUG(softc->dev, "[KID:%u]: TCE BAD:"
	+ " tcp_seqno: %u record_no: %lu enobufs: %u\n",
	+ kctx_tx->kid, mb_seq, rec_sn, kctx_tx->enobufs);
	+ return EINVAL;
	+ case BNXT_TLS_STATE_FREE:
	+ case BNXT_TLS_STATE_FREED:
	+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_TCE_FREE], 1);
	+ /* FALLTHROUGH */
	+ default:
	+ return EINVAL;
	+ }
	+
	+ *kid = kctx_tx->kid;
	+ *lflags \|= TX_BD_LONG_LFLAGS_CRYPTO_EN;
	+
	+ if (kctx_tx->tcp_seq_no == mb_seq) {
	+ kctx_tx->inorder_rec_sn = rec_sn;
	+ kctx_tx->inorder_paylen = mb->m_pkthdr.len - header_size;
	+ kctx_tx->inorder_seq = kctx_tx->tcp_seq_no;
	+ kctx_tx->tcp_seq_no += mb->m_pkthdr.len - header_size;
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_HW_PKT], 1);
	+ } else {
	+ uint32_t payload_len;
	+
	+ payload_len = mb->m_pkthdr.len - header_size;
	+
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_OOO], 1);
	+
	+ rc = bnxt_ktls_tx_ooo(softc, txr, mb, payload_len, mb_seq, kctx_tx, pi);
	+ if (rc) {
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_FAILED], 1);
	+ BNXT_DEBUG(softc->dev, "[KID:%u]: kTLS OOO failed, rc=%d\n", kctx_tx->kid, rc);
	+ return EAGAIN;
	+ }
	+ }
	+
	+ return (0);
	+}
	+
	+#else /* KERN_TLS */
	+
	+void bnxt_free_ktls_counters(struct bnxt_softc* softc)
	+{
	+}
	+
	+void bnxt_alloc_ktls_counters(struct bnxt_softc* softc)
	+{
	+}
	+
	+void bnxt_alloc_ktls_info(struct bnxt_softc* softc, uint16_t max_keys,
	+ bool partition_cap, uint16_t ctxs_per_partition)
	+{
	+}
	+
	+void bnxt_free_ktls_info(struct bnxt_softc* softc)
	+{
	+}
	+
	+void bnxt_hwrm_reserve_pf_key_ctxs(struct bnxt_softc* softc,
	+ struct hwrm_func_cfg_input* req,
	+ uint8_t type)
	+{
	+}
	+
	+int bnxt_ktls_init(struct bnxt_softc* softc)
	+{
	+ return 0;
	+}
	+
	+void bnxt_ktls_mpc_cmp(struct bnxt_softc* softc, uint32_t client,
	+ unsigned long handle, struct bnxt_cmpl_entry cmpl[],
	+ uint32_t entries)
	+{
	+}
	+
	+int bnxt_ktls_xmit(struct bnxt_softc* softc, struct bnxt_ring* txr,
	+ struct mbuf** ppmb, uint16_t* lflags, uint32_t* kid, if_pkt_info_t pi)
	+{
	+ return 0;
	+}
	+
	+void bnxt_ktls_del_all(struct bnxt_softc *softc)
	+{
	+}
	+
	+void bnxt_clear_ktls(struct bnxt_softc *softc)
	+{
	+}
	+
	+int
	+bnxt_alloc_ktls_rexmit_mbuf(struct bnxt_softc softc, struct bnxt_ring txr)
	+{
	+ return 0;
	+}
	+
	+void inline
	+bnxt_free_ktls_rexmit_mbuf(struct bnxt_ring *txr)
	+{
	+}
	+
	+#endif /* KERN_TLS */
	Index: sys/dev/bnxt/bnxt_en/bnxt_sysctl.h
	===================================================================
	--- sys/dev/bnxt/bnxt_en/bnxt_sysctl.h
	+++ sys/dev/bnxt/bnxt_en/bnxt_sysctl.h
	@@ -42,3 +42,6 @@
	int bnxt_create_pause_fc_sysctls(struct bnxt_softc *softc);
	int bnxt_create_dcb_sysctls(struct bnxt_softc *softc);
	int bnxt_create_dcb_ets_sysctls(struct bnxt_softc *softc);
	+int bnxt_create_ktls_sysctls(struct bnxt_softc *softc);
	+int bnxt_ktls_sysctls(struct bnxt_softc *softc);
	+int bnxt_create_mpc_cmp_time_sysctls(struct bnxt_softc *softc);
	Index: sys/dev/bnxt/bnxt_en/bnxt_sysctl.c
	===================================================================
	--- sys/dev/bnxt/bnxt_en/bnxt_sysctl.c
	+++ sys/dev/bnxt/bnxt_en/bnxt_sysctl.c
	@@ -34,6 +34,82 @@
	#include "bnxt.h"
	#include "bnxt_hwrm.h"
	#include "bnxt_sysctl.h"
	+#include "bnxt_ktls.h"
	+
	+struct bnxt_ktls_counters_list {
	+ enum bnxt_ktls_counters id;
	+ const char *label;
	+ const char *desc;
	+};
	+static const struct bnxt_ktls_counters_list ktls_counters[] = {
	+ { BNXT_KTLS_TX_ADD, "add_tx_session", "KTLS sessions add count" },
	+ { BNXT_KTLS_TX_DEL, "delete_tx_session", "KTLS sessions delete count" },
	+ { BNXT_KTLS_TX_HW_PKT, "hw_tx_pkts", "HW KTLS TX count" },
	+ { BNXT_KTLS_TX_FAILED, "failed_tx_pkts", "SW KTLS TX count" },
	+ { BNXT_KTLS_TX_OOO, "ooo_tx_pkts", "OOO KTLS TX count" },
	+ { BNXT_KTLS_TX_RETRANS, "retrans_tx_pkts", "Retransmitted KTLS TX count" },
	+ { BNXT_KTLS_TX_REPLAY, "replay_tx_pkts", "Replay KTLS TX count" },
	+ { BNXT_KTLS_TX_SEQ_FWD, "seq_fwd_tx_pkts", "Seq forward KTLS Tx count" },
	+ { BNXT_KTLS_TX_SEQ_FWD_TLS_HDR, "seq_fwd_tx_tls_hdr_pkts", "Seq forward Tx packets with TLS header" },
	+ { BNXT_KTLS_TX_SEQ_FWD_SILENT_DROPS, "seq_fwd_tx_silent_drops_pkts", "Silently dropped seq forward packets" },
	+ { BNXT_KTLS_TX_SEQ_FWD, "seq_fwd_tx_replay_pkts", "Replayed Seq forward packets" },
	+ { BNXT_KTLS_TX_MBUF_ALLOCS, "mbuf_allocs", "Alloc mbufs count" },
	+ { BNXT_KTLS_TX_MBUF_FREES, "mbuf_frees", "Free mbufs count" },
	+ { BNXT_KTLS_TX_MBUF_ERRORS, "mbuf_errors", "Error mbufs count" },
	+};
	+static const int ktls_counters_count = sizeof(ktls_counters) / sizeof(ktls_counters[0]);
	+
	+struct bnxt_ktls_err_counters_list {
	+ enum bnxt_ktls_err_counters id;
	+ const char *label;
	+ const char *desc;
	+};
	+static const struct bnxt_ktls_err_counters_list ktls_err_counters[] = {
	+ { BNXT_KTLS_TX_DEV_ADD_FAILED, "add_tx_session_failure", "Tx KTLS device add failure count" },
	+ { BNXT_KTLS_TX_MPC_TIMEOUT, "mpc_cmd_timeout", "Timeout MPC commands count" },
	+ { BNXT_KTLS_TX_UNINITIALIZED, "ktls_tx_uninit", "snd_tag alloc failure due to ktls not initialized" },
	+ { BNXT_KTLS_TX_CTX_ALLOC_FAILED, "ctx_alloc_failures", "snd_tag alloc failure due to ctx alloc failed" },
	+ { BNXT_KTLS_TX_MPC_FAILED, "mpc_failures", "MPC command failures count" },
	+ { BNXT_KTLS_TX_TCE_NOT_READY, "tce_not_ready", "Tx crypto engine(TCE) is not ready" },
	+ { BNXT_KTLS_TX_TCE_READY_DELAYED, "tce_ready_delayed", "TCE getting ready with delay" },
	+ { BNXT_KTLS_TX_TCE_BAD, "tce_bad", "TCE is in bad state" },
	+ { BNXT_KTLS_TX_UNSUPPORTED_TLS_HDR, "unsupported_tls_hdr", "snd_tag alloc failure due to unsupported TLS header" },
	+ { BNXT_KTLS_TX_INVALID_TAG, "invalid_tag", "Invalid snd_tag" },
	+ { BNXT_KTLS_TX_ZERO_TCP_PAYLEN, "zero_payload", "Packet/s with zero TCP payload" },
	+ { BNXT_KTLS_TX_INVALID_REC_SN, "invalid_rec_sn", "Invalid Record" },
	+ { BNXT_KTLS_TX_TCE_FREE, "tce_free", "Packets received when crypto delete is ongoing"},
	+};
	+
	+static const int ktls_err_counters_count = sizeof(ktls_err_counters) / sizeof(ktls_err_counters[0]);
	+
	+struct bnxt_mpc_cmp_time_counters_list {
	+ enum bnxt_mpc_cmp_time_counters id;
	+ const char *label;
	+ const char *desc;
	+};
	+static const struct bnxt_mpc_cmp_time_counters_list mpc_cmp_counters[] = {
	+ { BNXT_MPC_CMP_TIME_1US, "<=1us", "MPC command completion <= 1us" },
	+ { BNXT_MPC_CMP_TIME_1_5US, "1-5us", "MPC command completion between 1-5us" },
	+ { BNXT_MPC_CMP_TIME_5_10US, "5-10us", "MPC command completion between 5-10us" },
	+ { BNXT_MPC_CMP_TIME_10_15US, "10-15us", "MPC command completion between 10-15us" },
	+ { BNXT_MPC_CMP_TIME_15_20US, "15-20us", "MPC command completion between 15-20us" },
	+ { BNXT_MPC_CMP_TIME_20_25US, "20-25us", "MPC command completion between 20-25us" },
	+ { BNXT_MPC_CMP_TIME_25_50US, "25-50us", "MPC command completion between 25-50us" },
	+ { BNXT_MPC_CMP_TIME_50_100US, "50-100us", "MPC command completion between 50-100us" },
	+ { BNXT_MPC_CMP_TIME_100_200US, "100-200us", "MPC command completion between 100-200us" },
	+ { BNXT_MPC_CMP_TIME_200_500US, "200-500us", "MPC command completion between 200-500us" },
	+ { BNXT_MPC_CMP_TIME_500_1000US, "500-1000us", "MPC command completion between 500-1000us" },
	+ { BNXT_MPC_CMP_TIME_1000_1500US, "1000-1500us", "MPC command completion between 1000-1500us" },
	+ { BNXT_MPC_CMP_TIME_1500_2000US, "1500-2000us", "MPC command completion between 1500-2000us" },
	+ { BNXT_MPC_CMP_TIME_2000_2500US, "2000-2500us", "MPC command completion between 2000-2500us" },
	+ { BNXT_MPC_CMP_TIME_2500_3000US, "2500-3000us", "MPC command completion between 2500-3000us" },
	+ { BNXT_MPC_CMP_TIME_3000_3500US, "3000-3500us", "MPC command completion between 3000-3500us" },
	+ { BNXT_MPC_CMP_TIME_3500_4000US, "3500-4000us", "MPC command completion between 3500-4000us" },
	+ { BNXT_MPC_CMP_TIME_4000_4500US, "4000-4500us", "MPC command completion between 4000-4500us" },
	+ { BNXT_MPC_CMP_TIME_4500_5000US, "4500-5000us", "MPC command completion between 4500-5000us" },
	+ { BNXT_MPC_CMP_TIME_5000_PLUS_US, "5000+us", "MPC command completion >= 5000us" },
	+};
	+static const int mpc_cmp_counters_count = sizeof(mpc_cmp_counters) / sizeof(mpc_cmp_counters[0]);

	DEFINE_MUTEX(tmp_mutex); /* mutex lock for driver */
	extern void bnxt_fw_reset(struct bnxt_softc *bp);
	@@ -121,6 +197,27 @@
	return ENOMEM;
	}

	+ sysctl_ctx_init(&softc->ktls_stats);
	+ ctx = device_get_sysctl_ctx(softc->dev);
	+ softc->ktls_stats_oid = SYSCTL_ADD_NODE(ctx,
	+ SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO,
	+ "ktls_stats", CTLFLAG_RD \| CTLFLAG_MPSAFE, 0, "kTLS statistics");
	+ if (!softc->ktls_stats_oid) {
	+ sysctl_ctx_free(&softc->ktls_stats);
	+ return ENOMEM;
	+ }
	+
	+ sysctl_ctx_init(&softc->mpc_cmp_time_stats);
	+ ctx = device_get_sysctl_ctx(softc->dev);
	+ softc->mpc_cmp_time_stats_oid = SYSCTL_ADD_NODE(ctx,
	+ SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO,
	+ "mpc_cmp_time_stats", CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	+ "MPC completion time statistics");
	+ if (!softc->mpc_cmp_time_stats_oid) {
	+ sysctl_ctx_free(&softc->mpc_cmp_time_stats);
	+ return ENOMEM;
	+ }
	+
	return 0;
	}

	@@ -175,6 +272,22 @@
	softc->dcb_oid = NULL;
	}

	+ if (softc->ktls_stats_oid != NULL) {
	+ orc = sysctl_ctx_free(&softc->ktls_stats);
	+ if (orc)
	+ rc = orc;
	+ else
	+ softc->ktls_stats_oid = NULL;
	+ }
	+
	+ if (softc->mpc_cmp_time_stats_oid != NULL) {
	+ orc = sysctl_ctx_free(&softc->mpc_cmp_time_stats);
	+ if (orc)
	+ rc = orc;
	+ else
	+ softc->mpc_cmp_time_stats_oid = NULL;
	+ }
	+
	return rc;
	}

	@@ -2245,3 +2358,107 @@
	/* Nothing for now, meant for future expansion */
	return 0;
	}
	+
	+static int
	+sysctl_counter_u64_handler(SYSCTL_HANDLER_ARGS)
	+{
	+ counter_u64_t counter = (counter_u64_t )arg1;
	+ uint64_t val = counter_u64_fetch(*counter);
	+
	+ return sysctl_handle_64(oidp, &val, 0, req);
	+}
	+
	+int
	+bnxt_create_ktls_sysctls(struct bnxt_softc *softc)
	+{
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ struct sysctl_oid *oid;
	+
	+ if (!ktls)
	+ return 0;
	+
	+ oid = SYSCTL_ADD_NODE(&softc->ktls_stats,
	+ SYSCTL_CHILDREN(softc->ktls_stats_oid), OID_AUTO,
	+ "ktls_stats", CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	+ "In-kernel TLS statistics");
	+ if (!oid)
	+ return ENOMEM;
	+
	+ for (int i = 0; i < ktls_counters_count; i++) {
	+ const struct bnxt_ktls_counters_list *counters = &ktls_counters[i];
	+
	+ SYSCTL_ADD_PROC(&softc->ktls_stats,
	+ SYSCTL_CHILDREN(oid), OID_AUTO,
	+ counters->label,
	+ CTLTYPE_U64 \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
	+ &ktls->counters[counters->id],
	+ 0,
	+ sysctl_counter_u64_handler,
	+ "Q",
	+ counters->desc);
	+ }
	+
	+ for (int i = 0; i < ktls_err_counters_count; i++) {
	+ const struct bnxt_ktls_err_counters_list *err_counters = &ktls_err_counters[i];
	+
	+ SYSCTL_ADD_PROC(&softc->ktls_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ err_counters->label,
	+ CTLTYPE_U64 \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
	+ &ktls->err_counters[i], 0,
	+ sysctl_counter_u64_handler, "Q",
	+ err_counters->desc);
	+ }
	+
	+ return 0;
	+}
	+
	+int
	+bnxt_ktls_sysctls(struct bnxt_softc *softc)
	+{
	+ struct sysctl_ctx_list *ctx;
	+ struct sysctl_oid_list *children;
	+ struct sysctl_oid *oid;
	+
	+ ctx = device_get_sysctl_ctx(softc->dev);
	+ children = SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev));
	+
	+ oid = SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "max_ktls_entries",
	+ CTLFLAG_RDTUN, &softc->max_ktls_entries, 0,
	+ "Maximum number of TLS offload entries to preallocate (default: 102400, max: 512000)");
	+ if (!oid)
	+ return ENOMEM;
	+
	+ return 0;
	+}
	+
	+int
	+bnxt_create_mpc_cmp_time_sysctls(struct bnxt_softc *softc)
	+{
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ struct sysctl_oid *oid;
	+
	+ if (!ktls)
	+ return 0;
	+
	+ oid = SYSCTL_ADD_NODE(&softc->mpc_cmp_time_stats,
	+ SYSCTL_CHILDREN(softc->mpc_cmp_time_stats_oid), OID_AUTO,
	+ "mpc_cmp_stats", CTLFLAG_RD \| CTLFLAG_MPSAFE, 0, "MPC CMP time statistics");
	+ if (!oid)
	+ return ENOMEM;
	+
	+ for (int i = 0; i < mpc_cmp_counters_count; i++) {
	+ const struct bnxt_mpc_cmp_time_counters_list *mpc_counters = &mpc_cmp_counters[i];
	+
	+ SYSCTL_ADD_PROC(&softc->mpc_cmp_time_stats,
	+ SYSCTL_CHILDREN(oid), OID_AUTO,
	+ mpc_counters->label,
	+ CTLTYPE_U64 \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
	+ &ktls->mpc_cmp_time[i],
	+ 0,
	+ sysctl_counter_u64_handler,
	+ "Q",
	+ mpc_counters->desc);
	+ }
	+
	+ return 0;
	+}
	Index: sys/dev/bnxt/bnxt_en/bnxt_txrx.c
	===================================================================
	--- sys/dev/bnxt/bnxt_en/bnxt_txrx.c
	+++ sys/dev/bnxt/bnxt_en/bnxt_txrx.c
	@@ -37,10 +37,13 @@
	#include "opt_inet.h"
	#include "opt_inet6.h"
	#include "opt_rss.h"
	+#include "opt_kern_tls.h"

	#include <machine/atomic.h>
	#include "bnxt.h"
	#include "bnxt_ptp.h"
	+#include "bnxt_ktls.h"
	+#include "bnxt_log.h"

	/*
	* Function prototypes
	@@ -79,7 +82,7 @@
	* Device Dependent Packet Transmit and Receive Functions
	*/

	-static const uint16_t bnxt_tx_lhint[] = {
	+const uint16_t bnxt_tx_lhint[] = {
	TX_BD_SHORT_FLAGS_LHINT_LT512,
	TX_BD_SHORT_FLAGS_LHINT_LT1K,
	TX_BD_SHORT_FLAGS_LHINT_LT2K,
	@@ -97,12 +100,27 @@
	bool need_cpl = false;
	bool need_hi = false;
	uint16_t flags_type;
	- uint16_t lflags;
	+ uint16_t lflags = 0;
	uint32_t cfa_meta;
	int seg = 0;
	uint8_t wrap = 0;
	struct tx_bd_opaque *opq;

	+ txr->prod = pi->ipi_pidx;
	+#ifdef KTLS_IFLIB_SUPPORT
	+ uint32_t kid = 0;
	+ int ret;
	+
	+ if (pi->ipi_mbuf != NULL) {
	+ ret = bnxt_ktls_xmit(softc, txr, &pi->ipi_mbuf, &lflags, &kid, pi);
	+
	+ if (ret) {
	+ BNXT_DEBUG(softc->dev, "kTLS xmit failed, ret=%d\n", ret);
	+ return ret;
	+ }
	+ }
	+#endif
	+
	if ((pi->ipi_flags & IPI_TX_INTR) != 0)
	need_cpl = true;

	@@ -121,7 +139,7 @@
	/* TODO: Devices before Cu+B1 need to not mix long and short BDs */
	need_hi = true;

	- pi->ipi_new_pidx = pi->ipi_pidx;
	+ pi->ipi_new_pidx = txr->prod;
	tbd = &((struct tx_bd_long *)txr->vaddr)[pi->ipi_new_pidx];
	pi->ipi_ndescs = 0;
	opq = (struct tx_bd_opaque *)&tbd->opaque;
	@@ -165,11 +183,21 @@
	txr->epoch_arr[pi->ipi_new_pidx] = txr->epoch_bit;

	tbdh = &((struct tx_bd_long_hi *)txr->vaddr)[pi->ipi_new_pidx];
	+#ifdef KTLS_IFLIB_SUPPORT
	+ tbdh->kid_or_ts_high_mss = htole32((pi->ipi_tso_segsz & TX_BD_LONG_MSS_MASK) \|
	+ ((KID_HIGH(kid) << TX_BD_LONG_KID_OR_TS_HIGH_SFT) &
	+ TX_BD_LONG_KID_OR_TS_HIGH_MASK));
	+
	+ tbdh->kid_or_ts_low_hdr_size = htole16((((pi->ipi_ehdrlen + pi->ipi_ip_hlen +
	+ pi->ipi_tcp_hlen) >> 1) & TX_BD_LONG_HDR_SIZE_MASK) \|
	+ ((KID_LOW(kid) << TX_BD_LONG_KID_OR_TS_LOW_SFT) &
	+ TX_BD_LONG_KID_OR_TS_LOW_MASK));
	+#else
	tbdh->kid_or_ts_high_mss = htole16(pi->ipi_tso_segsz);
	tbdh->kid_or_ts_low_hdr_size = htole16((pi->ipi_ehdrlen + pi->ipi_ip_hlen +
	pi->ipi_tcp_hlen) >> 1);
	+#endif
	tbdh->cfa_action = 0;
	- lflags = 0;
	cfa_meta = 0;
	if (pi->ipi_mflags & M_VLANTAG) {
	/* TODO: Do we need to byte-swap the vtag here? */
	@@ -231,6 +259,7 @@
	if (pi->ipi_new_pidx < EPOCH_ARR_SZ)
	txr->epoch_arr[pi->ipi_new_pidx] = txr->epoch_bit;

	+ txr->prod = pi->ipi_new_pidx;
	return 0;
	}

	@@ -252,6 +281,7 @@
	struct bnxt_cp_ring *cpr = &softc->tx_cp_rings[txqid];
	struct tx_cmpl cmpl = (struct tx_cmpl )cpr->ring.vaddr;
	struct bnxt_ring *txr;
	+ struct bnxt_sw_tx_bd *tx_buf;
	int avail = 0;
	uint32_t cons = cpr->cons;
	uint32_t raw_cons = cpr->raw_cons;
	@@ -294,6 +324,32 @@
	*/
	if (!clear)
	goto done;
	+
	+ if (__predict_false(opq->ktls_replay)) {
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ struct bnxt_replay_pkt *pkt;
	+ uint32_t idx = opq->idx;
	+
	+ txr = &softc->tx_rings[txqid];
	+ if (idx >= txr->ring_size) {
	+ bnxt_log_live(softc, BNXT_LOGGER_L2,
	+ "%s: Invalid replay idx %u (ring_size=%u) "
	+ "on TXQ %u, opaque 0x%x\n",
	+ __func__, idx, txr->ring_size, txqid,
	+ cmpl[cons].opaque);
	+ break;
	+ }
	+ tx_buf = &txr->tx_buf_ring[idx];
	+ pkt = &txr->replay_pkt[idx];
	+ MPASS(tx_buf->is_replay);
	+ bus_dmamap_sync(ktls->dma_tag, pkt->dma_map,
	+ BUS_DMASYNC_POSTWRITE);
	+ bus_dmamap_unload(ktls->dma_tag, pkt->dma_map);
	+ m_freem(pkt->mbuf);
	+ counter_u64_add(ktls->counters[BNXT_KTLS_TX_MBUF_FREES], 1);
	+ pkt->mbuf = NULL;
	+ tx_buf->is_replay = 0;
	+ }
	break;
	case TX_CMPL_COAL_TYPE_TX_L2_COAL:
	if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL) {
	Index: sys/dev/bnxt/bnxt_en/if_bnxt.c
	===================================================================
	--- sys/dev/bnxt/bnxt_en/if_bnxt.c
	+++ sys/dev/bnxt/bnxt_en/if_bnxt.c
	@@ -60,6 +60,7 @@
	#include "opt_inet.h"
	#include "opt_inet6.h"
	#include "opt_rss.h"
	+#include "opt_kern_tls.h"

	#include "ifdi_if.h"

	@@ -76,6 +77,7 @@
	#include "bnxt_coredump.h"
	#include "bnxt_ptp.h"
	#include "bnxt_mpc.h"
	+#include "bnxt_ktls.h"

	/*
	* PCI Device ID Table
	@@ -648,6 +650,12 @@
	softc->tx_rings[i].db_ring_mask = softc->tx_rings[i].ring_size - 1;
	softc->tx_rings[i].vaddr = vaddrs[i * ntxqs + 1];
	softc->tx_rings[i].paddr = paddrs[i * ntxqs + 1];
	+ softc->tx_rings[i].tx_buf_ring =
	+ malloc(softc->tx_rings[i].ring_size * sizeof(struct bnxt_sw_tx_bd),
	+ M_DEVBUF, M_NOWAIT \| M_ZERO);
	+
	+ if (!softc->tx_rings[i].tx_buf_ring)
	+ return -ENOMEM;

	bnxt_create_tx_sysctls(softc, i);

	@@ -666,6 +674,27 @@
	softc->nq_rings[i].ring.vaddr = vaddrs[i * ntxqs + 2];
	softc->nq_rings[i].ring.paddr = paddrs[i * ntxqs + 2];
	softc->nq_rings[i].type = Q_TYPE_TX;
	+
	+#if defined(KERN_TLS)
	+ int j;
	+ int err;
	+
	+ if (softc->ktls_info) {
	+ err = bnxt_alloc_ktls_rexmit_mbuf(softc, &softc->tx_rings[i]);
	+ if (err) {
	+ /* Disable KTLs entirely when enough memory is not avaiable */
	+ for (j = i - 1; j >=0; j--)
	+ bnxt_free_ktls_rexmit_mbuf(&softc->tx_rings[j]);
	+
	+ device_printf(softc->dev,
	+ "Error:%d, kTLS support is disabled\n", err);
	+
	+ if_setcapenablebit(iflib_get_ifp(ctx), 0,
	+ (IFCAP_TXTLS4 \| IFCAP_TXTLS6));
	+ bnxt_free_ktls_info(softc);
	+ }
	+ }
	+#endif
	}
	}

	@@ -700,8 +729,10 @@
	int i;

	// Free TX queues
	- for (i=0; i<softc->ntxqsets; i++)
	+ for (i=0; i<softc->ntxqsets; i++) {
	iflib_dma_free(&softc->tx_stats[i]);
	+ free(softc->tx_rings[i].tx_buf_ring, M_DEVBUF);
	+ }
	free(softc->tx_rings, M_DEVBUF);
	softc->tx_rings = NULL;
	free(softc->tx_cp_rings, M_DEVBUF);
	@@ -1325,6 +1356,22 @@
	last_type = BNXT_CTX_MTQM;
	}

	+ if (BNXT_PF(softc)) {
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ uint32_t max_tx_ctx = 0;
	+
	+ if (ktls)
	+ max_tx_ctx = ktls->tck.max_ctx;
	+
	+ if (max_tx_ctx) {
	+ ctxm = &ctx->ctx_arr[BNXT_CTX_TCK];
	+ rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, max_tx_ctx, 1);
	+ if (rc)
	+ return rc;
	+ last_type = BNXT_CTX_TCK;
	+ }
	+ }
	+
	if (BNXT_PF(softc)) {
	for (type = BNXT_CTX_SRT_TRACE; type <= BNXT_CTX_ROCE_HWRM_TRACE; type++) {
	ctxm = &ctx->ctx_arr[type];
	@@ -1923,7 +1970,9 @@
	}
	if (pci_is_enabled(bp->pdev))
	pci_disable_device(bp->pdev);
	+ bnxt_ktls_del_all(bp);
	pci_disable_busmaster(bp->dev);
	+ bnxt_clear_ktls(bp);
	bnxt_free_ctx_mem(bp);
	}

	@@ -2761,6 +2810,28 @@
	memcpy(softc->rx_q_ids, softc->tx_q_ids, sizeof(softc->rx_q_ids));
	}

	+#if defined(KERN_TLS)
	+ if (BNXT_CHIP_P7(softc)) {
	+ rc = bnxt_ktls_sysctls(softc);
	+ if (rc) {
	+ device_printf(softc->dev, "attach: max_ktls_entries sysctl creation failed\n");
	+ goto failed;
	+ }
	+
	+ if (softc->max_ktls_entries == 0 \|\|
	+ softc->max_ktls_entries > BNXT_KTLS_MAX_ENTRIES ) {
	+
	+ if (softc->max_ktls_entries > BNXT_KTLS_MAX_ENTRIES)
	+ device_printf(softc->dev,
	+ "attach: max_ktls_entries (%d) exceeds allowed max (%d), resetting to default (%d)\n",
	+ softc->max_ktls_entries, BNXT_KTLS_MAX_ENTRIES, BNXT_KTLS_ENTRIES_DEFAULT);
	+
	+ softc->max_ktls_entries = BNXT_KTLS_ENTRIES_DEFAULT;
	+ }
	+
	+ }
	+#endif
	+
	/* Get the HW capabilities */
	rc = bnxt_hwrm_func_qcaps(softc);
	if (rc)
	@@ -2840,6 +2911,26 @@
	if (rc)
	goto failed;

	+#if defined(KERN_TLS)
	+ /* Initialize KTLS */
	+ rc = bnxt_ktls_init(softc);
	+ if (!rc && softc->ktls_info) {
	+ if_t ifp = iflib_get_ifp(ctx);
	+
	+ bnxt_alloc_ktls_counters(softc);
	+ if_setsndtagallocfn(ifp, bnxt_tls_snd_tag_alloc);
	+ device_printf(softc->dev, "kTLS is supported ifp:%p ctx:%p dev: %p softc: %p\n",
	+ ifp, ctx, iflib_get_dev(ctx), softc);
	+
	+ scctx->isc_capabilities \|= IFCAP_TXTLS4 \| IFCAP_TXTLS6;
	+ scctx->isc_capenable \|= IFCAP_TXTLS4 \| IFCAP_TXTLS6;
	+#ifdef KTLS_IFLIB_SUPPORT
	+ scctx->isc_tx_pad = BNXT_MAX_NUM_SEGS + 5;
	+#endif
	+ } else
	+ bnxt_free_ktls_info(softc);
	+#endif
	+
	/* Now set up iflib sc */
	scctx->isc_tx_nsegments = 31,
	scctx->isc_tx_tso_segments_max = 31;
	@@ -2957,6 +3048,16 @@
	if (rc)
	goto failed;

	+ if (softc->fw_cap & BNXT_FW_CAP_KTLS_SUPPORTED) {
	+ rc = bnxt_create_ktls_sysctls(softc);
	+ if (rc)
	+ goto failed;
	+
	+ rc = bnxt_create_mpc_cmp_time_sysctls(softc);
	+ if (rc)
	+ goto failed;
	+ }
	+
	if (softc->fw_cap & BNXT_FW_CAP_PTP) {
	rc = bnxt_hwrm_ptp_qcfg(softc);
	if (!rc) {
	@@ -3076,12 +3177,48 @@
	cancel_delayed_work_sync(&softc->fw_reset_task);
	cancel_work_sync(&softc->sp_task);
	bnxt_dcb_free(softc);
	+
	+#ifdef KERN_TLS
	+ {
	+ struct bnxt_tls_info *ktls = softc->ktls_info;
	+ int wait;
	+
	+ if (ktls) {
	+ for (i = 0; i < softc->ntxqsets; i++)
	+ bnxt_free_ktls_rexmit_mbuf(&softc->tx_rings[i]);
	+
	+ /* wait for all threads in bnxt_tls_snd_tag_alloc() to exit */
	+ while (atomic_load_32(&ktls->snd_tag_alloc_ref) != 0) {
	+ pause("W", hz);
	+ }
	+
	+ device_printf(softc->dev, "Active sessions before wait: %u\n",
	+ uma_zone_get_cur(ktls->zone));
	+ /* wait for all TLS tags to get freed */
	+ while (ktls->init != 0 &&
	+ uma_zone_get_cur(ktls->zone) != 0) {
	+ pause("W", hz);
	+ }
	+
	+ wait = 30;
	+ while (wait && atomic_load_32(&ktls->pending)) {
	+ device_printf(softc->dev, "KTLS device add/delete pending: %u"
	+ " waittime[%d]\n", wait, atomic_load_32(&ktls->pending));
	+ pause_sbt("bnxt_detach_sleep", mstosbt(1000 * 2), 0, C_HARDCLOCK);
	+ wait--;
	+ }
	+ }
	+ }
	+#endif
	+
	SLIST_REMOVE(&pf_list, &softc->list, bnxt_softc_list, next);
	bnxt_num_pfs--;
	bnxt_wol_config(ctx);
	bnxt_do_disable_intr(&softc->def_cp_ring);
	bnxt_ptp_free(softc);
	bnxt_free_sysctl_ctx(softc);
	+ bnxt_free_ktls_counters(softc);
	+ bnxt_free_ktls_info(softc);
	bnxt_free_crash_dump_mem(softc);
	bnxt_hwrm_func_reset(softc);
	/*
	@@ -3127,8 +3264,10 @@
	bnxt_unregister_logger(softc, BNXT_LOGGER_L2);
	mtx_destroy(&softc->log_lock);

	- if (!bnxt_num_pfs && bnxt_pf_wq)
	+ if (!bnxt_num_pfs && bnxt_pf_wq) {
	destroy_workqueue(bnxt_pf_wq);
	+ bnxt_pf_wq = NULL;
	+ }

	if (softc->pdev)
	linux_pci_detach_device(softc->pdev);
	Index: sys/modules/bnxt/bnxt_en/Makefile
	===================================================================
	--- sys/modules/bnxt/bnxt_en/Makefile
	+++ sys/modules/bnxt/bnxt_en/Makefile
	@@ -2,7 +2,7 @@

	KMOD = if_bnxt
	SRCS = device_if.h bus_if.h pci_if.h pci_iov_if.h ifdi_if.h
	-SRCS += opt_inet.h opt_inet6.h opt_rss.h
	+SRCS += opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h opt_kern_tls.h
	SRCS += bnxt_txrx.c if_bnxt.c
	SRCS += bnxt_hwrm.c
	SRCS += bnxt_sysctl.c
	@@ -16,6 +16,7 @@
	SRCS += bnxt_coredump.c bnxt_coredump.h
	SRCS += bnxt_ptp.c bnxt_ptp.h
	SRCS += bnxt_mpc.c bnxt_mpc.h
	+SRCS += bnxt_ktls.c bnxt_ktls.h
	SRCS += bnxt_compat.h
	SRCS += ${LINUXKPI_GENSRCS}

File Metadata

Mime Type: text/plain
Expires: Tue, Nov 10, 3:03 AM (5 h, 28 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 41625240
Default Alt Text: D58606.1794279800.diff (100 KB)

D58606.1794279800.diffNo OneTemporaryActions

D58606.1794279800.diffView Options

File Metadata

Event Timeline

D58606.1794279800.diff
No OneTemporary
Actions

D58606.1794279800.diff
View Options