Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/cxgbe/t4_main.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show All 27 Lines | |||||
*/ | */ | ||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include "opt_ddb.h" | #include "opt_ddb.h" | ||||
#include "opt_inet.h" | #include "opt_inet.h" | ||||
#include "opt_inet6.h" | #include "opt_inet6.h" | ||||
#include "opt_kern_tls.h" | |||||
#include "opt_ratelimit.h" | #include "opt_ratelimit.h" | ||||
#include "opt_rss.h" | #include "opt_rss.h" | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/conf.h> | #include <sys/conf.h> | ||||
#include <sys/priv.h> | #include <sys/priv.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/bus.h> | #include <sys/bus.h> | ||||
Show All 16 Lines | |||||
#include <net/if_types.h> | #include <net/if_types.h> | ||||
#include <net/if_dl.h> | #include <net/if_dl.h> | ||||
#include <net/if_vlan_var.h> | #include <net/if_vlan_var.h> | ||||
#ifdef RSS | #ifdef RSS | ||||
#include <net/rss_config.h> | #include <net/rss_config.h> | ||||
#endif | #endif | ||||
#include <netinet/in.h> | #include <netinet/in.h> | ||||
#include <netinet/ip.h> | #include <netinet/ip.h> | ||||
#ifdef KERN_TLS | |||||
#include <netinet/tcp_seq.h> | |||||
#endif | |||||
#if defined(__i386__) || defined(__amd64__) | #if defined(__i386__) || defined(__amd64__) | ||||
#include <machine/md_var.h> | #include <machine/md_var.h> | ||||
#include <machine/cputypes.h> | #include <machine/cputypes.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#endif | #endif | ||||
#include <crypto/rijndael/rijndael.h> | #include <crypto/rijndael/rijndael.h> | ||||
#ifdef DDB | #ifdef DDB | ||||
▲ Show 20 Lines • Show All 149 Lines • ▼ Show 20 Lines | static driver_t vcc_driver = { | ||||
sizeof(struct vi_info) | sizeof(struct vi_info) | ||||
}; | }; | ||||
/* ifnet interface */ | /* ifnet interface */ | ||||
static void cxgbe_init(void *); | static void cxgbe_init(void *); | ||||
static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); | static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); | ||||
static int cxgbe_transmit(struct ifnet *, struct mbuf *); | static int cxgbe_transmit(struct ifnet *, struct mbuf *); | ||||
static void cxgbe_qflush(struct ifnet *); | static void cxgbe_qflush(struct ifnet *); | ||||
#ifdef RATELIMIT | #if defined(KERN_TLS) || defined(RATELIMIT) | ||||
static int cxgbe_snd_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *, | static int cxgbe_snd_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *, | ||||
struct m_snd_tag **); | struct m_snd_tag **); | ||||
static int cxgbe_snd_tag_modify(struct m_snd_tag *, | static int cxgbe_snd_tag_modify(struct m_snd_tag *, | ||||
union if_snd_tag_modify_params *); | union if_snd_tag_modify_params *); | ||||
static int cxgbe_snd_tag_query(struct m_snd_tag *, | static int cxgbe_snd_tag_query(struct m_snd_tag *, | ||||
union if_snd_tag_query_params *); | union if_snd_tag_query_params *); | ||||
static void cxgbe_snd_tag_free(struct m_snd_tag *); | static void cxgbe_snd_tag_free(struct m_snd_tag *); | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 330 Lines • ▼ Show 20 Lines | |||||
* TOE tunables. | * TOE tunables. | ||||
*/ | */ | ||||
static int t4_cop_managed_offloading = 0; | static int t4_cop_managed_offloading = 0; | ||||
SYSCTL_INT(_hw_cxgbe, OID_AUTO, cop_managed_offloading, CTLFLAG_RDTUN, | SYSCTL_INT(_hw_cxgbe, OID_AUTO, cop_managed_offloading, CTLFLAG_RDTUN, | ||||
&t4_cop_managed_offloading, 0, | &t4_cop_managed_offloading, 0, | ||||
"COP (Connection Offload Policy) controls all TOE offload"); | "COP (Connection Offload Policy) controls all TOE offload"); | ||||
#endif | #endif | ||||
#ifdef KERN_TLS | |||||
/* | |||||
* This enables KERN_TLS for all adapters if set. | |||||
*/ | |||||
static int t4_kern_tls = 0; | |||||
SYSCTL_INT(_hw_cxgbe, OID_AUTO, kern_tls, CTLFLAG_RDTUN, &t4_kern_tls, 0, | |||||
"Enable KERN_TLS mode for all supported adapters"); | |||||
SYSCTL_NODE(_hw_cxgbe, OID_AUTO, tls, CTLFLAG_RD, 0, | |||||
"cxgbe(4) KERN_TLS parameters"); | |||||
static int t4_tls_inline_keys = 0; | |||||
SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, inline_keys, CTLFLAG_RDTUN, | |||||
&t4_tls_inline_keys, 0, | |||||
"Always pass TLS keys in work requests (1) or attempt to store TLS keys " | |||||
"in card memory."); | |||||
static int t4_tls_combo_wrs = 0; | |||||
SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, combo_wrs, CTLFLAG_RDTUN, &t4_tls_combo_wrs, | |||||
0, "Attempt to combine TCB field updates with TLS record work requests."); | |||||
#endif | |||||
/* Functions used by VIs to obtain unique MAC addresses for each VI. */ | /* Functions used by VIs to obtain unique MAC addresses for each VI. */ | ||||
static int vi_mac_funcs[] = { | static int vi_mac_funcs[] = { | ||||
FW_VI_FUNC_ETH, | FW_VI_FUNC_ETH, | ||||
FW_VI_FUNC_OFLD, | FW_VI_FUNC_OFLD, | ||||
FW_VI_FUNC_IWARP, | FW_VI_FUNC_IWARP, | ||||
FW_VI_FUNC_OPENISCSI, | FW_VI_FUNC_OPENISCSI, | ||||
FW_VI_FUNC_OPENFCOE, | FW_VI_FUNC_OPENFCOE, | ||||
FW_VI_FUNC_FOISCSI, | FW_VI_FUNC_FOISCSI, | ||||
▲ Show 20 Lines • Show All 418 Lines • ▼ Show 20 Lines | #endif | ||||
TAILQ_INIT(&sc->sfl); | TAILQ_INIT(&sc->sfl); | ||||
callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); | callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); | ||||
mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); | mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); | ||||
sc->policy = NULL; | sc->policy = NULL; | ||||
rw_init(&sc->policy_lock, "connection offload policy"); | rw_init(&sc->policy_lock, "connection offload policy"); | ||||
callout_init(&sc->ktls_tick, 1); | |||||
rc = t4_map_bars_0_and_4(sc); | rc = t4_map_bars_0_and_4(sc); | ||||
if (rc != 0) | if (rc != 0) | ||||
goto done; /* error message displayed already */ | goto done; /* error message displayed already */ | ||||
memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); | memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); | ||||
/* Prepare the adapter for operation. */ | /* Prepare the adapter for operation. */ | ||||
buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK); | buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK); | ||||
▲ Show 20 Lines • Show All 220 Lines • ▼ Show 20 Lines | s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, | ||||
M_ZERO | M_WAITOK); | M_ZERO | M_WAITOK); | ||||
sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, | sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, | ||||
M_ZERO | M_WAITOK); | M_ZERO | M_WAITOK); | ||||
t4_init_l2t(sc, M_WAITOK); | t4_init_l2t(sc, M_WAITOK); | ||||
t4_init_smt(sc, M_WAITOK); | t4_init_smt(sc, M_WAITOK); | ||||
t4_init_tx_sched(sc); | t4_init_tx_sched(sc); | ||||
t4_init_atid_table(sc); | t4_init_atid_table(sc); | ||||
jhb: The atid table change I could break out into a separate commit first if that's better. | |||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
t4_init_etid_table(sc); | t4_init_etid_table(sc); | ||||
#endif | #endif | ||||
#ifdef INET6 | #ifdef INET6 | ||||
t4_init_clip_table(sc); | t4_init_clip_table(sc); | ||||
#endif | #endif | ||||
if (sc->vres.key.size != 0) | if (sc->vres.key.size != 0) | ||||
sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start, | sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start, | ||||
▲ Show 20 Lines • Show All 321 Lines • ▼ Show 20 Lines | #endif | ||||
free(sc->sge.eqmap, M_CXGBE); | free(sc->sge.eqmap, M_CXGBE); | ||||
free(sc->tids.ftid_tab, M_CXGBE); | free(sc->tids.ftid_tab, M_CXGBE); | ||||
free(sc->tids.hpftid_tab, M_CXGBE); | free(sc->tids.hpftid_tab, M_CXGBE); | ||||
free_hftid_hash(&sc->tids); | free_hftid_hash(&sc->tids); | ||||
free(sc->tids.tid_tab, M_CXGBE); | free(sc->tids.tid_tab, M_CXGBE); | ||||
free(sc->tt.tls_rx_ports, M_CXGBE); | free(sc->tt.tls_rx_ports, M_CXGBE); | ||||
t4_destroy_dma_tag(sc); | t4_destroy_dma_tag(sc); | ||||
callout_drain(&sc->ktls_tick); | |||||
callout_drain(&sc->sfl_callout); | callout_drain(&sc->sfl_callout); | ||||
if (mtx_initialized(&sc->tids.ftid_lock)) { | if (mtx_initialized(&sc->tids.ftid_lock)) { | ||||
mtx_destroy(&sc->tids.ftid_lock); | mtx_destroy(&sc->tids.ftid_lock); | ||||
cv_destroy(&sc->tids.ftid_cv); | cv_destroy(&sc->tids.ftid_cv); | ||||
} | } | ||||
if (mtx_initialized(&sc->tids.atid_lock)) | if (mtx_initialized(&sc->tids.atid_lock)) | ||||
mtx_destroy(&sc->tids.atid_lock); | mtx_destroy(&sc->tids.atid_lock); | ||||
if (mtx_initialized(&sc->ifp_lock)) | if (mtx_initialized(&sc->ifp_lock)) | ||||
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | cxgbe_vi_attach(device_t dev, struct vi_info *vi) | ||||
if_initname(ifp, device_get_name(dev), device_get_unit(dev)); | if_initname(ifp, device_get_name(dev), device_get_unit(dev)); | ||||
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; | ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; | ||||
ifp->if_init = cxgbe_init; | ifp->if_init = cxgbe_init; | ||||
ifp->if_ioctl = cxgbe_ioctl; | ifp->if_ioctl = cxgbe_ioctl; | ||||
ifp->if_transmit = cxgbe_transmit; | ifp->if_transmit = cxgbe_transmit; | ||||
ifp->if_qflush = cxgbe_qflush; | ifp->if_qflush = cxgbe_qflush; | ||||
ifp->if_get_counter = cxgbe_get_counter; | ifp->if_get_counter = cxgbe_get_counter; | ||||
#ifdef RATELIMIT | #if defined(KERN_TLS) || defined(RATELIMIT) | ||||
ifp->if_snd_tag_alloc = cxgbe_snd_tag_alloc; | ifp->if_snd_tag_alloc = cxgbe_snd_tag_alloc; | ||||
ifp->if_snd_tag_modify = cxgbe_snd_tag_modify; | ifp->if_snd_tag_modify = cxgbe_snd_tag_modify; | ||||
ifp->if_snd_tag_query = cxgbe_snd_tag_query; | ifp->if_snd_tag_query = cxgbe_snd_tag_query; | ||||
ifp->if_snd_tag_free = cxgbe_snd_tag_free; | ifp->if_snd_tag_free = cxgbe_snd_tag_free; | ||||
#endif | |||||
#ifdef RATELIMIT | |||||
ifp->if_ratelimit_query = cxgbe_ratelimit_query; | ifp->if_ratelimit_query = cxgbe_ratelimit_query; | ||||
#endif | #endif | ||||
ifp->if_capabilities = T4_CAP; | ifp->if_capabilities = T4_CAP; | ||||
ifp->if_capenable = T4_CAP_ENABLE; | ifp->if_capenable = T4_CAP_ENABLE; | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
if (vi->nofldrxq != 0) | if (vi->nofldrxq != 0 && (vi->pi->adapter->flags & KERN_TLS_OK) == 0) | ||||
ifp->if_capabilities |= IFCAP_TOE; | ifp->if_capabilities |= IFCAP_TOE; | ||||
#endif | #endif | ||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0) { | if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0) { | ||||
ifp->if_capabilities |= IFCAP_TXRTLMT; | ifp->if_capabilities |= IFCAP_TXRTLMT; | ||||
ifp->if_capenable |= IFCAP_TXRTLMT; | ifp->if_capenable |= IFCAP_TXRTLMT; | ||||
} | } | ||||
#endif | #endif | ||||
ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | | ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | | ||||
CSUM_UDP_IPV6 | CSUM_TCP_IPV6; | CSUM_UDP_IPV6 | CSUM_TCP_IPV6; | ||||
ifp->if_hw_tsomax = IP_MAXPACKET; | ifp->if_hw_tsomax = IP_MAXPACKET; | ||||
ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_TSO; | ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_TSO; | ||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0) | if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0) | ||||
ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_EO_TSO; | ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_EO_TSO; | ||||
#endif | #endif | ||||
ifp->if_hw_tsomaxsegsize = 65536; | ifp->if_hw_tsomaxsegsize = 65536; | ||||
#ifdef KERN_TLS | |||||
if (vi->pi->adapter->flags & KERN_TLS_OK) { | |||||
ifp->if_capabilities |= IFCAP_TXTLS; | |||||
ifp->if_capenable |= IFCAP_TXTLS; | |||||
} | |||||
#endif | |||||
ether_ifattach(ifp, vi->hw_addr); | ether_ifattach(ifp, vi->hw_addr); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
if (vi->nnmrxq != 0) | if (vi->nnmrxq != 0) | ||||
cxgbe_nm_attach(vi); | cxgbe_nm_attach(vi); | ||||
#endif | #endif | ||||
sb = sbuf_new_auto(); | sb = sbuf_new_auto(); | ||||
sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); | sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); | ||||
▲ Show 20 Lines • Show All 292 Lines • ▼ Show 20 Lines | if (mask & IFCAP_HWRXTSTMP) { | ||||
rxq->iq.flags |= IQ_RX_TIMESTAMP; | rxq->iq.flags |= IQ_RX_TIMESTAMP; | ||||
else | else | ||||
rxq->iq.flags &= ~IQ_RX_TIMESTAMP; | rxq->iq.flags &= ~IQ_RX_TIMESTAMP; | ||||
} | } | ||||
} | } | ||||
if (mask & IFCAP_NOMAP) | if (mask & IFCAP_NOMAP) | ||||
ifp->if_capenable ^= IFCAP_NOMAP; | ifp->if_capenable ^= IFCAP_NOMAP; | ||||
#ifdef KERN_TLS | |||||
if (mask & IFCAP_TXTLS) | |||||
ifp->if_capenable ^= (mask & IFCAP_TXTLS); | |||||
#endif | |||||
#ifdef VLAN_CAPABILITIES | #ifdef VLAN_CAPABILITIES | ||||
VLAN_CAPABILITIES(ifp); | VLAN_CAPABILITIES(ifp); | ||||
#endif | #endif | ||||
fail: | fail: | ||||
end_synchronized_op(sc, 0); | end_synchronized_op(sc, 0); | ||||
break; | break; | ||||
case SIOCSIFMEDIA: | case SIOCSIFMEDIA: | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
struct cxgbe_snd_tag *cst; | struct cxgbe_snd_tag *cst; | ||||
#endif | #endif | ||||
void *items[1]; | void *items[1]; | ||||
int rc; | int rc; | ||||
M_ASSERTPKTHDR(m); | M_ASSERTPKTHDR(m); | ||||
MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ | MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ | ||||
#ifdef RATELIMIT | #if defined(KERN_TLS) || defined(RATELIMIT) | ||||
if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) | if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) | ||||
MPASS(m->m_pkthdr.snd_tag->ifp == ifp); | MPASS(m->m_pkthdr.snd_tag->ifp == ifp); | ||||
#endif | #endif | ||||
if (__predict_false(pi->link_cfg.link_ok == false)) { | if (__predict_false(pi->link_cfg.link_ok == false)) { | ||||
m_freem(m); | m_freem(m); | ||||
return (ENETDOWN); | return (ENETDOWN); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Lines | cxgbe_get_counter(struct ifnet *ifp, ift_counter c) | ||||
} | } | ||||
default: | default: | ||||
return (if_get_counter_default(ifp, c)); | return (if_get_counter_default(ifp, c)); | ||||
} | } | ||||
} | } | ||||
#ifdef RATELIMIT | #if defined(KERN_TLS) || defined(RATELIMIT) | ||||
Done Inline ActionsI could perhaps break out the snd tag changes to split cxgbe_snd_tag from cxgbe_rate_tag out as well, basically keeping these wrapper routines but removing the KERN_TLS bits from them and handle the structure renames in other places for cxgbe_rate_tag. jhb: I could perhaps break out the snd tag changes to split cxgbe_snd_tag from cxgbe_rate_tag out as… | |||||
void | void | ||||
cxgbe_snd_tag_init(struct cxgbe_snd_tag *cst, struct ifnet *ifp, int type) | cxgbe_snd_tag_init(struct cxgbe_snd_tag *cst, struct ifnet *ifp, int type) | ||||
{ | { | ||||
m_snd_tag_init(&cst->com, ifp); | m_snd_tag_init(&cst->com, ifp); | ||||
cst->type = type; | cst->type = type; | ||||
} | } | ||||
static int | static int | ||||
cxgbe_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, | cxgbe_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, | ||||
struct m_snd_tag **pt) | struct m_snd_tag **pt) | ||||
{ | { | ||||
int error; | int error; | ||||
switch (params->hdr.type) { | switch (params->hdr.type) { | ||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
case IF_SND_TAG_TYPE_RATE_LIMIT: | case IF_SND_TAG_TYPE_RATE_LIMIT: | ||||
error = cxgbe_rate_tag_alloc(ifp, params, pt); | error = cxgbe_rate_tag_alloc(ifp, params, pt); | ||||
break; | break; | ||||
#endif | #endif | ||||
#ifdef KERN_TLS | |||||
case IF_SND_TAG_TYPE_TLS: | |||||
error = cxgbe_tls_tag_alloc(ifp, params, pt); | |||||
break; | |||||
#endif | |||||
default: | default: | ||||
error = EOPNOTSUPP; | error = EOPNOTSUPP; | ||||
} | } | ||||
if (error == 0) | if (error == 0) | ||||
MPASS(mst_to_cst(*pt)->type == params->hdr.type); | MPASS(mst_to_cst(*pt)->type == params->hdr.type); | ||||
return (error); | return (error); | ||||
} | } | ||||
Show All 38 Lines | cxgbe_snd_tag_free(struct m_snd_tag *mst) | ||||
cst = mst_to_cst(mst); | cst = mst_to_cst(mst); | ||||
switch (cst->type) { | switch (cst->type) { | ||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
case IF_SND_TAG_TYPE_RATE_LIMIT: | case IF_SND_TAG_TYPE_RATE_LIMIT: | ||||
cxgbe_rate_tag_free(mst); | cxgbe_rate_tag_free(mst); | ||||
return; | return; | ||||
#endif | #endif | ||||
#ifdef KERN_TLS | |||||
case IF_SND_TAG_TYPE_TLS: | |||||
cxgbe_tls_tag_free(mst); | |||||
return; | |||||
#endif | |||||
default: | default: | ||||
panic("shouldn't get here"); | panic("shouldn't get here"); | ||||
} | } | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* The kernel picks a media from the list we had provided but we still validate | * The kernel picks a media from the list we had provided but we still validate | ||||
▲ Show 20 Lines • Show All 2,193 Lines • ▼ Show 20 Lines | #define READ_CAPS(x) do { \ | ||||
* We've got the params we wanted to query via the firmware. Now grab | * We've got the params we wanted to query via the firmware. Now grab | ||||
* some others directly from the chip. | * some others directly from the chip. | ||||
*/ | */ | ||||
rc = t4_read_chip_settings(sc); | rc = t4_read_chip_settings(sc); | ||||
return (rc); | return (rc); | ||||
} | } | ||||
#ifdef KERN_TLS | |||||
static void | |||||
ktls_tick(void *arg) | |||||
{ | |||||
struct adapter *sc; | |||||
uint32_t tstamp; | |||||
sc = arg; | |||||
tstamp = tcp_ts_getticks(); | |||||
t4_write_reg(sc, A_TP_SYNC_TIME_HI, tstamp >> 1); | |||||
t4_write_reg(sc, A_TP_SYNC_TIME_LO, tstamp << 31); | |||||
callout_schedule_sbt(&sc->ktls_tick, SBT_1MS, 0, C_HARDCLOCK); | |||||
} | |||||
static void | |||||
t4_enable_kern_tls(struct adapter *sc) | |||||
{ | |||||
uint32_t m, v; | |||||
m = F_ENABLECBYP; | |||||
v = F_ENABLECBYP; | |||||
t4_set_reg_field(sc, A_TP_PARA_REG6, m, v); | |||||
m = F_CPL_FLAGS_UPDATE_EN | F_SEQ_UPDATE_EN; | |||||
v = F_CPL_FLAGS_UPDATE_EN | F_SEQ_UPDATE_EN; | |||||
t4_set_reg_field(sc, A_ULP_TX_CONFIG, m, v); | |||||
m = F_NICMODE; | |||||
v = F_NICMODE; | |||||
t4_set_reg_field(sc, A_TP_IN_CONFIG, m, v); | |||||
m = F_LOOKUPEVERYPKT; | |||||
v = 0; | |||||
t4_set_reg_field(sc, A_TP_INGRESS_CONFIG, m, v); | |||||
m = F_TXDEFERENABLE | F_DISABLEWINDOWPSH | F_DISABLESEPPSHFLAG; | |||||
v = F_DISABLEWINDOWPSH; | |||||
t4_set_reg_field(sc, A_TP_PC_CONFIG, m, v); | |||||
m = V_TIMESTAMPRESOLUTION(M_TIMESTAMPRESOLUTION); | |||||
v = V_TIMESTAMPRESOLUTION(0x1f); | |||||
t4_set_reg_field(sc, A_TP_TIMER_RESOLUTION, m, v); | |||||
sc->flags |= KERN_TLS_OK; | |||||
sc->tlst.inline_keys = t4_tls_inline_keys; | |||||
sc->tlst.combo_wrs = t4_tls_combo_wrs; | |||||
} | |||||
#endif | |||||
static int | static int | ||||
set_params__post_init(struct adapter *sc) | set_params__post_init(struct adapter *sc) | ||||
{ | { | ||||
uint32_t param, val; | uint32_t param, val; | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
int i, v, shift; | int i, v, shift; | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | for (i = 0; i < nitems(t4_toe_rexmt_backoff); i++) { | ||||
if (t4_toe_rexmt_backoff[i] != -1) { | if (t4_toe_rexmt_backoff[i] != -1) { | ||||
v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0; | v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0; | ||||
shift = (i & 3) << 3; | shift = (i & 3) << 3; | ||||
t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3), | t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3), | ||||
M_TIMERBACKOFFINDEX0 << shift, v << shift); | M_TIMERBACKOFFINDEX0 << shift, v << shift); | ||||
} | } | ||||
} | } | ||||
#endif | #endif | ||||
#ifdef KERN_TLS | |||||
if (t4_kern_tls != 0 && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS && | |||||
sc->toecaps & FW_CAPS_CONFIG_TOE) | |||||
t4_enable_kern_tls(sc); | |||||
#endif | |||||
return (0); | return (0); | ||||
} | } | ||||
#undef FW_PARAM_PFVF | #undef FW_PARAM_PFVF | ||||
#undef FW_PARAM_DEV | #undef FW_PARAM_DEV | ||||
static void | static void | ||||
t4_set_desc(struct adapter *sc) | t4_set_desc(struct adapter *sc) | ||||
▲ Show 20 Lines • Show All 861 Lines • ▼ Show 20 Lines | #ifdef RSS | ||||
for (i = 0; i < nitems(rss_key); i++) { | for (i = 0; i < nitems(rss_key); i++) { | ||||
rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); | rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); | ||||
} | } | ||||
t4_write_rss_key(sc, &rss_key[0], -1, 1); | t4_write_rss_key(sc, &rss_key[0], -1, 1); | ||||
#endif | #endif | ||||
if (!(sc->flags & IS_VF)) | if (!(sc->flags & IS_VF)) | ||||
t4_intr_enable(sc); | t4_intr_enable(sc); | ||||
#ifdef KERN_TLS | |||||
if (sc->flags & KERN_TLS_OK) | |||||
callout_reset_sbt(&sc->ktls_tick, SBT_1MS, 0, ktls_tick, sc, | |||||
C_HARDCLOCK); | |||||
#endif | |||||
sc->flags |= FULL_INIT_DONE; | sc->flags |= FULL_INIT_DONE; | ||||
done: | done: | ||||
if (rc != 0) | if (rc != 0) | ||||
adapter_full_uninit(sc); | adapter_full_uninit(sc); | ||||
return (rc); | return (rc); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 851 Lines • ▼ Show 20 Lines | SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", | ||||
sysctl_ulprx_la, "A", "ULPRX logic analyzer"); | sysctl_ulprx_la, "A", "ULPRX logic analyzer"); | ||||
if (chip_id(sc) >= CHELSIO_T5) { | if (chip_id(sc) >= CHELSIO_T5) { | ||||
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", | SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", | ||||
CTLTYPE_STRING | CTLFLAG_RD, sc, 0, | CTLTYPE_STRING | CTLFLAG_RD, sc, 0, | ||||
sysctl_wcwr_stats, "A", "write combined work requests"); | sysctl_wcwr_stats, "A", "write combined work requests"); | ||||
} | } | ||||
#ifdef KERN_TLS | |||||
if (sc->flags & KERN_TLS_OK) { | |||||
/* | |||||
* dev.t4nex.0.tls. | |||||
*/ | |||||
oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "tls", CTLFLAG_RD, | |||||
NULL, "KERN_TLS parameters"); | |||||
children = SYSCTL_CHILDREN(oid); | |||||
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "inline_keys", | |||||
CTLFLAG_RW, &sc->tlst.inline_keys, 0, "Always pass TLS " | |||||
"keys in work requests (1) or attempt to store TLS keys " | |||||
"in card memory."); | |||||
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "combo_wrs", | |||||
CTLFLAG_RW, &sc->tlst.combo_wrs, 0, "Attempt to combine " | |||||
"TCB field updates with TLS record work requests."); | |||||
} | |||||
#endif | |||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
if (is_offload(sc)) { | if (is_offload(sc)) { | ||||
int i; | int i; | ||||
char s[4]; | char s[4]; | ||||
/* | /* | ||||
* dev.t4nex.X.toe. | * dev.t4nex.X.toe. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 454 Lines • ▼ Show 20 Lines | SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, | ||||
"# of buffer-group 2 truncated packets"); | "# of buffer-group 2 truncated packets"); | ||||
SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, | SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, | ||||
"# of buffer-group 3 truncated packets"); | "# of buffer-group 3 truncated packets"); | ||||
#undef SYSCTL_ADD_T4_PORTSTAT | #undef SYSCTL_ADD_T4_PORTSTAT | ||||
SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_records", | SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_records", | ||||
CTLFLAG_RD, &pi->tx_tls_records, | CTLFLAG_RD, &pi->tx_tls_records, | ||||
"# of TLS records transmitted"); | "# of TOE TLS records transmitted"); | ||||
SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_octets", | SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_octets", | ||||
CTLFLAG_RD, &pi->tx_tls_octets, | CTLFLAG_RD, &pi->tx_tls_octets, | ||||
"# of payload octets in transmitted TLS records"); | "# of payload octets in transmitted TOE TLS records"); | ||||
SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_records", | SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_records", | ||||
CTLFLAG_RD, &pi->rx_tls_records, | CTLFLAG_RD, &pi->rx_tls_records, | ||||
"# of TLS records received"); | "# of TOE TLS records received"); | ||||
SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_octets", | SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_octets", | ||||
CTLFLAG_RD, &pi->rx_tls_octets, | CTLFLAG_RD, &pi->rx_tls_octets, | ||||
"# of payload octets in received TLS records"); | "# of payload octets in received TOE TLS records"); | ||||
} | } | ||||
static int | static int | ||||
sysctl_int_array(SYSCTL_HANDLER_ARGS) | sysctl_int_array(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
int rc, *i, space = 0; | int rc, *i, space = 0; | ||||
struct sbuf sb; | struct sbuf sb; | ||||
▲ Show 20 Lines • Show All 3,208 Lines • ▼ Show 20 Lines | #endif | ||||
txq->imm_wrs = 0; | txq->imm_wrs = 0; | ||||
txq->sgl_wrs = 0; | txq->sgl_wrs = 0; | ||||
txq->txpkt_wrs = 0; | txq->txpkt_wrs = 0; | ||||
txq->txpkts0_wrs = 0; | txq->txpkts0_wrs = 0; | ||||
txq->txpkts1_wrs = 0; | txq->txpkts1_wrs = 0; | ||||
txq->txpkts0_pkts = 0; | txq->txpkts0_pkts = 0; | ||||
txq->txpkts1_pkts = 0; | txq->txpkts1_pkts = 0; | ||||
txq->raw_wrs = 0; | txq->raw_wrs = 0; | ||||
txq->tls_wrs = 0; | |||||
txq->kern_tls_records = 0; | |||||
txq->kern_tls_short = 0; | |||||
txq->kern_tls_partial = 0; | |||||
txq->kern_tls_full = 0; | |||||
txq->kern_tls_octets = 0; | |||||
txq->kern_tls_waste = 0; | |||||
txq->kern_tls_options = 0; | |||||
txq->kern_tls_header = 0; | |||||
txq->kern_tls_fin = 0; | |||||
txq->kern_tls_fin_short = 0; | |||||
txq->kern_tls_cbc = 0; | |||||
txq->kern_tls_gcm = 0; | |||||
mp_ring_reset_stats(txq->r); | mp_ring_reset_stats(txq->r); | ||||
} | } | ||||
#if defined(TCP_OFFLOAD) || defined(RATELIMIT) | #if defined(TCP_OFFLOAD) || defined(RATELIMIT) | ||||
for_each_ofld_txq(vi, i, wrq) { | for_each_ofld_txq(vi, i, wrq) { | ||||
wrq->tx_wrs_direct = 0; | wrq->tx_wrs_direct = 0; | ||||
wrq->tx_wrs_copied = 0; | wrq->tx_wrs_copied = 0; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 509 Lines • ▼ Show 20 Lines | |||||
#if defined(TCP_OFFLOAD) || defined(RATELIMIT) | #if defined(TCP_OFFLOAD) || defined(RATELIMIT) | ||||
calculate_nqueues(&t4_nofldtxq, nc, NOFLDTXQ); | calculate_nqueues(&t4_nofldtxq, nc, NOFLDTXQ); | ||||
calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI); | calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI); | ||||
#endif | #endif | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
calculate_nqueues(&t4_nofldrxq, nc, NOFLDRXQ); | calculate_nqueues(&t4_nofldrxq, nc, NOFLDRXQ); | ||||
calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI); | calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI); | ||||
#endif | |||||
#if defined(TCP_OFFLOAD) || defined(KERN_TLS) | |||||
if (t4_toecaps_allowed == -1) | if (t4_toecaps_allowed == -1) | ||||
t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; | t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; | ||||
#else | |||||
if (t4_toecaps_allowed == -1) | |||||
t4_toecaps_allowed = 0; | |||||
#endif | |||||
#ifdef TCP_OFFLOAD | |||||
if (t4_rdmacaps_allowed == -1) { | if (t4_rdmacaps_allowed == -1) { | ||||
t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP | | t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP | | ||||
FW_CAPS_CONFIG_RDMA_RDMAC; | FW_CAPS_CONFIG_RDMA_RDMAC; | ||||
} | } | ||||
if (t4_iscsicaps_allowed == -1) { | if (t4_iscsicaps_allowed == -1) { | ||||
t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU | | t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU | | ||||
FW_CAPS_CONFIG_ISCSI_TARGET_PDU | | FW_CAPS_CONFIG_ISCSI_TARGET_PDU | | ||||
FW_CAPS_CONFIG_ISCSI_T10DIF; | FW_CAPS_CONFIG_ISCSI_T10DIF; | ||||
} | } | ||||
if (t4_tmr_idx_ofld < 0 || t4_tmr_idx_ofld >= SGE_NTIMERS) | if (t4_tmr_idx_ofld < 0 || t4_tmr_idx_ofld >= SGE_NTIMERS) | ||||
t4_tmr_idx_ofld = TMR_IDX_OFLD; | t4_tmr_idx_ofld = TMR_IDX_OFLD; | ||||
if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS) | if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS) | ||||
t4_pktc_idx_ofld = PKTC_IDX_OFLD; | t4_pktc_idx_ofld = PKTC_IDX_OFLD; | ||||
#else | #else | ||||
if (t4_toecaps_allowed == -1) | |||||
t4_toecaps_allowed = 0; | |||||
if (t4_rdmacaps_allowed == -1) | if (t4_rdmacaps_allowed == -1) | ||||
t4_rdmacaps_allowed = 0; | t4_rdmacaps_allowed = 0; | ||||
if (t4_iscsicaps_allowed == -1) | if (t4_iscsicaps_allowed == -1) | ||||
t4_iscsicaps_allowed = 0; | t4_iscsicaps_allowed = 0; | ||||
#endif | #endif | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
▲ Show 20 Lines • Show All 286 Lines • ▼ Show 20 Lines | if (loaded++ == 0) { | ||||
callout_init(&fatal_callout, 1); | callout_init(&fatal_callout, 1); | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); | sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); | ||||
SLIST_INIT(&t4_uld_list); | SLIST_INIT(&t4_uld_list); | ||||
#endif | #endif | ||||
#ifdef INET6 | #ifdef INET6 | ||||
t4_clip_modload(); | t4_clip_modload(); | ||||
#endif | #endif | ||||
#ifdef KERN_TLS | |||||
t6_ktls_modload(); | |||||
#endif | |||||
t4_tracer_modload(); | t4_tracer_modload(); | ||||
tweak_tunables(); | tweak_tunables(); | ||||
} | } | ||||
sx_xunlock(&mlu); | sx_xunlock(&mlu); | ||||
break; | break; | ||||
case MOD_UNLOAD: | case MOD_UNLOAD: | ||||
sx_xlock(&mlu); | sx_xlock(&mlu); | ||||
Show All 23 Lines | #endif | ||||
} | } | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
sx_sunlock(&t4_uld_list_lock); | sx_sunlock(&t4_uld_list_lock); | ||||
#endif | #endif | ||||
sx_sunlock(&t4_list_lock); | sx_sunlock(&t4_list_lock); | ||||
if (t4_sge_extfree_refs() == 0) { | if (t4_sge_extfree_refs() == 0) { | ||||
t4_tracer_modunload(); | t4_tracer_modunload(); | ||||
#ifdef KERN_TLS | |||||
t6_ktls_modunload(); | |||||
#endif | |||||
#ifdef INET6 | #ifdef INET6 | ||||
t4_clip_modunload(); | t4_clip_modunload(); | ||||
#endif | #endif | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
sx_destroy(&t4_uld_list_lock); | sx_destroy(&t4_uld_list_lock); | ||||
#endif | #endif | ||||
sx_destroy(&t4_list_lock); | sx_destroy(&t4_list_lock); | ||||
t4_sge_modunload(); | t4_sge_modunload(); | ||||
▲ Show 20 Lines • Show All 56 Lines • Show Last 20 Lines |
The atid table change I could break out into a separate commit first if that's better.