Index: stable/11/sys/dev/cxgbe/adapter.h =================================================================== --- stable/11/sys/dev/cxgbe/adapter.h (revision 346862) +++ stable/11/sys/dev/cxgbe/adapter.h (revision 346863) @@ -1,1259 +1,1259 @@ /*- * Copyright (c) 2011 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __T4_ADAPTER_H__ #define __T4_ADAPTER_H__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "offload.h" #include "t4_ioctl.h" #include "common/t4_msg.h" #include "firmware/t4fw_interface.h" #define KTR_CXGBE KTR_SPARE3 MALLOC_DECLARE(M_CXGBE); #define CXGBE_UNIMPLEMENTED(s) \ panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) #if defined(__i386__) || defined(__amd64__) static __inline void prefetch(void *x) { __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); } #else #define prefetch(x) __builtin_prefetch(x) #endif #ifndef SYSCTL_ADD_UQUAD #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD #define sysctl_handle_64 sysctl_handle_quad #define CTLTYPE_U64 CTLTYPE_QUAD #endif #if (__FreeBSD_version >= 900030) || \ ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000)) #define SBUF_DRAIN 1 #endif SYSCTL_DECL(_hw_cxgbe); struct adapter; typedef struct adapter adapter_t; enum { /* * All ingress queues use this entry size. Note that the firmware event * queue and any iq expecting CPL_RX_PKT in the descriptor needs this to * be at least 64. */ IQ_ESIZE = 64, /* Default queue sizes for all kinds of ingress queues */ FW_IQ_QSIZE = 256, RX_IQ_QSIZE = 1024, /* All egress queues use this entry size */ EQ_ESIZE = 64, /* Default queue sizes for all kinds of egress queues */ CTRL_EQ_QSIZE = 128, TX_EQ_QSIZE = 1024, #if MJUMPAGESIZE != MCLBYTES SW_ZONE_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */ #else SW_ZONE_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ #endif CL_METADATA_SIZE = CACHE_LINE_SIZE, SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */ TX_SGL_SEGS = 39, TX_SGL_SEGS_TSO = 38, TX_WR_FLITS = SGE_MAX_WR_LEN / 8 }; enum { /* adapter intr_type */ INTR_INTX = (1 << 0), INTR_MSI = (1 << 1), INTR_MSIX = (1 << 2) }; enum { XGMAC_MTU = (1 << 0), XGMAC_PROMISC = (1 << 1), XGMAC_ALLMULTI = (1 << 2), XGMAC_VLANEX = (1 << 3), XGMAC_UCADDR = (1 << 4), XGMAC_MCADDRS = (1 << 5), XGMAC_ALL = 0xffff }; enum { /* flags understood by begin_synchronized_op */ HOLD_LOCK = (1 << 0), SLEEP_OK = (1 << 1), INTR_OK = (1 << 2), /* flags understood by end_synchronized_op */ LOCK_HELD = HOLD_LOCK, }; enum { /* adapter flags */ FULL_INIT_DONE = (1 << 0), FW_OK = (1 << 1), CHK_MBOX_ACCESS = (1 << 2), MASTER_PF = (1 << 3), ADAP_SYSCTL_CTX = (1 << 4), /* TOM_INIT_DONE= (1 << 5), No longer used */ BUF_PACKING_OK = (1 << 6), IS_VF = (1 << 7), CXGBE_BUSY = (1 << 9), /* port flags */ HAS_TRACEQ = (1 << 3), FIXED_IFMEDIA = (1 << 4), /* ifmedia list doesn't change. */ /* VI flags */ DOOMED = (1 << 0), VI_INIT_DONE = (1 << 1), VI_SYSCTL_CTX = (1 << 2), /* adapter debug_flags */ DF_DUMP_MBOX = (1 << 0), /* Log all mbox cmd/rpl. */ DF_LOAD_FW_ANYTIME = (1 << 1), /* Allow LOAD_FW after init */ DF_DISABLE_TCB_CACHE = (1 << 2), /* Disable TCB cache (T6+) */ }; #define IS_DOOMED(vi) ((vi)->flags & DOOMED) #define SET_DOOMED(vi) do {(vi)->flags |= DOOMED;} while (0) #define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY) #define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0) #define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0) struct vi_info { device_t dev; struct port_info *pi; struct ifnet *ifp; unsigned long flags; int if_flags; uint16_t *rss, *nm_rss; int smt_idx; /* for convenience */ uint16_t viid; int16_t xact_addr_filt;/* index of exact MAC address filter */ uint16_t rss_size; /* size of VI's RSS table slice */ uint16_t rss_base; /* start of VI's RSS table slice */ eventhandler_tag vlan_c; int nintr; int first_intr; /* These need to be int as they are used in sysctl */ int ntxq; /* # of tx queues */ int first_txq; /* index of first tx queue */ int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */ int nrxq; /* # of rx queues */ int first_rxq; /* index of first rx queue */ int nofldtxq; /* # of offload tx queues */ int first_ofld_txq; /* index of first offload tx queue */ int nofldrxq; /* # of offload rx queues */ int first_ofld_rxq; /* index of first offload rx queue */ int nnmtxq; int first_nm_txq; int nnmrxq; int first_nm_rxq; int tmr_idx; int ofld_tmr_idx; int pktc_idx; int ofld_pktc_idx; int qsize_rxq; int qsize_txq; struct timeval last_refreshed; struct fw_vi_stats_vf stats; struct callout tick; struct sysctl_ctx_list ctx; /* from ifconfig up to driver detach */ uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ }; struct tx_ch_rl_params { enum fw_sched_params_rate ratemode; /* %port (REL) or kbps (ABS) */ uint32_t maxrate; }; enum { TX_CLRL_REFRESH = (1 << 0), /* Need to update hardware state. */ TX_CLRL_ERROR = (1 << 1), /* Error, hardware state unknown. */ }; struct tx_cl_rl_params { int refcount; u_int flags; enum fw_sched_params_rate ratemode; /* %port REL or ABS value */ enum fw_sched_params_unit rateunit; /* kbps or pps (when ABS) */ enum fw_sched_params_mode mode; /* aggr or per-flow */ uint32_t maxrate; uint16_t pktsize; }; /* Tx scheduler parameters for a channel/port */ struct tx_sched_params { /* Channel Rate Limiter */ struct tx_ch_rl_params ch_rl; /* Class WRR */ /* XXX */ /* Class Rate Limiter */ struct tx_cl_rl_params cl_rl[]; }; struct port_info { device_t dev; struct adapter *adapter; struct vi_info *vi; int nvi; int up_vis; int uld_vis; struct tx_sched_params *sched_params; struct mtx pi_lock; char lockname[16]; unsigned long flags; uint8_t lport; /* associated offload logical port */ int8_t mdio_addr; uint8_t port_type; uint8_t mod_type; uint8_t port_id; uint8_t tx_chan; uint8_t mps_bg_map; /* rx MPS buffer group bitmap */ uint8_t rx_e_chan_map; /* rx TP e-channel bitmap */ struct link_config link_cfg; struct ifmedia media; struct timeval last_refreshed; struct port_stats stats; u_int tnl_cong_drops; u_int tx_parse_error; u_long tx_tls_records; u_long tx_tls_octets; u_long rx_tls_records; u_long rx_tls_octets; struct callout tick; }; #define IS_MAIN_VI(vi) ((vi) == &((vi)->pi->vi[0])) /* Where the cluster came from, how it has been carved up. */ struct cluster_layout { int8_t zidx; int8_t hwidx; uint16_t region1; /* mbufs laid out within this region */ /* region2 is the DMA region */ uint16_t region3; /* cluster_metadata within this region */ }; struct cluster_metadata { u_int refcount; struct fl_sdesc *sd; /* For debug only. Could easily be stale */ }; struct fl_sdesc { caddr_t cl; uint16_t nmbuf; /* # of driver originated mbufs with ref on cluster */ struct cluster_layout cll; }; struct tx_desc { __be64 flit[8]; }; struct tx_sdesc { struct mbuf *m; /* m_nextpkt linked chain of frames */ uint8_t desc_used; /* # of hardware descriptors used by the WR */ }; #define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header)) struct iq_desc { struct rss_header rss; uint8_t cpl[IQ_PAD]; struct rsp_ctrl rsp; }; #undef IQ_PAD CTASSERT(sizeof(struct iq_desc) == IQ_ESIZE); enum { /* iq flags */ IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */ IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */ /* 1 << 2 Used to be IQ_INTR */ IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */ IQ_ADJ_CREDIT = (1 << 4), /* hw is off by 1 credit for this iq */ /* iq state */ IQS_DISABLED = 0, IQS_BUSY = 1, IQS_IDLE = 2, /* netmap related flags */ NM_OFF = 0, NM_ON = 1, NM_BUSY = 2, }; enum { CPL_COOKIE_RESERVED = 0, CPL_COOKIE_FILTER, CPL_COOKIE_DDP0, CPL_COOKIE_DDP1, CPL_COOKIE_TOM, CPL_COOKIE_HASHFILTER, - CPL_COOKIE_AVAILABLE2, + CPL_COOKIE_ETHOFLD, CPL_COOKIE_AVAILABLE3, NUM_CPL_COOKIES = 8 /* Limited by M_COOKIE. Do not increase. */ }; struct sge_iq; struct rss_header; typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, struct mbuf *); typedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *); typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *); /* * Ingress Queue: T4 is producer, driver is consumer. */ struct sge_iq { uint32_t flags; volatile int state; struct adapter *adapter; struct iq_desc *desc; /* KVA of descriptor ring */ int8_t intr_pktc_idx; /* packet count threshold index */ uint8_t gen; /* generation bit */ uint8_t intr_params; /* interrupt holdoff parameters */ uint8_t intr_next; /* XXX: holdoff for next interrupt */ uint16_t qsize; /* size (# of entries) of the queue */ uint16_t sidx; /* index of the entry with the status page */ uint16_t cidx; /* consumer index */ uint16_t cntxt_id; /* SGE context id for the iq */ uint16_t abs_id; /* absolute SGE id for the iq */ STAILQ_ENTRY(sge_iq) link; bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; bus_addr_t ba; /* bus address of descriptor ring */ }; enum { EQ_CTRL = 1, EQ_ETH = 2, EQ_OFLD = 3, /* eq flags */ EQ_TYPEMASK = 0x3, /* 2 lsbits hold the type (see above) */ EQ_ALLOCATED = (1 << 2), /* firmware resources allocated */ EQ_ENABLED = (1 << 3), /* open for business */ EQ_QFLUSH = (1 << 4), /* if_qflush in progress */ }; /* Listed in order of preference. Update t4_sysctls too if you change these */ enum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; /* * Egress Queue: driver is producer, T4 is consumer. * * Note: A free list is an egress queue (driver produces the buffers and T4 * consumes them) but it's special enough to have its own struct (see sge_fl). */ struct sge_eq { unsigned int flags; /* MUST be first */ unsigned int cntxt_id; /* SGE context id for the eq */ unsigned int abs_id; /* absolute SGE id for the eq */ struct mtx eq_lock; struct tx_desc *desc; /* KVA of descriptor ring */ uint8_t doorbells; volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */ u_int udb_qid; /* relative qid within the doorbell page */ uint16_t sidx; /* index of the entry with the status page */ uint16_t cidx; /* consumer idx (desc idx) */ uint16_t pidx; /* producer idx (desc idx) */ uint16_t equeqidx; /* EQUEQ last requested at this pidx */ uint16_t dbidx; /* pidx of the most recent doorbell */ uint16_t iqid; /* iq that gets egr_update for the eq */ uint8_t tx_chan; /* tx channel used by the eq */ volatile u_int equiq; /* EQUIQ outstanding */ bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; bus_addr_t ba; /* bus address of descriptor ring */ char lockname[16]; }; struct sw_zone_info { uma_zone_t zone; /* zone that this cluster comes from */ int size; /* size of cluster: 2K, 4K, 9K, 16K, etc. */ int type; /* EXT_xxx type of the cluster */ int8_t head_hwidx; int8_t tail_hwidx; }; struct hw_buf_info { int8_t zidx; /* backpointer to zone; -ve means unused */ int8_t next; /* next hwidx for this zone; -1 means no more */ int size; }; enum { NUM_MEMWIN = 3, MEMWIN0_APERTURE = 2048, MEMWIN0_BASE = 0x1b800, MEMWIN1_APERTURE = 32768, MEMWIN1_BASE = 0x28000, MEMWIN2_APERTURE_T4 = 65536, MEMWIN2_BASE_T4 = 0x30000, MEMWIN2_APERTURE_T5 = 128 * 1024, MEMWIN2_BASE_T5 = 0x60000, }; struct memwin { struct rwlock mw_lock __aligned(CACHE_LINE_SIZE); uint32_t mw_base; /* constant after setup_memwin */ uint32_t mw_aperture; /* ditto */ uint32_t mw_curpos; /* protected by mw_lock */ }; enum { FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */ FL_DOOMED = (1 << 1), /* about to be destroyed */ FL_BUF_PACKING = (1 << 2), /* buffer packing enabled */ FL_BUF_RESUME = (1 << 3), /* resume from the middle of the frame */ }; #define FL_RUNNING_LOW(fl) \ (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat) #define FL_NOT_RUNNING_LOW(fl) \ (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat) struct sge_fl { struct mtx fl_lock; __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ struct cluster_layout cll_def; /* default refill zone, layout */ uint16_t lowat; /* # of buffers <= this means fl needs help */ int flags; uint16_t buf_boundary; /* The 16b idx all deal with hw descriptors */ uint16_t dbidx; /* hw pidx after last doorbell */ uint16_t sidx; /* index of status page */ volatile uint16_t hw_cidx; /* The 32b idx are all buffer idx, not hardware descriptor idx */ uint32_t cidx; /* consumer index */ uint32_t pidx; /* producer index */ uint32_t dbval; u_int rx_offset; /* offset in fl buf (when buffer packing) */ volatile uint32_t *udb; uint64_t mbuf_allocated;/* # of mbuf allocated from zone_mbuf */ uint64_t mbuf_inlined; /* # of mbuf created within clusters */ uint64_t cl_allocated; /* # of clusters allocated */ uint64_t cl_recycled; /* # of clusters recycled */ uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */ /* These 3 are valid when FL_BUF_RESUME is set, stale otherwise. */ struct mbuf *m0; struct mbuf **pnext; u_int remaining; uint16_t qsize; /* # of hw descriptors (status page included) */ uint16_t cntxt_id; /* SGE context id for the freelist */ TAILQ_ENTRY(sge_fl) link; /* All starving freelists */ bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; char lockname[16]; bus_addr_t ba; /* bus address of descriptor ring */ struct cluster_layout cll_alt; /* alternate refill zone, layout */ }; struct mp_ring; /* txq: SGE egress queue + what's needed for Ethernet NIC */ struct sge_txq { struct sge_eq eq; /* MUST be first */ struct ifnet *ifp; /* the interface this txq belongs to */ struct mp_ring *r; /* tx software ring */ struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ struct sglist *gl; __be32 cpl_ctrl0; /* for convenience */ int tc_idx; /* traffic class */ struct task tx_reclaim_task; /* stats for common events first */ uint64_t txcsum; /* # of times hardware assisted with checksum */ uint64_t tso_wrs; /* # of TSO work requests */ uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ uint64_t imm_wrs; /* # of work requests with immediate data */ uint64_t sgl_wrs; /* # of work requests with direct SGL */ uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ uint64_t txpkts0_wrs; /* # of type0 coalesced tx work requests */ uint64_t txpkts1_wrs; /* # of type1 coalesced tx work requests */ uint64_t txpkts0_pkts; /* # of frames in type0 coalesced tx WRs */ uint64_t txpkts1_pkts; /* # of frames in type1 coalesced tx WRs */ /* stats for not-that-common events */ } __aligned(CACHE_LINE_SIZE); /* rxq: SGE ingress queue + SGE free list + miscellaneous items */ struct sge_rxq { struct sge_iq iq; /* MUST be first */ struct sge_fl fl; /* MUST follow iq */ struct ifnet *ifp; /* the interface this rxq belongs to */ #if defined(INET) || defined(INET6) struct lro_ctrl lro; /* LRO state */ #endif /* stats for common events first */ uint64_t rxcsum; /* # of times hardware assisted with checksum */ uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ /* stats for not-that-common events */ } __aligned(CACHE_LINE_SIZE); static inline struct sge_rxq * iq_to_rxq(struct sge_iq *iq) { return (__containerof(iq, struct sge_rxq, iq)); } /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */ struct sge_ofld_rxq { struct sge_iq iq; /* MUST be first */ struct sge_fl fl; /* MUST follow iq */ } __aligned(CACHE_LINE_SIZE); static inline struct sge_ofld_rxq * iq_to_ofld_rxq(struct sge_iq *iq) { return (__containerof(iq, struct sge_ofld_rxq, iq)); } struct wrqe { STAILQ_ENTRY(wrqe) link; struct sge_wrq *wrq; int wr_len; char wr[] __aligned(16); }; struct wrq_cookie { TAILQ_ENTRY(wrq_cookie) link; int ndesc; int pidx; }; /* * wrq: SGE egress queue that is given prebuilt work requests. Both the control * and offload tx queues are of this type. */ struct sge_wrq { struct sge_eq eq; /* MUST be first */ struct adapter *adapter; struct task wrq_tx_task; /* Tx desc reserved but WR not "committed" yet. */ TAILQ_HEAD(wrq_incomplete_wrs , wrq_cookie) incomplete_wrs; /* List of WRs ready to go out as soon as descriptors are available. */ STAILQ_HEAD(, wrqe) wr_list; u_int nwr_pending; u_int ndesc_needed; /* stats for common events first */ uint64_t tx_wrs_direct; /* # of WRs written directly to desc ring. */ uint64_t tx_wrs_ss; /* # of WRs copied from scratch space. */ uint64_t tx_wrs_copied; /* # of WRs queued and copied to desc ring. */ /* stats for not-that-common events */ /* * Scratch space for work requests that wrap around after reaching the * status page, and some information about the last WR that used it. */ uint16_t ss_pidx; uint16_t ss_len; uint8_t ss[SGE_MAX_WR_LEN]; } __aligned(CACHE_LINE_SIZE); struct sge_nm_rxq { struct vi_info *vi; struct iq_desc *iq_desc; uint16_t iq_abs_id; uint16_t iq_cntxt_id; uint16_t iq_cidx; uint16_t iq_sidx; uint8_t iq_gen; __be64 *fl_desc; uint16_t fl_cntxt_id; uint32_t fl_cidx; uint32_t fl_pidx; uint32_t fl_sidx; uint32_t fl_db_val; u_int fl_hwidx:4; u_int nid; /* netmap ring # for this queue */ /* infrequently used items after this */ bus_dma_tag_t iq_desc_tag; bus_dmamap_t iq_desc_map; bus_addr_t iq_ba; int intr_idx; bus_dma_tag_t fl_desc_tag; bus_dmamap_t fl_desc_map; bus_addr_t fl_ba; } __aligned(CACHE_LINE_SIZE); struct sge_nm_txq { struct tx_desc *desc; uint16_t cidx; uint16_t pidx; uint16_t sidx; uint16_t equiqidx; /* EQUIQ last requested at this pidx */ uint16_t equeqidx; /* EQUEQ last requested at this pidx */ uint16_t dbidx; /* pidx of the most recent doorbell */ uint8_t doorbells; volatile uint32_t *udb; u_int udb_qid; u_int cntxt_id; __be32 cpl_ctrl0; /* for convenience */ u_int nid; /* netmap ring # for this queue */ /* infrequently used items after this */ bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; bus_addr_t ba; int iqidx; } __aligned(CACHE_LINE_SIZE); struct sge { int nrxq; /* total # of Ethernet rx queues */ int ntxq; /* total # of Ethernet tx queues */ int nofldrxq; /* total # of TOE rx queues */ int nofldtxq; /* total # of TOE tx queues */ int nnmrxq; /* total # of netmap rx queues */ int nnmtxq; /* total # of netmap tx queues */ int niq; /* total # of ingress queues */ int neq; /* total # of egress queues */ struct sge_iq fwq; /* Firmware event queue */ struct sge_wrq mgmtq; /* Management queue (control queue) */ struct sge_wrq *ctrlq; /* Control queues */ struct sge_txq *txq; /* NIC tx queues */ struct sge_rxq *rxq; /* NIC rx queues */ struct sge_wrq *ofld_txq; /* TOE tx queues */ struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */ struct sge_nm_txq *nm_txq; /* netmap tx queues */ struct sge_nm_rxq *nm_rxq; /* netmap rx queues */ uint16_t iq_start; /* first cntxt_id */ uint16_t iq_base; /* first abs_id */ int eq_start; /* first cntxt_id */ int eq_base; /* first abs_id */ struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ int8_t safe_hwidx1; /* may not have room for metadata */ int8_t safe_hwidx2; /* with room for metadata and maybe more */ struct sw_zone_info sw_zone_info[SW_ZONE_SIZES]; struct hw_buf_info hw_buf_info[SGE_FLBUF_SIZES]; }; struct devnames { const char *nexus_name; const char *ifnet_name; const char *vi_ifnet_name; const char *pf03_drv_name; const char *vf_nexus_name; const char *vf_ifnet_name; }; struct adapter { SLIST_ENTRY(adapter) link; device_t dev; struct cdev *cdev; const struct devnames *names; /* PCIe register resources */ int regs_rid; struct resource *regs_res; int msix_rid; struct resource *msix_res; bus_space_handle_t bh; bus_space_tag_t bt; bus_size_t mmio_len; int udbs_rid; struct resource *udbs_res; volatile uint8_t *udbs_base; unsigned int pf; unsigned int mbox; unsigned int vpd_busy; unsigned int vpd_flag; /* Interrupt information */ int intr_type; int intr_count; struct irq { struct resource *res; int rid; volatile int nm_state; /* NM_OFF, NM_ON, or NM_BUSY */ void *tag; struct sge_rxq *rxq; struct sge_nm_rxq *nm_rxq; } __aligned(CACHE_LINE_SIZE) *irq; int sge_gts_reg; int sge_kdoorbell_reg; bus_dma_tag_t dmat; /* Parent DMA tag */ struct sge sge; int lro_timeout; int sc_do_rxcopy; struct taskqueue *tq[MAX_NCHAN]; /* General purpose taskqueues */ struct port_info *port[MAX_NPORTS]; uint8_t chan_map[MAX_NCHAN]; /* channel -> port */ void *tom_softc; /* (struct tom_data *) */ struct tom_tunables tt; struct t4_offload_policy *policy; struct rwlock policy_lock; void *iwarp_softc; /* (struct c4iw_dev *) */ struct iw_tunables iwt; void *iscsi_ulp_softc; /* (struct cxgbei_data *) */ void *ccr_softc; /* (struct ccr_softc *) */ struct l2t_data *l2t; /* L2 table */ struct smt_data *smt; /* Source MAC Table */ struct tid_info tids; vmem_t *key_map; uint8_t doorbells; int offload_map; /* ports with IFCAP_TOE enabled */ int active_ulds; /* ULDs activated on this adapter */ int flags; int debug_flags; char ifp_lockname[16]; struct mtx ifp_lock; struct ifnet *ifp; /* tracer ifp */ struct ifmedia media; int traceq; /* iq used by all tracers, -1 if none */ int tracer_valid; /* bitmap of valid tracers */ int tracer_enabled; /* bitmap of enabled tracers */ char fw_version[16]; char tp_version[16]; char er_version[16]; char bs_version[16]; char cfg_file[32]; u_int cfcsum; struct adapter_params params; const struct chip_params *chip_params; struct t4_virt_res vres; uint16_t nbmcaps; uint16_t linkcaps; uint16_t switchcaps; uint16_t niccaps; uint16_t toecaps; uint16_t rdmacaps; uint16_t cryptocaps; uint16_t iscsicaps; uint16_t fcoecaps; struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */ struct mtx sc_lock; char lockname[16]; /* Starving free lists */ struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */ TAILQ_HEAD(, sge_fl) sfl; struct callout sfl_callout; struct mtx reg_lock; /* for indirect register access */ struct memwin memwin[NUM_MEMWIN]; /* memory windows */ struct mtx tc_lock; struct task tc_task; const char *last_op; const void *last_op_thr; int last_op_flags; }; #define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) #define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) #define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) #define ASSERT_SYNCHRONIZED_OP(sc) \ KASSERT(IS_BUSY(sc) && \ (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \ ("%s: operation not synchronized.", __func__)) #define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) #define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) #define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) #define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) #define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) #define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) #define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) #define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) #define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) #define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) #define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) #define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) #define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) #define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) #define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) #define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) #define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) #define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) #define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) #define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) #define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) #define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) #define CH_DUMP_MBOX(sc, mbox, data_reg) \ do { \ if (sc->debug_flags & DF_DUMP_MBOX) { \ log(LOG_NOTICE, \ "%s mbox %u: %016llx %016llx %016llx %016llx " \ "%016llx %016llx %016llx %016llx\n", \ device_get_nameunit(sc->dev), mbox, \ (unsigned long long)t4_read_reg64(sc, data_reg), \ (unsigned long long)t4_read_reg64(sc, data_reg + 8), \ (unsigned long long)t4_read_reg64(sc, data_reg + 16), \ (unsigned long long)t4_read_reg64(sc, data_reg + 24), \ (unsigned long long)t4_read_reg64(sc, data_reg + 32), \ (unsigned long long)t4_read_reg64(sc, data_reg + 40), \ (unsigned long long)t4_read_reg64(sc, data_reg + 48), \ (unsigned long long)t4_read_reg64(sc, data_reg + 56)); \ } \ } while (0) #define for_each_txq(vi, iter, q) \ for (q = &vi->pi->adapter->sge.txq[vi->first_txq], iter = 0; \ iter < vi->ntxq; ++iter, ++q) #define for_each_rxq(vi, iter, q) \ for (q = &vi->pi->adapter->sge.rxq[vi->first_rxq], iter = 0; \ iter < vi->nrxq; ++iter, ++q) #define for_each_ofld_txq(vi, iter, q) \ for (q = &vi->pi->adapter->sge.ofld_txq[vi->first_ofld_txq], iter = 0; \ iter < vi->nofldtxq; ++iter, ++q) #define for_each_ofld_rxq(vi, iter, q) \ for (q = &vi->pi->adapter->sge.ofld_rxq[vi->first_ofld_rxq], iter = 0; \ iter < vi->nofldrxq; ++iter, ++q) #define for_each_nm_txq(vi, iter, q) \ for (q = &vi->pi->adapter->sge.nm_txq[vi->first_nm_txq], iter = 0; \ iter < vi->nnmtxq; ++iter, ++q) #define for_each_nm_rxq(vi, iter, q) \ for (q = &vi->pi->adapter->sge.nm_rxq[vi->first_nm_rxq], iter = 0; \ iter < vi->nnmrxq; ++iter, ++q) #define for_each_vi(_pi, _iter, _vi) \ for ((_vi) = (_pi)->vi, (_iter) = 0; (_iter) < (_pi)->nvi; \ ++(_iter), ++(_vi)) #define IDXINCR(idx, incr, wrap) do { \ idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \ } while (0) #define IDXDIFF(head, tail, wrap) \ ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head)) /* One for errors, one for firmware events */ #define T4_EXTRA_INTR 2 /* One for firmware events */ #define T4VF_EXTRA_INTR 1 static inline int forwarding_intr_to_fwq(struct adapter *sc) { return (sc->intr_count == 1); } static inline uint32_t t4_read_reg(struct adapter *sc, uint32_t reg) { return bus_space_read_4(sc->bt, sc->bh, reg); } static inline void t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) { bus_space_write_4(sc->bt, sc->bh, reg, val); } static inline uint64_t t4_read_reg64(struct adapter *sc, uint32_t reg) { #ifdef __LP64__ return bus_space_read_8(sc->bt, sc->bh, reg); #else return (uint64_t)bus_space_read_4(sc->bt, sc->bh, reg) + ((uint64_t)bus_space_read_4(sc->bt, sc->bh, reg + 4) << 32); #endif } static inline void t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) { #ifdef __LP64__ bus_space_write_8(sc->bt, sc->bh, reg, val); #else bus_space_write_4(sc->bt, sc->bh, reg, val); bus_space_write_4(sc->bt, sc->bh, reg + 4, val>> 32); #endif } static inline void t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) { *val = pci_read_config(sc->dev, reg, 1); } static inline void t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) { pci_write_config(sc->dev, reg, val, 1); } static inline void t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) { *val = pci_read_config(sc->dev, reg, 2); } static inline void t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) { pci_write_config(sc->dev, reg, val, 2); } static inline void t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) { *val = pci_read_config(sc->dev, reg, 4); } static inline void t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) { pci_write_config(sc->dev, reg, val, 4); } static inline struct port_info * adap2pinfo(struct adapter *sc, int idx) { return (sc->port[idx]); } static inline void t4_os_set_hw_addr(struct port_info *pi, uint8_t hw_addr[]) { bcopy(hw_addr, pi->vi[0].hw_addr, ETHER_ADDR_LEN); } static inline int tx_resume_threshold(struct sge_eq *eq) { /* not quite the same as qsize / 4, but this will do. */ return (eq->sidx / 4); } static inline int t4_use_ldst(struct adapter *sc) { #ifdef notyet return (sc->flags & FW_OK || !sc->use_bd); #else return (0); #endif } /* t4_main.c */ extern int t4_ntxq; extern int t4_nrxq; extern int t4_intr_types; extern int t4_tmr_idx; extern int t4_pktc_idx; extern unsigned int t4_qsize_rxq; extern unsigned int t4_qsize_txq; extern device_method_t cxgbe_methods[]; int t4_os_find_pci_capability(struct adapter *, int); int t4_os_pci_save_state(struct adapter *); int t4_os_pci_restore_state(struct adapter *); void t4_os_portmod_changed(struct port_info *); void t4_os_link_changed(struct port_info *); void t4_iterate(void (*)(struct adapter *, void *), void *); void t4_init_devnames(struct adapter *); void t4_add_adapter(struct adapter *); void t4_aes_getdeckey(void *, const void *, unsigned int); int t4_detach_common(device_t); int t4_map_bars_0_and_4(struct adapter *); int t4_map_bar_2(struct adapter *); int t4_setup_intr_handlers(struct adapter *); void t4_sysctls(struct adapter *); int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *); void doom_vi(struct adapter *, struct vi_info *); void end_synchronized_op(struct adapter *, int); int update_mac_settings(struct ifnet *, int); int adapter_full_init(struct adapter *); int adapter_full_uninit(struct adapter *); uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); int vi_full_init(struct vi_info *); int vi_full_uninit(struct vi_info *); void vi_sysctls(struct vi_info *); void vi_tick(void *); int rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int); int alloc_atid_tab(struct tid_info *, int); void free_atid_tab(struct tid_info *); int alloc_atid(struct adapter *, void *); void *lookup_atid(struct adapter *, int); void free_atid(struct adapter *, int); void release_tid(struct adapter *, int, struct sge_wrq *); #ifdef DEV_NETMAP /* t4_netmap.c */ void cxgbe_nm_attach(struct vi_info *); void cxgbe_nm_detach(struct vi_info *); void t4_nm_intr(void *); #endif /* t4_sge.c */ void t4_sge_modload(void); void t4_sge_modunload(void); uint64_t t4_sge_extfree_refs(void); void t4_tweak_chip_settings(struct adapter *); int t4_read_chip_settings(struct adapter *); int t4_create_dma_tag(struct adapter *); void t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, struct sysctl_oid_list *); int t4_destroy_dma_tag(struct adapter *); int t4_setup_adapter_queues(struct adapter *); int t4_teardown_adapter_queues(struct adapter *); int t4_setup_vi_queues(struct vi_info *); int t4_teardown_vi_queues(struct vi_info *); void t4_intr_all(void *); void t4_intr(void *); void t4_vi_intr(void *); void t4_intr_err(void *); void t4_intr_evt(void *); void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *); void t4_update_fl_bufsize(struct ifnet *); int parse_pkt(struct adapter *, struct mbuf **); void *start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *); void commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *); int tnl_cong(struct port_info *, int); void t4_register_an_handler(an_handler_t); void t4_register_fw_msg_handler(int, fw_msg_handler_t); void t4_register_cpl_handler(int, cpl_handler_t); void t4_register_shared_cpl_handler(int, cpl_handler_t, int); /* t4_tracer.c */ struct t4_tracer; void t4_tracer_modload(void); void t4_tracer_modunload(void); void t4_tracer_port_detach(struct adapter *); int t4_get_tracer(struct adapter *, struct t4_tracer *); int t4_set_tracer(struct adapter *, struct t4_tracer *); int t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); int t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); /* t4_sched.c */ int t4_set_sched_class(struct adapter *, struct t4_sched_params *); int t4_set_sched_queue(struct adapter *, struct t4_sched_queue *); int t4_init_tx_sched(struct adapter *); int t4_free_tx_sched(struct adapter *); void t4_update_tx_sched(struct adapter *); int t4_reserve_cl_rl_kbps(struct adapter *, int, u_int, int *); void t4_release_cl_rl_kbps(struct adapter *, int, int); /* t4_filter.c */ int get_filter_mode(struct adapter *, uint32_t *); int set_filter_mode(struct adapter *, uint32_t); int get_filter(struct adapter *, struct t4_filter *); int set_filter(struct adapter *, struct t4_filter *); int del_filter(struct adapter *, struct t4_filter *); int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); int t4_hashfilter_ao_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); int t4_hashfilter_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); int t4_del_hashfilter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); void free_hftid_tab(struct tid_info *); static inline struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq) { int len = offsetof(struct wrqe, wr) + wr_len; struct wrqe *wr; wr = malloc(len, M_CXGBE, M_NOWAIT); if (__predict_false(wr == NULL)) return (NULL); wr->wr_len = wr_len; wr->wrq = wrq; return (wr); } static inline void * wrtod(struct wrqe *wr) { return (&wr->wr[0]); } static inline void free_wrqe(struct wrqe *wr) { free(wr, M_CXGBE); } static inline void t4_wrq_tx(struct adapter *sc, struct wrqe *wr) { struct sge_wrq *wrq = wr->wrq; TXQ_LOCK(wrq); t4_wrq_tx_locked(sc, wrq, wr); TXQ_UNLOCK(wrq); } static inline int read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, int len) { return (rw_via_memwin(sc, idx, addr, val, len, 0)); } static inline int write_via_memwin(struct adapter *sc, int idx, uint32_t addr, const uint32_t *val, int len) { return (rw_via_memwin(sc, idx, addr, (void *)(uintptr_t)val, len, 1)); } #endif Index: stable/11/sys/dev/cxgbe/common/t4_msg.h =================================================================== --- stable/11/sys/dev/cxgbe/common/t4_msg.h (revision 346862) +++ stable/11/sys/dev/cxgbe/common/t4_msg.h (revision 346863) @@ -1,3699 +1,3723 @@ /*- * Copyright (c) 2011, 2016 Chelsio Communications, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef T4_MSG_H #define T4_MSG_H enum { CPL_PASS_OPEN_REQ = 0x1, CPL_PASS_ACCEPT_RPL = 0x2, CPL_ACT_OPEN_REQ = 0x3, CPL_SET_TCB = 0x4, CPL_SET_TCB_FIELD = 0x5, CPL_GET_TCB = 0x6, CPL_CLOSE_CON_REQ = 0x8, CPL_CLOSE_LISTSRV_REQ = 0x9, CPL_ABORT_REQ = 0xA, CPL_ABORT_RPL = 0xB, CPL_TX_DATA = 0xC, CPL_RX_DATA_ACK = 0xD, CPL_TX_PKT = 0xE, CPL_RTE_DELETE_REQ = 0xF, CPL_RTE_WRITE_REQ = 0x10, CPL_RTE_READ_REQ = 0x11, CPL_L2T_WRITE_REQ = 0x12, CPL_L2T_READ_REQ = 0x13, CPL_SMT_WRITE_REQ = 0x14, CPL_SMT_READ_REQ = 0x15, CPL_TAG_WRITE_REQ = 0x16, CPL_BARRIER = 0x18, CPL_TID_RELEASE = 0x1A, CPL_TAG_READ_REQ = 0x1B, CPL_SRQ_TABLE_REQ = 0x1C, CPL_TX_PKT_FSO = 0x1E, CPL_TX_DATA_ISO = 0x1F, CPL_CLOSE_LISTSRV_RPL = 0x20, CPL_ERROR = 0x21, CPL_GET_TCB_RPL = 0x22, CPL_L2T_WRITE_RPL = 0x23, CPL_PASS_OPEN_RPL = 0x24, CPL_ACT_OPEN_RPL = 0x25, CPL_PEER_CLOSE = 0x26, CPL_RTE_DELETE_RPL = 0x27, CPL_RTE_WRITE_RPL = 0x28, CPL_RX_URG_PKT = 0x29, CPL_TAG_WRITE_RPL = 0x2A, CPL_ABORT_REQ_RSS = 0x2B, CPL_RX_URG_NOTIFY = 0x2C, CPL_ABORT_RPL_RSS = 0x2D, CPL_SMT_WRITE_RPL = 0x2E, CPL_TX_DATA_ACK = 0x2F, CPL_RX_PHYS_ADDR = 0x30, CPL_PCMD_READ_RPL = 0x31, CPL_CLOSE_CON_RPL = 0x32, CPL_ISCSI_HDR = 0x33, CPL_L2T_READ_RPL = 0x34, CPL_RDMA_CQE = 0x35, CPL_RDMA_CQE_READ_RSP = 0x36, CPL_RDMA_CQE_ERR = 0x37, CPL_RTE_READ_RPL = 0x38, CPL_RX_DATA = 0x39, CPL_SET_TCB_RPL = 0x3A, CPL_RX_PKT = 0x3B, CPL_TAG_READ_RPL = 0x3C, CPL_HIT_NOTIFY = 0x3D, CPL_PKT_NOTIFY = 0x3E, CPL_RX_DDP_COMPLETE = 0x3F, CPL_ACT_ESTABLISH = 0x40, CPL_PASS_ESTABLISH = 0x41, CPL_RX_DATA_DDP = 0x42, CPL_SMT_READ_RPL = 0x43, CPL_PASS_ACCEPT_REQ = 0x44, CPL_RX_ISCSI_CMP = 0x45, CPL_RX_FCOE_DDP = 0x46, CPL_FCOE_HDR = 0x47, CPL_T5_TRACE_PKT = 0x48, CPL_RX_ISCSI_DDP = 0x49, CPL_RX_FCOE_DIF = 0x4A, CPL_RX_DATA_DIF = 0x4B, CPL_ERR_NOTIFY = 0x4D, CPL_RX_TLS_CMP = 0x4E, CPL_RDMA_READ_REQ = 0x60, CPL_RX_ISCSI_DIF = 0x60, CPL_SET_LE_REQ = 0x80, CPL_PASS_OPEN_REQ6 = 0x81, CPL_ACT_OPEN_REQ6 = 0x83, CPL_TX_TLS_PDU = 0x88, CPL_TX_TLS_SFO = 0x89, CPL_TX_SEC_PDU = 0x8A, CPL_TX_TLS_ACK = 0x8B, CPL_RDMA_TERMINATE = 0xA2, CPL_RDMA_WRITE = 0xA4, CPL_SGE_EGR_UPDATE = 0xA5, CPL_SET_LE_RPL = 0xA6, CPL_FW2_MSG = 0xA7, CPL_FW2_PLD = 0xA8, CPL_T5_RDMA_READ_REQ = 0xA9, CPL_RDMA_ATOMIC_REQ = 0xAA, CPL_RDMA_ATOMIC_RPL = 0xAB, CPL_RDMA_IMM_DATA = 0xAC, CPL_RDMA_IMM_DATA_SE = 0xAD, CPL_RX_MPS_PKT = 0xAF, CPL_TRACE_PKT = 0xB0, CPL_RX2TX_DATA = 0xB1, CPL_TLS_DATA = 0xB1, CPL_ISCSI_DATA = 0xB2, CPL_FCOE_DATA = 0xB3, CPL_FW4_MSG = 0xC0, CPL_FW4_PLD = 0xC1, CPL_FW4_ACK = 0xC3, CPL_SRQ_TABLE_RPL = 0xCC, CPL_RX_PHYS_DSGL = 0xD0, CPL_FW6_MSG = 0xE0, CPL_FW6_PLD = 0xE1, CPL_TX_TNL_LSO = 0xEC, CPL_TX_PKT_LSO = 0xED, CPL_TX_PKT_XT = 0xEE, NUM_CPL_CMDS /* must be last and previous entries must be sorted */ }; enum CPL_error { CPL_ERR_NONE = 0, CPL_ERR_TCAM_PARITY = 1, CPL_ERR_TCAM_MISS = 2, CPL_ERR_TCAM_FULL = 3, CPL_ERR_BAD_LENGTH = 15, CPL_ERR_BAD_ROUTE = 18, CPL_ERR_CONN_RESET = 20, CPL_ERR_CONN_EXIST_SYNRECV = 21, CPL_ERR_CONN_EXIST = 22, CPL_ERR_ARP_MISS = 23, CPL_ERR_BAD_SYN = 24, CPL_ERR_CONN_TIMEDOUT = 30, CPL_ERR_XMIT_TIMEDOUT = 31, CPL_ERR_PERSIST_TIMEDOUT = 32, CPL_ERR_FINWAIT2_TIMEDOUT = 33, CPL_ERR_KEEPALIVE_TIMEDOUT = 34, CPL_ERR_RTX_NEG_ADVICE = 35, CPL_ERR_PERSIST_NEG_ADVICE = 36, CPL_ERR_KEEPALV_NEG_ADVICE = 37, CPL_ERR_WAIT_ARP_RPL = 41, CPL_ERR_ABORT_FAILED = 42, CPL_ERR_IWARP_FLM = 50, CPL_CONTAINS_READ_RPL = 60, CPL_CONTAINS_WRITE_RPL = 61, }; /* * Some of the error codes above implicitly indicate that there is no TID * allocated with the result of an ACT_OPEN. We use this predicate to make * that explicit. */ static inline int act_open_has_tid(int status) { return (status != CPL_ERR_TCAM_PARITY && status != CPL_ERR_TCAM_MISS && status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST_SYNRECV && status != CPL_ERR_CONN_EXIST); } /* * Convert an ACT_OPEN_RPL status to an errno. */ static inline int act_open_rpl_status_to_errno(int status) { switch (status) { case CPL_ERR_CONN_RESET: return (ECONNREFUSED); case CPL_ERR_ARP_MISS: return (EHOSTUNREACH); case CPL_ERR_CONN_TIMEDOUT: return (ETIMEDOUT); case CPL_ERR_TCAM_FULL: return (EAGAIN); case CPL_ERR_CONN_EXIST: return (EAGAIN); default: return (EIO); } } enum { CPL_CONN_POLICY_AUTO = 0, CPL_CONN_POLICY_ASK = 1, CPL_CONN_POLICY_FILTER = 2, CPL_CONN_POLICY_DENY = 3 }; enum { ULP_MODE_NONE = 0, ULP_MODE_ISCSI = 2, ULP_MODE_RDMA = 4, ULP_MODE_TCPDDP = 5, ULP_MODE_FCOE = 6, ULP_MODE_TLS = 8, }; enum { ULP_CRC_HEADER = 1 << 0, ULP_CRC_DATA = 1 << 1 }; enum { CPL_PASS_OPEN_ACCEPT, CPL_PASS_OPEN_REJECT, CPL_PASS_OPEN_ACCEPT_TNL }; enum { CPL_ABORT_SEND_RST = 0, CPL_ABORT_NO_RST, }; enum { /* TX_PKT_XT checksum types */ TX_CSUM_TCP = 0, TX_CSUM_UDP = 1, TX_CSUM_CRC16 = 4, TX_CSUM_CRC32 = 5, TX_CSUM_CRC32C = 6, TX_CSUM_FCOE = 7, TX_CSUM_TCPIP = 8, TX_CSUM_UDPIP = 9, TX_CSUM_TCPIP6 = 10, TX_CSUM_UDPIP6 = 11, TX_CSUM_IP = 12, }; enum { /* packet type in CPL_RX_PKT */ PKTYPE_XACT_UCAST = 0, PKTYPE_HASH_UCAST = 1, PKTYPE_XACT_MCAST = 2, PKTYPE_HASH_MCAST = 3, PKTYPE_PROMISC = 4, PKTYPE_HPROMISC = 5, PKTYPE_BCAST = 6 }; enum { /* DMAC type in CPL_RX_PKT */ DATYPE_UCAST, DATYPE_MCAST, DATYPE_BCAST }; enum { /* TCP congestion control algorithms */ CONG_ALG_RENO, CONG_ALG_TAHOE, CONG_ALG_NEWRENO, CONG_ALG_HIGHSPEED }; enum { /* RSS hash type */ RSS_HASH_NONE = 0, /* no hash computed */ RSS_HASH_IP = 1, /* IP or IPv6 2-tuple hash */ RSS_HASH_TCP = 2, /* TCP 4-tuple hash */ RSS_HASH_UDP = 3 /* UDP 4-tuple hash */ }; enum { /* LE commands */ LE_CMD_READ = 0x4, LE_CMD_WRITE = 0xb }; enum { /* LE request size */ LE_SZ_NONE = 0, LE_SZ_33 = 1, LE_SZ_66 = 2, LE_SZ_132 = 3, LE_SZ_264 = 4, LE_SZ_528 = 5 }; union opcode_tid { __be32 opcode_tid; __u8 opcode; }; #define S_CPL_OPCODE 24 #define V_CPL_OPCODE(x) ((x) << S_CPL_OPCODE) #define G_CPL_OPCODE(x) (((x) >> S_CPL_OPCODE) & 0xFF) #define G_TID(x) ((x) & 0xFFFFFF) /* tid is assumed to be 24-bits */ #define MK_OPCODE_TID(opcode, tid) (V_CPL_OPCODE(opcode) | (tid)) #define OPCODE_TID(cmd) ((cmd)->ot.opcode_tid) /* extract the TID from a CPL command */ #define GET_TID(cmd) (G_TID(ntohl(OPCODE_TID(cmd)))) #define GET_OPCODE(cmd) ((cmd)->ot.opcode) /* partitioning of TID fields that also carry a queue id */ #define S_TID_TID 0 #define M_TID_TID 0x7ff #define V_TID_TID(x) ((x) << S_TID_TID) #define G_TID_TID(x) (((x) >> S_TID_TID) & M_TID_TID) #define S_TID_COOKIE 11 #define M_TID_COOKIE 0x7 #define V_TID_COOKIE(x) ((x) << S_TID_COOKIE) #define G_TID_COOKIE(x) (((x) >> S_TID_COOKIE) & M_TID_COOKIE) #define S_TID_QID 14 #define M_TID_QID 0x3ff #define V_TID_QID(x) ((x) << S_TID_QID) #define G_TID_QID(x) (((x) >> S_TID_QID) & M_TID_QID) union opcode_info { __be64 opcode_info; __u8 opcode; }; struct tcp_options { __be16 mss; __u8 wsf; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 :4; __u8 unknown:1; __u8 ecn:1; __u8 sack:1; __u8 tstamp:1; #else __u8 tstamp:1; __u8 sack:1; __u8 ecn:1; __u8 unknown:1; __u8 :4; #endif }; struct rss_header { __u8 opcode; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 channel:2; __u8 filter_hit:1; __u8 filter_tid:1; __u8 hash_type:2; __u8 ipv6:1; __u8 send2fw:1; #else __u8 send2fw:1; __u8 ipv6:1; __u8 hash_type:2; __u8 filter_tid:1; __u8 filter_hit:1; __u8 channel:2; #endif __be16 qid; __be32 hash_val; }; #define S_HASHTYPE 20 #define M_HASHTYPE 0x3 #define G_HASHTYPE(x) (((x) >> S_HASHTYPE) & M_HASHTYPE) #define S_QNUM 0 #define M_QNUM 0xFFFF #define G_QNUM(x) (((x) >> S_QNUM) & M_QNUM) #if defined(RSS_HDR_VLD) || defined(CHELSIO_FW) # define RSS_HDR struct rss_header rss_hdr; #else # define RSS_HDR #endif #ifndef CHELSIO_FW struct work_request_hdr { __be32 wr_hi; __be32 wr_mid; __be64 wr_lo; }; /* wr_mid fields */ #define S_WR_LEN16 0 #define M_WR_LEN16 0xFF #define V_WR_LEN16(x) ((x) << S_WR_LEN16) #define G_WR_LEN16(x) (((x) >> S_WR_LEN16) & M_WR_LEN16) /* wr_hi fields */ #define S_WR_OP 24 #define M_WR_OP 0xFF #define V_WR_OP(x) ((__u64)(x) << S_WR_OP) #define G_WR_OP(x) (((x) >> S_WR_OP) & M_WR_OP) # define WR_HDR struct work_request_hdr wr # define WR_HDR_SIZE sizeof(struct work_request_hdr) #else # define WR_HDR # define WR_HDR_SIZE 0 #endif /* option 0 fields */ #define S_ACCEPT_MODE 0 #define M_ACCEPT_MODE 0x3 #define V_ACCEPT_MODE(x) ((x) << S_ACCEPT_MODE) #define G_ACCEPT_MODE(x) (((x) >> S_ACCEPT_MODE) & M_ACCEPT_MODE) #define S_TX_CHAN 2 #define M_TX_CHAN 0x3 #define V_TX_CHAN(x) ((x) << S_TX_CHAN) #define G_TX_CHAN(x) (((x) >> S_TX_CHAN) & M_TX_CHAN) #define S_NO_CONG 4 #define V_NO_CONG(x) ((x) << S_NO_CONG) #define F_NO_CONG V_NO_CONG(1U) #define S_DELACK 5 #define V_DELACK(x) ((x) << S_DELACK) #define F_DELACK V_DELACK(1U) #define S_INJECT_TIMER 6 #define V_INJECT_TIMER(x) ((x) << S_INJECT_TIMER) #define F_INJECT_TIMER V_INJECT_TIMER(1U) #define S_NON_OFFLOAD 7 #define V_NON_OFFLOAD(x) ((x) << S_NON_OFFLOAD) #define F_NON_OFFLOAD V_NON_OFFLOAD(1U) #define S_ULP_MODE 8 #define M_ULP_MODE 0xF #define V_ULP_MODE(x) ((x) << S_ULP_MODE) #define G_ULP_MODE(x) (((x) >> S_ULP_MODE) & M_ULP_MODE) #define S_RCV_BUFSIZ 12 #define M_RCV_BUFSIZ 0x3FFU #define V_RCV_BUFSIZ(x) ((x) << S_RCV_BUFSIZ) #define G_RCV_BUFSIZ(x) (((x) >> S_RCV_BUFSIZ) & M_RCV_BUFSIZ) #define S_DSCP 22 #define M_DSCP 0x3F #define V_DSCP(x) ((x) << S_DSCP) #define G_DSCP(x) (((x) >> S_DSCP) & M_DSCP) #define S_SMAC_SEL 28 #define M_SMAC_SEL 0xFF #define V_SMAC_SEL(x) ((__u64)(x) << S_SMAC_SEL) #define G_SMAC_SEL(x) (((x) >> S_SMAC_SEL) & M_SMAC_SEL) #define S_L2T_IDX 36 #define M_L2T_IDX 0xFFF #define V_L2T_IDX(x) ((__u64)(x) << S_L2T_IDX) #define G_L2T_IDX(x) (((x) >> S_L2T_IDX) & M_L2T_IDX) #define S_TCAM_BYPASS 48 #define V_TCAM_BYPASS(x) ((__u64)(x) << S_TCAM_BYPASS) #define F_TCAM_BYPASS V_TCAM_BYPASS(1ULL) #define S_NAGLE 49 #define V_NAGLE(x) ((__u64)(x) << S_NAGLE) #define F_NAGLE V_NAGLE(1ULL) #define S_WND_SCALE 50 #define M_WND_SCALE 0xF #define V_WND_SCALE(x) ((__u64)(x) << S_WND_SCALE) #define G_WND_SCALE(x) (((x) >> S_WND_SCALE) & M_WND_SCALE) #define S_KEEP_ALIVE 54 #define V_KEEP_ALIVE(x) ((__u64)(x) << S_KEEP_ALIVE) #define F_KEEP_ALIVE V_KEEP_ALIVE(1ULL) #define S_MAX_RT 55 #define M_MAX_RT 0xF #define V_MAX_RT(x) ((__u64)(x) << S_MAX_RT) #define G_MAX_RT(x) (((x) >> S_MAX_RT) & M_MAX_RT) #define S_MAX_RT_OVERRIDE 59 #define V_MAX_RT_OVERRIDE(x) ((__u64)(x) << S_MAX_RT_OVERRIDE) #define F_MAX_RT_OVERRIDE V_MAX_RT_OVERRIDE(1ULL) #define S_MSS_IDX 60 #define M_MSS_IDX 0xF #define V_MSS_IDX(x) ((__u64)(x) << S_MSS_IDX) #define G_MSS_IDX(x) (((x) >> S_MSS_IDX) & M_MSS_IDX) /* option 1 fields */ #define S_SYN_RSS_ENABLE 0 #define V_SYN_RSS_ENABLE(x) ((x) << S_SYN_RSS_ENABLE) #define F_SYN_RSS_ENABLE V_SYN_RSS_ENABLE(1U) #define S_SYN_RSS_USE_HASH 1 #define V_SYN_RSS_USE_HASH(x) ((x) << S_SYN_RSS_USE_HASH) #define F_SYN_RSS_USE_HASH V_SYN_RSS_USE_HASH(1U) #define S_SYN_RSS_QUEUE 2 #define M_SYN_RSS_QUEUE 0x3FF #define V_SYN_RSS_QUEUE(x) ((x) << S_SYN_RSS_QUEUE) #define G_SYN_RSS_QUEUE(x) (((x) >> S_SYN_RSS_QUEUE) & M_SYN_RSS_QUEUE) #define S_LISTEN_INTF 12 #define M_LISTEN_INTF 0xFF #define V_LISTEN_INTF(x) ((x) << S_LISTEN_INTF) #define G_LISTEN_INTF(x) (((x) >> S_LISTEN_INTF) & M_LISTEN_INTF) #define S_LISTEN_FILTER 20 #define V_LISTEN_FILTER(x) ((x) << S_LISTEN_FILTER) #define F_LISTEN_FILTER V_LISTEN_FILTER(1U) #define S_SYN_DEFENSE 21 #define V_SYN_DEFENSE(x) ((x) << S_SYN_DEFENSE) #define F_SYN_DEFENSE V_SYN_DEFENSE(1U) #define S_CONN_POLICY 22 #define M_CONN_POLICY 0x3 #define V_CONN_POLICY(x) ((x) << S_CONN_POLICY) #define G_CONN_POLICY(x) (((x) >> S_CONN_POLICY) & M_CONN_POLICY) #define S_T5_FILT_INFO 24 #define M_T5_FILT_INFO 0xffffffffffULL #define V_T5_FILT_INFO(x) ((x) << S_T5_FILT_INFO) #define G_T5_FILT_INFO(x) (((x) >> S_T5_FILT_INFO) & M_T5_FILT_INFO) #define S_FILT_INFO 28 #define M_FILT_INFO 0xfffffffffULL #define V_FILT_INFO(x) ((x) << S_FILT_INFO) #define G_FILT_INFO(x) (((x) >> S_FILT_INFO) & M_FILT_INFO) /* option 2 fields */ #define S_RSS_QUEUE 0 #define M_RSS_QUEUE 0x3FF #define V_RSS_QUEUE(x) ((x) << S_RSS_QUEUE) #define G_RSS_QUEUE(x) (((x) >> S_RSS_QUEUE) & M_RSS_QUEUE) #define S_RSS_QUEUE_VALID 10 #define V_RSS_QUEUE_VALID(x) ((x) << S_RSS_QUEUE_VALID) #define F_RSS_QUEUE_VALID V_RSS_QUEUE_VALID(1U) #define S_RX_COALESCE_VALID 11 #define V_RX_COALESCE_VALID(x) ((x) << S_RX_COALESCE_VALID) #define F_RX_COALESCE_VALID V_RX_COALESCE_VALID(1U) #define S_RX_COALESCE 12 #define M_RX_COALESCE 0x3 #define V_RX_COALESCE(x) ((x) << S_RX_COALESCE) #define G_RX_COALESCE(x) (((x) >> S_RX_COALESCE) & M_RX_COALESCE) #define S_CONG_CNTRL 14 #define M_CONG_CNTRL 0x3 #define V_CONG_CNTRL(x) ((x) << S_CONG_CNTRL) #define G_CONG_CNTRL(x) (((x) >> S_CONG_CNTRL) & M_CONG_CNTRL) #define S_PACE 16 #define M_PACE 0x3 #define V_PACE(x) ((x) << S_PACE) #define G_PACE(x) (((x) >> S_PACE) & M_PACE) #define S_CONG_CNTRL_VALID 18 #define V_CONG_CNTRL_VALID(x) ((x) << S_CONG_CNTRL_VALID) #define F_CONG_CNTRL_VALID V_CONG_CNTRL_VALID(1U) #define S_T5_ISS 18 #define V_T5_ISS(x) ((x) << S_T5_ISS) #define F_T5_ISS V_T5_ISS(1U) #define S_PACE_VALID 19 #define V_PACE_VALID(x) ((x) << S_PACE_VALID) #define F_PACE_VALID V_PACE_VALID(1U) #define S_RX_FC_DISABLE 20 #define V_RX_FC_DISABLE(x) ((x) << S_RX_FC_DISABLE) #define F_RX_FC_DISABLE V_RX_FC_DISABLE(1U) #define S_RX_FC_DDP 21 #define V_RX_FC_DDP(x) ((x) << S_RX_FC_DDP) #define F_RX_FC_DDP V_RX_FC_DDP(1U) #define S_RX_FC_VALID 22 #define V_RX_FC_VALID(x) ((x) << S_RX_FC_VALID) #define F_RX_FC_VALID V_RX_FC_VALID(1U) #define S_TX_QUEUE 23 #define M_TX_QUEUE 0x7 #define V_TX_QUEUE(x) ((x) << S_TX_QUEUE) #define G_TX_QUEUE(x) (((x) >> S_TX_QUEUE) & M_TX_QUEUE) #define S_RX_CHANNEL 26 #define V_RX_CHANNEL(x) ((x) << S_RX_CHANNEL) #define F_RX_CHANNEL V_RX_CHANNEL(1U) #define S_CCTRL_ECN 27 #define V_CCTRL_ECN(x) ((x) << S_CCTRL_ECN) #define F_CCTRL_ECN V_CCTRL_ECN(1U) #define S_WND_SCALE_EN 28 #define V_WND_SCALE_EN(x) ((x) << S_WND_SCALE_EN) #define F_WND_SCALE_EN V_WND_SCALE_EN(1U) #define S_TSTAMPS_EN 29 #define V_TSTAMPS_EN(x) ((x) << S_TSTAMPS_EN) #define F_TSTAMPS_EN V_TSTAMPS_EN(1U) #define S_SACK_EN 30 #define V_SACK_EN(x) ((x) << S_SACK_EN) #define F_SACK_EN V_SACK_EN(1U) #define S_T5_OPT_2_VALID 31 #define V_T5_OPT_2_VALID(x) ((x) << S_T5_OPT_2_VALID) #define F_T5_OPT_2_VALID V_T5_OPT_2_VALID(1U) struct cpl_pass_open_req { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be64 opt0; __be64 opt1; }; struct cpl_pass_open_req6 { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be64 local_ip_hi; __be64 local_ip_lo; __be64 peer_ip_hi; __be64 peer_ip_lo; __be64 opt0; __be64 opt1; }; struct cpl_pass_open_rpl { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; }; struct cpl_pass_establish { RSS_HDR union opcode_tid ot; __be32 rsvd; __be32 tos_stid; __be16 mac_idx; __be16 tcp_opt; __be32 snd_isn; __be32 rcv_isn; }; /* cpl_pass_establish.tos_stid fields */ #define S_PASS_OPEN_TID 0 #define M_PASS_OPEN_TID 0xFFFFFF #define V_PASS_OPEN_TID(x) ((x) << S_PASS_OPEN_TID) #define G_PASS_OPEN_TID(x) (((x) >> S_PASS_OPEN_TID) & M_PASS_OPEN_TID) #define S_PASS_OPEN_TOS 24 #define M_PASS_OPEN_TOS 0xFF #define V_PASS_OPEN_TOS(x) ((x) << S_PASS_OPEN_TOS) #define G_PASS_OPEN_TOS(x) (((x) >> S_PASS_OPEN_TOS) & M_PASS_OPEN_TOS) /* cpl_pass_establish.tcp_opt fields (also applies to act_open_establish) */ #define S_TCPOPT_WSCALE_OK 5 #define M_TCPOPT_WSCALE_OK 0x1 #define V_TCPOPT_WSCALE_OK(x) ((x) << S_TCPOPT_WSCALE_OK) #define G_TCPOPT_WSCALE_OK(x) (((x) >> S_TCPOPT_WSCALE_OK) & M_TCPOPT_WSCALE_OK) #define S_TCPOPT_SACK 6 #define M_TCPOPT_SACK 0x1 #define V_TCPOPT_SACK(x) ((x) << S_TCPOPT_SACK) #define G_TCPOPT_SACK(x) (((x) >> S_TCPOPT_SACK) & M_TCPOPT_SACK) #define S_TCPOPT_TSTAMP 7 #define M_TCPOPT_TSTAMP 0x1 #define V_TCPOPT_TSTAMP(x) ((x) << S_TCPOPT_TSTAMP) #define G_TCPOPT_TSTAMP(x) (((x) >> S_TCPOPT_TSTAMP) & M_TCPOPT_TSTAMP) #define S_TCPOPT_SND_WSCALE 8 #define M_TCPOPT_SND_WSCALE 0xF #define V_TCPOPT_SND_WSCALE(x) ((x) << S_TCPOPT_SND_WSCALE) #define G_TCPOPT_SND_WSCALE(x) (((x) >> S_TCPOPT_SND_WSCALE) & M_TCPOPT_SND_WSCALE) #define S_TCPOPT_MSS 12 #define M_TCPOPT_MSS 0xF #define V_TCPOPT_MSS(x) ((x) << S_TCPOPT_MSS) #define G_TCPOPT_MSS(x) (((x) >> S_TCPOPT_MSS) & M_TCPOPT_MSS) struct cpl_pass_accept_req { RSS_HDR union opcode_tid ot; __be16 rsvd; __be16 len; __be32 hdr_len; __be16 vlan; __be16 l2info; __be32 tos_stid; struct tcp_options tcpopt; }; /* cpl_pass_accept_req.hdr_len fields */ #define S_SYN_RX_CHAN 0 #define M_SYN_RX_CHAN 0xF #define V_SYN_RX_CHAN(x) ((x) << S_SYN_RX_CHAN) #define G_SYN_RX_CHAN(x) (((x) >> S_SYN_RX_CHAN) & M_SYN_RX_CHAN) #define S_TCP_HDR_LEN 10 #define M_TCP_HDR_LEN 0x3F #define V_TCP_HDR_LEN(x) ((x) << S_TCP_HDR_LEN) #define G_TCP_HDR_LEN(x) (((x) >> S_TCP_HDR_LEN) & M_TCP_HDR_LEN) #define S_T6_TCP_HDR_LEN 8 #define V_T6_TCP_HDR_LEN(x) ((x) << S_T6_TCP_HDR_LEN) #define G_T6_TCP_HDR_LEN(x) (((x) >> S_T6_TCP_HDR_LEN) & M_TCP_HDR_LEN) #define S_IP_HDR_LEN 16 #define M_IP_HDR_LEN 0x3FF #define V_IP_HDR_LEN(x) ((x) << S_IP_HDR_LEN) #define G_IP_HDR_LEN(x) (((x) >> S_IP_HDR_LEN) & M_IP_HDR_LEN) #define S_T6_IP_HDR_LEN 14 #define V_T6_IP_HDR_LEN(x) ((x) << S_T6_IP_HDR_LEN) #define G_T6_IP_HDR_LEN(x) (((x) >> S_T6_IP_HDR_LEN) & M_IP_HDR_LEN) #define S_ETH_HDR_LEN 26 #define M_ETH_HDR_LEN 0x3F #define V_ETH_HDR_LEN(x) ((x) << S_ETH_HDR_LEN) #define G_ETH_HDR_LEN(x) (((x) >> S_ETH_HDR_LEN) & M_ETH_HDR_LEN) #define S_T6_ETH_HDR_LEN 24 #define M_T6_ETH_HDR_LEN 0xFF #define V_T6_ETH_HDR_LEN(x) ((x) << S_T6_ETH_HDR_LEN) #define G_T6_ETH_HDR_LEN(x) (((x) >> S_T6_ETH_HDR_LEN) & M_T6_ETH_HDR_LEN) /* cpl_pass_accept_req.l2info fields */ #define S_SYN_MAC_IDX 0 #define M_SYN_MAC_IDX 0x1FF #define V_SYN_MAC_IDX(x) ((x) << S_SYN_MAC_IDX) #define G_SYN_MAC_IDX(x) (((x) >> S_SYN_MAC_IDX) & M_SYN_MAC_IDX) #define S_SYN_XACT_MATCH 9 #define V_SYN_XACT_MATCH(x) ((x) << S_SYN_XACT_MATCH) #define F_SYN_XACT_MATCH V_SYN_XACT_MATCH(1U) #define S_SYN_INTF 12 #define M_SYN_INTF 0xF #define V_SYN_INTF(x) ((x) << S_SYN_INTF) #define G_SYN_INTF(x) (((x) >> S_SYN_INTF) & M_SYN_INTF) struct cpl_pass_accept_rpl { WR_HDR; union opcode_tid ot; __be32 opt2; __be64 opt0; }; struct cpl_t5_pass_accept_rpl { WR_HDR; union opcode_tid ot; __be32 opt2; __be64 opt0; __be32 iss; union { __be32 rsvd; /* T5 */ __be32 opt3; /* T6 */ } u; }; struct cpl_act_open_req { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be64 opt0; __be32 params; __be32 opt2; }; #define S_FILTER_TUPLE 24 #define M_FILTER_TUPLE 0xFFFFFFFFFF #define V_FILTER_TUPLE(x) ((x) << S_FILTER_TUPLE) #define G_FILTER_TUPLE(x) (((x) >> S_FILTER_TUPLE) & M_FILTER_TUPLE) struct cpl_t5_act_open_req { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be64 opt0; __be32 iss; __be32 opt2; __be64 params; }; struct cpl_t6_act_open_req { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be32 local_ip; __be32 peer_ip; __be64 opt0; __be32 iss; __be32 opt2; __be64 params; __be32 rsvd2; __be32 opt3; }; /* cpl_{t5,t6}_act_open_req.params field */ #define S_AOPEN_FCOEMASK 0 #define V_AOPEN_FCOEMASK(x) ((x) << S_AOPEN_FCOEMASK) #define F_AOPEN_FCOEMASK V_AOPEN_FCOEMASK(1U) struct cpl_act_open_req6 { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be64 local_ip_hi; __be64 local_ip_lo; __be64 peer_ip_hi; __be64 peer_ip_lo; __be64 opt0; __be32 params; __be32 opt2; }; struct cpl_t5_act_open_req6 { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be64 local_ip_hi; __be64 local_ip_lo; __be64 peer_ip_hi; __be64 peer_ip_lo; __be64 opt0; __be32 iss; __be32 opt2; __be64 params; }; struct cpl_t6_act_open_req6 { WR_HDR; union opcode_tid ot; __be16 local_port; __be16 peer_port; __be64 local_ip_hi; __be64 local_ip_lo; __be64 peer_ip_hi; __be64 peer_ip_lo; __be64 opt0; __be32 iss; __be32 opt2; __be64 params; __be32 rsvd2; __be32 opt3; }; struct cpl_act_open_rpl { RSS_HDR union opcode_tid ot; __be32 atid_status; }; /* cpl_act_open_rpl.atid_status fields */ #define S_AOPEN_STATUS 0 #define M_AOPEN_STATUS 0xFF #define V_AOPEN_STATUS(x) ((x) << S_AOPEN_STATUS) #define G_AOPEN_STATUS(x) (((x) >> S_AOPEN_STATUS) & M_AOPEN_STATUS) #define S_AOPEN_ATID 8 #define M_AOPEN_ATID 0xFFFFFF #define V_AOPEN_ATID(x) ((x) << S_AOPEN_ATID) #define G_AOPEN_ATID(x) (((x) >> S_AOPEN_ATID) & M_AOPEN_ATID) struct cpl_act_establish { RSS_HDR union opcode_tid ot; __be32 rsvd; __be32 tos_atid; __be16 mac_idx; __be16 tcp_opt; __be32 snd_isn; __be32 rcv_isn; }; struct cpl_get_tcb { WR_HDR; union opcode_tid ot; __be16 reply_ctrl; __be16 cookie; }; /* cpl_get_tcb.reply_ctrl fields */ #define S_QUEUENO 0 #define M_QUEUENO 0x3FF #define V_QUEUENO(x) ((x) << S_QUEUENO) #define G_QUEUENO(x) (((x) >> S_QUEUENO) & M_QUEUENO) #define S_REPLY_CHAN 14 #define V_REPLY_CHAN(x) ((x) << S_REPLY_CHAN) #define F_REPLY_CHAN V_REPLY_CHAN(1U) #define S_NO_REPLY 15 #define V_NO_REPLY(x) ((x) << S_NO_REPLY) #define F_NO_REPLY V_NO_REPLY(1U) struct cpl_get_tcb_rpl { RSS_HDR union opcode_tid ot; __u8 cookie; __u8 status; __be16 len; }; struct cpl_set_tcb { WR_HDR; union opcode_tid ot; __be16 reply_ctrl; __be16 cookie; }; struct cpl_set_tcb_field { WR_HDR; union opcode_tid ot; __be16 reply_ctrl; __be16 word_cookie; __be64 mask; __be64 val; }; struct cpl_set_tcb_field_core { union opcode_tid ot; __be16 reply_ctrl; __be16 word_cookie; __be64 mask; __be64 val; }; /* cpl_set_tcb_field.word_cookie fields */ #define S_WORD 0 #define M_WORD 0x1F #define V_WORD(x) ((x) << S_WORD) #define G_WORD(x) (((x) >> S_WORD) & M_WORD) #define S_COOKIE 5 #define M_COOKIE 0x7 #define V_COOKIE(x) ((x) << S_COOKIE) #define G_COOKIE(x) (((x) >> S_COOKIE) & M_COOKIE) struct cpl_set_tcb_rpl { RSS_HDR union opcode_tid ot; __be16 rsvd; __u8 cookie; __u8 status; __be64 oldval; }; struct cpl_close_con_req { WR_HDR; union opcode_tid ot; __be32 rsvd; }; struct cpl_close_con_rpl { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; __be32 snd_nxt; __be32 rcv_nxt; }; struct cpl_close_listsvr_req { WR_HDR; union opcode_tid ot; __be16 reply_ctrl; __be16 rsvd; }; /* additional cpl_close_listsvr_req.reply_ctrl field */ #define S_LISTSVR_IPV6 14 #define V_LISTSVR_IPV6(x) ((x) << S_LISTSVR_IPV6) #define F_LISTSVR_IPV6 V_LISTSVR_IPV6(1U) struct cpl_close_listsvr_rpl { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; }; struct cpl_abort_req_rss { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; }; struct cpl_abort_req_rss6 { RSS_HDR union opcode_tid ot; __u32 srqidx_status; }; #define S_ABORT_RSS_STATUS 0 #define M_ABORT_RSS_STATUS 0xff #define V_ABORT_RSS_STATUS(x) ((x) << S_ABORT_RSS_STATUS) #define G_ABORT_RSS_STATUS(x) (((x) >> S_ABORT_RSS_STATUS) & M_ABORT_RSS_STATUS) #define S_ABORT_RSS_SRQIDX 8 #define M_ABORT_RSS_SRQIDX 0xffffff #define V_ABORT_RSS_SRQIDX(x) ((x) << S_ABORT_RSS_SRQIDX) #define G_ABORT_RSS_SRQIDX(x) (((x) >> S_ABORT_RSS_SRQIDX) & M_ABORT_RSS_SRQIDX) /* cpl_abort_req status command code in case of T6, * bit[0] specifies whether to send RST (0) to remote peer or suppress it (1) * bit[1] indicates ABORT_REQ was sent after a CLOSE_CON_REQ * bit[2] specifies whether to disable the mmgr (1) or not (0) */ struct cpl_abort_req { WR_HDR; union opcode_tid ot; __be32 rsvd0; __u8 rsvd1; __u8 cmd; __u8 rsvd2[6]; }; struct cpl_abort_req_core { union opcode_tid ot; __be32 rsvd0; __u8 rsvd1; __u8 cmd; __u8 rsvd2[6]; }; struct cpl_abort_rpl_rss { RSS_HDR union opcode_tid ot; __u8 rsvd[3]; __u8 status; }; struct cpl_abort_rpl_rss6 { RSS_HDR union opcode_tid ot; __u32 srqidx_status; }; struct cpl_abort_rpl { WR_HDR; union opcode_tid ot; __be32 rsvd0; __u8 rsvd1; __u8 cmd; __u8 rsvd2[6]; }; struct cpl_abort_rpl_core { union opcode_tid ot; __be32 rsvd0; __u8 rsvd1; __u8 cmd; __u8 rsvd2[6]; }; struct cpl_peer_close { RSS_HDR union opcode_tid ot; __be32 rcv_nxt; }; struct cpl_tid_release { WR_HDR; union opcode_tid ot; __be32 rsvd; }; struct tx_data_wr { __be32 wr_hi; __be32 wr_lo; __be32 len; __be32 flags; __be32 sndseq; __be32 param; }; /* tx_data_wr.flags fields */ #define S_TX_ACK_PAGES 21 #define M_TX_ACK_PAGES 0x7 #define V_TX_ACK_PAGES(x) ((x) << S_TX_ACK_PAGES) #define G_TX_ACK_PAGES(x) (((x) >> S_TX_ACK_PAGES) & M_TX_ACK_PAGES) /* tx_data_wr.param fields */ #define S_TX_PORT 0 #define M_TX_PORT 0x7 #define V_TX_PORT(x) ((x) << S_TX_PORT) #define G_TX_PORT(x) (((x) >> S_TX_PORT) & M_TX_PORT) #define S_TX_MSS 4 #define M_TX_MSS 0xF #define V_TX_MSS(x) ((x) << S_TX_MSS) #define G_TX_MSS(x) (((x) >> S_TX_MSS) & M_TX_MSS) #define S_TX_QOS 8 #define M_TX_QOS 0xFF #define V_TX_QOS(x) ((x) << S_TX_QOS) #define G_TX_QOS(x) (((x) >> S_TX_QOS) & M_TX_QOS) #define S_TX_SNDBUF 16 #define M_TX_SNDBUF 0xFFFF #define V_TX_SNDBUF(x) ((x) << S_TX_SNDBUF) #define G_TX_SNDBUF(x) (((x) >> S_TX_SNDBUF) & M_TX_SNDBUF) struct cpl_tx_data { union opcode_tid ot; __be32 len; __be32 rsvd; __be32 flags; }; /* cpl_tx_data.flags fields */ #define S_TX_PROXY 5 #define V_TX_PROXY(x) ((x) << S_TX_PROXY) #define F_TX_PROXY V_TX_PROXY(1U) #define S_TX_ULP_SUBMODE 6 #define M_TX_ULP_SUBMODE 0xF #define V_TX_ULP_SUBMODE(x) ((x) << S_TX_ULP_SUBMODE) #define G_TX_ULP_SUBMODE(x) (((x) >> S_TX_ULP_SUBMODE) & M_TX_ULP_SUBMODE) #define S_TX_ULP_MODE 10 #define M_TX_ULP_MODE 0x7 #define V_TX_ULP_MODE(x) ((x) << S_TX_ULP_MODE) #define G_TX_ULP_MODE(x) (((x) >> S_TX_ULP_MODE) & M_TX_ULP_MODE) #define S_TX_FORCE 13 #define V_TX_FORCE(x) ((x) << S_TX_FORCE) #define F_TX_FORCE V_TX_FORCE(1U) #define S_TX_SHOVE 14 #define V_TX_SHOVE(x) ((x) << S_TX_SHOVE) #define F_TX_SHOVE V_TX_SHOVE(1U) #define S_TX_MORE 15 #define V_TX_MORE(x) ((x) << S_TX_MORE) #define F_TX_MORE V_TX_MORE(1U) #define S_TX_URG 16 #define V_TX_URG(x) ((x) << S_TX_URG) #define F_TX_URG V_TX_URG(1U) #define S_TX_FLUSH 17 #define V_TX_FLUSH(x) ((x) << S_TX_FLUSH) #define F_TX_FLUSH V_TX_FLUSH(1U) #define S_TX_SAVE 18 #define V_TX_SAVE(x) ((x) << S_TX_SAVE) #define F_TX_SAVE V_TX_SAVE(1U) #define S_TX_TNL 19 #define V_TX_TNL(x) ((x) << S_TX_TNL) #define F_TX_TNL V_TX_TNL(1U) #define S_T6_TX_FORCE 20 #define V_T6_TX_FORCE(x) ((x) << S_T6_TX_FORCE) #define F_T6_TX_FORCE V_T6_TX_FORCE(1U) /* additional tx_data_wr.flags fields */ #define S_TX_CPU_IDX 0 #define M_TX_CPU_IDX 0x3F #define V_TX_CPU_IDX(x) ((x) << S_TX_CPU_IDX) #define G_TX_CPU_IDX(x) (((x) >> S_TX_CPU_IDX) & M_TX_CPU_IDX) #define S_TX_CLOSE 17 #define V_TX_CLOSE(x) ((x) << S_TX_CLOSE) #define F_TX_CLOSE V_TX_CLOSE(1U) #define S_TX_INIT 18 #define V_TX_INIT(x) ((x) << S_TX_INIT) #define F_TX_INIT V_TX_INIT(1U) #define S_TX_IMM_ACK 19 #define V_TX_IMM_ACK(x) ((x) << S_TX_IMM_ACK) #define F_TX_IMM_ACK V_TX_IMM_ACK(1U) #define S_TX_IMM_DMA 20 #define V_TX_IMM_DMA(x) ((x) << S_TX_IMM_DMA) #define F_TX_IMM_DMA V_TX_IMM_DMA(1U) struct cpl_tx_data_ack { RSS_HDR union opcode_tid ot; __be32 snd_una; }; struct cpl_wr_ack { /* XXX */ RSS_HDR union opcode_tid ot; __be16 credits; __be16 rsvd; __be32 snd_nxt; __be32 snd_una; }; struct cpl_tx_pkt_core { __be32 ctrl0; __be16 pack; __be16 len; __be64 ctrl1; }; struct cpl_tx_pkt { WR_HDR; struct cpl_tx_pkt_core c; }; #define cpl_tx_pkt_xt cpl_tx_pkt /* cpl_tx_pkt_core.ctrl0 fields */ #define S_TXPKT_VF 0 #define M_TXPKT_VF 0xFF #define V_TXPKT_VF(x) ((x) << S_TXPKT_VF) #define G_TXPKT_VF(x) (((x) >> S_TXPKT_VF) & M_TXPKT_VF) #define S_TXPKT_PF 8 #define M_TXPKT_PF 0x7 #define V_TXPKT_PF(x) ((x) << S_TXPKT_PF) #define G_TXPKT_PF(x) (((x) >> S_TXPKT_PF) & M_TXPKT_PF) #define S_TXPKT_VF_VLD 11 #define V_TXPKT_VF_VLD(x) ((x) << S_TXPKT_VF_VLD) #define F_TXPKT_VF_VLD V_TXPKT_VF_VLD(1U) #define S_TXPKT_OVLAN_IDX 12 #define M_TXPKT_OVLAN_IDX 0xF #define V_TXPKT_OVLAN_IDX(x) ((x) << S_TXPKT_OVLAN_IDX) #define G_TXPKT_OVLAN_IDX(x) (((x) >> S_TXPKT_OVLAN_IDX) & M_TXPKT_OVLAN_IDX) #define S_TXPKT_T5_OVLAN_IDX 12 #define M_TXPKT_T5_OVLAN_IDX 0x7 #define V_TXPKT_T5_OVLAN_IDX(x) ((x) << S_TXPKT_T5_OVLAN_IDX) #define G_TXPKT_T5_OVLAN_IDX(x) (((x) >> S_TXPKT_T5_OVLAN_IDX) & \ M_TXPKT_T5_OVLAN_IDX) #define S_TXPKT_INTF 16 #define M_TXPKT_INTF 0xF #define V_TXPKT_INTF(x) ((x) << S_TXPKT_INTF) #define G_TXPKT_INTF(x) (((x) >> S_TXPKT_INTF) & M_TXPKT_INTF) #define S_TXPKT_SPECIAL_STAT 20 #define V_TXPKT_SPECIAL_STAT(x) ((x) << S_TXPKT_SPECIAL_STAT) #define F_TXPKT_SPECIAL_STAT V_TXPKT_SPECIAL_STAT(1U) #define S_TXPKT_T5_FCS_DIS 21 #define V_TXPKT_T5_FCS_DIS(x) ((x) << S_TXPKT_T5_FCS_DIS) #define F_TXPKT_T5_FCS_DIS V_TXPKT_T5_FCS_DIS(1U) #define S_TXPKT_INS_OVLAN 21 #define V_TXPKT_INS_OVLAN(x) ((x) << S_TXPKT_INS_OVLAN) #define F_TXPKT_INS_OVLAN V_TXPKT_INS_OVLAN(1U) #define S_TXPKT_T5_INS_OVLAN 15 #define V_TXPKT_T5_INS_OVLAN(x) ((x) << S_TXPKT_T5_INS_OVLAN) #define F_TXPKT_T5_INS_OVLAN V_TXPKT_T5_INS_OVLAN(1U) #define S_TXPKT_STAT_DIS 22 #define V_TXPKT_STAT_DIS(x) ((x) << S_TXPKT_STAT_DIS) #define F_TXPKT_STAT_DIS V_TXPKT_STAT_DIS(1U) #define S_TXPKT_LOOPBACK 23 #define V_TXPKT_LOOPBACK(x) ((x) << S_TXPKT_LOOPBACK) #define F_TXPKT_LOOPBACK V_TXPKT_LOOPBACK(1U) #define S_TXPKT_TSTAMP 23 #define V_TXPKT_TSTAMP(x) ((x) << S_TXPKT_TSTAMP) #define F_TXPKT_TSTAMP V_TXPKT_TSTAMP(1U) #define S_TXPKT_OPCODE 24 #define M_TXPKT_OPCODE 0xFF #define V_TXPKT_OPCODE(x) ((x) << S_TXPKT_OPCODE) #define G_TXPKT_OPCODE(x) (((x) >> S_TXPKT_OPCODE) & M_TXPKT_OPCODE) /* cpl_tx_pkt_core.ctrl1 fields */ #define S_TXPKT_SA_IDX 0 #define M_TXPKT_SA_IDX 0xFFF #define V_TXPKT_SA_IDX(x) ((x) << S_TXPKT_SA_IDX) #define G_TXPKT_SA_IDX(x) (((x) >> S_TXPKT_SA_IDX) & M_TXPKT_SA_IDX) #define S_TXPKT_CSUM_END 12 #define M_TXPKT_CSUM_END 0xFF #define V_TXPKT_CSUM_END(x) ((x) << S_TXPKT_CSUM_END) #define G_TXPKT_CSUM_END(x) (((x) >> S_TXPKT_CSUM_END) & M_TXPKT_CSUM_END) #define S_TXPKT_CSUM_START 20 #define M_TXPKT_CSUM_START 0x3FF #define V_TXPKT_CSUM_START(x) ((x) << S_TXPKT_CSUM_START) #define G_TXPKT_CSUM_START(x) (((x) >> S_TXPKT_CSUM_START) & M_TXPKT_CSUM_START) #define S_TXPKT_IPHDR_LEN 20 #define M_TXPKT_IPHDR_LEN 0x3FFF #define V_TXPKT_IPHDR_LEN(x) ((__u64)(x) << S_TXPKT_IPHDR_LEN) #define G_TXPKT_IPHDR_LEN(x) (((x) >> S_TXPKT_IPHDR_LEN) & M_TXPKT_IPHDR_LEN) #define M_T6_TXPKT_IPHDR_LEN 0xFFF #define G_T6_TXPKT_IPHDR_LEN(x) \ (((x) >> S_TXPKT_IPHDR_LEN) & M_T6_TXPKT_IPHDR_LEN) #define S_TXPKT_CSUM_LOC 30 #define M_TXPKT_CSUM_LOC 0x3FF #define V_TXPKT_CSUM_LOC(x) ((__u64)(x) << S_TXPKT_CSUM_LOC) #define G_TXPKT_CSUM_LOC(x) (((x) >> S_TXPKT_CSUM_LOC) & M_TXPKT_CSUM_LOC) #define S_TXPKT_ETHHDR_LEN 34 #define M_TXPKT_ETHHDR_LEN 0x3F #define V_TXPKT_ETHHDR_LEN(x) ((__u64)(x) << S_TXPKT_ETHHDR_LEN) #define G_TXPKT_ETHHDR_LEN(x) (((x) >> S_TXPKT_ETHHDR_LEN) & M_TXPKT_ETHHDR_LEN) #define S_T6_TXPKT_ETHHDR_LEN 32 #define M_T6_TXPKT_ETHHDR_LEN 0xFF #define V_T6_TXPKT_ETHHDR_LEN(x) ((__u64)(x) << S_T6_TXPKT_ETHHDR_LEN) #define G_T6_TXPKT_ETHHDR_LEN(x) \ (((x) >> S_T6_TXPKT_ETHHDR_LEN) & M_T6_TXPKT_ETHHDR_LEN) #define S_TXPKT_CSUM_TYPE 40 #define M_TXPKT_CSUM_TYPE 0xF #define V_TXPKT_CSUM_TYPE(x) ((__u64)(x) << S_TXPKT_CSUM_TYPE) #define G_TXPKT_CSUM_TYPE(x) (((x) >> S_TXPKT_CSUM_TYPE) & M_TXPKT_CSUM_TYPE) #define S_TXPKT_VLAN 44 #define M_TXPKT_VLAN 0xFFFF #define V_TXPKT_VLAN(x) ((__u64)(x) << S_TXPKT_VLAN) #define G_TXPKT_VLAN(x) (((x) >> S_TXPKT_VLAN) & M_TXPKT_VLAN) #define S_TXPKT_VLAN_VLD 60 #define V_TXPKT_VLAN_VLD(x) ((__u64)(x) << S_TXPKT_VLAN_VLD) #define F_TXPKT_VLAN_VLD V_TXPKT_VLAN_VLD(1ULL) #define S_TXPKT_IPSEC 61 #define V_TXPKT_IPSEC(x) ((__u64)(x) << S_TXPKT_IPSEC) #define F_TXPKT_IPSEC V_TXPKT_IPSEC(1ULL) #define S_TXPKT_IPCSUM_DIS 62 #define V_TXPKT_IPCSUM_DIS(x) ((__u64)(x) << S_TXPKT_IPCSUM_DIS) #define F_TXPKT_IPCSUM_DIS V_TXPKT_IPCSUM_DIS(1ULL) #define S_TXPKT_L4CSUM_DIS 63 #define V_TXPKT_L4CSUM_DIS(x) ((__u64)(x) << S_TXPKT_L4CSUM_DIS) #define F_TXPKT_L4CSUM_DIS V_TXPKT_L4CSUM_DIS(1ULL) struct cpl_tx_pkt_lso_core { __be32 lso_ctrl; __be16 ipid_ofst; __be16 mss; __be32 seqno_offset; __be32 len; /* encapsulated CPL (TX_PKT, TX_PKT_XT or TX_DATA) follows here */ }; struct cpl_tx_pkt_lso { WR_HDR; struct cpl_tx_pkt_lso_core c; /* encapsulated CPL (TX_PKT, TX_PKT_XT or TX_DATA) follows here */ }; struct cpl_tx_pkt_ufo_core { __be16 ethlen; __be16 iplen; __be16 udplen; __be16 mss; __be32 len; __be32 r1; /* encapsulated CPL (TX_PKT, TX_PKT_XT or TX_DATA) follows here */ }; struct cpl_tx_pkt_ufo { WR_HDR; struct cpl_tx_pkt_ufo_core c; /* encapsulated CPL (TX_PKT, TX_PKT_XT or TX_DATA) follows here */ }; /* cpl_tx_pkt_lso_core.lso_ctrl fields */ #define S_LSO_TCPHDR_LEN 0 #define M_LSO_TCPHDR_LEN 0xF #define V_LSO_TCPHDR_LEN(x) ((x) << S_LSO_TCPHDR_LEN) #define G_LSO_TCPHDR_LEN(x) (((x) >> S_LSO_TCPHDR_LEN) & M_LSO_TCPHDR_LEN) #define S_LSO_IPHDR_LEN 4 #define M_LSO_IPHDR_LEN 0xFFF #define V_LSO_IPHDR_LEN(x) ((x) << S_LSO_IPHDR_LEN) #define G_LSO_IPHDR_LEN(x) (((x) >> S_LSO_IPHDR_LEN) & M_LSO_IPHDR_LEN) #define S_LSO_ETHHDR_LEN 16 #define M_LSO_ETHHDR_LEN 0xF #define V_LSO_ETHHDR_LEN(x) ((x) << S_LSO_ETHHDR_LEN) #define G_LSO_ETHHDR_LEN(x) (((x) >> S_LSO_ETHHDR_LEN) & M_LSO_ETHHDR_LEN) #define S_LSO_IPV6 20 #define V_LSO_IPV6(x) ((x) << S_LSO_IPV6) #define F_LSO_IPV6 V_LSO_IPV6(1U) #define S_LSO_OFLD_ENCAP 21 #define V_LSO_OFLD_ENCAP(x) ((x) << S_LSO_OFLD_ENCAP) #define F_LSO_OFLD_ENCAP V_LSO_OFLD_ENCAP(1U) #define S_LSO_LAST_SLICE 22 #define V_LSO_LAST_SLICE(x) ((x) << S_LSO_LAST_SLICE) #define F_LSO_LAST_SLICE V_LSO_LAST_SLICE(1U) #define S_LSO_FIRST_SLICE 23 #define V_LSO_FIRST_SLICE(x) ((x) << S_LSO_FIRST_SLICE) #define F_LSO_FIRST_SLICE V_LSO_FIRST_SLICE(1U) #define S_LSO_OPCODE 24 #define M_LSO_OPCODE 0xFF #define V_LSO_OPCODE(x) ((x) << S_LSO_OPCODE) #define G_LSO_OPCODE(x) (((x) >> S_LSO_OPCODE) & M_LSO_OPCODE) #define S_LSO_T5_XFER_SIZE 0 #define M_LSO_T5_XFER_SIZE 0xFFFFFFF #define V_LSO_T5_XFER_SIZE(x) ((x) << S_LSO_T5_XFER_SIZE) #define G_LSO_T5_XFER_SIZE(x) (((x) >> S_LSO_T5_XFER_SIZE) & M_LSO_T5_XFER_SIZE) /* cpl_tx_pkt_lso_core.mss fields */ #define S_LSO_MSS 0 #define M_LSO_MSS 0x3FFF #define V_LSO_MSS(x) ((x) << S_LSO_MSS) #define G_LSO_MSS(x) (((x) >> S_LSO_MSS) & M_LSO_MSS) #define S_LSO_IPID_SPLIT 15 #define V_LSO_IPID_SPLIT(x) ((x) << S_LSO_IPID_SPLIT) #define F_LSO_IPID_SPLIT V_LSO_IPID_SPLIT(1U) struct cpl_tx_pkt_fso { WR_HDR; __be32 fso_ctrl; __be16 seqcnt_ofst; __be16 mtu; __be32 param_offset; __be32 len; /* encapsulated CPL (TX_PKT or TX_PKT_XT) follows here */ }; /* cpl_tx_pkt_fso.fso_ctrl fields different from cpl_tx_pkt_lso.lso_ctrl */ #define S_FSO_XCHG_CLASS 21 #define V_FSO_XCHG_CLASS(x) ((x) << S_FSO_XCHG_CLASS) #define F_FSO_XCHG_CLASS V_FSO_XCHG_CLASS(1U) #define S_FSO_INITIATOR 20 #define V_FSO_INITIATOR(x) ((x) << S_FSO_INITIATOR) #define F_FSO_INITIATOR V_FSO_INITIATOR(1U) #define S_FSO_FCHDR_LEN 12 #define M_FSO_FCHDR_LEN 0xF #define V_FSO_FCHDR_LEN(x) ((x) << S_FSO_FCHDR_LEN) #define G_FSO_FCHDR_LEN(x) (((x) >> S_FSO_FCHDR_LEN) & M_FSO_FCHDR_LEN) struct cpl_iscsi_hdr_no_rss { union opcode_tid ot; __be16 pdu_len_ddp; __be16 len; __be32 seq; __be16 urg; __u8 rsvd; __u8 status; }; struct cpl_tx_data_iso { __be32 op_to_scsi; __u8 reserved1; __u8 ahs_len; __be16 mpdu; __be32 burst_size; __be32 len; __be32 reserved2_seglen_offset; __be32 datasn_offset; __be32 buffer_offset; __be32 reserved3; /* encapsulated CPL_TX_DATA follows here */ }; /* cpl_tx_data_iso.op_to_scsi fields */ #define S_CPL_TX_DATA_ISO_OP 24 #define M_CPL_TX_DATA_ISO_OP 0xff #define V_CPL_TX_DATA_ISO_OP(x) ((x) << S_CPL_TX_DATA_ISO_OP) #define G_CPL_TX_DATA_ISO_OP(x) \ (((x) >> S_CPL_TX_DATA_ISO_OP) & M_CPL_TX_DATA_ISO_OP) #define S_CPL_TX_DATA_ISO_FIRST 23 #define M_CPL_TX_DATA_ISO_FIRST 0x1 #define V_CPL_TX_DATA_ISO_FIRST(x) ((x) << S_CPL_TX_DATA_ISO_FIRST) #define G_CPL_TX_DATA_ISO_FIRST(x) \ (((x) >> S_CPL_TX_DATA_ISO_FIRST) & M_CPL_TX_DATA_ISO_FIRST) #define F_CPL_TX_DATA_ISO_FIRST V_CPL_TX_DATA_ISO_FIRST(1U) #define S_CPL_TX_DATA_ISO_LAST 22 #define M_CPL_TX_DATA_ISO_LAST 0x1 #define V_CPL_TX_DATA_ISO_LAST(x) ((x) << S_CPL_TX_DATA_ISO_LAST) #define G_CPL_TX_DATA_ISO_LAST(x) \ (((x) >> S_CPL_TX_DATA_ISO_LAST) & M_CPL_TX_DATA_ISO_LAST) #define F_CPL_TX_DATA_ISO_LAST V_CPL_TX_DATA_ISO_LAST(1U) #define S_CPL_TX_DATA_ISO_CPLHDRLEN 21 #define M_CPL_TX_DATA_ISO_CPLHDRLEN 0x1 #define V_CPL_TX_DATA_ISO_CPLHDRLEN(x) ((x) << S_CPL_TX_DATA_ISO_CPLHDRLEN) #define G_CPL_TX_DATA_ISO_CPLHDRLEN(x) \ (((x) >> S_CPL_TX_DATA_ISO_CPLHDRLEN) & M_CPL_TX_DATA_ISO_CPLHDRLEN) #define F_CPL_TX_DATA_ISO_CPLHDRLEN V_CPL_TX_DATA_ISO_CPLHDRLEN(1U) #define S_CPL_TX_DATA_ISO_HDRCRC 20 #define M_CPL_TX_DATA_ISO_HDRCRC 0x1 #define V_CPL_TX_DATA_ISO_HDRCRC(x) ((x) << S_CPL_TX_DATA_ISO_HDRCRC) #define G_CPL_TX_DATA_ISO_HDRCRC(x) \ (((x) >> S_CPL_TX_DATA_ISO_HDRCRC) & M_CPL_TX_DATA_ISO_HDRCRC) #define F_CPL_TX_DATA_ISO_HDRCRC V_CPL_TX_DATA_ISO_HDRCRC(1U) #define S_CPL_TX_DATA_ISO_PLDCRC 19 #define M_CPL_TX_DATA_ISO_PLDCRC 0x1 #define V_CPL_TX_DATA_ISO_PLDCRC(x) ((x) << S_CPL_TX_DATA_ISO_PLDCRC) #define G_CPL_TX_DATA_ISO_PLDCRC(x) \ (((x) >> S_CPL_TX_DATA_ISO_PLDCRC) & M_CPL_TX_DATA_ISO_PLDCRC) #define F_CPL_TX_DATA_ISO_PLDCRC V_CPL_TX_DATA_ISO_PLDCRC(1U) #define S_CPL_TX_DATA_ISO_IMMEDIATE 18 #define M_CPL_TX_DATA_ISO_IMMEDIATE 0x1 #define V_CPL_TX_DATA_ISO_IMMEDIATE(x) ((x) << S_CPL_TX_DATA_ISO_IMMEDIATE) #define G_CPL_TX_DATA_ISO_IMMEDIATE(x) \ (((x) >> S_CPL_TX_DATA_ISO_IMMEDIATE) & M_CPL_TX_DATA_ISO_IMMEDIATE) #define F_CPL_TX_DATA_ISO_IMMEDIATE V_CPL_TX_DATA_ISO_IMMEDIATE(1U) #define S_CPL_TX_DATA_ISO_SCSI 16 #define M_CPL_TX_DATA_ISO_SCSI 0x3 #define V_CPL_TX_DATA_ISO_SCSI(x) ((x) << S_CPL_TX_DATA_ISO_SCSI) #define G_CPL_TX_DATA_ISO_SCSI(x) \ (((x) >> S_CPL_TX_DATA_ISO_SCSI) & M_CPL_TX_DATA_ISO_SCSI) /* cpl_tx_data_iso.reserved2_seglen_offset fields */ #define S_CPL_TX_DATA_ISO_SEGLEN_OFFSET 0 #define M_CPL_TX_DATA_ISO_SEGLEN_OFFSET 0xffffff #define V_CPL_TX_DATA_ISO_SEGLEN_OFFSET(x) \ ((x) << S_CPL_TX_DATA_ISO_SEGLEN_OFFSET) #define G_CPL_TX_DATA_ISO_SEGLEN_OFFSET(x) \ (((x) >> S_CPL_TX_DATA_ISO_SEGLEN_OFFSET) & \ M_CPL_TX_DATA_ISO_SEGLEN_OFFSET) struct cpl_iscsi_hdr { RSS_HDR union opcode_tid ot; __be16 pdu_len_ddp; __be16 len; __be32 seq; __be16 urg; __u8 rsvd; __u8 status; }; /* cpl_iscsi_hdr.pdu_len_ddp fields */ #define S_ISCSI_PDU_LEN 0 #define M_ISCSI_PDU_LEN 0x7FFF #define V_ISCSI_PDU_LEN(x) ((x) << S_ISCSI_PDU_LEN) #define G_ISCSI_PDU_LEN(x) (((x) >> S_ISCSI_PDU_LEN) & M_ISCSI_PDU_LEN) #define S_ISCSI_DDP 15 #define V_ISCSI_DDP(x) ((x) << S_ISCSI_DDP) #define F_ISCSI_DDP V_ISCSI_DDP(1U) struct cpl_iscsi_data { RSS_HDR union opcode_tid ot; __u8 rsvd0[2]; __be16 len; __be32 seq; __be16 urg; __u8 rsvd1; __u8 status; }; struct cpl_rx_data { RSS_HDR union opcode_tid ot; __be16 rsvd; __be16 len; __be32 seq; __be16 urg; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 dack_mode:2; __u8 psh:1; __u8 heartbeat:1; __u8 ddp_off:1; __u8 :3; #else __u8 :3; __u8 ddp_off:1; __u8 heartbeat:1; __u8 psh:1; __u8 dack_mode:2; #endif __u8 status; }; struct cpl_fcoe_hdr { RSS_HDR union opcode_tid ot; __be16 oxid; __be16 len; __be32 rctl_fctl; __u8 cs_ctl; __u8 df_ctl; __u8 sof; __u8 eof; __be16 seq_cnt; __u8 seq_id; __u8 type; __be32 param; }; /* cpl_fcoe_hdr.rctl_fctl fields */ #define S_FCOE_FCHDR_RCTL 24 #define M_FCOE_FCHDR_RCTL 0xff #define V_FCOE_FCHDR_RCTL(x) ((x) << S_FCOE_FCHDR_RCTL) #define G_FCOE_FCHDR_RCTL(x) \ (((x) >> S_FCOE_FCHDR_RCTL) & M_FCOE_FCHDR_RCTL) #define S_FCOE_FCHDR_FCTL 0 #define M_FCOE_FCHDR_FCTL 0xffffff #define V_FCOE_FCHDR_FCTL(x) ((x) << S_FCOE_FCHDR_FCTL) #define G_FCOE_FCHDR_FCTL(x) \ (((x) >> S_FCOE_FCHDR_FCTL) & M_FCOE_FCHDR_FCTL) struct cpl_fcoe_data { RSS_HDR union opcode_tid ot; __u8 rsvd0[2]; __be16 len; __be32 seq; __u8 rsvd1[3]; __u8 status; }; struct cpl_rx_urg_notify { RSS_HDR union opcode_tid ot; __be32 seq; }; struct cpl_rx_urg_pkt { RSS_HDR union opcode_tid ot; __be16 rsvd; __be16 len; }; struct cpl_rx_data_ack { WR_HDR; union opcode_tid ot; __be32 credit_dack; }; struct cpl_rx_data_ack_core { union opcode_tid ot; __be32 credit_dack; }; /* cpl_rx_data_ack.ack_seq fields */ #define S_RX_CREDITS 0 #define M_RX_CREDITS 0x3FFFFFF #define V_RX_CREDITS(x) ((x) << S_RX_CREDITS) #define G_RX_CREDITS(x) (((x) >> S_RX_CREDITS) & M_RX_CREDITS) #define S_RX_MODULATE_TX 26 #define V_RX_MODULATE_TX(x) ((x) << S_RX_MODULATE_TX) #define F_RX_MODULATE_TX V_RX_MODULATE_TX(1U) #define S_RX_MODULATE_RX 27 #define V_RX_MODULATE_RX(x) ((x) << S_RX_MODULATE_RX) #define F_RX_MODULATE_RX V_RX_MODULATE_RX(1U) #define S_RX_FORCE_ACK 28 #define V_RX_FORCE_ACK(x) ((x) << S_RX_FORCE_ACK) #define F_RX_FORCE_ACK V_RX_FORCE_ACK(1U) #define S_RX_DACK_MODE 29 #define M_RX_DACK_MODE 0x3 #define V_RX_DACK_MODE(x) ((x) << S_RX_DACK_MODE) #define G_RX_DACK_MODE(x) (((x) >> S_RX_DACK_MODE) & M_RX_DACK_MODE) #define S_RX_DACK_CHANGE 31 #define V_RX_DACK_CHANGE(x) ((x) << S_RX_DACK_CHANGE) #define F_RX_DACK_CHANGE V_RX_DACK_CHANGE(1U) struct cpl_rx_ddp_complete { RSS_HDR union opcode_tid ot; __be32 ddp_report; __be32 rcv_nxt; __be32 rsvd; }; struct cpl_rx_data_ddp { RSS_HDR union opcode_tid ot; __be16 urg; __be16 len; __be32 seq; union { __be32 nxt_seq; __be32 ddp_report; } u; __be32 ulp_crc; __be32 ddpvld; }; #define cpl_rx_iscsi_ddp cpl_rx_data_ddp struct cpl_rx_fcoe_ddp { RSS_HDR union opcode_tid ot; __be16 rsvd; __be16 len; __be32 seq; __be32 ddp_report; __be32 ulp_crc; __be32 ddpvld; }; struct cpl_rx_data_dif { RSS_HDR union opcode_tid ot; __be16 ddp_len; __be16 msg_len; __be32 seq; union { __be32 nxt_seq; __be32 ddp_report; } u; __be32 err_vec; __be32 ddpvld; }; struct cpl_rx_iscsi_dif { RSS_HDR union opcode_tid ot; __be16 ddp_len; __be16 msg_len; __be32 seq; union { __be32 nxt_seq; __be32 ddp_report; } u; __be32 ulp_crc; __be32 ddpvld; __u8 rsvd0[8]; __be32 err_vec; __u8 rsvd1[4]; }; struct cpl_rx_iscsi_cmp { RSS_HDR union opcode_tid ot; __be16 pdu_len_ddp; __be16 len; __be32 seq; __be16 urg; __u8 rsvd; __u8 status; __be32 ulp_crc; __be32 ddpvld; }; struct cpl_rx_fcoe_dif { RSS_HDR union opcode_tid ot; __be16 ddp_len; __be16 msg_len; __be32 seq; __be32 ddp_report; __be32 err_vec; __be32 ddpvld; }; /* cpl_rx_{data,iscsi,fcoe}_{ddp,dif}.ddpvld fields */ #define S_DDP_VALID 15 #define M_DDP_VALID 0x1FFFF #define V_DDP_VALID(x) ((x) << S_DDP_VALID) #define G_DDP_VALID(x) (((x) >> S_DDP_VALID) & M_DDP_VALID) #define S_DDP_PPOD_MISMATCH 15 #define V_DDP_PPOD_MISMATCH(x) ((x) << S_DDP_PPOD_MISMATCH) #define F_DDP_PPOD_MISMATCH V_DDP_PPOD_MISMATCH(1U) #define S_DDP_PDU 16 #define V_DDP_PDU(x) ((x) << S_DDP_PDU) #define F_DDP_PDU V_DDP_PDU(1U) #define S_DDP_LLIMIT_ERR 17 #define V_DDP_LLIMIT_ERR(x) ((x) << S_DDP_LLIMIT_ERR) #define F_DDP_LLIMIT_ERR V_DDP_LLIMIT_ERR(1U) #define S_DDP_PPOD_PARITY_ERR 18 #define V_DDP_PPOD_PARITY_ERR(x) ((x) << S_DDP_PPOD_PARITY_ERR) #define F_DDP_PPOD_PARITY_ERR V_DDP_PPOD_PARITY_ERR(1U) #define S_DDP_PADDING_ERR 19 #define V_DDP_PADDING_ERR(x) ((x) << S_DDP_PADDING_ERR) #define F_DDP_PADDING_ERR V_DDP_PADDING_ERR(1U) #define S_DDP_HDRCRC_ERR 20 #define V_DDP_HDRCRC_ERR(x) ((x) << S_DDP_HDRCRC_ERR) #define F_DDP_HDRCRC_ERR V_DDP_HDRCRC_ERR(1U) #define S_DDP_DATACRC_ERR 21 #define V_DDP_DATACRC_ERR(x) ((x) << S_DDP_DATACRC_ERR) #define F_DDP_DATACRC_ERR V_DDP_DATACRC_ERR(1U) #define S_DDP_INVALID_TAG 22 #define V_DDP_INVALID_TAG(x) ((x) << S_DDP_INVALID_TAG) #define F_DDP_INVALID_TAG V_DDP_INVALID_TAG(1U) #define S_DDP_ULIMIT_ERR 23 #define V_DDP_ULIMIT_ERR(x) ((x) << S_DDP_ULIMIT_ERR) #define F_DDP_ULIMIT_ERR V_DDP_ULIMIT_ERR(1U) #define S_DDP_OFFSET_ERR 24 #define V_DDP_OFFSET_ERR(x) ((x) << S_DDP_OFFSET_ERR) #define F_DDP_OFFSET_ERR V_DDP_OFFSET_ERR(1U) #define S_DDP_COLOR_ERR 25 #define V_DDP_COLOR_ERR(x) ((x) << S_DDP_COLOR_ERR) #define F_DDP_COLOR_ERR V_DDP_COLOR_ERR(1U) #define S_DDP_TID_MISMATCH 26 #define V_DDP_TID_MISMATCH(x) ((x) << S_DDP_TID_MISMATCH) #define F_DDP_TID_MISMATCH V_DDP_TID_MISMATCH(1U) #define S_DDP_INVALID_PPOD 27 #define V_DDP_INVALID_PPOD(x) ((x) << S_DDP_INVALID_PPOD) #define F_DDP_INVALID_PPOD V_DDP_INVALID_PPOD(1U) #define S_DDP_ULP_MODE 28 #define M_DDP_ULP_MODE 0xF #define V_DDP_ULP_MODE(x) ((x) << S_DDP_ULP_MODE) #define G_DDP_ULP_MODE(x) (((x) >> S_DDP_ULP_MODE) & M_DDP_ULP_MODE) /* cpl_rx_{data,iscsi,fcoe}_{ddp,dif}.ddp_report fields */ #define S_DDP_OFFSET 0 #define M_DDP_OFFSET 0xFFFFFF #define V_DDP_OFFSET(x) ((x) << S_DDP_OFFSET) #define G_DDP_OFFSET(x) (((x) >> S_DDP_OFFSET) & M_DDP_OFFSET) #define S_DDP_DACK_MODE 24 #define M_DDP_DACK_MODE 0x3 #define V_DDP_DACK_MODE(x) ((x) << S_DDP_DACK_MODE) #define G_DDP_DACK_MODE(x) (((x) >> S_DDP_DACK_MODE) & M_DDP_DACK_MODE) #define S_DDP_BUF_IDX 26 #define V_DDP_BUF_IDX(x) ((x) << S_DDP_BUF_IDX) #define F_DDP_BUF_IDX V_DDP_BUF_IDX(1U) #define S_DDP_URG 27 #define V_DDP_URG(x) ((x) << S_DDP_URG) #define F_DDP_URG V_DDP_URG(1U) #define S_DDP_PSH 28 #define V_DDP_PSH(x) ((x) << S_DDP_PSH) #define F_DDP_PSH V_DDP_PSH(1U) #define S_DDP_BUF_COMPLETE 29 #define V_DDP_BUF_COMPLETE(x) ((x) << S_DDP_BUF_COMPLETE) #define F_DDP_BUF_COMPLETE V_DDP_BUF_COMPLETE(1U) #define S_DDP_BUF_TIMED_OUT 30 #define V_DDP_BUF_TIMED_OUT(x) ((x) << S_DDP_BUF_TIMED_OUT) #define F_DDP_BUF_TIMED_OUT V_DDP_BUF_TIMED_OUT(1U) #define S_DDP_INV 31 #define V_DDP_INV(x) ((x) << S_DDP_INV) #define F_DDP_INV V_DDP_INV(1U) struct cpl_rx_pkt { RSS_HDR __u8 opcode; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 iff:4; __u8 csum_calc:1; __u8 ipmi_pkt:1; __u8 vlan_ex:1; __u8 ip_frag:1; #else __u8 ip_frag:1; __u8 vlan_ex:1; __u8 ipmi_pkt:1; __u8 csum_calc:1; __u8 iff:4; #endif __be16 csum; __be16 vlan; __be16 len; __be32 l2info; __be16 hdr_len; __be16 err_vec; }; /* rx_pkt.l2info fields */ #define S_RX_ETHHDR_LEN 0 #define M_RX_ETHHDR_LEN 0x1F #define V_RX_ETHHDR_LEN(x) ((x) << S_RX_ETHHDR_LEN) #define G_RX_ETHHDR_LEN(x) (((x) >> S_RX_ETHHDR_LEN) & M_RX_ETHHDR_LEN) #define S_RX_T5_ETHHDR_LEN 0 #define M_RX_T5_ETHHDR_LEN 0x3F #define V_RX_T5_ETHHDR_LEN(x) ((x) << S_RX_T5_ETHHDR_LEN) #define G_RX_T5_ETHHDR_LEN(x) (((x) >> S_RX_T5_ETHHDR_LEN) & M_RX_T5_ETHHDR_LEN) #define M_RX_T6_ETHHDR_LEN 0xFF #define G_RX_T6_ETHHDR_LEN(x) (((x) >> S_RX_ETHHDR_LEN) & M_RX_T6_ETHHDR_LEN) #define S_RX_PKTYPE 5 #define M_RX_PKTYPE 0x7 #define V_RX_PKTYPE(x) ((x) << S_RX_PKTYPE) #define G_RX_PKTYPE(x) (((x) >> S_RX_PKTYPE) & M_RX_PKTYPE) #define S_RX_T5_DATYPE 6 #define M_RX_T5_DATYPE 0x3 #define V_RX_T5_DATYPE(x) ((x) << S_RX_T5_DATYPE) #define G_RX_T5_DATYPE(x) (((x) >> S_RX_T5_DATYPE) & M_RX_T5_DATYPE) #define S_RX_MACIDX 8 #define M_RX_MACIDX 0x1FF #define V_RX_MACIDX(x) ((x) << S_RX_MACIDX) #define G_RX_MACIDX(x) (((x) >> S_RX_MACIDX) & M_RX_MACIDX) #define S_RX_T5_PKTYPE 17 #define M_RX_T5_PKTYPE 0x7 #define V_RX_T5_PKTYPE(x) ((x) << S_RX_T5_PKTYPE) #define G_RX_T5_PKTYPE(x) (((x) >> S_RX_T5_PKTYPE) & M_RX_T5_PKTYPE) #define S_RX_DATYPE 18 #define M_RX_DATYPE 0x3 #define V_RX_DATYPE(x) ((x) << S_RX_DATYPE) #define G_RX_DATYPE(x) (((x) >> S_RX_DATYPE) & M_RX_DATYPE) #define S_RXF_PSH 20 #define V_RXF_PSH(x) ((x) << S_RXF_PSH) #define F_RXF_PSH V_RXF_PSH(1U) #define S_RXF_SYN 21 #define V_RXF_SYN(x) ((x) << S_RXF_SYN) #define F_RXF_SYN V_RXF_SYN(1U) #define S_RXF_UDP 22 #define V_RXF_UDP(x) ((x) << S_RXF_UDP) #define F_RXF_UDP V_RXF_UDP(1U) #define S_RXF_TCP 23 #define V_RXF_TCP(x) ((x) << S_RXF_TCP) #define F_RXF_TCP V_RXF_TCP(1U) #define S_RXF_IP 24 #define V_RXF_IP(x) ((x) << S_RXF_IP) #define F_RXF_IP V_RXF_IP(1U) #define S_RXF_IP6 25 #define V_RXF_IP6(x) ((x) << S_RXF_IP6) #define F_RXF_IP6 V_RXF_IP6(1U) #define S_RXF_SYN_COOKIE 26 #define V_RXF_SYN_COOKIE(x) ((x) << S_RXF_SYN_COOKIE) #define F_RXF_SYN_COOKIE V_RXF_SYN_COOKIE(1U) #define S_RXF_FCOE 26 #define V_RXF_FCOE(x) ((x) << S_RXF_FCOE) #define F_RXF_FCOE V_RXF_FCOE(1U) #define S_RXF_LRO 27 #define V_RXF_LRO(x) ((x) << S_RXF_LRO) #define F_RXF_LRO V_RXF_LRO(1U) #define S_RX_CHAN 28 #define M_RX_CHAN 0xF #define V_RX_CHAN(x) ((x) << S_RX_CHAN) #define G_RX_CHAN(x) (((x) >> S_RX_CHAN) & M_RX_CHAN) /* rx_pkt.hdr_len fields */ #define S_RX_TCPHDR_LEN 0 #define M_RX_TCPHDR_LEN 0x3F #define V_RX_TCPHDR_LEN(x) ((x) << S_RX_TCPHDR_LEN) #define G_RX_TCPHDR_LEN(x) (((x) >> S_RX_TCPHDR_LEN) & M_RX_TCPHDR_LEN) #define S_RX_IPHDR_LEN 6 #define M_RX_IPHDR_LEN 0x3FF #define V_RX_IPHDR_LEN(x) ((x) << S_RX_IPHDR_LEN) #define G_RX_IPHDR_LEN(x) (((x) >> S_RX_IPHDR_LEN) & M_RX_IPHDR_LEN) /* rx_pkt.err_vec fields */ #define S_RXERR_OR 0 #define V_RXERR_OR(x) ((x) << S_RXERR_OR) #define F_RXERR_OR V_RXERR_OR(1U) #define S_RXERR_MAC 1 #define V_RXERR_MAC(x) ((x) << S_RXERR_MAC) #define F_RXERR_MAC V_RXERR_MAC(1U) #define S_RXERR_IPVERS 2 #define V_RXERR_IPVERS(x) ((x) << S_RXERR_IPVERS) #define F_RXERR_IPVERS V_RXERR_IPVERS(1U) #define S_RXERR_FRAG 3 #define V_RXERR_FRAG(x) ((x) << S_RXERR_FRAG) #define F_RXERR_FRAG V_RXERR_FRAG(1U) #define S_RXERR_ATTACK 4 #define V_RXERR_ATTACK(x) ((x) << S_RXERR_ATTACK) #define F_RXERR_ATTACK V_RXERR_ATTACK(1U) #define S_RXERR_ETHHDR_LEN 5 #define V_RXERR_ETHHDR_LEN(x) ((x) << S_RXERR_ETHHDR_LEN) #define F_RXERR_ETHHDR_LEN V_RXERR_ETHHDR_LEN(1U) #define S_RXERR_IPHDR_LEN 6 #define V_RXERR_IPHDR_LEN(x) ((x) << S_RXERR_IPHDR_LEN) #define F_RXERR_IPHDR_LEN V_RXERR_IPHDR_LEN(1U) #define S_RXERR_TCPHDR_LEN 7 #define V_RXERR_TCPHDR_LEN(x) ((x) << S_RXERR_TCPHDR_LEN) #define F_RXERR_TCPHDR_LEN V_RXERR_TCPHDR_LEN(1U) #define S_RXERR_PKT_LEN 8 #define V_RXERR_PKT_LEN(x) ((x) << S_RXERR_PKT_LEN) #define F_RXERR_PKT_LEN V_RXERR_PKT_LEN(1U) #define S_RXERR_TCP_OPT 9 #define V_RXERR_TCP_OPT(x) ((x) << S_RXERR_TCP_OPT) #define F_RXERR_TCP_OPT V_RXERR_TCP_OPT(1U) #define S_RXERR_IPCSUM 12 #define V_RXERR_IPCSUM(x) ((x) << S_RXERR_IPCSUM) #define F_RXERR_IPCSUM V_RXERR_IPCSUM(1U) #define S_RXERR_CSUM 13 #define V_RXERR_CSUM(x) ((x) << S_RXERR_CSUM) #define F_RXERR_CSUM V_RXERR_CSUM(1U) #define S_RXERR_PING 14 #define V_RXERR_PING(x) ((x) << S_RXERR_PING) #define F_RXERR_PING V_RXERR_PING(1U) /* In T6, rx_pkt.err_vec indicates * RxError Error vector (16b) or * Encapsulating header length (8b), * Outer encapsulation type (2b) and * compressed error vector (6b) if CRxPktEnc is * enabled in TP_OUT_CONFIG */ #define S_T6_COMPR_RXERR_VEC 0 #define M_T6_COMPR_RXERR_VEC 0x3F #define V_T6_COMPR_RXERR_VEC(x) ((x) << S_T6_COMPR_RXERR_VEC) #define G_T6_COMPR_RXERR_VEC(x) \ (((x) >> S_T6_COMPR_RXERR_VEC) & M_T6_COMPR_RXERR_VEC) #define S_T6_COMPR_RXERR_MAC 0 #define V_T6_COMPR_RXERR_MAC(x) ((x) << S_T6_COMPR_RXERR_MAC) #define F_T6_COMPR_RXERR_MAC V_T6_COMPR_RXERR_MAC(1U) /* Logical OR of RX_ERROR_PKT_LEN, RX_ERROR_TCP_HDR_LEN * RX_ERROR_IP_HDR_LEN, RX_ERROR_ETH_HDR_LEN */ #define S_T6_COMPR_RXERR_LEN 1 #define V_T6_COMPR_RXERR_LEN(x) ((x) << S_T6_COMPR_RXERR_LEN) #define F_T6_COMPR_RXERR_LEN V_COMPR_T6_RXERR_LEN(1U) #define S_T6_COMPR_RXERR_TCP_OPT 2 #define V_T6_COMPR_RXERR_TCP_OPT(x) ((x) << S_T6_COMPR_RXERR_TCP_OPT) #define F_T6_COMPR_RXERR_TCP_OPT V_T6_COMPR_RXERR_TCP_OPT(1U) #define S_T6_COMPR_RXERR_IPV6_EXT 3 #define V_T6_COMPR_RXERR_IPV6_EXT(x) ((x) << S_T6_COMPR_RXERR_IPV6_EXT) #define F_T6_COMPR_RXERR_IPV6_EXT V_T6_COMPR_RXERR_IPV6_EXT(1U) /* Logical OR of RX_ERROR_CSUM, RX_ERROR_CSIP */ #define S_T6_COMPR_RXERR_SUM 4 #define V_T6_COMPR_RXERR_SUM(x) ((x) << S_T6_COMPR_RXERR_SUM) #define F_T6_COMPR_RXERR_SUM V_T6_COMPR_RXERR_SUM(1U) /* Logical OR of RX_ERROR_FPMA, RX_ERROR_PING_DROP, * RX_ERROR_ATTACK, RX_ERROR_FRAG,RX_ERROR_IPVERSION */ #define S_T6_COMPR_RXERR_MISC 5 #define V_T6_COMPR_RXERR_MISC(x) ((x) << S_T6_COMPR_RXERR_MISC) #define F_T6_COMPR_RXERR_MISC V_T6_COMPR_RXERR_MISC(1U) #define S_T6_RX_TNL_TYPE 6 #define M_T6_RX_TNL_TYPE 0x3 #define V_T6_RX_TNL_TYPE(x) ((x) << S_T6_RX_TNL_TYPE) #define G_T6_RX_TNL_TYPE(x) (((x) >> S_T6_RX_TNL_TYPE) & M_T6_RX_TNL_TYPE) #define RX_PKT_TNL_TYPE_NVGRE 1 #define RX_PKT_TNL_TYPE_VXLAN 2 #define RX_PKT_TNL_TYPE_GENEVE 3 #define S_T6_RX_TNLHDR_LEN 8 #define M_T6_RX_TNLHDR_LEN 0xFF #define V_T6_RX_TNLHDR_LEN(x) ((x) << S_T6_RX_TNLHDR_LEN) #define G_T6_RX_TNLHDR_LEN(x) (((x) >> S_T6_RX_TNLHDR_LEN) & M_T6_RX_TNLHDR_LEN) struct cpl_trace_pkt { RSS_HDR __u8 opcode; __u8 intf; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 runt:4; __u8 filter_hit:4; __u8 :6; __u8 err:1; __u8 trunc:1; #else __u8 filter_hit:4; __u8 runt:4; __u8 trunc:1; __u8 err:1; __u8 :6; #endif __be16 rsvd; __be16 len; __be64 tstamp; }; struct cpl_t5_trace_pkt { RSS_HDR __u8 opcode; __u8 intf; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 runt:4; __u8 filter_hit:4; __u8 :6; __u8 err:1; __u8 trunc:1; #else __u8 filter_hit:4; __u8 runt:4; __u8 trunc:1; __u8 err:1; __u8 :6; #endif __be16 rsvd; __be16 len; __be64 tstamp; __be64 rsvd1; }; struct cpl_rte_delete_req { WR_HDR; union opcode_tid ot; __be32 params; }; /* {cpl_rte_delete_req, cpl_rte_read_req}.params fields */ #define S_RTE_REQ_LUT_IX 8 #define M_RTE_REQ_LUT_IX 0x7FF #define V_RTE_REQ_LUT_IX(x) ((x) << S_RTE_REQ_LUT_IX) #define G_RTE_REQ_LUT_IX(x) (((x) >> S_RTE_REQ_LUT_IX) & M_RTE_REQ_LUT_IX) #define S_RTE_REQ_LUT_BASE 19 #define M_RTE_REQ_LUT_BASE 0x7FF #define V_RTE_REQ_LUT_BASE(x) ((x) << S_RTE_REQ_LUT_BASE) #define G_RTE_REQ_LUT_BASE(x) (((x) >> S_RTE_REQ_LUT_BASE) & M_RTE_REQ_LUT_BASE) #define S_RTE_READ_REQ_SELECT 31 #define V_RTE_READ_REQ_SELECT(x) ((x) << S_RTE_READ_REQ_SELECT) #define F_RTE_READ_REQ_SELECT V_RTE_READ_REQ_SELECT(1U) struct cpl_rte_delete_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[3]; }; struct cpl_rte_write_req { WR_HDR; union opcode_tid ot; __u32 write_sel; __be32 lut_params; __be32 l2t_idx; __be32 netmask; __be32 faddr; }; /* cpl_rte_write_req.write_sel fields */ #define S_RTE_WR_L2TIDX 31 #define V_RTE_WR_L2TIDX(x) ((x) << S_RTE_WR_L2TIDX) #define F_RTE_WR_L2TIDX V_RTE_WR_L2TIDX(1U) #define S_RTE_WR_FADDR 30 #define V_RTE_WR_FADDR(x) ((x) << S_RTE_WR_FADDR) #define F_RTE_WR_FADDR V_RTE_WR_FADDR(1U) /* cpl_rte_write_req.lut_params fields */ #define S_RTE_WR_LUT_IX 10 #define M_RTE_WR_LUT_IX 0x7FF #define V_RTE_WR_LUT_IX(x) ((x) << S_RTE_WR_LUT_IX) #define G_RTE_WR_LUT_IX(x) (((x) >> S_RTE_WR_LUT_IX) & M_RTE_WR_LUT_IX) #define S_RTE_WR_LUT_BASE 21 #define M_RTE_WR_LUT_BASE 0x7FF #define V_RTE_WR_LUT_BASE(x) ((x) << S_RTE_WR_LUT_BASE) #define G_RTE_WR_LUT_BASE(x) (((x) >> S_RTE_WR_LUT_BASE) & M_RTE_WR_LUT_BASE) struct cpl_rte_write_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[3]; }; struct cpl_rte_read_req { WR_HDR; union opcode_tid ot; __be32 params; }; struct cpl_rte_read_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd; __be16 l2t_idx; #if defined(__LITTLE_ENDIAN_BITFIELD) __u32 :30; __u32 select:1; #else __u32 select:1; __u32 :30; #endif __be32 addr; }; struct cpl_l2t_write_req { WR_HDR; union opcode_tid ot; __be16 params; __be16 l2t_idx; __be16 vlan; __u8 dst_mac[6]; }; /* cpl_l2t_write_req.params fields */ #define S_L2T_W_INFO 2 #define M_L2T_W_INFO 0x3F #define V_L2T_W_INFO(x) ((x) << S_L2T_W_INFO) #define G_L2T_W_INFO(x) (((x) >> S_L2T_W_INFO) & M_L2T_W_INFO) #define S_L2T_W_PORT 8 #define M_L2T_W_PORT 0x3 #define V_L2T_W_PORT(x) ((x) << S_L2T_W_PORT) #define G_L2T_W_PORT(x) (((x) >> S_L2T_W_PORT) & M_L2T_W_PORT) #define S_L2T_W_LPBK 10 #define V_L2T_W_LPBK(x) ((x) << S_L2T_W_LPBK) #define F_L2T_W_PKBK V_L2T_W_LPBK(1U) #define S_L2T_W_ARPMISS 11 #define V_L2T_W_ARPMISS(x) ((x) << S_L2T_W_ARPMISS) #define F_L2T_W_ARPMISS V_L2T_W_ARPMISS(1U) #define S_L2T_W_NOREPLY 15 #define V_L2T_W_NOREPLY(x) ((x) << S_L2T_W_NOREPLY) #define F_L2T_W_NOREPLY V_L2T_W_NOREPLY(1U) #define CPL_L2T_VLAN_NONE 0xfff struct cpl_l2t_write_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[3]; }; struct cpl_l2t_read_req { WR_HDR; union opcode_tid ot; __be32 l2t_idx; }; struct cpl_l2t_read_rpl { RSS_HDR union opcode_tid ot; __u8 status; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 :4; __u8 iff:4; #else __u8 iff:4; __u8 :4; #endif __be16 vlan; __be16 info; __u8 dst_mac[6]; }; struct cpl_srq_table_req { WR_HDR; union opcode_tid ot; __u8 status; __u8 rsvd[2]; __u8 idx; __be64 rsvd_pdid; __be32 qlen_qbase; __be16 cur_msn; __be16 max_msn; }; struct cpl_srq_table_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[2]; __u8 idx; __be64 rsvd_pdid; __be32 qlen_qbase; __be16 cur_msn; __be16 max_msn; }; /* cpl_srq_table_{req,rpl}.params fields */ #define S_SRQT_QLEN 28 #define M_SRQT_QLEN 0xF #define V_SRQT_QLEN(x) ((x) << S_SRQT_QLEN) #define G_SRQT_QLEN(x) (((x) >> S_SRQT_QLEN) & M_SRQT_QLEN) #define S_SRQT_QBASE 0 #define M_SRQT_QBASE 0x3FFFFFF #define V_SRQT_QBASE(x) ((x) << S_SRQT_QBASE) #define G_SRQT_QBASE(x) (((x) >> S_SRQT_QBASE) & M_SRQT_QBASE) #define S_SRQT_PDID 0 #define M_SRQT_PDID 0xFF #define V_SRQT_PDID(x) ((x) << S_SRQT_PDID) #define G_SRQT_PDID(x) (((x) >> S_SRQT_PDID) & M_SRQT_PDID) #define S_SRQT_IDX 0 #define M_SRQT_IDX 0xF #define V_SRQT_IDX(x) ((x) << S_SRQT_IDX) #define G_SRQT_IDX(x) (((x) >> S_SRQT_IDX) & M_SRQT_IDX) struct cpl_smt_write_req { WR_HDR; union opcode_tid ot; __be32 params; __be16 pfvf1; __u8 src_mac1[6]; __be16 pfvf0; __u8 src_mac0[6]; }; struct cpl_t6_smt_write_req { WR_HDR; union opcode_tid ot; __be32 params; __be64 tag; __be16 pfvf0; __u8 src_mac0[6]; __be32 local_ip; __be32 rsvd; }; struct cpl_smt_write_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[3]; }; struct cpl_smt_read_req { WR_HDR; union opcode_tid ot; __be32 params; }; struct cpl_smt_read_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 ovlan_idx; __be16 rsvd; __be16 pfvf1; __u8 src_mac1[6]; __be16 pfvf0; __u8 src_mac0[6]; }; /* cpl_smt_{read,write}_req.params fields */ #define S_SMTW_OVLAN_IDX 16 #define M_SMTW_OVLAN_IDX 0xF #define V_SMTW_OVLAN_IDX(x) ((x) << S_SMTW_OVLAN_IDX) #define G_SMTW_OVLAN_IDX(x) (((x) >> S_SMTW_OVLAN_IDX) & M_SMTW_OVLAN_IDX) #define S_SMTW_IDX 20 #define M_SMTW_IDX 0x7F #define V_SMTW_IDX(x) ((x) << S_SMTW_IDX) #define G_SMTW_IDX(x) (((x) >> S_SMTW_IDX) & M_SMTW_IDX) #define M_T6_SMTW_IDX 0xFF #define G_T6_SMTW_IDX(x) (((x) >> S_SMTW_IDX) & M_T6_SMTW_IDX) #define S_SMTW_NORPL 31 #define V_SMTW_NORPL(x) ((x) << S_SMTW_NORPL) #define F_SMTW_NORPL V_SMTW_NORPL(1U) /* cpl_smt_{read,write}_req.pfvf? fields */ #define S_SMTW_VF 0 #define M_SMTW_VF 0xFF #define V_SMTW_VF(x) ((x) << S_SMTW_VF) #define G_SMTW_VF(x) (((x) >> S_SMTW_VF) & M_SMTW_VF) #define S_SMTW_PF 8 #define M_SMTW_PF 0x7 #define V_SMTW_PF(x) ((x) << S_SMTW_PF) #define G_SMTW_PF(x) (((x) >> S_SMTW_PF) & M_SMTW_PF) #define S_SMTW_VF_VLD 11 #define V_SMTW_VF_VLD(x) ((x) << S_SMTW_VF_VLD) #define F_SMTW_VF_VLD V_SMTW_VF_VLD(1U) struct cpl_tag_write_req { WR_HDR; union opcode_tid ot; __be32 params; __be64 tag_val; }; struct cpl_tag_write_rpl { RSS_HDR union opcode_tid ot; __u8 status; __u8 rsvd[2]; __u8 idx; }; struct cpl_tag_read_req { WR_HDR; union opcode_tid ot; __be32 params; }; struct cpl_tag_read_rpl { RSS_HDR union opcode_tid ot; __u8 status; #if defined(__LITTLE_ENDIAN_BITFIELD) __u8 :4; __u8 tag_len:1; __u8 :2; __u8 ins_enable:1; #else __u8 ins_enable:1; __u8 :2; __u8 tag_len:1; __u8 :4; #endif __u8 rsvd; __u8 tag_idx; __be64 tag_val; }; /* cpl_tag{read,write}_req.params fields */ #define S_TAGW_IDX 0 #define M_TAGW_IDX 0x7F #define V_TAGW_IDX(x) ((x) << S_TAGW_IDX) #define G_TAGW_IDX(x) (((x) >> S_TAGW_IDX) & M_TAGW_IDX) #define S_TAGW_LEN 20 #define V_TAGW_LEN(x) ((x) << S_TAGW_LEN) #define F_TAGW_LEN V_TAGW_LEN(1U) #define S_TAGW_INS_ENABLE 23 #define V_TAGW_INS_ENABLE(x) ((x) << S_TAGW_INS_ENABLE) #define F_TAGW_INS_ENABLE V_TAGW_INS_ENABLE(1U) #define S_TAGW_NORPL 31 #define V_TAGW_NORPL(x) ((x) << S_TAGW_NORPL) #define F_TAGW_NORPL V_TAGW_NORPL(1U) struct cpl_barrier { WR_HDR; __u8 opcode; __u8 chan_map; __be16 rsvd0; __be32 rsvd1; }; /* cpl_barrier.chan_map fields */ #define S_CHAN_MAP 4 #define M_CHAN_MAP 0xF #define V_CHAN_MAP(x) ((x) << S_CHAN_MAP) #define G_CHAN_MAP(x) (((x) >> S_CHAN_MAP) & M_CHAN_MAP) struct cpl_error { RSS_HDR union opcode_tid ot; __be32 error; }; struct cpl_hit_notify { RSS_HDR union opcode_tid ot; __be32 rsvd; __be32 info; __be32 reason; }; struct cpl_pkt_notify { RSS_HDR union opcode_tid ot; __be16 rsvd; __be16 len; __be32 info; __be32 reason; }; /* cpl_{hit,pkt}_notify.info fields */ #define S_NTFY_MAC_IDX 0 #define M_NTFY_MAC_IDX 0x1FF #define V_NTFY_MAC_IDX(x) ((x) << S_NTFY_MAC_IDX) #define G_NTFY_MAC_IDX(x) (((x) >> S_NTFY_MAC_IDX) & M_NTFY_MAC_IDX) #define S_NTFY_INTF 10 #define M_NTFY_INTF 0xF #define V_NTFY_INTF(x) ((x) << S_NTFY_INTF) #define G_NTFY_INTF(x) (((x) >> S_NTFY_INTF) & M_NTFY_INTF) #define S_NTFY_TCPHDR_LEN 14 #define M_NTFY_TCPHDR_LEN 0xF #define V_NTFY_TCPHDR_LEN(x) ((x) << S_NTFY_TCPHDR_LEN) #define G_NTFY_TCPHDR_LEN(x) (((x) >> S_NTFY_TCPHDR_LEN) & M_NTFY_TCPHDR_LEN) #define S_NTFY_IPHDR_LEN 18 #define M_NTFY_IPHDR_LEN 0x1FF #define V_NTFY_IPHDR_LEN(x) ((x) << S_NTFY_IPHDR_LEN) #define G_NTFY_IPHDR_LEN(x) (((x) >> S_NTFY_IPHDR_LEN) & M_NTFY_IPHDR_LEN) #define S_NTFY_ETHHDR_LEN 27 #define M_NTFY_ETHHDR_LEN 0x1F #define V_NTFY_ETHHDR_LEN(x) ((x) << S_NTFY_ETHHDR_LEN) #define G_NTFY_ETHHDR_LEN(x) (((x) >> S_NTFY_ETHHDR_LEN) & M_NTFY_ETHHDR_LEN) #define S_NTFY_T5_IPHDR_LEN 18 #define M_NTFY_T5_IPHDR_LEN 0xFF #define V_NTFY_T5_IPHDR_LEN(x) ((x) << S_NTFY_T5_IPHDR_LEN) #define G_NTFY_T5_IPHDR_LEN(x) (((x) >> S_NTFY_T5_IPHDR_LEN) & M_NTFY_T5_IPHDR_LEN) #define S_NTFY_T5_ETHHDR_LEN 26 #define M_NTFY_T5_ETHHDR_LEN 0x3F #define V_NTFY_T5_ETHHDR_LEN(x) ((x) << S_NTFY_T5_ETHHDR_LEN) #define G_NTFY_T5_ETHHDR_LEN(x) (((x) >> S_NTFY_T5_ETHHDR_LEN) & M_NTFY_T5_ETHHDR_LEN) struct cpl_rdma_terminate { RSS_HDR union opcode_tid ot; __be16 rsvd; __be16 len; }; struct cpl_set_le_req { WR_HDR; union opcode_tid ot; __be16 reply_ctrl; __be16 params; __be64 mask_hi; __be64 mask_lo; __be64 val_hi; __be64 val_lo; }; /* cpl_set_le_req.reply_ctrl additional fields */ #define S_LE_REQ_IP6 13 #define V_LE_REQ_IP6(x) ((x) << S_LE_REQ_IP6) #define F_LE_REQ_IP6 V_LE_REQ_IP6(1U) /* cpl_set_le_req.params fields */ #define S_LE_CHAN 0 #define M_LE_CHAN 0x3 #define V_LE_CHAN(x) ((x) << S_LE_CHAN) #define G_LE_CHAN(x) (((x) >> S_LE_CHAN) & M_LE_CHAN) #define S_LE_OFFSET 5 #define M_LE_OFFSET 0x7 #define V_LE_OFFSET(x) ((x) << S_LE_OFFSET) #define G_LE_OFFSET(x) (((x) >> S_LE_OFFSET) & M_LE_OFFSET) #define S_LE_MORE 8 #define V_LE_MORE(x) ((x) << S_LE_MORE) #define F_LE_MORE V_LE_MORE(1U) #define S_LE_REQSIZE 9 #define M_LE_REQSIZE 0x7 #define V_LE_REQSIZE(x) ((x) << S_LE_REQSIZE) #define G_LE_REQSIZE(x) (((x) >> S_LE_REQSIZE) & M_LE_REQSIZE) #define S_LE_REQCMD 12 #define M_LE_REQCMD 0xF #define V_LE_REQCMD(x) ((x) << S_LE_REQCMD) #define G_LE_REQCMD(x) (((x) >> S_LE_REQCMD) & M_LE_REQCMD) struct cpl_set_le_rpl { RSS_HDR union opcode_tid ot; __u8 chan; __u8 info; __be16 len; }; /* cpl_set_le_rpl.info fields */ #define S_LE_RSPCMD 0 #define M_LE_RSPCMD 0xF #define V_LE_RSPCMD(x) ((x) << S_LE_RSPCMD) #define G_LE_RSPCMD(x) (((x) >> S_LE_RSPCMD) & M_LE_RSPCMD) #define S_LE_RSPSIZE 4 #define M_LE_RSPSIZE 0x7 #define V_LE_RSPSIZE(x) ((x) << S_LE_RSPSIZE) #define G_LE_RSPSIZE(x) (((x) >> S_LE_RSPSIZE) & M_LE_RSPSIZE) #define S_LE_RSPTYPE 7 #define V_LE_RSPTYPE(x) ((x) << S_LE_RSPTYPE) #define F_LE_RSPTYPE V_LE_RSPTYPE(1U) struct cpl_sge_egr_update { RSS_HDR __be32 opcode_qid; __be16 cidx; __be16 pidx; }; /* cpl_sge_egr_update.ot fields */ #define S_AUTOEQU 22 #define M_AUTOEQU 0x1 #define V_AUTOEQU(x) ((x) << S_AUTOEQU) #define G_AUTOEQU(x) (((x) >> S_AUTOEQU) & M_AUTOEQU) #define S_EGR_QID 0 #define M_EGR_QID 0x1FFFF #define V_EGR_QID(x) ((x) << S_EGR_QID) #define G_EGR_QID(x) (((x) >> S_EGR_QID) & M_EGR_QID) /* cpl_fw*.type values */ enum { FW_TYPE_CMD_RPL = 0, FW_TYPE_WR_RPL = 1, FW_TYPE_CQE = 2, FW_TYPE_OFLD_CONNECTION_WR_RPL = 3, FW_TYPE_RSSCPL = 4, FW_TYPE_WRERR_RPL = 5, FW_TYPE_PI_ERR = 6, FW_TYPE_TLS_KEY = 7, }; struct cpl_fw2_pld { RSS_HDR u8 opcode; u8 rsvd[5]; __be16 len; }; struct cpl_fw4_pld { RSS_HDR u8 opcode; u8 rsvd0[3]; u8 type; u8 rsvd1; __be16 len; __be64 data; __be64 rsvd2; }; struct cpl_fw6_pld { RSS_HDR u8 opcode; u8 rsvd[5]; __be16 len; __be64 data[4]; }; struct cpl_fw2_msg { RSS_HDR union opcode_info oi; }; struct cpl_fw4_msg { RSS_HDR u8 opcode; u8 type; __be16 rsvd0; __be32 rsvd1; __be64 data[2]; }; struct cpl_fw4_ack { RSS_HDR union opcode_tid ot; u8 credits; u8 rsvd0[2]; u8 flags; __be32 snd_nxt; __be32 snd_una; __be64 rsvd1; }; enum { CPL_FW4_ACK_FLAGS_SEQVAL = 0x1, /* seqn valid */ CPL_FW4_ACK_FLAGS_CH = 0x2, /* channel change complete */ CPL_FW4_ACK_FLAGS_FLOWC = 0x4, /* fw_flowc_wr complete */ }; +#define S_CPL_FW4_ACK_OPCODE 24 +#define M_CPL_FW4_ACK_OPCODE 0xff +#define V_CPL_FW4_ACK_OPCODE(x) ((x) << S_CPL_FW4_ACK_OPCODE) +#define G_CPL_FW4_ACK_OPCODE(x) \ + (((x) >> S_CPL_FW4_ACK_OPCODE) & M_CPL_FW4_ACK_OPCODE) + +#define S_CPL_FW4_ACK_FLOWID 0 +#define M_CPL_FW4_ACK_FLOWID 0xffffff +#define V_CPL_FW4_ACK_FLOWID(x) ((x) << S_CPL_FW4_ACK_FLOWID) +#define G_CPL_FW4_ACK_FLOWID(x) \ + (((x) >> S_CPL_FW4_ACK_FLOWID) & M_CPL_FW4_ACK_FLOWID) + +#define S_CPL_FW4_ACK_CR 24 +#define M_CPL_FW4_ACK_CR 0xff +#define V_CPL_FW4_ACK_CR(x) ((x) << S_CPL_FW4_ACK_CR) +#define G_CPL_FW4_ACK_CR(x) (((x) >> S_CPL_FW4_ACK_CR) & M_CPL_FW4_ACK_CR) + +#define S_CPL_FW4_ACK_SEQVAL 0 +#define M_CPL_FW4_ACK_SEQVAL 0x1 +#define V_CPL_FW4_ACK_SEQVAL(x) ((x) << S_CPL_FW4_ACK_SEQVAL) +#define G_CPL_FW4_ACK_SEQVAL(x) \ + (((x) >> S_CPL_FW4_ACK_SEQVAL) & M_CPL_FW4_ACK_SEQVAL) +#define F_CPL_FW4_ACK_SEQVAL V_CPL_FW4_ACK_SEQVAL(1U) + struct cpl_fw6_msg { RSS_HDR u8 opcode; u8 type; __be16 rsvd0; __be32 rsvd1; __be64 data[4]; }; /* cpl_fw6_msg.type values */ enum { FW6_TYPE_CMD_RPL = FW_TYPE_CMD_RPL, FW6_TYPE_WR_RPL = FW_TYPE_WR_RPL, FW6_TYPE_CQE = FW_TYPE_CQE, FW6_TYPE_OFLD_CONNECTION_WR_RPL = FW_TYPE_OFLD_CONNECTION_WR_RPL, FW6_TYPE_RSSCPL = FW_TYPE_RSSCPL, FW6_TYPE_WRERR_RPL = FW_TYPE_WRERR_RPL, FW6_TYPE_PI_ERR = FW_TYPE_PI_ERR, NUM_FW6_TYPES }; struct cpl_fw6_msg_ofld_connection_wr_rpl { __u64 cookie; __be32 tid; /* or atid in case of active failure */ __u8 t_state; __u8 retval; __u8 rsvd[2]; }; /* ULP_TX opcodes */ enum { ULP_TX_MEM_READ = 2, ULP_TX_MEM_WRITE = 3, ULP_TX_PKT = 4 }; enum { ULP_TX_SC_NOOP = 0x80, ULP_TX_SC_IMM = 0x81, ULP_TX_SC_DSGL = 0x82, ULP_TX_SC_ISGL = 0x83, ULP_TX_SC_PICTRL = 0x84, ULP_TX_SC_MEMRD = 0x86 }; #define S_ULPTX_CMD 24 #define M_ULPTX_CMD 0xFF #define V_ULPTX_CMD(x) ((x) << S_ULPTX_CMD) #define S_ULPTX_LEN16 0 #define M_ULPTX_LEN16 0xFF #define V_ULPTX_LEN16(x) ((x) << S_ULPTX_LEN16) #define S_ULP_TX_SC_MORE 23 #define V_ULP_TX_SC_MORE(x) ((x) << S_ULP_TX_SC_MORE) #define F_ULP_TX_SC_MORE V_ULP_TX_SC_MORE(1U) struct ulptx_sge_pair { __be32 len[2]; __be64 addr[2]; }; struct ulptx_sgl { __be32 cmd_nsge; __be32 len0; __be64 addr0; #if !(defined C99_NOT_SUPPORTED) struct ulptx_sge_pair sge[0]; #endif }; struct ulptx_isge { __be32 stag; __be32 len; __be64 target_ofst; }; struct ulptx_isgl { __be32 cmd_nisge; __be32 rsvd; #if !(defined C99_NOT_SUPPORTED) struct ulptx_isge sge[0]; #endif }; struct ulptx_idata { __be32 cmd_more; __be32 len; }; #define S_ULPTX_NSGE 0 #define M_ULPTX_NSGE 0xFFFF #define V_ULPTX_NSGE(x) ((x) << S_ULPTX_NSGE) #define G_ULPTX_NSGE(x) (((x) >> S_ULPTX_NSGE) & M_ULPTX_NSGE) struct ulptx_sc_memrd { __be32 cmd_to_len; __be32 addr; }; struct ulp_mem_io { WR_HDR; __be32 cmd; __be32 len16; /* command length */ __be32 dlen; /* data length in 32-byte units */ __be32 lock_addr; }; /* additional ulp_mem_io.cmd fields */ #define S_ULP_MEMIO_ORDER 23 #define V_ULP_MEMIO_ORDER(x) ((x) << S_ULP_MEMIO_ORDER) #define F_ULP_MEMIO_ORDER V_ULP_MEMIO_ORDER(1U) #define S_T5_ULP_MEMIO_IMM 23 #define V_T5_ULP_MEMIO_IMM(x) ((x) << S_T5_ULP_MEMIO_IMM) #define F_T5_ULP_MEMIO_IMM V_T5_ULP_MEMIO_IMM(1U) #define S_T5_ULP_MEMIO_ORDER 22 #define V_T5_ULP_MEMIO_ORDER(x) ((x) << S_T5_ULP_MEMIO_ORDER) #define F_T5_ULP_MEMIO_ORDER V_T5_ULP_MEMIO_ORDER(1U) #define S_T5_ULP_MEMIO_FID 4 #define M_T5_ULP_MEMIO_FID 0x7ff #define V_T5_ULP_MEMIO_FID(x) ((x) << S_T5_ULP_MEMIO_FID) /* ulp_mem_io.lock_addr fields */ #define S_ULP_MEMIO_ADDR 0 #define M_ULP_MEMIO_ADDR 0x7FFFFFF #define V_ULP_MEMIO_ADDR(x) ((x) << S_ULP_MEMIO_ADDR) #define S_ULP_MEMIO_LOCK 31 #define V_ULP_MEMIO_LOCK(x) ((x) << S_ULP_MEMIO_LOCK) #define F_ULP_MEMIO_LOCK V_ULP_MEMIO_LOCK(1U) /* ulp_mem_io.dlen fields */ #define S_ULP_MEMIO_DATA_LEN 0 #define M_ULP_MEMIO_DATA_LEN 0x1F #define V_ULP_MEMIO_DATA_LEN(x) ((x) << S_ULP_MEMIO_DATA_LEN) /* ULP_TXPKT field values */ enum { ULP_TXPKT_DEST_TP = 0, ULP_TXPKT_DEST_SGE, ULP_TXPKT_DEST_UP, ULP_TXPKT_DEST_DEVNULL, }; struct ulp_txpkt { __be32 cmd_dest; __be32 len; }; /* ulp_txpkt.cmd_dest fields */ #define S_ULP_TXPKT_DATAMODIFY 23 #define M_ULP_TXPKT_DATAMODIFY 0x1 #define V_ULP_TXPKT_DATAMODIFY(x) ((x) << S_ULP_TXPKT_DATAMODIFY) #define G_ULP_TXPKT_DATAMODIFY(x) \ (((x) >> S_ULP_TXPKT_DATAMODIFY) & M_ULP_TXPKT_DATAMODIFY_) #define F_ULP_TXPKT_DATAMODIFY V_ULP_TXPKT_DATAMODIFY(1U) #define S_ULP_TXPKT_CHANNELID 22 #define M_ULP_TXPKT_CHANNELID 0x1 #define V_ULP_TXPKT_CHANNELID(x) ((x) << S_ULP_TXPKT_CHANNELID) #define G_ULP_TXPKT_CHANNELID(x) \ (((x) >> S_ULP_TXPKT_CHANNELID) & M_ULP_TXPKT_CHANNELID) #define F_ULP_TXPKT_CHANNELID V_ULP_TXPKT_CHANNELID(1U) /* ulp_txpkt.cmd_dest fields */ #define S_ULP_TXPKT_DEST 16 #define M_ULP_TXPKT_DEST 0x3 #define V_ULP_TXPKT_DEST(x) ((x) << S_ULP_TXPKT_DEST) #define S_ULP_TXPKT_FID 4 #define M_ULP_TXPKT_FID 0x7ff #define V_ULP_TXPKT_FID(x) ((x) << S_ULP_TXPKT_FID) #define S_ULP_TXPKT_RO 3 #define V_ULP_TXPKT_RO(x) ((x) << S_ULP_TXPKT_RO) #define F_ULP_TXPKT_RO V_ULP_TXPKT_RO(1U) enum cpl_tx_tnl_lso_type { TX_TNL_TYPE_OPAQUE, TX_TNL_TYPE_NVGRE, TX_TNL_TYPE_VXLAN, TX_TNL_TYPE_GENEVE, }; struct cpl_tx_tnl_lso { __be32 op_to_IpIdSplitOut; __be16 IpIdOffsetOut; __be16 UdpLenSetOut_to_TnlHdrLen; __be64 r1; __be32 Flow_to_TcpHdrLen; __be16 IpIdOffset; __be16 IpIdSplit_to_Mss; __be32 TCPSeqOffset; __be32 EthLenOffset_Size; /* encapsulated CPL (TX_PKT_XT) follows here */ }; #define S_CPL_TX_TNL_LSO_OPCODE 24 #define M_CPL_TX_TNL_LSO_OPCODE 0xff #define V_CPL_TX_TNL_LSO_OPCODE(x) ((x) << S_CPL_TX_TNL_LSO_OPCODE) #define G_CPL_TX_TNL_LSO_OPCODE(x) \ (((x) >> S_CPL_TX_TNL_LSO_OPCODE) & M_CPL_TX_TNL_LSO_OPCODE) #define S_CPL_TX_TNL_LSO_FIRST 23 #define M_CPL_TX_TNL_LSO_FIRST 0x1 #define V_CPL_TX_TNL_LSO_FIRST(x) ((x) << S_CPL_TX_TNL_LSO_FIRST) #define G_CPL_TX_TNL_LSO_FIRST(x) \ (((x) >> S_CPL_TX_TNL_LSO_FIRST) & M_CPL_TX_TNL_LSO_FIRST) #define F_CPL_TX_TNL_LSO_FIRST V_CPL_TX_TNL_LSO_FIRST(1U) #define S_CPL_TX_TNL_LSO_LAST 22 #define M_CPL_TX_TNL_LSO_LAST 0x1 #define V_CPL_TX_TNL_LSO_LAST(x) ((x) << S_CPL_TX_TNL_LSO_LAST) #define G_CPL_TX_TNL_LSO_LAST(x) \ (((x) >> S_CPL_TX_TNL_LSO_LAST) & M_CPL_TX_TNL_LSO_LAST) #define F_CPL_TX_TNL_LSO_LAST V_CPL_TX_TNL_LSO_LAST(1U) #define S_CPL_TX_TNL_LSO_ETHHDRLENXOUT 21 #define M_CPL_TX_TNL_LSO_ETHHDRLENXOUT 0x1 #define V_CPL_TX_TNL_LSO_ETHHDRLENXOUT(x) \ ((x) << S_CPL_TX_TNL_LSO_ETHHDRLENXOUT) #define G_CPL_TX_TNL_LSO_ETHHDRLENXOUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_ETHHDRLENXOUT) & M_CPL_TX_TNL_LSO_ETHHDRLENXOUT) #define F_CPL_TX_TNL_LSO_ETHHDRLENXOUT V_CPL_TX_TNL_LSO_ETHHDRLENXOUT(1U) #define S_CPL_TX_TNL_LSO_IPV6OUT 20 #define M_CPL_TX_TNL_LSO_IPV6OUT 0x1 #define V_CPL_TX_TNL_LSO_IPV6OUT(x) ((x) << S_CPL_TX_TNL_LSO_IPV6OUT) #define G_CPL_TX_TNL_LSO_IPV6OUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPV6OUT) & M_CPL_TX_TNL_LSO_IPV6OUT) #define F_CPL_TX_TNL_LSO_IPV6OUT V_CPL_TX_TNL_LSO_IPV6OUT(1U) #define S_CPL_TX_TNL_LSO_ETHHDRLENOUT 16 #define M_CPL_TX_TNL_LSO_ETHHDRLENOUT 0xf #define V_CPL_TX_TNL_LSO_ETHHDRLENOUT(x) \ ((x) << S_CPL_TX_TNL_LSO_ETHHDRLENOUT) #define G_CPL_TX_TNL_LSO_ETHHDRLENOUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_ETHHDRLENOUT) & M_CPL_TX_TNL_LSO_ETHHDRLENOUT) #define S_CPL_TX_TNL_LSO_IPHDRLENOUT 4 #define M_CPL_TX_TNL_LSO_IPHDRLENOUT 0xfff #define V_CPL_TX_TNL_LSO_IPHDRLENOUT(x) ((x) << S_CPL_TX_TNL_LSO_IPHDRLENOUT) #define G_CPL_TX_TNL_LSO_IPHDRLENOUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPHDRLENOUT) & M_CPL_TX_TNL_LSO_IPHDRLENOUT) #define S_CPL_TX_TNL_LSO_IPHDRCHKOUT 3 #define M_CPL_TX_TNL_LSO_IPHDRCHKOUT 0x1 #define V_CPL_TX_TNL_LSO_IPHDRCHKOUT(x) ((x) << S_CPL_TX_TNL_LSO_IPHDRCHKOUT) #define G_CPL_TX_TNL_LSO_IPHDRCHKOUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPHDRCHKOUT) & M_CPL_TX_TNL_LSO_IPHDRCHKOUT) #define F_CPL_TX_TNL_LSO_IPHDRCHKOUT V_CPL_TX_TNL_LSO_IPHDRCHKOUT(1U) #define S_CPL_TX_TNL_LSO_IPLENSETOUT 2 #define M_CPL_TX_TNL_LSO_IPLENSETOUT 0x1 #define V_CPL_TX_TNL_LSO_IPLENSETOUT(x) ((x) << S_CPL_TX_TNL_LSO_IPLENSETOUT) #define G_CPL_TX_TNL_LSO_IPLENSETOUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPLENSETOUT) & M_CPL_TX_TNL_LSO_IPLENSETOUT) #define F_CPL_TX_TNL_LSO_IPLENSETOUT V_CPL_TX_TNL_LSO_IPLENSETOUT(1U) #define S_CPL_TX_TNL_LSO_IPIDINCOUT 1 #define M_CPL_TX_TNL_LSO_IPIDINCOUT 0x1 #define V_CPL_TX_TNL_LSO_IPIDINCOUT(x) ((x) << S_CPL_TX_TNL_LSO_IPIDINCOUT) #define G_CPL_TX_TNL_LSO_IPIDINCOUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPIDINCOUT) & M_CPL_TX_TNL_LSO_IPIDINCOUT) #define F_CPL_TX_TNL_LSO_IPIDINCOUT V_CPL_TX_TNL_LSO_IPIDINCOUT(1U) #define S_CPL_TX_TNL_LSO_IPIDSPLITOUT 0 #define M_CPL_TX_TNL_LSO_IPIDSPLITOUT 0x1 #define V_CPL_TX_TNL_LSO_IPIDSPLITOUT(x) \ ((x) << S_CPL_TX_TNL_LSO_IPIDSPLITOUT) #define G_CPL_TX_TNL_LSO_IPIDSPLITOUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPIDSPLITOUT) & M_CPL_TX_TNL_LSO_IPIDSPLITOUT) #define F_CPL_TX_TNL_LSO_IPIDSPLITOUT V_CPL_TX_TNL_LSO_IPIDSPLITOUT(1U) #define S_CPL_TX_TNL_LSO_UDPLENSETOUT 15 #define M_CPL_TX_TNL_LSO_UDPLENSETOUT 0x1 #define V_CPL_TX_TNL_LSO_UDPLENSETOUT(x) \ ((x) << S_CPL_TX_TNL_LSO_UDPLENSETOUT) #define G_CPL_TX_TNL_LSO_UDPLENSETOUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_UDPLENSETOUT) & M_CPL_TX_TNL_LSO_UDPLENSETOUT) #define F_CPL_TX_TNL_LSO_UDPLENSETOUT V_CPL_TX_TNL_LSO_UDPLENSETOUT(1U) #define S_CPL_TX_TNL_LSO_UDPCHKCLROUT 14 #define M_CPL_TX_TNL_LSO_UDPCHKCLROUT 0x1 #define V_CPL_TX_TNL_LSO_UDPCHKCLROUT(x) \ ((x) << S_CPL_TX_TNL_LSO_UDPCHKCLROUT) #define G_CPL_TX_TNL_LSO_UDPCHKCLROUT(x) \ (((x) >> S_CPL_TX_TNL_LSO_UDPCHKCLROUT) & M_CPL_TX_TNL_LSO_UDPCHKCLROUT) #define F_CPL_TX_TNL_LSO_UDPCHKCLROUT V_CPL_TX_TNL_LSO_UDPCHKCLROUT(1U) #define S_CPL_TX_TNL_LSO_TNLTYPE 12 #define M_CPL_TX_TNL_LSO_TNLTYPE 0x3 #define V_CPL_TX_TNL_LSO_TNLTYPE(x) ((x) << S_CPL_TX_TNL_LSO_TNLTYPE) #define G_CPL_TX_TNL_LSO_TNLTYPE(x) \ (((x) >> S_CPL_TX_TNL_LSO_TNLTYPE) & M_CPL_TX_TNL_LSO_TNLTYPE) #define S_CPL_TX_TNL_LSO_TNLHDRLEN 0 #define M_CPL_TX_TNL_LSO_TNLHDRLEN 0xfff #define V_CPL_TX_TNL_LSO_TNLHDRLEN(x) ((x) << S_CPL_TX_TNL_LSO_TNLHDRLEN) #define G_CPL_TX_TNL_LSO_TNLHDRLEN(x) \ (((x) >> S_CPL_TX_TNL_LSO_TNLHDRLEN) & M_CPL_TX_TNL_LSO_TNLHDRLEN) #define S_CPL_TX_TNL_LSO_FLOW 21 #define M_CPL_TX_TNL_LSO_FLOW 0x1 #define V_CPL_TX_TNL_LSO_FLOW(x) ((x) << S_CPL_TX_TNL_LSO_FLOW) #define G_CPL_TX_TNL_LSO_FLOW(x) \ (((x) >> S_CPL_TX_TNL_LSO_FLOW) & M_CPL_TX_TNL_LSO_FLOW) #define F_CPL_TX_TNL_LSO_FLOW V_CPL_TX_TNL_LSO_FLOW(1U) #define S_CPL_TX_TNL_LSO_IPV6 20 #define M_CPL_TX_TNL_LSO_IPV6 0x1 #define V_CPL_TX_TNL_LSO_IPV6(x) ((x) << S_CPL_TX_TNL_LSO_IPV6) #define G_CPL_TX_TNL_LSO_IPV6(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPV6) & M_CPL_TX_TNL_LSO_IPV6) #define F_CPL_TX_TNL_LSO_IPV6 V_CPL_TX_TNL_LSO_IPV6(1U) #define S_CPL_TX_TNL_LSO_ETHHDRLEN 16 #define M_CPL_TX_TNL_LSO_ETHHDRLEN 0xf #define V_CPL_TX_TNL_LSO_ETHHDRLEN(x) ((x) << S_CPL_TX_TNL_LSO_ETHHDRLEN) #define G_CPL_TX_TNL_LSO_ETHHDRLEN(x) \ (((x) >> S_CPL_TX_TNL_LSO_ETHHDRLEN) & M_CPL_TX_TNL_LSO_ETHHDRLEN) #define S_CPL_TX_TNL_LSO_IPHDRLEN 4 #define M_CPL_TX_TNL_LSO_IPHDRLEN 0xfff #define V_CPL_TX_TNL_LSO_IPHDRLEN(x) ((x) << S_CPL_TX_TNL_LSO_IPHDRLEN) #define G_CPL_TX_TNL_LSO_IPHDRLEN(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPHDRLEN) & M_CPL_TX_TNL_LSO_IPHDRLEN) #define S_CPL_TX_TNL_LSO_TCPHDRLEN 0 #define M_CPL_TX_TNL_LSO_TCPHDRLEN 0xf #define V_CPL_TX_TNL_LSO_TCPHDRLEN(x) ((x) << S_CPL_TX_TNL_LSO_TCPHDRLEN) #define G_CPL_TX_TNL_LSO_TCPHDRLEN(x) \ (((x) >> S_CPL_TX_TNL_LSO_TCPHDRLEN) & M_CPL_TX_TNL_LSO_TCPHDRLEN) #define S_CPL_TX_TNL_LSO_IPIDSPLIT 15 #define M_CPL_TX_TNL_LSO_IPIDSPLIT 0x1 #define V_CPL_TX_TNL_LSO_IPIDSPLIT(x) ((x) << S_CPL_TX_TNL_LSO_IPIDSPLIT) #define G_CPL_TX_TNL_LSO_IPIDSPLIT(x) \ (((x) >> S_CPL_TX_TNL_LSO_IPIDSPLIT) & M_CPL_TX_TNL_LSO_IPIDSPLIT) #define F_CPL_TX_TNL_LSO_IPIDSPLIT V_CPL_TX_TNL_LSO_IPIDSPLIT(1U) #define S_CPL_TX_TNL_LSO_ETHHDRLENX 14 #define M_CPL_TX_TNL_LSO_ETHHDRLENX 0x1 #define V_CPL_TX_TNL_LSO_ETHHDRLENX(x) ((x) << S_CPL_TX_TNL_LSO_ETHHDRLENX) #define G_CPL_TX_TNL_LSO_ETHHDRLENX(x) \ (((x) >> S_CPL_TX_TNL_LSO_ETHHDRLENX) & M_CPL_TX_TNL_LSO_ETHHDRLENX) #define F_CPL_TX_TNL_LSO_ETHHDRLENX V_CPL_TX_TNL_LSO_ETHHDRLENX(1U) #define S_CPL_TX_TNL_LSO_MSS 0 #define M_CPL_TX_TNL_LSO_MSS 0x3fff #define V_CPL_TX_TNL_LSO_MSS(x) ((x) << S_CPL_TX_TNL_LSO_MSS) #define G_CPL_TX_TNL_LSO_MSS(x) \ (((x) >> S_CPL_TX_TNL_LSO_MSS) & M_CPL_TX_TNL_LSO_MSS) #define S_CPL_TX_TNL_LSO_ETHLENOFFSET 28 #define M_CPL_TX_TNL_LSO_ETHLENOFFSET 0xf #define V_CPL_TX_TNL_LSO_ETHLENOFFSET(x) \ ((x) << S_CPL_TX_TNL_LSO_ETHLENOFFSET) #define G_CPL_TX_TNL_LSO_ETHLENOFFSET(x) \ (((x) >> S_CPL_TX_TNL_LSO_ETHLENOFFSET) & M_CPL_TX_TNL_LSO_ETHLENOFFSET) #define S_CPL_TX_TNL_LSO_SIZE 0 #define M_CPL_TX_TNL_LSO_SIZE 0xfffffff #define V_CPL_TX_TNL_LSO_SIZE(x) ((x) << S_CPL_TX_TNL_LSO_SIZE) #define G_CPL_TX_TNL_LSO_SIZE(x) \ (((x) >> S_CPL_TX_TNL_LSO_SIZE) & M_CPL_TX_TNL_LSO_SIZE) struct cpl_rx_mps_pkt { __be32 op_to_r1_hi; __be32 r1_lo_length; }; #define S_CPL_RX_MPS_PKT_OP 24 #define M_CPL_RX_MPS_PKT_OP 0xff #define V_CPL_RX_MPS_PKT_OP(x) ((x) << S_CPL_RX_MPS_PKT_OP) #define G_CPL_RX_MPS_PKT_OP(x) \ (((x) >> S_CPL_RX_MPS_PKT_OP) & M_CPL_RX_MPS_PKT_OP) #define S_CPL_RX_MPS_PKT_TYPE 20 #define M_CPL_RX_MPS_PKT_TYPE 0xf #define V_CPL_RX_MPS_PKT_TYPE(x) ((x) << S_CPL_RX_MPS_PKT_TYPE) #define G_CPL_RX_MPS_PKT_TYPE(x) \ (((x) >> S_CPL_RX_MPS_PKT_TYPE) & M_CPL_RX_MPS_PKT_TYPE) /* * Values for CPL_RX_MPS_PKT_TYPE, a bit-wise orthogonal field. */ #define X_CPL_RX_MPS_PKT_TYPE_PAUSE (1 << 0) #define X_CPL_RX_MPS_PKT_TYPE_PPP (1 << 1) #define X_CPL_RX_MPS_PKT_TYPE_QFC (1 << 2) #define X_CPL_RX_MPS_PKT_TYPE_PTP (1 << 3) struct cpl_tx_tls_sfo { __be32 op_to_seg_len; __be32 pld_len; __be32 type_protover; __be32 r1_lo; __be32 seqno_numivs; __be32 ivgen_hdrlen; __be64 scmd1; }; /* cpl_tx_tls_sfo macros */ #define S_CPL_TX_TLS_SFO_OPCODE 24 #define M_CPL_TX_TLS_SFO_OPCODE 0xff #define V_CPL_TX_TLS_SFO_OPCODE(x) ((x) << S_CPL_TX_TLS_SFO_OPCODE) #define G_CPL_TX_TLS_SFO_OPCODE(x) \ (((x) >> S_CPL_TX_TLS_SFO_OPCODE) & M_CPL_TX_TLS_SFO_OPCODE) #define S_CPL_TX_TLS_SFO_DATA_TYPE 20 #define M_CPL_TX_TLS_SFO_DATA_TYPE 0xf #define V_CPL_TX_TLS_SFO_DATA_TYPE(x) ((x) << S_CPL_TX_TLS_SFO_DATA_TYPE) #define G_CPL_TX_TLS_SFO_DATA_TYPE(x) \ (((x) >> S_CPL_TX_TLS_SFO_DATA_TYPE) & M_CPL_TX_TLS_SFO_DATA_TYPE) #define S_CPL_TX_TLS_SFO_CPL_LEN 16 #define M_CPL_TX_TLS_SFO_CPL_LEN 0xf #define V_CPL_TX_TLS_SFO_CPL_LEN(x) ((x) << S_CPL_TX_TLS_SFO_CPL_LEN) #define G_CPL_TX_TLS_SFO_CPL_LEN(x) \ (((x) >> S_CPL_TX_TLS_SFO_CPL_LEN) & M_CPL_TX_TLS_SFO_CPL_LEN) #define S_CPL_TX_TLS_SFO_SEG_LEN 0 #define M_CPL_TX_TLS_SFO_SEG_LEN 0xffff #define V_CPL_TX_TLS_SFO_SEG_LEN(x) ((x) << S_CPL_TX_TLS_SFO_SEG_LEN) #define G_CPL_TX_TLS_SFO_SEG_LEN(x) \ (((x) >> S_CPL_TX_TLS_SFO_SEG_LEN) & M_CPL_TX_TLS_SFO_SEG_LEN) #define S_CPL_TX_TLS_SFO_TYPE 24 #define M_CPL_TX_TLS_SFO_TYPE 0xff #define V_CPL_TX_TLS_SFO_TYPE(x) ((x) << S_CPL_TX_TLS_SFO_TYPE) #define G_CPL_TX_TLS_SFO_TYPE(x) \ (((x) >> S_CPL_TX_TLS_SFO_TYPE) & M_CPL_TX_TLS_SFO_TYPE) #define S_CPL_TX_TLS_SFO_PROTOVER 8 #define M_CPL_TX_TLS_SFO_PROTOVER 0xffff #define V_CPL_TX_TLS_SFO_PROTOVER(x) ((x) << S_CPL_TX_TLS_SFO_PROTOVER) #define G_CPL_TX_TLS_SFO_PROTOVER(x) \ (((x) >> S_CPL_TX_TLS_SFO_PROTOVER) & M_CPL_TX_TLS_SFO_PROTOVER) struct cpl_tls_data { RSS_HDR union opcode_tid ot; __be32 length_pkd; __be32 seq; __be32 r1; }; #define S_CPL_TLS_DATA_OPCODE 24 #define M_CPL_TLS_DATA_OPCODE 0xff #define V_CPL_TLS_DATA_OPCODE(x) ((x) << S_CPL_TLS_DATA_OPCODE) #define G_CPL_TLS_DATA_OPCODE(x) \ (((x) >> S_CPL_TLS_DATA_OPCODE) & M_CPL_TLS_DATA_OPCODE) #define S_CPL_TLS_DATA_TID 0 #define M_CPL_TLS_DATA_TID 0xffffff #define V_CPL_TLS_DATA_TID(x) ((x) << S_CPL_TLS_DATA_TID) #define G_CPL_TLS_DATA_TID(x) \ (((x) >> S_CPL_TLS_DATA_TID) & M_CPL_TLS_DATA_TID) #define S_CPL_TLS_DATA_LENGTH 0 #define M_CPL_TLS_DATA_LENGTH 0xffff #define V_CPL_TLS_DATA_LENGTH(x) ((x) << S_CPL_TLS_DATA_LENGTH) #define G_CPL_TLS_DATA_LENGTH(x) \ (((x) >> S_CPL_TLS_DATA_LENGTH) & M_CPL_TLS_DATA_LENGTH) struct cpl_rx_tls_cmp { RSS_HDR union opcode_tid ot; __be32 pdulength_length; __be32 seq; __be32 ddp_report; __be32 r; __be32 ddp_valid; }; #define S_CPL_RX_TLS_CMP_OPCODE 24 #define M_CPL_RX_TLS_CMP_OPCODE 0xff #define V_CPL_RX_TLS_CMP_OPCODE(x) ((x) << S_CPL_RX_TLS_CMP_OPCODE) #define G_CPL_RX_TLS_CMP_OPCODE(x) \ (((x) >> S_CPL_RX_TLS_CMP_OPCODE) & M_CPL_RX_TLS_CMP_OPCODE) #define S_CPL_RX_TLS_CMP_TID 0 #define M_CPL_RX_TLS_CMP_TID 0xffffff #define V_CPL_RX_TLS_CMP_TID(x) ((x) << S_CPL_RX_TLS_CMP_TID) #define G_CPL_RX_TLS_CMP_TID(x) \ (((x) >> S_CPL_RX_TLS_CMP_TID) & M_CPL_RX_TLS_CMP_TID) #define S_CPL_RX_TLS_CMP_PDULENGTH 16 #define M_CPL_RX_TLS_CMP_PDULENGTH 0xffff #define V_CPL_RX_TLS_CMP_PDULENGTH(x) ((x) << S_CPL_RX_TLS_CMP_PDULENGTH) #define G_CPL_RX_TLS_CMP_PDULENGTH(x) \ (((x) >> S_CPL_RX_TLS_CMP_PDULENGTH) & M_CPL_RX_TLS_CMP_PDULENGTH) #define S_CPL_RX_TLS_CMP_LENGTH 0 #define M_CPL_RX_TLS_CMP_LENGTH 0xffff #define V_CPL_RX_TLS_CMP_LENGTH(x) ((x) << S_CPL_RX_TLS_CMP_LENGTH) #define G_CPL_RX_TLS_CMP_LENGTH(x) \ (((x) >> S_CPL_RX_TLS_CMP_LENGTH) & M_CPL_RX_TLS_CMP_LENGTH) #define S_SCMD_SEQ_NO_CTRL 29 #define M_SCMD_SEQ_NO_CTRL 0x3 #define V_SCMD_SEQ_NO_CTRL(x) ((x) << S_SCMD_SEQ_NO_CTRL) #define G_SCMD_SEQ_NO_CTRL(x) \ (((x) >> S_SCMD_SEQ_NO_CTRL) & M_SCMD_SEQ_NO_CTRL) /* StsFieldPrsnt- Status field at the end of the TLS PDU */ #define S_SCMD_STATUS_PRESENT 28 #define M_SCMD_STATUS_PRESENT 0x1 #define V_SCMD_STATUS_PRESENT(x) ((x) << S_SCMD_STATUS_PRESENT) #define G_SCMD_STATUS_PRESENT(x) \ (((x) >> S_SCMD_STATUS_PRESENT) & M_SCMD_STATUS_PRESENT) #define F_SCMD_STATUS_PRESENT V_SCMD_STATUS_PRESENT(1U) /* ProtoVersion - Protocol Version 0: 1.2, 1:1.1, 2:DTLS, 3:Generic, * 3-15: Reserved. */ #define S_SCMD_PROTO_VERSION 24 #define M_SCMD_PROTO_VERSION 0xf #define V_SCMD_PROTO_VERSION(x) ((x) << S_SCMD_PROTO_VERSION) #define G_SCMD_PROTO_VERSION(x) \ (((x) >> S_SCMD_PROTO_VERSION) & M_SCMD_PROTO_VERSION) /* EncDecCtrl - Encryption/Decryption Control. 0: Encrypt, 1: Decrypt */ #define S_SCMD_ENC_DEC_CTRL 23 #define M_SCMD_ENC_DEC_CTRL 0x1 #define V_SCMD_ENC_DEC_CTRL(x) ((x) << S_SCMD_ENC_DEC_CTRL) #define G_SCMD_ENC_DEC_CTRL(x) \ (((x) >> S_SCMD_ENC_DEC_CTRL) & M_SCMD_ENC_DEC_CTRL) #define F_SCMD_ENC_DEC_CTRL V_SCMD_ENC_DEC_CTRL(1U) /* CipherAuthSeqCtrl - Cipher Authentication Sequence Control. */ #define S_SCMD_CIPH_AUTH_SEQ_CTRL 22 #define M_SCMD_CIPH_AUTH_SEQ_CTRL 0x1 #define V_SCMD_CIPH_AUTH_SEQ_CTRL(x) \ ((x) << S_SCMD_CIPH_AUTH_SEQ_CTRL) #define G_SCMD_CIPH_AUTH_SEQ_CTRL(x) \ (((x) >> S_SCMD_CIPH_AUTH_SEQ_CTRL) & M_SCMD_CIPH_AUTH_SEQ_CTRL) #define F_SCMD_CIPH_AUTH_SEQ_CTRL V_SCMD_CIPH_AUTH_SEQ_CTRL(1U) /* CiphMode - Cipher Mode. 0: NOP, 1:AES-CBC, 2:AES-GCM, 3:AES-CTR, * 4:Generic-AES, 5-15: Reserved. */ #define S_SCMD_CIPH_MODE 18 #define M_SCMD_CIPH_MODE 0xf #define V_SCMD_CIPH_MODE(x) ((x) << S_SCMD_CIPH_MODE) #define G_SCMD_CIPH_MODE(x) \ (((x) >> S_SCMD_CIPH_MODE) & M_SCMD_CIPH_MODE) /* AuthMode - Auth Mode. 0: NOP, 1:SHA1, 2:SHA2-224, 3:SHA2-256 * 4-15: Reserved */ #define S_SCMD_AUTH_MODE 14 #define M_SCMD_AUTH_MODE 0xf #define V_SCMD_AUTH_MODE(x) ((x) << S_SCMD_AUTH_MODE) #define G_SCMD_AUTH_MODE(x) \ (((x) >> S_SCMD_AUTH_MODE) & M_SCMD_AUTH_MODE) /* HmacCtrl - HMAC Control. 0:NOP, 1:No truncation, 2:Support HMAC Truncation * per RFC 4366, 3:IPSec 96 bits, 4-7:Reserved */ #define S_SCMD_HMAC_CTRL 11 #define M_SCMD_HMAC_CTRL 0x7 #define V_SCMD_HMAC_CTRL(x) ((x) << S_SCMD_HMAC_CTRL) #define G_SCMD_HMAC_CTRL(x) \ (((x) >> S_SCMD_HMAC_CTRL) & M_SCMD_HMAC_CTRL) /* IvSize - IV size in units of 2 bytes */ #define S_SCMD_IV_SIZE 7 #define M_SCMD_IV_SIZE 0xf #define V_SCMD_IV_SIZE(x) ((x) << S_SCMD_IV_SIZE) #define G_SCMD_IV_SIZE(x) \ (((x) >> S_SCMD_IV_SIZE) & M_SCMD_IV_SIZE) /* NumIVs - Number of IVs */ #define S_SCMD_NUM_IVS 0 #define M_SCMD_NUM_IVS 0x7f #define V_SCMD_NUM_IVS(x) ((x) << S_SCMD_NUM_IVS) #define G_SCMD_NUM_IVS(x) \ (((x) >> S_SCMD_NUM_IVS) & M_SCMD_NUM_IVS) /* EnbDbgId - If this is enabled upper 20 (63:44) bits if SeqNumber * (below) are used as Cid (connection id for debug status), these * bits are padded to zero for forming the 64 bit * sequence number for TLS */ #define S_SCMD_ENB_DBGID 31 #define M_SCMD_ENB_DBGID 0x1 #define V_SCMD_ENB_DBGID(x) ((x) << S_SCMD_ENB_DBGID) #define G_SCMD_ENB_DBGID(x) \ (((x) >> S_SCMD_ENB_DBGID) & M_SCMD_ENB_DBGID) /* IV generation in SW. */ #define S_SCMD_IV_GEN_CTRL 30 #define M_SCMD_IV_GEN_CTRL 0x1 #define V_SCMD_IV_GEN_CTRL(x) ((x) << S_SCMD_IV_GEN_CTRL) #define G_SCMD_IV_GEN_CTRL(x) \ (((x) >> S_SCMD_IV_GEN_CTRL) & M_SCMD_IV_GEN_CTRL) #define F_SCMD_IV_GEN_CTRL V_SCMD_IV_GEN_CTRL(1U) /* More frags */ #define S_SCMD_MORE_FRAGS 20 #define M_SCMD_MORE_FRAGS 0x1 #define V_SCMD_MORE_FRAGS(x) ((x) << S_SCMD_MORE_FRAGS) #define G_SCMD_MORE_FRAGS(x) (((x) >> S_SCMD_MORE_FRAGS) & M_SCMD_MORE_FRAGS) /*last frag */ #define S_SCMD_LAST_FRAG 19 #define M_SCMD_LAST_FRAG 0x1 #define V_SCMD_LAST_FRAG(x) ((x) << S_SCMD_LAST_FRAG) #define G_SCMD_LAST_FRAG(x) (((x) >> S_SCMD_LAST_FRAG) & M_SCMD_LAST_FRAG) /* TlsCompPdu */ #define S_SCMD_TLS_COMPPDU 18 #define M_SCMD_TLS_COMPPDU 0x1 #define V_SCMD_TLS_COMPPDU(x) ((x) << S_SCMD_TLS_COMPPDU) #define G_SCMD_TLS_COMPPDU(x) (((x) >> S_SCMD_TLS_COMPPDU) & M_SCMD_TLS_COMPPDU) /* KeyCntxtInline - Key context inline after the scmd OR PayloadOnly*/ #define S_SCMD_KEY_CTX_INLINE 17 #define M_SCMD_KEY_CTX_INLINE 0x1 #define V_SCMD_KEY_CTX_INLINE(x) ((x) << S_SCMD_KEY_CTX_INLINE) #define G_SCMD_KEY_CTX_INLINE(x) \ (((x) >> S_SCMD_KEY_CTX_INLINE) & M_SCMD_KEY_CTX_INLINE) #define F_SCMD_KEY_CTX_INLINE V_SCMD_KEY_CTX_INLINE(1U) /* TLSFragEnable - 0: Host created TLS PDUs, 1: TLS Framgmentation in ASIC */ #define S_SCMD_TLS_FRAG_ENABLE 16 #define M_SCMD_TLS_FRAG_ENABLE 0x1 #define V_SCMD_TLS_FRAG_ENABLE(x) ((x) << S_SCMD_TLS_FRAG_ENABLE) #define G_SCMD_TLS_FRAG_ENABLE(x) \ (((x) >> S_SCMD_TLS_FRAG_ENABLE) & M_SCMD_TLS_FRAG_ENABLE) #define F_SCMD_TLS_FRAG_ENABLE V_SCMD_TLS_FRAG_ENABLE(1U) /* MacOnly - Only send the MAC and discard PDU. This is valid for hash only * modes, in this case TLS_TX will drop the PDU and only * send back the MAC bytes. */ #define S_SCMD_MAC_ONLY 15 #define M_SCMD_MAC_ONLY 0x1 #define V_SCMD_MAC_ONLY(x) ((x) << S_SCMD_MAC_ONLY) #define G_SCMD_MAC_ONLY(x) \ (((x) >> S_SCMD_MAC_ONLY) & M_SCMD_MAC_ONLY) #define F_SCMD_MAC_ONLY V_SCMD_MAC_ONLY(1U) /* AadIVDrop - Drop the AAD and IV fields. Useful in protocols * which have complex AAD and IV formations Eg:AES-CCM */ #define S_SCMD_AADIVDROP 14 #define M_SCMD_AADIVDROP 0x1 #define V_SCMD_AADIVDROP(x) ((x) << S_SCMD_AADIVDROP) #define G_SCMD_AADIVDROP(x) \ (((x) >> S_SCMD_AADIVDROP) & M_SCMD_AADIVDROP) #define F_SCMD_AADIVDROP V_SCMD_AADIVDROP(1U) /* HdrLength - Length of all headers excluding TLS header * present before start of crypto PDU/payload. */ #define S_SCMD_HDR_LEN 0 #define M_SCMD_HDR_LEN 0x3fff #define V_SCMD_HDR_LEN(x) ((x) << S_SCMD_HDR_LEN) #define G_SCMD_HDR_LEN(x) \ (((x) >> S_SCMD_HDR_LEN) & M_SCMD_HDR_LEN) struct cpl_tx_sec_pdu { __be32 op_ivinsrtofst; __be32 pldlen; __be32 aadstart_cipherstop_hi; __be32 cipherstop_lo_authinsert; __be32 seqno_numivs; __be32 ivgen_hdrlen; __be64 scmd1; }; #define S_CPL_TX_SEC_PDU_OPCODE 24 #define M_CPL_TX_SEC_PDU_OPCODE 0xff #define V_CPL_TX_SEC_PDU_OPCODE(x) ((x) << S_CPL_TX_SEC_PDU_OPCODE) #define G_CPL_TX_SEC_PDU_OPCODE(x) \ (((x) >> S_CPL_TX_SEC_PDU_OPCODE) & M_CPL_TX_SEC_PDU_OPCODE) /* RX Channel Id */ #define S_CPL_TX_SEC_PDU_RXCHID 22 #define M_CPL_TX_SEC_PDU_RXCHID 0x1 #define V_CPL_TX_SEC_PDU_RXCHID(x) ((x) << S_CPL_TX_SEC_PDU_RXCHID) #define G_CPL_TX_SEC_PDU_RXCHID(x) \ (((x) >> S_CPL_TX_SEC_PDU_RXCHID) & M_CPL_TX_SEC_PDU_RXCHID) #define F_CPL_TX_SEC_PDU_RXCHID V_CPL_TX_SEC_PDU_RXCHID(1U) /* Ack Follows */ #define S_CPL_TX_SEC_PDU_ACKFOLLOWS 21 #define M_CPL_TX_SEC_PDU_ACKFOLLOWS 0x1 #define V_CPL_TX_SEC_PDU_ACKFOLLOWS(x) ((x) << S_CPL_TX_SEC_PDU_ACKFOLLOWS) #define G_CPL_TX_SEC_PDU_ACKFOLLOWS(x) \ (((x) >> S_CPL_TX_SEC_PDU_ACKFOLLOWS) & M_CPL_TX_SEC_PDU_ACKFOLLOWS) #define F_CPL_TX_SEC_PDU_ACKFOLLOWS V_CPL_TX_SEC_PDU_ACKFOLLOWS(1U) /* Loopback bit in cpl_tx_sec_pdu */ #define S_CPL_TX_SEC_PDU_ULPTXLPBK 20 #define M_CPL_TX_SEC_PDU_ULPTXLPBK 0x1 #define V_CPL_TX_SEC_PDU_ULPTXLPBK(x) ((x) << S_CPL_TX_SEC_PDU_ULPTXLPBK) #define G_CPL_TX_SEC_PDU_ULPTXLPBK(x) \ (((x) >> S_CPL_TX_SEC_PDU_ULPTXLPBK) & M_CPL_TX_SEC_PDU_ULPTXLPBK) #define F_CPL_TX_SEC_PDU_ULPTXLPBK V_CPL_TX_SEC_PDU_ULPTXLPBK(1U) /* Length of cpl header encapsulated */ #define S_CPL_TX_SEC_PDU_CPLLEN 16 #define M_CPL_TX_SEC_PDU_CPLLEN 0xf #define V_CPL_TX_SEC_PDU_CPLLEN(x) ((x) << S_CPL_TX_SEC_PDU_CPLLEN) #define G_CPL_TX_SEC_PDU_CPLLEN(x) \ (((x) >> S_CPL_TX_SEC_PDU_CPLLEN) & M_CPL_TX_SEC_PDU_CPLLEN) /* PlaceHolder */ #define S_CPL_TX_SEC_PDU_PLACEHOLDER 10 #define M_CPL_TX_SEC_PDU_PLACEHOLDER 0x1 #define V_CPL_TX_SEC_PDU_PLACEHOLDER(x) ((x) << S_CPL_TX_SEC_PDU_PLACEHOLDER) #define G_CPL_TX_SEC_PDU_PLACEHOLDER(x) \ (((x) >> S_CPL_TX_SEC_PDU_PLACEHOLDER) & \ M_CPL_TX_SEC_PDU_PLACEHOLDER) /* IvInsrtOffset: Insertion location for IV */ #define S_CPL_TX_SEC_PDU_IVINSRTOFST 0 #define M_CPL_TX_SEC_PDU_IVINSRTOFST 0x3ff #define V_CPL_TX_SEC_PDU_IVINSRTOFST(x) ((x) << S_CPL_TX_SEC_PDU_IVINSRTOFST) #define G_CPL_TX_SEC_PDU_IVINSRTOFST(x) \ (((x) >> S_CPL_TX_SEC_PDU_IVINSRTOFST) & \ M_CPL_TX_SEC_PDU_IVINSRTOFST) /* AadStartOffset: Offset in bytes for AAD start from * the first byte following * the pkt headers (0-255 * bytes) */ #define S_CPL_TX_SEC_PDU_AADSTART 24 #define M_CPL_TX_SEC_PDU_AADSTART 0xff #define V_CPL_TX_SEC_PDU_AADSTART(x) ((x) << S_CPL_TX_SEC_PDU_AADSTART) #define G_CPL_TX_SEC_PDU_AADSTART(x) \ (((x) >> S_CPL_TX_SEC_PDU_AADSTART) & \ M_CPL_TX_SEC_PDU_AADSTART) /* AadStopOffset: offset in bytes for AAD stop/end from the first byte following * the pkt headers (0-511 bytes) */ #define S_CPL_TX_SEC_PDU_AADSTOP 15 #define M_CPL_TX_SEC_PDU_AADSTOP 0x1ff #define V_CPL_TX_SEC_PDU_AADSTOP(x) ((x) << S_CPL_TX_SEC_PDU_AADSTOP) #define G_CPL_TX_SEC_PDU_AADSTOP(x) \ (((x) >> S_CPL_TX_SEC_PDU_AADSTOP) & M_CPL_TX_SEC_PDU_AADSTOP) /* CipherStartOffset: offset in bytes for encryption/decryption start from the * first byte following the pkt headers (0-1023 * bytes) */ #define S_CPL_TX_SEC_PDU_CIPHERSTART 5 #define M_CPL_TX_SEC_PDU_CIPHERSTART 0x3ff #define V_CPL_TX_SEC_PDU_CIPHERSTART(x) ((x) << S_CPL_TX_SEC_PDU_CIPHERSTART) #define G_CPL_TX_SEC_PDU_CIPHERSTART(x) \ (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTART) & \ M_CPL_TX_SEC_PDU_CIPHERSTART) /* CipherStopOffset: offset in bytes for encryption/decryption end * from end of the payload of this command (0-511 bytes) */ #define S_CPL_TX_SEC_PDU_CIPHERSTOP_HI 0 #define M_CPL_TX_SEC_PDU_CIPHERSTOP_HI 0x1f #define V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(x) \ ((x) << S_CPL_TX_SEC_PDU_CIPHERSTOP_HI) #define G_CPL_TX_SEC_PDU_CIPHERSTOP_HI(x) \ (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTOP_HI) & \ M_CPL_TX_SEC_PDU_CIPHERSTOP_HI) #define S_CPL_TX_SEC_PDU_CIPHERSTOP_LO 28 #define M_CPL_TX_SEC_PDU_CIPHERSTOP_LO 0xf #define V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(x) \ ((x) << S_CPL_TX_SEC_PDU_CIPHERSTOP_LO) #define G_CPL_TX_SEC_PDU_CIPHERSTOP_LO(x) \ (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTOP_LO) & \ M_CPL_TX_SEC_PDU_CIPHERSTOP_LO) /* AuthStartOffset: offset in bytes for authentication start from * the first byte following the pkt headers (0-1023) * */ #define S_CPL_TX_SEC_PDU_AUTHSTART 18 #define M_CPL_TX_SEC_PDU_AUTHSTART 0x3ff #define V_CPL_TX_SEC_PDU_AUTHSTART(x) ((x) << S_CPL_TX_SEC_PDU_AUTHSTART) #define G_CPL_TX_SEC_PDU_AUTHSTART(x) \ (((x) >> S_CPL_TX_SEC_PDU_AUTHSTART) & \ M_CPL_TX_SEC_PDU_AUTHSTART) /* AuthStopOffset: offset in bytes for authentication * end from end of the payload of this command (0-511 Bytes) */ #define S_CPL_TX_SEC_PDU_AUTHSTOP 9 #define M_CPL_TX_SEC_PDU_AUTHSTOP 0x1ff #define V_CPL_TX_SEC_PDU_AUTHSTOP(x) ((x) << S_CPL_TX_SEC_PDU_AUTHSTOP) #define G_CPL_TX_SEC_PDU_AUTHSTOP(x) \ (((x) >> S_CPL_TX_SEC_PDU_AUTHSTOP) & \ M_CPL_TX_SEC_PDU_AUTHSTOP) /* AuthInsrtOffset: offset in bytes for authentication insertion * from end of the payload of this command (0-511 bytes) */ #define S_CPL_TX_SEC_PDU_AUTHINSERT 0 #define M_CPL_TX_SEC_PDU_AUTHINSERT 0x1ff #define V_CPL_TX_SEC_PDU_AUTHINSERT(x) ((x) << S_CPL_TX_SEC_PDU_AUTHINSERT) #define G_CPL_TX_SEC_PDU_AUTHINSERT(x) \ (((x) >> S_CPL_TX_SEC_PDU_AUTHINSERT) & \ M_CPL_TX_SEC_PDU_AUTHINSERT) struct cpl_rx_phys_dsgl { __be32 op_to_tid; __be32 pcirlxorder_to_noofsgentr; struct rss_header rss_hdr_int; }; #define S_CPL_RX_PHYS_DSGL_OPCODE 24 #define M_CPL_RX_PHYS_DSGL_OPCODE 0xff #define V_CPL_RX_PHYS_DSGL_OPCODE(x) ((x) << S_CPL_RX_PHYS_DSGL_OPCODE) #define G_CPL_RX_PHYS_DSGL_OPCODE(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_OPCODE) & M_CPL_RX_PHYS_DSGL_OPCODE) #define S_CPL_RX_PHYS_DSGL_ISRDMA 23 #define M_CPL_RX_PHYS_DSGL_ISRDMA 0x1 #define V_CPL_RX_PHYS_DSGL_ISRDMA(x) ((x) << S_CPL_RX_PHYS_DSGL_ISRDMA) #define G_CPL_RX_PHYS_DSGL_ISRDMA(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_ISRDMA) & M_CPL_RX_PHYS_DSGL_ISRDMA) #define F_CPL_RX_PHYS_DSGL_ISRDMA V_CPL_RX_PHYS_DSGL_ISRDMA(1U) #define S_CPL_RX_PHYS_DSGL_RSVD1 20 #define M_CPL_RX_PHYS_DSGL_RSVD1 0x7 #define V_CPL_RX_PHYS_DSGL_RSVD1(x) ((x) << S_CPL_RX_PHYS_DSGL_RSVD1) #define G_CPL_RX_PHYS_DSGL_RSVD1(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_RSVD1) & M_CPL_RX_PHYS_DSGL_RSVD1) #define S_CPL_RX_PHYS_DSGL_PCIRLXORDER 31 #define M_CPL_RX_PHYS_DSGL_PCIRLXORDER 0x1 #define V_CPL_RX_PHYS_DSGL_PCIRLXORDER(x) \ ((x) << S_CPL_RX_PHYS_DSGL_PCIRLXORDER) #define G_CPL_RX_PHYS_DSGL_PCIRLXORDER(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_PCIRLXORDER) & \ M_CPL_RX_PHYS_DSGL_PCIRLXORDER) #define F_CPL_RX_PHYS_DSGL_PCIRLXORDER V_CPL_RX_PHYS_DSGL_PCIRLXORDER(1U) #define S_CPL_RX_PHYS_DSGL_PCINOSNOOP 30 #define M_CPL_RX_PHYS_DSGL_PCINOSNOOP 0x1 #define V_CPL_RX_PHYS_DSGL_PCINOSNOOP(x) \ ((x) << S_CPL_RX_PHYS_DSGL_PCINOSNOOP) #define G_CPL_RX_PHYS_DSGL_PCINOSNOOP(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_PCINOSNOOP) & \ M_CPL_RX_PHYS_DSGL_PCINOSNOOP) #define F_CPL_RX_PHYS_DSGL_PCINOSNOOP V_CPL_RX_PHYS_DSGL_PCINOSNOOP(1U) #define S_CPL_RX_PHYS_DSGL_PCITPHNTENB 29 #define M_CPL_RX_PHYS_DSGL_PCITPHNTENB 0x1 #define V_CPL_RX_PHYS_DSGL_PCITPHNTENB(x) \ ((x) << S_CPL_RX_PHYS_DSGL_PCITPHNTENB) #define G_CPL_RX_PHYS_DSGL_PCITPHNTENB(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_PCITPHNTENB) & \ M_CPL_RX_PHYS_DSGL_PCITPHNTENB) #define F_CPL_RX_PHYS_DSGL_PCITPHNTENB V_CPL_RX_PHYS_DSGL_PCITPHNTENB(1U) #define S_CPL_RX_PHYS_DSGL_PCITPHNT 27 #define M_CPL_RX_PHYS_DSGL_PCITPHNT 0x3 #define V_CPL_RX_PHYS_DSGL_PCITPHNT(x) ((x) << S_CPL_RX_PHYS_DSGL_PCITPHNT) #define G_CPL_RX_PHYS_DSGL_PCITPHNT(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_PCITPHNT) & \ M_CPL_RX_PHYS_DSGL_PCITPHNT) #define S_CPL_RX_PHYS_DSGL_DCAID 16 #define M_CPL_RX_PHYS_DSGL_DCAID 0x7ff #define V_CPL_RX_PHYS_DSGL_DCAID(x) ((x) << S_CPL_RX_PHYS_DSGL_DCAID) #define G_CPL_RX_PHYS_DSGL_DCAID(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_DCAID) & \ M_CPL_RX_PHYS_DSGL_DCAID) #define S_CPL_RX_PHYS_DSGL_NOOFSGENTR 0 #define M_CPL_RX_PHYS_DSGL_NOOFSGENTR 0xffff #define V_CPL_RX_PHYS_DSGL_NOOFSGENTR(x) \ ((x) << S_CPL_RX_PHYS_DSGL_NOOFSGENTR) #define G_CPL_RX_PHYS_DSGL_NOOFSGENTR(x) \ (((x) >> S_CPL_RX_PHYS_DSGL_NOOFSGENTR) & \ M_CPL_RX_PHYS_DSGL_NOOFSGENTR) /* CPL_TX_TLS_ACK */ struct cpl_tx_tls_ack { __be32 op_to_Rsvd2; __be32 PldLen; __be64 Rsvd3; }; #define S_CPL_TX_TLS_ACK_OPCODE 24 #define M_CPL_TX_TLS_ACK_OPCODE 0xff #define V_CPL_TX_TLS_ACK_OPCODE(x) ((x) << S_CPL_TX_TLS_ACK_OPCODE) #define G_CPL_TX_TLS_ACK_OPCODE(x) \ (((x) >> S_CPL_TX_TLS_ACK_OPCODE) & M_CPL_TX_TLS_ACK_OPCODE) #define S_CPL_TX_TLS_ACK_RSVD1 23 #define M_CPL_TX_TLS_ACK_RSVD1 0x1 #define V_CPL_TX_TLS_ACK_RSVD1(x) ((x) << S_CPL_TX_TLS_ACK_RSVD1) #define G_CPL_TX_TLS_ACK_RSVD1(x) \ (((x) >> S_CPL_TX_TLS_ACK_RSVD1) & M_CPL_TX_TLS_ACK_RSVD1) #define F_CPL_TX_TLS_ACK_RSVD1 V_CPL_TX_TLS_ACK_RSVD1(1U) #define S_CPL_TX_TLS_ACK_RXCHID 22 #define M_CPL_TX_TLS_ACK_RXCHID 0x1 #define V_CPL_TX_TLS_ACK_RXCHID(x) ((x) << S_CPL_TX_TLS_ACK_RXCHID) #define G_CPL_TX_TLS_ACK_RXCHID(x) \ (((x) >> S_CPL_TX_TLS_ACK_RXCHID) & M_CPL_TX_TLS_ACK_RXCHID) #define F_CPL_TX_TLS_ACK_RXCHID V_CPL_TX_TLS_ACK_RXCHID(1U) #define S_CPL_TX_TLS_ACK_FWMSG 21 #define M_CPL_TX_TLS_ACK_FWMSG 0x1 #define V_CPL_TX_TLS_ACK_FWMSG(x) ((x) << S_CPL_TX_TLS_ACK_FWMSG) #define G_CPL_TX_TLS_ACK_FWMSG(x) \ (((x) >> S_CPL_TX_TLS_ACK_FWMSG) & M_CPL_TX_TLS_ACK_FWMSG) #define F_CPL_TX_TLS_ACK_FWMSG V_CPL_TX_TLS_ACK_FWMSG(1U) #define S_CPL_TX_TLS_ACK_ULPTXLPBK 20 #define M_CPL_TX_TLS_ACK_ULPTXLPBK 0x1 #define V_CPL_TX_TLS_ACK_ULPTXLPBK(x) ((x) << S_CPL_TX_TLS_ACK_ULPTXLPBK) #define G_CPL_TX_TLS_ACK_ULPTXLPBK(x) \ (((x) >> S_CPL_TX_TLS_ACK_ULPTXLPBK) & M_CPL_TX_TLS_ACK_ULPTXLPBK) #define F_CPL_TX_TLS_ACK_ULPTXLPBK V_CPL_TX_TLS_ACK_ULPTXLPBK(1U) #define S_CPL_TX_TLS_ACK_CPLLEN 16 #define M_CPL_TX_TLS_ACK_CPLLEN 0xf #define V_CPL_TX_TLS_ACK_CPLLEN(x) ((x) << S_CPL_TX_TLS_ACK_CPLLEN) #define G_CPL_TX_TLS_ACK_CPLLEN(x) \ (((x) >> S_CPL_TX_TLS_ACK_CPLLEN) & M_CPL_TX_TLS_ACK_CPLLEN) #define S_CPL_TX_TLS_ACK_COMPLONERR 15 #define M_CPL_TX_TLS_ACK_COMPLONERR 0x1 #define V_CPL_TX_TLS_ACK_COMPLONERR(x) ((x) << S_CPL_TX_TLS_ACK_COMPLONERR) #define G_CPL_TX_TLS_ACK_COMPLONERR(x) \ (((x) >> S_CPL_TX_TLS_ACK_COMPLONERR) & M_CPL_TX_TLS_ACK_COMPLONERR) #define F_CPL_TX_TLS_ACK_COMPLONERR V_CPL_TX_TLS_ACK_COMPLONERR(1U) #define S_CPL_TX_TLS_ACK_LCB 14 #define M_CPL_TX_TLS_ACK_LCB 0x1 #define V_CPL_TX_TLS_ACK_LCB(x) ((x) << S_CPL_TX_TLS_ACK_LCB) #define G_CPL_TX_TLS_ACK_LCB(x) \ (((x) >> S_CPL_TX_TLS_ACK_LCB) & M_CPL_TX_TLS_ACK_LCB) #define F_CPL_TX_TLS_ACK_LCB V_CPL_TX_TLS_ACK_LCB(1U) #define S_CPL_TX_TLS_ACK_PHASH 13 #define M_CPL_TX_TLS_ACK_PHASH 0x1 #define V_CPL_TX_TLS_ACK_PHASH(x) ((x) << S_CPL_TX_TLS_ACK_PHASH) #define G_CPL_TX_TLS_ACK_PHASH(x) \ (((x) >> S_CPL_TX_TLS_ACK_PHASH) & M_CPL_TX_TLS_ACK_PHASH) #define F_CPL_TX_TLS_ACK_PHASH V_CPL_TX_TLS_ACK_PHASH(1U) #define S_CPL_TX_TLS_ACK_RSVD2 0 #define M_CPL_TX_TLS_ACK_RSVD2 0x1fff #define V_CPL_TX_TLS_ACK_RSVD2(x) ((x) << S_CPL_TX_TLS_ACK_RSVD2) #define G_CPL_TX_TLS_ACK_RSVD2(x) \ (((x) >> S_CPL_TX_TLS_ACK_RSVD2) & M_CPL_TX_TLS_ACK_RSVD2) #endif /* T4_MSG_H */ Index: stable/11/sys/dev/cxgbe/t4_sge.c =================================================================== --- stable/11/sys/dev/cxgbe/t4_sge.c (revision 346862) +++ stable/11/sys/dev/cxgbe/t4_sge.c (revision 346863) @@ -1,5264 +1,5286 @@ /*- * Copyright (c) 2011 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEV_NETMAP #include #include #include #include #include #endif #include "common/common.h" #include "common/t4_regs.h" #include "common/t4_regs_values.h" #include "common/t4_msg.h" #include "t4_l2t.h" #include "t4_mp_ring.h" #ifdef T4_PKT_TIMESTAMP #define RX_COPY_THRESHOLD (MINCLSIZE - 8) #else #define RX_COPY_THRESHOLD MINCLSIZE #endif /* * Ethernet frames are DMA'd at this byte offset into the freelist buffer. * 0-7 are valid values. */ static int fl_pktshift = 2; SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, "payload DMA offset in rx buffer (bytes)"); /* * Pad ethernet payload up to this boundary. * -1: driver should figure out a good value. * 0: disable padding. * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value. */ int fl_pad = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pad, CTLFLAG_RDTUN, &fl_pad, 0, "payload pad boundary (bytes)"); /* * Status page length. * -1: driver should figure out a good value. * 64 or 128 are the only other valid values. */ static int spg_len = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, spg_len, CTLFLAG_RDTUN, &spg_len, 0, "status page size (bytes)"); /* * Congestion drops. * -1: no congestion feedback (not recommended). * 0: backpressure the channel instead of dropping packets right away. * 1: no backpressure, drop packets for the congested queue immediately. */ static int cong_drop = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, cong_drop, CTLFLAG_RDTUN, &cong_drop, 0, "Congestion control for RX queues (0 = backpressure, 1 = drop"); /* * Deliver multiple frames in the same free list buffer if they fit. * -1: let the driver decide whether to enable buffer packing or not. * 0: disable buffer packing. * 1: enable buffer packing. */ static int buffer_packing = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, buffer_packing, CTLFLAG_RDTUN, &buffer_packing, 0, "Enable buffer packing"); /* * Start next frame in a packed buffer at this boundary. * -1: driver should figure out a good value. * T4: driver will ignore this and use the same value as fl_pad above. * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value. */ static int fl_pack = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pack, CTLFLAG_RDTUN, &fl_pack, 0, "payload pack boundary (bytes)"); /* * Allow the driver to create mbuf(s) in a cluster allocated for rx. * 0: never; always allocate mbufs from the zone_mbuf UMA zone. * 1: ok to create mbuf(s) within a cluster if there is room. */ static int allow_mbufs_in_cluster = 1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, allow_mbufs_in_cluster, CTLFLAG_RDTUN, &allow_mbufs_in_cluster, 0, "Allow driver to create mbufs within a rx cluster"); /* * Largest rx cluster size that the driver is allowed to allocate. */ static int largest_rx_cluster = MJUM16BYTES; SYSCTL_INT(_hw_cxgbe, OID_AUTO, largest_rx_cluster, CTLFLAG_RDTUN, &largest_rx_cluster, 0, "Largest rx cluster (bytes)"); /* * Size of cluster allocation that's most likely to succeed. The driver will * fall back to this size if it fails to allocate clusters larger than this. */ static int safest_rx_cluster = PAGE_SIZE; SYSCTL_INT(_hw_cxgbe, OID_AUTO, safest_rx_cluster, CTLFLAG_RDTUN, &safest_rx_cluster, 0, "Safe rx cluster (bytes)"); /* * The interrupt holdoff timers are multiplied by this value on T6+. * 1 and 3-17 (both inclusive) are legal values. */ static int tscale = 1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, tscale, CTLFLAG_RDTUN, &tscale, 0, "Interrupt holdoff timer scale on T6+"); /* * Number of LRO entries in the lro_ctrl structure per rx queue. */ static int lro_entries = TCP_LRO_ENTRIES; SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_entries, CTLFLAG_RDTUN, &lro_entries, 0, "Number of LRO entries per RX queue"); /* * This enables presorting of frames before they're fed into tcp_lro_rx. */ static int lro_mbufs = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_mbufs, CTLFLAG_RDTUN, &lro_mbufs, 0, "Enable presorting of LRO frames"); struct txpkts { u_int wr_type; /* type 0 or type 1 */ u_int npkt; /* # of packets in this work request */ u_int plen; /* total payload (sum of all packets) */ u_int len16; /* # of 16B pieces used by this work request */ }; /* A packet's SGL. This + m_pkthdr has all info needed for tx */ struct sgl { struct sglist sg; struct sglist_seg seg[TX_SGL_SEGS]; }; static int service_iq(struct sge_iq *, int); static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *); static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int); static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *); static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t, uint16_t, char *); static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *, bus_addr_t *, void **); static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, void *); static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *, int, int); static int free_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *); static void add_iq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_iq *); static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_fl *); static int alloc_fwq(struct adapter *); static int free_fwq(struct adapter *); static int alloc_mgmtq(struct adapter *); static int free_mgmtq(struct adapter *); static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, struct sysctl_oid *); static int free_rxq(struct vi_info *, struct sge_rxq *); #ifdef TCP_OFFLOAD static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int, struct sysctl_oid *); static int free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *); #endif #ifdef DEV_NETMAP static int alloc_nm_rxq(struct vi_info *, struct sge_nm_rxq *, int, int, struct sysctl_oid *); static int free_nm_rxq(struct vi_info *, struct sge_nm_rxq *); static int alloc_nm_txq(struct vi_info *, struct sge_nm_txq *, int, int, struct sysctl_oid *); static int free_nm_txq(struct vi_info *, struct sge_nm_txq *); #endif static int ctrl_eq_alloc(struct adapter *, struct sge_eq *); static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); #ifdef TCP_OFFLOAD static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); #endif static int alloc_eq(struct adapter *, struct vi_info *, struct sge_eq *); static int free_eq(struct adapter *, struct sge_eq *); static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, struct sysctl_oid *); static int free_wrq(struct adapter *, struct sge_wrq *); static int alloc_txq(struct vi_info *, struct sge_txq *, int, struct sysctl_oid *); static int free_txq(struct vi_info *, struct sge_txq *); static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int); static inline void ring_fl_db(struct adapter *, struct sge_fl *); static int refill_fl(struct adapter *, struct sge_fl *, int); static void refill_sfl(void *); static int alloc_fl_sdesc(struct sge_fl *); static void free_fl_sdesc(struct adapter *, struct sge_fl *); static void find_best_refill_source(struct adapter *, struct sge_fl *, int); static void find_safe_refill_source(struct adapter *, struct sge_fl *); static void add_fl_to_sfl(struct adapter *, struct sge_fl *); static inline void get_pkt_gl(struct mbuf *, struct sglist *); static inline u_int txpkt_len16(u_int, u_int); static inline u_int txpkt_vm_len16(u_int, u_int); static inline u_int txpkts0_len16(u_int); static inline u_int txpkts1_len16(void); static u_int write_txpkt_wr(struct sge_txq *, struct fw_eth_tx_pkt_wr *, struct mbuf *, u_int); static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *, struct fw_eth_tx_pkt_vm_wr *, struct mbuf *, u_int); static int try_txpkts(struct mbuf *, struct mbuf *, struct txpkts *, u_int); static int add_to_txpkts(struct mbuf *, struct txpkts *, u_int); static u_int write_txpkts_wr(struct sge_txq *, struct fw_eth_tx_pkts_wr *, struct mbuf *, const struct txpkts *, u_int); static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int); static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int); static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int); static inline uint16_t read_hw_cidx(struct sge_eq *); static inline u_int reclaimable_tx_desc(struct sge_eq *); static inline u_int total_available_tx_desc(struct sge_eq *); static u_int reclaim_tx_descs(struct sge_txq *, u_int); static void tx_reclaim(void *, int); static __be64 get_flit(struct sglist_seg *, int, int); static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, struct mbuf *); static int handle_fw_msg(struct sge_iq *, const struct rss_header *, struct mbuf *); static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *); static void wrq_tx_drain(void *, int); static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *); static int sysctl_uint16(SYSCTL_HANDLER_ARGS); static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS); static int sysctl_tc(SYSCTL_HANDLER_ARGS); static counter_u64_t extfree_refs; static counter_u64_t extfree_rels; an_handler_t t4_an_handler; fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES]; cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS]; cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES]; cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES]; cpl_handler_t act_open_rpl_handlers[NUM_CPL_COOKIES]; cpl_handler_t abort_rpl_rss_handlers[NUM_CPL_COOKIES]; +cpl_handler_t fw4_ack_handlers[NUM_CPL_COOKIES]; void t4_register_an_handler(an_handler_t h) { uintptr_t *loc; MPASS(h == NULL || t4_an_handler == NULL); loc = (uintptr_t *)&t4_an_handler; atomic_store_rel_ptr(loc, (uintptr_t)h); } void t4_register_fw_msg_handler(int type, fw_msg_handler_t h) { uintptr_t *loc; MPASS(type < nitems(t4_fw_msg_handler)); MPASS(h == NULL || t4_fw_msg_handler[type] == NULL); /* * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL * handler dispatch table. Reject any attempt to install a handler for * this subtype. */ MPASS(type != FW_TYPE_RSSCPL); MPASS(type != FW6_TYPE_RSSCPL); loc = (uintptr_t *)&t4_fw_msg_handler[type]; atomic_store_rel_ptr(loc, (uintptr_t)h); } void t4_register_cpl_handler(int opcode, cpl_handler_t h) { uintptr_t *loc; MPASS(opcode < nitems(t4_cpl_handler)); MPASS(h == NULL || t4_cpl_handler[opcode] == NULL); loc = (uintptr_t *)&t4_cpl_handler[opcode]; atomic_store_rel_ptr(loc, (uintptr_t)h); } static int set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); u_int tid; int cookie; MPASS(m == NULL); tid = GET_TID(cpl); if (is_ftid(iq->adapter, tid)) { /* * The return code for filter-write is put in the CPL cookie so * we have to rely on the hardware tid (is_ftid) to determine * that this is a response to a filter. */ cookie = CPL_COOKIE_FILTER; } else { cookie = G_COOKIE(cpl->cookie); } MPASS(cookie > CPL_COOKIE_RESERVED); MPASS(cookie < nitems(set_tcb_rpl_handlers)); return (set_tcb_rpl_handlers[cookie](iq, rss, m)); } static int l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1); unsigned int cookie; MPASS(m == NULL); cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER; return (l2t_write_rpl_handlers[cookie](iq, rss, m)); } static int act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); u_int cookie = G_TID_COOKIE(G_AOPEN_ATID(be32toh(cpl->atid_status))); MPASS(m == NULL); MPASS(cookie != CPL_COOKIE_RESERVED); return (act_open_rpl_handlers[cookie](iq, rss, m)); } static int abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; u_int cookie; MPASS(m == NULL); if (is_hashfilter(sc)) cookie = CPL_COOKIE_HASHFILTER; else cookie = CPL_COOKIE_TOM; return (abort_rpl_rss_handlers[cookie](iq, rss, m)); } +static int +fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) +{ + struct adapter *sc = iq->adapter; + const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); + unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); + u_int cookie; + + MPASS(m == NULL); + if (is_etid(sc, tid)) + cookie = CPL_COOKIE_ETHOFLD; + else + cookie = CPL_COOKIE_TOM; + + return (fw4_ack_handlers[cookie](iq, rss, m)); +} + static void t4_init_shared_cpl_handlers(void) { t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler); t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler); t4_register_cpl_handler(CPL_ACT_OPEN_RPL, act_open_rpl_handler); t4_register_cpl_handler(CPL_ABORT_RPL_RSS, abort_rpl_rss_handler); + t4_register_cpl_handler(CPL_FW4_ACK, fw4_ack_handler); } void t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie) { uintptr_t *loc; MPASS(opcode < nitems(t4_cpl_handler)); MPASS(cookie > CPL_COOKIE_RESERVED); MPASS(cookie < NUM_CPL_COOKIES); MPASS(t4_cpl_handler[opcode] != NULL); switch (opcode) { case CPL_SET_TCB_RPL: loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie]; break; case CPL_L2T_WRITE_RPL: loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie]; break; case CPL_ACT_OPEN_RPL: loc = (uintptr_t *)&act_open_rpl_handlers[cookie]; break; case CPL_ABORT_RPL_RSS: loc = (uintptr_t *)&abort_rpl_rss_handlers[cookie]; + break; + case CPL_FW4_ACK: + loc = (uintptr_t *)&fw4_ack_handlers[cookie]; break; default: MPASS(0); return; } MPASS(h == NULL || *loc == (uintptr_t)NULL); atomic_store_rel_ptr(loc, (uintptr_t)h); } /* * Called on MOD_LOAD. Validates and calculates the SGE tunables. */ void t4_sge_modload(void) { if (fl_pktshift < 0 || fl_pktshift > 7) { printf("Invalid hw.cxgbe.fl_pktshift value (%d)," " using 2 instead.\n", fl_pktshift); fl_pktshift = 2; } if (spg_len != 64 && spg_len != 128) { int len; #if defined(__i386__) || defined(__amd64__) len = cpu_clflush_line_size > 64 ? 128 : 64; #else len = 64; #endif if (spg_len != -1) { printf("Invalid hw.cxgbe.spg_len value (%d)," " using %d instead.\n", spg_len, len); } spg_len = len; } if (cong_drop < -1 || cong_drop > 1) { printf("Invalid hw.cxgbe.cong_drop value (%d)," " using 0 instead.\n", cong_drop); cong_drop = 0; } if (tscale != 1 && (tscale < 3 || tscale > 17)) { printf("Invalid hw.cxgbe.tscale value (%d)," " using 1 instead.\n", tscale); tscale = 1; } extfree_refs = counter_u64_alloc(M_WAITOK); extfree_rels = counter_u64_alloc(M_WAITOK); counter_u64_zero(extfree_refs); counter_u64_zero(extfree_rels); t4_init_shared_cpl_handlers(); t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg); t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg); t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update); t4_register_cpl_handler(CPL_RX_PKT, t4_eth_rx); t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl); t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl); } void t4_sge_modunload(void) { counter_u64_free(extfree_refs); counter_u64_free(extfree_rels); } uint64_t t4_sge_extfree_refs(void) { uint64_t refs, rels; rels = counter_u64_fetch(extfree_rels); refs = counter_u64_fetch(extfree_refs); return (refs - rels); } static inline void setup_pad_and_pack_boundaries(struct adapter *sc) { uint32_t v, m; int pad, pack, pad_shift; pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT : X_INGPADBOUNDARY_SHIFT; pad = fl_pad; if (fl_pad < (1 << pad_shift) || fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) || !powerof2(fl_pad)) { /* * If there is any chance that we might use buffer packing and * the chip is a T4, then pick 64 as the pad/pack boundary. Set * it to the minimum allowed in all other cases. */ pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift; /* * For fl_pad = 0 we'll still write a reasonable value to the * register but all the freelists will opt out of padding. * We'll complain here only if the user tried to set it to a * value greater than 0 that was invalid. */ if (fl_pad > 0) { device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value" " (%d), using %d instead.\n", fl_pad, pad); } } m = V_INGPADBOUNDARY(M_INGPADBOUNDARY); v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift); t4_set_reg_field(sc, A_SGE_CONTROL, m, v); if (is_t4(sc)) { if (fl_pack != -1 && fl_pack != pad) { /* Complain but carry on. */ device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored," " using %d instead.\n", fl_pack, pad); } return; } pack = fl_pack; if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 || !powerof2(fl_pack)) { pack = max(sc->params.pci.mps, CACHE_LINE_SIZE); MPASS(powerof2(pack)); if (pack < 16) pack = 16; if (pack == 32) pack = 64; if (pack > 4096) pack = 4096; if (fl_pack != -1) { device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value" " (%d), using %d instead.\n", fl_pack, pack); } } m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY); if (pack == 16) v = V_INGPACKBOUNDARY(0); else v = V_INGPACKBOUNDARY(ilog2(pack) - 5); MPASS(!is_t4(sc)); /* T4 doesn't have SGE_CONTROL2 */ t4_set_reg_field(sc, A_SGE_CONTROL2, m, v); } /* * adap->params.vpd.cclk must be set up before this is called. */ void t4_tweak_chip_settings(struct adapter *sc) { int i; uint32_t v, m; int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200}; int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk; int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); static int sge_flbuf_sizes[] = { MCLBYTES, #if MJUMPAGESIZE != MCLBYTES MJUMPAGESIZE, MJUMPAGESIZE - CL_METADATA_SIZE, MJUMPAGESIZE - 2 * MSIZE - CL_METADATA_SIZE, #endif MJUM9BYTES, MJUM16BYTES, MCLBYTES - MSIZE - CL_METADATA_SIZE, MJUM9BYTES - CL_METADATA_SIZE, MJUM16BYTES - CL_METADATA_SIZE, }; KASSERT(sc->flags & MASTER_PF, ("%s: trying to change chip settings when not master.", __func__)); m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE; v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE | V_EGRSTATUSPAGESIZE(spg_len == 128); t4_set_reg_field(sc, A_SGE_CONTROL, m, v); setup_pad_and_pack_boundaries(sc); v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) | V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10); t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v); KASSERT(nitems(sge_flbuf_sizes) <= SGE_FLBUF_SIZES, ("%s: hw buffer size table too big", __func__)); for (i = 0; i < min(nitems(sge_flbuf_sizes), SGE_FLBUF_SIZES); i++) { t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i), sge_flbuf_sizes[i]); } v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) | V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]); t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v); KASSERT(intr_timer[0] <= timer_max, ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0], timer_max)); for (i = 1; i < nitems(intr_timer); i++) { KASSERT(intr_timer[i] >= intr_timer[i - 1], ("%s: timers not listed in increasing order (%d)", __func__, i)); while (intr_timer[i] > timer_max) { if (i == nitems(intr_timer) - 1) { intr_timer[i] = timer_max; break; } intr_timer[i] += intr_timer[i - 1]; intr_timer[i] /= 2; } } v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) | V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1])); t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v); v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) | V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3])); t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v); v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) | V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5])); t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v); if (chip_id(sc) >= CHELSIO_T6) { m = V_TSCALE(M_TSCALE); if (tscale == 1) v = 0; else v = V_TSCALE(tscale - 2); t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v); if (sc->debug_flags & DF_DISABLE_TCB_CACHE) { m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN | V_WRTHRTHRESH(M_WRTHRTHRESH); t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1); v &= ~m; v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN | V_WRTHRTHRESH(16); t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1); } } /* 4K, 16K, 64K, 256K DDP "page sizes" */ v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v); m = v = F_TDDPTAGTCB; t4_set_reg_field(sc, A_ULP_RX_CTL, m, v); m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; t4_set_reg_field(sc, A_TP_PARA_REG5, m, v); } /* * SGE wants the buffer to be at least 64B and then a multiple of 16. If * padding is in use, the buffer's start and end need to be aligned to the pad * boundary as well. We'll just make sure that the size is a multiple of the * boundary here, it is up to the buffer allocation code to make sure the start * of the buffer is aligned as well. */ static inline int hwsz_ok(struct adapter *sc, int hwsz) { int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1; return (hwsz >= 64 && (hwsz & mask) == 0); } /* * XXX: driver really should be able to deal with unexpected settings. */ int t4_read_chip_settings(struct adapter *sc) { struct sge *s = &sc->sge; struct sge_params *sp = &sc->params.sge; int i, j, n, rc = 0; uint32_t m, v, r; uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); static int sw_buf_sizes[] = { /* Sorted by size */ MCLBYTES, #if MJUMPAGESIZE != MCLBYTES MJUMPAGESIZE, #endif MJUM9BYTES, MJUM16BYTES }; struct sw_zone_info *swz, *safe_swz; struct hw_buf_info *hwb; m = F_RXPKTCPLMODE; v = F_RXPKTCPLMODE; r = sc->params.sge.sge_control; if ((r & m) != v) { device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r); rc = EINVAL; } /* * If this changes then every single use of PAGE_SHIFT in the driver * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift. */ if (sp->page_shift != PAGE_SHIFT) { device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r); rc = EINVAL; } /* Filter out unusable hw buffer sizes entirely (mark with -2). */ hwb = &s->hw_buf_info[0]; for (i = 0; i < nitems(s->hw_buf_info); i++, hwb++) { r = sc->params.sge.sge_fl_buffer_size[i]; hwb->size = r; hwb->zidx = hwsz_ok(sc, r) ? -1 : -2; hwb->next = -1; } /* * Create a sorted list in decreasing order of hw buffer sizes (and so * increasing order of spare area) for each software zone. * * If padding is enabled then the start and end of the buffer must align * to the pad boundary; if packing is enabled then they must align with * the pack boundary as well. Allocations from the cluster zones are * aligned to min(size, 4K), so the buffer starts at that alignment and * ends at hwb->size alignment. If mbuf inlining is allowed the * starting alignment will be reduced to MSIZE and the driver will * exercise appropriate caution when deciding on the best buffer layout * to use. */ n = 0; /* no usable buffer size to begin with */ swz = &s->sw_zone_info[0]; safe_swz = NULL; for (i = 0; i < SW_ZONE_SIZES; i++, swz++) { int8_t head = -1, tail = -1; swz->size = sw_buf_sizes[i]; swz->zone = m_getzone(swz->size); swz->type = m_gettype(swz->size); if (swz->size < PAGE_SIZE) { MPASS(powerof2(swz->size)); if (fl_pad && (swz->size % sp->pad_boundary != 0)) continue; } if (swz->size == safest_rx_cluster) safe_swz = swz; hwb = &s->hw_buf_info[0]; for (j = 0; j < SGE_FLBUF_SIZES; j++, hwb++) { if (hwb->zidx != -1 || hwb->size > swz->size) continue; #ifdef INVARIANTS if (fl_pad) MPASS(hwb->size % sp->pad_boundary == 0); #endif hwb->zidx = i; if (head == -1) head = tail = j; else if (hwb->size < s->hw_buf_info[tail].size) { s->hw_buf_info[tail].next = j; tail = j; } else { int8_t *cur; struct hw_buf_info *t; for (cur = &head; *cur != -1; cur = &t->next) { t = &s->hw_buf_info[*cur]; if (hwb->size == t->size) { hwb->zidx = -2; break; } if (hwb->size > t->size) { hwb->next = *cur; *cur = j; break; } } } } swz->head_hwidx = head; swz->tail_hwidx = tail; if (tail != -1) { n++; if (swz->size - s->hw_buf_info[tail].size >= CL_METADATA_SIZE) sc->flags |= BUF_PACKING_OK; } } if (n == 0) { device_printf(sc->dev, "no usable SGE FL buffer size.\n"); rc = EINVAL; } s->safe_hwidx1 = -1; s->safe_hwidx2 = -1; if (safe_swz != NULL) { s->safe_hwidx1 = safe_swz->head_hwidx; for (i = safe_swz->head_hwidx; i != -1; i = hwb->next) { int spare; hwb = &s->hw_buf_info[i]; #ifdef INVARIANTS if (fl_pad) MPASS(hwb->size % sp->pad_boundary == 0); #endif spare = safe_swz->size - hwb->size; if (spare >= CL_METADATA_SIZE) { s->safe_hwidx2 = i; break; } } } if (sc->flags & IS_VF) return (0); v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ); if (r != v) { device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r); rc = EINVAL; } m = v = F_TDDPTAGTCB; r = t4_read_reg(sc, A_ULP_RX_CTL); if ((r & m) != v) { device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r); rc = EINVAL; } m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; r = t4_read_reg(sc, A_TP_PARA_REG5); if ((r & m) != v) { device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r); rc = EINVAL; } t4_init_tp_params(sc, 1); t4_read_mtu_tbl(sc, sc->params.mtus, NULL); t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd); return (rc); } int t4_create_dma_tag(struct adapter *sc) { int rc; rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->dmat); if (rc != 0) { device_printf(sc->dev, "failed to create main DMA tag: %d\n", rc); } return (rc); } void t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, struct sysctl_oid_list *children) { struct sge_params *sp = &sc->params.sge; SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes", CTLTYPE_STRING | CTLFLAG_RD, &sc->sge, 0, sysctl_bufsizes, "A", "freelist buffer sizes"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD, NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD, NULL, sp->pad_boundary, "payload pad boundary (bytes)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD, NULL, sp->spg_len, "status page size (bytes)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD, NULL, cong_drop, "congestion drop setting"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD, NULL, sp->pack_boundary, "payload pack boundary (bytes)"); } int t4_destroy_dma_tag(struct adapter *sc) { if (sc->dmat) bus_dma_tag_destroy(sc->dmat); return (0); } /* * Allocate and initialize the firmware event queue and the management queue. * * Returns errno on failure. Resources allocated up to that point may still be * allocated. Caller is responsible for cleanup in case this function fails. */ int t4_setup_adapter_queues(struct adapter *sc) { int rc; ADAPTER_LOCK_ASSERT_NOTOWNED(sc); sysctl_ctx_init(&sc->ctx); sc->flags |= ADAP_SYSCTL_CTX; /* * Firmware event queue */ rc = alloc_fwq(sc); if (rc != 0) return (rc); /* * Management queue. This is just a control queue that uses the fwq as * its associated iq. */ if (!(sc->flags & IS_VF)) rc = alloc_mgmtq(sc); return (rc); } /* * Idempotent */ int t4_teardown_adapter_queues(struct adapter *sc) { ADAPTER_LOCK_ASSERT_NOTOWNED(sc); /* Do this before freeing the queue */ if (sc->flags & ADAP_SYSCTL_CTX) { sysctl_ctx_free(&sc->ctx); sc->flags &= ~ADAP_SYSCTL_CTX; } free_mgmtq(sc); free_fwq(sc); return (0); } /* Maximum payload that can be delivered with a single iq descriptor */ static inline int mtu_to_max_payload(struct adapter *sc, int mtu, const int toe) { int payload; #ifdef TCP_OFFLOAD if (toe) { int rxcs = G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); /* Note that COP can set rx_coalesce on/off per connection. */ payload = max(mtu, rxcs); } else { #endif /* large enough even when hw VLAN extraction is disabled */ payload = sc->params.sge.fl_pktshift + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + mtu; #ifdef TCP_OFFLOAD } #endif return (payload); } int t4_setup_vi_queues(struct vi_info *vi) { int rc = 0, i, intr_idx, iqidx; struct sge_rxq *rxq; struct sge_txq *txq; struct sge_wrq *ctrlq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; struct sge_wrq *ofld_txq; #endif #ifdef DEV_NETMAP int saved_idx; struct sge_nm_rxq *nm_rxq; struct sge_nm_txq *nm_txq; #endif char name[16]; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct ifnet *ifp = vi->ifp; struct sysctl_oid *oid = device_get_sysctl_tree(vi->dev); struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); int maxp, mtu = ifp->if_mtu; /* Interrupt vector to start from (when using multiple vectors) */ intr_idx = vi->first_intr; #ifdef DEV_NETMAP saved_idx = intr_idx; if (ifp->if_capabilities & IFCAP_NETMAP) { /* netmap is supported with direct interrupts only. */ MPASS(!forwarding_intr_to_fwq(sc)); /* * We don't have buffers to back the netmap rx queues * right now so we create the queues in a way that * doesn't set off any congestion signal in the chip. */ oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_rxq", CTLFLAG_RD, NULL, "rx queues"); for_each_nm_rxq(vi, i, nm_rxq) { rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i, oid); if (rc != 0) goto done; intr_idx++; } oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_txq", CTLFLAG_RD, NULL, "tx queues"); for_each_nm_txq(vi, i, nm_txq) { iqidx = vi->first_nm_rxq + (i % vi->nnmrxq); rc = alloc_nm_txq(vi, nm_txq, iqidx, i, oid); if (rc != 0) goto done; } } /* Normal rx queues and netmap rx queues share the same interrupts. */ intr_idx = saved_idx; #endif /* * Allocate rx queues first because a default iqid is required when * creating a tx queue. */ maxp = mtu_to_max_payload(sc, mtu, 0); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD, NULL, "rx queues"); for_each_rxq(vi, i, rxq) { init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq); snprintf(name, sizeof(name), "%s rxq%d-fl", device_get_nameunit(vi->dev), i); init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name); rc = alloc_rxq(vi, rxq, forwarding_intr_to_fwq(sc) ? -1 : intr_idx, i, oid); if (rc != 0) goto done; intr_idx++; } #ifdef DEV_NETMAP if (ifp->if_capabilities & IFCAP_NETMAP) intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq); #endif #ifdef TCP_OFFLOAD maxp = mtu_to_max_payload(sc, mtu, 1); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq", CTLFLAG_RD, NULL, "rx queues for offloaded TCP connections"); for_each_ofld_rxq(vi, i, ofld_rxq) { init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx, vi->qsize_rxq); snprintf(name, sizeof(name), "%s ofld_rxq%d-fl", device_get_nameunit(vi->dev), i); init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name); rc = alloc_ofld_rxq(vi, ofld_rxq, forwarding_intr_to_fwq(sc) ? -1 : intr_idx, i, oid); if (rc != 0) goto done; intr_idx++; } #endif /* * Now the tx queues. */ oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD, NULL, "tx queues"); for_each_txq(vi, i, txq) { iqidx = vi->first_rxq + (i % vi->nrxq); snprintf(name, sizeof(name), "%s txq%d", device_get_nameunit(vi->dev), i); init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan, sc->sge.rxq[iqidx].iq.cntxt_id, name); rc = alloc_txq(vi, txq, i, oid); if (rc != 0) goto done; } #ifdef TCP_OFFLOAD oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq", CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections"); for_each_ofld_txq(vi, i, ofld_txq) { struct sysctl_oid *oid2; iqidx = vi->first_ofld_rxq + (i % vi->nofldrxq); snprintf(name, sizeof(name), "%s ofld_txq%d", device_get_nameunit(vi->dev), i); init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan, sc->sge.ofld_rxq[iqidx].iq.cntxt_id, name); snprintf(name, sizeof(name), "%d", i); oid2 = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL, "offload tx queue"); rc = alloc_wrq(sc, vi, ofld_txq, oid2); if (rc != 0) goto done; } #endif /* * Finally, the control queue. */ if (!IS_MAIN_VI(vi) || sc->flags & IS_VF) goto done; oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD, NULL, "ctrl queue"); ctrlq = &sc->sge.ctrlq[pi->port_id]; snprintf(name, sizeof(name), "%s ctrlq", device_get_nameunit(vi->dev)); init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, sc->sge.rxq[vi->first_rxq].iq.cntxt_id, name); rc = alloc_wrq(sc, vi, ctrlq, oid); done: if (rc) t4_teardown_vi_queues(vi); return (rc); } /* * Idempotent */ int t4_teardown_vi_queues(struct vi_info *vi) { int i; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct sge_rxq *rxq; struct sge_txq *txq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; struct sge_wrq *ofld_txq; #endif #ifdef DEV_NETMAP struct sge_nm_rxq *nm_rxq; struct sge_nm_txq *nm_txq; #endif /* Do this before freeing the queues */ if (vi->flags & VI_SYSCTL_CTX) { sysctl_ctx_free(&vi->ctx); vi->flags &= ~VI_SYSCTL_CTX; } #ifdef DEV_NETMAP if (vi->ifp->if_capabilities & IFCAP_NETMAP) { for_each_nm_txq(vi, i, nm_txq) { free_nm_txq(vi, nm_txq); } for_each_nm_rxq(vi, i, nm_rxq) { free_nm_rxq(vi, nm_rxq); } } #endif /* * Take down all the tx queues first, as they reference the rx queues * (for egress updates, etc.). */ if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF)) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]); for_each_txq(vi, i, txq) { free_txq(vi, txq); } #ifdef TCP_OFFLOAD for_each_ofld_txq(vi, i, ofld_txq) { free_wrq(sc, ofld_txq); } #endif /* * Then take down the rx queues. */ for_each_rxq(vi, i, rxq) { free_rxq(vi, rxq); } #ifdef TCP_OFFLOAD for_each_ofld_rxq(vi, i, ofld_rxq) { free_ofld_rxq(vi, ofld_rxq); } #endif return (0); } /* * Deals with errors and the firmware event queue. All data rx queues forward * their interrupt to the firmware event queue. */ void t4_intr_all(void *arg) { struct adapter *sc = arg; struct sge_iq *fwq = &sc->sge.fwq; t4_intr_err(arg); if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) { service_iq(fwq, 0); atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE); } } /* Deals with error interrupts */ void t4_intr_err(void *arg) { struct adapter *sc = arg; t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); t4_slow_intr_handler(sc); } void t4_intr_evt(void *arg) { struct sge_iq *iq = arg; if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { service_iq(iq, 0); atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); } } void t4_intr(void *arg) { struct sge_iq *iq = arg; if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { service_iq(iq, 0); atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); } } void t4_vi_intr(void *arg) { struct irq *irq = arg; #ifdef DEV_NETMAP if (atomic_cmpset_int(&irq->nm_state, NM_ON, NM_BUSY)) { t4_nm_intr(irq->nm_rxq); atomic_cmpset_int(&irq->nm_state, NM_BUSY, NM_ON); } #endif if (irq->rxq != NULL) t4_intr(irq->rxq); } static inline int sort_before_lro(struct lro_ctrl *lro) { return (lro->lro_mbuf_max != 0); } /* * Deals with anything and everything on the given ingress queue. */ static int service_iq(struct sge_iq *iq, int budget) { struct sge_iq *q; struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ struct sge_fl *fl; /* Use iff IQ_HAS_FL */ struct adapter *sc = iq->adapter; struct iq_desc *d = &iq->desc[iq->cidx]; int ndescs = 0, limit; int rsp_type, refill; uint32_t lq; uint16_t fl_hw_cidx; struct mbuf *m0; STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); #if defined(INET) || defined(INET6) const struct timeval lro_timeout = {0, sc->lro_timeout}; struct lro_ctrl *lro = &rxq->lro; #endif KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); limit = budget ? budget : iq->qsize / 16; if (iq->flags & IQ_HAS_FL) { fl = &rxq->fl; fl_hw_cidx = fl->hw_cidx; /* stable snapshot */ } else { fl = NULL; fl_hw_cidx = 0; /* to silence gcc warning */ } #if defined(INET) || defined(INET6) if (iq->flags & IQ_ADJ_CREDIT) { MPASS(sort_before_lro(lro)); iq->flags &= ~IQ_ADJ_CREDIT; if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) { tcp_lro_flush_all(lro); t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) | V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); return (0); } ndescs = 1; } #else MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); #endif /* * We always come back and check the descriptor ring for new indirect * interrupts and other responses after running a single handler. */ for (;;) { while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { rmb(); refill = 0; m0 = NULL; rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); lq = be32toh(d->rsp.pldbuflen_qid); switch (rsp_type) { case X_RSPD_TYPE_FLBUF: KASSERT(iq->flags & IQ_HAS_FL, ("%s: data for an iq (%p) with no freelist", __func__, iq)); m0 = get_fl_payload(sc, fl, lq); if (__predict_false(m0 == NULL)) goto process_iql; refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2; #ifdef T4_PKT_TIMESTAMP /* * 60 bit timestamp for the payload is * *(uint64_t *)m0->m_pktdat. Note that it is * in the leading free-space in the mbuf. The * kernel can clobber it during a pullup, * m_copymdata, etc. You need to make sure that * the mbuf reaches you unmolested if you care * about the timestamp. */ *(uint64_t *)m0->m_pktdat = be64toh(ctrl->u.last_flit) & 0xfffffffffffffff; #endif /* fall through */ case X_RSPD_TYPE_CPL: KASSERT(d->rss.opcode < NUM_CPL_CMDS, ("%s: bad opcode %02x.", __func__, d->rss.opcode)); t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0); break; case X_RSPD_TYPE_INTR: /* * Interrupts should be forwarded only to queues * that are not forwarding their interrupts. * This means service_iq can recurse but only 1 * level deep. */ KASSERT(budget == 0, ("%s: budget %u, rsp_type %u", __func__, budget, rsp_type)); /* * There are 1K interrupt-capable queues (qids 0 * through 1023). A response type indicating a * forwarded interrupt with a qid >= 1K is an * iWARP async notification. */ if (lq >= 1024) { t4_an_handler(iq, &d->rsp); break; } q = sc->sge.iqmap[lq - sc->sge.iq_start - sc->sge.iq_base]; if (atomic_cmpset_int(&q->state, IQS_IDLE, IQS_BUSY)) { if (service_iq(q, q->qsize / 16) == 0) { atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); } else { STAILQ_INSERT_TAIL(&iql, q, link); } } break; default: KASSERT(0, ("%s: illegal response type %d on iq %p", __func__, rsp_type, iq)); log(LOG_ERR, "%s: illegal response type %d on iq %p", device_get_nameunit(sc->dev), rsp_type, iq); break; } d++; if (__predict_false(++iq->cidx == iq->sidx)) { iq->cidx = 0; iq->gen ^= F_RSPD_GEN; d = &iq->desc[0]; } if (__predict_false(++ndescs == limit)) { t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); ndescs = 0; #if defined(INET) || defined(INET6) if (iq->flags & IQ_LRO_ENABLED && !sort_before_lro(lro) && sc->lro_timeout != 0) { tcp_lro_flush_inactive(lro, &lro_timeout); } #endif if (budget) { if (iq->flags & IQ_HAS_FL) { FL_LOCK(fl); refill_fl(sc, fl, 32); FL_UNLOCK(fl); } return (EINPROGRESS); } } if (refill) { FL_LOCK(fl); refill_fl(sc, fl, 32); FL_UNLOCK(fl); fl_hw_cidx = fl->hw_cidx; } } process_iql: if (STAILQ_EMPTY(&iql)) break; /* * Process the head only, and send it to the back of the list if * it's still not done. */ q = STAILQ_FIRST(&iql); STAILQ_REMOVE_HEAD(&iql, link); if (service_iq(q, q->qsize / 8) == 0) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); else STAILQ_INSERT_TAIL(&iql, q, link); } #if defined(INET) || defined(INET6) if (iq->flags & IQ_LRO_ENABLED) { if (ndescs > 0 && lro->lro_mbuf_count > 8) { MPASS(sort_before_lro(lro)); /* hold back one credit and don't flush LRO state */ iq->flags |= IQ_ADJ_CREDIT; ndescs--; } else { tcp_lro_flush_all(lro); } } #endif t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); if (iq->flags & IQ_HAS_FL) { int starved; FL_LOCK(fl); starved = refill_fl(sc, fl, 64); FL_UNLOCK(fl); if (__predict_false(starved != 0)) add_fl_to_sfl(sc, fl); } return (0); } static inline int cl_has_metadata(struct sge_fl *fl, struct cluster_layout *cll) { int rc = fl->flags & FL_BUF_PACKING || cll->region1 > 0; if (rc) MPASS(cll->region3 >= CL_METADATA_SIZE); return (rc); } static inline struct cluster_metadata * cl_metadata(struct adapter *sc, struct sge_fl *fl, struct cluster_layout *cll, caddr_t cl) { if (cl_has_metadata(fl, cll)) { struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx]; return ((struct cluster_metadata *)(cl + swz->size) - 1); } return (NULL); } static void rxb_free(struct mbuf *m, void *arg1, void *arg2) { uma_zone_t zone = arg1; caddr_t cl = arg2; uma_zfree(zone, cl); counter_u64_add(extfree_rels, 1); } /* * The mbuf returned by this function could be allocated from zone_mbuf or * constructed in spare room in the cluster. * * The mbuf carries the payload in one of these ways * a) frame inside the mbuf (mbuf from zone_mbuf) * b) m_cljset (for clusters without metadata) zone_mbuf * c) m_extaddref (cluster with metadata) inline mbuf * d) m_extaddref (cluster with metadata) zone_mbuf */ static struct mbuf * get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, int remaining) { struct mbuf *m; struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; struct cluster_layout *cll = &sd->cll; struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx]; struct hw_buf_info *hwb = &sc->sge.hw_buf_info[cll->hwidx]; struct cluster_metadata *clm = cl_metadata(sc, fl, cll, sd->cl); int len, blen; caddr_t payload; blen = hwb->size - fl->rx_offset; /* max possible in this buf */ len = min(remaining, blen); payload = sd->cl + cll->region1 + fl->rx_offset; if (fl->flags & FL_BUF_PACKING) { const u_int l = fr_offset + len; const u_int pad = roundup2(l, fl->buf_boundary) - l; if (fl->rx_offset + len + pad < hwb->size) blen = len + pad; MPASS(fl->rx_offset + blen <= hwb->size); } else { MPASS(fl->rx_offset == 0); /* not packing */ } if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) { /* * Copy payload into a freshly allocated mbuf. */ m = fr_offset == 0 ? m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA); if (m == NULL) return (NULL); fl->mbuf_allocated++; #ifdef T4_PKT_TIMESTAMP /* Leave room for a timestamp */ m->m_data += 8; #endif /* copy data to mbuf */ bcopy(payload, mtod(m, caddr_t), len); } else if (sd->nmbuf * MSIZE < cll->region1) { /* * There's spare room in the cluster for an mbuf. Create one * and associate it with the payload that's in the cluster. */ MPASS(clm != NULL); m = (struct mbuf *)(sd->cl + sd->nmbuf * MSIZE); /* No bzero required */ if (m_init(m, M_NOWAIT, MT_DATA, fr_offset == 0 ? M_PKTHDR | M_NOFREE : M_NOFREE)) return (NULL); fl->mbuf_inlined++; m_extaddref(m, payload, blen, &clm->refcount, rxb_free, swz->zone, sd->cl); if (sd->nmbuf++ == 0) counter_u64_add(extfree_refs, 1); } else { /* * Grab an mbuf from zone_mbuf and associate it with the * payload in the cluster. */ m = fr_offset == 0 ? m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA); if (m == NULL) return (NULL); fl->mbuf_allocated++; if (clm != NULL) { m_extaddref(m, payload, blen, &clm->refcount, rxb_free, swz->zone, sd->cl); if (sd->nmbuf++ == 0) counter_u64_add(extfree_refs, 1); } else { m_cljset(m, sd->cl, swz->type); sd->cl = NULL; /* consumed, not a recycle candidate */ } } if (fr_offset == 0) m->m_pkthdr.len = remaining; m->m_len = len; if (fl->flags & FL_BUF_PACKING) { fl->rx_offset += blen; MPASS(fl->rx_offset <= hwb->size); if (fl->rx_offset < hwb->size) return (m); /* without advancing the cidx */ } if (__predict_false(++fl->cidx % 8 == 0)) { uint16_t cidx = fl->cidx / 8; if (__predict_false(cidx == fl->sidx)) fl->cidx = cidx = 0; fl->hw_cidx = cidx; } fl->rx_offset = 0; return (m); } static struct mbuf * get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf) { struct mbuf *m0, *m, **pnext; u_int remaining; const u_int total = G_RSPD_LEN(len_newbuf); if (__predict_false(fl->flags & FL_BUF_RESUME)) { M_ASSERTPKTHDR(fl->m0); MPASS(fl->m0->m_pkthdr.len == total); MPASS(fl->remaining < total); m0 = fl->m0; pnext = fl->pnext; remaining = fl->remaining; fl->flags &= ~FL_BUF_RESUME; goto get_segment; } if (fl->rx_offset > 0 && len_newbuf & F_RSPD_NEWBUF) { fl->rx_offset = 0; if (__predict_false(++fl->cidx % 8 == 0)) { uint16_t cidx = fl->cidx / 8; if (__predict_false(cidx == fl->sidx)) fl->cidx = cidx = 0; fl->hw_cidx = cidx; } } /* * Payload starts at rx_offset in the current hw buffer. Its length is * 'len' and it may span multiple hw buffers. */ m0 = get_scatter_segment(sc, fl, 0, total); if (m0 == NULL) return (NULL); remaining = total - m0->m_len; pnext = &m0->m_next; while (remaining > 0) { get_segment: MPASS(fl->rx_offset == 0); m = get_scatter_segment(sc, fl, total - remaining, remaining); if (__predict_false(m == NULL)) { fl->m0 = m0; fl->pnext = pnext; fl->remaining = remaining; fl->flags |= FL_BUF_RESUME; return (NULL); } *pnext = m; pnext = &m->m_next; remaining -= m->m_len; } *pnext = NULL; M_ASSERTPKTHDR(m0); return (m0); } static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0) { struct sge_rxq *rxq = iq_to_rxq(iq); struct ifnet *ifp = rxq->ifp; struct adapter *sc = iq->adapter; const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); #if defined(INET) || defined(INET6) struct lro_ctrl *lro = &rxq->lro; #endif static const int sw_hashtype[4][2] = { {M_HASHTYPE_NONE, M_HASHTYPE_NONE}, {M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6}, {M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6}, {M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6}, }; KASSERT(m0 != NULL, ("%s: no payload with opcode %02x", __func__, rss->opcode)); m0->m_pkthdr.len -= sc->params.sge.fl_pktshift; m0->m_len -= sc->params.sge.fl_pktshift; m0->m_data += sc->params.sge.fl_pktshift; m0->m_pkthdr.rcvif = ifp; M_HASHTYPE_SET(m0, sw_hashtype[rss->hash_type][rss->ipv6]); m0->m_pkthdr.flowid = be32toh(rss->hash_val); if (cpl->csum_calc && !(cpl->err_vec & sc->params.tp.err_vec_mask)) { if (ifp->if_capenable & IFCAP_RXCSUM && cpl->l2info & htobe32(F_RXF_IP)) { m0->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR); rxq->rxcsum++; } else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 && cpl->l2info & htobe32(F_RXF_IP6)) { m0->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 | CSUM_PSEUDO_HDR); rxq->rxcsum++; } if (__predict_false(cpl->ip_frag)) m0->m_pkthdr.csum_data = be16toh(cpl->csum); else m0->m_pkthdr.csum_data = 0xffff; } if (cpl->vlan_ex) { m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan); m0->m_flags |= M_VLANTAG; rxq->vlan_extraction++; } #if defined(INET) || defined(INET6) if (iq->flags & IQ_LRO_ENABLED) { if (sort_before_lro(lro)) { tcp_lro_queue_mbuf(lro, m0); return (0); /* queued for sort, then LRO */ } if (tcp_lro_rx(lro, m0, 0) == 0) return (0); /* queued for LRO */ } #endif ifp->if_input(ifp, m0); return (0); } /* * Must drain the wrq or make sure that someone else will. */ static void wrq_tx_drain(void *arg, int n) { struct sge_wrq *wrq = arg; struct sge_eq *eq = &wrq->eq; EQ_LOCK(eq); if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) drain_wrq_wr_list(wrq->adapter, wrq); EQ_UNLOCK(eq); } static void drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq) { struct sge_eq *eq = &wrq->eq; u_int available, dbdiff; /* # of hardware descriptors */ u_int n; struct wrqe *wr; struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ EQ_LOCK_ASSERT_OWNED(eq); MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs)); wr = STAILQ_FIRST(&wrq->wr_list); MPASS(wr != NULL); /* Must be called with something useful to do */ MPASS(eq->pidx == eq->dbidx); dbdiff = 0; do { eq->cidx = read_hw_cidx(eq); if (eq->pidx == eq->cidx) available = eq->sidx - 1; else available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; MPASS(wr->wrq == wrq); n = howmany(wr->wr_len, EQ_ESIZE); if (available < n) break; dst = (void *)&eq->desc[eq->pidx]; if (__predict_true(eq->sidx - eq->pidx > n)) { /* Won't wrap, won't end exactly at the status page. */ bcopy(&wr->wr[0], dst, wr->wr_len); eq->pidx += n; } else { int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE; bcopy(&wr->wr[0], dst, first_portion); if (wr->wr_len > first_portion) { bcopy(&wr->wr[first_portion], &eq->desc[0], wr->wr_len - first_portion); } eq->pidx = n - (eq->sidx - eq->pidx); } wrq->tx_wrs_copied++; if (available < eq->sidx / 4 && atomic_cmpset_int(&eq->equiq, 0, 1)) { dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | F_FW_WR_EQUEQ); eq->equeqidx = eq->pidx; } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); eq->equeqidx = eq->pidx; } dbdiff += n; if (dbdiff >= 16) { ring_eq_db(sc, eq, dbdiff); dbdiff = 0; } STAILQ_REMOVE_HEAD(&wrq->wr_list, link); free_wrqe(wr); MPASS(wrq->nwr_pending > 0); wrq->nwr_pending--; MPASS(wrq->ndesc_needed >= n); wrq->ndesc_needed -= n; } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL); if (dbdiff) ring_eq_db(sc, eq, dbdiff); } /* * Doesn't fail. Holds on to work requests it can't send right away. */ void t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr) { #ifdef INVARIANTS struct sge_eq *eq = &wrq->eq; #endif EQ_LOCK_ASSERT_OWNED(eq); MPASS(wr != NULL); MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN); MPASS((wr->wr_len & 0x7) == 0); STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link); wrq->nwr_pending++; wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE); if (!TAILQ_EMPTY(&wrq->incomplete_wrs)) return; /* commit_wrq_wr will drain wr_list as well. */ drain_wrq_wr_list(sc, wrq); /* Doorbell must have caught up to the pidx. */ MPASS(eq->pidx == eq->dbidx); } void t4_update_fl_bufsize(struct ifnet *ifp) { struct vi_info *vi = ifp->if_softc; struct adapter *sc = vi->pi->adapter; struct sge_rxq *rxq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; #endif struct sge_fl *fl; int i, maxp, mtu = ifp->if_mtu; maxp = mtu_to_max_payload(sc, mtu, 0); for_each_rxq(vi, i, rxq) { fl = &rxq->fl; FL_LOCK(fl); find_best_refill_source(sc, fl, maxp); FL_UNLOCK(fl); } #ifdef TCP_OFFLOAD maxp = mtu_to_max_payload(sc, mtu, 1); for_each_ofld_rxq(vi, i, ofld_rxq) { fl = &ofld_rxq->fl; FL_LOCK(fl); find_best_refill_source(sc, fl, maxp); FL_UNLOCK(fl); } #endif } static inline int mbuf_nsegs(struct mbuf *m) { M_ASSERTPKTHDR(m); KASSERT(m->m_pkthdr.l5hlen > 0, ("%s: mbuf %p missing information on # of segments.", __func__, m)); return (m->m_pkthdr.l5hlen); } static inline void set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs) { M_ASSERTPKTHDR(m); m->m_pkthdr.l5hlen = nsegs; } static inline int mbuf_len16(struct mbuf *m) { int n; M_ASSERTPKTHDR(m); n = m->m_pkthdr.PH_loc.eight[0]; MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); return (n); } static inline void set_mbuf_len16(struct mbuf *m, uint8_t len16) { M_ASSERTPKTHDR(m); m->m_pkthdr.PH_loc.eight[0] = len16; } static inline int needs_tso(struct mbuf *m) { M_ASSERTPKTHDR(m); return (m->m_pkthdr.csum_flags & CSUM_TSO); } static inline int needs_l3_csum(struct mbuf *m) { M_ASSERTPKTHDR(m); return (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)); } static inline int needs_l4_csum(struct mbuf *m) { M_ASSERTPKTHDR(m); return (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)); } static inline int needs_vlan_insertion(struct mbuf *m) { M_ASSERTPKTHDR(m); return (m->m_flags & M_VLANTAG); } static void * m_advance(struct mbuf **pm, int *poffset, int len) { struct mbuf *m = *pm; int offset = *poffset; uintptr_t p = 0; MPASS(len > 0); for (;;) { if (offset + len < m->m_len) { offset += len; p = mtod(m, uintptr_t) + offset; break; } len -= m->m_len - offset; m = m->m_next; offset = 0; MPASS(m != NULL); } *poffset = offset; *pm = m; return ((void *)p); } /* * Can deal with empty mbufs in the chain that have m_len = 0, but the chain * must have at least one mbuf that's not empty. */ static inline int count_mbuf_nsegs(struct mbuf *m) { vm_paddr_t lastb, next; vm_offset_t va; int len, nsegs; MPASS(m != NULL); nsegs = 0; lastb = 0; for (; m; m = m->m_next) { len = m->m_len; if (__predict_false(len == 0)) continue; va = mtod(m, vm_offset_t); next = pmap_kextract(va); nsegs += sglist_count(m->m_data, len); if (lastb + 1 == next) nsegs--; lastb = pmap_kextract(va + len - 1); } MPASS(nsegs > 0); return (nsegs); } /* * Analyze the mbuf to determine its tx needs. The mbuf passed in may change: * a) caller can assume it's been freed if this function returns with an error. * b) it may get defragged up if the gather list is too long for the hardware. */ int parse_pkt(struct adapter *sc, struct mbuf **mp) { struct mbuf *m0 = *mp, *m; int rc, nsegs, defragged = 0, offset; struct ether_header *eh; void *l3hdr; #if defined(INET) || defined(INET6) struct tcphdr *tcp; #endif uint16_t eh_type; M_ASSERTPKTHDR(m0); if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { rc = EINVAL; fail: m_freem(m0); *mp = NULL; return (rc); } restart: /* * First count the number of gather list segments in the payload. * Defrag the mbuf if nsegs exceeds the hardware limit. */ M_ASSERTPKTHDR(m0); MPASS(m0->m_pkthdr.len > 0); nsegs = count_mbuf_nsegs(m0); if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { rc = EFBIG; goto fail; } *mp = m0 = m; /* update caller's copy after defrag */ goto restart; } if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN)) { m0 = m_pullup(m0, m0->m_pkthdr.len); if (m0 == NULL) { /* Should have left well enough alone. */ rc = EFBIG; goto fail; } *mp = m0; /* update caller's copy after pullup */ goto restart; } set_mbuf_nsegs(m0, nsegs); if (sc->flags & IS_VF) set_mbuf_len16(m0, txpkt_vm_len16(nsegs, needs_tso(m0))); else set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0))); if (!needs_tso(m0) && !(sc->flags & IS_VF && (needs_l3_csum(m0) || needs_l4_csum(m0)))) return (0); m = m0; eh = mtod(m, struct ether_header *); eh_type = ntohs(eh->ether_type); if (eh_type == ETHERTYPE_VLAN) { struct ether_vlan_header *evh = (void *)eh; eh_type = ntohs(evh->evl_proto); m0->m_pkthdr.l2hlen = sizeof(*evh); } else m0->m_pkthdr.l2hlen = sizeof(*eh); offset = 0; l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen); switch (eh_type) { #ifdef INET6 case ETHERTYPE_IPV6: { struct ip6_hdr *ip6 = l3hdr; MPASS(!needs_tso(m0) || ip6->ip6_nxt == IPPROTO_TCP); m0->m_pkthdr.l3hlen = sizeof(*ip6); break; } #endif #ifdef INET case ETHERTYPE_IP: { struct ip *ip = l3hdr; m0->m_pkthdr.l3hlen = ip->ip_hl * 4; break; } #endif default: panic("%s: ethertype 0x%04x unknown. if_cxgbe must be compiled" " with the same INET/INET6 options as the kernel.", __func__, eh_type); } #if defined(INET) || defined(INET6) if (needs_tso(m0)) { tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen); m0->m_pkthdr.l4hlen = tcp->th_off * 4; } #endif MPASS(m0 == *mp); return (0); } void * start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie) { struct sge_eq *eq = &wrq->eq; struct adapter *sc = wrq->adapter; int ndesc, available; struct wrqe *wr; void *w; MPASS(len16 > 0); ndesc = howmany(len16, EQ_ESIZE / 16); MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC); EQ_LOCK(eq); if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) drain_wrq_wr_list(sc, wrq); if (!STAILQ_EMPTY(&wrq->wr_list)) { slowpath: EQ_UNLOCK(eq); wr = alloc_wrqe(len16 * 16, wrq); if (__predict_false(wr == NULL)) return (NULL); cookie->pidx = -1; cookie->ndesc = ndesc; return (&wr->wr); } eq->cidx = read_hw_cidx(eq); if (eq->pidx == eq->cidx) available = eq->sidx - 1; else available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; if (available < ndesc) goto slowpath; cookie->pidx = eq->pidx; cookie->ndesc = ndesc; TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link); w = &eq->desc[eq->pidx]; IDXINCR(eq->pidx, ndesc, eq->sidx); if (__predict_false(cookie->pidx + ndesc > eq->sidx)) { w = &wrq->ss[0]; wrq->ss_pidx = cookie->pidx; wrq->ss_len = len16 * 16; } EQ_UNLOCK(eq); return (w); } void commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie) { struct sge_eq *eq = &wrq->eq; struct adapter *sc = wrq->adapter; int ndesc, pidx; struct wrq_cookie *prev, *next; if (cookie->pidx == -1) { struct wrqe *wr = __containerof(w, struct wrqe, wr); t4_wrq_tx(sc, wr); return; } if (__predict_false(w == &wrq->ss[0])) { int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE; MPASS(wrq->ss_len > n); /* WR had better wrap around. */ bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n); bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n); wrq->tx_wrs_ss++; } else wrq->tx_wrs_direct++; EQ_LOCK(eq); ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */ pidx = cookie->pidx; MPASS(pidx >= 0 && pidx < eq->sidx); prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link); next = TAILQ_NEXT(cookie, link); if (prev == NULL) { MPASS(pidx == eq->dbidx); if (next == NULL || ndesc >= 16) { int available; struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ /* * Note that the WR via which we'll request tx updates * is at pidx and not eq->pidx, which has moved on * already. */ dst = (void *)&eq->desc[pidx]; available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; if (available < eq->sidx / 4 && atomic_cmpset_int(&eq->equiq, 0, 1)) { dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | F_FW_WR_EQUEQ); eq->equeqidx = pidx; } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); eq->equeqidx = pidx; } ring_eq_db(wrq->adapter, eq, ndesc); } else { MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc); next->pidx = pidx; next->ndesc += ndesc; } } else { MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc); prev->ndesc += ndesc; } TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link); if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) drain_wrq_wr_list(sc, wrq); #ifdef INVARIANTS if (TAILQ_EMPTY(&wrq->incomplete_wrs)) { /* Doorbell must have caught up to the pidx. */ MPASS(wrq->eq.pidx == wrq->eq.dbidx); } #endif EQ_UNLOCK(eq); } static u_int can_resume_eth_tx(struct mp_ring *r) { struct sge_eq *eq = r->cookie; return (total_available_tx_desc(eq) > eq->sidx / 8); } static inline int cannot_use_txpkts(struct mbuf *m) { /* maybe put a GL limit too, to avoid silliness? */ return (needs_tso(m)); } static inline int discard_tx(struct sge_eq *eq) { return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED); } /* * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to * be consumed. Return the actual number consumed. 0 indicates a stall. */ static u_int eth_tx(struct mp_ring *r, u_int cidx, u_int pidx) { struct sge_txq *txq = r->cookie; struct sge_eq *eq = &txq->eq; struct ifnet *ifp = txq->ifp; struct vi_info *vi = ifp->if_softc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; u_int total, remaining; /* # of packets */ u_int available, dbdiff; /* # of hardware descriptors */ u_int n, next_cidx; struct mbuf *m0, *tail; struct txpkts txp; struct fw_eth_tx_pkts_wr *wr; /* any fw WR struct will do */ remaining = IDXDIFF(pidx, cidx, r->size); MPASS(remaining > 0); /* Must not be called without work to do. */ total = 0; TXQ_LOCK(txq); if (__predict_false(discard_tx(eq))) { while (cidx != pidx) { m0 = r->items[cidx]; m_freem(m0); if (++cidx == r->size) cidx = 0; } reclaim_tx_descs(txq, 2048); total = remaining; goto done; } /* How many hardware descriptors do we have readily available. */ if (eq->pidx == eq->cidx) available = eq->sidx - 1; else available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx); while (remaining > 0) { m0 = r->items[cidx]; M_ASSERTPKTHDR(m0); MPASS(m0->m_nextpkt == NULL); if (available < SGE_MAX_WR_NDESC) { available += reclaim_tx_descs(txq, 64); if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16)) break; /* out of descriptors */ } next_cidx = cidx + 1; if (__predict_false(next_cidx == r->size)) next_cidx = 0; wr = (void *)&eq->desc[eq->pidx]; if (sc->flags & IS_VF) { total++; remaining--; ETHER_BPF_MTAP(ifp, m0); n = write_txpkt_vm_wr(sc, txq, (void *)wr, m0, available); } else if (remaining > 1 && try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) { /* pkts at cidx, next_cidx should both be in txp. */ MPASS(txp.npkt == 2); tail = r->items[next_cidx]; MPASS(tail->m_nextpkt == NULL); ETHER_BPF_MTAP(ifp, m0); ETHER_BPF_MTAP(ifp, tail); m0->m_nextpkt = tail; if (__predict_false(++next_cidx == r->size)) next_cidx = 0; while (next_cidx != pidx) { if (add_to_txpkts(r->items[next_cidx], &txp, available) != 0) break; tail->m_nextpkt = r->items[next_cidx]; tail = tail->m_nextpkt; ETHER_BPF_MTAP(ifp, tail); if (__predict_false(++next_cidx == r->size)) next_cidx = 0; } n = write_txpkts_wr(txq, wr, m0, &txp, available); total += txp.npkt; remaining -= txp.npkt; } else { total++; remaining--; ETHER_BPF_MTAP(ifp, m0); n = write_txpkt_wr(txq, (void *)wr, m0, available); } MPASS(n >= 1 && n <= available && n <= SGE_MAX_WR_NDESC); available -= n; dbdiff += n; IDXINCR(eq->pidx, n, eq->sidx); if (total_available_tx_desc(eq) < eq->sidx / 4 && atomic_cmpset_int(&eq->equiq, 0, 1)) { wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | F_FW_WR_EQUEQ); eq->equeqidx = eq->pidx; } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); eq->equeqidx = eq->pidx; } if (dbdiff >= 16 && remaining >= 4) { ring_eq_db(sc, eq, dbdiff); available += reclaim_tx_descs(txq, 4 * dbdiff); dbdiff = 0; } cidx = next_cidx; } if (dbdiff != 0) { ring_eq_db(sc, eq, dbdiff); reclaim_tx_descs(txq, 32); } done: TXQ_UNLOCK(txq); return (total); } static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, int qsize) { KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS, ("%s: bad tmr_idx %d", __func__, tmr_idx)); KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */ ("%s: bad pktc_idx %d", __func__, pktc_idx)); iq->flags = 0; iq->adapter = sc; iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); iq->intr_pktc_idx = SGE_NCOUNTERS - 1; if (pktc_idx >= 0) { iq->intr_params |= F_QINTR_CNT_EN; iq->intr_pktc_idx = pktc_idx; } iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */ iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE; } static inline void init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name) { fl->qsize = qsize; fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; strlcpy(fl->lockname, name, sizeof(fl->lockname)); if (sc->flags & BUF_PACKING_OK && ((!is_t4(sc) && buffer_packing) || /* T5+: enabled unless 0 */ (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */ fl->flags |= FL_BUF_PACKING; find_best_refill_source(sc, fl, maxp); find_safe_refill_source(sc, fl); } static inline void init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize, uint8_t tx_chan, uint16_t iqid, char *name) { KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype)); eq->flags = eqtype & EQ_TYPEMASK; eq->tx_chan = tx_chan; eq->iqid = iqid; eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; strlcpy(eq->lockname, name, sizeof(eq->lockname)); } static int alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, bus_dmamap_t *map, bus_addr_t *pa, void **va) { int rc; rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag); if (rc != 0) { device_printf(sc->dev, "cannot allocate DMA tag: %d\n", rc); goto done; } rc = bus_dmamem_alloc(*tag, va, BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map); if (rc != 0) { device_printf(sc->dev, "cannot allocate DMA memory: %d\n", rc); goto done; } rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0); if (rc != 0) { device_printf(sc->dev, "cannot load DMA map: %d\n", rc); goto done; } done: if (rc) free_ring(sc, *tag, *map, *pa, *va); return (rc); } static int free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map, bus_addr_t pa, void *va) { if (pa) bus_dmamap_unload(tag, map); if (va) bus_dmamem_free(tag, va, map); if (tag) bus_dma_tag_destroy(tag); return (0); } /* * Allocates the ring for an ingress queue and an optional freelist. If the * freelist is specified it will be allocated and then associated with the * ingress queue. * * Returns errno on failure. Resources allocated up to that point may still be * allocated. Caller is responsible for cleanup in case this function fails. * * If the ingress queue will take interrupts directly then the intr_idx * specifies the vector, starting from 0. -1 means the interrupts for this * queue should be forwarded to the fwq. */ static int alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl, int intr_idx, int cong) { int rc, i, cntxt_id; size_t len; struct fw_iq_cmd c; struct port_info *pi = vi->pi; struct adapter *sc = iq->adapter; struct sge_params *sp = &sc->params.sge; __be32 v = 0; len = iq->qsize * IQ_ESIZE; rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba, (void **)&iq->desc); if (rc != 0) return (rc); bzero(&c, sizeof(c)); c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | V_FW_IQ_CMD_VFN(0)); c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | FW_LEN16(c)); /* Special handling for firmware event queue */ if (iq == &sc->sge.fwq) v |= F_FW_IQ_CMD_IQASYNCH; if (intr_idx < 0) { /* Forwarded interrupts, all headed to fwq */ v |= F_FW_IQ_CMD_IQANDST; v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.fwq.cntxt_id); } else { KASSERT(intr_idx < sc->intr_count, ("%s: invalid direct intr_idx %d", __func__, intr_idx)); v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); } c.type_to_iqandstindex = htobe32(v | V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | V_FW_IQ_CMD_VIID(vi->viid) | V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | F_FW_IQ_CMD_IQGTSMODE | V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4)); c.iqsize = htobe16(iq->qsize); c.iqaddr = htobe64(iq->ba); if (cong >= 0) c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN); if (fl) { mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF); len = fl->qsize * EQ_ESIZE; rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map, &fl->ba, (void **)&fl->desc); if (rc) return (rc); /* Allocate space for one software descriptor per buffer. */ rc = alloc_fl_sdesc(fl); if (rc != 0) { device_printf(sc->dev, "failed to setup fl software descriptors: %d\n", rc); return (rc); } if (fl->flags & FL_BUF_PACKING) { fl->lowat = roundup2(sp->fl_starve_threshold2, 8); fl->buf_boundary = sp->pack_boundary; } else { fl->lowat = roundup2(sp->fl_starve_threshold, 8); fl->buf_boundary = 16; } if (fl_pad && fl->buf_boundary < sp->pad_boundary) fl->buf_boundary = sp->pad_boundary; c.iqns_to_fl0congen |= htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO | (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) | (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN : 0)); if (cong >= 0) { c.iqns_to_fl0congen |= htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | F_FW_IQ_CMD_FL0CONGCIF | F_FW_IQ_CMD_FL0CONGEN); } c.fl0dcaen_to_fl0cidxfthresh = htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ? X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) | V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ? X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); c.fl0size = htobe16(fl->qsize); c.fl0addr = htobe64(fl->ba); } rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { device_printf(sc->dev, "failed to create ingress queue: %d\n", rc); return (rc); } iq->cidx = 0; iq->gen = F_RSPD_GEN; iq->intr_next = iq->intr_params; iq->cntxt_id = be16toh(c.iqid); iq->abs_id = be16toh(c.physiqid); iq->flags |= IQ_ALLOCATED; cntxt_id = iq->cntxt_id - sc->sge.iq_start; if (cntxt_id >= sc->sge.niq) { panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.niq - 1); } sc->sge.iqmap[cntxt_id] = iq; if (fl) { u_int qid; iq->flags |= IQ_HAS_FL; fl->cntxt_id = be16toh(c.fl0id); fl->pidx = fl->cidx = 0; cntxt_id = fl->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) { panic("%s: fl->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.neq - 1); } sc->sge.eqmap[cntxt_id] = (void *)fl; qid = fl->cntxt_id; if (isset(&sc->doorbells, DOORBELL_UDB)) { uint32_t s_qpp = sc->params.sge.eq_s_qpp; uint32_t mask = (1 << s_qpp) - 1; volatile uint8_t *udb; udb = sc->udbs_base + UDBS_DB_OFFSET; udb += (qid >> s_qpp) << PAGE_SHIFT; qid &= mask; if (qid < PAGE_SIZE / UDBS_SEG_SIZE) { udb += qid << UDBS_SEG_SHIFT; qid = 0; } fl->udb = (volatile void *)udb; } fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db; FL_LOCK(fl); /* Enough to make sure the SGE doesn't think it's starved */ refill_fl(sc, fl, fl->lowat); FL_UNLOCK(fl); } if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && cong >= 0) { uint32_t param, val; param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); if (cong == 0) val = 1 << 19; else { val = 2 << 19; for (i = 0; i < 4; i++) { if (cong & (1 << i)) val |= 1 << (i << 2); } } rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc != 0) { /* report error but carry on */ device_printf(sc->dev, "failed to set congestion manager context for " "ingress queue %d: %d\n", iq->cntxt_id, rc); } } /* Enable IQ interrupts */ atomic_store_rel_int(&iq->state, IQS_IDLE); t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); return (0); } static int free_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl) { int rc; struct adapter *sc = iq->adapter; device_t dev; if (sc == NULL) return (0); /* nothing to do */ dev = vi ? vi->dev : sc->dev; if (iq->flags & IQ_ALLOCATED) { rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff); if (rc != 0) { device_printf(dev, "failed to free queue %p: %d\n", iq, rc); return (rc); } iq->flags &= ~IQ_ALLOCATED; } free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc); bzero(iq, sizeof(*iq)); if (fl) { free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, fl->desc); if (fl->sdesc) free_fl_sdesc(sc, fl); if (mtx_initialized(&fl->fl_lock)) mtx_destroy(&fl->fl_lock); bzero(fl, sizeof(*fl)); } return (0); } static void add_iq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, struct sge_iq *iq) { struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &iq->ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, iq->qsize * IQ_ESIZE, "descriptor ring size in bytes"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id", CTLTYPE_INT | CTLFLAG_RD, &iq->abs_id, 0, sysctl_uint16, "I", "absolute id of the queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &iq->cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &iq->cidx, 0, sysctl_uint16, "I", "consumer index"); } static void add_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, struct sge_fl *fl) { struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL, "freelist"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &fl->ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, fl->sidx * EQ_ESIZE + sc->params.sge.spg_len, "desc ring size in bytes"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &fl->cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the freelist"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL, fl_pad ? 1 : 0, "padding enabled"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL, fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx, 0, "consumer index"); if (fl->flags & FL_BUF_PACKING) { SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset", CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset"); } SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx, 0, "producer index"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_allocated", CTLFLAG_RD, &fl->mbuf_allocated, "# of mbuf allocated"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_inlined", CTLFLAG_RD, &fl->mbuf_inlined, "# of mbuf inlined in clusters"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated", CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled", CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled", CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)"); } static int alloc_fwq(struct adapter *sc) { int rc, intr_idx; struct sge_iq *fwq = &sc->sge.fwq; struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev); struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE); if (sc->flags & IS_VF) intr_idx = 0; else intr_idx = sc->intr_count > 1 ? 1 : 0; rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1); if (rc != 0) { device_printf(sc->dev, "failed to create firmware event queue: %d\n", rc); return (rc); } oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", CTLFLAG_RD, NULL, "firmware event queue"); add_iq_sysctls(&sc->ctx, oid, fwq); return (0); } static int free_fwq(struct adapter *sc) { return free_iq_fl(NULL, &sc->sge.fwq, NULL); } static int alloc_mgmtq(struct adapter *sc) { int rc; struct sge_wrq *mgmtq = &sc->sge.mgmtq; char name[16]; struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev); struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "mgmtq", CTLFLAG_RD, NULL, "management queue"); snprintf(name, sizeof(name), "%s mgmtq", device_get_nameunit(sc->dev)); init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan, sc->sge.fwq.cntxt_id, name); rc = alloc_wrq(sc, NULL, mgmtq, oid); if (rc != 0) { device_printf(sc->dev, "failed to create management queue: %d\n", rc); return (rc); } return (0); } static int free_mgmtq(struct adapter *sc) { return free_wrq(sc, &sc->sge.mgmtq); } int tnl_cong(struct port_info *pi, int drop) { if (drop == -1) return (-1); else if (drop == 1) return (0); else return (pi->rx_e_chan_map); } static int alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int intr_idx, int idx, struct sysctl_oid *oid) { int rc; struct adapter *sc = vi->pi->adapter; struct sysctl_oid_list *children; char name[16]; rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, intr_idx, tnl_cong(vi->pi, cong_drop)); if (rc != 0) return (rc); if (idx == 0) sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id; else KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id, ("iq_base mismatch")); KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF, ("PF with non-zero iq_base")); /* * The freelist is just barely above the starvation threshold right now, * fill it up a bit more. */ FL_LOCK(&rxq->fl); refill_fl(sc, &rxq->fl, 128); FL_UNLOCK(&rxq->fl); #if defined(INET) || defined(INET6) rc = tcp_lro_init_args(&rxq->lro, vi->ifp, lro_entries, lro_mbufs); if (rc != 0) return (rc); MPASS(rxq->lro.ifp == vi->ifp); /* also indicates LRO init'ed */ if (vi->ifp->if_capenable & IFCAP_LRO) rxq->iq.flags |= IQ_LRO_ENABLED; #endif rxq->ifp = vi->ifp; children = SYSCTL_CHILDREN(oid); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "rx queue"); children = SYSCTL_CHILDREN(oid); add_iq_sysctls(&vi->ctx, oid, &rxq->iq); #if defined(INET) || defined(INET6) SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD, &rxq->lro.lro_queued, 0, NULL); SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD, &rxq->lro.lro_flushed, 0, NULL); #endif SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD, &rxq->rxcsum, "# of times hardware assisted with checksum"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_extraction", CTLFLAG_RD, &rxq->vlan_extraction, "# of times hardware extracted 802.1Q tag"); add_fl_sysctls(sc, &vi->ctx, oid, &rxq->fl); return (rc); } static int free_rxq(struct vi_info *vi, struct sge_rxq *rxq) { int rc; #if defined(INET) || defined(INET6) if (rxq->lro.ifp) { tcp_lro_free(&rxq->lro); rxq->lro.ifp = NULL; } #endif rc = free_iq_fl(vi, &rxq->iq, &rxq->fl); if (rc == 0) bzero(rxq, sizeof(*rxq)); return (rc); } #ifdef TCP_OFFLOAD static int alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int intr_idx, int idx, struct sysctl_oid *oid) { struct port_info *pi = vi->pi; int rc; struct sysctl_oid_list *children; char name[16]; rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 0); if (rc != 0) return (rc); children = SYSCTL_CHILDREN(oid); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "rx queue"); add_iq_sysctls(&vi->ctx, oid, &ofld_rxq->iq); add_fl_sysctls(pi->adapter, &vi->ctx, oid, &ofld_rxq->fl); return (rc); } static int free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq) { int rc; rc = free_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl); if (rc == 0) bzero(ofld_rxq, sizeof(*ofld_rxq)); return (rc); } #endif #ifdef DEV_NETMAP static int alloc_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int intr_idx, int idx, struct sysctl_oid *oid) { int rc; struct sysctl_oid_list *children; struct sysctl_ctx_list *ctx; char name[16]; size_t len; struct adapter *sc = vi->pi->adapter; struct netmap_adapter *na = NA(vi->ifp); MPASS(na != NULL); len = vi->qsize_rxq * IQ_ESIZE; rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map, &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc); if (rc != 0) return (rc); len = na->num_rx_desc * EQ_ESIZE + sc->params.sge.spg_len; rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map, &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc); if (rc != 0) return (rc); nm_rxq->vi = vi; nm_rxq->nid = idx; nm_rxq->iq_cidx = 0; nm_rxq->iq_sidx = vi->qsize_rxq - sc->params.sge.spg_len / IQ_ESIZE; nm_rxq->iq_gen = F_RSPD_GEN; nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0; nm_rxq->fl_sidx = na->num_rx_desc; nm_rxq->intr_idx = intr_idx; ctx = &vi->ctx; children = SYSCTL_CHILDREN(oid); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "rx queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id", CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_abs_id, 0, sysctl_uint16, "I", "absolute id of the queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cidx, 0, sysctl_uint16, "I", "consumer index"); children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL, "freelist"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->fl_cntxt_id, 0, sysctl_uint16, "I", "SGE context id of the freelist"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &nm_rxq->fl_cidx, 0, "consumer index"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &nm_rxq->fl_pidx, 0, "producer index"); return (rc); } static int free_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq) { struct adapter *sc = vi->pi->adapter; free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba, nm_rxq->iq_desc); free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba, nm_rxq->fl_desc); return (0); } static int alloc_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq, int iqidx, int idx, struct sysctl_oid *oid) { int rc; size_t len; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct netmap_adapter *na = NA(vi->ifp); char name[16]; struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); len = na->num_tx_desc * EQ_ESIZE + sc->params.sge.spg_len; rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map, &nm_txq->ba, (void **)&nm_txq->desc); if (rc) return (rc); nm_txq->pidx = nm_txq->cidx = 0; nm_txq->sidx = na->num_tx_desc; nm_txq->nid = idx; nm_txq->iqidx = iqidx; nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(G_FW_VIID_PFN(vi->viid)) | V_TXPKT_VF(G_FW_VIID_VIN(vi->viid)) | V_TXPKT_VF_VLD(G_FW_VIID_VIVLD(vi->viid))); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "netmap tx queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, &nm_txq->cntxt_id, 0, "SGE context id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &nm_txq->cidx, 0, sysctl_uint16, "I", "consumer index"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", CTLTYPE_INT | CTLFLAG_RD, &nm_txq->pidx, 0, sysctl_uint16, "I", "producer index"); return (rc); } static int free_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq) { struct adapter *sc = vi->pi->adapter; free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba, nm_txq->desc); return (0); } #endif static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) { int rc, cntxt_id; struct fw_eq_ctrl_cmd c; int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; bzero(&c, sizeof(c)); c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | V_FW_EQ_CTRL_CMD_VFN(0)); c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC | F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); c.physeqid_pkd = htobe32(0); c.fetchszm_to_iqid = htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | V_FW_EQ_CTRL_CMD_EQSIZE(qsize)); c.eqaddr = htobe64(eq->ba); rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { device_printf(sc->dev, "failed to create control queue %d: %d\n", eq->tx_chan, rc); return (rc); } eq->flags |= EQ_ALLOCATED; eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid)); cntxt_id = eq->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.neq - 1); sc->sge.eqmap[cntxt_id] = eq; return (rc); } static int eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) { int rc, cntxt_id; struct fw_eq_eth_cmd c; int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; bzero(&c, sizeof(c)); c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | V_FW_EQ_ETH_CMD_VFN(0)); c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC | F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid)); c.fetchszm_to_iqid = htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | V_FW_EQ_ETH_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | V_FW_EQ_ETH_CMD_EQSIZE(qsize)); c.eqaddr = htobe64(eq->ba); rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { device_printf(vi->dev, "failed to create Ethernet egress queue: %d\n", rc); return (rc); } eq->flags |= EQ_ALLOCATED; eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd)); eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); cntxt_id = eq->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.neq - 1); sc->sge.eqmap[cntxt_id] = eq; return (rc); } #ifdef TCP_OFFLOAD static int ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) { int rc, cntxt_id; struct fw_eq_ofld_cmd c; int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; bzero(&c, sizeof(c)); c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | V_FW_EQ_OFLD_CMD_VFN(0)); c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); c.fetchszm_to_iqid = htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | V_FW_EQ_OFLD_CMD_EQSIZE(qsize)); c.eqaddr = htobe64(eq->ba); rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { device_printf(vi->dev, "failed to create egress queue for TCP offload: %d\n", rc); return (rc); } eq->flags |= EQ_ALLOCATED; eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd)); cntxt_id = eq->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, cntxt_id, sc->sge.neq - 1); sc->sge.eqmap[cntxt_id] = eq; return (rc); } #endif static int alloc_eq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) { int rc, qsize; size_t len; mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF); qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; len = qsize * EQ_ESIZE; rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, &eq->ba, (void **)&eq->desc); if (rc) return (rc); eq->pidx = eq->cidx = 0; eq->equeqidx = eq->dbidx = 0; eq->doorbells = sc->doorbells; switch (eq->flags & EQ_TYPEMASK) { case EQ_CTRL: rc = ctrl_eq_alloc(sc, eq); break; case EQ_ETH: rc = eth_eq_alloc(sc, vi, eq); break; #ifdef TCP_OFFLOAD case EQ_OFLD: rc = ofld_eq_alloc(sc, vi, eq); break; #endif default: panic("%s: invalid eq type %d.", __func__, eq->flags & EQ_TYPEMASK); } if (rc != 0) { device_printf(sc->dev, "failed to allocate egress queue(%d): %d\n", eq->flags & EQ_TYPEMASK, rc); } if (isset(&eq->doorbells, DOORBELL_UDB) || isset(&eq->doorbells, DOORBELL_UDBWC) || isset(&eq->doorbells, DOORBELL_WCWR)) { uint32_t s_qpp = sc->params.sge.eq_s_qpp; uint32_t mask = (1 << s_qpp) - 1; volatile uint8_t *udb; udb = sc->udbs_base + UDBS_DB_OFFSET; udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ eq->udb_qid = eq->cntxt_id & mask; /* id in page */ if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE) clrbit(&eq->doorbells, DOORBELL_WCWR); else { udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ eq->udb_qid = 0; } eq->udb = (volatile void *)udb; } return (rc); } static int free_eq(struct adapter *sc, struct sge_eq *eq) { int rc; if (eq->flags & EQ_ALLOCATED) { switch (eq->flags & EQ_TYPEMASK) { case EQ_CTRL: rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); break; case EQ_ETH: rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); break; #ifdef TCP_OFFLOAD case EQ_OFLD: rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); break; #endif default: panic("%s: invalid eq type %d.", __func__, eq->flags & EQ_TYPEMASK); } if (rc != 0) { device_printf(sc->dev, "failed to free egress queue (%d): %d\n", eq->flags & EQ_TYPEMASK, rc); return (rc); } eq->flags &= ~EQ_ALLOCATED; } free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc); if (mtx_initialized(&eq->eq_lock)) mtx_destroy(&eq->eq_lock); bzero(eq, sizeof(*eq)); return (0); } static int alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq, struct sysctl_oid *oid) { int rc; struct sysctl_ctx_list *ctx = vi ? &vi->ctx : &sc->ctx; struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); rc = alloc_eq(sc, vi, &wrq->eq); if (rc) return (rc); wrq->adapter = sc; TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq); TAILQ_INIT(&wrq->incomplete_wrs); STAILQ_INIT(&wrq->wr_list); wrq->nwr_pending = 0; wrq->ndesc_needed = 0; SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &wrq->eq.ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, wrq->eq.sidx * EQ_ESIZE + sc->params.sge.spg_len, "desc ring size in bytes"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, &wrq->eq.cntxt_id, 0, "SGE context id of the queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I", "consumer index"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx", CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I", "producer index"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, wrq->eq.sidx, "status page index"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD, &wrq->tx_wrs_direct, "# of work requests (direct)"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD, &wrq->tx_wrs_copied, "# of work requests (copied)"); SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD, &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)"); return (rc); } static int free_wrq(struct adapter *sc, struct sge_wrq *wrq) { int rc; rc = free_eq(sc, &wrq->eq); if (rc) return (rc); bzero(wrq, sizeof(*wrq)); return (0); } static int alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx, struct sysctl_oid *oid) { int rc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct sge_eq *eq = &txq->eq; char name[16]; struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, can_resume_eth_tx, M_CXGBE, M_WAITOK); if (rc != 0) { device_printf(sc->dev, "failed to allocate mp_ring: %d\n", rc); return (rc); } rc = alloc_eq(sc, vi, eq); if (rc != 0) { mp_ring_free(txq->r); txq->r = NULL; return (rc); } /* Can't fail after this point. */ if (idx == 0) sc->sge.eq_base = eq->abs_id - eq->cntxt_id; else KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id, ("eq_base mismatch")); KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF, ("PF with non-zero eq_base")); TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq); txq->ifp = vi->ifp; txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK); if (sc->flags & IS_VF) txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | V_TXPKT_INTF(pi->tx_chan)); else txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(G_FW_VIID_PFN(vi->viid)) | V_TXPKT_VF(G_FW_VIID_VIN(vi->viid)) | V_TXPKT_VF_VLD(G_FW_VIID_VIVLD(vi->viid))); txq->tc_idx = -1; txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE, M_ZERO | M_WAITOK); snprintf(name, sizeof(name), "%d", idx); oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, "tx queue"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &eq->ba, "bus address of descriptor ring"); SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, eq->sidx * EQ_ESIZE + sc->params.sge.spg_len, "desc ring size in bytes"); SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD, &eq->abs_id, 0, "absolute id of the queue"); SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, &eq->cntxt_id, 0, "SGE context id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", CTLTYPE_INT | CTLFLAG_RD, &eq->cidx, 0, sysctl_uint16, "I", "consumer index"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", CTLTYPE_INT | CTLFLAG_RD, &eq->pidx, 0, sysctl_uint16, "I", "producer index"); SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, eq->sidx, "status page index"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "tc", CTLTYPE_INT | CTLFLAG_RW, vi, idx, sysctl_tc, "I", "traffic class (-1 means none)"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD, &txq->txcsum, "# of times hardware assisted with checksum"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_insertion", CTLFLAG_RD, &txq->vlan_insertion, "# of times hardware inserted 802.1Q tag"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD, &txq->tso_wrs, "# of TSO work requests"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD, &txq->imm_wrs, "# of work requests with immediate data"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD, &txq->sgl_wrs, "# of work requests with direct SGL"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD, &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_wrs", CTLFLAG_RD, &txq->txpkts0_wrs, "# of txpkts (type 0) work requests"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_wrs", CTLFLAG_RD, &txq->txpkts1_wrs, "# of txpkts (type 1) work requests"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_pkts", CTLFLAG_RD, &txq->txpkts0_pkts, "# of frames tx'd using type0 txpkts work requests"); SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_pkts", CTLFLAG_RD, &txq->txpkts1_pkts, "# of frames tx'd using type1 txpkts work requests"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_enqueues", CTLFLAG_RD, &txq->r->enqueues, "# of enqueues to the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_drops", CTLFLAG_RD, &txq->r->drops, "# of drops in the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_starts", CTLFLAG_RD, &txq->r->starts, "# of normal consumer starts in the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_stalls", CTLFLAG_RD, &txq->r->stalls, "# of consumer stalls in the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_restarts", CTLFLAG_RD, &txq->r->restarts, "# of consumer restarts in the mp_ring for this queue"); SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_abdications", CTLFLAG_RD, &txq->r->abdications, "# of consumer abdications in the mp_ring for this queue"); return (0); } static int free_txq(struct vi_info *vi, struct sge_txq *txq) { int rc; struct adapter *sc = vi->pi->adapter; struct sge_eq *eq = &txq->eq; rc = free_eq(sc, eq); if (rc) return (rc); sglist_free(txq->gl); free(txq->sdesc, M_CXGBE); mp_ring_free(txq->r); bzero(txq, sizeof(*txq)); return (0); } static void oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *ba = arg; KASSERT(nseg == 1, ("%s meant for single segment mappings only.", __func__)); *ba = error ? 0 : segs->ds_addr; } static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl) { uint32_t n, v; n = IDXDIFF(fl->pidx / 8, fl->dbidx, fl->sidx); MPASS(n > 0); wmb(); v = fl->dbval | V_PIDX(n); if (fl->udb) *fl->udb = htole32(v); else t4_write_reg(sc, sc->sge_kdoorbell_reg, v); IDXINCR(fl->dbidx, n, fl->sidx); } /* * Fills up the freelist by allocating up to 'n' buffers. Buffers that are * recycled do not count towards this allocation budget. * * Returns non-zero to indicate that this freelist should be added to the list * of starving freelists. */ static int refill_fl(struct adapter *sc, struct sge_fl *fl, int n) { __be64 *d; struct fl_sdesc *sd; uintptr_t pa; caddr_t cl; struct cluster_layout *cll; struct sw_zone_info *swz; struct cluster_metadata *clm; uint16_t max_pidx; uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */ FL_LOCK_ASSERT_OWNED(fl); /* * We always stop at the beginning of the hardware descriptor that's just * before the one with the hw cidx. This is to avoid hw pidx = hw cidx, * which would mean an empty freelist to the chip. */ max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1; if (fl->pidx == max_pidx * 8) return (0); d = &fl->desc[fl->pidx]; sd = &fl->sdesc[fl->pidx]; cll = &fl->cll_def; /* default layout */ swz = &sc->sge.sw_zone_info[cll->zidx]; while (n > 0) { if (sd->cl != NULL) { if (sd->nmbuf == 0) { /* * Fast recycle without involving any atomics on * the cluster's metadata (if the cluster has * metadata). This happens when all frames * received in the cluster were small enough to * fit within a single mbuf each. */ fl->cl_fast_recycled++; #ifdef INVARIANTS clm = cl_metadata(sc, fl, &sd->cll, sd->cl); if (clm != NULL) MPASS(clm->refcount == 1); #endif goto recycled_fast; } /* * Cluster is guaranteed to have metadata. Clusters * without metadata always take the fast recycle path * when they're recycled. */ clm = cl_metadata(sc, fl, &sd->cll, sd->cl); MPASS(clm != NULL); if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { fl->cl_recycled++; counter_u64_add(extfree_rels, 1); goto recycled; } sd->cl = NULL; /* gave up my reference */ } MPASS(sd->cl == NULL); alloc: cl = uma_zalloc(swz->zone, M_NOWAIT); if (__predict_false(cl == NULL)) { if (cll == &fl->cll_alt || fl->cll_alt.zidx == -1 || fl->cll_def.zidx == fl->cll_alt.zidx) break; /* fall back to the safe zone */ cll = &fl->cll_alt; swz = &sc->sge.sw_zone_info[cll->zidx]; goto alloc; } fl->cl_allocated++; n--; pa = pmap_kextract((vm_offset_t)cl); pa += cll->region1; sd->cl = cl; sd->cll = *cll; *d = htobe64(pa | cll->hwidx); clm = cl_metadata(sc, fl, cll, cl); if (clm != NULL) { recycled: #ifdef INVARIANTS clm->sd = sd; #endif clm->refcount = 1; } sd->nmbuf = 0; recycled_fast: d++; sd++; if (__predict_false(++fl->pidx % 8 == 0)) { uint16_t pidx = fl->pidx / 8; if (__predict_false(pidx == fl->sidx)) { fl->pidx = 0; pidx = 0; sd = fl->sdesc; d = fl->desc; } if (pidx == max_pidx) break; if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4) ring_fl_db(sc, fl); } } if (fl->pidx / 8 != fl->dbidx) ring_fl_db(sc, fl); return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); } /* * Attempt to refill all starving freelists. */ static void refill_sfl(void *arg) { struct adapter *sc = arg; struct sge_fl *fl, *fl_temp; mtx_assert(&sc->sfl_lock, MA_OWNED); TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { FL_LOCK(fl); refill_fl(sc, fl, 64); if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { TAILQ_REMOVE(&sc->sfl, fl, link); fl->flags &= ~FL_STARVING; } FL_UNLOCK(fl); } if (!TAILQ_EMPTY(&sc->sfl)) callout_schedule(&sc->sfl_callout, hz / 5); } static int alloc_fl_sdesc(struct sge_fl *fl) { fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE, M_ZERO | M_WAITOK); return (0); } static void free_fl_sdesc(struct adapter *sc, struct sge_fl *fl) { struct fl_sdesc *sd; struct cluster_metadata *clm; struct cluster_layout *cll; int i; sd = fl->sdesc; for (i = 0; i < fl->sidx * 8; i++, sd++) { if (sd->cl == NULL) continue; cll = &sd->cll; clm = cl_metadata(sc, fl, cll, sd->cl); if (sd->nmbuf == 0) uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl); else if (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1) { uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl); counter_u64_add(extfree_rels, 1); } sd->cl = NULL; } free(fl->sdesc, M_CXGBE); fl->sdesc = NULL; } static inline void get_pkt_gl(struct mbuf *m, struct sglist *gl) { int rc; M_ASSERTPKTHDR(m); sglist_reset(gl); rc = sglist_append_mbuf(gl, m); if (__predict_false(rc != 0)) { panic("%s: mbuf %p (%d segs) was vetted earlier but now fails " "with %d.", __func__, m, mbuf_nsegs(m), rc); } KASSERT(gl->sg_nseg == mbuf_nsegs(m), ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m, mbuf_nsegs(m), gl->sg_nseg)); KASSERT(gl->sg_nseg > 0 && gl->sg_nseg <= (needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS), ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__, gl->sg_nseg, needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)); } /* * len16 for a txpkt WR with a GL. Includes the firmware work request header. */ static inline u_int txpkt_len16(u_int nsegs, u_int tso) { u_int n; MPASS(nsegs > 0); nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); if (tso) n += sizeof(struct cpl_tx_pkt_lso_core); return (howmany(n, 16)); } /* * len16 for a txpkt_vm WR with a GL. Includes the firmware work * request header. */ static inline u_int txpkt_vm_len16(u_int nsegs, u_int tso) { u_int n; MPASS(nsegs > 0); nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct fw_eth_tx_pkt_vm_wr) + sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); if (tso) n += sizeof(struct cpl_tx_pkt_lso_core); return (howmany(n, 16)); } /* * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work * request header. */ static inline u_int txpkts0_len16(u_int nsegs) { u_int n; MPASS(nsegs > 0); nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) + sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); return (howmany(n, 16)); } /* * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work * request header. */ static inline u_int txpkts1_len16(void) { u_int n; n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl); return (howmany(n, 16)); } static inline u_int imm_payload(u_int ndesc) { u_int n; n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) - sizeof(struct cpl_tx_pkt_core); return (n); } /* * Write a VM txpkt WR for this packet to the hardware descriptors, update the * software descriptor, and advance the pidx. It is guaranteed that enough * descriptors are available. * * The return value is the # of hardware descriptors used. */ static u_int write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, struct fw_eth_tx_pkt_vm_wr *wr, struct mbuf *m0, u_int available) { struct sge_eq *eq = &txq->eq; struct tx_sdesc *txsd; struct cpl_tx_pkt_core *cpl; uint32_t ctrl; /* used in many unrelated places */ uint64_t ctrl1; int csum_type, len16, ndesc, pktlen, nsegs; caddr_t dst; TXQ_LOCK_ASSERT_OWNED(txq); M_ASSERTPKTHDR(m0); MPASS(available > 0 && available < eq->sidx); len16 = mbuf_len16(m0); nsegs = mbuf_nsegs(m0); pktlen = m0->m_pkthdr.len; ctrl = sizeof(struct cpl_tx_pkt_core); if (needs_tso(m0)) ctrl += sizeof(struct cpl_tx_pkt_lso_core); ndesc = howmany(len16, EQ_ESIZE / 16); MPASS(ndesc <= available); /* Firmware work request header */ MPASS(wr == (void *)&eq->desc[eq->pidx]); wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) | V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); ctrl = V_FW_WR_LEN16(len16); wr->equiq_to_len16 = htobe32(ctrl); wr->r3[0] = 0; wr->r3[1] = 0; /* * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci. * vlantci is ignored unless the ethtype is 0x8100, so it's * simpler to always copy it rather than making it * conditional. Also, it seems that we do not have to set * vlantci or fake the ethtype when doing VLAN tag insertion. */ m_copydata(m0, 0, sizeof(struct ether_header) + 2, wr->ethmacdst); csum_type = -1; if (needs_tso(m0)) { struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && m0->m_pkthdr.l4hlen > 0, ("%s: mbuf %p needs TSO but missing header lengths", __func__, m0)); ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header)) ctrl |= V_LSO_ETHHDR_LEN(1); if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) ctrl |= F_LSO_IPV6; lso->lso_ctrl = htobe32(ctrl); lso->ipid_ofst = htobe16(0); lso->mss = htobe16(m0->m_pkthdr.tso_segsz); lso->seqno_offset = htobe32(0); lso->len = htobe32(pktlen); if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) csum_type = TX_CSUM_TCPIP6; else csum_type = TX_CSUM_TCPIP; cpl = (void *)(lso + 1); txq->tso_wrs++; } else { if (m0->m_pkthdr.csum_flags & CSUM_IP_TCP) csum_type = TX_CSUM_TCPIP; else if (m0->m_pkthdr.csum_flags & CSUM_IP_UDP) csum_type = TX_CSUM_UDPIP; else if (m0->m_pkthdr.csum_flags & CSUM_IP6_TCP) csum_type = TX_CSUM_TCPIP6; else if (m0->m_pkthdr.csum_flags & CSUM_IP6_UDP) csum_type = TX_CSUM_UDPIP6; #if defined(INET) else if (m0->m_pkthdr.csum_flags & CSUM_IP) { /* * XXX: The firmware appears to stomp on the * fragment/flags field of the IP header when * using TX_CSUM_IP. Fall back to doing * software checksums. */ u_short *sump; struct mbuf *m; int offset; m = m0; offset = 0; sump = m_advance(&m, &offset, m0->m_pkthdr.l2hlen + offsetof(struct ip, ip_sum)); *sump = in_cksum_skip(m0, m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen, m0->m_pkthdr.l2hlen); m0->m_pkthdr.csum_flags &= ~CSUM_IP; } #endif cpl = (void *)(wr + 1); } /* Checksum offload */ ctrl1 = 0; if (needs_l3_csum(m0) == 0) ctrl1 |= F_TXPKT_IPCSUM_DIS; if (csum_type >= 0) { KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0, ("%s: mbuf %p needs checksum offload but missing header lengths", __func__, m0)); if (chip_id(sc) <= CHELSIO_T5) { ctrl1 |= V_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen - ETHER_HDR_LEN); } else { ctrl1 |= V_T6_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen - ETHER_HDR_LEN); } ctrl1 |= V_TXPKT_IPHDR_LEN(m0->m_pkthdr.l3hlen); ctrl1 |= V_TXPKT_CSUM_TYPE(csum_type); } else ctrl1 |= F_TXPKT_L4CSUM_DIS; if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) txq->txcsum++; /* some hardware assistance provided */ /* VLAN tag insertion */ if (needs_vlan_insertion(m0)) { ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); txq->vlan_insertion++; } /* CPL header */ cpl->ctrl0 = txq->cpl_ctrl0; cpl->pack = 0; cpl->len = htobe16(pktlen); cpl->ctrl1 = htobe64(ctrl1); /* SGL */ dst = (void *)(cpl + 1); /* * A packet using TSO will use up an entire descriptor for the * firmware work request header, LSO CPL, and TX_PKT_XT CPL. * If this descriptor is the last descriptor in the ring, wrap * around to the front of the ring explicitly for the start of * the sgl. */ if (dst == (void *)&eq->desc[eq->sidx]) { dst = (void *)&eq->desc[0]; write_gl_to_txd(txq, m0, &dst, 0); } else write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); txq->sgl_wrs++; txq->txpkt_wrs++; txsd = &txq->sdesc[eq->pidx]; txsd->m = m0; txsd->desc_used = ndesc; return (ndesc); } /* * Write a txpkt WR for this packet to the hardware descriptors, update the * software descriptor, and advance the pidx. It is guaranteed that enough * descriptors are available. * * The return value is the # of hardware descriptors used. */ static u_int write_txpkt_wr(struct sge_txq *txq, struct fw_eth_tx_pkt_wr *wr, struct mbuf *m0, u_int available) { struct sge_eq *eq = &txq->eq; struct tx_sdesc *txsd; struct cpl_tx_pkt_core *cpl; uint32_t ctrl; /* used in many unrelated places */ uint64_t ctrl1; int len16, ndesc, pktlen, nsegs; caddr_t dst; TXQ_LOCK_ASSERT_OWNED(txq); M_ASSERTPKTHDR(m0); MPASS(available > 0 && available < eq->sidx); len16 = mbuf_len16(m0); nsegs = mbuf_nsegs(m0); pktlen = m0->m_pkthdr.len; ctrl = sizeof(struct cpl_tx_pkt_core); if (needs_tso(m0)) ctrl += sizeof(struct cpl_tx_pkt_lso_core); else if (pktlen <= imm_payload(2) && available >= 2) { /* Immediate data. Recalculate len16 and set nsegs to 0. */ ctrl += pktlen; len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) + pktlen, 16); nsegs = 0; } ndesc = howmany(len16, EQ_ESIZE / 16); MPASS(ndesc <= available); /* Firmware work request header */ MPASS(wr == (void *)&eq->desc[eq->pidx]); wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); ctrl = V_FW_WR_LEN16(len16); wr->equiq_to_len16 = htobe32(ctrl); wr->r3 = 0; if (needs_tso(m0)) { struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && m0->m_pkthdr.l4hlen > 0, ("%s: mbuf %p needs TSO but missing header lengths", __func__, m0)); ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header)) ctrl |= V_LSO_ETHHDR_LEN(1); if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) ctrl |= F_LSO_IPV6; lso->lso_ctrl = htobe32(ctrl); lso->ipid_ofst = htobe16(0); lso->mss = htobe16(m0->m_pkthdr.tso_segsz); lso->seqno_offset = htobe32(0); lso->len = htobe32(pktlen); cpl = (void *)(lso + 1); txq->tso_wrs++; } else cpl = (void *)(wr + 1); /* Checksum offload */ ctrl1 = 0; if (needs_l3_csum(m0) == 0) ctrl1 |= F_TXPKT_IPCSUM_DIS; if (needs_l4_csum(m0) == 0) ctrl1 |= F_TXPKT_L4CSUM_DIS; if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) txq->txcsum++; /* some hardware assistance provided */ /* VLAN tag insertion */ if (needs_vlan_insertion(m0)) { ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); txq->vlan_insertion++; } /* CPL header */ cpl->ctrl0 = txq->cpl_ctrl0; cpl->pack = 0; cpl->len = htobe16(pktlen); cpl->ctrl1 = htobe64(ctrl1); /* SGL */ dst = (void *)(cpl + 1); if (nsegs > 0) { write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); txq->sgl_wrs++; } else { struct mbuf *m; for (m = m0; m != NULL; m = m->m_next) { copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); #ifdef INVARIANTS pktlen -= m->m_len; #endif } #ifdef INVARIANTS KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen)); #endif txq->imm_wrs++; } txq->txpkt_wrs++; txsd = &txq->sdesc[eq->pidx]; txsd->m = m0; txsd->desc_used = ndesc; return (ndesc); } static int try_txpkts(struct mbuf *m, struct mbuf *n, struct txpkts *txp, u_int available) { u_int needed, nsegs1, nsegs2, l1, l2; if (cannot_use_txpkts(m) || cannot_use_txpkts(n)) return (1); nsegs1 = mbuf_nsegs(m); nsegs2 = mbuf_nsegs(n); if (nsegs1 + nsegs2 == 2) { txp->wr_type = 1; l1 = l2 = txpkts1_len16(); } else { txp->wr_type = 0; l1 = txpkts0_len16(nsegs1); l2 = txpkts0_len16(nsegs2); } txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + l1 + l2; needed = howmany(txp->len16, EQ_ESIZE / 16); if (needed > SGE_MAX_WR_NDESC || needed > available) return (1); txp->plen = m->m_pkthdr.len + n->m_pkthdr.len; if (txp->plen > 65535) return (1); txp->npkt = 2; set_mbuf_len16(m, l1); set_mbuf_len16(n, l2); return (0); } static int add_to_txpkts(struct mbuf *m, struct txpkts *txp, u_int available) { u_int plen, len16, needed, nsegs; MPASS(txp->wr_type == 0 || txp->wr_type == 1); nsegs = mbuf_nsegs(m); if (needs_tso(m) || (txp->wr_type == 1 && nsegs != 1)) return (1); plen = txp->plen + m->m_pkthdr.len; if (plen > 65535) return (1); if (txp->wr_type == 0) len16 = txpkts0_len16(nsegs); else len16 = txpkts1_len16(); needed = howmany(txp->len16 + len16, EQ_ESIZE / 16); if (needed > SGE_MAX_WR_NDESC || needed > available) return (1); txp->npkt++; txp->plen = plen; txp->len16 += len16; set_mbuf_len16(m, len16); return (0); } /* * Write a txpkts WR for the packets in txp to the hardware descriptors, update * the software descriptor, and advance the pidx. It is guaranteed that enough * descriptors are available. * * The return value is the # of hardware descriptors used. */ static u_int write_txpkts_wr(struct sge_txq *txq, struct fw_eth_tx_pkts_wr *wr, struct mbuf *m0, const struct txpkts *txp, u_int available) { struct sge_eq *eq = &txq->eq; struct tx_sdesc *txsd; struct cpl_tx_pkt_core *cpl; uint32_t ctrl; uint64_t ctrl1; int ndesc, checkwrap; struct mbuf *m; void *flitp; TXQ_LOCK_ASSERT_OWNED(txq); MPASS(txp->npkt > 0); MPASS(txp->plen < 65536); MPASS(m0 != NULL); MPASS(m0->m_nextpkt != NULL); MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); MPASS(available > 0 && available < eq->sidx); ndesc = howmany(txp->len16, EQ_ESIZE / 16); MPASS(ndesc <= available); MPASS(wr == (void *)&eq->desc[eq->pidx]); wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); ctrl = V_FW_WR_LEN16(txp->len16); wr->equiq_to_len16 = htobe32(ctrl); wr->plen = htobe16(txp->plen); wr->npkt = txp->npkt; wr->r3 = 0; wr->type = txp->wr_type; flitp = wr + 1; /* * At this point we are 16B into a hardware descriptor. If checkwrap is * set then we know the WR is going to wrap around somewhere. We'll * check for that at appropriate points. */ checkwrap = eq->sidx - ndesc < eq->pidx; for (m = m0; m != NULL; m = m->m_nextpkt) { if (txp->wr_type == 0) { struct ulp_txpkt *ulpmc; struct ulptx_idata *ulpsc; /* ULP master command */ ulpmc = flitp; ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid)); ulpmc->len = htobe32(mbuf_len16(m)); /* ULP subcommand */ ulpsc = (void *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | F_ULP_TX_SC_MORE); ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core)); cpl = (void *)(ulpsc + 1); if (checkwrap && (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx]) cpl = (void *)&eq->desc[0]; } else { cpl = flitp; } /* Checksum offload */ ctrl1 = 0; if (needs_l3_csum(m) == 0) ctrl1 |= F_TXPKT_IPCSUM_DIS; if (needs_l4_csum(m) == 0) ctrl1 |= F_TXPKT_L4CSUM_DIS; if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) txq->txcsum++; /* some hardware assistance provided */ /* VLAN tag insertion */ if (needs_vlan_insertion(m)) { ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); txq->vlan_insertion++; } /* CPL header */ cpl->ctrl0 = txq->cpl_ctrl0; cpl->pack = 0; cpl->len = htobe16(m->m_pkthdr.len); cpl->ctrl1 = htobe64(ctrl1); flitp = cpl + 1; if (checkwrap && (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) flitp = (void *)&eq->desc[0]; write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap); } if (txp->wr_type == 0) { txq->txpkts0_pkts += txp->npkt; txq->txpkts0_wrs++; } else { txq->txpkts1_pkts += txp->npkt; txq->txpkts1_wrs++; } txsd = &txq->sdesc[eq->pidx]; txsd->m = m0; txsd->desc_used = ndesc; return (ndesc); } /* * If the SGL ends on an address that is not 16 byte aligned, this function will * add a 0 filled flit at the end. */ static void write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap) { struct sge_eq *eq = &txq->eq; struct sglist *gl = txq->gl; struct sglist_seg *seg; __be64 *flitp, *wrap; struct ulptx_sgl *usgl; int i, nflits, nsegs; KASSERT(((uintptr_t)(*to) & 0xf) == 0, ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to)); MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); get_pkt_gl(m, gl); nsegs = gl->sg_nseg; MPASS(nsegs > 0); nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; flitp = (__be64 *)(*to); wrap = (__be64 *)(&eq->desc[eq->sidx]); seg = &gl->sg_segs[0]; usgl = (void *)flitp; /* * We start at a 16 byte boundary somewhere inside the tx descriptor * ring, so we're at least 16 bytes away from the status page. There is * no chance of a wrap around in the middle of usgl (which is 16 bytes). */ usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(nsegs)); usgl->len0 = htobe32(seg->ss_len); usgl->addr0 = htobe64(seg->ss_paddr); seg++; if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) { /* Won't wrap around at all */ for (i = 0; i < nsegs - 1; i++, seg++) { usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); } if (i & 1) usgl->sge[i / 2].len[1] = htobe32(0); flitp += nflits; } else { /* Will wrap somewhere in the rest of the SGL */ /* 2 flits already written, write the rest flit by flit */ flitp = (void *)(usgl + 1); for (i = 0; i < nflits - 2; i++) { if (flitp == wrap) flitp = (void *)eq->desc; *flitp++ = get_flit(seg, nsegs - 1, i); } } if (nflits & 1) { MPASS(((uintptr_t)flitp) & 0xf); *flitp++ = 0; } MPASS((((uintptr_t)flitp) & 0xf) == 0); if (__predict_false(flitp == wrap)) *to = (void *)eq->desc; else *to = (void *)flitp; } static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) { MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); if (__predict_true((uintptr_t)(*to) + len <= (uintptr_t)&eq->desc[eq->sidx])) { bcopy(from, *to, len); (*to) += len; } else { int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); bcopy(from, *to, portion); from += portion; portion = len - portion; /* remaining */ bcopy(from, (void *)eq->desc, portion); (*to) = (caddr_t)eq->desc + portion; } } static inline void ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n) { u_int db; MPASS(n > 0); db = eq->doorbells; if (n > 1) clrbit(&db, DOORBELL_WCWR); wmb(); switch (ffs(db) - 1) { case DOORBELL_UDB: *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); break; case DOORBELL_WCWR: { volatile uint64_t *dst, *src; int i; /* * Queues whose 128B doorbell segment fits in the page do not * use relative qid (udb_qid is always 0). Only queues with * doorbell segments can do WCWR. */ KASSERT(eq->udb_qid == 0 && n == 1, ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p", __func__, eq->doorbells, n, eq->dbidx, eq)); dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET - UDBS_DB_OFFSET); i = eq->dbidx; src = (void *)&eq->desc[i]; while (src != (void *)&eq->desc[i + 1]) *dst++ = *src++; wmb(); break; } case DOORBELL_UDBWC: *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); wmb(); break; case DOORBELL_KDB: t4_write_reg(sc, sc->sge_kdoorbell_reg, V_QID(eq->cntxt_id) | V_PIDX(n)); break; } IDXINCR(eq->dbidx, n, eq->sidx); } static inline u_int reclaimable_tx_desc(struct sge_eq *eq) { uint16_t hw_cidx; hw_cidx = read_hw_cidx(eq); return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx)); } static inline u_int total_available_tx_desc(struct sge_eq *eq) { uint16_t hw_cidx, pidx; hw_cidx = read_hw_cidx(eq); pidx = eq->pidx; if (pidx == hw_cidx) return (eq->sidx - 1); else return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1); } static inline uint16_t read_hw_cidx(struct sge_eq *eq) { struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; uint16_t cidx = spg->cidx; /* stable snapshot */ return (be16toh(cidx)); } /* * Reclaim 'n' descriptors approximately. */ static u_int reclaim_tx_descs(struct sge_txq *txq, u_int n) { struct tx_sdesc *txsd; struct sge_eq *eq = &txq->eq; u_int can_reclaim, reclaimed; TXQ_LOCK_ASSERT_OWNED(txq); MPASS(n > 0); reclaimed = 0; can_reclaim = reclaimable_tx_desc(eq); while (can_reclaim && reclaimed < n) { int ndesc; struct mbuf *m, *nextpkt; txsd = &txq->sdesc[eq->cidx]; ndesc = txsd->desc_used; /* Firmware doesn't return "partial" credits. */ KASSERT(can_reclaim >= ndesc, ("%s: unexpected number of credits: %d, %d", __func__, can_reclaim, ndesc)); KASSERT(ndesc != 0, ("%s: descriptor with no credits: cidx %d", __func__, eq->cidx)); for (m = txsd->m; m != NULL; m = nextpkt) { nextpkt = m->m_nextpkt; m->m_nextpkt = NULL; m_freem(m); } reclaimed += ndesc; can_reclaim -= ndesc; IDXINCR(eq->cidx, ndesc, eq->sidx); } return (reclaimed); } static void tx_reclaim(void *arg, int n) { struct sge_txq *txq = arg; struct sge_eq *eq = &txq->eq; do { if (TXQ_TRYLOCK(txq) == 0) break; n = reclaim_tx_descs(txq, 32); if (eq->cidx == eq->pidx) eq->equeqidx = eq->pidx; TXQ_UNLOCK(txq); } while (n > 0); } static __be64 get_flit(struct sglist_seg *segs, int nsegs, int idx) { int i = (idx / 3) * 2; switch (idx % 3) { case 0: { uint64_t rc; rc = (uint64_t)segs[i].ss_len << 32; if (i + 1 < nsegs) rc |= (uint64_t)(segs[i + 1].ss_len); return (htobe64(rc)); } case 1: return (htobe64(segs[i].ss_paddr)); case 2: return (htobe64(segs[i + 1].ss_paddr)); } return (0); } static void find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp) { int8_t zidx, hwidx, idx; uint16_t region1, region3; int spare, spare_needed, n; struct sw_zone_info *swz; struct hw_buf_info *hwb, *hwb_list = &sc->sge.hw_buf_info[0]; /* * Buffer Packing: Look for PAGE_SIZE or larger zone which has a bufsize * large enough for the max payload and cluster metadata. Otherwise * settle for the largest bufsize that leaves enough room in the cluster * for metadata. * * Without buffer packing: Look for the smallest zone which has a * bufsize large enough for the max payload. Settle for the largest * bufsize available if there's nothing big enough for max payload. */ spare_needed = fl->flags & FL_BUF_PACKING ? CL_METADATA_SIZE : 0; swz = &sc->sge.sw_zone_info[0]; hwidx = -1; for (zidx = 0; zidx < SW_ZONE_SIZES; zidx++, swz++) { if (swz->size > largest_rx_cluster) { if (__predict_true(hwidx != -1)) break; /* * This is a misconfiguration. largest_rx_cluster is * preventing us from finding a refill source. See * dev.t5nex..buffer_sizes to figure out why. */ device_printf(sc->dev, "largest_rx_cluster=%u leaves no" " refill source for fl %p (dma %u). Ignored.\n", largest_rx_cluster, fl, maxp); } for (idx = swz->head_hwidx; idx != -1; idx = hwb->next) { hwb = &hwb_list[idx]; spare = swz->size - hwb->size; if (spare < spare_needed) continue; hwidx = idx; /* best option so far */ if (hwb->size >= maxp) { if ((fl->flags & FL_BUF_PACKING) == 0) goto done; /* stop looking (not packing) */ if (swz->size >= safest_rx_cluster) goto done; /* stop looking (packing) */ } break; /* keep looking, next zone */ } } done: /* A usable hwidx has been located. */ MPASS(hwidx != -1); hwb = &hwb_list[hwidx]; zidx = hwb->zidx; swz = &sc->sge.sw_zone_info[zidx]; region1 = 0; region3 = swz->size - hwb->size; /* * Stay within this zone and see if there is a better match when mbuf * inlining is allowed. Remember that the hwidx's are sorted in * decreasing order of size (so in increasing order of spare area). */ for (idx = hwidx; idx != -1; idx = hwb->next) { hwb = &hwb_list[idx]; spare = swz->size - hwb->size; if (allow_mbufs_in_cluster == 0 || hwb->size < maxp) break; /* * Do not inline mbufs if doing so would violate the pad/pack * boundary alignment requirement. */ if (fl_pad && (MSIZE % sc->params.sge.pad_boundary) != 0) continue; if (fl->flags & FL_BUF_PACKING && (MSIZE % sc->params.sge.pack_boundary) != 0) continue; if (spare < CL_METADATA_SIZE + MSIZE) continue; n = (spare - CL_METADATA_SIZE) / MSIZE; if (n > howmany(hwb->size, maxp)) break; hwidx = idx; if (fl->flags & FL_BUF_PACKING) { region1 = n * MSIZE; region3 = spare - region1; } else { region1 = MSIZE; region3 = spare - region1; break; } } KASSERT(zidx >= 0 && zidx < SW_ZONE_SIZES, ("%s: bad zone %d for fl %p, maxp %d", __func__, zidx, fl, maxp)); KASSERT(hwidx >= 0 && hwidx <= SGE_FLBUF_SIZES, ("%s: bad hwidx %d for fl %p, maxp %d", __func__, hwidx, fl, maxp)); KASSERT(region1 + sc->sge.hw_buf_info[hwidx].size + region3 == sc->sge.sw_zone_info[zidx].size, ("%s: bad buffer layout for fl %p, maxp %d. " "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, sc->sge.sw_zone_info[zidx].size, region1, sc->sge.hw_buf_info[hwidx].size, region3)); if (fl->flags & FL_BUF_PACKING || region1 > 0) { KASSERT(region3 >= CL_METADATA_SIZE, ("%s: no room for metadata. fl %p, maxp %d; " "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, sc->sge.sw_zone_info[zidx].size, region1, sc->sge.hw_buf_info[hwidx].size, region3)); KASSERT(region1 % MSIZE == 0, ("%s: bad mbuf region for fl %p, maxp %d. " "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, sc->sge.sw_zone_info[zidx].size, region1, sc->sge.hw_buf_info[hwidx].size, region3)); } fl->cll_def.zidx = zidx; fl->cll_def.hwidx = hwidx; fl->cll_def.region1 = region1; fl->cll_def.region3 = region3; } static void find_safe_refill_source(struct adapter *sc, struct sge_fl *fl) { struct sge *s = &sc->sge; struct hw_buf_info *hwb; struct sw_zone_info *swz; int spare; int8_t hwidx; if (fl->flags & FL_BUF_PACKING) hwidx = s->safe_hwidx2; /* with room for metadata */ else if (allow_mbufs_in_cluster && s->safe_hwidx2 != -1) { hwidx = s->safe_hwidx2; hwb = &s->hw_buf_info[hwidx]; swz = &s->sw_zone_info[hwb->zidx]; spare = swz->size - hwb->size; /* no good if there isn't room for an mbuf as well */ if (spare < CL_METADATA_SIZE + MSIZE) hwidx = s->safe_hwidx1; } else hwidx = s->safe_hwidx1; if (hwidx == -1) { /* No fallback source */ fl->cll_alt.hwidx = -1; fl->cll_alt.zidx = -1; return; } hwb = &s->hw_buf_info[hwidx]; swz = &s->sw_zone_info[hwb->zidx]; spare = swz->size - hwb->size; fl->cll_alt.hwidx = hwidx; fl->cll_alt.zidx = hwb->zidx; if (allow_mbufs_in_cluster && (fl_pad == 0 || (MSIZE % sc->params.sge.pad_boundary) == 0)) fl->cll_alt.region1 = ((spare - CL_METADATA_SIZE) / MSIZE) * MSIZE; else fl->cll_alt.region1 = 0; fl->cll_alt.region3 = spare - fl->cll_alt.region1; } static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) { mtx_lock(&sc->sfl_lock); FL_LOCK(fl); if ((fl->flags & FL_DOOMED) == 0) { fl->flags |= FL_STARVING; TAILQ_INSERT_TAIL(&sc->sfl, fl, link); callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc); } FL_UNLOCK(fl); mtx_unlock(&sc->sfl_lock); } static void handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq) { struct sge_wrq *wrq = (void *)eq; atomic_readandclear_int(&eq->equiq); taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task); } static void handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq) { struct sge_txq *txq = (void *)eq; MPASS((eq->flags & EQ_TYPEMASK) == EQ_ETH); atomic_readandclear_int(&eq->equiq); mp_ring_check_drainage(txq->r, 0); taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task); } static int handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); struct adapter *sc = iq->adapter; struct sge *s = &sc->sge; struct sge_eq *eq; static void (*h[])(struct adapter *, struct sge_eq *) = {NULL, &handle_wrq_egr_update, &handle_eth_egr_update, &handle_wrq_egr_update}; KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, rss->opcode)); eq = s->eqmap[qid - s->eq_start - s->eq_base]; (*h[eq->flags & EQ_TYPEMASK])(sc, eq); return (0); } /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */ CTASSERT(offsetof(struct cpl_fw4_msg, data) == \ offsetof(struct cpl_fw6_msg, data)); static int handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, rss->opcode)); if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { const struct rss_header *rss2; rss2 = (const struct rss_header *)&cpl->data[0]; return (t4_cpl_handler[rss2->opcode](iq, rss2, m)); } return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0])); } /** * t4_handle_wrerr_rpl - process a FW work request error message * @adap: the adapter * @rpl: start of the FW message */ static int t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl) { u8 opcode = *(const u8 *)rpl; const struct fw_error_cmd *e = (const void *)rpl; unsigned int i; if (opcode != FW_ERROR_CMD) { log(LOG_ERR, "%s: Received WRERR_RPL message with opcode %#x\n", device_get_nameunit(adap->dev), opcode); return (EINVAL); } log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev), G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" : "non-fatal"); switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) { case FW_ERROR_TYPE_EXCEPTION: log(LOG_ERR, "exception info:\n"); for (i = 0; i < nitems(e->u.exception.info); i++) log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ", be32toh(e->u.exception.info[i])); log(LOG_ERR, "\n"); break; case FW_ERROR_TYPE_HWMODULE: log(LOG_ERR, "HW module regaddr %08x regval %08x\n", be32toh(e->u.hwmodule.regaddr), be32toh(e->u.hwmodule.regval)); break; case FW_ERROR_TYPE_WR: log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n", be16toh(e->u.wr.cidx), G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)), G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)), be32toh(e->u.wr.eqid)); for (i = 0; i < nitems(e->u.wr.wrhdr); i++) log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ", e->u.wr.wrhdr[i]); log(LOG_ERR, "\n"); break; case FW_ERROR_TYPE_ACL: log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s", be16toh(e->u.acl.cidx), G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)), G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)), be32toh(e->u.acl.eqid), G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" : "MAC"); for (i = 0; i < nitems(e->u.acl.val); i++) log(LOG_ERR, " %02x", e->u.acl.val[i]); log(LOG_ERR, "\n"); break; default: log(LOG_ERR, "type %#x\n", G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))); return (EINVAL); } return (0); } static int sysctl_uint16(SYSCTL_HANDLER_ARGS) { uint16_t *id = arg1; int i = *id; return sysctl_handle_int(oidp, &i, 0, req); } static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS) { struct sge *s = arg1; struct hw_buf_info *hwb = &s->hw_buf_info[0]; struct sw_zone_info *swz = &s->sw_zone_info[0]; int i, rc; struct sbuf sb; char c; sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) { if (hwb->zidx >= 0 && swz[hwb->zidx].size <= largest_rx_cluster) c = '*'; else c = '\0'; sbuf_printf(&sb, "%u%c ", hwb->size, c); } sbuf_trim(&sb); sbuf_finish(&sb); rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); sbuf_delete(&sb); return (rc); } static int sysctl_tc(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct port_info *pi; struct adapter *sc; struct sge_txq *txq; struct tx_cl_rl_params *tc; int qidx = arg2, rc, tc_idx; uint32_t fw_queue, fw_class; MPASS(qidx >= 0 && qidx < vi->ntxq); pi = vi->pi; sc = pi->adapter; txq = &sc->sge.txq[vi->first_txq + qidx]; tc_idx = txq->tc_idx; rc = sysctl_handle_int(oidp, &tc_idx, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (sc->flags & IS_VF) return (EPERM); /* Note that -1 is legitimate input (it means unbind). */ if (tc_idx < -1 || tc_idx >= sc->chip_params->nsched_cls) return (EINVAL); mtx_lock(&sc->tc_lock); if (tc_idx == txq->tc_idx) { rc = 0; /* No change, nothing to do. */ goto done; } fw_queue = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH) | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id); if (tc_idx == -1) fw_class = 0xffffffff; /* Unbind. */ else { /* * Bind to a different class. */ tc = &pi->sched_params->cl_rl[tc_idx]; if (tc->flags & TX_CLRL_ERROR) { /* Previous attempt to set the cl-rl params failed. */ rc = EIO; goto done; } else { /* * Ok to proceed. Place a reference on the new class * while still holding on to the reference on the * previous class, if any. */ fw_class = tc_idx; tc->refcount++; } } mtx_unlock(&sc->tc_lock); rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4stc"); if (rc) return (rc); rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, &fw_class); end_synchronized_op(sc, 0); mtx_lock(&sc->tc_lock); if (rc == 0) { if (txq->tc_idx != -1) { tc = &pi->sched_params->cl_rl[txq->tc_idx]; MPASS(tc->refcount > 0); tc->refcount--; } txq->tc_idx = tc_idx; } else if (tc_idx != -1) { tc = &pi->sched_params->cl_rl[tc_idx]; MPASS(tc->refcount > 0); tc->refcount--; } done: mtx_unlock(&sc->tc_lock); return (rc); } Index: stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c =================================================================== --- stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c (revision 346862) +++ stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c (revision 346863) @@ -1,2379 +1,2355 @@ /*- * Copyright (c) 2012, 2015 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #ifdef TCP_OFFLOAD #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define TCPSTATES #include #include #include #include #include #include #include #include #include #include #include "common/common.h" #include "common/t4_msg.h" #include "common/t4_regs.h" #include "common/t4_tcb.h" #include "tom/t4_tom_l2t.h" #include "tom/t4_tom.h" VNET_DECLARE(int, tcp_do_autosndbuf); #define V_tcp_do_autosndbuf VNET(tcp_do_autosndbuf) VNET_DECLARE(int, tcp_autosndbuf_inc); #define V_tcp_autosndbuf_inc VNET(tcp_autosndbuf_inc) VNET_DECLARE(int, tcp_autosndbuf_max); #define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max) VNET_DECLARE(int, tcp_do_autorcvbuf); #define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) VNET_DECLARE(int, tcp_autorcvbuf_inc); #define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) VNET_DECLARE(int, tcp_autorcvbuf_max); #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) static void t4_aiotx_cancel(struct kaiocb *job); static void t4_aiotx_queue_toep(struct toepcb *toep); static size_t aiotx_mbuf_pgoff(struct mbuf *m) { struct aiotx_buffer *ab; MPASS(IS_AIOTX_MBUF(m)); ab = m->m_ext.ext_arg1; return ((ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) % PAGE_SIZE); } static vm_page_t * aiotx_mbuf_pages(struct mbuf *m) { struct aiotx_buffer *ab; int npages; MPASS(IS_AIOTX_MBUF(m)); ab = m->m_ext.ext_arg1; npages = (ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) / PAGE_SIZE; return (ab->ps.pages + npages); } void send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) { struct wrqe *wr; struct fw_flowc_wr *flowc; unsigned int nparams, flowclen, paramidx; struct vi_info *vi = toep->vi; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; unsigned int pfvf = G_FW_VIID_PFN(vi->viid) << S_FW_VIID_PFN; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT), ("%s: flowc for tid %u sent already", __func__, toep->tid)); if (ftxp != NULL) nparams = 8; else nparams = 6; if (toep->ulp_mode == ULP_MODE_TLS) nparams++; if (toep->tls.fcplenmax != 0) nparams++; if (toep->tc_idx != -1) { MPASS(toep->tc_idx >= 0 && toep->tc_idx < sc->chip_params->nsched_cls); nparams++; } flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } flowc = wrtod(wr); memset(flowc, 0, wr->wr_len); flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | V_FW_FLOWC_WR_NPARAMS(nparams)); flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | V_FW_WR_FLOWID(toep->tid)); #define FLOWC_PARAM(__m, __v) \ do { \ flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \ flowc->mnemval[paramidx].val = htobe32(__v); \ paramidx++; \ } while (0) paramidx = 0; FLOWC_PARAM(PFNVFN, pfvf); FLOWC_PARAM(CH, pi->tx_chan); FLOWC_PARAM(PORT, pi->tx_chan); FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id); if (ftxp) { uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf); FLOWC_PARAM(SNDNXT, ftxp->snd_nxt); FLOWC_PARAM(RCVNXT, ftxp->rcv_nxt); FLOWC_PARAM(SNDBUF, sndbuf); FLOWC_PARAM(MSS, ftxp->mss); CTR6(KTR_CXGBE, "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x", __func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt, ftxp->rcv_nxt); } else { FLOWC_PARAM(SNDBUF, 512); FLOWC_PARAM(MSS, 512); CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid); } if (toep->ulp_mode == ULP_MODE_TLS) FLOWC_PARAM(ULP_MODE, toep->ulp_mode); if (toep->tls.fcplenmax != 0) FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax); if (toep->tc_idx != -1) FLOWC_PARAM(SCHEDCLASS, toep->tc_idx); #undef FLOWC_PARAM KASSERT(paramidx == nparams, ("nparams mismatch")); txsd->tx_credits = howmany(flowclen, 16); txsd->plen = 0; KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, ("%s: not enough credits (%d)", __func__, toep->tx_credits)); toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; toep->flags |= TPF_FLOWC_WR_SENT; t4_wrq_tx(sc, wr); } #ifdef RATELIMIT /* * Input is Bytes/second (so_max_pacing-rate), chip counts in Kilobits/second. */ static int update_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps) { int tc_idx, rc; const u_int kbps = (u_int) (uint64_t)Bps * 8ULL / 1000; const int port_id = toep->vi->pi->port_id; CTR3(KTR_CXGBE, "%s: tid %u, rate %uKbps", __func__, toep->tid, kbps); if (kbps == 0) { /* unbind */ tc_idx = -1; } else { rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx); if (rc != 0) return (rc); MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls); } if (toep->tc_idx != tc_idx) { struct wrqe *wr; struct fw_flowc_wr *flowc; int nparams = 1, flowclen, flowclen16; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); flowclen16 = howmany(flowclen, 16); if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0 || (wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq)) == NULL) { if (tc_idx >= 0) t4_release_cl_rl_kbps(sc, port_id, tc_idx); return (ENOMEM); } flowc = wrtod(wr); memset(flowc, 0, wr->wr_len); flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | V_FW_FLOWC_WR_NPARAMS(nparams)); flowc->flowid_len16 = htonl(V_FW_WR_LEN16(flowclen16) | V_FW_WR_FLOWID(toep->tid)); flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; if (tc_idx == -1) flowc->mnemval[0].val = htobe32(0xff); else flowc->mnemval[0].val = htobe32(tc_idx); txsd->tx_credits = flowclen16; txsd->plen = 0; toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; t4_wrq_tx(sc, wr); } if (toep->tc_idx >= 0) t4_release_cl_rl_kbps(sc, port_id, toep->tc_idx); toep->tc_idx = tc_idx; return (0); } #endif void send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt) { struct wrqe *wr; struct cpl_abort_req *req; int tid = toep->tid; struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */ INP_WLOCK_ASSERT(inp); CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s", __func__, toep->tid, inp->inp_flags & INP_DROPPED ? "inp dropped" : tcpstates[tp->t_state], toep->flags, inp->inp_flags, toep->flags & TPF_ABORT_SHUTDOWN ? " (abort already in progress)" : ""); if (toep->flags & TPF_ABORT_SHUTDOWN) return; /* abort already in progress */ toep->flags |= TPF_ABORT_SHUTDOWN; KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %d.", __func__, tid)); wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid); if (inp->inp_flags & INP_DROPPED) req->rsvd0 = htobe32(snd_nxt); else req->rsvd0 = htobe32(tp->snd_nxt); req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT); req->cmd = CPL_ABORT_SEND_RST; /* * XXX: What's the correct way to tell that the inp hasn't been detached * from its socket? Should I even be flushing the snd buffer here? */ if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { struct socket *so = inp->inp_socket; if (so != NULL) /* because I'm not sure. See comment above */ sbflush(&so->so_snd); } t4_l2t_send(sc, wr, toep->l2te); } /* * Called when a connection is established to translate the TCP options * reported by HW to FreeBSD's native format. */ static void assign_rxopt(struct tcpcb *tp, unsigned int opt) { struct toepcb *toep = tp->t_toe; struct inpcb *inp = tp->t_inpcb; struct adapter *sc = td_adapter(toep->td); int n; INP_LOCK_ASSERT(inp); if (inp->inp_inc.inc_flags & INC_ISIPV6) n = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); else n = sizeof(struct ip) + sizeof(struct tcphdr); tp->t_maxseg = sc->params.mtus[G_TCPOPT_MSS(opt)] - n; if (G_TCPOPT_TSTAMP(opt)) { tp->t_flags |= TF_RCVD_TSTMP; /* timestamps ok */ tp->ts_recent = 0; /* hmmm */ tp->ts_recent_age = tcp_ts_getticks(); tp->t_maxseg -= TCPOLEN_TSTAMP_APPA; } CTR5(KTR_CXGBE, "%s: tid %d, mtu_idx %u (%u), mss %u", __func__, toep->tid, G_TCPOPT_MSS(opt), sc->params.mtus[G_TCPOPT_MSS(opt)], tp->t_maxseg); if (G_TCPOPT_SACK(opt)) tp->t_flags |= TF_SACK_PERMIT; /* should already be set */ else tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */ if (G_TCPOPT_WSCALE_OK(opt)) tp->t_flags |= TF_RCVD_SCALE; /* Doing window scaling? */ if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == (TF_RCVD_SCALE | TF_REQ_SCALE)) { tp->rcv_scale = tp->request_r_scale; tp->snd_scale = G_TCPOPT_SND_WSCALE(opt); } } /* * Completes some final bits of initialization for just established connections * and changes their state to TCPS_ESTABLISHED. * * The ISNs are from after the exchange of SYNs. i.e., the true ISN + 1. */ void make_established(struct toepcb *toep, uint32_t snd_isn, uint32_t rcv_isn, uint16_t opt) { struct inpcb *inp = toep->inp; struct socket *so = inp->inp_socket; struct tcpcb *tp = intotcpcb(inp); long bufsize; uint32_t iss = be32toh(snd_isn) - 1; /* true ISS */ uint32_t irs = be32toh(rcv_isn) - 1; /* true IRS */ uint16_t tcpopt = be16toh(opt); struct flowc_tx_params ftxp; INP_WLOCK_ASSERT(inp); KASSERT(tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_SYN_RECEIVED, ("%s: TCP state %s", __func__, tcpstates[tp->t_state])); CTR6(KTR_CXGBE, "%s: tid %d, so %p, inp %p, tp %p, toep %p", __func__, toep->tid, so, inp, tp, toep); tcp_state_change(tp, TCPS_ESTABLISHED); tp->t_starttime = ticks; TCPSTAT_INC(tcps_connects); tp->irs = irs; tcp_rcvseqinit(tp); tp->rcv_wnd = toep->rx_credits << 10; tp->rcv_adv += tp->rcv_wnd; tp->last_ack_sent = tp->rcv_nxt; /* * If we were unable to send all rx credits via opt0, save the remainder * in rx_credits so that they can be handed over with the next credit * update. */ SOCKBUF_LOCK(&so->so_rcv); bufsize = select_rcv_wnd(so); SOCKBUF_UNLOCK(&so->so_rcv); toep->rx_credits = bufsize - tp->rcv_wnd; tp->iss = iss; tcp_sendseqinit(tp); tp->snd_una = iss + 1; tp->snd_nxt = iss + 1; tp->snd_max = iss + 1; assign_rxopt(tp, tcpopt); SOCKBUF_LOCK(&so->so_snd); if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf) bufsize = V_tcp_autosndbuf_max; else bufsize = sbspace(&so->so_snd); SOCKBUF_UNLOCK(&so->so_snd); ftxp.snd_nxt = tp->snd_nxt; ftxp.rcv_nxt = tp->rcv_nxt; ftxp.snd_space = bufsize; ftxp.mss = tp->t_maxseg; send_flowc_wr(toep, &ftxp); soisconnected(so); } int send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits) { struct wrqe *wr; struct cpl_rx_data_ack *req; uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); KASSERT(credits >= 0, ("%s: %d credits", __func__, credits)); wr = alloc_wrqe(sizeof(*req), toep->ctrlq); if (wr == NULL) return (0); req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); req->credit_dack = htobe32(dack | V_RX_CREDITS(credits)); t4_wrq_tx(sc, wr); return (credits); } void send_rx_modulate(struct adapter *sc, struct toepcb *toep) { struct wrqe *wr; struct cpl_rx_data_ack *req; wr = alloc_wrqe(sizeof(*req), toep->ctrlq); if (wr == NULL) return; req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); req->credit_dack = htobe32(F_RX_MODULATE_RX); t4_wrq_tx(sc, wr); } void t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp) { struct adapter *sc = tod->tod_softc; struct inpcb *inp = tp->t_inpcb; struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_rcv; struct toepcb *toep = tp->t_toe; int credits; INP_WLOCK_ASSERT(inp); SOCKBUF_LOCK_ASSERT(sb); KASSERT(toep->sb_cc >= sbused(sb), ("%s: sb %p has more data (%d) than last time (%d).", __func__, sb, sbused(sb), toep->sb_cc)); credits = toep->sb_cc - sbused(sb); toep->sb_cc = sbused(sb); if (toep->ulp_mode == ULP_MODE_TLS) { if (toep->tls.rcv_over >= credits) { toep->tls.rcv_over -= credits; credits = 0; } else { credits -= toep->tls.rcv_over; toep->tls.rcv_over = 0; } } toep->rx_credits += credits; if (toep->rx_credits > 0 && (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 || (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || toep->sb_cc + tp->rcv_wnd < sb->sb_lowat)) { credits = send_rx_credits(sc, toep, toep->rx_credits); toep->rx_credits -= credits; tp->rcv_wnd += credits; tp->rcv_adv += credits; } else if (toep->flags & TPF_FORCE_CREDITS) send_rx_modulate(sc, toep); } void t4_rcvd(struct toedev *tod, struct tcpcb *tp) { struct inpcb *inp = tp->t_inpcb; struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_rcv; SOCKBUF_LOCK(sb); t4_rcvd_locked(tod, tp); SOCKBUF_UNLOCK(sb); } /* * Close a connection by sending a CPL_CLOSE_CON_REQ message. */ int t4_close_conn(struct adapter *sc, struct toepcb *toep) { struct wrqe *wr; struct cpl_close_con_req *req; unsigned int tid = toep->tid; CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid, toep->flags & TPF_FIN_SENT ? ", IGNORED" : ""); if (toep->flags & TPF_FIN_SENT) return (0); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, tid)); wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } req = wrtod(wr); req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) | V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr))); req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) | V_FW_WR_FLOWID(tid)); req->wr.wr_lo = cpu_to_be64(0); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); req->rsvd = 0; toep->flags |= TPF_FIN_SENT; toep->flags &= ~TPF_SEND_FIN; t4_l2t_send(sc, wr, toep->l2te); return (0); } #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) #define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16)) /* Maximum amount of immediate data we could stuff in a WR */ static inline int max_imm_payload(int tx_credits) { const int n = 2; /* Use only up to 2 desc for imm. data WR */ KASSERT(tx_credits >= 0 && tx_credits <= MAX_OFLD_TX_CREDITS, ("%s: %d credits", __func__, tx_credits)); if (tx_credits < MIN_OFLD_TX_CREDITS) return (0); if (tx_credits >= (n * EQ_ESIZE) / 16) return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr)); else return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr)); } /* Maximum number of SGL entries we could stuff in a WR */ static inline int max_dsgl_nsegs(int tx_credits) { int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */ int sge_pair_credits = tx_credits - MIN_OFLD_TX_CREDITS; KASSERT(tx_credits >= 0 && tx_credits <= MAX_OFLD_TX_CREDITS, ("%s: %d credits", __func__, tx_credits)); if (tx_credits < MIN_OFLD_TX_CREDITS) return (0); nseg += 2 * (sge_pair_credits * 16 / 24); if ((sge_pair_credits * 16) % 24 == 16) nseg++; return (nseg); } static inline void write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign) { struct fw_ofld_tx_data_wr *txwr = dst; txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | V_FW_WR_IMMDLEN(immdlen)); txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | V_FW_WR_LEN16(credits)); txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) | V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove)); txwr->plen = htobe32(plen); if (txalign > 0) { struct tcpcb *tp = intotcpcb(toep->inp); if (plen < 2 * tp->t_maxseg) txwr->lsodisable_to_flags |= htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE); else txwr->lsodisable_to_flags |= htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD | (tp->t_flags & TF_NODELAY ? 0 : F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE)); } } /* * Generate a DSGL from a starting mbuf. The total number of segments and the * maximum segments in any one mbuf are provided. */ static void write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) { struct mbuf *m; struct ulptx_sgl *usgl = dst; int i, j, rc; struct sglist sg; struct sglist_seg segs[n]; KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); sglist_init(&sg, n, segs); usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(nsegs)); i = -1; for (m = start; m != stop; m = m->m_next) { if (IS_AIOTX_MBUF(m)) rc = sglist_append_vmpages(&sg, aiotx_mbuf_pages(m), aiotx_mbuf_pgoff(m), m->m_len); else rc = sglist_append(&sg, mtod(m, void *), m->m_len); if (__predict_false(rc != 0)) panic("%s: sglist_append %d", __func__, rc); for (j = 0; j < sg.sg_nseg; i++, j++) { if (i < 0) { usgl->len0 = htobe32(segs[j].ss_len); usgl->addr0 = htobe64(segs[j].ss_paddr); } else { usgl->sge[i / 2].len[i & 1] = htobe32(segs[j].ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(segs[j].ss_paddr); } #ifdef INVARIANTS nsegs--; #endif } sglist_reset(&sg); } if (i & 1) usgl->sge[i / 2].len[1] = htobe32(0); KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p", __func__, nsegs, start, stop)); } /* * Max number of SGL entries an offload tx work request can have. This is 41 * (1 + 40) for a full 512B work request. * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40) */ #define OFLD_SGL_LEN (41) /* * Send data and/or a FIN to the peer. * * The socket's so_snd buffer consists of a stream of data starting with sb_mb * and linked together with m_next. sb_sndptr, if set, is the last mbuf that * was transmitted. * * drop indicates the number of bytes that should be dropped from the head of * the send buffer. It is an optimization that lets do_fw4_ack avoid creating * contention on the send buffer lock (before this change it used to do * sowwakeup and then t4_push_frames right after that when recovering from tx * stalls). When drop is set this function MUST drop the bytes and wake up any * writers. */ void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) { struct mbuf *sndptr, *m, *sb_sndptr; struct fw_ofld_tx_data_wr *txwr; struct wrqe *wr; u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_snd; int tx_credits, shove, compl, sowwakeup; struct ofld_tx_sdesc *txsd; bool aiotx_mbuf_seen; INP_WLOCK_ASSERT(inp); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); KASSERT(toep->ulp_mode == ULP_MODE_NONE || toep->ulp_mode == ULP_MODE_TCPDDP || toep->ulp_mode == ULP_MODE_TLS || toep->ulp_mode == ULP_MODE_RDMA, ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", __func__, toep->tid, toep->flags, tp->t_flags); #endif if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) return; #ifdef RATELIMIT if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; } #endif /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. */ if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { KASSERT(drop == 0, ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); return; } txsd = &toep->txsd[toep->txsd_pidx]; do { tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); max_imm = max_imm_payload(tx_credits); max_nsegs = max_dsgl_nsegs(tx_credits); SOCKBUF_LOCK(sb); sowwakeup = drop; if (drop) { sbdrop_locked(sb, drop); drop = 0; } sb_sndptr = sb->sb_sndptr; sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; plen = 0; nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ aiotx_mbuf_seen = false; for (m = sndptr; m != NULL; m = m->m_next) { int n; if (IS_AIOTX_MBUF(m)) n = sglist_count_vmpages(aiotx_mbuf_pages(m), aiotx_mbuf_pgoff(m), m->m_len); else n = sglist_count(mtod(m, void *), m->m_len); nsegs += n; plen += m->m_len; /* This mbuf sent us _over_ the nsegs limit, back out */ if (plen > max_imm && nsegs > max_nsegs) { nsegs -= n; plen -= m->m_len; if (plen == 0) { /* Too few credits */ toep->flags |= TPF_TX_SUSPENDED; if (sowwakeup) { if (!TAILQ_EMPTY( &toep->aiotx_jobq)) t4_aiotx_queue_toep( toep); sowwakeup_locked(so); } else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); return; } break; } if (IS_AIOTX_MBUF(m)) aiotx_mbuf_seen = true; if (max_nsegs_1mbuf < n) max_nsegs_1mbuf = n; sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ /* This mbuf put us right at the max_nsegs limit */ if (plen > max_imm && nsegs == max_nsegs) { m = m->m_next; break; } } if (sbused(sb) > sb->sb_hiwat * 5 / 8 && toep->plen_nocompl + plen >= sb->sb_hiwat / 4) compl = 1; else compl = 0; if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf && sb->sb_hiwat < V_tcp_autosndbuf_max && sbused(sb) >= sb->sb_hiwat * 7 / 8) { int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, V_tcp_autosndbuf_max); if (!sbreserve_locked(sb, newsize, so, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; else sowwakeup = 1; /* room available */ } if (sowwakeup) { if (!TAILQ_EMPTY(&toep->aiotx_jobq)) t4_aiotx_queue_toep(toep); sowwakeup_locked(so); } else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); /* nothing to send */ if (plen == 0) { KASSERT(m == NULL, ("%s: nothing to send, but m != NULL", __func__)); break; } if (__predict_false(toep->flags & TPF_FIN_SENT)) panic("%s: excess tx.", __func__); shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); if (plen <= max_imm && !aiotx_mbuf_seen) { /* Immediate data tx */ wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; return; } txwr = wrtod(wr); credits = howmany(wr->wr_len, 16); write_tx_wr(txwr, toep, plen, plen, credits, shove, 0, sc->tt.tx_align); m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); nsegs = 0; } else { int wr_len; /* DSGL tx */ wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; return; } txwr = wrtod(wr); credits = howmany(wr_len, 16); write_tx_wr(txwr, toep, 0, plen, credits, shove, 0, sc->tt.tx_align); write_tx_sgl(txwr + 1, sndptr, m, nsegs, max_nsegs_1mbuf); if (wr_len & 0xf) { uint64_t *pad = (uint64_t *) ((uintptr_t)txwr + wr_len); *pad = 0; } } KASSERT(toep->tx_credits >= credits, ("%s: not enough credits", __func__)); toep->tx_credits -= credits; toep->tx_nocompl += credits; toep->plen_nocompl += plen; if (toep->tx_credits <= toep->tx_total * 3 / 8 && toep->tx_nocompl >= toep->tx_total / 4) compl = 1; if (compl || toep->ulp_mode == ULP_MODE_RDMA) { txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); toep->tx_nocompl = 0; toep->plen_nocompl = 0; } tp->snd_nxt += plen; tp->snd_max += plen; SOCKBUF_LOCK(sb); KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__)); sb->sb_sndptr = sb_sndptr; SOCKBUF_UNLOCK(sb); toep->flags |= TPF_TX_DATA_SENT; if (toep->tx_credits < MIN_OFLD_TX_CREDITS) toep->flags |= TPF_TX_SUSPENDED; KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); txsd->plen = plen; txsd->tx_credits = credits; txsd++; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { toep->txsd_pidx = 0; txsd = &toep->txsd[0]; } toep->txsd_avail--; t4_l2t_send(sc, wr, toep->l2te); } while (m != NULL); /* Send a FIN if requested, but only if there's no more data to send */ if (m == NULL && toep->flags & TPF_SEND_FIN) t4_close_conn(sc, toep); } static inline void rqdrop_locked(struct mbufq *q, int plen) { struct mbuf *m; while (plen > 0) { m = mbufq_dequeue(q); /* Too many credits. */ MPASS(m != NULL); M_ASSERTPKTHDR(m); /* Partial credits. */ MPASS(plen >= m->m_pkthdr.len); plen -= m->m_pkthdr.len; m_freem(m); } } void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) { struct mbuf *sndptr, *m; struct fw_ofld_tx_data_wr *txwr; struct wrqe *wr; u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; u_int adjusted_plen, ulp_submode; struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); int tx_credits, shove; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; struct mbufq *pduq = &toep->ulp_pduq; static const u_int ulp_extra_len[] = {0, 4, 4, 8}; INP_WLOCK_ASSERT(inp); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); KASSERT(toep->ulp_mode == ULP_MODE_ISCSI, ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) return; /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. */ if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { KASSERT(drop == 0, ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); return; } if (drop) rqdrop_locked(&toep->ulp_pdu_reclaimq, drop); while ((sndptr = mbufq_first(pduq)) != NULL) { M_ASSERTPKTHDR(sndptr); tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); max_imm = max_imm_payload(tx_credits); max_nsegs = max_dsgl_nsegs(tx_credits); plen = 0; nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ for (m = sndptr; m != NULL; m = m->m_next) { int n = sglist_count(mtod(m, void *), m->m_len); nsegs += n; plen += m->m_len; /* * This mbuf would send us _over_ the nsegs limit. * Suspend tx because the PDU can't be sent out. */ if (plen > max_imm && nsegs > max_nsegs) { toep->flags |= TPF_TX_SUSPENDED; return; } if (max_nsegs_1mbuf < n) max_nsegs_1mbuf = n; } if (__predict_false(toep->flags & TPF_FIN_SENT)) panic("%s: excess tx.", __func__); /* * We have a PDU to send. All of it goes out in one WR so 'm' * is NULL. A PDU's length is always a multiple of 4. */ MPASS(m == NULL); MPASS((plen & 3) == 0); MPASS(sndptr->m_pkthdr.len == plen); shove = !(tp->t_flags & TF_MORETOCOME); ulp_submode = mbuf_ulp_submode(sndptr); MPASS(ulp_submode < nitems(ulp_extra_len)); /* * plen doesn't include header and data digests, which are * generated and inserted in the right places by the TOE, but * they do occupy TCP sequence space and need to be accounted * for. */ adjusted_plen = plen + ulp_extra_len[ulp_submode]; if (plen <= max_imm) { /* Immediate data tx */ wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; return; } txwr = wrtod(wr); credits = howmany(wr->wr_len, 16); write_tx_wr(txwr, toep, plen, adjusted_plen, credits, shove, ulp_submode, sc->tt.tx_align); m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); nsegs = 0; } else { int wr_len; /* DSGL tx */ wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; return; } txwr = wrtod(wr); credits = howmany(wr_len, 16); write_tx_wr(txwr, toep, 0, adjusted_plen, credits, shove, ulp_submode, sc->tt.tx_align); write_tx_sgl(txwr + 1, sndptr, m, nsegs, max_nsegs_1mbuf); if (wr_len & 0xf) { uint64_t *pad = (uint64_t *) ((uintptr_t)txwr + wr_len); *pad = 0; } } KASSERT(toep->tx_credits >= credits, ("%s: not enough credits", __func__)); m = mbufq_dequeue(pduq); MPASS(m == sndptr); mbufq_enqueue(&toep->ulp_pdu_reclaimq, m); toep->tx_credits -= credits; toep->tx_nocompl += credits; toep->plen_nocompl += plen; if (toep->tx_credits <= toep->tx_total * 3 / 8 && toep->tx_nocompl >= toep->tx_total / 4) { txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); toep->tx_nocompl = 0; toep->plen_nocompl = 0; } tp->snd_nxt += adjusted_plen; tp->snd_max += adjusted_plen; toep->flags |= TPF_TX_DATA_SENT; if (toep->tx_credits < MIN_OFLD_TX_CREDITS) toep->flags |= TPF_TX_SUSPENDED; KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); txsd->plen = plen; txsd->tx_credits = credits; txsd++; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { toep->txsd_pidx = 0; txsd = &toep->txsd[0]; } toep->txsd_avail--; t4_l2t_send(sc, wr, toep->l2te); } /* Send a FIN if requested, but only if there are no more PDUs to send */ if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) t4_close_conn(sc, toep); } int t4_tod_output(struct toedev *tod, struct tcpcb *tp) { struct adapter *sc = tod->tod_softc; #ifdef INVARIANTS struct inpcb *inp = tp->t_inpcb; #endif struct toepcb *toep = tp->t_toe; INP_WLOCK_ASSERT(inp); KASSERT((inp->inp_flags & INP_DROPPED) == 0, ("%s: inp %p dropped.", __func__, inp)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); if (toep->ulp_mode == ULP_MODE_ISCSI) t4_push_pdus(sc, toep, 0); else if (tls_tx_key(toep)) t4_push_tls_records(sc, toep, 0); else t4_push_frames(sc, toep, 0); return (0); } int t4_send_fin(struct toedev *tod, struct tcpcb *tp) { struct adapter *sc = tod->tod_softc; #ifdef INVARIANTS struct inpcb *inp = tp->t_inpcb; #endif struct toepcb *toep = tp->t_toe; INP_WLOCK_ASSERT(inp); KASSERT((inp->inp_flags & INP_DROPPED) == 0, ("%s: inp %p dropped.", __func__, inp)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); toep->flags |= TPF_SEND_FIN; if (tp->t_state >= TCPS_ESTABLISHED) { if (toep->ulp_mode == ULP_MODE_ISCSI) t4_push_pdus(sc, toep, 0); else if (tls_tx_key(toep)) t4_push_tls_records(sc, toep, 0); else t4_push_frames(sc, toep, 0); } return (0); } int t4_send_rst(struct toedev *tod, struct tcpcb *tp) { struct adapter *sc = tod->tod_softc; #if defined(INVARIANTS) struct inpcb *inp = tp->t_inpcb; #endif struct toepcb *toep = tp->t_toe; INP_WLOCK_ASSERT(inp); KASSERT((inp->inp_flags & INP_DROPPED) == 0, ("%s: inp %p dropped.", __func__, inp)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); /* hmmmm */ KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc for tid %u [%s] not sent already", __func__, toep->tid, tcpstates[tp->t_state])); send_reset(sc, toep, 0); return (0); } /* * Peer has sent us a FIN. */ static int do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_peer_close *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; struct tcpcb *tp = NULL; struct socket *so; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif KASSERT(opcode == CPL_PEER_CLOSE, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); if (__predict_false(toep->flags & TPF_SYNQE)) { #ifdef INVARIANTS struct synq_entry *synqe = (void *)toep; INP_WLOCK(synqe->lctx->inp); if (synqe->flags & TPF_SYNQE_HAS_L2TE) { KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN, ("%s: listen socket closed but tid %u not aborted.", __func__, tid)); } else { /* * do_pass_accept_req is still running and will * eventually take care of this tid. */ } INP_WUNLOCK(synqe->lctx->inp); #endif CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, toep, toep->flags); return (0); } KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); CURVNET_SET(toep->vnet); INP_INFO_RLOCK(&V_tcbinfo); INP_WLOCK(inp); tp = intotcpcb(inp); CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp); if (toep->flags & TPF_ABORT_SHUTDOWN) goto done; tp->rcv_nxt++; /* FIN */ so = inp->inp_socket; if (toep->ulp_mode == ULP_MODE_TCPDDP) { DDP_LOCK(toep); if (__predict_false(toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) handle_ddp_close(toep, tp, cpl->rcv_nxt); DDP_UNLOCK(toep); } socantrcvmore(so); if (toep->ulp_mode != ULP_MODE_RDMA) { KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt), ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt, be32toh(cpl->rcv_nxt))); } switch (tp->t_state) { case TCPS_SYN_RECEIVED: tp->t_starttime = ticks; /* FALLTHROUGH */ case TCPS_ESTABLISHED: tcp_state_change(tp, TCPS_CLOSE_WAIT); break; case TCPS_FIN_WAIT_1: tcp_state_change(tp, TCPS_CLOSING); break; case TCPS_FIN_WAIT_2: tcp_twstart(tp); INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ INP_INFO_RUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); INP_WLOCK(inp); final_cpl_received(toep); return (0); default: log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n", __func__, tid, tp->t_state); } done: INP_WUNLOCK(inp); INP_INFO_RUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); return (0); } /* * Peer has ACK'd our FIN. */ static int do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; struct tcpcb *tp = NULL; struct socket *so = NULL; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif KASSERT(opcode == CPL_CLOSE_CON_RPL, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); CURVNET_SET(toep->vnet); INP_INFO_RLOCK(&V_tcbinfo); INP_WLOCK(inp); tp = intotcpcb(inp); CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x", __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags); if (toep->flags & TPF_ABORT_SHUTDOWN) goto done; so = inp->inp_socket; tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */ switch (tp->t_state) { case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */ tcp_twstart(tp); release: INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ INP_INFO_RUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); INP_WLOCK(inp); final_cpl_received(toep); /* no more CPLs expected */ return (0); case TCPS_LAST_ACK: if (tcp_close(tp)) INP_WUNLOCK(inp); goto release; case TCPS_FIN_WAIT_1: if (so->so_rcv.sb_state & SBS_CANTRCVMORE) soisdisconnected(so); tcp_state_change(tp, TCPS_FIN_WAIT_2); break; default: log(LOG_ERR, "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n", __func__, tid, tcpstates[tp->t_state]); } done: INP_WUNLOCK(inp); INP_INFO_RUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); return (0); } void send_abort_rpl(struct adapter *sc, struct sge_wrq *ofld_txq, int tid, int rst_status) { struct wrqe *wr; struct cpl_abort_rpl *cpl; wr = alloc_wrqe(sizeof(*cpl), ofld_txq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } cpl = wrtod(wr); INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid); cpl->cmd = rst_status; t4_wrq_tx(sc, wr); } static int abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason) { switch (abort_reason) { case CPL_ERR_BAD_SYN: case CPL_ERR_CONN_RESET: return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET); case CPL_ERR_XMIT_TIMEDOUT: case CPL_ERR_PERSIST_TIMEDOUT: case CPL_ERR_FINWAIT2_TIMEDOUT: case CPL_ERR_KEEPALIVE_TIMEDOUT: return (ETIMEDOUT); default: return (EIO); } } /* * TCP RST from the peer, timeout, or some other such critical error. */ static int do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct sge_wrq *ofld_txq = toep->ofld_txq; struct inpcb *inp; struct tcpcb *tp; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif KASSERT(opcode == CPL_ABORT_REQ_RSS, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); if (toep->flags & TPF_SYNQE) return (do_abort_req_synqe(iq, rss, m)); KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); if (negative_advice(cpl->status)) { CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)", __func__, cpl->status, tid, toep->flags); return (0); /* Ignore negative advice */ } inp = toep->inp; CURVNET_SET(toep->vnet); INP_INFO_RLOCK(&V_tcbinfo); /* for tcp_close */ INP_WLOCK(inp); tp = intotcpcb(inp); CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d", __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp->inp_flags, cpl->status); /* * If we'd initiated an abort earlier the reply to it is responsible for * cleaning up resources. Otherwise we tear everything down right here * right now. We owe the T4 a CPL_ABORT_RPL no matter what. */ if (toep->flags & TPF_ABORT_SHUTDOWN) { INP_WUNLOCK(inp); goto done; } toep->flags |= TPF_ABORT_SHUTDOWN; if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { struct socket *so = inp->inp_socket; if (so != NULL) so_error_set(so, abort_status_to_errno(tp, cpl->status)); tp = tcp_close(tp); if (tp == NULL) INP_WLOCK(inp); /* re-acquire */ } final_cpl_received(toep); done: INP_INFO_RUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST); return (0); } /* * Reply to the CPL_ABORT_REQ (send_reset) */ static int do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif KASSERT(opcode == CPL_ABORT_RPL_RSS, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); if (toep->flags & TPF_SYNQE) return (do_abort_rpl_synqe(iq, rss, m)); KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d", __func__, tid, toep, inp, cpl->status); KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, ("%s: wasn't expecting abort reply", __func__)); INP_WLOCK(inp); final_cpl_received(toep); return (0); } static int do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_rx_data *cpl = mtod(m, const void *); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; struct tcpcb *tp; struct socket *so; struct sockbuf *sb; int len; uint32_t ddp_placed = 0; if (__predict_false(toep->flags & TPF_SYNQE)) { #ifdef INVARIANTS struct synq_entry *synqe = (void *)toep; INP_WLOCK(synqe->lctx->inp); if (synqe->flags & TPF_SYNQE_HAS_L2TE) { KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN, ("%s: listen socket closed but tid %u not aborted.", __func__, tid)); } else { /* * do_pass_accept_req is still running and will * eventually take care of this tid. */ } INP_WUNLOCK(synqe->lctx->inp); #endif CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, toep, toep->flags); m_freem(m); return (0); } KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); /* strip off CPL header */ m_adj(m, sizeof(*cpl)); len = m->m_pkthdr.len; INP_WLOCK(inp); if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", __func__, tid, len, inp->inp_flags); INP_WUNLOCK(inp); m_freem(m); return (0); } tp = intotcpcb(inp); if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq))) ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt; tp->rcv_nxt += len; if (tp->rcv_wnd < len) { KASSERT(toep->ulp_mode == ULP_MODE_RDMA, ("%s: negative window size", __func__)); } tp->rcv_wnd -= len; tp->t_rcvtime = ticks; if (toep->ulp_mode == ULP_MODE_TCPDDP) DDP_LOCK(toep); so = inp_inpcbtosocket(inp); sb = &so->so_rcv; SOCKBUF_LOCK(sb); if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", __func__, tid, len); m_freem(m); SOCKBUF_UNLOCK(sb); if (toep->ulp_mode == ULP_MODE_TCPDDP) DDP_UNLOCK(toep); INP_WUNLOCK(inp); CURVNET_SET(toep->vnet); INP_INFO_RLOCK(&V_tcbinfo); INP_WLOCK(inp); tp = tcp_drop(tp, ECONNRESET); if (tp) INP_WUNLOCK(inp); INP_INFO_RUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); return (0); } /* receive buffer autosize */ MPASS(toep->vnet == so->so_vnet); CURVNET_SET(toep->vnet); if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autorcvbuf && sb->sb_hiwat < V_tcp_autorcvbuf_max && len > (sbspace(sb) / 8 * 7)) { unsigned int hiwat = sb->sb_hiwat; unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, V_tcp_autorcvbuf_max); if (!sbreserve_locked(sb, newsize, so, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; else toep->rx_credits += newsize - hiwat; } if (toep->ulp_mode == ULP_MODE_TCPDDP) { int changed = !(toep->ddp.flags & DDP_ON) ^ cpl->ddp_off; if (toep->ddp.waiting_count != 0 || toep->ddp.active_count != 0) CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)", __func__, tid, len); if (changed) { if (toep->ddp.flags & DDP_SC_REQ) toep->ddp.flags ^= DDP_ON | DDP_SC_REQ; else { KASSERT(cpl->ddp_off == 1, ("%s: DDP switched on by itself.", __func__)); /* Fell out of DDP mode */ toep->ddp.flags &= ~DDP_ON; CTR1(KTR_CXGBE, "%s: fell out of DDP mode", __func__); insert_ddp_data(toep, ddp_placed); } } if (toep->ddp.flags & DDP_ON) { /* * CPL_RX_DATA with DDP on can only be an indicate. * Start posting queued AIO requests via DDP. The * payload that arrived in this indicate is appended * to the socket buffer as usual. */ handle_ddp_indicate(toep); } } KASSERT(toep->sb_cc >= sbused(sb), ("%s: sb %p has more data (%d) than last time (%d).", __func__, sb, sbused(sb), toep->sb_cc)); toep->rx_credits += toep->sb_cc - sbused(sb); sbappendstream_locked(sb, m, 0); toep->sb_cc = sbused(sb); if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) { int credits; credits = send_rx_credits(sc, toep, toep->rx_credits); toep->rx_credits -= credits; tp->rcv_wnd += credits; tp->rcv_adv += credits; } if (toep->ulp_mode == ULP_MODE_TCPDDP && toep->ddp.waiting_count > 0 && sbavail(sb) != 0) { CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__, tid); ddp_queue_toep(toep); } sorwakeup_locked(so); SOCKBUF_UNLOCK_ASSERT(sb); if (toep->ulp_mode == ULP_MODE_TCPDDP) DDP_UNLOCK(toep); INP_WUNLOCK(inp); CURVNET_RESTORE(); return (0); } -#define S_CPL_FW4_ACK_OPCODE 24 -#define M_CPL_FW4_ACK_OPCODE 0xff -#define V_CPL_FW4_ACK_OPCODE(x) ((x) << S_CPL_FW4_ACK_OPCODE) -#define G_CPL_FW4_ACK_OPCODE(x) \ - (((x) >> S_CPL_FW4_ACK_OPCODE) & M_CPL_FW4_ACK_OPCODE) - -#define S_CPL_FW4_ACK_FLOWID 0 -#define M_CPL_FW4_ACK_FLOWID 0xffffff -#define V_CPL_FW4_ACK_FLOWID(x) ((x) << S_CPL_FW4_ACK_FLOWID) -#define G_CPL_FW4_ACK_FLOWID(x) \ - (((x) >> S_CPL_FW4_ACK_FLOWID) & M_CPL_FW4_ACK_FLOWID) - -#define S_CPL_FW4_ACK_CR 24 -#define M_CPL_FW4_ACK_CR 0xff -#define V_CPL_FW4_ACK_CR(x) ((x) << S_CPL_FW4_ACK_CR) -#define G_CPL_FW4_ACK_CR(x) (((x) >> S_CPL_FW4_ACK_CR) & M_CPL_FW4_ACK_CR) - -#define S_CPL_FW4_ACK_SEQVAL 0 -#define M_CPL_FW4_ACK_SEQVAL 0x1 -#define V_CPL_FW4_ACK_SEQVAL(x) ((x) << S_CPL_FW4_ACK_SEQVAL) -#define G_CPL_FW4_ACK_SEQVAL(x) \ - (((x) >> S_CPL_FW4_ACK_SEQVAL) & M_CPL_FW4_ACK_SEQVAL) -#define F_CPL_FW4_ACK_SEQVAL V_CPL_FW4_ACK_SEQVAL(1U) - static int do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp; struct tcpcb *tp; struct socket *so; uint8_t credits = cpl->credits; struct ofld_tx_sdesc *txsd; int plen; #ifdef INVARIANTS unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl))); #endif /* * Very unusual case: we'd sent a flowc + abort_req for a synq entry and * now this comes back carrying the credits for the flowc. */ if (__predict_false(toep->flags & TPF_SYNQE)) { KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, ("%s: credits for a synq entry %p", __func__, toep)); return (0); } inp = toep->inp; KASSERT(opcode == CPL_FW4_ACK, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); INP_WLOCK(inp); if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) { INP_WUNLOCK(inp); return (0); } KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0, ("%s: inp_flags 0x%x", __func__, inp->inp_flags)); tp = intotcpcb(inp); if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) { tcp_seq snd_una = be32toh(cpl->snd_una); #ifdef INVARIANTS if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) { log(LOG_ERR, "%s: unexpected seq# %x for TID %u, snd_una %x\n", __func__, snd_una, toep->tid, tp->snd_una); } #endif if (tp->snd_una != snd_una) { tp->snd_una = snd_una; tp->ts_recent_age = tcp_ts_getticks(); } } #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: tid %d credits %u", __func__, tid, credits); #endif so = inp->inp_socket; txsd = &toep->txsd[toep->txsd_cidx]; plen = 0; while (credits) { KASSERT(credits >= txsd->tx_credits, ("%s: too many (or partial) credits", __func__)); credits -= txsd->tx_credits; toep->tx_credits += txsd->tx_credits; plen += txsd->plen; if (txsd->iv_buffer) { free(txsd->iv_buffer, M_CXGBE); txsd->iv_buffer = NULL; } txsd++; toep->txsd_avail++; KASSERT(toep->txsd_avail <= toep->txsd_total, ("%s: txsd avail > total", __func__)); if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) { txsd = &toep->txsd[0]; toep->txsd_cidx = 0; } } if (toep->tx_credits == toep->tx_total) { toep->tx_nocompl = 0; toep->plen_nocompl = 0; } if (toep->flags & TPF_TX_SUSPENDED && toep->tx_credits >= toep->tx_total / 4) { #ifdef VERBOSE_TRACES CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__, tid); #endif toep->flags &= ~TPF_TX_SUSPENDED; CURVNET_SET(toep->vnet); if (toep->ulp_mode == ULP_MODE_ISCSI) t4_push_pdus(sc, toep, plen); else if (tls_tx_key(toep)) t4_push_tls_records(sc, toep, plen); else t4_push_frames(sc, toep, plen); CURVNET_RESTORE(); } else if (plen > 0) { struct sockbuf *sb = &so->so_snd; int sbu; SOCKBUF_LOCK(sb); sbu = sbused(sb); if (toep->ulp_mode == ULP_MODE_ISCSI) { if (__predict_false(sbu > 0)) { /* * The data trasmitted before the tid's ULP mode * changed to ISCSI is still in so_snd. * Incoming credits should account for so_snd * first. */ sbdrop_locked(sb, min(sbu, plen)); plen -= min(sbu, plen); } sowwakeup_locked(so); /* unlocks so_snd */ rqdrop_locked(&toep->ulp_pdu_reclaimq, plen); } else { #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__, tid, plen); #endif sbdrop_locked(sb, plen); if (tls_tx_key(toep)) { struct tls_ofld_info *tls_ofld = &toep->tls; MPASS(tls_ofld->sb_off >= plen); tls_ofld->sb_off -= plen; } if (!TAILQ_EMPTY(&toep->aiotx_jobq)) t4_aiotx_queue_toep(toep); sowwakeup_locked(so); /* unlocks so_snd */ } SOCKBUF_UNLOCK_ASSERT(sb); } INP_WUNLOCK(inp); return (0); } void t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep, uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie) { struct wrqe *wr; struct cpl_set_tcb_field *req; struct ofld_tx_sdesc *txsd; MPASS((cookie & ~M_COOKIE) == 0); if (reply) { MPASS(cookie != CPL_COOKIE_RESERVED); } wr = alloc_wrqe(sizeof(*req), wrq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id)); if (reply == 0) req->reply_ctrl |= htobe16(F_NO_REPLY); req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie)); req->mask = htobe64(mask); req->val = htobe64(val); if ((wrq->eq.flags & EQ_TYPEMASK) == EQ_OFLD) { txsd = &toep->txsd[toep->txsd_pidx]; txsd->tx_credits = howmany(sizeof(*req), 16); txsd->plen = 0; KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, ("%s: not enough credits (%d)", __func__, toep->tx_credits)); toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; } t4_wrq_tx(sc, wr); } void t4_init_cpl_io_handlers(void) { t4_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close); t4_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl); t4_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req); t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl, CPL_COOKIE_TOM); t4_register_cpl_handler(CPL_RX_DATA, do_rx_data); - t4_register_cpl_handler(CPL_FW4_ACK, do_fw4_ack); + t4_register_shared_cpl_handler(CPL_FW4_ACK, do_fw4_ack, CPL_COOKIE_TOM); } void t4_uninit_cpl_io_handlers(void) { t4_register_cpl_handler(CPL_PEER_CLOSE, NULL); t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL); t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL); t4_register_cpl_handler(CPL_ABORT_RPL_RSS, NULL); t4_register_cpl_handler(CPL_RX_DATA, NULL); - t4_register_cpl_handler(CPL_FW4_ACK, NULL); + t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM); } /* * Use the 'backend3' field in AIO jobs to store the amount of data * sent by the AIO job so far and the 'backend4' field to hold an * error that should be reported when the job is completed. */ #define aio_sent backend3 #define aio_error backend4 #define jobtotid(job) \ (((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid) static void free_aiotx_buffer(struct aiotx_buffer *ab) { struct kaiocb *job; long status; int error; if (refcount_release(&ab->refcount) == 0) return; job = ab->job; error = job->aio_error; status = job->aio_sent; vm_page_unhold_pages(ab->ps.pages, ab->ps.npages); free(ab, M_CXGBE); #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__, jobtotid(job), job, status, error); #endif if (error == ECANCELED && status != 0) error = 0; if (error == ECANCELED) aio_cancel(job); else if (error) aio_complete(job, -1, error); else aio_complete(job, status, 0); } static void t4_aiotx_mbuf_free(struct mbuf *m, void *buffer, void *arg) { struct aiotx_buffer *ab = buffer; #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__, m->m_len, jobtotid(ab->job)); #endif free_aiotx_buffer(ab); } /* * Hold the buffer backing an AIO request and return an AIO transmit * buffer. */ static int hold_aio(struct kaiocb *job) { struct aiotx_buffer *ab; struct vmspace *vm; vm_map_t map; vm_offset_t start, end, pgoff; int n; MPASS(job->backend1 == NULL); /* * The AIO subsystem will cancel and drain all requests before * permitting a process to exit or exec, so p_vmspace should * be stable here. */ vm = job->userproc->p_vmspace; map = &vm->vm_map; start = (uintptr_t)job->uaiocb.aio_buf; pgoff = start & PAGE_MASK; end = round_page(start + job->uaiocb.aio_nbytes); start = trunc_page(start); n = atop(end - start); ab = malloc(sizeof(*ab) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK | M_ZERO); refcount_init(&ab->refcount, 1); ab->ps.pages = (vm_page_t *)(ab + 1); ab->ps.npages = vm_fault_quick_hold_pages(map, start, end - start, VM_PROT_WRITE, ab->ps.pages, n); if (ab->ps.npages < 0) { free(ab, M_CXGBE); return (EFAULT); } KASSERT(ab->ps.npages == n, ("hold_aio: page count mismatch: %d vs %d", ab->ps.npages, n)); ab->ps.offset = pgoff; ab->ps.len = job->uaiocb.aio_nbytes; ab->job = job; job->backend1 = ab; #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d", __func__, jobtotid(job), &ab->ps, job, ab->ps.npages); #endif return (0); } static void t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job) { struct adapter *sc; struct sockbuf *sb; struct file *fp; struct aiotx_buffer *ab; struct inpcb *inp; struct tcpcb *tp; struct mbuf *m; int error; bool moretocome, sendmore; sc = td_adapter(toep->td); sb = &so->so_snd; SOCKBUF_UNLOCK(sb); fp = job->fd_file; ab = job->backend1; m = NULL; #ifdef MAC error = mac_socket_check_send(fp->f_cred, so); if (error != 0) goto out; #endif if (ab == NULL) { error = hold_aio(job); if (error != 0) goto out; ab = job->backend1; } /* Inline sosend_generic(). */ job->msgsnd = 1; error = sblock(sb, SBL_WAIT); MPASS(error == 0); sendanother: m = m_get(M_WAITOK, MT_DATA); SOCKBUF_LOCK(sb); if (so->so_snd.sb_state & SBS_CANTSENDMORE) { SOCKBUF_UNLOCK(sb); sbunlock(sb); if ((so->so_options & SO_NOSIGPIPE) == 0) { PROC_LOCK(job->userproc); kern_psignal(job->userproc, SIGPIPE); PROC_UNLOCK(job->userproc); } error = EPIPE; goto out; } if (so->so_error) { error = so->so_error; so->so_error = 0; SOCKBUF_UNLOCK(sb); sbunlock(sb); goto out; } if ((so->so_state & SS_ISCONNECTED) == 0) { SOCKBUF_UNLOCK(sb); sbunlock(sb); error = ENOTCONN; goto out; } if (sbspace(sb) < sb->sb_lowat) { MPASS(job->aio_sent == 0 || !(so->so_state & SS_NBIO)); /* * Don't block if there is too little room in the socket * buffer. Instead, requeue the request. */ if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { SOCKBUF_UNLOCK(sb); sbunlock(sb); error = ECANCELED; goto out; } TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); SOCKBUF_UNLOCK(sb); sbunlock(sb); goto out; } /* * Write as much data as the socket permits, but no more than a * a single sndbuf at a time. */ m->m_len = sbspace(sb); if (m->m_len > ab->ps.len - job->aio_sent) { m->m_len = ab->ps.len - job->aio_sent; moretocome = false; } else moretocome = true; if (m->m_len > sc->tt.sndbuf) { m->m_len = sc->tt.sndbuf; sendmore = true; } else sendmore = false; if (!TAILQ_EMPTY(&toep->aiotx_jobq)) moretocome = true; SOCKBUF_UNLOCK(sb); MPASS(m->m_len != 0); /* Inlined tcp_usr_send(). */ inp = toep->inp; INP_WLOCK(inp); if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { INP_WUNLOCK(inp); sbunlock(sb); error = ECONNRESET; goto out; } refcount_acquire(&ab->refcount); m_extadd(m, NULL, ab->ps.len, t4_aiotx_mbuf_free, ab, (void *)(uintptr_t)job->aio_sent, 0, EXT_NET_DRV); m->m_ext.ext_flags |= EXT_FLAG_AIOTX; job->aio_sent += m->m_len; sbappendstream(sb, m, 0); m = NULL; if (!(inp->inp_flags & INP_DROPPED)) { tp = intotcpcb(inp); if (moretocome) tp->t_flags |= TF_MORETOCOME; error = tp->t_fb->tfb_tcp_output(tp); if (moretocome) tp->t_flags &= ~TF_MORETOCOME; } INP_WUNLOCK(inp); if (sendmore) goto sendanother; sbunlock(sb); if (error) goto out; /* * If this is a non-blocking socket and the request has not * been fully completed, requeue it until the socket is ready * again. */ if (job->aio_sent < job->uaiocb.aio_nbytes && !(so->so_state & SS_NBIO)) { SOCKBUF_LOCK(sb); if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { SOCKBUF_UNLOCK(sb); error = ECANCELED; goto out; } TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); return; } /* * If the request will not be requeued, drop a reference on * the the aiotx buffer. Any mbufs in flight should still * contain a reference, but this drops the reference that the * job owns while it is waiting to queue mbufs to the socket. */ free_aiotx_buffer(ab); out: if (error) { if (ab != NULL) { job->aio_error = error; free_aiotx_buffer(ab); } else { MPASS(job->aio_sent == 0); aio_complete(job, -1, error); } } if (m != NULL) m_free(m); SOCKBUF_LOCK(sb); } static void t4_aiotx_task(void *context, int pending) { struct toepcb *toep = context; struct inpcb *inp = toep->inp; struct socket *so = inp->inp_socket; struct kaiocb *job; CURVNET_SET(toep->vnet); SOCKBUF_LOCK(&so->so_snd); while (!TAILQ_EMPTY(&toep->aiotx_jobq) && sowriteable(so)) { job = TAILQ_FIRST(&toep->aiotx_jobq); TAILQ_REMOVE(&toep->aiotx_jobq, job, list); if (!aio_clear_cancel_function(job)) continue; t4_aiotx_process_job(toep, so, job); } toep->aiotx_task_active = false; SOCKBUF_UNLOCK(&so->so_snd); CURVNET_RESTORE(); free_toepcb(toep); } static void t4_aiotx_queue_toep(struct toepcb *toep) { SOCKBUF_LOCK_ASSERT(&toep->inp->inp_socket->so_snd); #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: queueing aiotx task for tid %d, active = %s", __func__, toep->tid, toep->aiotx_task_active ? "true" : "false"); #endif if (toep->aiotx_task_active) return; toep->aiotx_task_active = true; hold_toepcb(toep); soaio_enqueue(&toep->aiotx_task); } static void t4_aiotx_cancel(struct kaiocb *job) { struct aiotx_buffer *ab; struct socket *so; struct sockbuf *sb; struct tcpcb *tp; struct toepcb *toep; so = job->fd_file->f_data; tp = so_sototcpcb(so); toep = tp->t_toe; MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE); sb = &so->so_snd; SOCKBUF_LOCK(sb); if (!aio_cancel_cleared(job)) TAILQ_REMOVE(&toep->aiotx_jobq, job, list); SOCKBUF_UNLOCK(sb); ab = job->backend1; if (ab != NULL) free_aiotx_buffer(ab); else aio_cancel(job); } int t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job) { struct tcpcb *tp = so_sototcpcb(so); struct toepcb *toep = tp->t_toe; struct adapter *sc = td_adapter(toep->td); /* This only handles writes. */ if (job->uaiocb.aio_lio_opcode != LIO_WRITE) return (EOPNOTSUPP); if (!sc->tt.tx_zcopy) return (EOPNOTSUPP); if (tls_tx_key(toep)) return (EOPNOTSUPP); SOCKBUF_LOCK(&so->so_snd); #ifdef VERBOSE_TRACES CTR2(KTR_CXGBE, "%s: queueing %p", __func__, job); #endif if (!aio_set_cancel_function(job, t4_aiotx_cancel)) panic("new job was cancelled"); TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list); if (sowriteable(so)) t4_aiotx_queue_toep(toep); SOCKBUF_UNLOCK(&so->so_snd); return (0); } void aiotx_init_toep(struct toepcb *toep) { TAILQ_INIT(&toep->aiotx_jobq); TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep); } #endif Index: stable/11 =================================================================== --- stable/11 (revision 346862) +++ stable/11 (revision 346863) Property changes on: stable/11 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r334138