Page MenuHomeFreeBSD
Differential D30073 Diff 88931 sys/dev/e1000/igb_txrx.c

Changeset View

Standalone View

View Options

sys/dev/e1000/igb_txrx.c

Show First 20 Lines • Show All 41 Lines • ▼ Show 20 Lines
* Local Function prototypes * Local Function prototypes
*********************************************************************/ *********************************************************************/
static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi); static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi);
static void igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); static void igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear);
static void igb_isc_rxd_refill(void *arg, if_rxd_update_t iru); static void igb_isc_rxd_refill(void *arg, if_rxd_update_t iru);
static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx); static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
static int igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget); qidx_t pidx);
static int igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
qidx_t budget);
static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi,
static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); uint32_t *cmd_type_len, uint32_t *olinfo_status);
static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi,
uint32_t *cmd_type_len, uint32_t *olinfo_status);
static void igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype); static void igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype);
static int igb_determine_rsstype(u16 pkt_info); static int igb_determine_rsstype(uint16_t pkt_info);
extern void igb_if_enable_intr(if_ctx_t ctx); extern void igb_if_enable_intr(if_ctx_t ctx);
extern int em_intr(void *arg); extern int em_intr(void *arg);
struct if_txrx igb_txrx = { struct if_txrx igb_txrx = {
.ift_txd_encap = igb_isc_txd_encap, .ift_txd_encap = igb_isc_txd_encap,
.ift_txd_flush = igb_isc_txd_flush, .ift_txd_flush = igb_isc_txd_flush,
.ift_txd_credits_update = igb_isc_txd_credits_update, .ift_txd_credits_update = igb_isc_txd_credits_update,
.ift_rxd_available = igb_isc_rxd_available, .ift_rxd_available = igb_isc_rxd_available,
.ift_rxd_pkt_get = igb_isc_rxd_pkt_get, .ift_rxd_pkt_get = igb_isc_rxd_pkt_get,
.ift_rxd_refill = igb_isc_rxd_refill, .ift_rxd_refill = igb_isc_rxd_refill,
.ift_rxd_flush = igb_isc_rxd_flush, .ift_rxd_flush = igb_isc_rxd_flush,
.ift_legacy_intr = em_intr .ift_legacy_intr = em_intr
}; };
/********************************************************************** /**********************************************************************
* *
* Setup work for hardware segmentation offload (TSO) on * Setup work for hardware segmentation offload (TSO) on
* adapters using advanced tx descriptors * adapters using advanced tx descriptors
* *
**********************************************************************/ **********************************************************************/
static int static int
igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, uint32_t *cmd_type_len,
uint32_t *olinfo_status)
{ {
struct e1000_adv_tx_context_desc *TXD; struct e1000_adv_tx_context_desc *TXD;
struct adapter *adapter = txr->adapter; struct adapter *adapter = txr->adapter;
u32 type_tucmd_mlhl = 0, vlan_macip_lens = 0; uint32_t type_tucmd_mlhl = 0, vlan_macip_lens = 0;
u32 mss_l4len_idx = 0; uint32_t mss_l4len_idx = 0;
u32 paylen; uint32_t paylen;
switch(pi->ipi_etype) { switch(pi->ipi_etype) {
case ETHERTYPE_IPV6: case ETHERTYPE_IPV6:
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
break; break;
case ETHERTYPE_IP: case ETHERTYPE_IP:
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
/* Tell transmit desc to also do IPv4 checksum. */ /* Tell transmit desc to also do IPv4 checksum. */
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines
} }
/********************************************************************* /*********************************************************************
* *
* Advanced Context Descriptor setup for VLAN, CSUM or TSO * Advanced Context Descriptor setup for VLAN, CSUM or TSO
* *
**********************************************************************/ **********************************************************************/
static int static int
igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, uint32_t *cmd_type_len,
uint32_t *olinfo_status)
{ {
struct e1000_adv_tx_context_desc *TXD; struct e1000_adv_tx_context_desc *TXD;
struct adapter *adapter = txr->adapter; struct adapter *adapter = txr->adapter;
u32 vlan_macip_lens, type_tucmd_mlhl; uint32_t vlan_macip_lens, type_tucmd_mlhl;
u32 mss_l4len_idx; uint32_t mss_l4len_idx;
mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0; mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0;
/* First check if TSO is to be used */ /* First check if TSO is to be used */
if (pi->ipi_csum_flags & CSUM_TSO) if (pi->ipi_csum_flags & CSUM_TSO)
return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status)); return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status));
/* Indicate the whole packet as payload when not doing TSO */ /* Indicate the whole packet as payload when not doing TSO */
*olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT; *olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT;
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Linesigb_isc_txd_encap(void *arg, if_pkt_info_t pi)
struct adapter *sc = arg; struct adapter *sc = arg;
if_softc_ctx_t scctx = sc->shared; if_softc_ctx_t scctx = sc->shared;
struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
struct tx_ring *txr = &que->txr; struct tx_ring *txr = &que->txr;
int nsegs = pi->ipi_nsegs; int nsegs = pi->ipi_nsegs;
bus_dma_segment_t *segs = pi->ipi_segs; bus_dma_segment_t *segs = pi->ipi_segs;
union e1000_adv_tx_desc *txd = NULL; union e1000_adv_tx_desc *txd = NULL;
int i, j, pidx_last; int i, j, pidx_last;
u32 olinfo_status, cmd_type_len, txd_flags; uint32_t olinfo_status, cmd_type_len, txd_flags;
qidx_t ntxd; qidx_t ntxd;
pidx_last = olinfo_status = 0; pidx_last = olinfo_status = 0;
/* Basic descriptor defines */ /* Basic descriptor defines */
cmd_type_len = (E1000_ADVTXD_DTYP_DATA | cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
if (pi->ipi_mflags & M_VLANTAG) if (pi->ipi_mflags & M_VLANTAG)
▲ Show 20 Lines • Show All 142 Lines • ▼ Show 20 Lines
static int static int
igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
{ {
struct adapter *sc = arg; struct adapter *sc = arg;
if_softc_ctx_t scctx = sc->shared; if_softc_ctx_t scctx = sc->shared;
struct em_rx_queue *que = &sc->rx_queues[rxqid]; struct em_rx_queue *que = &sc->rx_queues[rxqid];
struct rx_ring *rxr = &que->rxr; struct rx_ring *rxr = &que->rxr;
union e1000_adv_rx_desc *rxd; union e1000_adv_rx_desc *rxd;
u32 staterr = 0; uint32_t staterr = 0;
int cnt, i; int cnt, i;
for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i]; rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i];
staterr = le32toh(rxd->wb.upper.status_error); staterr = le32toh(rxd->wb.upper.status_error);
if ((staterr & E1000_RXD_STAT_DD) == 0) if ((staterr & E1000_RXD_STAT_DD) == 0)
break; break;
Show All 14 Lines
static int static int
igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
{ {
struct adapter *adapter = arg; struct adapter *adapter = arg;
if_softc_ctx_t scctx = adapter->shared; if_softc_ctx_t scctx = adapter->shared;
struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
struct rx_ring *rxr = &que->rxr; struct rx_ring *rxr = &que->rxr;
struct ifnet *ifp = iflib_get_ifp(adapter->ctx);
union e1000_adv_rx_desc *rxd; union e1000_adv_rx_desc *rxd;
u16 pkt_info, len; uint16_t pkt_info, len, vtag;
u16 vtag = 0; uint32_t ptype;
u32 ptype; uint32_t staterr;
u32 staterr = 0;
bool eop; bool eop;
int i = 0; int i, cidx;
int cidx = ri->iri_cidx;
staterr = i = vtag = 0;
cidx = ri->iri_cidx;
do { do {
rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx]; rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx];
staterr = le32toh(rxd->wb.upper.status_error); staterr = le32toh(rxd->wb.upper.status_error);
pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info); pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info);
MPASS ((staterr & E1000_RXD_STAT_DD) != 0); MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
len = le16toh(rxd->wb.upper.length); len = le16toh(rxd->wb.upper.length);
Show All 32 Lines if (rxr->hdr_split == TRUE) {
cidx = 0; cidx = 0;
} }
#endif #endif
i++; i++;
} while (!eop); } while (!eop);
rxr->rx_packets++; rxr->rx_packets++;
if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) ri->iri_csum_flags = 0;
markjUnsubmitted
Not Done
Inline Actions

This assignment isn't needed.

markj: This assignment isn't needed.
if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
igb_rx_checksum(staterr, ri, ptype); igb_rx_checksum(staterr, ri, ptype);
if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && if ((scctx->isc_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
(staterr & E1000_RXD_STAT_VP) != 0) { (staterr & E1000_RXD_STAT_VP) != 0) {
ri->iri_vtag = vtag; ri->iri_vtag = vtag;
ri->iri_flags |= M_VLANTAG; ri->iri_flags |= M_VLANTAG;
} }
ri->iri_flowid = ri->iri_flowid =
le32toh(rxd->wb.lower.hi_dword.rss); le32toh(rxd->wb.lower.hi_dword.rss);
ri->iri_rsstype = igb_determine_rsstype(pkt_info); ri->iri_rsstype = igb_determine_rsstype(pkt_info);
ri->iri_nfrags = i; ri->iri_nfrags = i;
return (0); return (0);
} }
/********************************************************************* /*********************************************************************
* *
* Verify that the hardware indicated that the checksum is valid. * Verify that the hardware indicated that the checksum is valid.
* Inform the stack about the status of checksum so that stack * Inform the stack about the status of checksum so that stack
* doesn't spend time verifying the checksum. * doesn't spend time verifying the checksum.
* *
*********************************************************************/ *********************************************************************/
static void static void
igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype) igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype)
{ {
u16 status = (u16)staterr; uint16_t status = (uint16_t)staterr;
u8 errors = (u8) (staterr >> 24); uint8_t errors = (uint8_t)(staterr >> 24);
bool sctp = FALSE;
/* Ignore Checksum bit is set */ /* Ignore Checksum bit is set */
if (status & E1000_RXD_STAT_IXSM) { if (__predict_false(status & E1000_RXD_STAT_IXSM))
ri->iri_csum_flags = 0;
return; return;
}
if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && /* If there is a layer 3 or 4 error we are done */
(ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) if (errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE))
gallatinUnsubmitted
Not Done
Inline Actions

Maybe this is also worth a __predict_false()

gallatin: Maybe this is also worth a __predict_false()
sctp = 1; return;
else
sctp = 0;
if (status & E1000_RXD_STAT_IPCS) {
/* Did it pass? */
if (!(errors & E1000_RXD_ERR_IPE)) {
/* IP Checksum Good */ /* IP Checksum Good */
ri->iri_csum_flags = CSUM_IP_CHECKED; if (status & E1000_RXD_STAT_IPCS)
gallatinUnsubmitted
Not Done
Inline Actions

I would not use a __predict here, as there is no v6 l3 checksum, so I assume that v6 packets would take the unpredicted path, and be penalized.

gallatin: I would not use a __predict here, as there is no v6 l3 checksum, so I assume that v6 packets…
ri->iri_csum_flags |= CSUM_IP_VALID; ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
} else
ri->iri_csum_flags = 0;
}
if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { /* Valid L4E checksum */
u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); if (__predict_true(status &
if (sctp) /* reassign */ (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) {
type = CSUM_SCTP_VALID; /* SCTP header present.
/* Did it pass? */ * XXXKB: ETQF doesn't appear to be used in igb?
if (!(errors & E1000_RXD_ERR_TCPE)) { */
ri->iri_csum_flags |= type; if (__predict_false((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
if (sctp == 0) (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)) {
markjUnsubmitted
Not Done
Inline Actions

Indentation here should be by four spaces.

markj: Indentation here should be by four spaces.
ri->iri_csum_flags |= CSUM_SCTP_VALID;
} else {
ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
ri->iri_csum_data = htons(0xffff); ri->iri_csum_data = htons(0xffff);
} }
} }
return;
} }
/******************************************************************** /********************************************************************
* *
* Parse the packet type to determine the appropriate hash * Parse the packet type to determine the appropriate hash
* *
******************************************************************/ ******************************************************************/
static int static int
igb_determine_rsstype(u16 pkt_info) igb_determine_rsstype(uint16_t pkt_info)
{ {
switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
case E1000_RXDADV_RSSTYPE_IPV4_TCP: case E1000_RXDADV_RSSTYPE_IPV4_TCP:
return M_HASHTYPE_RSS_TCP_IPV4; return M_HASHTYPE_RSS_TCP_IPV4;
case E1000_RXDADV_RSSTYPE_IPV4: case E1000_RXDADV_RSSTYPE_IPV4:
return M_HASHTYPE_RSS_IPV4; return M_HASHTYPE_RSS_IPV4;
case E1000_RXDADV_RSSTYPE_IPV6_TCP: case E1000_RXDADV_RSSTYPE_IPV6_TCP:
return M_HASHTYPE_RSS_TCP_IPV6; return M_HASHTYPE_RSS_TCP_IPV6;
Show All 10 Lines