Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/e1000/igb_txrx.c
/*- | /*- | ||||
* SPDX-License-Identifier: BSD-2-Clause | |||||
* | |||||
* Copyright (c) 2016 Matthew Macy <mmacy@mattmacy.io> | * Copyright (c) 2016 Matthew Macy <mmacy@mattmacy.io> | ||||
* All rights reserved. | * All rights reserved. | ||||
* | * | ||||
* Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||
* modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||
* are met: | * are met: | ||||
* 1. Redistributions of source code must retain the above copyright | * 1. Redistributions of source code must retain the above copyright | ||||
* notice, this list of conditions and the following disclaimer. | * notice, this list of conditions and the following disclaimer. | ||||
Show All 32 Lines | |||||
* Local Function prototypes | * Local Function prototypes | ||||
*********************************************************************/ | *********************************************************************/ | ||||
static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi); | static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi); | ||||
static void igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); | static void igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); | ||||
static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); | static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); | ||||
static void igb_isc_rxd_refill(void *arg, if_rxd_update_t iru); | static void igb_isc_rxd_refill(void *arg, if_rxd_update_t iru); | ||||
static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx); | static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, | ||||
static int igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget); | qidx_t pidx); | ||||
static int igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, | |||||
qidx_t budget); | |||||
static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | ||||
static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); | static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, | ||||
static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); | uint32_t *cmd_type_len, uint32_t *olinfo_status); | ||||
static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, | |||||
uint32_t *cmd_type_len, uint32_t *olinfo_status); | |||||
static void igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype); | static void igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype); | ||||
static int igb_determine_rsstype(u16 pkt_info); | static int igb_determine_rsstype(uint16_t pkt_info); | ||||
extern void igb_if_enable_intr(if_ctx_t ctx); | extern void igb_if_enable_intr(if_ctx_t ctx); | ||||
extern int em_intr(void *arg); | extern int em_intr(void *arg); | ||||
struct if_txrx igb_txrx = { | struct if_txrx igb_txrx = { | ||||
.ift_txd_encap = igb_isc_txd_encap, | .ift_txd_encap = igb_isc_txd_encap, | ||||
.ift_txd_flush = igb_isc_txd_flush, | .ift_txd_flush = igb_isc_txd_flush, | ||||
.ift_txd_credits_update = igb_isc_txd_credits_update, | .ift_txd_credits_update = igb_isc_txd_credits_update, | ||||
.ift_rxd_available = igb_isc_rxd_available, | .ift_rxd_available = igb_isc_rxd_available, | ||||
.ift_rxd_pkt_get = igb_isc_rxd_pkt_get, | .ift_rxd_pkt_get = igb_isc_rxd_pkt_get, | ||||
.ift_rxd_refill = igb_isc_rxd_refill, | .ift_rxd_refill = igb_isc_rxd_refill, | ||||
.ift_rxd_flush = igb_isc_rxd_flush, | .ift_rxd_flush = igb_isc_rxd_flush, | ||||
.ift_legacy_intr = em_intr | .ift_legacy_intr = em_intr | ||||
}; | }; | ||||
/********************************************************************** | /********************************************************************** | ||||
* | * | ||||
* Setup work for hardware segmentation offload (TSO) on | * Setup work for hardware segmentation offload (TSO) on | ||||
* adapters using advanced tx descriptors | * adapters using advanced tx descriptors | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) | igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, uint32_t *cmd_type_len, | ||||
uint32_t *olinfo_status) | |||||
{ | { | ||||
struct e1000_adv_tx_context_desc *TXD; | struct e1000_adv_tx_context_desc *TXD; | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
u32 type_tucmd_mlhl = 0, vlan_macip_lens = 0; | uint32_t type_tucmd_mlhl = 0, vlan_macip_lens = 0; | ||||
u32 mss_l4len_idx = 0; | uint32_t mss_l4len_idx = 0; | ||||
u32 paylen; | uint32_t paylen; | ||||
switch(pi->ipi_etype) { | switch(pi->ipi_etype) { | ||||
case ETHERTYPE_IPV6: | case ETHERTYPE_IPV6: | ||||
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; | type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; | ||||
break; | break; | ||||
case ETHERTYPE_IP: | case ETHERTYPE_IP: | ||||
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; | type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; | ||||
/* Tell transmit desc to also do IPv4 checksum. */ | /* Tell transmit desc to also do IPv4 checksum. */ | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Advanced Context Descriptor setup for VLAN, CSUM or TSO | * Advanced Context Descriptor setup for VLAN, CSUM or TSO | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) | igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, uint32_t *cmd_type_len, | ||||
uint32_t *olinfo_status) | |||||
{ | { | ||||
struct e1000_adv_tx_context_desc *TXD; | struct e1000_adv_tx_context_desc *TXD; | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
u32 vlan_macip_lens, type_tucmd_mlhl; | uint32_t vlan_macip_lens, type_tucmd_mlhl; | ||||
u32 mss_l4len_idx; | uint32_t mss_l4len_idx; | ||||
mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0; | mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0; | ||||
/* First check if TSO is to be used */ | /* First check if TSO is to be used */ | ||||
if (pi->ipi_csum_flags & CSUM_TSO) | if (pi->ipi_csum_flags & CSUM_TSO) | ||||
return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status)); | return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status)); | ||||
/* Indicate the whole packet as payload when not doing TSO */ | /* Indicate the whole packet as payload when not doing TSO */ | ||||
*olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT; | *olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT; | ||||
/* Now ready a context descriptor */ | /* Now ready a context descriptor */ | ||||
TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx]; | TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx]; | ||||
/* | /* | ||||
** In advanced descriptors the vlan tag must | ** In advanced descriptors the vlan tag must | ||||
** be placed into the context descriptor. Hence | ** be placed into the context descriptor. Hence | ||||
** we need to make one even if not doing offloads. | ** we need to make one even if not doing offloads. | ||||
*/ | */ | ||||
if (pi->ipi_mflags & M_VLANTAG) { | if (pi->ipi_mflags & M_VLANTAG) { | ||||
vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT); | vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT); | ||||
} else if ((pi->ipi_csum_flags & IGB_CSUM_OFFLOAD) == 0) { | } else if ((pi->ipi_csum_flags & IGB_CSUM_OFFLOAD) == 0) { | ||||
return (0); | return (0); | ||||
} | } | ||||
/* Set the ether header length */ | /* Set the ether header length */ | ||||
vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; | vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; | ||||
switch(pi->ipi_etype) { | switch(pi->ipi_etype) { | ||||
case ETHERTYPE_IP: | case ETHERTYPE_IP: | ||||
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; | type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; | ||||
break; | break; | ||||
case ETHERTYPE_IPV6: | case ETHERTYPE_IPV6: | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | igb_isc_txd_encap(void *arg, if_pkt_info_t pi) | ||||
struct adapter *sc = arg; | struct adapter *sc = arg; | ||||
if_softc_ctx_t scctx = sc->shared; | if_softc_ctx_t scctx = sc->shared; | ||||
struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; | struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; | ||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
int nsegs = pi->ipi_nsegs; | int nsegs = pi->ipi_nsegs; | ||||
bus_dma_segment_t *segs = pi->ipi_segs; | bus_dma_segment_t *segs = pi->ipi_segs; | ||||
union e1000_adv_tx_desc *txd = NULL; | union e1000_adv_tx_desc *txd = NULL; | ||||
int i, j, pidx_last; | int i, j, pidx_last; | ||||
u32 olinfo_status, cmd_type_len, txd_flags; | uint32_t olinfo_status, cmd_type_len, txd_flags; | ||||
qidx_t ntxd; | qidx_t ntxd; | ||||
pidx_last = olinfo_status = 0; | pidx_last = olinfo_status = 0; | ||||
/* Basic descriptor defines */ | /* Basic descriptor defines */ | ||||
cmd_type_len = (E1000_ADVTXD_DTYP_DATA | | cmd_type_len = (E1000_ADVTXD_DTYP_DATA | | ||||
E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); | E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); | ||||
if (pi->ipi_mflags & M_VLANTAG) | if (pi->ipi_mflags & M_VLANTAG) | ||||
▲ Show 20 Lines • Show All 142 Lines • ▼ Show 20 Lines | |||||
static int | static int | ||||
igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) | igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) | ||||
{ | { | ||||
struct adapter *sc = arg; | struct adapter *sc = arg; | ||||
if_softc_ctx_t scctx = sc->shared; | if_softc_ctx_t scctx = sc->shared; | ||||
struct em_rx_queue *que = &sc->rx_queues[rxqid]; | struct em_rx_queue *que = &sc->rx_queues[rxqid]; | ||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||
union e1000_adv_rx_desc *rxd; | union e1000_adv_rx_desc *rxd; | ||||
u32 staterr = 0; | uint32_t staterr = 0; | ||||
int cnt, i; | int cnt, i; | ||||
for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { | for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { | ||||
rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i]; | rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i]; | ||||
staterr = le32toh(rxd->wb.upper.status_error); | staterr = le32toh(rxd->wb.upper.status_error); | ||||
if ((staterr & E1000_RXD_STAT_DD) == 0) | if ((staterr & E1000_RXD_STAT_DD) == 0) | ||||
break; | break; | ||||
Show All 14 Lines | |||||
static int | static int | ||||
igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | ||||
{ | { | ||||
struct adapter *adapter = arg; | struct adapter *adapter = arg; | ||||
if_softc_ctx_t scctx = adapter->shared; | if_softc_ctx_t scctx = adapter->shared; | ||||
struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; | struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; | ||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||
struct ifnet *ifp = iflib_get_ifp(adapter->ctx); | |||||
union e1000_adv_rx_desc *rxd; | union e1000_adv_rx_desc *rxd; | ||||
u16 pkt_info, len; | uint16_t pkt_info, len, vtag; | ||||
u16 vtag = 0; | uint32_t ptype, staterr; | ||||
u32 ptype; | int i, cidx; | ||||
u32 staterr = 0; | |||||
bool eop; | bool eop; | ||||
int i = 0; | |||||
int cidx = ri->iri_cidx; | |||||
staterr = i = vtag = 0; | |||||
cidx = ri->iri_cidx; | |||||
do { | do { | ||||
rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx]; | rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx]; | ||||
staterr = le32toh(rxd->wb.upper.status_error); | staterr = le32toh(rxd->wb.upper.status_error); | ||||
pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info); | pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info); | ||||
MPASS ((staterr & E1000_RXD_STAT_DD) != 0); | MPASS ((staterr & E1000_RXD_STAT_DD) != 0); | ||||
len = le16toh(rxd->wb.upper.length); | len = le16toh(rxd->wb.upper.length); | ||||
Show All 32 Lines | if (rxr->hdr_split == TRUE) { | ||||
cidx = 0; | cidx = 0; | ||||
} | } | ||||
#endif | #endif | ||||
i++; | i++; | ||||
} while (!eop); | } while (!eop); | ||||
rxr->rx_packets++; | rxr->rx_packets++; | ||||
if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) | if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) | ||||
markj: This assignment isn't needed. | |||||
igb_rx_checksum(staterr, ri, ptype); | igb_rx_checksum(staterr, ri, ptype); | ||||
if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && | if ((scctx->isc_capenable & IFCAP_VLAN_HWTAGGING) != 0 && | ||||
(staterr & E1000_RXD_STAT_VP) != 0) { | (staterr & E1000_RXD_STAT_VP) != 0) { | ||||
ri->iri_vtag = vtag; | ri->iri_vtag = vtag; | ||||
ri->iri_flags |= M_VLANTAG; | ri->iri_flags |= M_VLANTAG; | ||||
} | } | ||||
ri->iri_flowid = | ri->iri_flowid = | ||||
le32toh(rxd->wb.lower.hi_dword.rss); | le32toh(rxd->wb.lower.hi_dword.rss); | ||||
ri->iri_rsstype = igb_determine_rsstype(pkt_info); | ri->iri_rsstype = igb_determine_rsstype(pkt_info); | ||||
ri->iri_nfrags = i; | ri->iri_nfrags = i; | ||||
return (0); | return (0); | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Verify that the hardware indicated that the checksum is valid. | * Verify that the hardware indicated that the checksum is valid. | ||||
* Inform the stack about the status of checksum so that stack | * Inform the stack about the status of checksum so that stack | ||||
* doesn't spend time verifying the checksum. | * doesn't spend time verifying the checksum. | ||||
* | * | ||||
*********************************************************************/ | *********************************************************************/ | ||||
static void | static void | ||||
igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype) | igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype) | ||||
{ | { | ||||
u16 status = (u16)staterr; | uint16_t status = (uint16_t)staterr; | ||||
u8 errors = (u8) (staterr >> 24); | uint8_t errors = (uint8_t)(staterr >> 24); | ||||
bool sctp = FALSE; | |||||
/* Ignore Checksum bit is set */ | /* Ignore Checksum bit is set */ | ||||
if (status & E1000_RXD_STAT_IXSM) { | if (__predict_false(status & E1000_RXD_STAT_IXSM)) | ||||
ri->iri_csum_flags = 0; | |||||
return; | return; | ||||
} | |||||
if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && | /* If there is a layer 3 or 4 error we are done */ | ||||
(ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) | if (errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE)) | ||||
Not Done Inline ActionsMaybe this is also worth a __predict_false() gallatin: Maybe this is also worth a __predict_false() | |||||
sctp = 1; | return; | ||||
else | |||||
sctp = 0; | |||||
if (status & E1000_RXD_STAT_IPCS) { | |||||
/* Did it pass? */ | |||||
if (!(errors & E1000_RXD_ERR_IPE)) { | |||||
/* IP Checksum Good */ | /* IP Checksum Good */ | ||||
ri->iri_csum_flags = CSUM_IP_CHECKED; | if (status & E1000_RXD_STAT_IPCS) | ||||
Not Done Inline ActionsI would not use a __predict here, as there is no v6 l3 checksum, so I assume that v6 packets would take the unpredicted path, and be penalized. gallatin: I would not use a __predict here, as there is no v6 l3 checksum, so I assume that v6 packets… | |||||
ri->iri_csum_flags |= CSUM_IP_VALID; | ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); | ||||
} else | |||||
ri->iri_csum_flags = 0; | |||||
} | |||||
if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { | /* Valid L4E checksum */ | ||||
u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); | if (__predict_true(status & | ||||
if (sctp) /* reassign */ | (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) { | ||||
type = CSUM_SCTP_VALID; | /* SCTP header present. | ||||
/* Did it pass? */ | * XXXKB: ETQF doesn't appear to be used in igb? | ||||
if (!(errors & E1000_RXD_ERR_TCPE)) { | */ | ||||
ri->iri_csum_flags |= type; | if (__predict_false((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && | ||||
if (sctp == 0) | (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)) { | ||||
Not Done Inline ActionsIndentation here should be by four spaces. markj: Indentation here should be by four spaces. | |||||
ri->iri_csum_flags |= CSUM_SCTP_VALID; | |||||
} else { | |||||
ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; | |||||
ri->iri_csum_data = htons(0xffff); | ri->iri_csum_data = htons(0xffff); | ||||
} | } | ||||
} | } | ||||
return; | |||||
} | } | ||||
/******************************************************************** | /******************************************************************** | ||||
* | * | ||||
* Parse the packet type to determine the appropriate hash | * Parse the packet type to determine the appropriate hash | ||||
* | * | ||||
******************************************************************/ | ******************************************************************/ | ||||
static int | static int | ||||
igb_determine_rsstype(u16 pkt_info) | igb_determine_rsstype(uint16_t pkt_info) | ||||
{ | { | ||||
switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { | switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { | ||||
case E1000_RXDADV_RSSTYPE_IPV4_TCP: | case E1000_RXDADV_RSSTYPE_IPV4_TCP: | ||||
return M_HASHTYPE_RSS_TCP_IPV4; | return M_HASHTYPE_RSS_TCP_IPV4; | ||||
case E1000_RXDADV_RSSTYPE_IPV4: | case E1000_RXDADV_RSSTYPE_IPV4: | ||||
return M_HASHTYPE_RSS_IPV4; | return M_HASHTYPE_RSS_IPV4; | ||||
case E1000_RXDADV_RSSTYPE_IPV6_TCP: | case E1000_RXDADV_RSSTYPE_IPV6_TCP: | ||||
return M_HASHTYPE_RSS_TCP_IPV6; | return M_HASHTYPE_RSS_TCP_IPV6; | ||||
Show All 10 Lines |
This assignment isn't needed.