Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/ixgbe/ix_txrx.c
Show All 39 Lines | |||||
#endif | #endif | ||||
#include "ixgbe.h" | #include "ixgbe.h" | ||||
/************************************************************************ | /************************************************************************ | ||||
* Local Function prototypes | * Local Function prototypes | ||||
************************************************************************/ | ************************************************************************/ | ||||
static int ixgbe_isc_txd_encap(void *arg, if_pkt_info_t pi); | static int ixgbe_isc_txd_encap(void *, if_pkt_info_t); | ||||
static void ixgbe_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); | static void ixgbe_isc_txd_flush(void *, uint16_t, qidx_t); | ||||
static int ixgbe_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); | static int ixgbe_isc_txd_credits_update(void *, uint16_t, bool); | ||||
static void ixgbe_isc_rxd_refill(void *arg, if_rxd_update_t iru); | static void ixgbe_isc_rxd_refill(void *, if_rxd_update_t); | ||||
static void ixgbe_isc_rxd_flush(void *arg, uint16_t qsidx, | static void ixgbe_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t); | ||||
uint8_t flidx __unused, qidx_t pidx); | static int ixgbe_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t); | ||||
static int ixgbe_isc_rxd_available(void *arg, uint16_t qsidx, qidx_t pidx, | static int ixgbe_isc_rxd_pkt_get(void *, if_rxd_info_t); | ||||
qidx_t budget); | |||||
static int ixgbe_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | |||||
static void ixgbe_rx_checksum(uint32_t staterr, if_rxd_info_t ri, | static void ixgbe_rx_checksum(uint32_t, if_rxd_info_t, uint32_t); | ||||
uint32_t ptype); | |||||
static int ixgbe_tx_ctx_setup(struct ixgbe_adv_tx_context_desc *, | static int ixgbe_tx_ctx_setup(struct ixgbe_adv_tx_context_desc *, | ||||
if_pkt_info_t); | if_pkt_info_t); | ||||
extern void ixgbe_if_enable_intr(if_ctx_t ctx); | extern void ixgbe_if_enable_intr(if_ctx_t ctx); | ||||
static int ixgbe_determine_rsstype(uint16_t pkt_info); | static int ixgbe_determine_rsstype(uint16_t pkt_info); | ||||
struct if_txrx ixgbe_txrx = { | struct if_txrx ixgbe_txrx = { | ||||
.ift_txd_encap = ixgbe_isc_txd_encap, | .ift_txd_encap = ixgbe_isc_txd_encap, | ||||
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines | |||||
} /* ixgbe_tx_ctx_setup */ | } /* ixgbe_tx_ctx_setup */ | ||||
/************************************************************************ | /************************************************************************ | ||||
* ixgbe_isc_txd_encap | * ixgbe_isc_txd_encap | ||||
************************************************************************/ | ************************************************************************/ | ||||
static int | static int | ||||
ixgbe_isc_txd_encap(void *arg, if_pkt_info_t pi) | ixgbe_isc_txd_encap(void *arg, if_pkt_info_t pi) | ||||
{ | { | ||||
struct adapter *sc = arg; | struct ixgbe_softc *sc = arg; | ||||
if_softc_ctx_t scctx = sc->shared; | if_softc_ctx_t scctx = sc->shared; | ||||
struct ix_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; | struct ix_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; | ||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
int nsegs = pi->ipi_nsegs; | int nsegs = pi->ipi_nsegs; | ||||
bus_dma_segment_t *segs = pi->ipi_segs; | bus_dma_segment_t *segs = pi->ipi_segs; | ||||
union ixgbe_adv_tx_desc *txd = NULL; | union ixgbe_adv_tx_desc *txd = NULL; | ||||
struct ixgbe_adv_tx_context_desc *TXD; | struct ixgbe_adv_tx_context_desc *TXD; | ||||
int i, j, first, pidx_last; | int i, j, first, pidx_last; | ||||
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | |||||
} /* ixgbe_isc_txd_encap */ | } /* ixgbe_isc_txd_encap */ | ||||
/************************************************************************ | /************************************************************************ | ||||
* ixgbe_isc_txd_flush | * ixgbe_isc_txd_flush | ||||
************************************************************************/ | ************************************************************************/ | ||||
static void | static void | ||||
ixgbe_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) | ixgbe_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) | ||||
{ | { | ||||
struct adapter *sc = arg; | struct ixgbe_softc *sc = arg; | ||||
struct ix_tx_queue *que = &sc->tx_queues[txqid]; | struct ix_tx_queue *que = &sc->tx_queues[txqid]; | ||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
IXGBE_WRITE_REG(&sc->hw, txr->tail, pidx); | IXGBE_WRITE_REG(&sc->hw, txr->tail, pidx); | ||||
} /* ixgbe_isc_txd_flush */ | } /* ixgbe_isc_txd_flush */ | ||||
/************************************************************************ | /************************************************************************ | ||||
* ixgbe_isc_txd_credits_update | * ixgbe_isc_txd_credits_update | ||||
************************************************************************/ | ************************************************************************/ | ||||
static int | static int | ||||
ixgbe_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) | ixgbe_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) | ||||
{ | { | ||||
struct adapter *sc = arg; | struct ixgbe_softc *sc = arg; | ||||
if_softc_ctx_t scctx = sc->shared; | if_softc_ctx_t scctx = sc->shared; | ||||
struct ix_tx_queue *que = &sc->tx_queues[txqid]; | struct ix_tx_queue *que = &sc->tx_queues[txqid]; | ||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
qidx_t processed = 0; | qidx_t processed = 0; | ||||
int updated; | int updated; | ||||
qidx_t cur, prev, ntxd, rs_cidx; | qidx_t cur, prev, ntxd, rs_cidx; | ||||
int32_t delta; | int32_t delta; | ||||
uint8_t status; | uint8_t status; | ||||
Show All 40 Lines | |||||
} /* ixgbe_isc_txd_credits_update */ | } /* ixgbe_isc_txd_credits_update */ | ||||
/************************************************************************ | /************************************************************************ | ||||
* ixgbe_isc_rxd_refill | * ixgbe_isc_rxd_refill | ||||
************************************************************************/ | ************************************************************************/ | ||||
static void | static void | ||||
ixgbe_isc_rxd_refill(void *arg, if_rxd_update_t iru) | ixgbe_isc_rxd_refill(void *arg, if_rxd_update_t iru) | ||||
{ | { | ||||
struct adapter *sc = arg; | struct ixgbe_softc *sc = arg; | ||||
struct ix_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; | struct ix_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; | ||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||
uint64_t *paddrs; | uint64_t *paddrs; | ||||
int i; | int i; | ||||
uint32_t next_pidx, pidx; | uint32_t next_pidx, pidx; | ||||
uint16_t count; | uint16_t count; | ||||
paddrs = iru->iru_paddrs; | paddrs = iru->iru_paddrs; | ||||
pidx = iru->iru_pidx; | pidx = iru->iru_pidx; | ||||
count = iru->iru_count; | count = iru->iru_count; | ||||
for (i = 0, next_pidx = pidx; i < count; i++) { | for (i = 0, next_pidx = pidx; i < count; i++) { | ||||
rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); | rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); | ||||
if (++next_pidx == sc->shared->isc_nrxd[0]) | if (++next_pidx == sc->shared->isc_nrxd[0]) | ||||
next_pidx = 0; | next_pidx = 0; | ||||
} | } | ||||
} /* ixgbe_isc_rxd_refill */ | } /* ixgbe_isc_rxd_refill */ | ||||
/************************************************************************ | /************************************************************************ | ||||
* ixgbe_isc_rxd_flush | * ixgbe_isc_rxd_flush | ||||
************************************************************************/ | ************************************************************************/ | ||||
static void | static void | ||||
ixgbe_isc_rxd_flush(void *arg, uint16_t qsidx, uint8_t flidx __unused, qidx_t pidx) | ixgbe_isc_rxd_flush(void *arg, uint16_t qsidx, uint8_t flidx __unused, qidx_t pidx) | ||||
{ | { | ||||
struct adapter *sc = arg; | struct ixgbe_softc *sc = arg; | ||||
struct ix_rx_queue *que = &sc->rx_queues[qsidx]; | struct ix_rx_queue *que = &sc->rx_queues[qsidx]; | ||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||
IXGBE_WRITE_REG(&sc->hw, rxr->tail, pidx); | IXGBE_WRITE_REG(&sc->hw, rxr->tail, pidx); | ||||
} /* ixgbe_isc_rxd_flush */ | } /* ixgbe_isc_rxd_flush */ | ||||
/************************************************************************ | /************************************************************************ | ||||
* ixgbe_isc_rxd_available | * ixgbe_isc_rxd_available | ||||
************************************************************************/ | ************************************************************************/ | ||||
static int | static int | ||||
ixgbe_isc_rxd_available(void *arg, uint16_t qsidx, qidx_t pidx, qidx_t budget) | ixgbe_isc_rxd_available(void *arg, uint16_t qsidx, qidx_t pidx, qidx_t budget) | ||||
{ | { | ||||
struct adapter *sc = arg; | struct ixgbe_softc *sc = arg; | ||||
struct ix_rx_queue *que = &sc->rx_queues[qsidx]; | struct ix_rx_queue *que = &sc->rx_queues[qsidx]; | ||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||
union ixgbe_adv_rx_desc *rxd; | union ixgbe_adv_rx_desc *rxd; | ||||
uint32_t staterr; | uint32_t staterr; | ||||
int cnt, i, nrxd; | int cnt, i, nrxd; | ||||
nrxd = sc->shared->isc_nrxd[0]; | nrxd = sc->shared->isc_nrxd[0]; | ||||
for (cnt = 0, i = pidx; cnt < nrxd && cnt <= budget;) { | for (cnt = 0, i = pidx; cnt < nrxd && cnt <= budget;) { | ||||
Show All 17 Lines | |||||
* to upper layer. Initialize ri structure. | * to upper layer. Initialize ri structure. | ||||
* | * | ||||
* Returns 0 upon success, errno on failure | * Returns 0 upon success, errno on failure | ||||
************************************************************************/ | ************************************************************************/ | ||||
static int | static int | ||||
ixgbe_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | ixgbe_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | ||||
{ | { | ||||
struct adapter *adapter = arg; | struct ixgbe_softc *sc = arg; | ||||
struct ix_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; | struct ix_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; | ||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||
struct ifnet *ifp = iflib_get_ifp(adapter->ctx); | struct ifnet *ifp = iflib_get_ifp(sc->ctx); | ||||
union ixgbe_adv_rx_desc *rxd; | union ixgbe_adv_rx_desc *rxd; | ||||
uint16_t pkt_info, len, cidx, i; | uint16_t pkt_info, len, cidx, i; | ||||
uint16_t vtag = 0; | uint16_t vtag = 0; | ||||
uint32_t ptype; | uint32_t ptype; | ||||
uint32_t staterr = 0; | uint32_t staterr = 0; | ||||
bool eop; | bool eop; | ||||
Show All 20 Lines | do { | ||||
if ( (rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP) ) { | if ( (rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP) ) { | ||||
vtag = le16toh(rxd->wb.upper.vlan); | vtag = le16toh(rxd->wb.upper.vlan); | ||||
} else { | } else { | ||||
vtag = 0; | vtag = 0; | ||||
} | } | ||||
/* Make sure bad packets are discarded */ | /* Make sure bad packets are discarded */ | ||||
if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) { | if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) { | ||||
if (adapter->feat_en & IXGBE_FEATURE_VF) | if (sc->feat_en & IXGBE_FEATURE_VF) | ||||
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); | if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); | ||||
rxr->rx_discarded++; | rxr->rx_discarded++; | ||||
return (EBADMSG); | return (EBADMSG); | ||||
} | } | ||||
ri->iri_frags[i].irf_flid = 0; | ri->iri_frags[i].irf_flid = 0; | ||||
ri->iri_frags[i].irf_idx = cidx; | ri->iri_frags[i].irf_idx = cidx; | ||||
ri->iri_frags[i].irf_len = len; | ri->iri_frags[i].irf_len = len; | ||||
if (++cidx == adapter->shared->isc_nrxd[0]) | if (++cidx == sc->shared->isc_nrxd[0]) | ||||
cidx = 0; | cidx = 0; | ||||
i++; | i++; | ||||
/* even a 16K packet shouldn't consume more than 8 clusters */ | /* even a 16K packet shouldn't consume more than 8 clusters */ | ||||
MPASS(i < 9); | MPASS(i < 9); | ||||
} while (!eop); | } while (!eop); | ||||
rxr->rx_packets++; | rxr->rx_packets++; | ||||
rxr->packets++; | rxr->packets++; | ||||
rxr->rx_bytes += ri->iri_len; | rxr->rx_bytes += ri->iri_len; | ||||
if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) | if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) | ||||
ixgbe_rx_checksum(staterr, ri, ptype); | ixgbe_rx_checksum(staterr, ri, ptype); | ||||
ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); | ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); | ||||
ri->iri_rsstype = ixgbe_determine_rsstype(pkt_info); | ri->iri_rsstype = ixgbe_determine_rsstype(pkt_info); | ||||
if ((adapter->feat_en & IXGBE_FEATURE_RSS) == 0) { | if ((sc->feat_en & IXGBE_FEATURE_RSS) == 0) { | ||||
if (ri->iri_rsstype == M_HASHTYPE_OPAQUE) | if (ri->iri_rsstype == M_HASHTYPE_OPAQUE) | ||||
ri->iri_rsstype = M_HASHTYPE_NONE; | ri->iri_rsstype = M_HASHTYPE_NONE; | ||||
else | else | ||||
ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH; | ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH; | ||||
} | } | ||||
ri->iri_vtag = vtag; | ri->iri_vtag = vtag; | ||||
ri->iri_nfrags = i; | ri->iri_nfrags = i; | ||||
if (vtag) | if (vtag) | ||||
▲ Show 20 Lines • Show All 69 Lines • Show Last 20 Lines |