Changeset View
Standalone View
sys/dev/e1000/em_txrx.c
/*- | /*- | ||||||||||
* SPDX-License-Identifier: BSD-2-Clause | |||||||||||
* | |||||||||||
* Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> | * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> | ||||||||||
* Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> | * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> | ||||||||||
* All rights reserved. | * All rights reserved. | ||||||||||
* | * | ||||||||||
* Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||||||||
* modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||||||||
* are met: | * are met: | ||||||||||
* 1. Redistributions of source code must retain the above copyright | * 1. Redistributions of source code must retain the above copyright | ||||||||||
Show All 27 Lines | |||||||||||
#define DPRINTF device_printf | #define DPRINTF device_printf | ||||||||||
#else | #else | ||||||||||
#define DPRINTF(...) | #define DPRINTF(...) | ||||||||||
#endif | #endif | ||||||||||
/********************************************************************* | /********************************************************************* | ||||||||||
* Local Function prototypes | * Local Function prototypes | ||||||||||
*********************************************************************/ | *********************************************************************/ | ||||||||||
static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, | static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, | ||||||||||
u32 *txd_lower); | uint32_t *txd_upper, uint32_t *txd_lower); | ||||||||||
static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, | static int em_transmit_checksum_setup(struct adapter *adapter, | ||||||||||
u32 *txd_upper, u32 *txd_lower); | if_pkt_info_t pi, uint32_t *txd_upper, uint32_t *txd_lower); | ||||||||||
static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); | static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); | ||||||||||
static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); | static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); | ||||||||||
static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); | static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); | ||||||||||
static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru); | static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru); | ||||||||||
static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, | static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, | ||||||||||
qidx_t pidx); | qidx_t pidx); | ||||||||||
static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, | static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, | ||||||||||
qidx_t budget); | qidx_t budget); | ||||||||||
static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | ||||||||||
static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru); | static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru); | ||||||||||
static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, | static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, | ||||||||||
qidx_t budget); | qidx_t budget); | ||||||||||
static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | ||||||||||
static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t); | static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t); | ||||||||||
static int em_determine_rsstype(u32 pkt_info); | static int em_determine_rsstype(uint32_t pkt_info); | ||||||||||
extern int em_intr(void *arg); | extern int em_intr(void *arg); | ||||||||||
struct if_txrx em_txrx = { | struct if_txrx em_txrx = { | ||||||||||
.ift_txd_encap = em_isc_txd_encap, | .ift_txd_encap = em_isc_txd_encap, | ||||||||||
.ift_txd_flush = em_isc_txd_flush, | .ift_txd_flush = em_isc_txd_flush, | ||||||||||
.ift_txd_credits_update = em_isc_txd_credits_update, | .ift_txd_credits_update = em_isc_txd_credits_update, | ||||||||||
.ift_rxd_available = em_isc_rxd_available, | .ift_rxd_available = em_isc_rxd_available, | ||||||||||
.ift_rxd_pkt_get = em_isc_rxd_pkt_get, | .ift_rxd_pkt_get = em_isc_rxd_pkt_get, | ||||||||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | |||||||||||
/********************************************************************** | /********************************************************************** | ||||||||||
* | * | ||||||||||
* Setup work for hardware segmentation offload (TSO) on | * Setup work for hardware segmentation offload (TSO) on | ||||||||||
* adapters using advanced tx descriptors | * adapters using advanced tx descriptors | ||||||||||
* | * | ||||||||||
**********************************************************************/ | **********************************************************************/ | ||||||||||
static int | static int | ||||||||||
em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) | em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, uint32_t *txd_upper, | ||||||||||
uint32_t *txd_lower) | |||||||||||
{ | { | ||||||||||
if_softc_ctx_t scctx = adapter->shared; | if_softc_ctx_t scctx = adapter->shared; | ||||||||||
struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; | struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; | ||||||||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||||||||
struct e1000_hw *hw = &adapter->hw; | |||||||||||
struct e1000_context_desc *TXD; | struct e1000_context_desc *TXD; | ||||||||||
int cur, hdr_len; | int cur, hdr_len; | ||||||||||
uint32_t cmd_type_len; | |||||||||||
hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; | hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; | ||||||||||
*txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ | *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ | ||||||||||
E1000_TXD_DTYP_D | /* Data descr type */ | E1000_TXD_DTYP_D | /* Data descr type */ | ||||||||||
E1000_TXD_CMD_TSE); /* Do TSE on this packet */ | E1000_TXD_CMD_TSE); /* Do TSE on this packet */ | ||||||||||
/* IP and/or TCP header checksum calculation and insertion. */ | /* IP and/or TCP header checksum calculation and insertion. */ | ||||||||||
*txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; | *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; | ||||||||||
Show All 23 Lines | em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, uint32_t *txd_upper, | ||||||||||
/* | /* | ||||||||||
* Payload size per packet w/o any headers. | * Payload size per packet w/o any headers. | ||||||||||
* Length of all headers up to payload. | * Length of all headers up to payload. | ||||||||||
*/ | */ | ||||||||||
TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); | TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); | ||||||||||
TXD->tcp_seg_setup.fields.hdr_len = hdr_len; | TXD->tcp_seg_setup.fields.hdr_len = hdr_len; | ||||||||||
TXD->cmd_and_length = htole32(adapter->txd_cmd | | /* 8254x SDM4.0 page 45, and PCIe GbE SDM2.5 page 63 | ||||||||||
markjUnsubmitted Not Done Inline Actions
markj: | |||||||||||
* - Set up basic TUCMDs | |||||||||||
* - Enable IP bit on 82544. | |||||||||||
* - For others on indicates IPv4, while off indicates IPv6 | |||||||||||
*/ | |||||||||||
cmd_type_len = adapter->txd_cmd | | |||||||||||
E1000_TXD_CMD_DEXT | /* Extended descr */ | E1000_TXD_CMD_DEXT | /* Extended descr */ | ||||||||||
E1000_TXD_CMD_TSE | /* TSE context */ | E1000_TXD_CMD_TSE | /* TSE context */ | ||||||||||
E1000_TXD_CMD_IP | /* Do IP csum */ | E1000_TXD_CMD_TCP; /* Do TCP checksum */ | ||||||||||
E1000_TXD_CMD_TCP | /* Do TCP checksum */ | if (hw->mac.type == e1000_82544) | ||||||||||
cmd_type_len |= E1000_TXD_CMD_IP; | |||||||||||
else if (pi->ipi_etype == ETHERTYPE_IP) | |||||||||||
cmd_type_len |= E1000_TXD_CMD_IP; | |||||||||||
TXD->cmd_and_length = htole32(cmd_type_len | | |||||||||||
(pi->ipi_len - hdr_len)); /* Total len */ | (pi->ipi_len - hdr_len)); /* Total len */ | ||||||||||
txr->tx_tso = TRUE; | txr->tx_tso = TRUE; | ||||||||||
if (++cur == scctx->isc_ntxd[0]) { | if (++cur == scctx->isc_ntxd[0]) { | ||||||||||
cur = 0; | cur = 0; | ||||||||||
} | } | ||||||||||
DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur); | DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, | ||||||||||
pi->ipi_pidx, cur); | |||||||||||
return (cur); | return (cur); | ||||||||||
} | } | ||||||||||
#define TSO_WORKAROUND 4 | static int | ||||||||||
#define DONT_FORCE_CTX 1 | em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, | ||||||||||
uint32_t *txd_upper, uint32_t *txd_lower) | |||||||||||
{ | |||||||||||
struct e1000_context_desc *TXD = NULL; | |||||||||||
if_softc_ctx_t scctx = adapter->shared; | |||||||||||
struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; | |||||||||||
struct tx_ring *txr = &que->txr; | |||||||||||
int csum_flags = pi->ipi_csum_flags; | |||||||||||
int cur, hdr_len; | |||||||||||
uint32_t cmd; | |||||||||||
cur = pi->ipi_pidx; | |||||||||||
hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; | |||||||||||
cmd = adapter->txd_cmd; | |||||||||||
/********************************************************************* | /********************************************************************* | ||||||||||
* The offload context is protocol specific (TCP/UDP) and thus | * The offload context is protocol specific (TCP/UDP) and thus | ||||||||||
* only needs to be set when the protocol changes. The occasion | * only needs to be set when the protocol changes. The occasion | ||||||||||
* of a context change can be a performance detriment, and | * of a context change can be a performance detriment, and | ||||||||||
* might be better just disabled. The reason arises in the way | * might be better just disabled. The reason arises in the way | ||||||||||
* in which the controller supports pipelined requests from the | * in which the controller supports pipelined requests from the | ||||||||||
* Tx data DMA. Up to four requests can be pipelined, and they may | * Tx data DMA. Up to four requests can be pipelined, and they may | ||||||||||
* belong to the same packet or to multiple packets. However all | * belong to the same packet or to multiple packets. However all | ||||||||||
* requests for one packet are issued before a request is issued | * requests for one packet are issued before a request is issued | ||||||||||
* for a subsequent packet and if a request for the next packet | * for a subsequent packet and if a request for the next packet | ||||||||||
* requires a context change, that request will be stalled | * requires a context change, that request will be stalled | ||||||||||
* until the previous request completes. This means setting up | * until the previous request completes. This means setting up | ||||||||||
* a new context effectively disables pipelined Tx data DMA which | * a new context effectively disables pipelined Tx data DMA which | ||||||||||
* in turn greatly slow down performance to send small sized | * in turn greatly slow down performance to send small sized | ||||||||||
* frames. | * frames. | ||||||||||
**********************************************************************/ | * | ||||||||||
static int | |||||||||||
em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) | |||||||||||
{ | |||||||||||
struct e1000_context_desc *TXD = NULL; | |||||||||||
if_softc_ctx_t scctx = adapter->shared; | |||||||||||
struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; | |||||||||||
struct tx_ring *txr = &que->txr; | |||||||||||
int csum_flags = pi->ipi_csum_flags; | |||||||||||
int cur, hdr_len; | |||||||||||
u32 cmd; | |||||||||||
cur = pi->ipi_pidx; | |||||||||||
hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; | |||||||||||
cmd = adapter->txd_cmd; | |||||||||||
/* | |||||||||||
* The 82574L can only remember the *last* context used | * The 82574L can only remember the *last* context used | ||||||||||
* regardless of queue that it was use for. We cannot reuse | * regardless of queue that it was use for. We cannot reuse | ||||||||||
* contexts on this hardware platform and must generate a new | * contexts on this hardware platform and must generate a new | ||||||||||
* context every time. 82574L hardware spec, section 7.2.6, | * context every time. 82574L hardware spec, section 7.2.6, | ||||||||||
* second note. | * second note. | ||||||||||
*/ | **********************************************************************/ | ||||||||||
#define DONT_FORCE_CTX 1 | |||||||||||
if (DONT_FORCE_CTX && | if (DONT_FORCE_CTX && | ||||||||||
adapter->tx_num_queues == 1 && | adapter->tx_num_queues == 1 && | ||||||||||
txr->csum_lhlen == pi->ipi_ehdrlen && | txr->csum_lhlen == pi->ipi_ehdrlen && | ||||||||||
txr->csum_iphlen == pi->ipi_ip_hlen && | txr->csum_iphlen == pi->ipi_ip_hlen && | ||||||||||
txr->csum_flags == csum_flags) { | txr->csum_flags == csum_flags) { | ||||||||||
/* | /* | ||||||||||
* Same csum offload context as the previous packets; | * Same csum offload context as the previous packets; | ||||||||||
* just return. | * just return. | ||||||||||
*/ | */ | ||||||||||
*txd_upper = txr->csum_txd_upper; | *txd_upper = txr->csum_txd_upper; | ||||||||||
*txd_lower = txr->csum_txd_lower; | *txd_lower = txr->csum_txd_lower; | ||||||||||
return (cur); | return (cur); | ||||||||||
} | } | ||||||||||
TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; | TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; | ||||||||||
if (csum_flags & CSUM_IP) { | if (csum_flags & CSUM_IP) { | ||||||||||
*txd_upper |= E1000_TXD_POPTS_IXSM << 8; | *txd_upper |= E1000_TXD_POPTS_IXSM << 8; | ||||||||||
/* | /* | ||||||||||
* Start offset for header checksum calculation. | * Start offset for header checksum calculation. | ||||||||||
* End offset for header checksum calculation. | * End offset for header checksum calculation. | ||||||||||
* Offset of place to put the checksum. | * Offset of place to put the checksum. | ||||||||||
*/ | */ | ||||||||||
TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; | TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; | ||||||||||
TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); | TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); | ||||||||||
TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); | TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + | ||||||||||
offsetof(struct ip, ip_sum); | |||||||||||
cmd |= E1000_TXD_CMD_IP; | cmd |= E1000_TXD_CMD_IP; | ||||||||||
} | } | ||||||||||
if (csum_flags & (CSUM_TCP|CSUM_UDP)) { | if (csum_flags & (CSUM_TCP|CSUM_UDP)) { | ||||||||||
uint8_t tucso; | uint8_t tucso; | ||||||||||
*txd_upper |= E1000_TXD_POPTS_TXSM << 8; | *txd_upper |= E1000_TXD_POPTS_TXSM << 8; | ||||||||||
*txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; | *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; | ||||||||||
Show All 32 Lines | em_isc_txd_encap(void *arg, if_pkt_info_t pi) | ||||||||||
struct adapter *sc = arg; | struct adapter *sc = arg; | ||||||||||
if_softc_ctx_t scctx = sc->shared; | if_softc_ctx_t scctx = sc->shared; | ||||||||||
struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; | struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; | ||||||||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||||||||
bus_dma_segment_t *segs = pi->ipi_segs; | bus_dma_segment_t *segs = pi->ipi_segs; | ||||||||||
int nsegs = pi->ipi_nsegs; | int nsegs = pi->ipi_nsegs; | ||||||||||
int csum_flags = pi->ipi_csum_flags; | int csum_flags = pi->ipi_csum_flags; | ||||||||||
int i, j, first, pidx_last; | int i, j, first, pidx_last; | ||||||||||
u32 txd_flags, txd_upper = 0, txd_lower = 0; | uint32_t txd_flags, txd_upper = 0, txd_lower = 0; | ||||||||||
struct e1000_tx_desc *ctxd = NULL; | struct e1000_tx_desc *ctxd = NULL; | ||||||||||
bool do_tso, tso_desc; | bool do_tso, tso_desc; | ||||||||||
qidx_t ntxd; | qidx_t ntxd; | ||||||||||
txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; | txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; | ||||||||||
i = first = pi->ipi_pidx; | i = first = pi->ipi_pidx; | ||||||||||
do_tso = (csum_flags & CSUM_TSO); | do_tso = (csum_flags & CSUM_TSO); | ||||||||||
Show All 14 Lines | em_isc_txd_encap(void *arg, if_pkt_info_t pi) | ||||||||||
/* Do hardware assists */ | /* Do hardware assists */ | ||||||||||
if (do_tso) { | if (do_tso) { | ||||||||||
i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); | i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); | ||||||||||
tso_desc = TRUE; | tso_desc = TRUE; | ||||||||||
} else if (csum_flags & EM_CSUM_OFFLOAD) { | } else if (csum_flags & EM_CSUM_OFFLOAD) { | ||||||||||
i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); | i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); | ||||||||||
} | } | ||||||||||
if (pi->ipi_mflags & M_VLANTAG) { | if (pi->ipi_mflags & M_VLANTAG && !do_tso) { | ||||||||||
markjUnsubmitted Not Done Inline ActionsWhy is this change needed? markj: Why is this change needed? | |||||||||||
kbowlingAuthorUnsubmitted Not Done Inline ActionsIt's insufficient as is, we need to properly handle legacy, context and data descriptors for lem/em and I haven't wrapped my brain around it yet to make the encap code look and work nice. The primary issue is we are supposed to put the vlan tag and mark the VLE bit in the last data descriptor following a context descriptor. The lem versus em datasheet is confusing, it says put the tag and VLE in the first data descriptor if using the context/data descriptors for hwcsum non-TSE (aka non-TSO), and last if TSE (TSO) so I'm still trying to understand it. kbowling: It's insufficient as is, we need to properly handle legacy, context and data descriptors for… | |||||||||||
/* Set the vlan id. */ | /* Set the vlan id. */ | ||||||||||
txd_upper |= htole16(pi->ipi_vtag) << 16; | txd_upper |= htole16(pi->ipi_vtag) << 16; | ||||||||||
/* Tell hardware to add tag */ | /* Tell hardware to add tag */ | ||||||||||
txd_lower |= htole32(E1000_TXD_CMD_VLE); | txd_lower |= htole32(E1000_TXD_CMD_VLE); | ||||||||||
} | } | ||||||||||
DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); | DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); | ||||||||||
/* XXX adapter->pcix_82544 -- lem_fill_descriptors */ | /* XXX adapter->pcix_82544 -- lem_fill_descriptors */ | ||||||||||
Show All 9 Lines | for (j = 0; j < nsegs; j++) { | ||||||||||
seg_len = segs[j].ds_len; | seg_len = segs[j].ds_len; | ||||||||||
cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; | cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; | ||||||||||
/* | /* | ||||||||||
* TSO Workaround: | * TSO Workaround: | ||||||||||
* If this is the last descriptor, we want to | * If this is the last descriptor, we want to | ||||||||||
* split it so we have a small final sentinel | * split it so we have a small final sentinel | ||||||||||
*/ | */ | ||||||||||
#define TSO_WORKAROUND 4 | |||||||||||
if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { | if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { | ||||||||||
seg_len -= TSO_WORKAROUND; | seg_len -= TSO_WORKAROUND; | ||||||||||
ctxd->buffer_addr = htole64(seg_addr); | ctxd->buffer_addr = htole64(seg_addr); | ||||||||||
ctxd->lower.data = htole32(cmd | txd_lower | seg_len); | ctxd->lower.data = htole32(cmd | txd_lower | seg_len); | ||||||||||
ctxd->upper.data = htole32(txd_upper); | ctxd->upper.data = htole32(txd_upper); | ||||||||||
if (++i == scctx->isc_ntxd[0]) | if (++i == scctx->isc_ntxd[0]) | ||||||||||
i = 0; | i = 0; | ||||||||||
▲ Show 20 Lines • Show All 173 Lines • ▼ Show 20 Lines | |||||||||||
static int | static int | ||||||||||
lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) | lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) | ||||||||||
{ | { | ||||||||||
struct adapter *sc = arg; | struct adapter *sc = arg; | ||||||||||
if_softc_ctx_t scctx = sc->shared; | if_softc_ctx_t scctx = sc->shared; | ||||||||||
struct em_rx_queue *que = &sc->rx_queues[rxqid]; | struct em_rx_queue *que = &sc->rx_queues[rxqid]; | ||||||||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||||||||
struct e1000_rx_desc *rxd; | struct e1000_rx_desc *rxd; | ||||||||||
u32 staterr = 0; | uint32_t staterr = 0; | ||||||||||
int cnt, i; | int cnt, i; | ||||||||||
for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { | for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { | ||||||||||
rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; | rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; | ||||||||||
staterr = rxd->status; | staterr = rxd->status; | ||||||||||
if ((staterr & E1000_RXD_STAT_DD) == 0) | if ((staterr & E1000_RXD_STAT_DD) == 0) | ||||||||||
break; | break; | ||||||||||
if (++i == scctx->isc_nrxd[0]) | if (++i == scctx->isc_nrxd[0]) | ||||||||||
i = 0; | i = 0; | ||||||||||
if (staterr & E1000_RXD_STAT_EOP) | if (staterr & E1000_RXD_STAT_EOP) | ||||||||||
cnt++; | cnt++; | ||||||||||
} | } | ||||||||||
return (cnt); | return (cnt); | ||||||||||
} | } | ||||||||||
static int | static int | ||||||||||
em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) | em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) | ||||||||||
{ | { | ||||||||||
struct adapter *sc = arg; | struct adapter *sc = arg; | ||||||||||
if_softc_ctx_t scctx = sc->shared; | if_softc_ctx_t scctx = sc->shared; | ||||||||||
struct em_rx_queue *que = &sc->rx_queues[rxqid]; | struct em_rx_queue *que = &sc->rx_queues[rxqid]; | ||||||||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||||||||
union e1000_rx_desc_extended *rxd; | union e1000_rx_desc_extended *rxd; | ||||||||||
u32 staterr = 0; | uint32_t staterr = 0; | ||||||||||
int cnt, i; | int cnt, i; | ||||||||||
for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { | for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { | ||||||||||
rxd = &rxr->rx_base[i]; | rxd = &rxr->rx_base[i]; | ||||||||||
staterr = le32toh(rxd->wb.upper.status_error); | staterr = le32toh(rxd->wb.upper.status_error); | ||||||||||
if ((staterr & E1000_RXD_STAT_DD) == 0) | if ((staterr & E1000_RXD_STAT_DD) == 0) | ||||||||||
break; | break; | ||||||||||
if (++i == scctx->isc_nrxd[0]) | if (++i == scctx->isc_nrxd[0]) | ||||||||||
i = 0; | i = 0; | ||||||||||
if (staterr & E1000_RXD_STAT_EOP) | if (staterr & E1000_RXD_STAT_EOP) | ||||||||||
cnt++; | cnt++; | ||||||||||
} | } | ||||||||||
return (cnt); | return (cnt); | ||||||||||
} | } | ||||||||||
static int | static int | ||||||||||
lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | ||||||||||
{ | { | ||||||||||
struct adapter *adapter = arg; | struct adapter *adapter = arg; | ||||||||||
if_softc_ctx_t scctx = adapter->shared; | if_softc_ctx_t scctx = adapter->shared; | ||||||||||
Done Inline ActionsI would have gone for the ifp too, but the "iflib way" is to avoid accessing this directly when possible. Can you get what you need from scctx->isc_capenable ? gallatin: I would have gone for the ifp too, but the "iflib way" is to avoid accessing this directly when… | |||||||||||
struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; | struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; | ||||||||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||||||||
struct e1000_rx_desc *rxd; | struct e1000_rx_desc *rxd; | ||||||||||
u16 len; | uint16_t len, vtag; | ||||||||||
u32 status, errors; | uint32_t status, errors; | ||||||||||
bool eop; | bool eop; | ||||||||||
int i, cidx; | int i, cidx; | ||||||||||
status = errors = i = 0; | status = errors = i = vtag = 0; | ||||||||||
cidx = ri->iri_cidx; | cidx = ri->iri_cidx; | ||||||||||
do { | do { | ||||||||||
rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; | rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; | ||||||||||
status = rxd->status; | status = rxd->status; | ||||||||||
errors = rxd->errors; | errors = rxd->errors; | ||||||||||
/* Error Checking then decrement count */ | /* Error Checking then decrement count */ | ||||||||||
MPASS ((status & E1000_RXD_STAT_DD) != 0); | MPASS ((status & E1000_RXD_STAT_DD) != 0); | ||||||||||
len = le16toh(rxd->length); | len = le16toh(rxd->length); | ||||||||||
ri->iri_len += len; | ri->iri_len += len; | ||||||||||
eop = (status & E1000_RXD_STAT_EOP) != 0; | eop = (status & E1000_RXD_STAT_EOP) != 0; | ||||||||||
vtag = le16toh(rxd->special & E1000_RXD_SPC_VLAN_MASK); | |||||||||||
markjUnsubmitted Not Done Inline ActionsWhy fetch it from each descriptor? markj: Why fetch it from each descriptor? | |||||||||||
kbowlingAuthorUnsubmitted Not Done Inline ActionsI had the same concern, trying to unify the igb and ixgbe txrx so I will prep an update for those to move it outside the loop. kbowling: I had the same concern, trying to unify the igb and ixgbe txrx so I will prep an update for… | |||||||||||
/* Make sure bad packets are discarded */ | /* Make sure bad packets are discarded */ | ||||||||||
if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { | if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { | ||||||||||
adapter->dropped_pkts++; | adapter->dropped_pkts++; | ||||||||||
/* XXX fixup if common */ | /* XXX fixup if common */ | ||||||||||
return (EBADMSG); | return (EBADMSG); | ||||||||||
} | } | ||||||||||
ri->iri_frags[i].irf_flid = 0; | ri->iri_frags[i].irf_flid = 0; | ||||||||||
ri->iri_frags[i].irf_idx = cidx; | ri->iri_frags[i].irf_idx = cidx; | ||||||||||
ri->iri_frags[i].irf_len = len; | ri->iri_frags[i].irf_len = len; | ||||||||||
/* Zero out the receive descriptors status. */ | /* Zero out the receive descriptors status. */ | ||||||||||
rxd->status = 0; | rxd->status = 0; | ||||||||||
if (++cidx == scctx->isc_nrxd[0]) | if (++cidx == scctx->isc_nrxd[0]) | ||||||||||
cidx = 0; | cidx = 0; | ||||||||||
i++; | i++; | ||||||||||
} while (!eop); | } while (!eop); | ||||||||||
Done Inline ActionsI believe iflib ensures that this field is zeroed. Otherwise, note that we assume ri->iri_flags is zeroed when ORing M_VLANTAG below. markj: I believe iflib ensures that this field is zeroed. Otherwise, note that we assume `ri… | |||||||||||
/* XXX add a faster way to look this up */ | if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) | ||||||||||
if (adapter->hw.mac.type >= e1000_82543) | |||||||||||
em_receive_checksum(status, errors, ri); | em_receive_checksum(status, errors, ri); | ||||||||||
if (status & E1000_RXD_STAT_VP) { | if ((scctx->isc_capenable & IFCAP_VLAN_HWTAGGING) != 0 && | ||||||||||
ri->iri_vtag = le16toh(rxd->special); | (status & E1000_RXD_STAT_VP) != 0) { | ||||||||||
markjUnsubmitted Not Done Inline Actions
markj: | |||||||||||
ri->iri_vtag = vtag; | |||||||||||
ri->iri_flags |= M_VLANTAG; | ri->iri_flags |= M_VLANTAG; | ||||||||||
} | } | ||||||||||
ri->iri_nfrags = i; | ri->iri_nfrags = i; | ||||||||||
return (0); | return (0); | ||||||||||
} | } | ||||||||||
static int | static int | ||||||||||
em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | ||||||||||
{ | { | ||||||||||
struct adapter *adapter = arg; | struct adapter *adapter = arg; | ||||||||||
if_softc_ctx_t scctx = adapter->shared; | if_softc_ctx_t scctx = adapter->shared; | ||||||||||
struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; | struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; | ||||||||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||||||||
union e1000_rx_desc_extended *rxd; | union e1000_rx_desc_extended *rxd; | ||||||||||
u16 len; | uint16_t len, vtag; | ||||||||||
u32 pkt_info; | uint32_t pkt_info; | ||||||||||
u32 staterr = 0; | uint32_t staterr; | ||||||||||
bool eop; | bool eop; | ||||||||||
int i, cidx, vtag; | int i, cidx; | ||||||||||
i = vtag = 0; | staterr = i = vtag = 0; | ||||||||||
cidx = ri->iri_cidx; | cidx = ri->iri_cidx; | ||||||||||
do { | do { | ||||||||||
rxd = &rxr->rx_base[cidx]; | rxd = &rxr->rx_base[cidx]; | ||||||||||
staterr = le32toh(rxd->wb.upper.status_error); | staterr = le32toh(rxd->wb.upper.status_error); | ||||||||||
pkt_info = le32toh(rxd->wb.lower.mrq); | pkt_info = le32toh(rxd->wb.lower.mrq); | ||||||||||
/* Error Checking then decrement count */ | /* Error Checking then decrement count */ | ||||||||||
MPASS ((staterr & E1000_RXD_STAT_DD) != 0); | MPASS ((staterr & E1000_RXD_STAT_DD) != 0); | ||||||||||
len = le16toh(rxd->wb.upper.length); | len = le16toh(rxd->wb.upper.length); | ||||||||||
ri->iri_len += len; | ri->iri_len += len; | ||||||||||
eop = (staterr & E1000_RXD_STAT_EOP) != 0; | eop = (staterr & E1000_RXD_STAT_EOP) != 0; | ||||||||||
vtag = le16toh(rxd->wb.upper.vlan); | |||||||||||
/* Make sure bad packets are discarded */ | /* Make sure bad packets are discarded */ | ||||||||||
if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { | if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { | ||||||||||
adapter->dropped_pkts++; | adapter->dropped_pkts++; | ||||||||||
return EBADMSG; | return EBADMSG; | ||||||||||
} | } | ||||||||||
ri->iri_frags[i].irf_flid = 0; | ri->iri_frags[i].irf_flid = 0; | ||||||||||
ri->iri_frags[i].irf_idx = cidx; | ri->iri_frags[i].irf_idx = cidx; | ||||||||||
ri->iri_frags[i].irf_len = len; | ri->iri_frags[i].irf_len = len; | ||||||||||
/* Zero out the receive descriptors status. */ | /* Zero out the receive descriptors status. */ | ||||||||||
rxd->wb.upper.status_error &= htole32(~0xFF); | rxd->wb.upper.status_error &= htole32(~0xFF); | ||||||||||
if (++cidx == scctx->isc_nrxd[0]) | if (++cidx == scctx->isc_nrxd[0]) | ||||||||||
cidx = 0; | cidx = 0; | ||||||||||
i++; | i++; | ||||||||||
} while (!eop); | } while (!eop); | ||||||||||
if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) | |||||||||||
em_receive_checksum(staterr, staterr >> 24, ri); | em_receive_checksum(staterr, staterr >> 24, ri); | ||||||||||
if (staterr & E1000_RXD_STAT_VP) { | if ((scctx->isc_capenable & IFCAP_VLAN_HWTAGGING) != 0 && | ||||||||||
vtag = le16toh(rxd->wb.upper.vlan); | (staterr & E1000_RXD_STAT_VP) != 0) { | ||||||||||
markjUnsubmitted Not Done Inline Actions
markj: | |||||||||||
} | |||||||||||
ri->iri_vtag = vtag; | ri->iri_vtag = vtag; | ||||||||||
if (vtag) | |||||||||||
ri->iri_flags |= M_VLANTAG; | ri->iri_flags |= M_VLANTAG; | ||||||||||
} | |||||||||||
ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); | ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); | ||||||||||
ri->iri_rsstype = em_determine_rsstype(pkt_info); | ri->iri_rsstype = em_determine_rsstype(pkt_info); | ||||||||||
ri->iri_nfrags = i; | ri->iri_nfrags = i; | ||||||||||
return (0); | return (0); | ||||||||||
} | } | ||||||||||
/********************************************************************* | /********************************************************************* | ||||||||||
* | * | ||||||||||
* Verify that the hardware indicated that the checksum is valid. | * Verify that the hardware indicated that the checksum is valid. | ||||||||||
* Inform the stack about the status of checksum so that stack | * Inform the stack about the status of checksum so that stack | ||||||||||
* doesn't spend time verifying the checksum. | * doesn't spend time verifying the checksum. | ||||||||||
* | * | ||||||||||
*********************************************************************/ | *********************************************************************/ | ||||||||||
static void | static void | ||||||||||
em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri) | em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri) | ||||||||||
{ | { | ||||||||||
if (__predict_false(status & E1000_RXD_STAT_IXSM)) | if (__predict_false(status & E1000_RXD_STAT_IXSM)) | ||||||||||
return; | return; | ||||||||||
/* If there is a layer 3 or 4 error we are done */ | /* If there is a layer 3 or 4 error we are done */ | ||||||||||
if (__predict_false(errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE))) | if (__predict_false(errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE))) | ||||||||||
return; | return; | ||||||||||
Done Inline ActionsI'd expect this to be rare too? markj: I'd expect this to be rare too? | |||||||||||
/* IP Checksum Good */ | /* IP Checksum Good */ | ||||||||||
if (status & E1000_RXD_STAT_IPCS) | if (status & E1000_RXD_STAT_IPCS) | ||||||||||
ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); | ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); | ||||||||||
/* Valid L4E checksum */ | /* Valid L4E checksum */ | ||||||||||
if (__predict_true(status & | if (__predict_true(status & | ||||||||||
(E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) { | (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) { | ||||||||||
ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; | ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; | ||||||||||
ri->iri_csum_data = htons(0xffff); | ri->iri_csum_data = htons(0xffff); | ||||||||||
} | } | ||||||||||
} | } | ||||||||||
/******************************************************************** | /******************************************************************** | ||||||||||
* | * | ||||||||||
* Parse the packet type to determine the appropriate hash | * Parse the packet type to determine the appropriate hash | ||||||||||
* | * | ||||||||||
******************************************************************/ | ******************************************************************/ | ||||||||||
static int | static int | ||||||||||
em_determine_rsstype(u32 pkt_info) | em_determine_rsstype(uint32_t pkt_info) | ||||||||||
{ | { | ||||||||||
switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { | switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { | ||||||||||
case E1000_RXDADV_RSSTYPE_IPV4_TCP: | case E1000_RXDADV_RSSTYPE_IPV4_TCP: | ||||||||||
return M_HASHTYPE_RSS_TCP_IPV4; | return M_HASHTYPE_RSS_TCP_IPV4; | ||||||||||
case E1000_RXDADV_RSSTYPE_IPV4: | case E1000_RXDADV_RSSTYPE_IPV4: | ||||||||||
return M_HASHTYPE_RSS_IPV4; | return M_HASHTYPE_RSS_IPV4; | ||||||||||
case E1000_RXDADV_RSSTYPE_IPV6_TCP: | case E1000_RXDADV_RSSTYPE_IPV6_TCP: | ||||||||||
return M_HASHTYPE_RSS_TCP_IPV6; | return M_HASHTYPE_RSS_TCP_IPV6; | ||||||||||
case E1000_RXDADV_RSSTYPE_IPV6_EX: | case E1000_RXDADV_RSSTYPE_IPV6_EX: | ||||||||||
return M_HASHTYPE_RSS_IPV6_EX; | return M_HASHTYPE_RSS_IPV6_EX; | ||||||||||
case E1000_RXDADV_RSSTYPE_IPV6: | case E1000_RXDADV_RSSTYPE_IPV6: | ||||||||||
return M_HASHTYPE_RSS_IPV6; | return M_HASHTYPE_RSS_IPV6; | ||||||||||
case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: | case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: | ||||||||||
return M_HASHTYPE_RSS_TCP_IPV6_EX; | return M_HASHTYPE_RSS_TCP_IPV6_EX; | ||||||||||
default: | default: | ||||||||||
return M_HASHTYPE_OPAQUE; | return M_HASHTYPE_OPAQUE; | ||||||||||
markjUnsubmitted Not Done Inline ActionsThe indentation was right before. markj: The indentation was right before. | |||||||||||
} | } | ||||||||||
} | } | ||||||||||
Not Done Inline Actionscase should be on the same indentation level as switch. markj: `case` should be on the same indentation level as `switch`. |