Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/ixl/ixl_txrx.c
Show First 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
#include "ixl.h" | #include "ixl.h" | ||||
#ifdef RSS | #ifdef RSS | ||||
#include <net/rss_config.h> | #include <net/rss_config.h> | ||||
#endif | #endif | ||||
/* Local Prototypes */ | /* Local Prototypes */ | ||||
static void ixl_rx_checksum(struct mbuf *, u32, u32, u8); | static void ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype); | ||||
static void ixl_refresh_mbufs(struct ixl_queue *, int); | |||||
static int ixl_xmit(struct ixl_queue *, struct mbuf **); | |||||
static int ixl_tx_setup_offload(struct ixl_queue *, | |||||
struct mbuf *, u32 *, u32 *); | |||||
static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *); | |||||
static __inline void ixl_rx_discard(struct rx_ring *, int); | static int ixl_isc_txd_encap(void *arg, if_pkt_info_t pi); | ||||
static __inline void ixl_rx_input(struct rx_ring *, struct ifnet *, | static void ixl_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx); | ||||
struct mbuf *, u8); | static int ixl_isc_txd_credits_update(void *arg, uint16_t qid, uint32_t cidx); | ||||
#ifdef DEV_NETMAP | static void ixl_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, | ||||
#include <dev/netmap/if_ixl_netmap.h> | uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count); | ||||
#endif /* DEV_NETMAP */ | static void ixl_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx); | ||||
static int ixl_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx); | |||||
static int ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | |||||
/* | extern int ixl_intr(void *arg); | ||||
** Multiqueue Transmit driver | |||||
*/ | |||||
int | |||||
ixl_mq_start(struct ifnet *ifp, struct mbuf *m) | |||||
{ | |||||
struct ixl_vsi *vsi = ifp->if_softc; | |||||
struct ixl_queue *que; | |||||
struct tx_ring *txr; | |||||
int err, i; | |||||
#ifdef RSS | |||||
u32 bucket_id; | |||||
#endif | |||||
/* | struct if_txrx ixl_txrx = { | ||||
** Which queue to use: | ixl_isc_txd_encap, | ||||
** | ixl_isc_txd_flush, | ||||
** When doing RSS, map it to the same outbound | ixl_isc_txd_credits_update, | ||||
** queue as the incoming flow would be mapped to. | ixl_isc_rxd_available, | ||||
** If everything is setup correctly, it should be | ixl_isc_rxd_pkt_get, | ||||
** the same bucket that the current CPU we're on is. | ixl_isc_rxd_refill, | ||||
*/ | ixl_isc_rxd_flush, | ||||
if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { | ixl_intr | ||||
#ifdef RSS | }; | ||||
if (rss_hash2bucket(m->m_pkthdr.flowid, | |||||
M_HASHTYPE_GET(m), &bucket_id) == 0) { | |||||
i = bucket_id % vsi->num_queues; | |||||
} else | |||||
#endif | |||||
i = m->m_pkthdr.flowid % vsi->num_queues; | |||||
} else | |||||
i = curcpu % vsi->num_queues; | |||||
/* | |||||
** This may not be perfect, but until something | |||||
** better comes along it will keep from scheduling | |||||
** on stalled queues. | |||||
*/ | |||||
if (((1 << i) & vsi->active_queues) == 0) | |||||
i = ffsl(vsi->active_queues); | |||||
que = &vsi->queues[i]; | extern if_shared_ctx_t ixl_sctx; | ||||
txr = &que->txr; | |||||
err = drbr_enqueue(ifp, txr->br, m); | |||||
if (err) | |||||
return (err); | |||||
if (IXL_TX_TRYLOCK(txr)) { | |||||
ixl_mq_start_locked(ifp, txr); | |||||
IXL_TX_UNLOCK(txr); | |||||
} else | |||||
taskqueue_enqueue(que->tq, &que->tx_task); | |||||
return (0); | #ifdef notyet | ||||
} | |||||
int | |||||
ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) | |||||
{ | |||||
struct ixl_queue *que = txr->que; | |||||
struct ixl_vsi *vsi = que->vsi; | |||||
struct mbuf *next; | |||||
int err = 0; | |||||
if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || | |||||
vsi->link_active == 0) | |||||
return (ENETDOWN); | |||||
/* Process the transmit queue */ | |||||
while ((next = drbr_peek(ifp, txr->br)) != NULL) { | |||||
if ((err = ixl_xmit(que, &next)) != 0) { | |||||
if (next == NULL) | |||||
drbr_advance(ifp, txr->br); | |||||
else | |||||
drbr_putback(ifp, txr->br, next); | |||||
break; | |||||
} | |||||
drbr_advance(ifp, txr->br); | |||||
/* Send a copy of the frame to the BPF listener */ | |||||
ETHER_BPF_MTAP(ifp, next); | |||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) | |||||
break; | |||||
} | |||||
if (txr->avail < IXL_TX_CLEANUP_THRESHOLD) | |||||
ixl_txeof(que); | |||||
return (err); | |||||
} | |||||
/* | /* | ||||
* Called from a taskqueue to drain queued transmit packets. | |||||
*/ | |||||
void | |||||
ixl_deferred_mq_start(void *arg, int pending) | |||||
{ | |||||
struct ixl_queue *que = arg; | |||||
struct tx_ring *txr = &que->txr; | |||||
struct ixl_vsi *vsi = que->vsi; | |||||
struct ifnet *ifp = vsi->ifp; | |||||
IXL_TX_LOCK(txr); | |||||
if (!drbr_empty(ifp, txr->br)) | |||||
ixl_mq_start_locked(ifp, txr); | |||||
IXL_TX_UNLOCK(txr); | |||||
} | |||||
/* | |||||
** Flush all queue ring buffers | |||||
*/ | |||||
void | |||||
ixl_qflush(struct ifnet *ifp) | |||||
{ | |||||
struct ixl_vsi *vsi = ifp->if_softc; | |||||
for (int i = 0; i < vsi->num_queues; i++) { | |||||
struct ixl_queue *que = &vsi->queues[i]; | |||||
struct tx_ring *txr = &que->txr; | |||||
struct mbuf *m; | |||||
IXL_TX_LOCK(txr); | |||||
while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) | |||||
m_freem(m); | |||||
IXL_TX_UNLOCK(txr); | |||||
} | |||||
if_qflush(ifp); | |||||
} | |||||
/* | |||||
** Find mbuf chains passed to the driver | ** Find mbuf chains passed to the driver | ||||
** that are 'sparse', using more than 8 | ** that are 'sparse', using more than 8 | ||||
** mbufs to deliver an mss-size chunk of data | ** mbufs to deliver an mss-size chunk of data | ||||
*/ | */ | ||||
static inline bool | static inline bool | ||||
ixl_tso_detect_sparse(struct mbuf *mp) | ixl_tso_detect_sparse(struct mbuf *mp) | ||||
{ | { | ||||
struct mbuf *m; | struct mbuf *m; | ||||
Show All 9 Lines | for (m = mp->m_next; m != NULL; m = m->m_next) { | ||||
if (m->m_next == NULL) | if (m->m_next == NULL) | ||||
break; | break; | ||||
} | } | ||||
if (num > IXL_SPARSE_CHAIN) | if (num > IXL_SPARSE_CHAIN) | ||||
ret = TRUE; | ret = TRUE; | ||||
return (ret); | return (ret); | ||||
} | } | ||||
#endif | |||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Setup descriptor for hw offloads | |||||
* | |||||
**********************************************************************/ | |||||
static void | |||||
ixl_tx_setup_offload(struct ixl_queue *que, | |||||
if_pkt_info_t pi, u32 *cmd, u32 *off) | |||||
{ | |||||
switch (pi->ipi_etype) { | |||||
#ifdef INET | |||||
case ETHERTYPE_IP: | |||||
/* The IP checksum must be recalculated with TSO */ | |||||
if (pi->ipi_csum_flags & CSUM_TSO) | |||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; | |||||
else | |||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; | |||||
break; | |||||
#endif | |||||
#ifdef INET6 | |||||
case ETHERTYPE_IPV6: | |||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; | |||||
break; | |||||
#endif | |||||
default: | |||||
break; | |||||
} | |||||
*off |= (pi->ipi_ehdrlen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; | |||||
*off |= (pi->ipi_ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; | |||||
switch (pi->ipi_ipproto) { | |||||
case IPPROTO_TCP: | |||||
if (pi->ipi_csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) { | |||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; | |||||
*off |= (pi->ipi_tcp_hlen >> 2) << | |||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | |||||
} | |||||
#ifdef IXL_FDIR | |||||
ixl_atr(que, pi->ipi_tcp_hflags, pi->ipi_etype); | |||||
#endif | |||||
break; | |||||
case IPPROTO_UDP: | |||||
if (pi->ipi_csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) { | |||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; | |||||
*off |= (sizeof(struct udphdr) >> 2) << | |||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | |||||
} | |||||
break; | |||||
case IPPROTO_SCTP: | |||||
if (pi->ipi_csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) { | |||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; | |||||
*off |= (sizeof(struct sctphdr) >> 2) << | |||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | |||||
} | |||||
/* Fall Thru */ | |||||
default: | |||||
break; | |||||
} | |||||
} | |||||
/********************************************************************** | |||||
* | |||||
* Setup context for hardware segmentation offload (TSO) | |||||
* | |||||
**********************************************************************/ | |||||
static int | |||||
ixl_tso_setup(struct ixl_queue *que, if_pkt_info_t pi) | |||||
{ | |||||
struct tx_ring *txr = &que->txr; | |||||
struct i40e_tx_context_desc *TXD; | |||||
struct ixl_tx_buf *buf; | |||||
u32 cmd, mss, type, tsolen; | |||||
int idx; | |||||
u64 type_cmd_tso_mss; | |||||
idx = pi->ipi_pidx; | |||||
buf = &txr->tx_buffers[idx]; | |||||
TXD = (struct i40e_tx_context_desc *) &txr->tx_base[idx]; | |||||
tsolen = pi->ipi_len - (pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen); | |||||
type = I40E_TX_DESC_DTYPE_CONTEXT; | |||||
cmd = I40E_TX_CTX_DESC_TSO; | |||||
mss = pi->ipi_tso_segsz; | |||||
type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | | |||||
((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | | |||||
((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | | |||||
((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); | |||||
TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); | |||||
TXD->tunneling_params = htole32(0); | |||||
buf->eop_index = -1; | |||||
if (++idx == ixl_sctx->isc_ntxd) | |||||
idx = 0; | |||||
return (idx); | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* This routine maps the mbufs to tx descriptors, allowing the | * This routine maps the mbufs to tx descriptors, allowing the | ||||
* TX engine to transmit the packets. | * TX engine to transmit the packets. | ||||
* - return 0 on success, positive on failure | * - return 0 on success, positive on failure | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
#define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) | #define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) | ||||
static int | static int | ||||
ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) | ixl_isc_txd_encap(void *arg, if_pkt_info_t pi) | ||||
{ | { | ||||
struct ixl_vsi *vsi = que->vsi; | struct ixl_vsi *vsi = arg; | ||||
struct i40e_hw *hw = vsi->hw; | struct ixl_queue *que = &vsi->queues[pi->ipi_qsidx]; | ||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
int nsegs = pi->ipi_nsegs; | |||||
bus_dma_segment_t *segs = pi->ipi_segs; | |||||
struct ixl_tx_buf *buf; | struct ixl_tx_buf *buf; | ||||
struct i40e_tx_desc *txd = NULL; | struct i40e_tx_desc *txd = NULL; | ||||
struct mbuf *m_head, *m; | int i, j; | ||||
int i, j, error, nsegs, maxsegs; | |||||
int first, last = 0; | int first, last = 0; | ||||
u16 vtag = 0; | |||||
u32 cmd, off; | u32 cmd, off; | ||||
bus_dmamap_t map; | |||||
bus_dma_tag_t tag; | |||||
bus_dma_segment_t segs[IXL_MAX_TSO_SEGS]; | |||||
cmd = off = 0; | cmd = off = 0; | ||||
m_head = *m_headp; | |||||
/* | /* | ||||
* Important to capture the first descriptor | * Important to capture the first descriptor | ||||
* used because it will contain the index of | * used because it will contain the index of | ||||
* the one we tell the hardware to report back | * the one we tell the hardware to report back | ||||
*/ | */ | ||||
first = txr->next_avail; | first = pi->ipi_pidx; | ||||
buf = &txr->buffers[first]; | buf = &txr->tx_buffers[first]; | ||||
map = buf->map; | |||||
tag = txr->tx_tag; | |||||
maxsegs = IXL_MAX_TX_SEGS; | |||||
if (pi->ipi_flags & IPI_TX_INTR) | |||||
cmd |= (I40E_TX_DESC_CMD_RS << I40E_TXD_QW1_CMD_SHIFT); | |||||
#ifdef notyet | |||||
/* add this check to iflib - shouldn't actually happen in practice */ | |||||
if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | ||||
/* Use larger mapping for TSO */ | /* Use larger mapping for TSO */ | ||||
tag = txr->tso_tag; | tag = txr->tso_tag; | ||||
maxsegs = IXL_MAX_TSO_SEGS; | maxsegs = IXL_MAX_TSO_SEGS; | ||||
if (ixl_tso_detect_sparse(m_head)) { | if (ixl_tso_detect_sparse(m_head)) { | ||||
m = m_defrag(m_head, M_NOWAIT); | m = m_defrag(m_head, M_NOWAIT); | ||||
if (m == NULL) { | if (m == NULL) { | ||||
m_freem(*m_headp); | m_freem(*m_headp); | ||||
*m_headp = NULL; | *m_headp = NULL; | ||||
return (ENOBUFS); | return (ENOBUFS); | ||||
} | } | ||||
*m_headp = m; | *m_headp = m; | ||||
} | } | ||||
} | } | ||||
#endif | |||||
/* | |||||
* Map the packet for DMA. | |||||
*/ | |||||
error = bus_dmamap_load_mbuf_sg(tag, map, | |||||
*m_headp, segs, &nsegs, BUS_DMA_NOWAIT); | |||||
if (error == EFBIG) { | |||||
struct mbuf *m; | |||||
m = m_collapse(*m_headp, M_NOWAIT, maxsegs); | |||||
if (m == NULL) { | |||||
que->mbuf_defrag_failed++; | |||||
m_freem(*m_headp); | |||||
*m_headp = NULL; | |||||
return (ENOBUFS); | |||||
} | |||||
*m_headp = m; | |||||
/* Try it again */ | |||||
error = bus_dmamap_load_mbuf_sg(tag, map, | |||||
*m_headp, segs, &nsegs, BUS_DMA_NOWAIT); | |||||
if (error == ENOMEM) { | |||||
que->tx_dma_setup++; | |||||
return (error); | |||||
} else if (error != 0) { | |||||
que->tx_dma_setup++; | |||||
m_freem(*m_headp); | |||||
*m_headp = NULL; | |||||
return (error); | |||||
} | |||||
} else if (error == ENOMEM) { | |||||
que->tx_dma_setup++; | |||||
return (error); | |||||
} else if (error != 0) { | |||||
que->tx_dma_setup++; | |||||
m_freem(*m_headp); | |||||
*m_headp = NULL; | |||||
return (error); | |||||
} | |||||
/* Make certain there are enough descriptors */ | |||||
if (nsegs > txr->avail - 2) { | |||||
txr->no_desc++; | |||||
error = ENOBUFS; | |||||
goto xmit_fail; | |||||
} | |||||
m_head = *m_headp; | |||||
/* Set up the TSO/CSUM offload */ | /* Set up the TSO/CSUM offload */ | ||||
if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { | if (pi->ipi_csum_flags & CSUM_OFFLOAD) { | ||||
error = ixl_tx_setup_offload(que, m_head, &cmd, &off); | /* Set up the TSO context descriptor if required */ | ||||
if (error) | if (pi->ipi_csum_flags & CSUM_TSO) | ||||
goto xmit_fail; | first = ixl_tso_setup(que, pi); | ||||
ixl_tx_setup_offload(que, pi, &cmd, &off); | |||||
} | } | ||||
cmd |= I40E_TX_DESC_CMD_ICRC; | if (pi->ipi_mflags & M_VLANTAG) | ||||
/* Grab the VLAN tag */ | |||||
if (m_head->m_flags & M_VLANTAG) { | |||||
cmd |= I40E_TX_DESC_CMD_IL2TAG1; | cmd |= I40E_TX_DESC_CMD_IL2TAG1; | ||||
vtag = htole16(m_head->m_pkthdr.ether_vtag); | |||||
} | |||||
i = txr->next_avail; | cmd |= I40E_TX_DESC_CMD_ICRC; | ||||
i = first; | |||||
for (j = 0; j < nsegs; j++) { | for (j = 0; j < nsegs; j++) { | ||||
bus_size_t seglen; | bus_size_t seglen; | ||||
buf = &txr->buffers[i]; | buf = &txr->tx_buffers[i]; | ||||
buf->tag = tag; /* Keep track of the type tag */ | txd = &txr->tx_base[i]; | ||||
txd = &txr->base[i]; | |||||
seglen = segs[j].ds_len; | seglen = segs[j].ds_len; | ||||
txd->buffer_addr = htole64(segs[j].ds_addr); | txd->buffer_addr = htole64(segs[j].ds_addr); | ||||
txd->cmd_type_offset_bsz = | txd->cmd_type_offset_bsz = | ||||
htole64(I40E_TX_DESC_DTYPE_DATA | htole64(I40E_TX_DESC_DTYPE_DATA | ||||
| ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) | | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) | ||||
| ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) | | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) | ||||
| ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | ||||
| ((u64)vtag << I40E_TXD_QW1_L2TAG1_SHIFT)); | | ((u64)htole16(pi->ipi_vtag) << I40E_TXD_QW1_L2TAG1_SHIFT)); | ||||
last = i; /* descriptor that will get completion IRQ */ | last = i; /* descriptor that will get completion IRQ */ | ||||
if (++i == que->num_desc) | if (++i == ixl_sctx->isc_ntxd) | ||||
i = 0; | i = 0; | ||||
buf->m_head = NULL; | |||||
buf->eop_index = -1; | buf->eop_index = -1; | ||||
} | } | ||||
/* Set the last descriptor for report */ | /* Set the last descriptor for report */ | ||||
txd->cmd_type_offset_bsz |= | txd->cmd_type_offset_bsz |= | ||||
htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); | htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); | ||||
txr->avail -= nsegs; | pi->ipi_new_pidx = i; | ||||
txr->next_avail = i; | |||||
buf->m_head = m_head; | |||||
/* Swap the dma map between the first and last descriptor */ | |||||
txr->buffers[first].map = buf->map; | |||||
buf->map = map; | |||||
bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE); | |||||
/* Set the index of the descriptor that will be marked done */ | /* Set the index of the descriptor that will be marked done */ | ||||
buf = &txr->buffers[first]; | buf = &txr->tx_buffers[first]; | ||||
buf->eop_index = last; | buf->eop_index = last; | ||||
bus_dmamap_sync(txr->dma.tag, txr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
/* | |||||
* Advance the Transmit Descriptor Tail (Tdt), this tells the | |||||
* hardware that this frame is available to transmit. | |||||
*/ | |||||
++txr->total_packets; | ++txr->total_packets; | ||||
wr32(hw, txr->tail, i); | |||||
ixl_flush(hw); | |||||
/* Mark outstanding work */ | |||||
if (que->busy == 0) | |||||
que->busy = 1; | |||||
return (0); | return (0); | ||||
xmit_fail: | |||||
bus_dmamap_unload(tag, buf->map); | |||||
return (error); | |||||
} | } | ||||
static void | |||||
/********************************************************************* | ixl_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx) | ||||
* | |||||
* Allocate memory for tx_buffer structures. The tx_buffer stores all | |||||
* the information needed to transmit a packet on the wire. This is | |||||
* called only once at attach, setup is done every reset. | |||||
* | |||||
**********************************************************************/ | |||||
int | |||||
ixl_allocate_tx_data(struct ixl_queue *que) | |||||
{ | { | ||||
struct tx_ring *txr = &que->txr; | struct ixl_vsi *vsi = arg; | ||||
struct ixl_vsi *vsi = que->vsi; | struct tx_ring *txr = &vsi->queues[txqid].txr; | ||||
device_t dev = vsi->dev; | |||||
struct ixl_tx_buf *buf; | |||||
int error = 0; | |||||
/* | /* | ||||
* Setup DMA descriptor areas. | * Advance the Transmit Descriptor Tail (Tdt), this tells the | ||||
* hardware that this frame is available to transmit. | |||||
*/ | */ | ||||
if ((error = bus_dma_tag_create(NULL, /* parent */ | wr32(vsi->hw, txr->tail, pidx); | ||||
1, 0, /* alignment, bounds */ | |||||
BUS_SPACE_MAXADDR, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filter, filterarg */ | |||||
IXL_TSO_SIZE, /* maxsize */ | |||||
IXL_MAX_TX_SEGS, /* nsegments */ | |||||
PAGE_SIZE, /* maxsegsize */ | |||||
0, /* flags */ | |||||
NULL, /* lockfunc */ | |||||
NULL, /* lockfuncarg */ | |||||
&txr->tx_tag))) { | |||||
device_printf(dev,"Unable to allocate TX DMA tag\n"); | |||||
goto fail; | |||||
} | } | ||||
/* Make a special tag for TSO */ | |||||
if ((error = bus_dma_tag_create(NULL, /* parent */ | |||||
1, 0, /* alignment, bounds */ | |||||
BUS_SPACE_MAXADDR, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filter, filterarg */ | |||||
IXL_TSO_SIZE, /* maxsize */ | |||||
IXL_MAX_TSO_SEGS, /* nsegments */ | |||||
PAGE_SIZE, /* maxsegsize */ | |||||
0, /* flags */ | |||||
NULL, /* lockfunc */ | |||||
NULL, /* lockfuncarg */ | |||||
&txr->tso_tag))) { | |||||
device_printf(dev,"Unable to allocate TX TSO DMA tag\n"); | |||||
goto fail; | |||||
} | |||||
if (!(txr->buffers = | |||||
(struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) * | |||||
que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { | |||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | |||||
error = ENOMEM; | |||||
goto fail; | |||||
} | |||||
/* Create the descriptor buffer default dma maps */ | |||||
buf = txr->buffers; | |||||
for (int i = 0; i < que->num_desc; i++, buf++) { | |||||
buf->tag = txr->tx_tag; | |||||
error = bus_dmamap_create(buf->tag, 0, &buf->map); | |||||
if (error != 0) { | |||||
device_printf(dev, "Unable to create TX DMA map\n"); | |||||
goto fail; | |||||
} | |||||
} | |||||
fail: | |||||
return (error); | |||||
} | |||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* (Re)Initialize a queue transmit ring. | * (Re)Initialize a queue transmit ring. | ||||
* - called by init, it clears the descriptor ring, | * - called by init, it clears the descriptor ring, | ||||
* and frees any stale mbufs | * and frees any stale mbufs | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
void | void | ||||
ixl_init_tx_ring(struct ixl_queue *que) | ixl_init_tx_ring(struct ixl_queue *que) | ||||
{ | { | ||||
#ifdef DEV_NETMAP | |||||
struct netmap_adapter *na = NA(que->vsi->ifp); | |||||
struct netmap_slot *slot; | |||||
#endif /* DEV_NETMAP */ | |||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
struct ixl_tx_buf *buf; | struct ixl_tx_buf *buf; | ||||
/* Clear the old ring contents */ | /* Clear the old ring contents */ | ||||
IXL_TX_LOCK(txr); | bzero((void *)txr->tx_base, | ||||
(sizeof(struct i40e_tx_desc)) * ixl_sctx->isc_ntxd); | |||||
#ifdef DEV_NETMAP | |||||
/* | |||||
* (under lock): if in netmap mode, do some consistency | |||||
* checks and set slot to entry 0 of the netmap ring. | |||||
*/ | |||||
slot = netmap_reset(na, NR_TX, que->me, 0); | |||||
#endif /* DEV_NETMAP */ | |||||
bzero((void *)txr->base, | |||||
(sizeof(struct i40e_tx_desc)) * que->num_desc); | |||||
/* Reset indices */ | |||||
txr->next_avail = 0; | |||||
txr->next_to_clean = 0; | |||||
#ifdef IXL_FDIR | #ifdef IXL_FDIR | ||||
/* Initialize flow director */ | /* Initialize flow director */ | ||||
txr->atr_rate = ixl_atr_rate; | txr->atr_rate = ixl_atr_rate; | ||||
txr->atr_count = 0; | txr->atr_count = 0; | ||||
#endif | #endif | ||||
/* Free any existing tx mbufs. */ | buf = txr->tx_buffers; | ||||
buf = txr->buffers; | for (int i = 0; i < ixl_sctx->isc_ntxd; i++, buf++) { | ||||
for (int i = 0; i < que->num_desc; i++, buf++) { | |||||
if (buf->m_head != NULL) { | |||||
bus_dmamap_sync(buf->tag, buf->map, | |||||
BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_unload(buf->tag, buf->map); | |||||
m_freem(buf->m_head); | |||||
buf->m_head = NULL; | |||||
} | |||||
#ifdef DEV_NETMAP | |||||
/* | |||||
* In netmap mode, set the map for the packet buffer. | |||||
* NOTE: Some drivers (not this one) also need to set | |||||
* the physical buffer address in the NIC ring. | |||||
* netmap_idx_n2k() maps a nic index, i, into the corresponding | |||||
* netmap slot index, si | |||||
*/ | |||||
if (slot) { | |||||
int si = netmap_idx_n2k(&na->tx_rings[que->me], i); | |||||
netmap_load_map(na, buf->tag, buf->map, NMB(na, slot + si)); | |||||
} | |||||
#endif /* DEV_NETMAP */ | |||||
/* Clear the EOP index */ | /* Clear the EOP index */ | ||||
buf->eop_index = -1; | buf->eop_index = -1; | ||||
} | } | ||||
/* Set number of descriptors available */ | |||||
txr->avail = que->num_desc; | |||||
bus_dmamap_sync(txr->dma.tag, txr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
IXL_TX_UNLOCK(txr); | |||||
} | } | ||||
/********************************************************************* | |||||
* | |||||
* Free transmit ring related data structures. | |||||
* | |||||
**********************************************************************/ | |||||
void | |||||
ixl_free_que_tx(struct ixl_queue *que) | |||||
{ | |||||
struct tx_ring *txr = &que->txr; | |||||
struct ixl_tx_buf *buf; | |||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); | |||||
for (int i = 0; i < que->num_desc; i++) { | |||||
buf = &txr->buffers[i]; | |||||
if (buf->m_head != NULL) { | |||||
bus_dmamap_sync(buf->tag, buf->map, | |||||
BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_unload(buf->tag, | |||||
buf->map); | |||||
m_freem(buf->m_head); | |||||
buf->m_head = NULL; | |||||
if (buf->map != NULL) { | |||||
bus_dmamap_destroy(buf->tag, | |||||
buf->map); | |||||
buf->map = NULL; | |||||
} | |||||
} else if (buf->map != NULL) { | |||||
bus_dmamap_unload(buf->tag, | |||||
buf->map); | |||||
bus_dmamap_destroy(buf->tag, | |||||
buf->map); | |||||
buf->map = NULL; | |||||
} | |||||
} | |||||
if (txr->br != NULL) | |||||
buf_ring_free(txr->br, M_DEVBUF); | |||||
if (txr->buffers != NULL) { | |||||
free(txr->buffers, M_DEVBUF); | |||||
txr->buffers = NULL; | |||||
} | |||||
if (txr->tx_tag != NULL) { | |||||
bus_dma_tag_destroy(txr->tx_tag); | |||||
txr->tx_tag = NULL; | |||||
} | |||||
if (txr->tso_tag != NULL) { | |||||
bus_dma_tag_destroy(txr->tso_tag); | |||||
txr->tso_tag = NULL; | |||||
} | |||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); | |||||
return; | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* Setup descriptor for hw offloads | |||||
* | |||||
**********************************************************************/ | |||||
static int | |||||
ixl_tx_setup_offload(struct ixl_queue *que, | |||||
struct mbuf *mp, u32 *cmd, u32 *off) | |||||
{ | |||||
struct ether_vlan_header *eh; | |||||
#ifdef INET | |||||
struct ip *ip = NULL; | |||||
#endif | |||||
struct tcphdr *th = NULL; | |||||
#ifdef INET6 | |||||
struct ip6_hdr *ip6; | |||||
#endif | |||||
int elen, ip_hlen = 0, tcp_hlen; | |||||
u16 etype; | |||||
u8 ipproto = 0; | |||||
bool tso = FALSE; | |||||
/* Set up the TSO context descriptor if required */ | |||||
if (mp->m_pkthdr.csum_flags & CSUM_TSO) { | |||||
tso = ixl_tso_setup(que, mp); | |||||
if (tso) | |||||
++que->tso; | |||||
else | |||||
return (ENXIO); | |||||
} | |||||
/* | /* | ||||
* Determine where frame payload starts. | |||||
* Jump over vlan headers if already present, | |||||
* helpful for QinQ too. | |||||
*/ | |||||
eh = mtod(mp, struct ether_vlan_header *); | |||||
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | |||||
etype = ntohs(eh->evl_proto); | |||||
elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | |||||
} else { | |||||
etype = ntohs(eh->evl_encap_proto); | |||||
elen = ETHER_HDR_LEN; | |||||
} | |||||
switch (etype) { | |||||
#ifdef INET | |||||
case ETHERTYPE_IP: | |||||
ip = (struct ip *)(mp->m_data + elen); | |||||
ip_hlen = ip->ip_hl << 2; | |||||
ipproto = ip->ip_p; | |||||
th = (struct tcphdr *)((caddr_t)ip + ip_hlen); | |||||
/* The IP checksum must be recalculated with TSO */ | |||||
if (tso) | |||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; | |||||
else | |||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; | |||||
break; | |||||
#endif | |||||
#ifdef INET6 | |||||
case ETHERTYPE_IPV6: | |||||
ip6 = (struct ip6_hdr *)(mp->m_data + elen); | |||||
ip_hlen = sizeof(struct ip6_hdr); | |||||
ipproto = ip6->ip6_nxt; | |||||
th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); | |||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; | |||||
break; | |||||
#endif | |||||
default: | |||||
break; | |||||
} | |||||
*off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; | |||||
*off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; | |||||
switch (ipproto) { | |||||
case IPPROTO_TCP: | |||||
tcp_hlen = th->th_off << 2; | |||||
if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) { | |||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; | |||||
*off |= (tcp_hlen >> 2) << | |||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | |||||
} | |||||
#ifdef IXL_FDIR | |||||
ixl_atr(que, th, etype); | |||||
#endif | |||||
break; | |||||
case IPPROTO_UDP: | |||||
if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) { | |||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; | |||||
*off |= (sizeof(struct udphdr) >> 2) << | |||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | |||||
} | |||||
break; | |||||
case IPPROTO_SCTP: | |||||
if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) { | |||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; | |||||
*off |= (sizeof(struct sctphdr) >> 2) << | |||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | |||||
} | |||||
/* Fall Thru */ | |||||
default: | |||||
break; | |||||
} | |||||
return (0); | |||||
} | |||||
/********************************************************************** | |||||
* | |||||
* Setup context for hardware segmentation offload (TSO) | |||||
* | |||||
**********************************************************************/ | |||||
static bool | |||||
ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp) | |||||
{ | |||||
struct tx_ring *txr = &que->txr; | |||||
struct i40e_tx_context_desc *TXD; | |||||
struct ixl_tx_buf *buf; | |||||
u32 cmd, mss, type, tsolen; | |||||
u16 etype; | |||||
int idx, elen, ip_hlen, tcp_hlen; | |||||
struct ether_vlan_header *eh; | |||||
#ifdef INET | |||||
struct ip *ip; | |||||
#endif | |||||
#ifdef INET6 | |||||
struct ip6_hdr *ip6; | |||||
#endif | |||||
#if defined(INET6) || defined(INET) | |||||
struct tcphdr *th; | |||||
#endif | |||||
u64 type_cmd_tso_mss; | |||||
/* | |||||
* Determine where frame payload starts. | |||||
* Jump over vlan headers if already present | |||||
*/ | |||||
eh = mtod(mp, struct ether_vlan_header *); | |||||
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | |||||
elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | |||||
etype = eh->evl_proto; | |||||
} else { | |||||
elen = ETHER_HDR_LEN; | |||||
etype = eh->evl_encap_proto; | |||||
} | |||||
switch (ntohs(etype)) { | |||||
#ifdef INET6 | |||||
case ETHERTYPE_IPV6: | |||||
ip6 = (struct ip6_hdr *)(mp->m_data + elen); | |||||
if (ip6->ip6_nxt != IPPROTO_TCP) | |||||
return (ENXIO); | |||||
ip_hlen = sizeof(struct ip6_hdr); | |||||
th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); | |||||
th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); | |||||
tcp_hlen = th->th_off << 2; | |||||
break; | |||||
#endif | |||||
#ifdef INET | |||||
case ETHERTYPE_IP: | |||||
ip = (struct ip *)(mp->m_data + elen); | |||||
if (ip->ip_p != IPPROTO_TCP) | |||||
return (ENXIO); | |||||
ip->ip_sum = 0; | |||||
ip_hlen = ip->ip_hl << 2; | |||||
th = (struct tcphdr *)((caddr_t)ip + ip_hlen); | |||||
th->th_sum = in_pseudo(ip->ip_src.s_addr, | |||||
ip->ip_dst.s_addr, htons(IPPROTO_TCP)); | |||||
tcp_hlen = th->th_off << 2; | |||||
break; | |||||
#endif | |||||
default: | |||||
printf("%s: CSUM_TSO but no supported IP version (0x%04x)", | |||||
__func__, ntohs(etype)); | |||||
return FALSE; | |||||
} | |||||
/* Ensure we have at least the IP+TCP header in the first mbuf. */ | |||||
if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr)) | |||||
return FALSE; | |||||
idx = txr->next_avail; | |||||
buf = &txr->buffers[idx]; | |||||
TXD = (struct i40e_tx_context_desc *) &txr->base[idx]; | |||||
tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen); | |||||
type = I40E_TX_DESC_DTYPE_CONTEXT; | |||||
cmd = I40E_TX_CTX_DESC_TSO; | |||||
mss = mp->m_pkthdr.tso_segsz; | |||||
type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | | |||||
((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | | |||||
((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | | |||||
((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); | |||||
TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); | |||||
TXD->tunneling_params = htole32(0); | |||||
buf->m_head = NULL; | |||||
buf->eop_index = -1; | |||||
if (++idx == que->num_desc) | |||||
idx = 0; | |||||
txr->avail--; | |||||
txr->next_avail = idx; | |||||
return TRUE; | |||||
} | |||||
/* | |||||
** ixl_get_tx_head - Retrieve the value from the | ** ixl_get_tx_head - Retrieve the value from the | ||||
** location the HW records its HEAD index | ** location the HW records its HEAD index | ||||
*/ | */ | ||||
static inline u32 | static inline u32 | ||||
ixl_get_tx_head(struct ixl_queue *que) | ixl_get_tx_head(struct ixl_queue *que) | ||||
{ | { | ||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
void *head = &txr->base[que->num_desc]; | void *head = &txr->tx_base[ixl_sctx->isc_ntxd]; | ||||
return LE32_TO_CPU(*(volatile __le32 *)head); | return LE32_TO_CPU(*(volatile __le32 *)head); | ||||
} | } | ||||
/********************************************************************** | /********************************************************************** | ||||
* | * | ||||
* Examine each tx_buffer in the used queue. If the hardware is done | * Examine each tx_buffer in the used queue. If the hardware is done | ||||
* processing the packet then free associated resources. The | * processing the packet then free associated resources. The | ||||
* tx_buffer is put back on the free queue. | * tx_buffer is put back on the free queue. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
bool | static int | ||||
ixl_txeof(struct ixl_queue *que) | ixl_isc_txd_credits_update(void *arg, uint16_t qid, uint32_t cidx) | ||||
{ | { | ||||
struct ixl_vsi *vsi = arg; | |||||
struct ixl_queue *que = &vsi->queues[qid]; | |||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
u32 first, last, head, done, processed; | u32 first, last, head, done, processed; | ||||
struct ixl_tx_buf *buf; | struct ixl_tx_buf *buf; | ||||
struct i40e_tx_desc *tx_desc, *eop_desc; | struct i40e_tx_desc *tx_desc, *eop_desc; | ||||
mtx_assert(&txr->mtx, MA_OWNED); | |||||
#ifdef DEV_NETMAP | |||||
// XXX todo: implement moderation | |||||
if (netmap_tx_irq(que->vsi->ifp, que->me)) | |||||
return FALSE; | |||||
#endif /* DEF_NETMAP */ | |||||
/* These are not the descriptors you seek, move along :) */ | |||||
if (txr->avail == que->num_desc) { | |||||
que->busy = 0; | |||||
return FALSE; | |||||
} | |||||
processed = 0; | processed = 0; | ||||
first = txr->next_to_clean; | first = cidx; | ||||
buf = &txr->buffers[first]; | buf = &txr->tx_buffers[first]; | ||||
tx_desc = (struct i40e_tx_desc *)&txr->base[first]; | tx_desc = (struct i40e_tx_desc *)&txr->tx_base[first]; | ||||
last = buf->eop_index; | last = buf->eop_index; | ||||
if (last == -1) | if (last == -1) | ||||
return FALSE; | return (0); | ||||
eop_desc = (struct i40e_tx_desc *)&txr->base[last]; | eop_desc = (struct i40e_tx_desc *)&txr->tx_base[last]; | ||||
/* Get the Head WB value */ | /* Get the Head WB value */ | ||||
head = ixl_get_tx_head(que); | head = ixl_get_tx_head(que); | ||||
/* | /* | ||||
** Get the index of the first descriptor | ** Get the index of the first descriptor | ||||
** BEYOND the EOP and call that 'done'. | ** BEYOND the EOP and call that 'done'. | ||||
** I do this so the comparison in the | ** I do this so the comparison in the | ||||
** inner while loop below can be simple | ** inner while loop below can be simple | ||||
*/ | */ | ||||
if (++last == que->num_desc) last = 0; | if (++last == ixl_sctx->isc_ntxd) last = 0; | ||||
done = last; | done = last; | ||||
bus_dmamap_sync(txr->dma.tag, txr->dma.map, | |||||
BUS_DMASYNC_POSTREAD); | |||||
/* | /* | ||||
** The HEAD index of the ring is written in a | ** The HEAD index of the ring is written in a | ||||
** defined location, this rather than a done bit | ** defined location, this rather than a done bit | ||||
** is what is used to keep track of what must be | ** is what is used to keep track of what must be | ||||
** 'cleaned'. | ** 'cleaned'. | ||||
*/ | */ | ||||
while (first != head) { | while (first != head) { | ||||
/* We clean the range of the packet */ | |||||
while (first != done) { | while (first != done) { | ||||
++txr->avail; | |||||
++processed; | ++processed; | ||||
if (buf->m_head) { | |||||
txr->bytes += /* for ITR adjustment */ | |||||
buf->m_head->m_pkthdr.len; | |||||
txr->tx_bytes += /* for TX stats */ | |||||
buf->m_head->m_pkthdr.len; | |||||
bus_dmamap_sync(buf->tag, | |||||
buf->map, | |||||
BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_unload(buf->tag, | |||||
buf->map); | |||||
m_freem(buf->m_head); | |||||
buf->m_head = NULL; | |||||
buf->map = NULL; | |||||
} | |||||
buf->eop_index = -1; | buf->eop_index = -1; | ||||
if (++first == ixl_sctx->isc_ntxd) | |||||
if (++first == que->num_desc) | |||||
first = 0; | first = 0; | ||||
buf = &txr->buffers[first]; | buf = &txr->tx_buffers[first]; | ||||
tx_desc = &txr->base[first]; | tx_desc = &txr->tx_base[first]; | ||||
} | } | ||||
++txr->packets; | ++txr->packets; | ||||
/* See if there is more work now */ | /* See if there is more work now */ | ||||
last = buf->eop_index; | last = buf->eop_index; | ||||
if (last != -1) { | if (last == -1) | ||||
eop_desc = &txr->base[last]; | break; | ||||
eop_desc = &txr->tx_base[last]; | |||||
/* Get next done point */ | /* Get next done point */ | ||||
if (++last == que->num_desc) last = 0; | if (++last == ixl_sctx->isc_ntxd) last = 0; | ||||
done = last; | done = last; | ||||
} else | |||||
break; | |||||
} | |||||
bus_dmamap_sync(txr->dma.tag, txr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
txr->next_to_clean = first; | |||||
/* | |||||
** Hang detection, we know there's | |||||
** work outstanding or the first return | |||||
** would have been taken, so indicate an | |||||
** unsuccessful pass, in local_timer if | |||||
** the value is too great the queue will | |||||
** be considered hung. If anything has been | |||||
** cleaned then reset the state. | |||||
*/ | |||||
if ((processed == 0) && (que->busy != IXL_QUEUE_HUNG)) | |||||
++que->busy; | |||||
if (processed) | |||||
que->busy = 1; /* Note this turns off HUNG */ | |||||
/* | |||||
* If there are no pending descriptors, clear the timeout. | |||||
*/ | |||||
if (txr->avail == que->num_desc) { | |||||
que->busy = 0; | |||||
return FALSE; | |||||
} | } | ||||
return (processed); | |||||
return TRUE; | |||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Refresh mbuf buffers for RX descriptor rings | * Refresh mbuf buffers for RX descriptor rings | ||||
* - now keeps its own state so discards due to resource | * - now keeps its own state so discards due to resource | ||||
* exhaustion are unnecessary, if an mbuf cannot be obtained | * exhaustion are unnecessary, if an mbuf cannot be obtained | ||||
* it just returns, keeping its placeholder, thus it can simply | * it just returns, keeping its placeholder, thus it can simply | ||||
* be recalled to try again. | * be recalled to try again. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
ixl_refresh_mbufs(struct ixl_queue *que, int limit) | ixl_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, | ||||
{ | uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count) | ||||
struct ixl_vsi *vsi = que->vsi; | |||||
struct rx_ring *rxr = &que->rxr; | |||||
bus_dma_segment_t hseg[1]; | |||||
bus_dma_segment_t pseg[1]; | |||||
struct ixl_rx_buf *buf; | |||||
struct mbuf *mh, *mp; | |||||
int i, j, nsegs, error; | |||||
bool refreshed = FALSE; | |||||
i = j = rxr->next_refresh; | |||||
/* Control the loop with one beyond */ | |||||
if (++j == que->num_desc) | |||||
j = 0; | |||||
while (j != limit) { | |||||
buf = &rxr->buffers[i]; | |||||
if (rxr->hdr_split == FALSE) | |||||
goto no_split; | |||||
if (buf->m_head == NULL) { | |||||
mh = m_gethdr(M_NOWAIT, MT_DATA); | |||||
if (mh == NULL) | |||||
goto update; | |||||
} else | |||||
mh = buf->m_head; | |||||
mh->m_pkthdr.len = mh->m_len = MHLEN; | |||||
mh->m_len = MHLEN; | |||||
mh->m_flags |= M_PKTHDR; | |||||
/* Get the memory mapping */ | |||||
error = bus_dmamap_load_mbuf_sg(rxr->htag, | |||||
buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) { | |||||
printf("Refresh mbufs: hdr dmamap load" | |||||
" failure - %d\n", error); | |||||
m_free(mh); | |||||
buf->m_head = NULL; | |||||
goto update; | |||||
} | |||||
buf->m_head = mh; | |||||
bus_dmamap_sync(rxr->htag, buf->hmap, | |||||
BUS_DMASYNC_PREREAD); | |||||
rxr->base[i].read.hdr_addr = | |||||
htole64(hseg[0].ds_addr); | |||||
no_split: | |||||
if (buf->m_pack == NULL) { | |||||
mp = m_getjcl(M_NOWAIT, MT_DATA, | |||||
M_PKTHDR, rxr->mbuf_sz); | |||||
if (mp == NULL) | |||||
goto update; | |||||
} else | |||||
mp = buf->m_pack; | |||||
mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; | |||||
/* Get the memory mapping */ | |||||
error = bus_dmamap_load_mbuf_sg(rxr->ptag, | |||||
buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) { | |||||
printf("Refresh mbufs: payload dmamap load" | |||||
" failure - %d\n", error); | |||||
m_free(mp); | |||||
buf->m_pack = NULL; | |||||
goto update; | |||||
} | |||||
buf->m_pack = mp; | |||||
bus_dmamap_sync(rxr->ptag, buf->pmap, | |||||
BUS_DMASYNC_PREREAD); | |||||
rxr->base[i].read.pkt_addr = | |||||
htole64(pseg[0].ds_addr); | |||||
/* Used only when doing header split */ | |||||
rxr->base[i].read.hdr_addr = 0; | |||||
refreshed = TRUE; | |||||
/* Next is precalculated */ | |||||
i = j; | |||||
rxr->next_refresh = i; | |||||
if (++j == que->num_desc) | |||||
j = 0; | |||||
} | |||||
update: | |||||
if (refreshed) /* Update hardware tail index */ | |||||
wr32(vsi->hw, rxr->tail, rxr->next_refresh); | |||||
return; | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* Allocate memory for rx_buffer structures. Since we use one | |||||
* rx_buffer per descriptor, the maximum number of rx_buffer's | |||||
* that we'll need is equal to the number of receive descriptors | |||||
* that we've defined. | |||||
* | |||||
**********************************************************************/ | |||||
int | |||||
ixl_allocate_rx_data(struct ixl_queue *que) | |||||
{ | { | ||||
struct rx_ring *rxr = &que->rxr; | struct ixl_vsi *vsi = arg; | ||||
struct ixl_vsi *vsi = que->vsi; | struct rx_ring *rxr = &vsi->queues[rxqid].rxr; | ||||
device_t dev = vsi->dev; | int i; | ||||
struct ixl_rx_buf *buf; | uint32_t next_pidx; | ||||
int i, bsize, error; | |||||
bsize = sizeof(struct ixl_rx_buf) * que->num_desc; | for (i = 0, next_pidx = pidx; i < count; i++) { | ||||
if (!(rxr->buffers = | rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); | ||||
(struct ixl_rx_buf *) malloc(bsize, | if (++next_pidx == ixl_sctx->isc_nrxd) | ||||
M_DEVBUF, M_NOWAIT | M_ZERO))) { | next_pidx = 0; | ||||
device_printf(dev, "Unable to allocate rx_buffer memory\n"); | |||||
error = ENOMEM; | |||||
return (error); | |||||
} | } | ||||
if ((error = bus_dma_tag_create(NULL, /* parent */ | |||||
1, 0, /* alignment, bounds */ | |||||
BUS_SPACE_MAXADDR, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filter, filterarg */ | |||||
MSIZE, /* maxsize */ | |||||
1, /* nsegments */ | |||||
MSIZE, /* maxsegsize */ | |||||
0, /* flags */ | |||||
NULL, /* lockfunc */ | |||||
NULL, /* lockfuncarg */ | |||||
&rxr->htag))) { | |||||
device_printf(dev, "Unable to create RX DMA htag\n"); | |||||
return (error); | |||||
} | } | ||||
if ((error = bus_dma_tag_create(NULL, /* parent */ | static void | ||||
1, 0, /* alignment, bounds */ | ixl_isc_rxd_flush(void * arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx) | ||||
BUS_SPACE_MAXADDR, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filter, filterarg */ | |||||
MJUM16BYTES, /* maxsize */ | |||||
1, /* nsegments */ | |||||
MJUM16BYTES, /* maxsegsize */ | |||||
0, /* flags */ | |||||
NULL, /* lockfunc */ | |||||
NULL, /* lockfuncarg */ | |||||
&rxr->ptag))) { | |||||
device_printf(dev, "Unable to create RX DMA ptag\n"); | |||||
return (error); | |||||
} | |||||
for (i = 0; i < que->num_desc; i++) { | |||||
buf = &rxr->buffers[i]; | |||||
error = bus_dmamap_create(rxr->htag, | |||||
BUS_DMA_NOWAIT, &buf->hmap); | |||||
if (error) { | |||||
device_printf(dev, "Unable to create RX head map\n"); | |||||
break; | |||||
} | |||||
error = bus_dmamap_create(rxr->ptag, | |||||
BUS_DMA_NOWAIT, &buf->pmap); | |||||
if (error) { | |||||
device_printf(dev, "Unable to create RX pkt map\n"); | |||||
break; | |||||
} | |||||
} | |||||
return (error); | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* (Re)Initialize the queue receive ring and its buffers. | |||||
* | |||||
**********************************************************************/ | |||||
int | |||||
ixl_init_rx_ring(struct ixl_queue *que) | |||||
{ | { | ||||
struct rx_ring *rxr = &que->rxr; | struct ixl_vsi *vsi = arg; | ||||
struct ixl_vsi *vsi = que->vsi; | struct rx_ring *rxr = &vsi->queues[rxqid].rxr; | ||||
#if defined(INET6) || defined(INET) | |||||
struct ifnet *ifp = vsi->ifp; | |||||
struct lro_ctrl *lro = &rxr->lro; | |||||
#endif | |||||
struct ixl_rx_buf *buf; | |||||
bus_dma_segment_t pseg[1], hseg[1]; | |||||
int rsize, nsegs, error = 0; | |||||
#ifdef DEV_NETMAP | |||||
struct netmap_adapter *na = NA(que->vsi->ifp); | |||||
struct netmap_slot *slot; | |||||
#endif /* DEV_NETMAP */ | |||||
IXL_RX_LOCK(rxr); | wr32(vsi->hw, rxr->tail, pidx); | ||||
#ifdef DEV_NETMAP | |||||
/* same as in ixl_init_tx_ring() */ | |||||
slot = netmap_reset(na, NR_RX, que->me, 0); | |||||
#endif /* DEV_NETMAP */ | |||||
/* Clear the ring contents */ | |||||
rsize = roundup2(que->num_desc * | |||||
sizeof(union i40e_rx_desc), DBA_ALIGN); | |||||
bzero((void *)rxr->base, rsize); | |||||
/* Cleanup any existing buffers */ | |||||
for (int i = 0; i < que->num_desc; i++) { | |||||
buf = &rxr->buffers[i]; | |||||
if (buf->m_head != NULL) { | |||||
bus_dmamap_sync(rxr->htag, buf->hmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->htag, buf->hmap); | |||||
buf->m_head->m_flags |= M_PKTHDR; | |||||
m_freem(buf->m_head); | |||||
} | } | ||||
if (buf->m_pack != NULL) { | |||||
bus_dmamap_sync(rxr->ptag, buf->pmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->ptag, buf->pmap); | |||||
buf->m_pack->m_flags |= M_PKTHDR; | |||||
m_freem(buf->m_pack); | |||||
} | |||||
buf->m_head = NULL; | |||||
buf->m_pack = NULL; | |||||
} | |||||
/* header split is off */ | static int | ||||
rxr->hdr_split = FALSE; | ixl_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx) | ||||
/* Now replenish the mbufs */ | |||||
for (int j = 0; j != que->num_desc; ++j) { | |||||
struct mbuf *mh, *mp; | |||||
buf = &rxr->buffers[j]; | |||||
#ifdef DEV_NETMAP | |||||
/* | |||||
* In netmap mode, fill the map and set the buffer | |||||
* address in the NIC ring, considering the offset | |||||
* between the netmap and NIC rings (see comment in | |||||
* ixgbe_setup_transmit_ring() ). No need to allocate | |||||
* an mbuf, so end the block with a continue; | |||||
*/ | |||||
if (slot) { | |||||
int sj = netmap_idx_n2k(&na->rx_rings[que->me], j); | |||||
uint64_t paddr; | |||||
void *addr; | |||||
addr = PNMB(na, slot + sj, &paddr); | |||||
netmap_load_map(na, rxr->dma.tag, buf->pmap, addr); | |||||
/* Update descriptor and the cached value */ | |||||
rxr->base[j].read.pkt_addr = htole64(paddr); | |||||
rxr->base[j].read.hdr_addr = 0; | |||||
continue; | |||||
} | |||||
#endif /* DEV_NETMAP */ | |||||
/* | |||||
** Don't allocate mbufs if not | |||||
** doing header split, its wasteful | |||||
*/ | |||||
if (rxr->hdr_split == FALSE) | |||||
goto skip_head; | |||||
/* First the header */ | |||||
buf->m_head = m_gethdr(M_NOWAIT, MT_DATA); | |||||
if (buf->m_head == NULL) { | |||||
error = ENOBUFS; | |||||
goto fail; | |||||
} | |||||
m_adj(buf->m_head, ETHER_ALIGN); | |||||
mh = buf->m_head; | |||||
mh->m_len = mh->m_pkthdr.len = MHLEN; | |||||
mh->m_flags |= M_PKTHDR; | |||||
/* Get the memory mapping */ | |||||
error = bus_dmamap_load_mbuf_sg(rxr->htag, | |||||
buf->hmap, buf->m_head, hseg, | |||||
&nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) /* Nothing elegant to do here */ | |||||
goto fail; | |||||
bus_dmamap_sync(rxr->htag, | |||||
buf->hmap, BUS_DMASYNC_PREREAD); | |||||
/* Update descriptor */ | |||||
rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr); | |||||
skip_head: | |||||
/* Now the payload cluster */ | |||||
buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, | |||||
M_PKTHDR, rxr->mbuf_sz); | |||||
if (buf->m_pack == NULL) { | |||||
error = ENOBUFS; | |||||
goto fail; | |||||
} | |||||
mp = buf->m_pack; | |||||
mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; | |||||
/* Get the memory mapping */ | |||||
error = bus_dmamap_load_mbuf_sg(rxr->ptag, | |||||
buf->pmap, mp, pseg, | |||||
&nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) | |||||
goto fail; | |||||
bus_dmamap_sync(rxr->ptag, | |||||
buf->pmap, BUS_DMASYNC_PREREAD); | |||||
/* Update descriptor */ | |||||
rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr); | |||||
rxr->base[j].read.hdr_addr = 0; | |||||
} | |||||
/* Setup our descriptor indices */ | |||||
rxr->next_check = 0; | |||||
rxr->next_refresh = 0; | |||||
rxr->lro_enabled = FALSE; | |||||
rxr->split = 0; | |||||
rxr->bytes = 0; | |||||
rxr->discard = FALSE; | |||||
wr32(vsi->hw, rxr->tail, que->num_desc - 1); | |||||
ixl_flush(vsi->hw); | |||||
#if defined(INET6) || defined(INET) | |||||
/* | |||||
** Now set up the LRO interface: | |||||
*/ | |||||
if (ifp->if_capenable & IFCAP_LRO) { | |||||
int err = tcp_lro_init(lro); | |||||
if (err) { | |||||
if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me); | |||||
goto fail; | |||||
} | |||||
INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me); | |||||
rxr->lro_enabled = TRUE; | |||||
lro->ifp = vsi->ifp; | |||||
} | |||||
#endif | |||||
bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
fail: | |||||
IXL_RX_UNLOCK(rxr); | |||||
return (error); | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* Free station receive ring data structures | |||||
* | |||||
**********************************************************************/ | |||||
void | |||||
ixl_free_que_rx(struct ixl_queue *que) | |||||
{ | { | ||||
struct rx_ring *rxr = &que->rxr; | struct ixl_vsi *vsi = arg; | ||||
struct ixl_rx_buf *buf; | struct rx_ring *rxr = &vsi->queues[rxqid].rxr; | ||||
union i40e_rx_desc *cur; | |||||
u64 qword; | |||||
uint32_t status; | |||||
int cnt, i; | |||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); | for (cnt = 0, i = idx; cnt < ixl_sctx->isc_nrxd;) { | ||||
cur = &rxr->rx_base[i]; | |||||
/* Cleanup any existing buffers */ | qword = le64toh(cur->wb.qword1.status_error_len); | ||||
if (rxr->buffers != NULL) { | status = (qword & I40E_RXD_QW1_STATUS_MASK) | ||||
for (int i = 0; i < que->num_desc; i++) { | >> I40E_RXD_QW1_STATUS_SHIFT; | ||||
buf = &rxr->buffers[i]; | if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) | ||||
if (buf->m_head != NULL) { | break; | ||||
bus_dmamap_sync(rxr->htag, buf->hmap, | cnt++; | ||||
BUS_DMASYNC_POSTREAD); | if (++i == ixl_sctx->isc_nrxd) | ||||
bus_dmamap_unload(rxr->htag, buf->hmap); | i = 0; | ||||
buf->m_head->m_flags |= M_PKTHDR; | |||||
m_freem(buf->m_head); | |||||
} | } | ||||
if (buf->m_pack != NULL) { | |||||
bus_dmamap_sync(rxr->ptag, buf->pmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->ptag, buf->pmap); | |||||
buf->m_pack->m_flags |= M_PKTHDR; | |||||
m_freem(buf->m_pack); | |||||
} | |||||
buf->m_head = NULL; | |||||
buf->m_pack = NULL; | |||||
if (buf->hmap != NULL) { | |||||
bus_dmamap_destroy(rxr->htag, buf->hmap); | |||||
buf->hmap = NULL; | |||||
} | |||||
if (buf->pmap != NULL) { | |||||
bus_dmamap_destroy(rxr->ptag, buf->pmap); | |||||
buf->pmap = NULL; | |||||
} | |||||
} | |||||
if (rxr->buffers != NULL) { | |||||
free(rxr->buffers, M_DEVBUF); | |||||
rxr->buffers = NULL; | |||||
} | |||||
} | |||||
if (rxr->htag != NULL) { | return (cnt); | ||||
bus_dma_tag_destroy(rxr->htag); | |||||
rxr->htag = NULL; | |||||
} | } | ||||
if (rxr->ptag != NULL) { | |||||
bus_dma_tag_destroy(rxr->ptag); | |||||
rxr->ptag = NULL; | |||||
} | |||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); | |||||
return; | |||||
} | |||||
static __inline void | |||||
ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) | |||||
{ | |||||
#if defined(INET6) || defined(INET) | |||||
/* | |||||
* ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet | |||||
* should be computed by hardware. Also it should not have VLAN tag in | |||||
* ethernet header. | |||||
*/ | |||||
if (rxr->lro_enabled && | |||||
(ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && | |||||
(m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == | |||||
(CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { | |||||
/* | |||||
* Send to the stack if: | |||||
** - LRO not enabled, or | |||||
** - no LRO resources, or | |||||
** - lro enqueue fails | |||||
*/ | |||||
if (rxr->lro.lro_cnt != 0) | |||||
if (tcp_lro_rx(&rxr->lro, m, 0) == 0) | |||||
return; | |||||
} | |||||
#endif | |||||
IXL_RX_UNLOCK(rxr); | |||||
(*ifp->if_input)(ifp, m); | |||||
IXL_RX_LOCK(rxr); | |||||
} | |||||
static __inline void | |||||
ixl_rx_discard(struct rx_ring *rxr, int i) | |||||
{ | |||||
struct ixl_rx_buf *rbuf; | |||||
rbuf = &rxr->buffers[i]; | |||||
if (rbuf->fmp != NULL) {/* Partial chain ? */ | |||||
rbuf->fmp->m_flags |= M_PKTHDR; | |||||
m_freem(rbuf->fmp); | |||||
rbuf->fmp = NULL; | |||||
} | |||||
/* | |||||
** With advanced descriptors the writeback | |||||
** clobbers the buffer addrs, so its easier | |||||
** to just free the existing mbufs and take | |||||
** the normal refresh path to get new buffers | |||||
** and mapping. | |||||
*/ | |||||
if (rbuf->m_head) { | |||||
m_free(rbuf->m_head); | |||||
rbuf->m_head = NULL; | |||||
} | |||||
if (rbuf->m_pack) { | |||||
m_free(rbuf->m_pack); | |||||
rbuf->m_pack = NULL; | |||||
} | |||||
return; | |||||
} | |||||
#ifdef RSS | #ifdef RSS | ||||
/* | /* | ||||
** i40e_ptype_to_hash: parse the packet type | ** i40e_ptype_to_hash: parse the packet type | ||||
** to determine the appropriate hash. | ** to determine the appropriate hash. | ||||
*/ | */ | ||||
static inline int | static inline int | ||||
ixl_ptype_to_hash(u8 ptype) | ixl_ptype_to_hash(u8 ptype) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | ixl_ptype_to_hash(u8 ptype) | ||||
} | } | ||||
/* We should never get here!! */ | /* We should never get here!! */ | ||||
return M_HASHTYPE_OPAQUE; | return M_HASHTYPE_OPAQUE; | ||||
} | } | ||||
#endif /* RSS */ | #endif /* RSS */ | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* This routine executes in interrupt context. It replenishes | * This routine executes in ithread context. It sends data which has been | ||||
* the mbufs in the descriptor and sends data which has been | |||||
* dma'ed into host memory to upper layer. | * dma'ed into host memory to upper layer. | ||||
* | * | ||||
* We loop at most count times if count is > 0, or until done if | * Returns 0 upon success, errno on failure | ||||
* count < 0. | |||||
* | |||||
* Return TRUE for more work, FALSE for all clean. | |||||
*********************************************************************/ | *********************************************************************/ | ||||
bool | |||||
ixl_rxeof(struct ixl_queue *que, int count) | static int | ||||
ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | |||||
{ | { | ||||
struct ixl_vsi *vsi = que->vsi; | struct ixl_vsi *vsi = arg; | ||||
struct ixl_queue *que = &vsi->queues[ri->iri_qsidx]; | |||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||
struct ifnet *ifp = vsi->ifp; | |||||
#if defined(INET6) || defined(INET) | |||||
struct lro_ctrl *lro = &rxr->lro; | |||||
struct lro_entry *queued; | |||||
#endif | |||||
int i, nextp, processed = 0; | |||||
union i40e_rx_desc *cur; | union i40e_rx_desc *cur; | ||||
struct ixl_rx_buf *rbuf, *nbuf; | u32 status, error; | ||||
IXL_RX_LOCK(rxr); | |||||
#ifdef DEV_NETMAP | |||||
if (netmap_rx_irq(ifp, que->me, &count)) { | |||||
IXL_RX_UNLOCK(rxr); | |||||
return (FALSE); | |||||
} | |||||
#endif /* DEV_NETMAP */ | |||||
for (i = rxr->next_check; count != 0;) { | |||||
struct mbuf *sendmp, *mh, *mp; | |||||
u32 rsc, status, error; | |||||
u16 hlen, plen, vtag; | u16 hlen, plen, vtag; | ||||
u64 qword; | u64 qword; | ||||
u8 ptype; | u8 ptype; | ||||
bool eop; | bool eop; | ||||
/* Sync the ring. */ | ri->iri_qidx = 0; | ||||
bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, | cur = &rxr->rx_base[ri->iri_cidx]; | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
cur = &rxr->base[i]; | |||||
qword = le64toh(cur->wb.qword1.status_error_len); | qword = le64toh(cur->wb.qword1.status_error_len); | ||||
status = (qword & I40E_RXD_QW1_STATUS_MASK) | status = (qword & I40E_RXD_QW1_STATUS_MASK) | ||||
>> I40E_RXD_QW1_STATUS_SHIFT; | >> I40E_RXD_QW1_STATUS_SHIFT; | ||||
error = (qword & I40E_RXD_QW1_ERROR_MASK) | error = (qword & I40E_RXD_QW1_ERROR_MASK) | ||||
>> I40E_RXD_QW1_ERROR_SHIFT; | >> I40E_RXD_QW1_ERROR_SHIFT; | ||||
plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) | plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) | ||||
>> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; | >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; | ||||
hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) | hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) | ||||
>> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; | >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; | ||||
ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) | ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) | ||||
>> I40E_RXD_QW1_PTYPE_SHIFT; | >> I40E_RXD_QW1_PTYPE_SHIFT; | ||||
if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) { | /* we should never be called without a valid descriptor */ | ||||
++rxr->not_done; | MPASS((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) != 0); | ||||
break; | |||||
} | |||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) | |||||
break; | |||||
count--; | ri->iri_len = plen; | ||||
sendmp = NULL; | rxr->rx_bytes += plen; | ||||
nbuf = NULL; | |||||
rsc = 0; | #ifdef notyet | ||||
/* XXX should be checked from avail */ | |||||
#endif | |||||
cur->wb.qword1.status_error_len = 0; | cur->wb.qword1.status_error_len = 0; | ||||
rbuf = &rxr->buffers[i]; | |||||
mh = rbuf->m_head; | |||||
mp = rbuf->m_pack; | |||||
eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); | eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); | ||||
if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) | if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) | ||||
vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); | vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); | ||||
else | else | ||||
vtag = 0; | vtag = 0; | ||||
/* | /* | ||||
** Make sure bad packets are discarded, | ** Make sure bad packets are discarded, | ||||
** note that only EOP descriptor has valid | ** note that only EOP descriptor has valid | ||||
** error results. | ** error results. | ||||
*/ | */ | ||||
if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { | if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { | ||||
rxr->discarded++; | rxr->discarded++; | ||||
ixl_rx_discard(rxr, i); | return (EBADMSG); | ||||
goto next_desc; | |||||
} | } | ||||
/* Prefetch the next buffer */ | /* Prefetch the next buffer */ | ||||
if (!eop) { | if (!eop) { | ||||
nextp = i + 1; | ri->iri_next_offset = 1; | ||||
if (nextp == que->num_desc) | |||||
nextp = 0; | |||||
nbuf = &rxr->buffers[nextp]; | |||||
prefetch(nbuf); | |||||
} | |||||
/* | |||||
** The header mbuf is ONLY used when header | |||||
** split is enabled, otherwise we get normal | |||||
** behavior, ie, both header and payload | |||||
** are DMA'd into the payload buffer. | |||||
** | |||||
** Rather than using the fmp/lmp global pointers | |||||
** we now keep the head of a packet chain in the | |||||
** buffer struct and pass this along from one | |||||
** descriptor to the next, until we get EOP. | |||||
*/ | |||||
if (rxr->hdr_split && (rbuf->fmp == NULL)) { | |||||
if (hlen > IXL_RX_HDR) | |||||
hlen = IXL_RX_HDR; | |||||
mh->m_len = hlen; | |||||
mh->m_flags |= M_PKTHDR; | |||||
mh->m_next = NULL; | |||||
mh->m_pkthdr.len = mh->m_len; | |||||
/* Null buf pointer so it is refreshed */ | |||||
rbuf->m_head = NULL; | |||||
/* | |||||
** Check the payload length, this | |||||
** could be zero if its a small | |||||
** packet. | |||||
*/ | |||||
if (plen > 0) { | |||||
mp->m_len = plen; | |||||
mp->m_next = NULL; | |||||
mp->m_flags &= ~M_PKTHDR; | |||||
mh->m_next = mp; | |||||
mh->m_pkthdr.len += mp->m_len; | |||||
/* Null buf pointer so it is refreshed */ | |||||
rbuf->m_pack = NULL; | |||||
rxr->split++; | |||||
} | |||||
/* | |||||
** Now create the forward | |||||
** chain so when complete | |||||
** we wont have to. | |||||
*/ | |||||
if (eop == 0) { | |||||
/* stash the chain head */ | |||||
nbuf->fmp = mh; | |||||
/* Make forward chain */ | |||||
if (plen) | |||||
mp->m_next = nbuf->m_pack; | |||||
else | |||||
mh->m_next = nbuf->m_pack; | |||||
} else { | } else { | ||||
/* Singlet, prepare to send */ | |||||
sendmp = mh; | |||||
if (vtag) { | |||||
sendmp->m_pkthdr.ether_vtag = vtag; | |||||
sendmp->m_flags |= M_VLANTAG; | |||||
} | |||||
} | |||||
} else { | |||||
/* | |||||
** Either no header split, or a | |||||
** secondary piece of a fragmented | |||||
** split packet. | |||||
*/ | |||||
mp->m_len = plen; | |||||
/* | |||||
** See if there is a stored head | |||||
** that determines what we are | |||||
*/ | |||||
sendmp = rbuf->fmp; | |||||
rbuf->m_pack = rbuf->fmp = NULL; | |||||
if (sendmp != NULL) /* secondary frag */ | |||||
sendmp->m_pkthdr.len += mp->m_len; | |||||
else { | |||||
/* first desc of a non-ps chain */ | |||||
sendmp = mp; | |||||
sendmp->m_flags |= M_PKTHDR; | |||||
sendmp->m_pkthdr.len = mp->m_len; | |||||
if (vtag) { | |||||
sendmp->m_pkthdr.ether_vtag = vtag; | |||||
sendmp->m_flags |= M_VLANTAG; | |||||
} | |||||
} | |||||
/* Pass the head pointer on */ | |||||
if (eop == 0) { | |||||
nbuf->fmp = sendmp; | |||||
sendmp = NULL; | |||||
mp->m_next = nbuf->m_pack; | |||||
} | |||||
} | |||||
++processed; | |||||
/* Sending this frame? */ | |||||
if (eop) { | |||||
sendmp->m_pkthdr.rcvif = ifp; | |||||
/* gather stats */ | |||||
rxr->rx_packets++; | rxr->rx_packets++; | ||||
rxr->rx_bytes += sendmp->m_pkthdr.len; | |||||
/* capture data for dynamic ITR adjustment */ | /* capture data for dynamic ITR adjustment */ | ||||
rxr->packets++; | rxr->packets++; | ||||
rxr->bytes += sendmp->m_pkthdr.len; | if ((vsi->ifp->if_capenable & IFCAP_RXCSUM) != 0) | ||||
if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) | ixl_rx_checksum(ri, status, error, ptype); | ||||
ixl_rx_checksum(sendmp, status, error, ptype); | |||||
#ifdef RSS | #ifdef RSS | ||||
sendmp->m_pkthdr.flowid = | ri->iri_flowid = | ||||
le32toh(cur->wb.qword0.hi_dword.rss); | le32toh(cur->wb.qword0.hi_dword.rss); | ||||
M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype)); | ri->iri_rsstype = ixl_ptype_to_hash(ptype); | ||||
#else | #else | ||||
sendmp->m_pkthdr.flowid = que->msix; | ri->iri_flowid = que->msix; | ||||
M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); | ri->iri_rsstype = M_HASHTYPE_OPAQUE; | ||||
#endif | #endif | ||||
if (vtag) { | |||||
ri->iri_vtag = vtag; | |||||
ri->iri_flags |= M_VLANTAG; | |||||
} | } | ||||
next_desc: | ri->iri_next_offset = 0; | ||||
bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
/* Advance our pointers to the next descriptor. */ | |||||
if (++i == que->num_desc) | |||||
i = 0; | |||||
/* Now send to the stack or do LRO */ | |||||
if (sendmp != NULL) { | |||||
rxr->next_check = i; | |||||
ixl_rx_input(rxr, ifp, sendmp, ptype); | |||||
i = rxr->next_check; | |||||
} | } | ||||
return (0); | |||||
/* Every 8 descriptors we go to refresh mbufs */ | |||||
if (processed == 8) { | |||||
ixl_refresh_mbufs(que, i); | |||||
processed = 0; | |||||
} | } | ||||
} | |||||
/* Refresh any remaining buf structs */ | |||||
if (ixl_rx_unrefreshed(que)) | |||||
ixl_refresh_mbufs(que, i); | |||||
rxr->next_check = i; | |||||
#if defined(INET6) || defined(INET) | |||||
/* | |||||
* Flush any outstanding LRO work | |||||
*/ | |||||
while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { | |||||
SLIST_REMOVE_HEAD(&lro->lro_active, next); | |||||
tcp_lro_flush(lro, queued); | |||||
} | |||||
#endif | |||||
IXL_RX_UNLOCK(rxr); | |||||
return (FALSE); | |||||
} | |||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Verify that the hardware indicated that the checksum is valid. | * Verify that the hardware indicated that the checksum is valid. | ||||
* Inform the stack about the status of checksum so that stack | * Inform the stack about the status of checksum so that stack | ||||
* doesn't spend time verifying the checksum. | * doesn't spend time verifying the checksum. | ||||
* | * | ||||
*********************************************************************/ | *********************************************************************/ | ||||
static void | static void | ||||
ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype) | ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype) | ||||
{ | { | ||||
struct i40e_rx_ptype_decoded decoded; | struct i40e_rx_ptype_decoded decoded; | ||||
decoded = decode_rx_desc_ptype(ptype); | decoded = decode_rx_desc_ptype(ptype); | ||||
/* Errors? */ | /* Errors? */ | ||||
if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | | if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | | ||||
(1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { | (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { | ||||
mp->m_pkthdr.csum_flags = 0; | ri->iri_csum_flags = 0; | ||||
return; | return; | ||||
} | } | ||||
/* IPv6 with extension headers likely have bad csum */ | /* IPv6 with extension headers likely have bad csum */ | ||||
if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && | if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && | ||||
decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) | decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) | ||||
if (status & | if (status & | ||||
(1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { | (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { | ||||
mp->m_pkthdr.csum_flags = 0; | ri->iri_csum_flags = 0; | ||||
return; | return; | ||||
} | } | ||||
/* IP Checksum Good */ | /* IP Checksum Good */ | ||||
mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; | ri->iri_csum_flags = CSUM_IP_CHECKED; | ||||
mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; | ri->iri_csum_flags |= CSUM_IP_VALID; | ||||
if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) { | if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) { | ||||
mp->m_pkthdr.csum_flags |= | ri->iri_csum_flags |= | ||||
(CSUM_DATA_VALID | CSUM_PSEUDO_HDR); | (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); | ||||
mp->m_pkthdr.csum_data |= htons(0xffff); | ri->iri_csum_data |= htons(0xffff); | ||||
} | } | ||||
return; | return; | ||||
} | } | ||||
#if __FreeBSD_version >= 1100000 | #if __FreeBSD_version >= 1100000 | ||||
uint64_t | uint64_t | ||||
ixl_get_counter(if_t ifp, ift_counter cnt) | ixl_get_counter(if_t ifp, ift_counter cnt) | ||||
{ | { | ||||
Show All 36 Lines |