Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/ixl/ixl_txrx.c
/****************************************************************************** | /****************************************************************************** | ||||
Copyright (c) 2013-2015, Intel Corporation | Copyright (c) 2013-2017, Intel Corporation | ||||
All rights reserved. | All rights reserved. | ||||
Redistribution and use in source and binary forms, with or without | Redistribution and use in source and binary forms, with or without | ||||
modification, are permitted provided that the following conditions are met: | modification, are permitted provided that the following conditions are met: | ||||
1. Redistributions of source code must retain the above copyright notice, | 1. Redistributions of source code must retain the above copyright notice, | ||||
this list of conditions and the following disclaimer. | this list of conditions and the following disclaimer. | ||||
Show All 34 Lines | |||||
#include "ixl.h" | #include "ixl.h" | ||||
#ifdef RSS | #ifdef RSS | ||||
#include <net/rss_config.h> | #include <net/rss_config.h> | ||||
#endif | #endif | ||||
/* Local Prototypes */ | /* Local Prototypes */ | ||||
static void ixl_rx_checksum(struct mbuf *, u32, u32, u8); | static void ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype); | ||||
static void ixl_refresh_mbufs(struct ixl_queue *, int); | |||||
static int ixl_xmit(struct ixl_queue *, struct mbuf **); | |||||
static int ixl_tx_setup_offload(struct ixl_queue *, | |||||
struct mbuf *, u32 *, u32 *); | |||||
static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *); | |||||
static inline void ixl_rx_discard(struct rx_ring *, int); | static int ixl_isc_txd_encap(void *arg, if_pkt_info_t pi); | ||||
static inline void ixl_rx_input(struct rx_ring *, struct ifnet *, | static void ixl_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); | ||||
struct mbuf *, u8); | static int ixl_isc_txd_credits_update_hwb(void *arg, uint16_t txqid, bool clear); | ||||
static int ixl_isc_txd_credits_update_dwb(void *arg, uint16_t txqid, bool clear); | |||||
static inline bool ixl_tso_detect_sparse(struct mbuf *mp); | static void ixl_isc_rxd_refill(void *arg, if_rxd_update_t iru); | ||||
static inline u32 ixl_get_tx_head(struct ixl_queue *que); | static void ixl_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, | ||||
qidx_t pidx); | |||||
static int ixl_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, | |||||
qidx_t budget); | |||||
static int ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); | |||||
#ifdef DEV_NETMAP | extern int ixl_intr(void *arg); | ||||
#include <dev/netmap/if_ixl_netmap.h> | |||||
int ixl_rx_miss, ixl_rx_miss_bufs, ixl_crcstrip = 1; | |||||
#endif /* DEV_NETMAP */ | |||||
struct if_txrx ixl_txrx_hwb = { | |||||
ixl_isc_txd_encap, | |||||
ixl_isc_txd_flush, | |||||
ixl_isc_txd_credits_update_hwb, | |||||
ixl_isc_rxd_available, | |||||
ixl_isc_rxd_pkt_get, | |||||
ixl_isc_rxd_refill, | |||||
ixl_isc_rxd_flush, | |||||
ixl_intr | |||||
}; | |||||
struct if_txrx ixl_txrx_dwb = { | |||||
ixl_isc_txd_encap, | |||||
ixl_isc_txd_flush, | |||||
ixl_isc_txd_credits_update_dwb, | |||||
ixl_isc_rxd_available, | |||||
ixl_isc_rxd_pkt_get, | |||||
ixl_isc_rxd_refill, | |||||
ixl_isc_rxd_flush, | |||||
ixl_intr | |||||
}; | |||||
/* | /* | ||||
* @key key is saved into this parameter | * @key key is saved into this parameter | ||||
*/ | */ | ||||
void | void | ||||
ixl_get_default_rss_key(u32 *key) | ixl_get_default_rss_key(u32 *key) | ||||
{ | { | ||||
MPASS(key != NULL); | MPASS(key != NULL); | ||||
u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687, | u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687, | ||||
0x183cfd8c, 0xce880440, 0x580cbc3c, | 0x183cfd8c, 0xce880440, 0x580cbc3c, | ||||
0x35897377, 0x328b25e1, 0x4fa98922, | 0x35897377, 0x328b25e1, 0x4fa98922, | ||||
0xb7d90c14, 0xd5bad70d, 0xcd15a2c1, | 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1, | ||||
0x0, 0x0, 0x0}; | 0x0, 0x0, 0x0}; | ||||
bcopy(rss_seed, key, IXL_RSS_KEY_SIZE); | bcopy(rss_seed, key, IXL_RSS_KEY_SIZE); | ||||
} | } | ||||
/* | /** | ||||
** Multiqueue Transmit driver | * i40e_vc_stat_str - convert virtchnl status err code to a string | ||||
*/ | * @hw: pointer to the HW structure | ||||
int | * @stat_err: the status error code to convert | ||||
ixl_mq_start(struct ifnet *ifp, struct mbuf *m) | **/ | ||||
const char * | |||||
i40e_vc_stat_str(struct i40e_hw *hw, enum virtchnl_status_code stat_err) | |||||
{ | { | ||||
struct ixl_vsi *vsi = ifp->if_softc; | switch (stat_err) { | ||||
struct ixl_queue *que; | case VIRTCHNL_STATUS_SUCCESS: | ||||
struct tx_ring *txr; | return "OK"; | ||||
int err, i; | case VIRTCHNL_ERR_PARAM: | ||||
#ifdef RSS | return "VIRTCHNL_ERR_PARAM"; | ||||
u32 bucket_id; | case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH: | ||||
#endif | return "VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH"; | ||||
case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR: | |||||
/* | return "VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR"; | ||||
** Which queue to use: | case VIRTCHNL_STATUS_ERR_INVALID_VF_ID: | ||||
** | return "VIRTCHNL_STATUS_ERR_INVALID_VF_ID"; | ||||
** When doing RSS, map it to the same outbound | case VIRTCHNL_STATUS_NOT_SUPPORTED: | ||||
** queue as the incoming flow would be mapped to. | return "VIRTCHNL_STATUS_NOT_SUPPORTED"; | ||||
** If everything is setup correctly, it should be | |||||
** the same bucket that the current CPU we're on is. | |||||
*/ | |||||
if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { | |||||
#ifdef RSS | |||||
if (rss_hash2bucket(m->m_pkthdr.flowid, | |||||
M_HASHTYPE_GET(m), &bucket_id) == 0) { | |||||
i = bucket_id % vsi->num_queues; | |||||
} else | |||||
#endif | |||||
i = m->m_pkthdr.flowid % vsi->num_queues; | |||||
} else | |||||
i = curcpu % vsi->num_queues; | |||||
que = &vsi->queues[i]; | |||||
txr = &que->txr; | |||||
err = drbr_enqueue(ifp, txr->br, m); | |||||
if (err) | |||||
return (err); | |||||
if (IXL_TX_TRYLOCK(txr)) { | |||||
ixl_mq_start_locked(ifp, txr); | |||||
IXL_TX_UNLOCK(txr); | |||||
} else | |||||
taskqueue_enqueue(que->tq, &que->tx_task); | |||||
return (0); | |||||
} | } | ||||
int | snprintf(hw->err_str, sizeof(hw->err_str), "%d", stat_err); | ||||
ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) | return hw->err_str; | ||||
{ | |||||
struct ixl_queue *que = txr->que; | |||||
struct ixl_vsi *vsi = que->vsi; | |||||
struct mbuf *next; | |||||
int err = 0; | |||||
if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || | |||||
vsi->link_active == 0) | |||||
return (ENETDOWN); | |||||
/* Process the transmit queue */ | |||||
while ((next = drbr_peek(ifp, txr->br)) != NULL) { | |||||
if ((err = ixl_xmit(que, &next)) != 0) { | |||||
if (next == NULL) | |||||
drbr_advance(ifp, txr->br); | |||||
else | |||||
drbr_putback(ifp, txr->br, next); | |||||
break; | |||||
} | } | ||||
drbr_advance(ifp, txr->br); | |||||
/* Send a copy of the frame to the BPF listener */ | |||||
ETHER_BPF_MTAP(ifp, next); | |||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) | |||||
break; | |||||
} | |||||
if (txr->avail < IXL_TX_CLEANUP_THRESHOLD) | |||||
ixl_txeof(que); | |||||
return (err); | |||||
} | |||||
/* | /* | ||||
* Called from a taskqueue to drain queued transmit packets. | * PCI BUSMASTER needs to be set for proper operation. | ||||
*/ | */ | ||||
void | void | ||||
ixl_deferred_mq_start(void *arg, int pending) | ixl_set_busmaster(device_t dev) | ||||
{ | { | ||||
struct ixl_queue *que = arg; | u16 pci_cmd_word; | ||||
struct tx_ring *txr = &que->txr; | |||||
struct ixl_vsi *vsi = que->vsi; | |||||
struct ifnet *ifp = vsi->ifp; | |||||
IXL_TX_LOCK(txr); | pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); | ||||
if (!drbr_empty(ifp, txr->br)) | pci_cmd_word |= PCIM_CMD_BUSMASTEREN; | ||||
ixl_mq_start_locked(ifp, txr); | pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); | ||||
IXL_TX_UNLOCK(txr); | |||||
} | } | ||||
/* | /* | ||||
** Flush all queue ring buffers | * Rewrite the ENABLE bit in the MSIX control register | ||||
*/ | */ | ||||
void | void | ||||
ixl_qflush(struct ifnet *ifp) | ixl_set_msix_enable(device_t dev) | ||||
{ | { | ||||
struct ixl_vsi *vsi = ifp->if_softc; | int msix_ctrl, rid; | ||||
for (int i = 0; i < vsi->num_queues; i++) { | pci_find_cap(dev, PCIY_MSIX, &rid); | ||||
struct ixl_queue *que = &vsi->queues[i]; | rid += PCIR_MSIX_CTRL; | ||||
struct tx_ring *txr = &que->txr; | msix_ctrl = pci_read_config(dev, rid, 2); | ||||
struct mbuf *m; | msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; | ||||
IXL_TX_LOCK(txr); | pci_write_config(dev, rid, msix_ctrl, 2); | ||||
while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) | |||||
m_freem(m); | |||||
IXL_TX_UNLOCK(txr); | |||||
} | } | ||||
if_qflush(ifp); | |||||
static bool | |||||
ixl_is_tx_desc_done(struct tx_ring *txr, int idx) | |||||
{ | |||||
return (((txr->tx_base[idx].cmd_type_offset_bsz >> I40E_TXD_QW1_DTYPE_SHIFT) | |||||
& I40E_TXD_QW1_DTYPE_MASK) == I40E_TX_DESC_DTYPE_DESC_DONE); | |||||
} | } | ||||
/* | // TODO: Might want to still incorporate this | ||||
** Find mbuf chains passed to the driver | // TODO: Compare this version of iflib with current version in OOT driver | ||||
** that are 'sparse', using more than 8 | #if 0 | ||||
** mbufs to deliver an mss-size chunk of data | |||||
*/ | |||||
static inline bool | static inline bool | ||||
ixl_tso_detect_sparse(struct mbuf *mp) | ixl_tso_detect_sparse(struct mbuf *mp) | ||||
{ | { | ||||
struct mbuf *m; | struct mbuf *m; | ||||
int num, mss; | int num, mss; | ||||
num = 0; | num = 0; | ||||
mss = mp->m_pkthdr.tso_segsz; | mss = mp->m_pkthdr.tso_segsz; | ||||
/* Exclude first mbuf; assume it contains all headers */ | /* Exclude first mbuf; assume it contains all headers */ | ||||
for (m = mp->m_next; m != NULL; m = m->m_next) { | for (m = mp->m_next; m != NULL; m = m->m_next) { | ||||
if (m == NULL) | if (m == NULL) | ||||
break; | break; | ||||
num++; | num++; | ||||
mss -= m->m_len % mp->m_pkthdr.tso_segsz; | mss -= m->m_len % mp->m_pkthdr.tso_segsz; | ||||
if (mss < 1) { | |||||
if (num > IXL_SPARSE_CHAIN) | if (num > IXL_SPARSE_CHAIN) | ||||
return (true); | return (true); | ||||
if (mss < 1) { | |||||
num = (mss == 0) ? 0 : 1; | num = (mss == 0) ? 0 : 1; | ||||
mss += mp->m_pkthdr.tso_segsz; | mss += mp->m_pkthdr.tso_segsz; | ||||
} | } | ||||
} | } | ||||
return (false); | return (false); | ||||
} | } | ||||
#endif | |||||
/********************************************************************* | |||||
* | |||||
* This routine maps the mbufs to tx descriptors, allowing the | |||||
* TX engine to transmit the packets. | |||||
* - return 0 on success, positive on failure | |||||
* | |||||
**********************************************************************/ | |||||
#define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) | |||||
static int | static int | ||||
ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) | ixl_tso_detect_sparse(bus_dma_segment_t *segs, int nsegs, int segsz) | ||||
{ | { | ||||
struct ixl_vsi *vsi = que->vsi; | int i, count, curseg; | ||||
struct i40e_hw *hw = vsi->hw; | |||||
struct tx_ring *txr = &que->txr; | |||||
struct ixl_tx_buf *buf; | |||||
struct i40e_tx_desc *txd = NULL; | |||||
struct mbuf *m_head, *m; | |||||
int i, j, error, nsegs; | |||||
int first, last = 0; | |||||
u16 vtag = 0; | |||||
u32 cmd, off; | |||||
bus_dmamap_t map; | |||||
bus_dma_tag_t tag; | |||||
bus_dma_segment_t segs[IXL_MAX_TSO_SEGS]; | |||||
cmd = off = 0; | if (nsegs <= IXL_MAX_TX_SEGS-2) | ||||
m_head = *m_headp; | return (0); | ||||
for (curseg = count = i = 0; i < nsegs; i++) { | |||||
/* | curseg += segs[i].ds_len; | ||||
* Important to capture the first descriptor | count++; | ||||
* used because it will contain the index of | if (__predict_false(count == IXL_MAX_TX_SEGS-2)) | ||||
* the one we tell the hardware to report back | return (1); | ||||
*/ | if (curseg > segsz) { | ||||
first = txr->next_avail; | curseg -= segsz; | ||||
buf = &txr->buffers[first]; | count = 1; | ||||
map = buf->map; | |||||
tag = txr->tx_tag; | |||||
if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | |||||
/* Use larger mapping for TSO */ | |||||
tag = txr->tso_tag; | |||||
if (ixl_tso_detect_sparse(m_head)) { | |||||
m = m_defrag(m_head, M_NOWAIT); | |||||
if (m == NULL) { | |||||
m_freem(*m_headp); | |||||
*m_headp = NULL; | |||||
return (ENOBUFS); | |||||
} | } | ||||
*m_headp = m; | if (curseg == segsz) | ||||
curseg = count = 0; | |||||
} | } | ||||
} | |||||
/* | |||||
* Map the packet for DMA. | |||||
*/ | |||||
error = bus_dmamap_load_mbuf_sg(tag, map, | |||||
*m_headp, segs, &nsegs, BUS_DMA_NOWAIT); | |||||
if (error == EFBIG) { | |||||
struct mbuf *m; | |||||
m = m_defrag(*m_headp, M_NOWAIT); | |||||
if (m == NULL) { | |||||
que->mbuf_defrag_failed++; | |||||
m_freem(*m_headp); | |||||
*m_headp = NULL; | |||||
return (ENOBUFS); | |||||
} | |||||
*m_headp = m; | |||||
/* Try it again */ | |||||
error = bus_dmamap_load_mbuf_sg(tag, map, | |||||
*m_headp, segs, &nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) { | |||||
que->tx_dmamap_failed++; | |||||
m_freem(*m_headp); | |||||
*m_headp = NULL; | |||||
return (error); | |||||
} | |||||
} else if (error != 0) { | |||||
que->tx_dmamap_failed++; | |||||
m_freem(*m_headp); | |||||
*m_headp = NULL; | |||||
return (error); | |||||
} | |||||
/* Make certain there are enough descriptors */ | |||||
if (nsegs > txr->avail - 2) { | |||||
txr->no_desc++; | |||||
error = ENOBUFS; | |||||
goto xmit_fail; | |||||
} | |||||
m_head = *m_headp; | |||||
/* Set up the TSO/CSUM offload */ | |||||
if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { | |||||
error = ixl_tx_setup_offload(que, m_head, &cmd, &off); | |||||
if (error) | |||||
goto xmit_fail; | |||||
} | |||||
cmd |= I40E_TX_DESC_CMD_ICRC; | |||||
/* Grab the VLAN tag */ | |||||
if (m_head->m_flags & M_VLANTAG) { | |||||
cmd |= I40E_TX_DESC_CMD_IL2TAG1; | |||||
vtag = htole16(m_head->m_pkthdr.ether_vtag); | |||||
} | |||||
i = txr->next_avail; | |||||
for (j = 0; j < nsegs; j++) { | |||||
bus_size_t seglen; | |||||
buf = &txr->buffers[i]; | |||||
buf->tag = tag; /* Keep track of the type tag */ | |||||
txd = &txr->base[i]; | |||||
seglen = segs[j].ds_len; | |||||
txd->buffer_addr = htole64(segs[j].ds_addr); | |||||
txd->cmd_type_offset_bsz = | |||||
htole64(I40E_TX_DESC_DTYPE_DATA | |||||
| ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) | |||||
| ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) | |||||
| ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | |||||
| ((u64)vtag << I40E_TXD_QW1_L2TAG1_SHIFT)); | |||||
last = i; /* descriptor that will get completion IRQ */ | |||||
if (++i == que->num_desc) | |||||
i = 0; | |||||
buf->m_head = NULL; | |||||
buf->eop_index = -1; | |||||
} | |||||
/* Set the last descriptor for report */ | |||||
txd->cmd_type_offset_bsz |= | |||||
htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); | |||||
txr->avail -= nsegs; | |||||
txr->next_avail = i; | |||||
buf->m_head = m_head; | |||||
/* Swap the dma map between the first and last descriptor */ | |||||
txr->buffers[first].map = buf->map; | |||||
buf->map = map; | |||||
bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE); | |||||
/* Set the index of the descriptor that will be marked done */ | |||||
buf = &txr->buffers[first]; | |||||
buf->eop_index = last; | |||||
bus_dmamap_sync(txr->dma.tag, txr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
/* | |||||
* Advance the Transmit Descriptor Tail (Tdt), this tells the | |||||
* hardware that this frame is available to transmit. | |||||
*/ | |||||
++txr->total_packets; | |||||
wr32(hw, txr->tail, i); | |||||
/* Mark outstanding work */ | |||||
atomic_store_rel_32(&txr->watchdog_timer, IXL_WATCHDOG); | |||||
return (0); | return (0); | ||||
xmit_fail: | |||||
bus_dmamap_unload(tag, buf->map); | |||||
return (error); | |||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Allocate memory for tx_buffer structures. The tx_buffer stores all | |||||
* the information needed to transmit a packet on the wire. This is | |||||
* called only once at attach, setup is done every reset. | |||||
* | |||||
**********************************************************************/ | |||||
int | |||||
ixl_allocate_tx_data(struct ixl_queue *que) | |||||
{ | |||||
struct tx_ring *txr = &que->txr; | |||||
struct ixl_vsi *vsi = que->vsi; | |||||
device_t dev = vsi->dev; | |||||
struct ixl_tx_buf *buf; | |||||
int error = 0; | |||||
/* | |||||
* Setup DMA descriptor areas. | |||||
*/ | |||||
if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | |||||
1, 0, /* alignment, bounds */ | |||||
BUS_SPACE_MAXADDR, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filter, filterarg */ | |||||
IXL_TSO_SIZE, /* maxsize */ | |||||
IXL_MAX_TX_SEGS, /* nsegments */ | |||||
PAGE_SIZE, /* maxsegsize */ | |||||
0, /* flags */ | |||||
NULL, /* lockfunc */ | |||||
NULL, /* lockfuncarg */ | |||||
&txr->tx_tag))) { | |||||
device_printf(dev,"Unable to allocate TX DMA tag\n"); | |||||
goto fail; | |||||
} | |||||
/* Make a special tag for TSO */ | |||||
if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | |||||
1, 0, /* alignment, bounds */ | |||||
BUS_SPACE_MAXADDR, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filter, filterarg */ | |||||
IXL_TSO_SIZE, /* maxsize */ | |||||
IXL_MAX_TSO_SEGS, /* nsegments */ | |||||
PAGE_SIZE, /* maxsegsize */ | |||||
0, /* flags */ | |||||
NULL, /* lockfunc */ | |||||
NULL, /* lockfuncarg */ | |||||
&txr->tso_tag))) { | |||||
device_printf(dev,"Unable to allocate TX TSO DMA tag\n"); | |||||
goto fail; | |||||
} | |||||
if (!(txr->buffers = | |||||
(struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) * | |||||
que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { | |||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | |||||
error = ENOMEM; | |||||
goto fail; | |||||
} | |||||
/* Create the descriptor buffer default dma maps */ | |||||
buf = txr->buffers; | |||||
for (int i = 0; i < que->num_desc; i++, buf++) { | |||||
buf->tag = txr->tx_tag; | |||||
error = bus_dmamap_create(buf->tag, 0, &buf->map); | |||||
if (error != 0) { | |||||
device_printf(dev, "Unable to create TX DMA map\n"); | |||||
goto fail; | |||||
} | |||||
} | |||||
fail: | |||||
return (error); | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* (Re)Initialize a queue transmit ring. | |||||
* - called by init, it clears the descriptor ring, | |||||
* and frees any stale mbufs | |||||
* | |||||
**********************************************************************/ | |||||
void | |||||
ixl_init_tx_ring(struct ixl_queue *que) | |||||
{ | |||||
#ifdef DEV_NETMAP | |||||
struct netmap_adapter *na = NA(que->vsi->ifp); | |||||
struct netmap_slot *slot; | |||||
#endif /* DEV_NETMAP */ | |||||
struct tx_ring *txr = &que->txr; | |||||
struct ixl_tx_buf *buf; | |||||
/* Clear the old ring contents */ | |||||
IXL_TX_LOCK(txr); | |||||
#ifdef DEV_NETMAP | |||||
/* | |||||
* (under lock): if in netmap mode, do some consistency | |||||
* checks and set slot to entry 0 of the netmap ring. | |||||
*/ | |||||
slot = netmap_reset(na, NR_TX, que->me, 0); | |||||
#endif /* DEV_NETMAP */ | |||||
bzero((void *)txr->base, | |||||
(sizeof(struct i40e_tx_desc)) * que->num_desc); | |||||
/* Reset indices */ | |||||
txr->next_avail = 0; | |||||
txr->next_to_clean = 0; | |||||
/* Reset watchdog status */ | |||||
txr->watchdog_timer = 0; | |||||
#ifdef IXL_FDIR | |||||
/* Initialize flow director */ | |||||
txr->atr_rate = ixl_atr_rate; | |||||
txr->atr_count = 0; | |||||
#endif | |||||
/* Free any existing tx mbufs. */ | |||||
buf = txr->buffers; | |||||
for (int i = 0; i < que->num_desc; i++, buf++) { | |||||
if (buf->m_head != NULL) { | |||||
bus_dmamap_sync(buf->tag, buf->map, | |||||
BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_unload(buf->tag, buf->map); | |||||
m_freem(buf->m_head); | |||||
buf->m_head = NULL; | |||||
} | |||||
#ifdef DEV_NETMAP | |||||
/* | |||||
* In netmap mode, set the map for the packet buffer. | |||||
* NOTE: Some drivers (not this one) also need to set | |||||
* the physical buffer address in the NIC ring. | |||||
* netmap_idx_n2k() maps a nic index, i, into the corresponding | |||||
* netmap slot index, si | |||||
*/ | |||||
if (slot) { | |||||
int si = netmap_idx_n2k(na->tx_rings[que->me], i); | |||||
netmap_load_map(na, buf->tag, buf->map, NMB(na, slot + si)); | |||||
} | |||||
#endif /* DEV_NETMAP */ | |||||
/* Clear the EOP index */ | |||||
buf->eop_index = -1; | |||||
} | |||||
/* Set number of descriptors available */ | |||||
txr->avail = que->num_desc; | |||||
bus_dmamap_sync(txr->dma.tag, txr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
IXL_TX_UNLOCK(txr); | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* Free transmit ring related data structures. | |||||
* | |||||
**********************************************************************/ | |||||
void | |||||
ixl_free_que_tx(struct ixl_queue *que) | |||||
{ | |||||
struct tx_ring *txr = &que->txr; | |||||
struct ixl_tx_buf *buf; | |||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); | |||||
for (int i = 0; i < que->num_desc; i++) { | |||||
buf = &txr->buffers[i]; | |||||
if (buf->m_head != NULL) { | |||||
bus_dmamap_sync(buf->tag, buf->map, | |||||
BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_unload(buf->tag, | |||||
buf->map); | |||||
m_freem(buf->m_head); | |||||
buf->m_head = NULL; | |||||
if (buf->map != NULL) { | |||||
bus_dmamap_destroy(buf->tag, | |||||
buf->map); | |||||
buf->map = NULL; | |||||
} | |||||
} else if (buf->map != NULL) { | |||||
bus_dmamap_unload(buf->tag, | |||||
buf->map); | |||||
bus_dmamap_destroy(buf->tag, | |||||
buf->map); | |||||
buf->map = NULL; | |||||
} | |||||
} | |||||
if (txr->br != NULL) | |||||
buf_ring_free(txr->br, M_DEVBUF); | |||||
if (txr->buffers != NULL) { | |||||
free(txr->buffers, M_DEVBUF); | |||||
txr->buffers = NULL; | |||||
} | |||||
if (txr->tx_tag != NULL) { | |||||
bus_dma_tag_destroy(txr->tx_tag); | |||||
txr->tx_tag = NULL; | |||||
} | |||||
if (txr->tso_tag != NULL) { | |||||
bus_dma_tag_destroy(txr->tso_tag); | |||||
txr->tso_tag = NULL; | |||||
} | |||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); | |||||
return; | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* Setup descriptor for hw offloads | * Setup descriptor for hw offloads | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static void | ||||
ixl_tx_setup_offload(struct ixl_queue *que, | ixl_tx_setup_offload(struct ixl_tx_queue *que, | ||||
struct mbuf *mp, u32 *cmd, u32 *off) | if_pkt_info_t pi, u32 *cmd, u32 *off) | ||||
{ | { | ||||
struct ether_vlan_header *eh; | switch (pi->ipi_etype) { | ||||
#ifdef INET | #ifdef INET | ||||
struct ip *ip = NULL; | |||||
#endif | |||||
struct tcphdr *th = NULL; | |||||
#ifdef INET6 | |||||
struct ip6_hdr *ip6; | |||||
#endif | |||||
int elen, ip_hlen = 0, tcp_hlen; | |||||
u16 etype; | |||||
u8 ipproto = 0; | |||||
bool tso = FALSE; | |||||
/* Set up the TSO context descriptor if required */ | |||||
if (mp->m_pkthdr.csum_flags & CSUM_TSO) { | |||||
tso = ixl_tso_setup(que, mp); | |||||
if (tso) | |||||
++que->tso; | |||||
else | |||||
return (ENXIO); | |||||
} | |||||
/* | |||||
* Determine where frame payload starts. | |||||
* Jump over vlan headers if already present, | |||||
* helpful for QinQ too. | |||||
*/ | |||||
eh = mtod(mp, struct ether_vlan_header *); | |||||
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | |||||
etype = ntohs(eh->evl_proto); | |||||
elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | |||||
} else { | |||||
etype = ntohs(eh->evl_encap_proto); | |||||
elen = ETHER_HDR_LEN; | |||||
} | |||||
switch (etype) { | |||||
#ifdef INET | |||||
case ETHERTYPE_IP: | case ETHERTYPE_IP: | ||||
ip = (struct ip *)(mp->m_data + elen); | if (pi->ipi_csum_flags & CSUM_IP) | ||||
ip_hlen = ip->ip_hl << 2; | |||||
ipproto = ip->ip_p; | |||||
th = (struct tcphdr *)((caddr_t)ip + ip_hlen); | |||||
/* The IP checksum must be recalculated with TSO */ | |||||
if (tso) | |||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; | *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; | ||||
else | else | ||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; | *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; | ||||
break; | break; | ||||
#endif | #endif | ||||
#ifdef INET6 | #ifdef INET6 | ||||
case ETHERTYPE_IPV6: | case ETHERTYPE_IPV6: | ||||
ip6 = (struct ip6_hdr *)(mp->m_data + elen); | |||||
ip_hlen = sizeof(struct ip6_hdr); | |||||
ipproto = ip6->ip6_nxt; | |||||
th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); | |||||
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; | *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; | ||||
break; | break; | ||||
#endif | #endif | ||||
default: | default: | ||||
break; | break; | ||||
} | } | ||||
*off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; | *off |= (pi->ipi_ehdrlen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; | ||||
*off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; | *off |= (pi->ipi_ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; | ||||
switch (ipproto) { | switch (pi->ipi_ipproto) { | ||||
case IPPROTO_TCP: | case IPPROTO_TCP: | ||||
tcp_hlen = th->th_off << 2; | if (pi->ipi_csum_flags & IXL_CSUM_TCP) { | ||||
if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) { | |||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; | *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; | ||||
*off |= (tcp_hlen >> 2) << | *off |= (pi->ipi_tcp_hlen >> 2) << | ||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | ||||
} | } | ||||
#ifdef IXL_FDIR | |||||
ixl_atr(que, th, etype); | |||||
#endif | |||||
break; | break; | ||||
case IPPROTO_UDP: | case IPPROTO_UDP: | ||||
if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) { | if (pi->ipi_csum_flags & IXL_CSUM_UDP) { | ||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; | *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; | ||||
*off |= (sizeof(struct udphdr) >> 2) << | *off |= (sizeof(struct udphdr) >> 2) << | ||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | ||||
} | } | ||||
break; | break; | ||||
case IPPROTO_SCTP: | case IPPROTO_SCTP: | ||||
if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) { | if (pi->ipi_csum_flags & IXL_CSUM_SCTP) { | ||||
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; | *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; | ||||
*off |= (sizeof(struct sctphdr) >> 2) << | *off |= (sizeof(struct sctphdr) >> 2) << | ||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | ||||
} | } | ||||
/* Fall Thru */ | /* Fall Thru */ | ||||
default: | default: | ||||
break; | break; | ||||
} | } | ||||
return (0); | |||||
} | } | ||||
/********************************************************************** | /********************************************************************** | ||||
* | * | ||||
* Setup context for hardware segmentation offload (TSO) | * Setup context for hardware segmentation offload (TSO) | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static bool | static int | ||||
ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp) | ixl_tso_setup(struct tx_ring *txr, if_pkt_info_t pi) | ||||
{ | { | ||||
struct tx_ring *txr = &que->txr; | if_softc_ctx_t scctx; | ||||
struct i40e_tx_context_desc *TXD; | struct i40e_tx_context_desc *TXD; | ||||
struct ixl_tx_buf *buf; | |||||
u32 cmd, mss, type, tsolen; | u32 cmd, mss, type, tsolen; | ||||
u16 etype; | int idx; | ||||
int idx, elen, ip_hlen, tcp_hlen; | |||||
struct ether_vlan_header *eh; | |||||
#ifdef INET | |||||
struct ip *ip; | |||||
#endif | |||||
#ifdef INET6 | |||||
struct ip6_hdr *ip6; | |||||
#endif | |||||
#if defined(INET6) || defined(INET) | |||||
struct tcphdr *th; | |||||
#endif | |||||
u64 type_cmd_tso_mss; | u64 type_cmd_tso_mss; | ||||
/* | idx = pi->ipi_pidx; | ||||
* Determine where frame payload starts. | TXD = (struct i40e_tx_context_desc *) &txr->tx_base[idx]; | ||||
* Jump over vlan headers if already present | tsolen = pi->ipi_len - (pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen); | ||||
*/ | scctx = txr->que->vsi->shared; | ||||
eh = mtod(mp, struct ether_vlan_header *); | |||||
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | |||||
elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | |||||
etype = eh->evl_proto; | |||||
} else { | |||||
elen = ETHER_HDR_LEN; | |||||
etype = eh->evl_encap_proto; | |||||
} | |||||
switch (ntohs(etype)) { | |||||
#ifdef INET6 | |||||
case ETHERTYPE_IPV6: | |||||
ip6 = (struct ip6_hdr *)(mp->m_data + elen); | |||||
if (ip6->ip6_nxt != IPPROTO_TCP) | |||||
return (ENXIO); | |||||
ip_hlen = sizeof(struct ip6_hdr); | |||||
th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); | |||||
th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); | |||||
tcp_hlen = th->th_off << 2; | |||||
/* | |||||
* The corresponding flag is set by the stack in the IPv4 | |||||
* TSO case, but not in IPv6 (at least in FreeBSD 10.2). | |||||
* So, set it here because the rest of the flow requires it. | |||||
*/ | |||||
mp->m_pkthdr.csum_flags |= CSUM_TCP_IPV6; | |||||
break; | |||||
#endif | |||||
#ifdef INET | |||||
case ETHERTYPE_IP: | |||||
ip = (struct ip *)(mp->m_data + elen); | |||||
if (ip->ip_p != IPPROTO_TCP) | |||||
return (ENXIO); | |||||
ip->ip_sum = 0; | |||||
ip_hlen = ip->ip_hl << 2; | |||||
th = (struct tcphdr *)((caddr_t)ip + ip_hlen); | |||||
th->th_sum = in_pseudo(ip->ip_src.s_addr, | |||||
ip->ip_dst.s_addr, htons(IPPROTO_TCP)); | |||||
tcp_hlen = th->th_off << 2; | |||||
break; | |||||
#endif | |||||
default: | |||||
printf("%s: CSUM_TSO but no supported IP version (0x%04x)", | |||||
__func__, ntohs(etype)); | |||||
return FALSE; | |||||
} | |||||
/* Ensure we have at least the IP+TCP header in the first mbuf. */ | |||||
if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr)) | |||||
return FALSE; | |||||
idx = txr->next_avail; | |||||
buf = &txr->buffers[idx]; | |||||
TXD = (struct i40e_tx_context_desc *) &txr->base[idx]; | |||||
tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen); | |||||
type = I40E_TX_DESC_DTYPE_CONTEXT; | type = I40E_TX_DESC_DTYPE_CONTEXT; | ||||
cmd = I40E_TX_CTX_DESC_TSO; | cmd = I40E_TX_CTX_DESC_TSO; | ||||
/* TSO MSS must not be less than 64 */ | /* TSO MSS must not be less than 64 */ | ||||
if (mp->m_pkthdr.tso_segsz < IXL_MIN_TSO_MSS) { | if (pi->ipi_tso_segsz < IXL_MIN_TSO_MSS) { | ||||
que->mss_too_small++; | txr->mss_too_small++; | ||||
mp->m_pkthdr.tso_segsz = IXL_MIN_TSO_MSS; | pi->ipi_tso_segsz = IXL_MIN_TSO_MSS; | ||||
} | } | ||||
mss = mp->m_pkthdr.tso_segsz; | mss = pi->ipi_tso_segsz; | ||||
type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | | type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | | ||||
((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | | ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | | ||||
((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | | ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | | ||||
((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); | ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); | ||||
TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); | TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); | ||||
TXD->tunneling_params = htole32(0); | TXD->tunneling_params = htole32(0); | ||||
buf->m_head = NULL; | |||||
buf->eop_index = -1; | |||||
if (++idx == que->num_desc) | return ((idx + 1) & (scctx->isc_ntxd[0]-1)); | ||||
idx = 0; | |||||
txr->avail--; | |||||
txr->next_avail = idx; | |||||
return TRUE; | |||||
} | } | ||||
/* | /********************************************************************* | ||||
** ixl_get_tx_head - Retrieve the value from the | |||||
** location the HW records its HEAD index | |||||
*/ | |||||
static inline u32 | |||||
ixl_get_tx_head(struct ixl_queue *que) | |||||
{ | |||||
struct tx_ring *txr = &que->txr; | |||||
void *head = &txr->base[que->num_desc]; | |||||
return LE32_TO_CPU(*(volatile __le32 *)head); | |||||
} | |||||
/********************************************************************** | |||||
* | * | ||||
* Examine each tx_buffer in the used queue. If the hardware is done | * This routine maps the mbufs to tx descriptors, allowing the | ||||
* processing the packet then free associated resources. The | * TX engine to transmit the packets. | ||||
* tx_buffer is put back on the free queue. | * - return 0 on success, positive on failure | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
bool | #define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) | ||||
ixl_txeof(struct ixl_queue *que) | |||||
static int | |||||
ixl_isc_txd_encap(void *arg, if_pkt_info_t pi) | |||||
{ | { | ||||
struct ixl_vsi *vsi = arg; | |||||
if_softc_ctx_t scctx = vsi->shared; | |||||
struct ixl_tx_queue *que = &vsi->tx_queues[pi->ipi_qsidx]; | |||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
u32 first, last, head, done, processed; | int nsegs = pi->ipi_nsegs; | ||||
struct ixl_tx_buf *buf; | bus_dma_segment_t *segs = pi->ipi_segs; | ||||
struct i40e_tx_desc *tx_desc, *eop_desc; | struct i40e_tx_desc *txd = NULL; | ||||
int i, j, mask, pidx_last; | |||||
u32 cmd, off, tx_intr; | |||||
// device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); | |||||
mtx_assert(&txr->mtx, MA_OWNED); | cmd = off = 0; | ||||
i = pi->ipi_pidx; | |||||
#ifdef DEV_NETMAP | tx_intr = (pi->ipi_flags & IPI_TX_INTR); | ||||
// XXX todo: implement moderation | #if 0 | ||||
if (netmap_tx_irq(que->vsi->ifp, que->me)) | device_printf(iflib_get_dev(vsi->ctx), "%s: tx_intr %d\n", __func__, tx_intr); | ||||
return FALSE; | #endif | ||||
#endif /* DEF_NETMAP */ | |||||
/* These are not the descriptors you seek, move along :) */ | /* Set up the TSO/CSUM offload */ | ||||
if (txr->avail == que->num_desc) { | if (pi->ipi_csum_flags & CSUM_OFFLOAD) { | ||||
atomic_store_rel_32(&txr->watchdog_timer, 0); | /* Set up the TSO context descriptor if required */ | ||||
return FALSE; | if (pi->ipi_csum_flags & CSUM_TSO) { | ||||
if (ixl_tso_detect_sparse(segs, nsegs, pi->ipi_tso_segsz)) | |||||
return (EFBIG); | |||||
i = ixl_tso_setup(txr, pi); | |||||
} | } | ||||
ixl_tx_setup_offload(que, pi, &cmd, &off); | |||||
} | |||||
if (pi->ipi_mflags & M_VLANTAG) | |||||
cmd |= I40E_TX_DESC_CMD_IL2TAG1; | |||||
processed = 0; | cmd |= I40E_TX_DESC_CMD_ICRC; | ||||
first = txr->next_to_clean; | mask = scctx->isc_ntxd[0] - 1; | ||||
buf = &txr->buffers[first]; | for (j = 0; j < nsegs; j++) { | ||||
tx_desc = (struct i40e_tx_desc *)&txr->base[first]; | bus_size_t seglen; | ||||
last = buf->eop_index; | |||||
if (last == -1) | |||||
return FALSE; | |||||
eop_desc = (struct i40e_tx_desc *)&txr->base[last]; | |||||
/* Get the Head WB value */ | txd = &txr->tx_base[i]; | ||||
head = ixl_get_tx_head(que); | seglen = segs[j].ds_len; | ||||
/* | txd->buffer_addr = htole64(segs[j].ds_addr); | ||||
** Get the index of the first descriptor | txd->cmd_type_offset_bsz = | ||||
** BEYOND the EOP and call that 'done'. | htole64(I40E_TX_DESC_DTYPE_DATA | ||||
** I do this so the comparison in the | | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) | ||||
** inner while loop below can be simple | | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) | ||||
*/ | | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | ||||
if (++last == que->num_desc) last = 0; | | ((u64)htole16(pi->ipi_vtag) << I40E_TXD_QW1_L2TAG1_SHIFT)); | ||||
done = last; | |||||
bus_dmamap_sync(txr->dma.tag, txr->dma.map, | txr->tx_bytes += seglen; | ||||
BUS_DMASYNC_POSTREAD); | pidx_last = i; | ||||
/* | i = (i+1) & mask; | ||||
** The HEAD index of the ring is written in a | |||||
** defined location, this rather than a done bit | |||||
** is what is used to keep track of what must be | |||||
** 'cleaned'. | |||||
*/ | |||||
while (first != head) { | |||||
/* We clean the range of the packet */ | |||||
while (first != done) { | |||||
++txr->avail; | |||||
++processed; | |||||
if (buf->m_head) { | |||||
txr->bytes += /* for ITR adjustment */ | |||||
buf->m_head->m_pkthdr.len; | |||||
txr->tx_bytes += /* for TX stats */ | |||||
buf->m_head->m_pkthdr.len; | |||||
bus_dmamap_sync(buf->tag, | |||||
buf->map, | |||||
BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_unload(buf->tag, | |||||
buf->map); | |||||
m_freem(buf->m_head); | |||||
buf->m_head = NULL; | |||||
} | } | ||||
buf->eop_index = -1; | /* Set the last descriptor for report */ | ||||
txd->cmd_type_offset_bsz |= | |||||
htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); | |||||
/* Add to report status array (if using TX interrupts) */ | |||||
if (!vsi->enable_head_writeback && tx_intr) { | |||||
txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; | |||||
txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & mask; | |||||
MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); | |||||
} | |||||
pi->ipi_new_pidx = i; | |||||
if (++first == que->num_desc) | ++txr->tx_packets; | ||||
first = 0; | return (0); | ||||
buf = &txr->buffers[first]; | |||||
tx_desc = &txr->base[first]; | |||||
} | } | ||||
++txr->packets; | |||||
/* See if there is more work now */ | |||||
last = buf->eop_index; | |||||
if (last != -1) { | |||||
eop_desc = &txr->base[last]; | |||||
/* Get next done point */ | |||||
if (++last == que->num_desc) last = 0; | |||||
done = last; | |||||
} else | |||||
break; | |||||
} | |||||
bus_dmamap_sync(txr->dma.tag, txr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
txr->next_to_clean = first; | static void | ||||
ixl_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) | |||||
{ | |||||
struct ixl_vsi *vsi = arg; | |||||
struct tx_ring *txr = &vsi->tx_queues[txqid].txr; | |||||
// device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); | |||||
/* | /* | ||||
* If there are no pending descriptors, clear the timeout. | * Advance the Transmit Descriptor Tail (Tdt), this tells the | ||||
* hardware that this frame is available to transmit. | |||||
*/ | */ | ||||
if (txr->avail == que->num_desc) { | wr32(vsi->hw, txr->tail, pidx); | ||||
atomic_store_rel_32(&txr->watchdog_timer, 0); | |||||
return FALSE; | |||||
} | } | ||||
return TRUE; | |||||
} | |||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Refresh mbuf buffers for RX descriptor rings | * (Re)Initialize a queue transmit ring. | ||||
* - now keeps its own state so discards due to resource | * - called by init, it clears the descriptor ring, | ||||
* exhaustion are unnecessary, if an mbuf cannot be obtained | * and frees any stale mbufs | ||||
* it just returns, keeping its placeholder, thus it can simply | |||||
* be recalled to try again. | |||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | void | ||||
ixl_refresh_mbufs(struct ixl_queue *que, int limit) | ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que) | ||||
{ | { | ||||
struct ixl_vsi *vsi = que->vsi; | struct tx_ring *txr = &que->txr; | ||||
struct rx_ring *rxr = &que->rxr; | |||||
bus_dma_segment_t hseg[1]; | |||||
bus_dma_segment_t pseg[1]; | |||||
struct ixl_rx_buf *buf; | |||||
struct mbuf *mh, *mp; | |||||
int i, j, nsegs, error; | |||||
bool refreshed = FALSE; | |||||
i = j = rxr->next_refresh; | // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); | ||||
/* Control the loop with one beyond */ | |||||
if (++j == que->num_desc) | |||||
j = 0; | |||||
while (j != limit) { | /* Clear the old ring contents */ | ||||
buf = &rxr->buffers[i]; | bzero((void *)txr->tx_base, | ||||
if (rxr->hdr_split == FALSE) | (sizeof(struct i40e_tx_desc)) * vsi->shared->isc_ntxd[0]); | ||||
goto no_split; | |||||
if (buf->m_head == NULL) { | // TODO: Write max descriptor index instead of 0? | ||||
mh = m_gethdr(M_NOWAIT, MT_DATA); | wr32(vsi->hw, txr->tail, 0); | ||||
if (mh == NULL) | wr32(vsi->hw, I40E_QTX_HEAD(txr->me), 0); | ||||
goto update; | |||||
} else | |||||
mh = buf->m_head; | |||||
mh->m_pkthdr.len = mh->m_len = MHLEN; | |||||
mh->m_len = MHLEN; | |||||
mh->m_flags |= M_PKTHDR; | |||||
/* Get the memory mapping */ | |||||
error = bus_dmamap_load_mbuf_sg(rxr->htag, | |||||
buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) { | |||||
printf("Refresh mbufs: hdr dmamap load" | |||||
" failure - %d\n", error); | |||||
m_free(mh); | |||||
buf->m_head = NULL; | |||||
goto update; | |||||
} | } | ||||
buf->m_head = mh; | |||||
bus_dmamap_sync(rxr->htag, buf->hmap, | |||||
BUS_DMASYNC_PREREAD); | |||||
rxr->base[i].read.hdr_addr = | |||||
htole64(hseg[0].ds_addr); | |||||
no_split: | /* | ||||
if (buf->m_pack == NULL) { | * ixl_get_tx_head - Retrieve the value from the | ||||
mp = m_getjcl(M_NOWAIT, MT_DATA, | * location the HW records its HEAD index | ||||
M_PKTHDR, rxr->mbuf_sz); | */ | ||||
if (mp == NULL) | static inline u32 | ||||
goto update; | ixl_get_tx_head(struct ixl_tx_queue *que) | ||||
} else | { | ||||
mp = buf->m_pack; | if_softc_ctx_t scctx = que->vsi->shared; | ||||
struct tx_ring *txr = &que->txr; | |||||
void *head = &txr->tx_base[scctx->isc_ntxd[0]]; | |||||
mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; | return LE32_TO_CPU(*(volatile __le32 *)head); | ||||
/* Get the memory mapping */ | |||||
error = bus_dmamap_load_mbuf_sg(rxr->ptag, | |||||
buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) { | |||||
printf("Refresh mbufs: payload dmamap load" | |||||
" failure - %d\n", error); | |||||
m_free(mp); | |||||
buf->m_pack = NULL; | |||||
goto update; | |||||
} | } | ||||
buf->m_pack = mp; | |||||
bus_dmamap_sync(rxr->ptag, buf->pmap, | |||||
BUS_DMASYNC_PREREAD); | |||||
rxr->base[i].read.pkt_addr = | |||||
htole64(pseg[0].ds_addr); | |||||
/* Used only when doing header split */ | |||||
rxr->base[i].read.hdr_addr = 0; | |||||
refreshed = TRUE; | static int | ||||
/* Next is precalculated */ | ixl_isc_txd_credits_update_hwb(void *arg, uint16_t qid, bool clear) | ||||
i = j; | |||||
rxr->next_refresh = i; | |||||
if (++j == que->num_desc) | |||||
j = 0; | |||||
} | |||||
update: | |||||
if (refreshed) /* Update hardware tail index */ | |||||
wr32(vsi->hw, rxr->tail, rxr->next_refresh); | |||||
return; | |||||
} | |||||
/********************************************************************* | |||||
* | |||||
* Allocate memory for rx_buffer structures. Since we use one | |||||
* rx_buffer per descriptor, the maximum number of rx_buffer's | |||||
* that we'll need is equal to the number of receive descriptors | |||||
* that we've defined. | |||||
* | |||||
**********************************************************************/ | |||||
int | |||||
ixl_allocate_rx_data(struct ixl_queue *que) | |||||
{ | { | ||||
struct rx_ring *rxr = &que->rxr; | struct ixl_vsi *vsi = arg; | ||||
struct ixl_vsi *vsi = que->vsi; | if_softc_ctx_t scctx = vsi->shared; | ||||
device_t dev = vsi->dev; | struct ixl_tx_queue *que = &vsi->tx_queues[qid]; | ||||
struct ixl_rx_buf *buf; | struct tx_ring *txr = &que->txr; | ||||
int i, bsize, error; | int head, credits; | ||||
bsize = sizeof(struct ixl_rx_buf) * que->num_desc; | /* Get the Head WB value */ | ||||
if (!(rxr->buffers = | head = ixl_get_tx_head(que); | ||||
(struct ixl_rx_buf *) malloc(bsize, | |||||
M_DEVBUF, M_NOWAIT | M_ZERO))) { | |||||
device_printf(dev, "Unable to allocate rx_buffer memory\n"); | |||||
error = ENOMEM; | |||||
return (error); | |||||
} | |||||
if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | credits = head - txr->tx_cidx_processed; | ||||
1, 0, /* alignment, bounds */ | if (credits < 0) | ||||
BUS_SPACE_MAXADDR, /* lowaddr */ | credits += scctx->isc_ntxd[0]; | ||||
BUS_SPACE_MAXADDR, /* highaddr */ | if (clear) | ||||
NULL, NULL, /* filter, filterarg */ | txr->tx_cidx_processed = head; | ||||
MSIZE, /* maxsize */ | |||||
1, /* nsegments */ | |||||
MSIZE, /* maxsegsize */ | |||||
0, /* flags */ | |||||
NULL, /* lockfunc */ | |||||
NULL, /* lockfuncarg */ | |||||
&rxr->htag))) { | |||||
device_printf(dev, "Unable to create RX DMA htag\n"); | |||||
return (error); | |||||
} | |||||
if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | return (credits); | ||||
1, 0, /* alignment, bounds */ | |||||
BUS_SPACE_MAXADDR, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filter, filterarg */ | |||||
MJUM16BYTES, /* maxsize */ | |||||
1, /* nsegments */ | |||||
MJUM16BYTES, /* maxsegsize */ | |||||
0, /* flags */ | |||||
NULL, /* lockfunc */ | |||||
NULL, /* lockfuncarg */ | |||||
&rxr->ptag))) { | |||||
device_printf(dev, "Unable to create RX DMA ptag\n"); | |||||
return (error); | |||||
} | } | ||||
for (i = 0; i < que->num_desc; i++) { | static int | ||||
buf = &rxr->buffers[i]; | ixl_isc_txd_credits_update_dwb(void *arg, uint16_t txqid, bool clear) | ||||
error = bus_dmamap_create(rxr->htag, | { | ||||
BUS_DMA_NOWAIT, &buf->hmap); | struct ixl_vsi *vsi = arg; | ||||
if (error) { | struct ixl_tx_queue *tx_que = &vsi->tx_queues[txqid]; | ||||
device_printf(dev, "Unable to create RX head map\n"); | if_softc_ctx_t scctx = vsi->shared; | ||||
break; | struct tx_ring *txr = &tx_que->txr; | ||||
} | |||||
error = bus_dmamap_create(rxr->ptag, | |||||
BUS_DMA_NOWAIT, &buf->pmap); | |||||
if (error) { | |||||
device_printf(dev, "Unable to create RX pkt map\n"); | |||||
break; | |||||
} | |||||
} | |||||
return (error); | qidx_t processed = 0; | ||||
} | qidx_t cur, prev, ntxd, rs_cidx; | ||||
int32_t delta; | |||||
bool is_done; | |||||
// device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); | |||||
/********************************************************************* | rs_cidx = txr->tx_rs_cidx; | ||||
* | #if 0 | ||||
* (Re)Initialize the queue receive ring and its buffers. | device_printf(iflib_get_dev(vsi->ctx), "%s: (q%d) rs_cidx %d, txr->tx_rs_pidx %d\n", __func__, | ||||
* | txr->me, rs_cidx, txr->tx_rs_pidx); | ||||
**********************************************************************/ | |||||
int | |||||
ixl_init_rx_ring(struct ixl_queue *que) | |||||
{ | |||||
struct rx_ring *rxr = &que->rxr; | |||||
struct ixl_vsi *vsi = que->vsi; | |||||
#if defined(INET6) || defined(INET) | |||||
struct ifnet *ifp = vsi->ifp; | |||||
struct lro_ctrl *lro = &rxr->lro; | |||||
#endif | #endif | ||||
struct ixl_rx_buf *buf; | if (rs_cidx == txr->tx_rs_pidx) | ||||
bus_dma_segment_t pseg[1], hseg[1]; | return (0); | ||||
int rsize, nsegs, error = 0; | cur = txr->tx_rsq[rs_cidx]; | ||||
#ifdef DEV_NETMAP | MPASS(cur != QIDX_INVALID); | ||||
struct netmap_adapter *na = NA(que->vsi->ifp); | is_done = ixl_is_tx_desc_done(txr, cur); | ||||
struct netmap_slot *slot; | |||||
#endif /* DEV_NETMAP */ | |||||
IXL_RX_LOCK(rxr); | if (clear == false || !is_done) | ||||
#ifdef DEV_NETMAP | return (0); | ||||
/* same as in ixl_init_tx_ring() */ | |||||
slot = netmap_reset(na, NR_RX, que->me, 0); | |||||
#endif /* DEV_NETMAP */ | |||||
/* Clear the ring contents */ | |||||
rsize = roundup2(que->num_desc * | |||||
sizeof(union i40e_rx_desc), DBA_ALIGN); | |||||
bzero((void *)rxr->base, rsize); | |||||
/* Cleanup any existing buffers */ | |||||
for (int i = 0; i < que->num_desc; i++) { | |||||
buf = &rxr->buffers[i]; | |||||
if (buf->m_head != NULL) { | |||||
bus_dmamap_sync(rxr->htag, buf->hmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->htag, buf->hmap); | |||||
buf->m_head->m_flags |= M_PKTHDR; | |||||
m_freem(buf->m_head); | |||||
} | |||||
if (buf->m_pack != NULL) { | |||||
bus_dmamap_sync(rxr->ptag, buf->pmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->ptag, buf->pmap); | |||||
buf->m_pack->m_flags |= M_PKTHDR; | |||||
m_freem(buf->m_pack); | |||||
} | |||||
buf->m_head = NULL; | |||||
buf->m_pack = NULL; | |||||
} | |||||
/* header split is off */ | prev = txr->tx_cidx_processed; | ||||
rxr->hdr_split = FALSE; | ntxd = scctx->isc_ntxd[0]; | ||||
do { | |||||
/* Now replenish the mbufs */ | delta = (int32_t)cur - (int32_t)prev; | ||||
for (int j = 0; j != que->num_desc; ++j) { | MPASS(prev == 0 || delta != 0); | ||||
struct mbuf *mh, *mp; | if (delta < 0) | ||||
delta += ntxd; | |||||
buf = &rxr->buffers[j]; | #if 0 | ||||
#ifdef DEV_NETMAP | device_printf(iflib_get_dev(vsi->ctx), | ||||
/* | "%s: (q%d) cidx_processed=%u cur=%u clear=%d delta=%d\n", | ||||
* In netmap mode, fill the map and set the buffer | __func__, txr->me, prev, cur, clear, delta); | ||||
* address in the NIC ring, considering the offset | |||||
* between the netmap and NIC rings (see comment in | |||||
* ixgbe_setup_transmit_ring() ). No need to allocate | |||||
* an mbuf, so end the block with a continue; | |||||
*/ | |||||
if (slot) { | |||||
int sj = netmap_idx_n2k(na->rx_rings[que->me], j); | |||||
uint64_t paddr; | |||||
void *addr; | |||||
addr = PNMB(na, slot + sj, &paddr); | |||||
netmap_load_map(na, rxr->dma.tag, buf->pmap, addr); | |||||
/* Update descriptor and the cached value */ | |||||
rxr->base[j].read.pkt_addr = htole64(paddr); | |||||
rxr->base[j].read.hdr_addr = 0; | |||||
continue; | |||||
} | |||||
#endif /* DEV_NETMAP */ | |||||
/* | |||||
** Don't allocate mbufs if not | |||||
** doing header split, its wasteful | |||||
*/ | |||||
if (rxr->hdr_split == FALSE) | |||||
goto skip_head; | |||||
/* First the header */ | |||||
buf->m_head = m_gethdr(M_NOWAIT, MT_DATA); | |||||
if (buf->m_head == NULL) { | |||||
error = ENOBUFS; | |||||
goto fail; | |||||
} | |||||
m_adj(buf->m_head, ETHER_ALIGN); | |||||
mh = buf->m_head; | |||||
mh->m_len = mh->m_pkthdr.len = MHLEN; | |||||
mh->m_flags |= M_PKTHDR; | |||||
/* Get the memory mapping */ | |||||
error = bus_dmamap_load_mbuf_sg(rxr->htag, | |||||
buf->hmap, buf->m_head, hseg, | |||||
&nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) /* Nothing elegant to do here */ | |||||
goto fail; | |||||
bus_dmamap_sync(rxr->htag, | |||||
buf->hmap, BUS_DMASYNC_PREREAD); | |||||
/* Update descriptor */ | |||||
rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr); | |||||
skip_head: | |||||
/* Now the payload cluster */ | |||||
buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, | |||||
M_PKTHDR, rxr->mbuf_sz); | |||||
if (buf->m_pack == NULL) { | |||||
error = ENOBUFS; | |||||
goto fail; | |||||
} | |||||
mp = buf->m_pack; | |||||
mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; | |||||
/* Get the memory mapping */ | |||||
error = bus_dmamap_load_mbuf_sg(rxr->ptag, | |||||
buf->pmap, mp, pseg, | |||||
&nsegs, BUS_DMA_NOWAIT); | |||||
if (error != 0) | |||||
goto fail; | |||||
bus_dmamap_sync(rxr->ptag, | |||||
buf->pmap, BUS_DMASYNC_PREREAD); | |||||
/* Update descriptor */ | |||||
rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr); | |||||
rxr->base[j].read.hdr_addr = 0; | |||||
} | |||||
/* Setup our descriptor indices */ | |||||
rxr->next_check = 0; | |||||
rxr->next_refresh = 0; | |||||
rxr->lro_enabled = FALSE; | |||||
rxr->split = 0; | |||||
rxr->bytes = 0; | |||||
rxr->discard = FALSE; | |||||
wr32(vsi->hw, rxr->tail, que->num_desc - 1); | |||||
ixl_flush(vsi->hw); | |||||
#if defined(INET6) || defined(INET) | |||||
/* | |||||
** Now set up the LRO interface: | |||||
*/ | |||||
if (ifp->if_capenable & IFCAP_LRO) { | |||||
int err = tcp_lro_init(lro); | |||||
if (err) { | |||||
if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me); | |||||
goto fail; | |||||
} | |||||
INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me); | |||||
rxr->lro_enabled = TRUE; | |||||
lro->ifp = vsi->ifp; | |||||
} | |||||
#endif | #endif | ||||
processed += delta; | |||||
prev = cur; | |||||
rs_cidx = (rs_cidx + 1) & (ntxd-1); | |||||
if (rs_cidx == txr->tx_rs_pidx) | |||||
break; | |||||
cur = txr->tx_rsq[rs_cidx]; | |||||
MPASS(cur != QIDX_INVALID); | |||||
is_done = ixl_is_tx_desc_done(txr, cur); | |||||
} while (is_done); | |||||
bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, | txr->tx_rs_cidx = rs_cidx; | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | txr->tx_cidx_processed = prev; | ||||
fail: | #if 0 | ||||
IXL_RX_UNLOCK(rxr); | device_printf(iflib_get_dev(vsi->ctx), "%s: (q%d) processed %d\n", __func__, txr->me, processed); | ||||
return (error); | #endif | ||||
return (processed); | |||||
} | } | ||||
static void | |||||
/********************************************************************* | ixl_isc_rxd_refill(void *arg, if_rxd_update_t iru) | ||||
* | |||||
* Free station receive ring data structures | |||||
* | |||||
**********************************************************************/ | |||||
void | |||||
ixl_free_que_rx(struct ixl_queue *que) | |||||
{ | { | ||||
struct rx_ring *rxr = &que->rxr; | struct ixl_vsi *vsi = arg; | ||||
struct ixl_rx_buf *buf; | if_softc_ctx_t scctx = vsi->shared; | ||||
struct rx_ring *rxr = &((vsi->rx_queues[iru->iru_qsidx]).rxr); | |||||
uint64_t *paddrs; | |||||
uint32_t next_pidx, pidx; | |||||
uint16_t count; | |||||
int i; | |||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); | paddrs = iru->iru_paddrs; | ||||
pidx = iru->iru_pidx; | |||||
count = iru->iru_count; | |||||
/* Cleanup any existing buffers */ | for (i = 0, next_pidx = pidx; i < count; i++) { | ||||
if (rxr->buffers != NULL) { | rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); | ||||
for (int i = 0; i < que->num_desc; i++) { | if (++next_pidx == scctx->isc_nrxd[0]) | ||||
buf = &rxr->buffers[i]; | next_pidx = 0; | ||||
if (buf->m_head != NULL) { | |||||
bus_dmamap_sync(rxr->htag, buf->hmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->htag, buf->hmap); | |||||
buf->m_head->m_flags |= M_PKTHDR; | |||||
m_freem(buf->m_head); | |||||
} | } | ||||
if (buf->m_pack != NULL) { | |||||
bus_dmamap_sync(rxr->ptag, buf->pmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->ptag, buf->pmap); | |||||
buf->m_pack->m_flags |= M_PKTHDR; | |||||
m_freem(buf->m_pack); | |||||
} | } | ||||
buf->m_head = NULL; | |||||
buf->m_pack = NULL; | |||||
if (buf->hmap != NULL) { | |||||
bus_dmamap_destroy(rxr->htag, buf->hmap); | |||||
buf->hmap = NULL; | |||||
} | |||||
if (buf->pmap != NULL) { | |||||
bus_dmamap_destroy(rxr->ptag, buf->pmap); | |||||
buf->pmap = NULL; | |||||
} | |||||
} | |||||
if (rxr->buffers != NULL) { | |||||
free(rxr->buffers, M_DEVBUF); | |||||
rxr->buffers = NULL; | |||||
} | |||||
} | |||||
if (rxr->htag != NULL) { | static void | ||||
bus_dma_tag_destroy(rxr->htag); | ixl_isc_rxd_flush(void * arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) | ||||
rxr->htag = NULL; | |||||
} | |||||
if (rxr->ptag != NULL) { | |||||
bus_dma_tag_destroy(rxr->ptag); | |||||
rxr->ptag = NULL; | |||||
} | |||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); | |||||
return; | |||||
} | |||||
static inline void | |||||
ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) | |||||
{ | { | ||||
struct ixl_vsi *vsi = arg; | |||||
struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; | |||||
#if defined(INET6) || defined(INET) | wr32(vsi->hw, rxr->tail, pidx); | ||||
/* | |||||
* ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet | |||||
* should be computed by hardware. Also it should not have VLAN tag in | |||||
* ethernet header. | |||||
*/ | |||||
if (rxr->lro_enabled && | |||||
(ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && | |||||
(m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == | |||||
(CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { | |||||
/* | |||||
* Send to the stack if: | |||||
** - LRO not enabled, or | |||||
** - no LRO resources, or | |||||
** - lro enqueue fails | |||||
*/ | |||||
if (rxr->lro.lro_cnt != 0) | |||||
if (tcp_lro_rx(&rxr->lro, m, 0) == 0) | |||||
return; | |||||
} | } | ||||
#endif | |||||
(*ifp->if_input)(ifp, m); | |||||
} | |||||
static int | |||||
static inline void | ixl_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) | ||||
ixl_rx_discard(struct rx_ring *rxr, int i) | |||||
{ | { | ||||
struct ixl_rx_buf *rbuf; | struct ixl_vsi *vsi = arg; | ||||
struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; | |||||
union i40e_rx_desc *rxd; | |||||
u64 qword; | |||||
uint32_t status; | |||||
int cnt, i, nrxd; | |||||
rbuf = &rxr->buffers[i]; | // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); | ||||
nrxd = vsi->shared->isc_nrxd[0]; | |||||
if (rbuf->fmp != NULL) {/* Partial chain ? */ | // Stolen from em | ||||
rbuf->fmp->m_flags |= M_PKTHDR; | if (budget == 1) { | ||||
m_freem(rbuf->fmp); | rxd = &rxr->rx_base[idx]; | ||||
rbuf->fmp = NULL; | qword = le64toh(rxd->wb.qword1.status_error_len); | ||||
status = (qword & I40E_RXD_QW1_STATUS_MASK) | |||||
>> I40E_RXD_QW1_STATUS_SHIFT; | |||||
return !!(status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)); | |||||
} | } | ||||
/* | for (cnt = 0, i = idx; cnt < nrxd - 1 && cnt <= budget;) { | ||||
** With advanced descriptors the writeback | rxd = &rxr->rx_base[i]; | ||||
** clobbers the buffer addrs, so its easier | qword = le64toh(rxd->wb.qword1.status_error_len); | ||||
** to just free the existing mbufs and take | status = (qword & I40E_RXD_QW1_STATUS_MASK) | ||||
** the normal refresh path to get new buffers | >> I40E_RXD_QW1_STATUS_SHIFT; | ||||
** and mapping. | |||||
*/ | |||||
if (rbuf->m_head) { | |||||
m_free(rbuf->m_head); | |||||
rbuf->m_head = NULL; | |||||
} | |||||
if (rbuf->m_pack) { | if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) | ||||
m_free(rbuf->m_pack); | break; | ||||
rbuf->m_pack = NULL; | if (++i == nrxd) | ||||
i = 0; | |||||
if (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)) | |||||
cnt++; | |||||
} | } | ||||
return; | return (cnt); | ||||
} | } | ||||
#ifdef RSS | |||||
/* | /* | ||||
** i40e_ptype_to_hash: parse the packet type | ** i40e_ptype_to_hash: parse the packet type | ||||
** to determine the appropriate hash. | ** to determine the appropriate hash. | ||||
*/ | */ | ||||
static inline int | static inline int | ||||
ixl_ptype_to_hash(u8 ptype) | ixl_ptype_to_hash(u8 ptype) | ||||
{ | { | ||||
struct i40e_rx_ptype_decoded decoded; | struct i40e_rx_ptype_decoded decoded; | ||||
decoded = decode_rx_desc_ptype(ptype); | decoded = decode_rx_desc_ptype(ptype); | ||||
if (!decoded.known) | if (!decoded.known) | ||||
return M_HASHTYPE_OPAQUE_HASH; | return M_HASHTYPE_OPAQUE; | ||||
if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2) | if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2) | ||||
return M_HASHTYPE_OPAQUE_HASH; | return M_HASHTYPE_OPAQUE; | ||||
/* Note: anything that gets to this point is IP */ | /* Note: anything that gets to this point is IP */ | ||||
if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { | if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { | ||||
switch (decoded.inner_prot) { | switch (decoded.inner_prot) { | ||||
case I40E_RX_PTYPE_INNER_PROT_TCP: | case I40E_RX_PTYPE_INNER_PROT_TCP: | ||||
return M_HASHTYPE_RSS_TCP_IPV6; | return M_HASHTYPE_RSS_TCP_IPV6; | ||||
case I40E_RX_PTYPE_INNER_PROT_UDP: | case I40E_RX_PTYPE_INNER_PROT_UDP: | ||||
return M_HASHTYPE_RSS_UDP_IPV6; | return M_HASHTYPE_RSS_UDP_IPV6; | ||||
default: | default: | ||||
return M_HASHTYPE_RSS_IPV6; | return M_HASHTYPE_RSS_IPV6; | ||||
} | } | ||||
} | } | ||||
if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { | if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { | ||||
switch (decoded.inner_prot) { | switch (decoded.inner_prot) { | ||||
case I40E_RX_PTYPE_INNER_PROT_TCP: | case I40E_RX_PTYPE_INNER_PROT_TCP: | ||||
return M_HASHTYPE_RSS_TCP_IPV4; | return M_HASHTYPE_RSS_TCP_IPV4; | ||||
case I40E_RX_PTYPE_INNER_PROT_UDP: | case I40E_RX_PTYPE_INNER_PROT_UDP: | ||||
return M_HASHTYPE_RSS_UDP_IPV4; | return M_HASHTYPE_RSS_UDP_IPV4; | ||||
default: | default: | ||||
return M_HASHTYPE_RSS_IPV4; | return M_HASHTYPE_RSS_IPV4; | ||||
} | } | ||||
} | } | ||||
/* We should never get here!! */ | /* We should never get here!! */ | ||||
return M_HASHTYPE_OPAQUE_HASH; | return M_HASHTYPE_OPAQUE; | ||||
} | } | ||||
#endif /* RSS */ | |||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* This routine executes in interrupt context. It replenishes | * This routine executes in ithread context. It sends data which has been | ||||
* the mbufs in the descriptor and sends data which has been | |||||
* dma'ed into host memory to upper layer. | * dma'ed into host memory to upper layer. | ||||
* | * | ||||
* We loop at most count times if count is > 0, or until done if | * Returns 0 upon success, errno on failure | ||||
* count < 0. | |||||
* | * | ||||
* Return TRUE for more work, FALSE for all clean. | |||||
*********************************************************************/ | *********************************************************************/ | ||||
bool | static int | ||||
ixl_rxeof(struct ixl_queue *que, int count) | ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | ||||
{ | { | ||||
struct ixl_vsi *vsi = que->vsi; | struct ixl_vsi *vsi = arg; | ||||
struct ixl_rx_queue *que = &vsi->rx_queues[ri->iri_qsidx]; | |||||
struct rx_ring *rxr = &que->rxr; | struct rx_ring *rxr = &que->rxr; | ||||
struct ifnet *ifp = vsi->ifp; | |||||
#if defined(INET6) || defined(INET) | |||||
struct lro_ctrl *lro = &rxr->lro; | |||||
#endif | |||||
int i, nextp, processed = 0; | |||||
union i40e_rx_desc *cur; | union i40e_rx_desc *cur; | ||||
struct ixl_rx_buf *rbuf, *nbuf; | |||||
IXL_RX_LOCK(rxr); | |||||
#ifdef DEV_NETMAP | |||||
if (netmap_rx_irq(ifp, que->me, &count)) { | |||||
IXL_RX_UNLOCK(rxr); | |||||
return (FALSE); | |||||
} | |||||
#endif /* DEV_NETMAP */ | |||||
for (i = rxr->next_check; count != 0;) { | |||||
struct mbuf *sendmp, *mh, *mp; | |||||
u32 status, error; | u32 status, error; | ||||
u16 hlen, plen, vtag; | u16 hlen, plen, vtag; | ||||
u64 qword; | u64 qword; | ||||
u8 ptype; | u8 ptype; | ||||
bool eop; | bool eop; | ||||
int i, cidx; | |||||
/* Sync the ring. */ | /* XXX: No packet split support, so hlen is unused */ | ||||
bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, | // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
cur = &rxr->base[i]; | cidx = ri->iri_cidx; | ||||
i = 0; | |||||
do { | |||||
cur = &rxr->rx_base[cidx]; | |||||
qword = le64toh(cur->wb.qword1.status_error_len); | qword = le64toh(cur->wb.qword1.status_error_len); | ||||
status = (qword & I40E_RXD_QW1_STATUS_MASK) | status = (qword & I40E_RXD_QW1_STATUS_MASK) | ||||
>> I40E_RXD_QW1_STATUS_SHIFT; | >> I40E_RXD_QW1_STATUS_SHIFT; | ||||
error = (qword & I40E_RXD_QW1_ERROR_MASK) | error = (qword & I40E_RXD_QW1_ERROR_MASK) | ||||
>> I40E_RXD_QW1_ERROR_SHIFT; | >> I40E_RXD_QW1_ERROR_SHIFT; | ||||
plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) | plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) | ||||
>> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; | >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; | ||||
hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) | hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) | ||||
>> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; | >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; | ||||
ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) | ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) | ||||
>> I40E_RXD_QW1_PTYPE_SHIFT; | >> I40E_RXD_QW1_PTYPE_SHIFT; | ||||
if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) { | /* we should never be called without a valid descriptor */ | ||||
++rxr->not_done; | MPASS((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) != 0); | ||||
break; | |||||
} | |||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) | |||||
break; | |||||
count--; | ri->iri_len += plen; | ||||
sendmp = NULL; | rxr->bytes += plen; | ||||
nbuf = NULL; | |||||
cur->wb.qword1.status_error_len = 0; | cur->wb.qword1.status_error_len = 0; | ||||
rbuf = &rxr->buffers[i]; | |||||
mh = rbuf->m_head; | |||||
mp = rbuf->m_pack; | |||||
eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); | eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); | ||||
if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) | if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) | ||||
vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); | vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); | ||||
else | else | ||||
vtag = 0; | vtag = 0; | ||||
/* Remove device access to the rx buffers. */ | |||||
if (rbuf->m_head != NULL) { | |||||
bus_dmamap_sync(rxr->htag, rbuf->hmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->htag, rbuf->hmap); | |||||
} | |||||
if (rbuf->m_pack != NULL) { | |||||
bus_dmamap_sync(rxr->ptag, rbuf->pmap, | |||||
BUS_DMASYNC_POSTREAD); | |||||
bus_dmamap_unload(rxr->ptag, rbuf->pmap); | |||||
} | |||||
/* | /* | ||||
** Make sure bad packets are discarded, | ** Make sure bad packets are discarded, | ||||
** note that only EOP descriptor has valid | ** note that only EOP descriptor has valid | ||||
** error results. | ** error results. | ||||
*/ | */ | ||||
if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { | if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { | ||||
rxr->desc_errs++; | rxr->desc_errs++; | ||||
ixl_rx_discard(rxr, i); | return (EBADMSG); | ||||
goto next_desc; | |||||
} | } | ||||
ri->iri_frags[i].irf_flid = 0; | |||||
ri->iri_frags[i].irf_idx = cidx; | |||||
ri->iri_frags[i].irf_len = plen; | |||||
if (++cidx == vsi->shared->isc_ntxd[0]) | |||||
cidx = 0; | |||||
i++; | |||||
/* even a 16K packet shouldn't consume more than 8 clusters */ | |||||
MPASS(i < 9); | |||||
} while (!eop); | |||||
/* Prefetch the next buffer */ | |||||
if (!eop) { | |||||
nextp = i + 1; | |||||
if (nextp == que->num_desc) | |||||
nextp = 0; | |||||
nbuf = &rxr->buffers[nextp]; | |||||
prefetch(nbuf); | |||||
} | |||||
/* | |||||
** The header mbuf is ONLY used when header | |||||
** split is enabled, otherwise we get normal | |||||
** behavior, ie, both header and payload | |||||
** are DMA'd into the payload buffer. | |||||
** | |||||
** Rather than using the fmp/lmp global pointers | |||||
** we now keep the head of a packet chain in the | |||||
** buffer struct and pass this along from one | |||||
** descriptor to the next, until we get EOP. | |||||
*/ | |||||
if (rxr->hdr_split && (rbuf->fmp == NULL)) { | |||||
if (hlen > IXL_RX_HDR) | |||||
hlen = IXL_RX_HDR; | |||||
mh->m_len = hlen; | |||||
mh->m_flags |= M_PKTHDR; | |||||
mh->m_next = NULL; | |||||
mh->m_pkthdr.len = mh->m_len; | |||||
/* Null buf pointer so it is refreshed */ | |||||
rbuf->m_head = NULL; | |||||
/* | |||||
** Check the payload length, this | |||||
** could be zero if its a small | |||||
** packet. | |||||
*/ | |||||
if (plen > 0) { | |||||
mp->m_len = plen; | |||||
mp->m_next = NULL; | |||||
mp->m_flags &= ~M_PKTHDR; | |||||
mh->m_next = mp; | |||||
mh->m_pkthdr.len += mp->m_len; | |||||
/* Null buf pointer so it is refreshed */ | |||||
rbuf->m_pack = NULL; | |||||
rxr->split++; | |||||
} | |||||
/* | |||||
** Now create the forward | |||||
** chain so when complete | |||||
** we wont have to. | |||||
*/ | |||||
if (eop == 0) { | |||||
/* stash the chain head */ | |||||
nbuf->fmp = mh; | |||||
/* Make forward chain */ | |||||
if (plen) | |||||
mp->m_next = nbuf->m_pack; | |||||
else | |||||
mh->m_next = nbuf->m_pack; | |||||
} else { | |||||
/* Singlet, prepare to send */ | |||||
sendmp = mh; | |||||
if (vtag) { | |||||
sendmp->m_pkthdr.ether_vtag = vtag; | |||||
sendmp->m_flags |= M_VLANTAG; | |||||
} | |||||
} | |||||
} else { | |||||
/* | |||||
** Either no header split, or a | |||||
** secondary piece of a fragmented | |||||
** split packet. | |||||
*/ | |||||
mp->m_len = plen; | |||||
/* | |||||
** See if there is a stored head | |||||
** that determines what we are | |||||
*/ | |||||
sendmp = rbuf->fmp; | |||||
rbuf->m_pack = rbuf->fmp = NULL; | |||||
if (sendmp != NULL) /* secondary frag */ | |||||
sendmp->m_pkthdr.len += mp->m_len; | |||||
else { | |||||
/* first desc of a non-ps chain */ | |||||
sendmp = mp; | |||||
sendmp->m_flags |= M_PKTHDR; | |||||
sendmp->m_pkthdr.len = mp->m_len; | |||||
} | |||||
/* Pass the head pointer on */ | |||||
if (eop == 0) { | |||||
nbuf->fmp = sendmp; | |||||
sendmp = NULL; | |||||
mp->m_next = nbuf->m_pack; | |||||
} | |||||
} | |||||
++processed; | |||||
/* Sending this frame? */ | |||||
if (eop) { | |||||
sendmp->m_pkthdr.rcvif = ifp; | |||||
/* gather stats */ | |||||
rxr->rx_packets++; | |||||
rxr->rx_bytes += sendmp->m_pkthdr.len; | |||||
/* capture data for dynamic ITR adjustment */ | /* capture data for dynamic ITR adjustment */ | ||||
// TODO: Figure out why these are repeated... | |||||
rxr->packets++; | rxr->packets++; | ||||
rxr->bytes += sendmp->m_pkthdr.len; | rxr->rx_packets++; | ||||
/* Set VLAN tag (field only valid in eop desc) */ | |||||
if (vtag) { | |||||
sendmp->m_pkthdr.ether_vtag = vtag; | |||||
sendmp->m_flags |= M_VLANTAG; | |||||
} | |||||
if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) | |||||
ixl_rx_checksum(sendmp, status, error, ptype); | |||||
#ifdef RSS | |||||
sendmp->m_pkthdr.flowid = | |||||
le32toh(cur->wb.qword0.hi_dword.rss); | |||||
M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype)); | |||||
#else | |||||
sendmp->m_pkthdr.flowid = que->msix; | |||||
M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); | |||||
#endif | |||||
} | |||||
next_desc: | |||||
bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
/* Advance our pointers to the next descriptor. */ | if ((vsi->ifp->if_capenable & IFCAP_RXCSUM) != 0) | ||||
if (++i == que->num_desc) | ixl_rx_checksum(ri, status, error, ptype); | ||||
i = 0; | ri->iri_flowid = le32toh(cur->wb.qword0.hi_dword.rss); | ||||
ri->iri_rsstype = ixl_ptype_to_hash(ptype); | |||||
/* Now send to the stack or do LRO */ | ri->iri_vtag = vtag; | ||||
if (sendmp != NULL) { | ri->iri_nfrags = i; | ||||
rxr->next_check = i; | if (vtag) | ||||
IXL_RX_UNLOCK(rxr); | ri->iri_flags |= M_VLANTAG; | ||||
ixl_rx_input(rxr, ifp, sendmp, ptype); | return (0); | ||||
IXL_RX_LOCK(rxr); | |||||
i = rxr->next_check; | |||||
} | } | ||||
/* Every 8 descriptors we go to refresh mbufs */ | |||||
if (processed == 8) { | |||||
ixl_refresh_mbufs(que, i); | |||||
processed = 0; | |||||
} | |||||
} | |||||
/* Refresh any remaining buf structs */ | |||||
if (ixl_rx_unrefreshed(que)) | |||||
ixl_refresh_mbufs(que, i); | |||||
rxr->next_check = i; | |||||
IXL_RX_UNLOCK(rxr); | |||||
#if defined(INET6) || defined(INET) | |||||
/* | |||||
* Flush any outstanding LRO work | |||||
*/ | |||||
#if __FreeBSD_version >= 1100105 | |||||
tcp_lro_flush_all(lro); | |||||
#else | |||||
struct lro_entry *queued; | |||||
while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { | |||||
SLIST_REMOVE_HEAD(&lro->lro_active, next); | |||||
tcp_lro_flush(lro, queued); | |||||
} | |||||
#endif | |||||
#endif /* defined(INET6) || defined(INET) */ | |||||
return (FALSE); | |||||
} | |||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Verify that the hardware indicated that the checksum is valid. | * Verify that the hardware indicated that the checksum is valid. | ||||
* Inform the stack about the status of checksum so that stack | * Inform the stack about the status of checksum so that stack | ||||
* doesn't spend time verifying the checksum. | * doesn't spend time verifying the checksum. | ||||
* | * | ||||
*********************************************************************/ | *********************************************************************/ | ||||
static void | static void | ||||
ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype) | ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype) | ||||
{ | { | ||||
struct i40e_rx_ptype_decoded decoded; | struct i40e_rx_ptype_decoded decoded; | ||||
decoded = decode_rx_desc_ptype(ptype); | ri->iri_csum_flags = 0; | ||||
/* Errors? */ | /* No L3 or L4 checksum was calculated */ | ||||
if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | | if (!(status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT))) | ||||
(1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { | |||||
mp->m_pkthdr.csum_flags = 0; | |||||
return; | return; | ||||
} | |||||
decoded = decode_rx_desc_ptype(ptype); | |||||
/* IPv6 with extension headers likely have bad csum */ | /* IPv6 with extension headers likely have bad csum */ | ||||
if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && | if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && | ||||
decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) | decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { | ||||
if (status & | if (status & | ||||
(1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { | (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { | ||||
mp->m_pkthdr.csum_flags = 0; | ri->iri_csum_flags = 0; | ||||
return; | return; | ||||
} | } | ||||
} | |||||
ri->iri_csum_flags |= CSUM_L3_CALC; | |||||
/* IP Checksum Good */ | /* IPv4 checksum error */ | ||||
mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; | if (error & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT)) | ||||
mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; | |||||
if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) { | |||||
mp->m_pkthdr.csum_flags |= | |||||
(CSUM_DATA_VALID | CSUM_PSEUDO_HDR); | |||||
mp->m_pkthdr.csum_data |= htons(0xffff); | |||||
} | |||||
return; | return; | ||||
} | |||||
#if __FreeBSD_version >= 1100000 | ri->iri_csum_flags |= CSUM_L3_VALID; | ||||
uint64_t | ri->iri_csum_flags |= CSUM_L4_CALC; | ||||
ixl_get_counter(if_t ifp, ift_counter cnt) | |||||
{ | |||||
struct ixl_vsi *vsi; | |||||
vsi = if_getsoftc(ifp); | /* L4 checksum error */ | ||||
if (error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)) | |||||
return; | |||||
switch (cnt) { | ri->iri_csum_flags |= CSUM_L4_VALID; | ||||
case IFCOUNTER_IPACKETS: | ri->iri_csum_data |= htons(0xffff); | ||||
return (vsi->ipackets); | |||||
case IFCOUNTER_IERRORS: | |||||
return (vsi->ierrors); | |||||
case IFCOUNTER_OPACKETS: | |||||
return (vsi->opackets); | |||||
case IFCOUNTER_OERRORS: | |||||
return (vsi->oerrors); | |||||
case IFCOUNTER_COLLISIONS: | |||||
/* Collisions are by standard impossible in 40G/10G Ethernet */ | |||||
return (0); | |||||
case IFCOUNTER_IBYTES: | |||||
return (vsi->ibytes); | |||||
case IFCOUNTER_OBYTES: | |||||
return (vsi->obytes); | |||||
case IFCOUNTER_IMCASTS: | |||||
return (vsi->imcasts); | |||||
case IFCOUNTER_OMCASTS: | |||||
return (vsi->omcasts); | |||||
case IFCOUNTER_IQDROPS: | |||||
return (vsi->iqdrops); | |||||
case IFCOUNTER_OQDROPS: | |||||
return (vsi->oqdrops); | |||||
case IFCOUNTER_NOPROTO: | |||||
return (vsi->noproto); | |||||
default: | |||||
return (if_get_counter_default(ifp, cnt)); | |||||
} | } | ||||
} | |||||
#endif | |||||