Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/ixl/ixl_txrx.c
Show All 29 Lines | /****************************************************************************** | ||||
POSSIBILITY OF SUCH DAMAGE. | POSSIBILITY OF SUCH DAMAGE. | ||||
******************************************************************************/ | ******************************************************************************/ | ||||
/*$FreeBSD$*/ | /*$FreeBSD$*/ | ||||
/* | /* | ||||
** IXL driver TX/RX Routines: | ** IXL driver TX/RX Routines: | ||||
** This was seperated to allow usage by | ** This was seperated to allow usage by | ||||
** both the BASE and the VF drivers. | ** both the PF and VF drivers. | ||||
*/ | */ | ||||
#ifndef IXL_STANDALONE_BUILD | #ifndef IXL_STANDALONE_BUILD | ||||
#include "opt_inet.h" | #include "opt_inet.h" | ||||
#include "opt_inet6.h" | #include "opt_inet6.h" | ||||
#include "opt_rss.h" | #include "opt_rss.h" | ||||
#endif | #endif | ||||
#include "ixl.h" | #include "ixl.h" | ||||
#ifdef RSS | #ifdef RSS | ||||
#include <net/rss_config.h> | #include <net/rss_config.h> | ||||
#endif | #endif | ||||
/* Local Prototypes */ | /* Local Prototypes */ | ||||
static void ixl_rx_checksum(struct mbuf *, u32, u32, u8); | static void ixl_rx_checksum(struct mbuf *, u32, u32, u8); | ||||
static void ixl_refresh_mbufs(struct ixl_queue *, int); | static void ixl_refresh_mbufs(struct ixl_queue *, int); | ||||
static int ixl_xmit(struct ixl_queue *, struct mbuf **); | static int ixl_xmit(struct ixl_queue *, struct mbuf **); | ||||
static int ixl_tx_setup_offload(struct ixl_queue *, | static int ixl_tx_setup_offload(struct ixl_queue *, | ||||
struct mbuf *, u32 *, u32 *); | struct mbuf *, u32 *, u32 *); | ||||
static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *); | static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *); | ||||
static __inline void ixl_rx_discard(struct rx_ring *, int); | static inline void ixl_rx_discard(struct rx_ring *, int); | ||||
static __inline void ixl_rx_input(struct rx_ring *, struct ifnet *, | static inline void ixl_rx_input(struct rx_ring *, struct ifnet *, | ||||
struct mbuf *, u8); | struct mbuf *, u8); | ||||
static inline bool ixl_tso_detect_sparse(struct mbuf *mp); | |||||
static int ixl_tx_setup_offload(struct ixl_queue *que, | |||||
struct mbuf *mp, u32 *cmd, u32 *off); | |||||
static inline u32 ixl_get_tx_head(struct ixl_queue *que); | |||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
#include <dev/netmap/if_ixl_netmap.h> | #include <dev/netmap/if_ixl_netmap.h> | ||||
#endif /* DEV_NETMAP */ | #endif /* DEV_NETMAP */ | ||||
/* | /* | ||||
* @key key is saved into this parameter | |||||
*/ | |||||
void | |||||
ixl_get_default_rss_key(u32 *key) | |||||
{ | |||||
MPASS(key != NULL); | |||||
u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687, | |||||
0x183cfd8c, 0xce880440, 0x580cbc3c, | |||||
0x35897377, 0x328b25e1, 0x4fa98922, | |||||
0xb7d90c14, 0xd5bad70d, 0xcd15a2c1, | |||||
0x0, 0x0, 0x0}; | |||||
bcopy(rss_seed, key, IXL_RSS_KEY_SIZE); | |||||
} | |||||
/* | |||||
** Multiqueue Transmit driver | ** Multiqueue Transmit driver | ||||
*/ | */ | ||||
int | int | ||||
ixl_mq_start(struct ifnet *ifp, struct mbuf *m) | ixl_mq_start(struct ifnet *ifp, struct mbuf *m) | ||||
{ | { | ||||
struct ixl_vsi *vsi = ifp->if_softc; | struct ixl_vsi *vsi = ifp->if_softc; | ||||
struct ixl_queue *que; | struct ixl_queue *que; | ||||
struct tx_ring *txr; | struct tx_ring *txr; | ||||
Show All 15 Lines | #ifdef RSS | ||||
if (rss_hash2bucket(m->m_pkthdr.flowid, | if (rss_hash2bucket(m->m_pkthdr.flowid, | ||||
M_HASHTYPE_GET(m), &bucket_id) == 0) { | M_HASHTYPE_GET(m), &bucket_id) == 0) { | ||||
i = bucket_id % vsi->num_queues; | i = bucket_id % vsi->num_queues; | ||||
} else | } else | ||||
#endif | #endif | ||||
i = m->m_pkthdr.flowid % vsi->num_queues; | i = m->m_pkthdr.flowid % vsi->num_queues; | ||||
} else | } else | ||||
i = curcpu % vsi->num_queues; | i = curcpu % vsi->num_queues; | ||||
/* | |||||
** This may not be perfect, but until something | |||||
** better comes along it will keep from scheduling | |||||
** on stalled queues. | |||||
*/ | |||||
if (((1 << i) & vsi->active_queues) == 0) | |||||
i = ffsl(vsi->active_queues); | |||||
que = &vsi->queues[i]; | que = &vsi->queues[i]; | ||||
txr = &que->txr; | txr = &que->txr; | ||||
err = drbr_enqueue(ifp, txr->br, m); | err = drbr_enqueue(ifp, txr->br, m); | ||||
if (err) | if (err) | ||||
return (err); | return (err); | ||||
if (IXL_TX_TRYLOCK(txr)) { | if (IXL_TX_TRYLOCK(txr)) { | ||||
▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Lines | |||||
ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) | ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) | ||||
{ | { | ||||
struct ixl_vsi *vsi = que->vsi; | struct ixl_vsi *vsi = que->vsi; | ||||
struct i40e_hw *hw = vsi->hw; | struct i40e_hw *hw = vsi->hw; | ||||
struct tx_ring *txr = &que->txr; | struct tx_ring *txr = &que->txr; | ||||
struct ixl_tx_buf *buf; | struct ixl_tx_buf *buf; | ||||
struct i40e_tx_desc *txd = NULL; | struct i40e_tx_desc *txd = NULL; | ||||
struct mbuf *m_head, *m; | struct mbuf *m_head, *m; | ||||
int i, j, error, nsegs, maxsegs; | int i, j, error, nsegs; | ||||
int first, last = 0; | int first, last = 0; | ||||
u16 vtag = 0; | u16 vtag = 0; | ||||
u32 cmd, off; | u32 cmd, off; | ||||
bus_dmamap_t map; | bus_dmamap_t map; | ||||
bus_dma_tag_t tag; | bus_dma_tag_t tag; | ||||
bus_dma_segment_t segs[IXL_MAX_TSO_SEGS]; | bus_dma_segment_t segs[IXL_MAX_TSO_SEGS]; | ||||
cmd = off = 0; | cmd = off = 0; | ||||
m_head = *m_headp; | m_head = *m_headp; | ||||
/* | /* | ||||
* Important to capture the first descriptor | * Important to capture the first descriptor | ||||
* used because it will contain the index of | * used because it will contain the index of | ||||
* the one we tell the hardware to report back | * the one we tell the hardware to report back | ||||
*/ | */ | ||||
first = txr->next_avail; | first = txr->next_avail; | ||||
buf = &txr->buffers[first]; | buf = &txr->buffers[first]; | ||||
map = buf->map; | map = buf->map; | ||||
tag = txr->tx_tag; | tag = txr->tx_tag; | ||||
maxsegs = IXL_MAX_TX_SEGS; | |||||
if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | ||||
/* Use larger mapping for TSO */ | /* Use larger mapping for TSO */ | ||||
tag = txr->tso_tag; | tag = txr->tso_tag; | ||||
maxsegs = IXL_MAX_TSO_SEGS; | |||||
if (ixl_tso_detect_sparse(m_head)) { | if (ixl_tso_detect_sparse(m_head)) { | ||||
m = m_defrag(m_head, M_NOWAIT); | m = m_defrag(m_head, M_NOWAIT); | ||||
if (m == NULL) { | if (m == NULL) { | ||||
m_freem(*m_headp); | m_freem(*m_headp); | ||||
*m_headp = NULL; | *m_headp = NULL; | ||||
return (ENOBUFS); | return (ENOBUFS); | ||||
} | } | ||||
*m_headp = m; | *m_headp = m; | ||||
Show All 18 Lines | if (error == EFBIG) { | ||||
} | } | ||||
*m_headp = m; | *m_headp = m; | ||||
/* Try it again */ | /* Try it again */ | ||||
error = bus_dmamap_load_mbuf_sg(tag, map, | error = bus_dmamap_load_mbuf_sg(tag, map, | ||||
*m_headp, segs, &nsegs, BUS_DMA_NOWAIT); | *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); | ||||
if (error == ENOMEM) { | if (error == ENOMEM) { | ||||
que->tx_dma_setup++; | que->tx_dmamap_failed++; | ||||
return (error); | return (error); | ||||
} else if (error != 0) { | } else if (error != 0) { | ||||
que->tx_dma_setup++; | que->tx_dmamap_failed++; | ||||
m_freem(*m_headp); | m_freem(*m_headp); | ||||
*m_headp = NULL; | *m_headp = NULL; | ||||
return (error); | return (error); | ||||
} | } | ||||
} else if (error == ENOMEM) { | } else if (error == ENOMEM) { | ||||
que->tx_dma_setup++; | que->tx_dmamap_failed++; | ||||
return (error); | return (error); | ||||
} else if (error != 0) { | } else if (error != 0) { | ||||
que->tx_dma_setup++; | que->tx_dmamap_failed++; | ||||
m_freem(*m_headp); | m_freem(*m_headp); | ||||
*m_headp = NULL; | *m_headp = NULL; | ||||
return (error); | return (error); | ||||
} | } | ||||
/* Make certain there are enough descriptors */ | /* Make certain there are enough descriptors */ | ||||
if (nsegs > txr->avail - 2) { | if (nsegs > txr->avail - 2) { | ||||
txr->no_desc++; | txr->no_desc++; | ||||
▲ Show 20 Lines • Show All 476 Lines • ▼ Show 20 Lines | #endif | ||||
idx = txr->next_avail; | idx = txr->next_avail; | ||||
buf = &txr->buffers[idx]; | buf = &txr->buffers[idx]; | ||||
TXD = (struct i40e_tx_context_desc *) &txr->base[idx]; | TXD = (struct i40e_tx_context_desc *) &txr->base[idx]; | ||||
tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen); | tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen); | ||||
type = I40E_TX_DESC_DTYPE_CONTEXT; | type = I40E_TX_DESC_DTYPE_CONTEXT; | ||||
cmd = I40E_TX_CTX_DESC_TSO; | cmd = I40E_TX_CTX_DESC_TSO; | ||||
/* ERJ: this must not be less than 64 */ | |||||
mss = mp->m_pkthdr.tso_segsz; | mss = mp->m_pkthdr.tso_segsz; | ||||
type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | | type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | | ||||
((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | | ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | | ||||
((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | | ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | | ||||
((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); | ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); | ||||
TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); | TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); | ||||
▲ Show 20 Lines • Show All 554 Lines • ▼ Show 20 Lines | if (rxr->ptag != NULL) { | ||||
bus_dma_tag_destroy(rxr->ptag); | bus_dma_tag_destroy(rxr->ptag); | ||||
rxr->ptag = NULL; | rxr->ptag = NULL; | ||||
} | } | ||||
INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); | INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); | ||||
return; | return; | ||||
} | } | ||||
static __inline void | static inline void | ||||
ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) | ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) | ||||
{ | { | ||||
#if defined(INET6) || defined(INET) | #if defined(INET6) || defined(INET) | ||||
/* | /* | ||||
* ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet | * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet | ||||
* should be computed by hardware. Also it should not have VLAN tag in | * should be computed by hardware. Also it should not have VLAN tag in | ||||
* ethernet header. | * ethernet header. | ||||
Show All 14 Lines | #if defined(INET6) || defined(INET) | ||||
} | } | ||||
#endif | #endif | ||||
IXL_RX_UNLOCK(rxr); | IXL_RX_UNLOCK(rxr); | ||||
(*ifp->if_input)(ifp, m); | (*ifp->if_input)(ifp, m); | ||||
IXL_RX_LOCK(rxr); | IXL_RX_LOCK(rxr); | ||||
} | } | ||||
static __inline void | static inline void | ||||
ixl_rx_discard(struct rx_ring *rxr, int i) | ixl_rx_discard(struct rx_ring *rxr, int i) | ||||
{ | { | ||||
struct ixl_rx_buf *rbuf; | struct ixl_rx_buf *rbuf; | ||||
rbuf = &rxr->buffers[i]; | rbuf = &rxr->buffers[i]; | ||||
if (rbuf->fmp != NULL) {/* Partial chain ? */ | if (rbuf->fmp != NULL) {/* Partial chain ? */ | ||||
rbuf->fmp->m_flags |= M_PKTHDR; | rbuf->fmp->m_flags |= M_PKTHDR; | ||||
▲ Show 20 Lines • Show All 110 Lines • ▼ Show 20 Lines | #ifdef DEV_NETMAP | ||||
if (netmap_rx_irq(ifp, que->me, &count)) { | if (netmap_rx_irq(ifp, que->me, &count)) { | ||||
IXL_RX_UNLOCK(rxr); | IXL_RX_UNLOCK(rxr); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
#endif /* DEV_NETMAP */ | #endif /* DEV_NETMAP */ | ||||
for (i = rxr->next_check; count != 0;) { | for (i = rxr->next_check; count != 0;) { | ||||
struct mbuf *sendmp, *mh, *mp; | struct mbuf *sendmp, *mh, *mp; | ||||
u32 rsc, status, error; | u32 status, error; | ||||
u16 hlen, plen, vtag; | u16 hlen, plen, vtag; | ||||
u64 qword; | u64 qword; | ||||
u8 ptype; | u8 ptype; | ||||
bool eop; | bool eop; | ||||
/* Sync the ring. */ | /* Sync the ring. */ | ||||
bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, | bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
Show All 16 Lines | if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) { | ||||
break; | break; | ||||
} | } | ||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) | if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) | ||||
break; | break; | ||||
count--; | count--; | ||||
sendmp = NULL; | sendmp = NULL; | ||||
nbuf = NULL; | nbuf = NULL; | ||||
rsc = 0; | |||||
cur->wb.qword1.status_error_len = 0; | cur->wb.qword1.status_error_len = 0; | ||||
rbuf = &rxr->buffers[i]; | rbuf = &rxr->buffers[i]; | ||||
mh = rbuf->m_head; | mh = rbuf->m_head; | ||||
mp = rbuf->m_pack; | mp = rbuf->m_pack; | ||||
eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); | eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); | ||||
if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) | if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) | ||||
vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); | vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); | ||||
else | else | ||||
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | if (rxr->hdr_split && (rbuf->fmp == NULL)) { | ||||
if (sendmp != NULL) /* secondary frag */ | if (sendmp != NULL) /* secondary frag */ | ||||
sendmp->m_pkthdr.len += mp->m_len; | sendmp->m_pkthdr.len += mp->m_len; | ||||
else { | else { | ||||
/* first desc of a non-ps chain */ | /* first desc of a non-ps chain */ | ||||
sendmp = mp; | sendmp = mp; | ||||
sendmp->m_flags |= M_PKTHDR; | sendmp->m_flags |= M_PKTHDR; | ||||
sendmp->m_pkthdr.len = mp->m_len; | sendmp->m_pkthdr.len = mp->m_len; | ||||
if (vtag) { | |||||
sendmp->m_pkthdr.ether_vtag = vtag; | |||||
sendmp->m_flags |= M_VLANTAG; | |||||
} | } | ||||
} | |||||
/* Pass the head pointer on */ | /* Pass the head pointer on */ | ||||
if (eop == 0) { | if (eop == 0) { | ||||
nbuf->fmp = sendmp; | nbuf->fmp = sendmp; | ||||
sendmp = NULL; | sendmp = NULL; | ||||
mp->m_next = nbuf->m_pack; | mp->m_next = nbuf->m_pack; | ||||
} | } | ||||
} | } | ||||
++processed; | ++processed; | ||||
/* Sending this frame? */ | /* Sending this frame? */ | ||||
if (eop) { | if (eop) { | ||||
sendmp->m_pkthdr.rcvif = ifp; | sendmp->m_pkthdr.rcvif = ifp; | ||||
/* gather stats */ | /* gather stats */ | ||||
rxr->rx_packets++; | rxr->rx_packets++; | ||||
rxr->rx_bytes += sendmp->m_pkthdr.len; | rxr->rx_bytes += sendmp->m_pkthdr.len; | ||||
/* capture data for dynamic ITR adjustment */ | /* capture data for dynamic ITR adjustment */ | ||||
rxr->packets++; | rxr->packets++; | ||||
rxr->bytes += sendmp->m_pkthdr.len; | rxr->bytes += sendmp->m_pkthdr.len; | ||||
/* Set VLAN tag (field only valid in eop desc) */ | |||||
if (vtag) { | |||||
sendmp->m_pkthdr.ether_vtag = vtag; | |||||
sendmp->m_flags |= M_VLANTAG; | |||||
} | |||||
if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) | if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) | ||||
ixl_rx_checksum(sendmp, status, error, ptype); | ixl_rx_checksum(sendmp, status, error, ptype); | ||||
#ifdef RSS | #ifdef RSS | ||||
sendmp->m_pkthdr.flowid = | sendmp->m_pkthdr.flowid = | ||||
le32toh(cur->wb.qword0.hi_dword.rss); | le32toh(cur->wb.qword0.hi_dword.rss); | ||||
M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype)); | M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype)); | ||||
#else | #else | ||||
sendmp->m_pkthdr.flowid = que->msix; | sendmp->m_pkthdr.flowid = que->msix; | ||||
▲ Show 20 Lines • Show All 126 Lines • Show Last 20 Lines |