Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/ixl/ixl_txrx.c
Show First 20 Lines • Show All 266 Lines • ▼ Show 20 Lines | case IPPROTO_SCTP: | ||||
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; | ||||
} | } | ||||
/* Fall Thru */ | /* Fall Thru */ | ||||
default: | default: | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
/********************************************************************** | #if defined(INET6) || defined(INET) | ||||
/** | |||||
* Setup context descriptor for TSO or VXLAN Offload | |||||
* @txr: TX ring which handles transmission | |||||
* @pi: information extracted from packet headers | |||||
* | * | ||||
* Setup context for hardware segmentation offload (TSO) | * Configure TX descriptor with information extracted | ||||
* | * from a packet header required for HW to calculate | ||||
**********************************************************************/ | * requested checksum and perform TCP segmentation. | ||||
*/ | |||||
static int | static int | ||||
ixl_tso_setup(struct tx_ring *txr, if_pkt_info_t pi) | ixl_ctxd_setup(struct tx_ring *txr, if_pkt_info_t pi) | ||||
{ | { | ||||
if_softc_ctx_t scctx; | if_softc_ctx_t scctx; | ||||
struct i40e_tx_context_desc *TXD; | struct i40e_tx_context_desc *TXD; | ||||
u32 cmd, mss, type, tsolen; | u32 cmd, mss, type, tsolen; | ||||
int idx, total_hdr_len; | int idx, total_hdr_len; | ||||
u64 type_cmd_tso_mss; | u64 type_cmd_tso_mss; | ||||
idx = pi->ipi_pidx; | idx = pi->ipi_pidx; | ||||
TXD = (struct i40e_tx_context_desc *) &txr->tx_base[idx]; | TXD = (struct i40e_tx_context_desc *) &txr->tx_base[idx]; | ||||
total_hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; | |||||
tsolen = pi->ipi_len - total_hdr_len; | |||||
scctx = txr->que->vsi->shared; | scctx = txr->que->vsi->shared; | ||||
type_cmd_tso_mss = | |||||
(u64)I40E_TX_DESC_DTYPE_CONTEXT << I40E_TXD_CTX_QW1_DTYPE_SHIFT; | |||||
type = I40E_TX_DESC_DTYPE_CONTEXT; | type = I40E_TX_DESC_DTYPE_CONTEXT; | ||||
if (pi->ipi_csum_flags & IXL_CSUM_TSO) { | |||||
cmd = I40E_TX_CTX_DESC_TSO; | cmd = I40E_TX_CTX_DESC_TSO; | ||||
/* | /* | ||||
* TSO MSS must not be less than 64; this prevents a | * TSO MSS must not be less than 64; this prevents a | ||||
* BAD_LSO_MSS MDD event when the MSS is too small. | * BAD_LSO_MSS MDD event when the MSS is too small. | ||||
*/ | */ | ||||
if (pi->ipi_tso_segsz < IXL_MIN_TSO_MSS) { | if (pi->ipi_tso_segsz < IXL_MIN_TSO_MSS) { | ||||
txr->mss_too_small++; | txr->mss_too_small++; | ||||
pi->ipi_tso_segsz = IXL_MIN_TSO_MSS; | pi->ipi_tso_segsz = IXL_MIN_TSO_MSS; | ||||
} | } | ||||
mss = pi->ipi_tso_segsz; | mss = pi->ipi_tso_segsz; | ||||
total_hdr_len = pi->ipi_ehdrlen + | |||||
pi->ipi_ip_hlen + pi->ipi_tcp_hlen + | |||||
pi->ipi_outer_ip_hlen + pi->ipi_tun_hlen; | |||||
tsolen = pi->ipi_len - total_hdr_len; | |||||
/* Check for BAD_LS0_MSS MDD event (mss too large) */ | /* Check for BAD_LS0_MSS MDD event (mss too large) */ | ||||
MPASS(mss <= IXL_MAX_TSO_MSS); | MPASS(mss <= IXL_MAX_TSO_MSS); | ||||
/* Check for NO_HEAD MDD event (header lengths are 0) */ | /* Check for NO_HEAD MDD event (header lengths are 0) */ | ||||
MPASS(pi->ipi_ehdrlen != 0); | MPASS(pi->ipi_ehdrlen != 0); | ||||
MPASS(pi->ipi_ip_hlen != 0); | MPASS(pi->ipi_ip_hlen != 0); | ||||
/* Partial check for BAD_LSO_LEN MDD event */ | /* Partial check for BAD_LSO_LEN MDD event */ | ||||
MPASS(tsolen != 0); | MPASS(tsolen != 0); | ||||
/* Partial check for WRONG_SIZE MDD event (during TSO) */ | /* Partial check for WRONG_SIZE MDD event (during TSO) */ | ||||
MPASS(total_hdr_len + mss <= IXL_MAX_FRAME); | MPASS(total_hdr_len + mss <= IXL_MAX_FRAME); | ||||
type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | | type_cmd_tso_mss |= | ||||
((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | | ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | | ||||
((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | | ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | | ||||
((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); | ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); | ||||
txr->que->tso++; | |||||
} | |||||
TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); | TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); | ||||
if (pi->ipi_csum_flags & CSUM_ENCAP_VXLAN) { | |||||
u32 tun_params = I40E_TXD_CTX_UDP_TUNNELING; | |||||
switch (pi->ipi_outer_etype) { | |||||
case ETHERTYPE_IP: | |||||
if (pi->ipi_csum_flags & CSUM_INNER_IP_TSO) | |||||
tun_params |= I40E_TX_CTX_EXT_IP_IPV4; | |||||
else | |||||
tun_params |= I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; | |||||
break; | |||||
case ETHERTYPE_IPV6: | |||||
tun_params |= I40E_TX_CTX_EXT_IP_IPV6; | |||||
break; | |||||
default: | |||||
break; | |||||
} | |||||
tun_params |= | |||||
(pi->ipi_outer_ip_hlen >> 2) << I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT | | |||||
(pi->ipi_tun_hlen >> 1) << I40E_TXD_CTX_QW0_NATLEN_SHIFT; | |||||
TXD->tunneling_params = htole32(tun_params); | |||||
txr->que->tx_vxlan++; | |||||
} else | |||||
TXD->tunneling_params = htole32(0); | TXD->tunneling_params = htole32(0); | ||||
txr->que->tso++; | |||||
return ((idx + 1) & (scctx->isc_ntxd[0]-1)); | return ((idx + 1) & (scctx->isc_ntxd[0]-1)); | ||||
} | } | ||||
#endif | |||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* This routine maps the mbufs to tx descriptors, allowing the | * This routine maps the mbufs to tx descriptors, allowing the | ||||
* TX engine to transmit the packets. | * TX engine to transmit the packets. | ||||
* - return 0 on success, positive on failure | * - return 0 on success, positive on failure | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
Show All 14 Lines | ixl_isc_txd_encap(void *arg, if_pkt_info_t pi) | ||||
cmd = off = 0; | cmd = off = 0; | ||||
i = pi->ipi_pidx; | i = pi->ipi_pidx; | ||||
tx_intr = (pi->ipi_flags & IPI_TX_INTR); | tx_intr = (pi->ipi_flags & IPI_TX_INTR); | ||||
/* Set up the TSO/CSUM offload */ | /* Set up the TSO/CSUM offload */ | ||||
if (pi->ipi_csum_flags & CSUM_OFFLOAD) { | if (pi->ipi_csum_flags & CSUM_OFFLOAD) { | ||||
/* Set up the TSO context descriptor if required */ | /* Set up the context descriptor if required */ | ||||
if (pi->ipi_csum_flags & CSUM_TSO) { | if (IXL_NEEDS_CTXD(pi->ipi_csum_flags)) { | ||||
/* Prevent MAX_BUFF MDD event (for TSO) */ | /* Prevent MAX_BUFF MDD event (for TSO) */ | ||||
if (ixl_tso_detect_sparse(segs, nsegs, pi)) | if (ixl_tso_detect_sparse(segs, nsegs, pi)) | ||||
return (EFBIG); | return (EFBIG); | ||||
i = ixl_tso_setup(txr, pi); | i = ixl_ctxd_setup(txr, pi); | ||||
} | } | ||||
ixl_tx_setup_offload(que, pi, &cmd, &off); | ixl_tx_setup_offload(que, pi, &cmd, &off); | ||||
} | } | ||||
if (pi->ipi_mflags & M_VLANTAG) | if (pi->ipi_mflags & M_VLANTAG) | ||||
cmd |= I40E_TX_DESC_CMD_IL2TAG1; | cmd |= I40E_TX_DESC_CMD_IL2TAG1; | ||||
cmd |= I40E_TX_DESC_CMD_ICRC; | cmd |= I40E_TX_DESC_CMD_ICRC; | ||||
mask = scctx->isc_ntxd[0] - 1; | mask = scctx->isc_ntxd[0] - 1; | ||||
▲ Show 20 Lines • Show All 353 Lines • ▼ Show 20 Lines | ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) | ||||
ri->iri_rsstype = ixl_ptype_to_hash(ptype); | ri->iri_rsstype = ixl_ptype_to_hash(ptype); | ||||
ri->iri_vtag = vtag; | ri->iri_vtag = vtag; | ||||
ri->iri_nfrags = i; | ri->iri_nfrags = i; | ||||
if (vtag) | if (vtag) | ||||
ri->iri_flags |= M_VLANTAG; | ri->iri_flags |= M_VLANTAG; | ||||
return (0); | return (0); | ||||
} | } | ||||
/********************************************************************* | /** | ||||
* ixl_rx_checksum - Verify that the hardware indicated that the checksum is valid or not | |||||
* @ri: iflib RXD info | |||||
* @status: RX descriptor status data | |||||
* @error: RX descriptor error data | |||||
* @ptype: packet type | |||||
* | * | ||||
* Verify that the hardware indicated that the checksum is valid. | * Determine whether the hardware indicated that RX checksums were verified | ||||
* Inform the stack about the status of checksum so that stack | * and are valid. Inform the stack about the status of checksum so that stack | ||||
* doesn't spend time verifying the checksum. | * doesn't spend time verifying them. | ||||
* | */ | ||||
*********************************************************************/ | |||||
static u8 | static u8 | ||||
ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype) | ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype) | ||||
{ | { | ||||
struct i40e_rx_ptype_decoded decoded; | struct i40e_rx_ptype_decoded decoded; | ||||
ri->iri_csum_flags = 0; | ri->iri_csum_flags = 0; | ||||
/* No L3 or L4 checksum was calculated */ | /* No L3 or L4 checksum was calculated */ | ||||
if (!(status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT))) | if (!(status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT))) | ||||
return (0); | return (0); | ||||
decoded = decode_rx_desc_ptype(ptype); | decoded = decode_rx_desc_ptype(ptype); | ||||
/* Cannot proceed if packet type is unknown or not an IP packet */ | |||||
if (decoded.known == 0 || decoded.outer_ip != I40E_RX_PTYPE_OUTER_IP) | |||||
return (0); | |||||
/* IPv6 with extension headers likely have bad csum */ | /* IPv6 with extension headers likely have bad csum */ | ||||
if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && | if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && | ||||
decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { | decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { | ||||
if (status & | if (status & | ||||
(1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { | (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { | ||||
ri->iri_csum_flags = 0; | ri->iri_csum_flags = 0; | ||||
return (1); | return (1); | ||||
} | } | ||||
} | } | ||||
switch (decoded.tunnel_type) { | |||||
case I40E_RX_PTYPE_TUNNEL_NONE: | |||||
/* L3 checksum is calculated only for IPv4 packets */ | |||||
if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { | |||||
ri->iri_csum_flags |= CSUM_L3_CALC; | ri->iri_csum_flags |= CSUM_L3_CALC; | ||||
/* IPv4 checksum error */ | /* IPv4 checksum error */ | ||||
if (error & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT)) | if (error & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT)) | ||||
return (1); | return (1); | ||||
ri->iri_csum_flags |= CSUM_L3_VALID; | ri->iri_csum_flags |= CSUM_L3_VALID; | ||||
ri->iri_csum_flags |= CSUM_L4_CALC; | } | ||||
switch (decoded.inner_prot) { | |||||
case I40E_RX_PTYPE_INNER_PROT_UDP: | |||||
case I40E_RX_PTYPE_INNER_PROT_TCP: | |||||
case I40E_RX_PTYPE_INNER_PROT_SCTP: | |||||
ri->iri_csum_flags |= CSUM_L4_CALC; | |||||
/* L4 checksum error */ | /* L4 checksum error */ | ||||
if (error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)) | if (error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)) | ||||
return (1); | return (1); | ||||
ri->iri_csum_flags |= CSUM_L4_VALID; | ri->iri_csum_flags |= CSUM_L4_VALID; | ||||
ri->iri_csum_data |= htons(0xffff); | ri->iri_csum_data |= htons(0xffff); | ||||
break; | |||||
default: | |||||
break; | |||||
} | |||||
break; | |||||
case I40E_RX_PTYPE_TUNNEL_IP_GRENAT: | |||||
case I40E_RX_PTYPE_TUNNEL_IP_GRENAT_MAC: | |||||
case I40E_RX_PTYPE_TUNNEL_IP_GRENAT_MAC_VLAN: | |||||
/* L3 checksum of outer IPv4 packets */ | |||||
if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { | |||||
ri->iri_csum_flags = CSUM_L3_CALC; | |||||
/* IP checksum error */ | |||||
if (error & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT)) | |||||
return (1); | |||||
ri->iri_csum_flags |= CSUM_L3_VALID; | |||||
} | |||||
/* L3 checksum of most inner IPv4 packets */ | |||||
if (decoded.tunnel_end_prot == I40E_RX_PTYPE_TUNNEL_END_IPV4) { | |||||
ri->iri_csum_flags = CSUM_INNER_L3_CALC; | |||||
/* IP checksum error */ | |||||
if (error & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT)) | |||||
return (1); | |||||
ri->iri_csum_flags |= CSUM_INNER_L3_VALID; | |||||
} | |||||
switch (decoded.inner_prot) { | |||||
case I40E_RX_PTYPE_INNER_PROT_UDP: | |||||
case I40E_RX_PTYPE_INNER_PROT_TCP: | |||||
case I40E_RX_PTYPE_INNER_PROT_SCTP: | |||||
ri->iri_csum_flags |= CSUM_INNER_L4_CALC; | |||||
if (error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)) | |||||
return (1); | |||||
ri->iri_csum_flags |= CSUM_INNER_L4_VALID; | |||||
ri->iri_csum_data |= htons(0xffff); | |||||
break; | |||||
default: | |||||
break; | |||||
} | |||||
break; | |||||
default: | |||||
break; | |||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
/* Set Report Status queue fields to 0 */ | /* Set Report Status queue fields to 0 */ | ||||
void | void | ||||
ixl_init_tx_rsqs(struct ixl_vsi *vsi) | ixl_init_tx_rsqs(struct ixl_vsi *vsi) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 159 Lines • ▼ Show 20 Lines | for (int q = 0; q < vsi->num_tx_queues; q++) { | ||||
queue_list = SYSCTL_CHILDREN(queue_node); | queue_list = SYSCTL_CHILDREN(queue_node); | ||||
tx_que = &(vsi->tx_queues[q]); | tx_que = &(vsi->tx_queues[q]); | ||||
txr = &(tx_que->txr); | txr = &(tx_que->txr); | ||||
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso", | SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso", | ||||
CTLFLAG_RD, &(tx_que->tso), | CTLFLAG_RD, &(tx_que->tso), | ||||
"TSO"); | "TSO"); | ||||
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_vxlan", | |||||
CTLFLAG_RD, &(tx_que->tx_vxlan), | |||||
"VXLAN HW Offload"); | |||||
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mss_too_small", | SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mss_too_small", | ||||
CTLFLAG_RD, &(txr->mss_too_small), | CTLFLAG_RD, &(txr->mss_too_small), | ||||
"TSO sends with an MSS less than 64"); | "TSO sends with an MSS less than 64"); | ||||
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "packets", | SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "packets", | ||||
CTLFLAG_RD, &(txr->tx_packets), | CTLFLAG_RD, &(txr->tx_packets), | ||||
"Queue Packets Transmitted"); | "Queue Packets Transmitted"); | ||||
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "bytes", | SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "bytes", | ||||
CTLFLAG_RD, &(txr->tx_bytes), | CTLFLAG_RD, &(txr->tx_bytes), | ||||
"Queue Bytes Transmitted"); | "Queue Bytes Transmitted"); | ||||
SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "itr", | SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "itr", | ||||
CTLFLAG_RD, &(txr->itr), 0, | CTLFLAG_RD, &(txr->itr), 0, | ||||
"Queue Tx ITR Interval"); | "Queue Tx ITR Interval"); | ||||
} | } | ||||
} | } |