Differential D21959 Diff 66316 sys/dev/ice/ice_iflib_txrx.c

Changeset View

Standalone View

sys/dev/ice/ice_iflib_txrx.c

This file was added.

				/* SPDX-License-Identifier: BSD-3-Clause */
				/* Copyright (c) 2020, Intel Corporation
				* All rights reserved.
				*
				* Redistribution and use in source and binary forms, with or without
				* modification, are permitted provided that the following conditions are met:
				*
				* 1. Redistributions of source code must retain the above copyright notice,
				* this list of conditions and the following disclaimer.
				*
				* 2. Redistributions in binary form must reproduce the above copyright
				* notice, this list of conditions and the following disclaimer in the
				* documentation and/or other materials provided with the distribution.
				*
				* 3. Neither the name of the Intel Corporation nor the names of its
				* contributors may be used to endorse or promote products derived from
				* this software without specific prior written permission.
				*
				* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
				* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
				* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
				* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
				* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
				* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
				* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
				* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
				* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
				* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
				* POSSIBILITY OF SUCH DAMAGE.
				*/
				/$FreeBSD$/

				/**
				* @file ice_iflib_txrx.c
				* @brief iflib Tx/Rx hotpath
				*
				* Main location for the iflib Tx/Rx hotpath implementation.
				*
				* Contains the implementation for the iflib function callbacks and the
				* if_txrx ops structure.
				*/

				#include "ice_iflib.h"

				/* Tx/Rx hotpath utility functions */
				#include "ice_common_txrx.h"

				/*
				* iflib txrx method declarations
				*/
				static int ice_ift_txd_encap(void *arg, if_pkt_info_t pi);
				static int ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri);
				static void ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
				static int ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear);
				static int ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget);
				static void ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx, qidx_t pidx);
				static void ice_ift_rxd_refill(void *arg, if_rxd_update_t iru);

				/* Macro to help extract the NIC mode flexible Rx descriptor fields from the
				* advanced 32byte Rx descriptors.
				*/
				#define RX_FLEX_NIC(desc, field) \
				(((struct ice_32b_rx_flex_desc_nic *)desc)->field)

				/**
				* @var ice_txrx
				* @brief Tx/Rx operations for the iflib stack
				*
				* Structure defining the Tx and Rx related operations that iflib can request
				* the driver to perform. These are the main entry points for the hot path of
				* the transmit and receive paths in the iflib driver.
				*/
				struct if_txrx ice_txrx = {
				.ift_txd_encap = ice_ift_txd_encap,
				.ift_txd_flush = ice_ift_txd_flush,
				.ift_txd_credits_update = ice_ift_txd_credits_update,
				.ift_rxd_available = ice_ift_rxd_available,
				.ift_rxd_pkt_get = ice_ift_rxd_pkt_get,
				.ift_rxd_refill = ice_ift_rxd_refill,
				.ift_rxd_flush = ice_ift_rxd_flush,
				};

				/**
				* ice_ift_txd_encap - prepare Tx descriptors for a packet
				* @arg: the iflib softc structure pointer
				* @pi: packet info
				*
				* Prepares and encapsulates the given packet into into Tx descriptors, in
				* preparation for sending to the transmit engine. Sets the necessary context
				* descriptors for TSO and other offloads, and prepares the last descriptor
				* for the writeback status.
				*
				* Return 0 on success, non-zero error code on failure.
				*/
				static int
				ice_ift_txd_encap(void *arg, if_pkt_info_t pi)
				{
				struct ice_softc sc = (struct ice_softc )arg;
				struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[pi->ipi_qsidx];
				int nsegs = pi->ipi_nsegs;
				bus_dma_segment_t *segs = pi->ipi_segs;
				struct ice_tx_desc *txd = NULL;
				int i, j, mask, pidx_last;
				u32 cmd, off;

				cmd = off = 0;
				i = pi->ipi_pidx;

				/* Set up the TSO/CSUM offload */
				if (pi->ipi_csum_flags & ICE_CSUM_OFFLOAD) {
				/* Set up the TSO context descriptor if required */
				if (pi->ipi_csum_flags & CSUM_TSO) {
				if (ice_tso_detect_sparse(pi))
				return (EFBIG);
				i = ice_tso_setup(txq, pi);
				}
				ice_tx_setup_offload(txq, pi, &cmd, &off);
				}
				if (pi->ipi_mflags & M_VLANTAG)
				cmd \|= ICE_TX_DESC_CMD_IL2TAG1;

				mask = txq->desc_count - 1;
				for (j = 0; j < nsegs; j++) {
				bus_size_t seglen;

				txd = &txq->tx_base[i];
				seglen = segs[j].ds_len;

				txd->buf_addr = htole64(segs[j].ds_addr);
				txd->cmd_type_offset_bsz =
				htole64(ICE_TX_DESC_DTYPE_DATA
				\| ((u64)cmd << ICE_TXD_QW1_CMD_S)
				\| ((u64)off << ICE_TXD_QW1_OFFSET_S)
				\| ((u64)seglen << ICE_TXD_QW1_TX_BUF_SZ_S)
				\| ((u64)htole16(pi->ipi_vtag) << ICE_TXD_QW1_L2TAG1_S));

				txq->stats.tx_bytes += seglen;
				pidx_last = i;
				i = (i+1) & mask;
				}

				/* Set the last descriptor for report */
				#define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP \| ICE_TX_DESC_CMD_RS)
				txd->cmd_type_offset_bsz \|=
				htole64(((u64)ICE_TXD_CMD << ICE_TXD_QW1_CMD_S));

				/* Add to report status array */
				txq->tx_rsq[txq->tx_rs_pidx] = pidx_last;
				txq->tx_rs_pidx = (txq->tx_rs_pidx+1) & mask;
				MPASS(txq->tx_rs_pidx != txq->tx_rs_cidx);

				pi->ipi_new_pidx = i;

				++txq->stats.tx_packets;
				return (0);
				}

				/**
				* ice_ift_txd_flush - Flush Tx descriptors to hardware
				* @arg: device specific softc pointer
				* @txqid: the Tx queue to flush
				* @pidx: descriptor index to advance tail to
				*
				* Advance the Transmit Descriptor Tail (TDT). This indicates to hardware that
				* frames are available for transmit.
				*/
				static void
				ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
				{
				struct ice_softc sc = (struct ice_softc )arg;
				struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];
				struct ice_hw *hw = &sc->hw;

				wr32(hw, txq->tail, pidx);
				}

				/**
				* ice_ift_txd_credits_update - cleanup Tx descriptors
				* @arg: device private softc
				* @txqid: the Tx queue to update
				* @clear: if false, only report, do not actually clean
				*
				* If clear is false, iflib is asking if we could clean up any Tx
				* descriptors.
				*
				* If clear is true, iflib is requesting to cleanup and reclaim used Tx
				* descriptors.
				*/
				static int
				ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear)
				{
				struct ice_softc sc = (struct ice_softc )arg;
				struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];

				qidx_t processed = 0;
				qidx_t cur, prev, ntxd, rs_cidx;
				int32_t delta;
				bool is_done;

				rs_cidx = txq->tx_rs_cidx;
				if (rs_cidx == txq->tx_rs_pidx)
				return (0);
				cur = txq->tx_rsq[rs_cidx];
				MPASS(cur != QIDX_INVALID);
				is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);

				if (!is_done)
				return (0);
				else if (clear == false)
				return (1);

				prev = txq->tx_cidx_processed;
				ntxd = txq->desc_count;
				do {
				MPASS(prev != cur);
				delta = (int32_t)cur - (int32_t)prev;
				if (delta < 0)
				delta += ntxd;
				MPASS(delta > 0);
				processed += delta;
				prev = cur;
				rs_cidx = (rs_cidx + 1) & (ntxd-1);
				if (rs_cidx == txq->tx_rs_pidx)
				break;
				cur = txq->tx_rsq[rs_cidx];
				MPASS(cur != QIDX_INVALID);
				is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);
				} while (is_done);

				txq->tx_rs_cidx = rs_cidx;
				txq->tx_cidx_processed = prev;

				return (processed);
				}

				/**
				* ice_ift_rxd_available - Return number of available Rx packets
				* @arg: device private softc
				* @rxqid: the Rx queue id
				* @pidx: descriptor start point
				* @budget: maximum Rx budget
				*
				* Determines how many Rx packets are available on the queue, up to a maximum
				* of the given budget.
				*/
				static int
				ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget)
				{
				struct ice_softc sc = (struct ice_softc )arg;
				struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];
				union ice_32b_rx_flex_desc *rxd;
				uint16_t status0;
				int cnt, i, nrxd;

				nrxd = rxq->desc_count;

				for (cnt = 0, i = pidx; cnt < nrxd - 1 && cnt < budget;) {
				rxd = &rxq->rx_base[i];
				status0 = le16toh(rxd->wb.status_error0);

				if ((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) == 0)
				break;
				if (++i == nrxd)
				i = 0;
				if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S))
				cnt++;
				}

				return (cnt);
				}

				/**
				* ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer
				* @arg: device specific softc
				* @ri: receive packet info
				*
				* This function is called by iflib, and executes in ithread context. It is
				* called by iflib to obtain data which has been DMA'ed into host memory.
				* Returns zero on success, and an error code on failure.
				*/
				static int
				ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri)
				{
				struct ice_softc sc = (struct ice_softc )arg;
				struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[ri->iri_qsidx];
				union ice_32b_rx_flex_desc *cur;
				u16 status0, plen, vtag, ptype;
				bool eop;
				size_t cidx;
				int i;

				cidx = ri->iri_cidx;
				i = 0;
				do {
				/* 5 descriptor receive limit */
				MPASS(i < ICE_MAX_RX_SEGS);

				cur = &rxq->rx_base[cidx];
				status0 = le16toh(cur->wb.status_error0);
				plen = le16toh(cur->wb.pkt_len) &
				ICE_RX_FLX_DESC_PKT_LEN_M;
				ptype = le16toh(cur->wb.ptype_flex_flags0) &
				ICE_RX_FLEX_DESC_PTYPE_M;

				/* we should never be called without a valid descriptor */
				MPASS((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) != 0);

				ri->iri_len += plen;

				cur->wb.status_error0 = 0;
				eop = (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S));
				if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S))
				vtag = le16toh(cur->wb.l2tag1);
				else
				vtag = 0;

				/*
				* Make sure packets with bad L2 values are discarded.
				* NOTE: Only the EOP descriptor has valid error results.
				*/
				if (eop && (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S))) {
				rxq->stats.desc_errs++;
				return (EBADMSG);
				}
				ri->iri_frags[i].irf_flid = 0;
				ri->iri_frags[i].irf_idx = cidx;
				ri->iri_frags[i].irf_len = plen;
				if (++cidx == rxq->desc_count)
				cidx = 0;
				i++;
				} while (!eop);

				/* capture soft statistics for this Rx queue */
				rxq->stats.rx_packets++;
				rxq->stats.rx_bytes += ri->iri_len;

				if ((iflib_get_ifp(sc->ctx)->if_capenable & IFCAP_RXCSUM) != 0)
				ice_rx_checksum(rxq, &ri->iri_csum_flags,
				&ri->iri_csum_data, status0, ptype);
				ri->iri_flowid = le32toh(RX_FLEX_NIC(&cur->wb, rss_hash));
				ri->iri_rsstype = ice_ptype_to_hash(ptype);
				ri->iri_vtag = vtag;
				ri->iri_nfrags = i;
				if (vtag)
				ri->iri_flags \|= M_VLANTAG;
				return (0);
				}

				/**
				* ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware
				* @arg: device specific softc structure
				* @iru: the Rx descriptor update structure
				*
				* Update the Rx descriptor indices for a given queue, assigning new physical
				* addresses to the descriptors, preparing them for re-use by the hardware.
				*/
				static void
				ice_ift_rxd_refill(void *arg, if_rxd_update_t iru)
				{
				struct ice_softc sc = (struct ice_softc )arg;
				struct ice_rx_queue *rxq;
				uint32_t next_pidx;
				int i;
				uint64_t *paddrs;
				uint32_t pidx;
				uint16_t qsidx, count;

				paddrs = iru->iru_paddrs;
				pidx = iru->iru_pidx;
				qsidx = iru->iru_qsidx;
				count = iru->iru_count;

				rxq = &(sc->pf_vsi.rx_queues[qsidx]);

				for (i = 0, next_pidx = pidx; i < count; i++) {
				rxq->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]);
				if (++next_pidx == (uint32_t)rxq->desc_count)
				next_pidx = 0;
				}
				}

				/**
				* ice_ift_rxd_flush - Flush Rx descriptors to hardware
				* @arg: device specific softc pointer
				* @rxqid: the Rx queue to flush
				* @flidx: unused parameter
				* @pidx: descriptor index to advance tail to
				*
				* Advance the Receive Descriptor Tail (RDT). This indicates to hardware that
				* software is done with the descriptor and it can be recycled.
				*/
				static void
				ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx __unused,
				qidx_t pidx)
				{
				struct ice_softc sc = (struct ice_softc )arg;
				struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];
				struct ice_hw *hw = &sc->hw;

				wr32(hw, rxq->tail, pidx);
				}