Differential D8299 Diff 23742 sys/dev/e1000/em_txrx.c

Changeset View

Standalone View

sys/dev/e1000/em_txrx.c

This file was added.

Property	Old Value	New Value
svn:eol-style	null	native \ No newline at end of property
svn:keywords	null	FreeBSD=%H \ No newline at end of property
svn:mime-type	null	text/plain \ No newline at end of property

				#ifndef KLD_MODULE
				#include "opt_iflib.h"
				#endif

				#include "if_em.h"

				#ifdef RSS
				#include <net/rss_config.h>
				#include <netinet/in_rss.h>
				#endif

				#ifdef VERBOSE_DEBUG
				#define DPRINTF device_printf
				#else
				#define DPRINTF(...)
				#endif

				/*********************************************************************
				* Local Function prototypes
				*********************************************************************/
				static int em_tso_setup(struct adapter adapter, if_pkt_info_t pi, u32 txd_upper, u32 *txd_lower);
				static int em_transmit_checksum_setup(struct adapter adapter, if_pkt_info_t pi, u32 txd_upper, u32 *txd_lower);
				static int em_isc_txd_encap(void *arg, if_pkt_info_t pi);
				static void em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx);
				static int em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear);
				static void em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
				uint32_t pidx, uint64_t paddrs, caddr_t vaddrs __unused, uint16_t count, uint16_t buflen __unused);
				static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx);
				static int em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx,
				int budget);
				static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);

				static void lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
				uint32_t pidx, uint64_t paddrs, caddr_t vaddrs __unused, uint16_t count, uint16_t buflen __unused);

				static int lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx,
				int budget);
				static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);

				static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri);
				static void em_receive_checksum(uint32_t status, if_rxd_info_t ri);
				extern int em_intr(void *arg);

				struct if_txrx em_txrx = {
				em_isc_txd_encap,
				em_isc_txd_flush,
				em_isc_txd_credits_update,
				em_isc_rxd_available,
				em_isc_rxd_pkt_get,
				em_isc_rxd_refill,
				em_isc_rxd_flush,
				em_intr
				};

				struct if_txrx lem_txrx = {
				em_isc_txd_encap,
				em_isc_txd_flush,
				em_isc_txd_credits_update,
				lem_isc_rxd_available,
				lem_isc_rxd_pkt_get,
				lem_isc_rxd_refill,
				em_isc_rxd_flush,
				em_intr
				};

				extern if_shared_ctx_t em_sctx;

				/**********************************************************************
				*
				* Setup work for hardware segmentation offload (TSO) on
				* adapters using advanced tx descriptors
				*
				**********************************************************************/
				static int
				em_tso_setup(struct adapter adapter, if_pkt_info_t pi, u32 txd_upper, u32 *txd_lower)
				{
				if_softc_ctx_t scctx = adapter->shared;
				struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx];
				struct tx_ring *txr = &que->txr;
				struct e1000_context_desc *TXD;
				struct em_txbuffer *tx_buffer;
				int cur, hdr_len;

				hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
				txd_lower = (E1000_TXD_CMD_DEXT \| / Extended descr type */
				E1000_TXD_DTYP_D \| /* Data descr type */
				E1000_TXD_CMD_TSE); /* Do TSE on this packet */

				/* IP and/or TCP header checksum calculation and insertion. */
				*txd_upper = (E1000_TXD_POPTS_IXSM \| E1000_TXD_POPTS_TXSM) << 8;

				cur = pi->ipi_pidx;
				TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
				tx_buffer = &txr->tx_buffers[cur];

				/*
				* Start offset for header checksum calculation.
				* End offset for header checksum calculation.
				* Offset of place put the checksum.
				*/
				TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
				TXD->lower_setup.ip_fields.ipcse =
				htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1);
				TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);

				/*
				* Start offset for payload checksum calculation.
				* End offset for payload checksum calculation.
				* Offset of place to put the checksum.
				*/
				TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
				TXD->upper_setup.tcp_fields.tucse = 0;
				TXD->upper_setup.tcp_fields.tucso =
				pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum);

				/*
				* Payload size per packet w/o any headers.
				* Length of all headers up to payload.
				*/
				TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz);
				TXD->tcp_seg_setup.fields.hdr_len = hdr_len;

				TXD->cmd_and_length = htole32(adapter->txd_cmd \|
				E1000_TXD_CMD_DEXT \| /* Extended descr */
				E1000_TXD_CMD_TSE \| /* TSE context */
				E1000_TXD_CMD_IP \| /* Do IP csum */
				E1000_TXD_CMD_TCP \| /* Do TCP checksum */
				(pi->ipi_len - hdr_len)); /* Total len */
				tx_buffer->eop = -1;
				txr->tx_tso = TRUE;

				if (++cur == scctx->isc_ntxd[0]) {
				cur = 0;
				}
				DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur);
				return (cur);
				}

				#define TSO_WORKAROUND 4
				#define DONT_FORCE_CTX 1


				/*********************************************************************
				* The offload context is protocol specific (TCP/UDP) and thus
				* only needs to be set when the protocol changes. The occasion
				* of a context change can be a performance detriment, and
				* might be better just disabled. The reason arises in the way
				* in which the controller supports pipelined requests from the
				* Tx data DMA. Up to four requests can be pipelined, and they may
				* belong to the same packet or to multiple packets. However all
				* requests for one packet are issued before a request is issued
				* for a subsequent packet and if a request for the next packet
				* requires a context change, that request will be stalled
				* until the previous request completes. This means setting up
				* a new context effectively disables pipelined Tx data DMA which
				* in turn greatly slow down performance to send small sized
				* frames.
				**********************************************************************/

				static int
				em_transmit_checksum_setup(struct adapter adapter, if_pkt_info_t pi, u32 txd_upper, u32 *txd_lower)
				{
				struct e1000_context_desc *TXD = NULL;
				if_softc_ctx_t scctx = adapter->shared;
				struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx];
				struct tx_ring *txr = &que->txr;
				struct em_txbuffer *tx_buffer;
				int csum_flags = pi->ipi_csum_flags;
				int cur, hdr_len;
				u32 cmd;

				cur = pi->ipi_pidx;
				hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
				cmd = adapter->txd_cmd;

				/*
				* The 82574L can only remember the last context used
				* regardless of queue that it was use for. We cannot reuse
				* contexts on this hardware platform and must generate a new
				* context every time. 82574L hardware spec, section 7.2.6,
				* second note.
				*/
				if (DONT_FORCE_CTX &&
				adapter->tx_num_queues == 1 &&
				txr->csum_lhlen == pi->ipi_ehdrlen &&
				txr->csum_iphlen == pi->ipi_ip_hlen &&
				txr->csum_flags == csum_flags) {
				/*
				* Same csum offload context as the previous packets;
				* just return.
				*/
				*txd_upper = txr->csum_txd_upper;
				*txd_lower = txr->csum_txd_lower;
				return (cur);
				}

				TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
				if (csum_flags & CSUM_IP) {
				*txd_upper \|= E1000_TXD_POPTS_IXSM << 8;
				/*
				* Start offset for header checksum calculation.
				* End offset for header checksum calculation.
				* Offset of place to put the checksum.
				*/
				TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
				TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
				TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
				cmd \|= E1000_TXD_CMD_IP;
				}

				if (csum_flags & (CSUM_TCP\|CSUM_UDP)) {
				uint8_t tucso;

				*txd_upper \|= E1000_TXD_POPTS_TXSM << 8;
				*txd_lower = E1000_TXD_CMD_DEXT \| E1000_TXD_DTYP_D;

				if (csum_flags & CSUM_TCP) {
				tucso = hdr_len + offsetof(struct tcphdr, th_sum);
				cmd \|= E1000_TXD_CMD_TCP;
				} else
				tucso = hdr_len + offsetof(struct udphdr, uh_sum);
				TXD->upper_setup.tcp_fields.tucss = hdr_len;
				TXD->upper_setup.tcp_fields.tucse = htole16(0);
				TXD->upper_setup.tcp_fields.tucso = tucso;
				}

				txr->csum_lhlen = pi->ipi_ehdrlen;
				txr->csum_iphlen = pi->ipi_ip_hlen;
				txr->csum_flags = csum_flags;
				txr->csum_txd_upper = *txd_upper;
				txr->csum_txd_lower = *txd_lower;

				TXD->tcp_seg_setup.data = htole32(0);
				TXD->cmd_and_length =
				htole32(E1000_TXD_CMD_IFCS \| E1000_TXD_CMD_DEXT \| cmd);

				tx_buffer = &txr->tx_buffers[cur];
				tx_buffer->eop = -1;

				if (++cur == scctx->isc_ntxd[0]) {
				cur = 0;
				}
				DPRINTF(iflib_get_dev(adapter->ctx), "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n",
				csum_flags, txd_upper, txd_lower, hdr_len, cmd);
				return (cur);
				}

				static int
				em_isc_txd_encap(void *arg, if_pkt_info_t pi)
				{
				struct adapter *sc = arg;
				if_softc_ctx_t scctx = sc->shared;
				struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
				struct tx_ring *txr = &que->txr;
				bus_dma_segment_t *segs = pi->ipi_segs;
				int nsegs = pi->ipi_nsegs;
				int csum_flags = pi->ipi_csum_flags;
				int i, j, first, pidx_last;
				u32 txd_upper = 0, txd_lower = 0;

				struct em_txbuffer *tx_buffer;
				struct e1000_tx_desc *ctxd = NULL;
				bool do_tso, tso_desc;

				i = first = pi->ipi_pidx;
				do_tso = (csum_flags & CSUM_TSO);
				tso_desc = FALSE;
				/*
				* TSO Hardware workaround, if this packet is not
				* TSO, and is only a single descriptor long, and
				* it follows a TSO burst, then we need to add a
				* sentinel descriptor to prevent premature writeback.
				*/
				if ((!do_tso) && (txr->tx_tso == TRUE)) {
				if (nsegs == 1)
				tso_desc = TRUE;
				txr->tx_tso = FALSE;
				}

				/* Do hardware assists */
				if (do_tso) {
				i = em_tso_setup(sc, pi, &txd_upper, &txd_lower);
				tso_desc = TRUE;
				} else if (csum_flags & CSUM_OFFLOAD) {
				i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower);
				}

				if (pi->ipi_mflags & M_VLANTAG) {
				/* Set the vlan id. */
				txd_upper \|= htole16(pi->ipi_vtag) << 16;
				/* Tell hardware to add tag */
				txd_lower \|= htole32(E1000_TXD_CMD_VLE);
				}

				DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i);
				/* XXX adapter->pcix_82544 -- lem_fill_descriptors */

				/* Set up our transmit descriptors */
				for (j = 0; j < nsegs; j++) {
				bus_size_t seg_len;
				bus_addr_t seg_addr;
				uint32_t cmd;

				ctxd = &txr->tx_base[i];
				tx_buffer = &txr->tx_buffers[i];
				seg_addr = segs[j].ds_addr;
				seg_len = segs[j].ds_len;
				cmd = E1000_TXD_CMD_IFCS \| sc->txd_cmd;

				/*
				** TSO Workaround:
				** If this is the last descriptor, we want to
				** split it so we have a small final sentinel
				*/
				if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
				seg_len -= TSO_WORKAROUND;
				ctxd->buffer_addr = htole64(seg_addr);
				ctxd->lower.data = htole32(cmd \| txd_lower \| seg_len);
				ctxd->upper.data = htole32(txd_upper);

				if (++i == scctx->isc_ntxd[0])
				i = 0;

				/* Now make the sentinel */
				ctxd = &txr->tx_base[i];
				tx_buffer = &txr->tx_buffers[i];
				ctxd->buffer_addr = htole64(seg_addr + seg_len);
				ctxd->lower.data = htole32(cmd \| txd_lower \| TSO_WORKAROUND);
				ctxd->upper.data = htole32(txd_upper);
				pidx_last = i;
				if (++i == scctx->isc_ntxd[0])
				i = 0;
				DPRINTF(iflib_get_dev(sc->ctx), "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]);
				} else {
				ctxd->buffer_addr = htole64(seg_addr);
				ctxd->lower.data = htole32(cmd \| txd_lower \| seg_len);
				ctxd->upper.data = htole32(txd_upper);
				pidx_last = i;
				if (++i == scctx->isc_ntxd[0])
				i = 0;
				DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]);
				}
				tx_buffer->eop = -1;
				}

				/*
				* Last Descriptor of Packet
				* needs End Of Packet (EOP)
				* and Report Status (RS)
				*/
				ctxd->lower.data \|=
				htole32(E1000_TXD_CMD_EOP \| E1000_TXD_CMD_RS);

				tx_buffer = &txr->tx_buffers[first];
				tx_buffer->eop = pidx_last;
				DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i);
				pi->ipi_new_pidx = i;

				return (0);
				}

				static void
				em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx)
				{
				struct adapter *adapter = arg;
				struct em_tx_queue *que = &adapter->tx_queues[txqid];
				struct tx_ring *txr = &que->txr;

				E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx);
				}

				static int
				em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear)
				{
				struct adapter *adapter = arg;
				if_softc_ctx_t scctx = adapter->shared;
				struct em_tx_queue *que = &adapter->tx_queues[txqid];
				struct tx_ring *txr = &que->txr;

				u32 cidx, processed = 0;
				int last, done;
				struct em_txbuffer *buf;
				struct e1000_tx_desc tx_desc, eop_desc;

				cidx = cidx_init;
				buf = &txr->tx_buffers[cidx];
				tx_desc = &txr->tx_base[cidx];
				last = buf->eop;
				eop_desc = &txr->tx_base[last];

				DPRINTF(iflib_get_dev(adapter->ctx), "credits_update: cidx_init=%d clear=%d last=%d\n",
				cidx_init, clear, last);
				/*
				* What this does is get the index of the
				* first descriptor AFTER the EOP of the
				* first packet, that way we can do the
				* simple comparison on the inner while loop.
				*/
				if (++last == scctx->isc_ntxd[0])
				last = 0;
				done = last;


				while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
				/* We clean the range of the packet */
				while (cidx != done) {
				if (clear) {
				tx_desc->upper.data = 0;
				tx_desc->lower.data = 0;
				tx_desc->buffer_addr = 0;
				buf->eop = -1;
				}
				tx_desc++;
				buf++;
				processed++;

				/* wrap the ring ? */
				if (++cidx == scctx->isc_ntxd[0]) {
				cidx = 0;
				}
				buf = &txr->tx_buffers[cidx];
				tx_desc = &txr->tx_base[cidx];
				}
				/* See if we can continue to the next packet */
				last = buf->eop;
				if (last == -1)
				break;
				eop_desc = &txr->tx_base[last];
				/* Get new done point */
				if (++last == scctx->isc_ntxd[0])
				last = 0;
				done = last;
				}

				DPRINTF(iflib_get_dev(adapter->ctx), "Processed %d credits update\n", processed);
				return(processed);
				}

				static void
				lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
				uint32_t pidx, uint64_t paddrs, caddr_t vaddrs __unused, uint16_t count, uint16_t buflen __unused)
				{
				struct adapter *sc = arg;
				if_softc_ctx_t scctx = sc->shared;
				struct em_rx_queue *que = &sc->rx_queues[rxqid];
				struct rx_ring *rxr = &que->rxr;
				struct e1000_rx_desc *rxd;
				int i;
				uint32_t next_pidx;

				for (i = 0, next_pidx = pidx; i < count; i++) {
				rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx];
				rxd->buffer_addr = htole64(paddrs[i]);
				/* status bits must be cleared */
				rxd->status = 0;

				if (++next_pidx == scctx->isc_nrxd[0])
				next_pidx = 0;
				}
				}

				static void
				em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
				uint32_t pidx, uint64_t paddrs, caddr_t vaddrs __unused, uint16_t count, uint16_t buflen __unused)
				{
				struct adapter *sc = arg;
				if_softc_ctx_t scctx = sc->shared;
				struct em_rx_queue *que = &sc->rx_queues[rxqid];
				struct rx_ring *rxr = &que->rxr;
				union e1000_rx_desc_extended *rxd;
				int i;
				uint32_t next_pidx;

				for (i = 0, next_pidx = pidx; i < count; i++) {
				rxd = &rxr->rx_base[next_pidx];
				rxd->read.buffer_addr = htole64(paddrs[i]);
				/* DD bits must be cleared */
				rxd->wb.upper.status_error = 0;

				if (++next_pidx == scctx->isc_nrxd[0])
				next_pidx = 0;
				}
				}

				static void
				em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx)
				{
				struct adapter *sc = arg;
				struct em_rx_queue *que = &sc->rx_queues[rxqid];
				struct rx_ring *rxr = &que->rxr;

				E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
				}

				static int
				lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget)
				{
				struct adapter *sc = arg;
				if_softc_ctx_t scctx = sc->shared;
				struct em_rx_queue *que = &sc->rx_queues[rxqid];
				struct rx_ring *rxr = &que->rxr;
				struct e1000_rx_desc *rxd;
				u32 staterr = 0;
				int cnt, i;

				for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
				rxd = (struct e1000_rx_desc *)&rxr->rx_base[i];
				staterr = rxd->status;

				if ((staterr & E1000_RXD_STAT_DD) == 0)
				break;

				if (++i == scctx->isc_nrxd[0])
				i = 0;

				if (staterr & E1000_RXD_STAT_EOP)
				cnt++;
				}
				return (cnt);
				}

				static int
				em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget)
				{
				struct adapter *sc = arg;
				if_softc_ctx_t scctx = sc->shared;
				struct em_rx_queue *que = &sc->rx_queues[rxqid];
				struct rx_ring *rxr = &que->rxr;
				union e1000_rx_desc_extended *rxd;
				u32 staterr = 0;
				int cnt, i;

				for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
				rxd = &rxr->rx_base[i];
				staterr = le32toh(rxd->wb.upper.status_error);

				if ((staterr & E1000_RXD_STAT_DD) == 0)
				break;

				if (++i == scctx->isc_nrxd[0]) {
				i = 0;
				}

				if (staterr & E1000_RXD_STAT_EOP)
				cnt++;

				}
				return (cnt);
				}

				static int
				lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
				{
				struct adapter *adapter = arg;
				if_softc_ctx_t scctx = adapter->shared;
				struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
				struct rx_ring *rxr = &que->rxr;
				struct e1000_rx_desc *rxd;
				u16 len;
				u32 status, errors;
				bool eop;
				int i, cidx;

				status = errors = i = 0;
				cidx = ri->iri_cidx;

				do {
				rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx];
				status = rxd->status;
				errors = rxd->errors;

				/* Error Checking then decrement count */
				MPASS ((status & E1000_RXD_STAT_DD) != 0);

				len = le16toh(rxd->length);
				ri->iri_len += len;

				eop = (status & E1000_RXD_STAT_EOP) != 0;

				/* Make sure bad packets are discarded */
				if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
				adapter->dropped_pkts++;
				/* XXX fixup if common */
				return (EBADMSG);
				}

				ri->iri_frags[i].irf_flid = 0;
				ri->iri_frags[i].irf_idx = cidx;
				ri->iri_frags[i].irf_len = len;
				/* Zero out the receive descriptors status. */
				rxd->status = 0;

				if (++cidx == scctx->isc_nrxd[0])
				cidx = 0;
				i++;
				} while (!eop);

				/* XXX add a faster way to look this up */
				if (adapter->hw.mac.type >= e1000_82543 && !(status & E1000_RXD_STAT_IXSM))
				lem_receive_checksum(status, errors, ri);

				if (status & E1000_RXD_STAT_VP) {
				ri->iri_vtag = le16toh(rxd->special);
				ri->iri_flags \|= M_VLANTAG;
				}

				ri->iri_nfrags = i;

				return (0);
				}

				static int
				em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
				{
				struct adapter *adapter = arg;
				if_softc_ctx_t scctx = adapter->shared;
				struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
				struct rx_ring *rxr = &que->rxr;
				union e1000_rx_desc_extended *rxd;

				u16 len;
				u32 staterr = 0;
				bool eop;
				int i, cidx, vtag;

				i = vtag = 0;
				cidx = ri->iri_cidx;

				do {
				rxd = &rxr->rx_base[cidx];
				staterr = le32toh(rxd->wb.upper.status_error);

				/* Error Checking then decrement count */
				MPASS ((staterr & E1000_RXD_STAT_DD) != 0);

				len = le16toh(rxd->wb.upper.length);
				ri->iri_len += len;

				eop = (staterr & E1000_RXD_STAT_EOP) != 0;

				/* Make sure bad packets are discarded */
				if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
				adapter->dropped_pkts++;
				return EBADMSG;
				}

				ri->iri_frags[i].irf_flid = 0;
				ri->iri_frags[i].irf_idx = cidx;
				ri->iri_frags[i].irf_len = len;
				/* Zero out the receive descriptors status. */
				rxd->wb.upper.status_error &= htole32(~0xFF);

				if (++cidx == scctx->isc_nrxd[0])
				cidx = 0;
				i++;
				} while (!eop);

				/* XXX add a faster way to look this up */
				if (adapter->hw.mac.type >= e1000_82543)
				em_receive_checksum(staterr, ri);

				if (staterr & E1000_RXD_STAT_VP) {
				vtag = le16toh(rxd->wb.upper.vlan);
				}

				ri->iri_vtag = vtag;
				ri->iri_nfrags = i;
				if (vtag)
				ri->iri_flags \|= M_VLANTAG;

				return (0);
				}

				/*********************************************************************
				*
				* Verify that the hardware indicated that the checksum is valid.
				* Inform the stack about the status of checksum so that stack
				* doesn't spend time verifying the checksum.
				*
				*********************************************************************/
				static void
				lem_receive_checksum(int status, int errors, if_rxd_info_t ri)
				{
				/* Did it pass? */
				if (status & E1000_RXD_STAT_IPCS && !(errors & E1000_RXD_ERR_IPE))
				ri->iri_csum_flags = (CSUM_IP_CHECKED\|CSUM_IP_VALID);

				if (status & E1000_RXD_STAT_TCPCS) {
				/* Did it pass? */
				if (!(errors & E1000_RXD_ERR_TCPE)) {
				ri->iri_csum_flags \|=
				(CSUM_DATA_VALID \| CSUM_PSEUDO_HDR);
				ri->iri_csum_data = htons(0xffff);
				}
				}
				}

				static void
				em_receive_checksum(uint32_t status, if_rxd_info_t ri)
				{
				ri->iri_csum_flags = 0;

				/* Ignore Checksum bit is set */
				if (status & E1000_RXD_STAT_IXSM)
				return;

				/* If the IP checksum exists and there is no IP Checksum error */
				if ((status & (E1000_RXD_STAT_IPCS \| E1000_RXDEXT_STATERR_IPE)) ==
				E1000_RXD_STAT_IPCS) {
				ri->iri_csum_flags = (CSUM_IP_CHECKED \| CSUM_IP_VALID);
				}

				/* TCP or UDP checksum */
				if ((status & (E1000_RXD_STAT_TCPCS \| E1000_RXDEXT_STATERR_TCPE)) ==
				E1000_RXD_STAT_TCPCS) {
				ri->iri_csum_flags \|= (CSUM_DATA_VALID \| CSUM_PSEUDO_HDR);
				ri->iri_csum_data = htons(0xffff);
				}
				if (status & E1000_RXD_STAT_UDPCS) {
				ri->iri_csum_flags \|= (CSUM_DATA_VALID \| CSUM_PSEUDO_HDR);
				ri->iri_csum_data = htons(0xffff);
				}
				}