Page MenuHomeFreeBSD
Files F150477210

D8299.id23800.diff
No OneTemporary
Actions

D8299.id23800.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
Index: UPDATING
===================================================================
--- UPDATING
+++ UPDATING
@@ -51,6 +51,11 @@
****************************** SPECIAL WARNING: ******************************
+20170105:
+ The igb(4), em(4) and lem(4) ethernet drivers are now implemented via
+ IFLIB. If you have a custom kernel configuration that excludes em(4)
+ but you use igb(4), you need to re-add em(4) to your custom configuration.
+
20161217:
Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 3.9.1.
Please see the 20141231 entry below for information about prerequisites
Index: sys/amd64/conf/GENERIC
===================================================================
--- sys/amd64/conf/GENERIC
+++ sys/amd64/conf/GENERIC
@@ -230,7 +230,6 @@
device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE
device de # DEC/Intel DC21x4x (``Tulip'')
device em # Intel PRO/1000 Gigabit Ethernet Family
-device igb # Intel PRO/1000 PCIE Server Gigabit Family
device ix # Intel PRO/10GbE PCIE PF Ethernet
device ixv # Intel PRO/10GbE PCIE VF Ethernet
device ixl # Intel XL710 40Gbe PCIE Ethernet
Index: sys/conf/files
===================================================================
--- sys/conf/files
+++ sys/conf/files
@@ -1572,43 +1572,43 @@
dev/eisa/eisaconf.c optional eisa
dev/e1000/if_em.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/if_lem.c optional em \
+dev/e1000/em_txrx.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/if_igb.c optional igb \
+dev/e1000/igb_txrx.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_80003es2lan.c optional em | igb \
+dev/e1000/e1000_80003es2lan.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_82540.c optional em | igb \
+dev/e1000/e1000_82540.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_82541.c optional em | igb \
+dev/e1000/e1000_82541.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_82542.c optional em | igb \
+dev/e1000/e1000_82542.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_82543.c optional em | igb \
+dev/e1000/e1000_82543.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_82571.c optional em | igb \
+dev/e1000/e1000_82571.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_82575.c optional em | igb \
+dev/e1000/e1000_82575.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_ich8lan.c optional em | igb \
+dev/e1000/e1000_ich8lan.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_i210.c optional em | igb \
+dev/e1000/e1000_i210.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_api.c optional em | igb \
+dev/e1000/e1000_api.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_mac.c optional em | igb \
+dev/e1000/e1000_mac.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_manage.c optional em | igb \
+dev/e1000/e1000_manage.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_nvm.c optional em | igb \
+dev/e1000/e1000_nvm.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_phy.c optional em | igb \
+dev/e1000/e1000_phy.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_vf.c optional em | igb \
+dev/e1000/e1000_vf.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_mbx.c optional em | igb \
+dev/e1000/e1000_mbx.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
-dev/e1000/e1000_osdep.c optional em | igb \
+dev/e1000/e1000_osdep.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
dev/et/if_et.c optional et
dev/en/if_en_pci.c optional en pci
Index: sys/dev/e1000/em_txrx.c
===================================================================
--- /dev/null
+++ sys/dev/e1000/em_txrx.c
@@ -0,0 +1,719 @@
+#include "if_em.h"
+
+#ifdef RSS
+#include <net/rss_config.h>
+#include <netinet/in_rss.h>
+#endif
+
+#ifdef VERBOSE_DEBUG
+#define DPRINTF device_printf
+#else
+#define DPRINTF(...)
+#endif
+
+/*********************************************************************
+ * Local Function prototypes
+ *********************************************************************/
+static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower);
+static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower);
+static int em_isc_txd_encap(void *arg, if_pkt_info_t pi);
+static void em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx);
+static int em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear);
+static void em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
+ uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused);
+static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx);
+static int em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx,
+ int budget);
+static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
+
+static void lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
+ uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused);
+
+static int lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx,
+ int budget);
+static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
+
+static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri);
+static void em_receive_checksum(uint32_t status, if_rxd_info_t ri);
+extern int em_intr(void *arg);
+
+struct if_txrx em_txrx = {
+ em_isc_txd_encap,
+ em_isc_txd_flush,
+ em_isc_txd_credits_update,
+ em_isc_rxd_available,
+ em_isc_rxd_pkt_get,
+ em_isc_rxd_refill,
+ em_isc_rxd_flush,
+ em_intr
+};
+
+struct if_txrx lem_txrx = {
+ em_isc_txd_encap,
+ em_isc_txd_flush,
+ em_isc_txd_credits_update,
+ lem_isc_rxd_available,
+ lem_isc_rxd_pkt_get,
+ lem_isc_rxd_refill,
+ em_isc_rxd_flush,
+ em_intr
+};
+
+extern if_shared_ctx_t em_sctx;
+
+/**********************************************************************
+ *
+ * Setup work for hardware segmentation offload (TSO) on
+ * adapters using advanced tx descriptors
+ *
+ **********************************************************************/
+static int
+em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower)
+{
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx];
+ struct tx_ring *txr = &que->txr;
+ struct e1000_context_desc *TXD;
+ struct em_txbuffer *tx_buffer;
+ int cur, hdr_len;
+
+ hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
+ *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
+ E1000_TXD_DTYP_D | /* Data descr type */
+ E1000_TXD_CMD_TSE); /* Do TSE on this packet */
+
+ /* IP and/or TCP header checksum calculation and insertion. */
+ *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
+
+ cur = pi->ipi_pidx;
+ TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
+ tx_buffer = &txr->tx_buffers[cur];
+
+ /*
+ * Start offset for header checksum calculation.
+ * End offset for header checksum calculation.
+ * Offset of place put the checksum.
+ */
+ TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
+ TXD->lower_setup.ip_fields.ipcse =
+ htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1);
+ TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
+
+ /*
+ * Start offset for payload checksum calculation.
+ * End offset for payload checksum calculation.
+ * Offset of place to put the checksum.
+ */
+ TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
+ TXD->upper_setup.tcp_fields.tucse = 0;
+ TXD->upper_setup.tcp_fields.tucso =
+ pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum);
+
+ /*
+ * Payload size per packet w/o any headers.
+ * Length of all headers up to payload.
+ */
+ TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz);
+ TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
+
+ TXD->cmd_and_length = htole32(adapter->txd_cmd |
+ E1000_TXD_CMD_DEXT | /* Extended descr */
+ E1000_TXD_CMD_TSE | /* TSE context */
+ E1000_TXD_CMD_IP | /* Do IP csum */
+ E1000_TXD_CMD_TCP | /* Do TCP checksum */
+ (pi->ipi_len - hdr_len)); /* Total len */
+ tx_buffer->eop = -1;
+ txr->tx_tso = TRUE;
+
+ if (++cur == scctx->isc_ntxd[0]) {
+ cur = 0;
+ }
+ DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur);
+ return (cur);
+}
+
+#define TSO_WORKAROUND 4
+#define DONT_FORCE_CTX 1
+
+
+/*********************************************************************
+ * The offload context is protocol specific (TCP/UDP) and thus
+ * only needs to be set when the protocol changes. The occasion
+ * of a context change can be a performance detriment, and
+ * might be better just disabled. The reason arises in the way
+ * in which the controller supports pipelined requests from the
+ * Tx data DMA. Up to four requests can be pipelined, and they may
+ * belong to the same packet or to multiple packets. However all
+ * requests for one packet are issued before a request is issued
+ * for a subsequent packet and if a request for the next packet
+ * requires a context change, that request will be stalled
+ * until the previous request completes. This means setting up
+ * a new context effectively disables pipelined Tx data DMA which
+ * in turn greatly slow down performance to send small sized
+ * frames.
+ **********************************************************************/
+
+static int
+em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower)
+{
+ struct e1000_context_desc *TXD = NULL;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx];
+ struct tx_ring *txr = &que->txr;
+ struct em_txbuffer *tx_buffer;
+ int csum_flags = pi->ipi_csum_flags;
+ int cur, hdr_len;
+ u32 cmd;
+
+ cur = pi->ipi_pidx;
+ hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
+ cmd = adapter->txd_cmd;
+
+ /*
+ * The 82574L can only remember the *last* context used
+ * regardless of queue that it was use for. We cannot reuse
+ * contexts on this hardware platform and must generate a new
+ * context every time. 82574L hardware spec, section 7.2.6,
+ * second note.
+ */
+ if (DONT_FORCE_CTX &&
+ adapter->tx_num_queues == 1 &&
+ txr->csum_lhlen == pi->ipi_ehdrlen &&
+ txr->csum_iphlen == pi->ipi_ip_hlen &&
+ txr->csum_flags == csum_flags) {
+ /*
+ * Same csum offload context as the previous packets;
+ * just return.
+ */
+ *txd_upper = txr->csum_txd_upper;
+ *txd_lower = txr->csum_txd_lower;
+ return (cur);
+ }
+
+ TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
+ if (csum_flags & CSUM_IP) {
+ *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
+ /*
+ * Start offset for header checksum calculation.
+ * End offset for header checksum calculation.
+ * Offset of place to put the checksum.
+ */
+ TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
+ TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
+ TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
+ cmd |= E1000_TXD_CMD_IP;
+ }
+
+ if (csum_flags & (CSUM_TCP|CSUM_UDP)) {
+ uint8_t tucso;
+
+ *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
+ *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
+
+ if (csum_flags & CSUM_TCP) {
+ tucso = hdr_len + offsetof(struct tcphdr, th_sum);
+ cmd |= E1000_TXD_CMD_TCP;
+ } else
+ tucso = hdr_len + offsetof(struct udphdr, uh_sum);
+ TXD->upper_setup.tcp_fields.tucss = hdr_len;
+ TXD->upper_setup.tcp_fields.tucse = htole16(0);
+ TXD->upper_setup.tcp_fields.tucso = tucso;
+ }
+
+ txr->csum_lhlen = pi->ipi_ehdrlen;
+ txr->csum_iphlen = pi->ipi_ip_hlen;
+ txr->csum_flags = csum_flags;
+ txr->csum_txd_upper = *txd_upper;
+ txr->csum_txd_lower = *txd_lower;
+
+ TXD->tcp_seg_setup.data = htole32(0);
+ TXD->cmd_and_length =
+ htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
+
+ tx_buffer = &txr->tx_buffers[cur];
+ tx_buffer->eop = -1;
+
+ if (++cur == scctx->isc_ntxd[0]) {
+ cur = 0;
+ }
+ DPRINTF(iflib_get_dev(adapter->ctx), "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n",
+ csum_flags, *txd_upper, *txd_lower, hdr_len, cmd);
+ return (cur);
+}
+
+static int
+em_isc_txd_encap(void *arg, if_pkt_info_t pi)
+{
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
+ struct tx_ring *txr = &que->txr;
+ bus_dma_segment_t *segs = pi->ipi_segs;
+ int nsegs = pi->ipi_nsegs;
+ int csum_flags = pi->ipi_csum_flags;
+ int i, j, first, pidx_last;
+ u32 txd_upper = 0, txd_lower = 0;
+
+ struct em_txbuffer *tx_buffer;
+ struct e1000_tx_desc *ctxd = NULL;
+ bool do_tso, tso_desc;
+
+ i = first = pi->ipi_pidx;
+ do_tso = (csum_flags & CSUM_TSO);
+ tso_desc = FALSE;
+ /*
+ * TSO Hardware workaround, if this packet is not
+ * TSO, and is only a single descriptor long, and
+ * it follows a TSO burst, then we need to add a
+ * sentinel descriptor to prevent premature writeback.
+ */
+ if ((!do_tso) && (txr->tx_tso == TRUE)) {
+ if (nsegs == 1)
+ tso_desc = TRUE;
+ txr->tx_tso = FALSE;
+ }
+
+ /* Do hardware assists */
+ if (do_tso) {
+ i = em_tso_setup(sc, pi, &txd_upper, &txd_lower);
+ tso_desc = TRUE;
+ } else if (csum_flags & CSUM_OFFLOAD) {
+ i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower);
+ }
+
+ if (pi->ipi_mflags & M_VLANTAG) {
+ /* Set the vlan id. */
+ txd_upper |= htole16(pi->ipi_vtag) << 16;
+ /* Tell hardware to add tag */
+ txd_lower |= htole32(E1000_TXD_CMD_VLE);
+ }
+
+ DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i);
+ /* XXX adapter->pcix_82544 -- lem_fill_descriptors */
+
+ /* Set up our transmit descriptors */
+ for (j = 0; j < nsegs; j++) {
+ bus_size_t seg_len;
+ bus_addr_t seg_addr;
+ uint32_t cmd;
+
+ ctxd = &txr->tx_base[i];
+ tx_buffer = &txr->tx_buffers[i];
+ seg_addr = segs[j].ds_addr;
+ seg_len = segs[j].ds_len;
+ cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd;
+
+ /*
+ ** TSO Workaround:
+ ** If this is the last descriptor, we want to
+ ** split it so we have a small final sentinel
+ */
+ if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
+ seg_len -= TSO_WORKAROUND;
+ ctxd->buffer_addr = htole64(seg_addr);
+ ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
+ ctxd->upper.data = htole32(txd_upper);
+
+ if (++i == scctx->isc_ntxd[0])
+ i = 0;
+
+ /* Now make the sentinel */
+ ctxd = &txr->tx_base[i];
+ tx_buffer = &txr->tx_buffers[i];
+ ctxd->buffer_addr = htole64(seg_addr + seg_len);
+ ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND);
+ ctxd->upper.data = htole32(txd_upper);
+ pidx_last = i;
+ if (++i == scctx->isc_ntxd[0])
+ i = 0;
+ DPRINTF(iflib_get_dev(sc->ctx), "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]);
+ } else {
+ ctxd->buffer_addr = htole64(seg_addr);
+ ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
+ ctxd->upper.data = htole32(txd_upper);
+ pidx_last = i;
+ if (++i == scctx->isc_ntxd[0])
+ i = 0;
+ DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]);
+ }
+ tx_buffer->eop = -1;
+ }
+
+ /*
+ * Last Descriptor of Packet
+ * needs End Of Packet (EOP)
+ * and Report Status (RS)
+ */
+ ctxd->lower.data |=
+ htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
+
+ tx_buffer = &txr->tx_buffers[first];
+ tx_buffer->eop = pidx_last;
+ DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i);
+ pi->ipi_new_pidx = i;
+
+ return (0);
+}
+
+static void
+em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx)
+{
+ struct adapter *adapter = arg;
+ struct em_tx_queue *que = &adapter->tx_queues[txqid];
+ struct tx_ring *txr = &que->txr;
+
+ E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx);
+}
+
+static int
+em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear)
+{
+ struct adapter *adapter = arg;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_tx_queue *que = &adapter->tx_queues[txqid];
+ struct tx_ring *txr = &que->txr;
+
+ u32 cidx, processed = 0;
+ int last, done;
+ struct em_txbuffer *buf;
+ struct e1000_tx_desc *tx_desc, *eop_desc;
+
+ cidx = cidx_init;
+ buf = &txr->tx_buffers[cidx];
+ tx_desc = &txr->tx_base[cidx];
+ last = buf->eop;
+ eop_desc = &txr->tx_base[last];
+
+ DPRINTF(iflib_get_dev(adapter->ctx), "credits_update: cidx_init=%d clear=%d last=%d\n",
+ cidx_init, clear, last);
+ /*
+ * What this does is get the index of the
+ * first descriptor AFTER the EOP of the
+ * first packet, that way we can do the
+ * simple comparison on the inner while loop.
+ */
+ if (++last == scctx->isc_ntxd[0])
+ last = 0;
+ done = last;
+
+
+ while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
+ /* We clean the range of the packet */
+ while (cidx != done) {
+ if (clear) {
+ tx_desc->upper.data = 0;
+ tx_desc->lower.data = 0;
+ tx_desc->buffer_addr = 0;
+ buf->eop = -1;
+ }
+ tx_desc++;
+ buf++;
+ processed++;
+
+ /* wrap the ring ? */
+ if (++cidx == scctx->isc_ntxd[0]) {
+ cidx = 0;
+ }
+ buf = &txr->tx_buffers[cidx];
+ tx_desc = &txr->tx_base[cidx];
+ }
+ /* See if we can continue to the next packet */
+ last = buf->eop;
+ if (last == -1)
+ break;
+ eop_desc = &txr->tx_base[last];
+ /* Get new done point */
+ if (++last == scctx->isc_ntxd[0])
+ last = 0;
+ done = last;
+ }
+
+ DPRINTF(iflib_get_dev(adapter->ctx), "Processed %d credits update\n", processed);
+ return(processed);
+}
+
+static void
+lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
+ uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused)
+{
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
+ struct e1000_rx_desc *rxd;
+ int i;
+ uint32_t next_pidx;
+
+ for (i = 0, next_pidx = pidx; i < count; i++) {
+ rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx];
+ rxd->buffer_addr = htole64(paddrs[i]);
+ /* status bits must be cleared */
+ rxd->status = 0;
+
+ if (++next_pidx == scctx->isc_nrxd[0])
+ next_pidx = 0;
+ }
+}
+
+static void
+em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
+ uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused)
+{
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
+ union e1000_rx_desc_extended *rxd;
+ int i;
+ uint32_t next_pidx;
+
+ for (i = 0, next_pidx = pidx; i < count; i++) {
+ rxd = &rxr->rx_base[next_pidx];
+ rxd->read.buffer_addr = htole64(paddrs[i]);
+ /* DD bits must be cleared */
+ rxd->wb.upper.status_error = 0;
+
+ if (++next_pidx == scctx->isc_nrxd[0])
+ next_pidx = 0;
+ }
+}
+
+static void
+em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx)
+{
+ struct adapter *sc = arg;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
+
+ E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
+}
+
+static int
+lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget)
+{
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
+ struct e1000_rx_desc *rxd;
+ u32 staterr = 0;
+ int cnt, i;
+
+ for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
+ rxd = (struct e1000_rx_desc *)&rxr->rx_base[i];
+ staterr = rxd->status;
+
+ if ((staterr & E1000_RXD_STAT_DD) == 0)
+ break;
+
+ if (++i == scctx->isc_nrxd[0])
+ i = 0;
+
+ if (staterr & E1000_RXD_STAT_EOP)
+ cnt++;
+ }
+ return (cnt);
+}
+
+static int
+em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget)
+{
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
+ union e1000_rx_desc_extended *rxd;
+ u32 staterr = 0;
+ int cnt, i;
+
+ for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
+ rxd = &rxr->rx_base[i];
+ staterr = le32toh(rxd->wb.upper.status_error);
+
+ if ((staterr & E1000_RXD_STAT_DD) == 0)
+ break;
+
+ if (++i == scctx->isc_nrxd[0]) {
+ i = 0;
+ }
+
+ if (staterr & E1000_RXD_STAT_EOP)
+ cnt++;
+
+ }
+ return (cnt);
+}
+
+static int
+lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
+{
+ struct adapter *adapter = arg;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
+ struct rx_ring *rxr = &que->rxr;
+ struct e1000_rx_desc *rxd;
+ u16 len;
+ u32 status, errors;
+ bool eop;
+ int i, cidx;
+
+ status = errors = i = 0;
+ cidx = ri->iri_cidx;
+
+ do {
+ rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx];
+ status = rxd->status;
+ errors = rxd->errors;
+
+ /* Error Checking then decrement count */
+ MPASS ((status & E1000_RXD_STAT_DD) != 0);
+
+ len = le16toh(rxd->length);
+ ri->iri_len += len;
+
+ eop = (status & E1000_RXD_STAT_EOP) != 0;
+
+ /* Make sure bad packets are discarded */
+ if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
+ adapter->dropped_pkts++;
+ /* XXX fixup if common */
+ return (EBADMSG);
+ }
+
+ ri->iri_frags[i].irf_flid = 0;
+ ri->iri_frags[i].irf_idx = cidx;
+ ri->iri_frags[i].irf_len = len;
+ /* Zero out the receive descriptors status. */
+ rxd->status = 0;
+
+ if (++cidx == scctx->isc_nrxd[0])
+ cidx = 0;
+ i++;
+ } while (!eop);
+
+ /* XXX add a faster way to look this up */
+ if (adapter->hw.mac.type >= e1000_82543 && !(status & E1000_RXD_STAT_IXSM))
+ lem_receive_checksum(status, errors, ri);
+
+ if (status & E1000_RXD_STAT_VP) {
+ ri->iri_vtag = le16toh(rxd->special);
+ ri->iri_flags |= M_VLANTAG;
+ }
+
+ ri->iri_nfrags = i;
+
+ return (0);
+}
+
+static int
+em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
+{
+ struct adapter *adapter = arg;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
+ struct rx_ring *rxr = &que->rxr;
+ union e1000_rx_desc_extended *rxd;
+
+ u16 len;
+ u32 staterr = 0;
+ bool eop;
+ int i, cidx, vtag;
+
+ i = vtag = 0;
+ cidx = ri->iri_cidx;
+
+ do {
+ rxd = &rxr->rx_base[cidx];
+ staterr = le32toh(rxd->wb.upper.status_error);
+
+ /* Error Checking then decrement count */
+ MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
+
+ len = le16toh(rxd->wb.upper.length);
+ ri->iri_len += len;
+
+ eop = (staterr & E1000_RXD_STAT_EOP) != 0;
+
+ /* Make sure bad packets are discarded */
+ if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
+ adapter->dropped_pkts++;
+ return EBADMSG;
+ }
+
+ ri->iri_frags[i].irf_flid = 0;
+ ri->iri_frags[i].irf_idx = cidx;
+ ri->iri_frags[i].irf_len = len;
+ /* Zero out the receive descriptors status. */
+ rxd->wb.upper.status_error &= htole32(~0xFF);
+
+ if (++cidx == scctx->isc_nrxd[0])
+ cidx = 0;
+ i++;
+ } while (!eop);
+
+ /* XXX add a faster way to look this up */
+ if (adapter->hw.mac.type >= e1000_82543)
+ em_receive_checksum(staterr, ri);
+
+ if (staterr & E1000_RXD_STAT_VP) {
+ vtag = le16toh(rxd->wb.upper.vlan);
+ }
+
+ ri->iri_vtag = vtag;
+ ri->iri_nfrags = i;
+ if (vtag)
+ ri->iri_flags |= M_VLANTAG;
+
+ return (0);
+}
+
+/*********************************************************************
+ *
+ * Verify that the hardware indicated that the checksum is valid.
+ * Inform the stack about the status of checksum so that stack
+ * doesn't spend time verifying the checksum.
+ *
+ *********************************************************************/
+static void
+lem_receive_checksum(int status, int errors, if_rxd_info_t ri)
+{
+ /* Did it pass? */
+ if (status & E1000_RXD_STAT_IPCS && !(errors & E1000_RXD_ERR_IPE))
+ ri->iri_csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID);
+
+ if (status & E1000_RXD_STAT_TCPCS) {
+ /* Did it pass? */
+ if (!(errors & E1000_RXD_ERR_TCPE)) {
+ ri->iri_csum_flags |=
+ (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
+ ri->iri_csum_data = htons(0xffff);
+ }
+ }
+}
+
+static void
+em_receive_checksum(uint32_t status, if_rxd_info_t ri)
+{
+ ri->iri_csum_flags = 0;
+
+ /* Ignore Checksum bit is set */
+ if (status & E1000_RXD_STAT_IXSM)
+ return;
+
+ /* If the IP checksum exists and there is no IP Checksum error */
+ if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) ==
+ E1000_RXD_STAT_IPCS) {
+ ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
+ }
+
+ /* TCP or UDP checksum */
+ if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) ==
+ E1000_RXD_STAT_TCPCS) {
+ ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
+ ri->iri_csum_data = htons(0xffff);
+ }
+ if (status & E1000_RXD_STAT_UDPCS) {
+ ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
+ ri->iri_csum_data = htons(0xffff);
+ }
+}
Index: sys/dev/e1000/if_em.h
===================================================================
--- sys/dev/e1000/if_em.h
+++ sys/dev/e1000/if_em.h
@@ -1,36 +1,67 @@
-/******************************************************************************
+/*$FreeBSD$*/
+#include "opt_em.h"
+#include "opt_ddb.h"
+#include "opt_inet.h"
+#include "opt_inet6.h"
- Copyright (c) 2001-2015, Intel Corporation
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
+#ifdef HAVE_KERNEL_OPTION_HEADERS
+#include "opt_device_polling.h"
+#endif
-******************************************************************************/
-/*$FreeBSD$*/
+#include <sys/param.h>
+#include <sys/systm.h>
+#ifdef DDB
+#include <sys/types.h>
+#include <ddb/ddb.h>
+#endif
+#if __FreeBSD_version >= 800000
+#include <sys/buf_ring.h>
+#endif
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/kthread.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
+#include <sys/eventhandler.h>
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <net/bpf.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/if_arp.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/iflib.h>
+
+#include <net/if_types.h>
+#include <net/if_vlan_var.h>
+
+#include <netinet/in_systm.h>
+#include <netinet/in.h>
+#include <netinet/if_ether.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+
+#include <machine/in_cksum.h>
+#include <dev/led/led.h>
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#include "e1000_api.h"
+#include "e1000_82571.h"
+#include "ifdi_if.h"
#ifndef _EM_H_DEFINED_
@@ -51,13 +82,10 @@
* desscriptors should meet the following condition.
* (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0
*/
-#define EM_MIN_TXD 80
+#define EM_MIN_TXD 128
#define EM_MAX_TXD 4096
-#ifdef EM_MULTIQUEUE
-#define EM_DEFAULT_TXD 4096
-#else
-#define EM_DEFAULT_TXD 1024
-#endif
+#define EM_DEFAULT_TXD 1024
+#define EM_DEFAULT_MULTI_TXD 4096
/*
* EM_RXD - Maximum number of receive Descriptors
@@ -72,13 +100,10 @@
* desscriptors should meet the following condition.
* (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0
*/
-#define EM_MIN_RXD 80
+#define EM_MIN_RXD 128
#define EM_MAX_RXD 4096
-#ifdef EM_MULTIQUEUE
-#define EM_DEFAULT_RXD 4096
-#else
-#define EM_DEFAULT_RXD 1024
-#endif
+#define EM_DEFAULT_RXD 1024
+#define EM_DEFAULT_MULTI_RXD 4096
/*
* EM_TIDV - Transmit Interrupt Delay Value
@@ -149,17 +174,6 @@
#endif
/*
- * This parameter controls the max duration of transmit watchdog.
- */
-#define EM_WATCHDOG (10 * hz)
-
-/*
- * This parameter controls when the driver calls the routine to reclaim
- * transmit descriptors.
- */
-#define EM_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 8)
-
-/*
* This parameter controls whether or not autonegotation is enabled.
* 0 - Disable autonegotiation
* 1 - Enable autonegotiation
@@ -221,6 +235,18 @@
#define PCICFG_DESC_RING_STATUS 0xe4
#define FLUSH_DESC_REQUIRED 0x100
+
+#define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \
+ ((hw->mac.type <= e1000_82576) ? 16 : 8))
+#define IGB_RX_HTHRESH 8
+#define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \
+ (adapter->intr_type == IFLIB_INTR_MSIX)) ? 1 : 4)
+
+#define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8)
+#define IGB_TX_HTHRESH 1
+#define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \
+ (adapter->intr_type == IFLIB_INTR_MSIX) ? 1 : 16)
+
/*
* TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be
* multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will
@@ -242,6 +268,7 @@
#define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK)
#define EM_BAR_TYPE_MASK 0x00000001
#define EM_BAR_TYPE_MMEM 0x00000000
+#define EM_BAR_TYPE_IO 0x00000001
#define EM_BAR_TYPE_FLASH 0x0014
#define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK)
#define EM_BAR_MEM_TYPE_MASK 0x00000006
@@ -279,6 +306,9 @@
#define ETH_ADDR_LEN 6
#define CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */
+#define IGB_PKTTYPE_MASK 0x0000FFF0
+#define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */
+
/*
* 82574 has a nonstandard address for EIAC
* and since its only used in MSIX, and in
@@ -295,19 +325,6 @@
#define EM_NVM_MSIX_N_MASK (0x7 << EM_NVM_MSIX_N_SHIFT)
#define EM_NVM_MSIX_N_SHIFT 7
-/*
- * Bus dma allocation structure used by
- * e1000_dma_malloc and e1000_dma_free.
- */
-struct em_dma_alloc {
- bus_addr_t dma_paddr;
- caddr_t dma_vaddr;
- bus_dma_tag_t dma_tag;
- bus_dmamap_t dma_map;
- bus_dma_segment_t dma_seg;
- int dma_nseg;
-};
-
struct adapter;
struct em_int_delay_info {
@@ -321,35 +338,31 @@
*/
struct tx_ring {
struct adapter *adapter;
- struct mtx tx_mtx;
- char mtx_name[16];
+ struct em_tx_queue *que;
u32 me;
- u32 msix;
- u32 ims;
int busy;
- struct em_dma_alloc txdma;
struct e1000_tx_desc *tx_base;
- struct task tx_task;
- struct taskqueue *tq;
- u32 next_avail_desc;
- u32 next_to_clean;
+ uint64_t tx_paddr;
struct em_txbuffer *tx_buffers;
- volatile u16 tx_avail;
u32 tx_tso; /* last tx was tso */
- u16 last_hw_offload;
- u8 last_hw_ipcso;
- u8 last_hw_ipcss;
- u8 last_hw_tucso;
- u8 last_hw_tucss;
-#if __FreeBSD_version >= 800000
- struct buf_ring *br;
-#endif
+
/* Interrupt resources */
- bus_dma_tag_t txtag;
void *tag;
struct resource *res;
unsigned long tx_irq;
unsigned long no_desc_avail;
+
+ /* Saved csum offloading context information */
+ int csum_flags;
+ int csum_lhlen;
+ int csum_iphlen;
+
+ int csum_thlen;
+ int csum_mss;
+ int csum_pktlen;
+
+ uint32_t csum_txd_upper;
+ uint32_t csum_txd_lower; /* last field */
};
/*
@@ -357,26 +370,15 @@
*/
struct rx_ring {
struct adapter *adapter;
+ struct em_rx_queue *que;
u32 me;
- u32 msix;
- u32 ims;
- struct mtx rx_mtx;
- char mtx_name[16];
u32 payload;
- struct task rx_task;
- struct taskqueue *tq;
union e1000_rx_desc_extended *rx_base;
- struct em_dma_alloc rxdma;
- u32 next_to_refresh;
- u32 next_to_check;
- struct em_rxbuffer *rx_buffers;
- struct mbuf *fmp;
- struct mbuf *lmp;
+ uint64_t rx_paddr;
/* Interrupt resources */
void *tag;
struct resource *res;
- bus_dma_tag_t rxtag;
bool discard;
/* Soft stats */
@@ -386,62 +388,68 @@
unsigned long rx_bytes;
};
+struct em_tx_queue {
+ struct adapter *adapter;
+ u32 msix;
+ u32 eims; /* This queue's EIMS bit */
+ u32 me;
+ struct tx_ring txr;
+};
+
+struct em_rx_queue {
+ struct adapter *adapter;
+ u32 me;
+ u32 msix;
+ u32 eims;
+ struct rx_ring rxr;
+ u64 irqs;
+ struct if_irq que_irq;
+};
/* Our adapter structure */
struct adapter {
- if_t ifp;
+ struct ifnet *ifp;
struct e1000_hw hw;
+ if_softc_ctx_t shared;
+ if_ctx_t ctx;
+#define tx_num_queues shared->isc_ntxqsets
+#define rx_num_queues shared->isc_nrxqsets
+#define intr_type shared->isc_intr
/* FreeBSD operating-system-specific structures. */
struct e1000_osdep osdep;
- device_t dev;
+ struct device *dev;
struct cdev *led_dev;
+ struct em_tx_queue *tx_queues;
+ struct em_rx_queue *rx_queues;
+ struct if_irq irq;
+
struct resource *memory;
struct resource *flash;
- struct resource *msix_mem;
+ struct resource *ioport;
+ int io_rid;
struct resource *res;
void *tag;
u32 linkvec;
u32 ivars;
- struct ifmedia media;
- struct callout timer;
+ struct ifmedia *media;
int msix;
int if_flags;
- int max_frame_size;
int min_frame_size;
- struct mtx core_mtx;
int em_insert_vlan_header;
u32 ims;
bool in_detach;
/* Task for FAST handling */
- struct task link_task;
- struct task que_task;
- struct taskqueue *tq; /* private task queue */
-
- eventhandler_tag vlan_attach;
- eventhandler_tag vlan_detach;
-
- u16 num_vlans;
- u8 num_queues;
-
- /*
- * Transmit rings:
- * Allocated at run time, an array of rings.
- */
- struct tx_ring *tx_rings;
- int num_tx_desc;
+ struct grouptask link_task;
+
+ u16 num_vlans;
u32 txd_cmd;
- /*
- * Receive rings:
- * Allocated at run time, an array of rings.
- */
- struct rx_ring *rx_rings;
- int num_rx_desc;
+ u32 tx_process_limit;
u32 rx_process_limit;
u32 rx_mbuf_sz;
@@ -467,7 +475,12 @@
u16 link_speed;
u16 link_duplex;
u32 smartspeed;
+ u32 dmac;
+ int link_mask;
+ u64 que_mask;
+
+
struct em_int_delay_info tx_int_delay;
struct em_int_delay_info tx_abs_int_delay;
struct em_int_delay_info rx_int_delay;
@@ -502,34 +515,10 @@
} em_vendor_info_t;
struct em_txbuffer {
- int next_eop; /* Index of the desc to watch */
- struct mbuf *m_head;
- bus_dmamap_t map; /* bus_dma map for packet */
-};
-
-struct em_rxbuffer {
- int next_eop; /* Index of the desc to watch */
- struct mbuf *m_head;
- bus_dmamap_t map; /* bus_dma map for packet */
- bus_addr_t paddr;
+ int eop;
};
-/*
-** Find the number of unrefreshed RX descriptors
-*/
-static inline u16
-e1000_rx_unrefreshed(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
-
- if (rxr->next_to_check > rxr->next_to_refresh)
- return (rxr->next_to_check - rxr->next_to_refresh - 1);
- else
- return ((adapter->num_rx_desc + rxr->next_to_check) -
- rxr->next_to_refresh - 1);
-}
-
#define EM_CORE_LOCK_INIT(_sc, _name) \
mtx_init(&(_sc)->core_mtx, _name, "EM Core Lock", MTX_DEF)
#define EM_TX_LOCK_INIT(_sc, _name) \
Index: sys/dev/e1000/if_em.c
===================================================================
--- sys/dev/e1000/if_em.c
+++ sys/dev/e1000/if_em.c
@@ -1,99 +1,10 @@
-/******************************************************************************
- Copyright (c) 2001-2015, Intel Corporation
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
-
-******************************************************************************/
-/*$FreeBSD$*/
-
-#include "opt_em.h"
-#include "opt_ddb.h"
-#include "opt_inet.h"
-#include "opt_inet6.h"
-
-#ifdef HAVE_KERNEL_OPTION_HEADERS
-#include "opt_device_polling.h"
-#endif
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#ifdef DDB
-#include <sys/types.h>
-#include <ddb/ddb.h>
-#endif
-#if __FreeBSD_version >= 800000
-#include <sys/buf_ring.h>
-#endif
-#include <sys/bus.h>
-#include <sys/endian.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/malloc.h>
-#include <sys/mbuf.h>
-#include <sys/module.h>
-#include <sys/rman.h>
-#include <sys/smp.h>
-#include <sys/socket.h>
-#include <sys/sockio.h>
-#include <sys/sysctl.h>
-#include <sys/taskqueue.h>
-#include <sys/eventhandler.h>
-#include <machine/bus.h>
-#include <machine/resource.h>
-
-#include <net/bpf.h>
-#include <net/ethernet.h>
-#include <net/if.h>
-#include <net/if_var.h>
-#include <net/if_arp.h>
-#include <net/if_dl.h>
-#include <net/if_media.h>
-
-#include <net/if_types.h>
-#include <net/if_vlan_var.h>
-
-#include <netinet/in_systm.h>
-#include <netinet/in.h>
-#include <netinet/if_ether.h>
-#include <netinet/ip.h>
-#include <netinet/ip6.h>
-#include <netinet/tcp.h>
-#include <netinet/udp.h>
-
-#include <machine/in_cksum.h>
-#include <dev/led/led.h>
-#include <dev/pci/pcivar.h>
-#include <dev/pci/pcireg.h>
-
-#include "e1000_api.h"
-#include "e1000_82571.h"
#include "if_em.h"
+#include <sys/sbuf.h>
+#include <machine/_inttypes.h>
+
+#define em_mac_min e1000_82547
+#define igb_mac_min e1000_82575
/*********************************************************************
* Driver version:
@@ -110,184 +21,213 @@
* { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
*********************************************************************/
-static em_vendor_info_t em_vendor_info_array[] =
+static pci_vendor_info_t em_vendor_info_array[] =
{
- /* Intel(R) PRO/1000 Network Connection */
- { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0},
-
- { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_ICH10_D_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH2_LV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH2_LV_V, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_LPT_I217_V, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_I218_LM2, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_I218_V2, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_I218_LM3, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_I218_V3, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5,
- PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, PCI_ANY_ID, PCI_ANY_ID, 0},
+ /* Intel(R) PRO/1000 Network Connection - Legacy em*/
+ PVID(0x8086, E1000_DEV_ID_82540EM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82540EM_LOM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82540EP, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82540EP_LOM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82540EP_LP, "Intel(R) PRO/1000 Network Connection"),
+
+ PVID(0x8086, E1000_DEV_ID_82541EI, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82541ER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82541ER_LOM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82541GI, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82541GI_LF, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, "Intel(R) PRO/1000 Network Connection"),
+
+ PVID(0x8086, E1000_DEV_ID_82542, "Intel(R) PRO/1000 Network Connection"),
+
+ PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, "Intel(R) PRO/1000 Network Connection"),
+
+ PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82544GC_LOM, "Intel(R) PRO/1000 Network Connection"),
+
+ PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, "Intel(R) PRO/1000 Network Connection"),
+
+ PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, "Intel(R) PRO/1000 Network Connection"),
+
+ PVID(0x8086, E1000_DEV_ID_82547EI, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82547GI, "Intel(R) PRO/1000 Network Connection"),
+
+ /* Intel(R) PRO/1000 Network Connection - em */
+ PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82573E, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82573E_IAMT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, "Intel(R) PRO/1000 Network Connection"),
+ PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, "Intel(R) PRO/1000 Network Connection"),
/* required last entry */
- { 0, 0, 0, 0, 0}
+ PVID_END
};
-/*********************************************************************
- * Table of branding strings for all supported NICs.
- *********************************************************************/
-
-static char *em_strings[] = {
- "Intel(R) PRO/1000 Network Connection"
+static pci_vendor_info_t igb_vendor_info_array[] =
+{
+ /* Intel(R) PRO/1000 Network Connection - em */
+ PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82576_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82580_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82580_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I350_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
+ /* required last entry */
+ PVID_END
};
/*********************************************************************
* Function prototypes
*********************************************************************/
-static int em_probe(device_t);
-static int em_attach(device_t);
-static int em_detach(device_t);
-static int em_shutdown(device_t);
-static int em_suspend(device_t);
-static int em_resume(device_t);
-#ifdef EM_MULTIQUEUE
-static int em_mq_start(if_t, struct mbuf *);
-static int em_mq_start_locked(if_t,
- struct tx_ring *);
-static void em_qflush(if_t);
-#else
-static void em_start(if_t);
-static void em_start_locked(if_t, struct tx_ring *);
-#endif
-static int em_ioctl(if_t, u_long, caddr_t);
-static uint64_t em_get_counter(if_t, ift_counter);
-static void em_init(void *);
-static void em_init_locked(struct adapter *);
-static void em_stop(void *);
-static void em_media_status(if_t, struct ifmediareq *);
-static int em_media_change(if_t);
-static void em_identify_hardware(struct adapter *);
-static int em_allocate_pci_resources(struct adapter *);
-static int em_allocate_legacy(struct adapter *);
-static int em_allocate_msix(struct adapter *);
-static int em_allocate_queues(struct adapter *);
-static int em_setup_msix(struct adapter *);
-static void em_free_pci_resources(struct adapter *);
-static void em_local_timer(void *);
-static void em_reset(struct adapter *);
-static int em_setup_interface(device_t, struct adapter *);
-static void em_flush_desc_rings(struct adapter *);
-
-static void em_setup_transmit_structures(struct adapter *);
-static void em_initialize_transmit_unit(struct adapter *);
-static int em_allocate_transmit_buffers(struct tx_ring *);
-static void em_free_transmit_structures(struct adapter *);
-static void em_free_transmit_buffers(struct tx_ring *);
-
-static int em_setup_receive_structures(struct adapter *);
-static int em_allocate_receive_buffers(struct rx_ring *);
-static void em_initialize_receive_unit(struct adapter *);
-static void em_free_receive_structures(struct adapter *);
-static void em_free_receive_buffers(struct rx_ring *);
-
-static void em_enable_intr(struct adapter *);
-static void em_disable_intr(struct adapter *);
+static void *em_register(device_t dev);
+static void *igb_register(device_t dev);
+static int em_if_attach_pre(if_ctx_t ctx);
+static int em_if_attach_post(if_ctx_t ctx);
+static int em_if_detach(if_ctx_t ctx);
+static int em_if_shutdown(if_ctx_t ctx);
+static int em_if_suspend(if_ctx_t ctx);
+static int em_if_resume(if_ctx_t ctx);
+
+static int em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets);
+static int em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets);
+static void em_if_queues_free(if_ctx_t ctx);
+
+static uint64_t em_if_get_counter(if_ctx_t, ift_counter);
+static void em_if_init(if_ctx_t ctx);
+static void em_if_stop(if_ctx_t ctx);
+static void em_if_media_status(if_ctx_t, struct ifmediareq *);
+static int em_if_media_change(if_ctx_t ctx);
+static int em_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
+static void em_if_timer(if_ctx_t ctx, uint16_t qid);
+static void em_if_vlan_register(if_ctx_t ctx, u16 vtag);
+static void em_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
+
+static void em_identify_hardware(if_ctx_t ctx);
+static int em_allocate_pci_resources(if_ctx_t ctx);
+static void em_free_pci_resources(if_ctx_t ctx);
+static void em_reset(if_ctx_t ctx);
+static int em_setup_interface(if_ctx_t ctx);
+static int em_setup_msix(if_ctx_t ctx);
+
+static void em_initialize_transmit_unit(if_ctx_t ctx);
+static void em_initialize_receive_unit(if_ctx_t ctx);
+
+static void em_if_enable_intr(if_ctx_t ctx);
+static void em_if_disable_intr(if_ctx_t ctx);
+static int em_if_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid);
+static void em_if_multi_set(if_ctx_t ctx);
+static void em_if_update_admin_status(if_ctx_t ctx);
static void em_update_stats_counters(struct adapter *);
static void em_add_hw_stats(struct adapter *adapter);
-static void em_txeof(struct tx_ring *);
-static bool em_rxeof(struct rx_ring *, int, int *);
-#ifndef __NO_STRICT_ALIGNMENT
-static int em_fixup_rx(struct rx_ring *);
-#endif
-static void em_setup_rxdesc(union e1000_rx_desc_extended *,
- const struct em_rxbuffer *rxbuf);
-static void em_receive_checksum(uint32_t status, struct mbuf *);
-static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int,
- struct ip *, u32 *, u32 *);
-static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *,
- struct tcphdr *, u32 *, u32 *);
-static void em_set_promisc(struct adapter *);
-static void em_disable_promisc(struct adapter *);
-static void em_set_multi(struct adapter *);
-static void em_update_link_status(struct adapter *);
-static void em_refresh_mbufs(struct rx_ring *, int);
-static void em_register_vlan(void *, if_t, u16);
-static void em_unregister_vlan(void *, if_t, u16);
+static int em_if_set_promisc(if_ctx_t ctx, int flags);
static void em_setup_vlan_hw_support(struct adapter *);
-static int em_xmit(struct tx_ring *, struct mbuf **);
-static int em_dma_malloc(struct adapter *, bus_size_t,
- struct em_dma_alloc *, int);
-static void em_dma_free(struct adapter *, struct em_dma_alloc *);
static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
static void em_print_nvm_info(struct adapter *);
static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
@@ -301,63 +241,122 @@
static void em_release_manageability(struct adapter *);
static void em_get_hw_control(struct adapter *);
static void em_release_hw_control(struct adapter *);
-static void em_get_wakeup(device_t);
-static void em_enable_wakeup(device_t);
+static void em_get_wakeup(if_ctx_t ctx);
+static void em_enable_wakeup(if_ctx_t ctx);
static int em_enable_phy_wakeup(struct adapter *);
-static void em_led_func(void *, int);
static void em_disable_aspm(struct adapter *);
-static int em_irq_fast(void *);
+int em_intr(void *arg);
+static void em_disable_promisc(if_ctx_t ctx);
/* MSIX handlers */
-static void em_msix_tx(void *);
-static void em_msix_rx(void *);
-static void em_msix_link(void *);
-static void em_handle_tx(void *context, int pending);
-static void em_handle_rx(void *context, int pending);
-static void em_handle_link(void *context, int pending);
+static int em_if_msix_intr_assign(if_ctx_t, int);
+static int em_msix_link(void *);
+static void em_handle_link(void *context);
-#ifdef EM_MULTIQUEUE
-static void em_enable_vectors_82574(struct adapter *);
-#endif
+static void em_enable_vectors_82574(if_ctx_t);
static void em_set_sysctl_value(struct adapter *, const char *,
const char *, int *, int);
static int em_set_flowcntl(SYSCTL_HANDLER_ARGS);
static int em_sysctl_eee(SYSCTL_HANDLER_ARGS);
+static void em_if_led_func(if_ctx_t ctx, int onoff);
+
+static void em_init_tx_ring(struct em_tx_queue *que);
+static int em_get_regs(SYSCTL_HANDLER_ARGS);
-static __inline void em_rx_discard(struct rx_ring *, int);
+static void lem_smartspeed(struct adapter *adapter);
+static void igb_configure_queues(struct adapter *adapter);
-#ifdef DEVICE_POLLING
-static poll_handler_t em_poll;
-#endif /* POLLING */
/*********************************************************************
* FreeBSD Device Interface Entry Points
*********************************************************************/
-
static device_method_t em_methods[] = {
/* Device interface */
- DEVMETHOD(device_probe, em_probe),
- DEVMETHOD(device_attach, em_attach),
- DEVMETHOD(device_detach, em_detach),
- DEVMETHOD(device_shutdown, em_shutdown),
- DEVMETHOD(device_suspend, em_suspend),
- DEVMETHOD(device_resume, em_resume),
- DEVMETHOD_END
+ DEVMETHOD(device_register, em_register),
+ DEVMETHOD(device_probe, iflib_device_probe),
+ DEVMETHOD(device_attach, iflib_device_attach),
+ DEVMETHOD(device_detach, iflib_device_detach),
+ DEVMETHOD(device_shutdown, iflib_device_shutdown),
+ DEVMETHOD(device_suspend, iflib_device_suspend),
+ DEVMETHOD(device_resume, iflib_device_resume),
+ DEVMETHOD_END
+};
+
+static device_method_t igb_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_register, igb_register),
+ DEVMETHOD(device_probe, iflib_device_probe),
+ DEVMETHOD(device_attach, iflib_device_attach),
+ DEVMETHOD(device_detach, iflib_device_detach),
+ DEVMETHOD(device_shutdown, iflib_device_shutdown),
+ DEVMETHOD(device_suspend, iflib_device_suspend),
+ DEVMETHOD(device_resume, iflib_device_resume),
+ DEVMETHOD_END
};
+
static driver_t em_driver = {
"em", em_methods, sizeof(struct adapter),
};
-devclass_t em_devclass;
+static devclass_t em_devclass;
DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
+
MODULE_DEPEND(em, pci, 1, 1, 1);
MODULE_DEPEND(em, ether, 1, 1, 1);
-#ifdef DEV_NETMAP
-MODULE_DEPEND(em, netmap, 1, 1, 1);
-#endif /* DEV_NETMAP */
+MODULE_DEPEND(em, iflib, 1, 1, 1);
+
+static driver_t igb_driver = {
+ "igb", igb_methods, sizeof(struct adapter),
+};
+
+static devclass_t igb_devclass;
+DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
+
+MODULE_DEPEND(igb, pci, 1, 1, 1);
+MODULE_DEPEND(igb, ether, 1, 1, 1);
+MODULE_DEPEND(igb, iflib, 1, 1, 1);
+
+
+static device_method_t em_if_methods[] = {
+ DEVMETHOD(ifdi_attach_pre, em_if_attach_pre),
+ DEVMETHOD(ifdi_attach_post, em_if_attach_post),
+ DEVMETHOD(ifdi_detach, em_if_detach),
+ DEVMETHOD(ifdi_shutdown, em_if_shutdown),
+ DEVMETHOD(ifdi_suspend, em_if_suspend),
+ DEVMETHOD(ifdi_resume, em_if_resume),
+ DEVMETHOD(ifdi_init, em_if_init),
+ DEVMETHOD(ifdi_stop, em_if_stop),
+ DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign),
+ DEVMETHOD(ifdi_intr_enable, em_if_enable_intr),
+ DEVMETHOD(ifdi_intr_disable, em_if_disable_intr),
+ DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc),
+ DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc),
+ DEVMETHOD(ifdi_queues_free, em_if_queues_free),
+ DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status),
+ DEVMETHOD(ifdi_multi_set, em_if_multi_set),
+ DEVMETHOD(ifdi_media_status, em_if_media_status),
+ DEVMETHOD(ifdi_media_change, em_if_media_change),
+ DEVMETHOD(ifdi_mtu_set, em_if_mtu_set),
+ DEVMETHOD(ifdi_promisc_set, em_if_set_promisc),
+ DEVMETHOD(ifdi_timer, em_if_timer),
+ DEVMETHOD(ifdi_vlan_register, em_if_vlan_register),
+ DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister),
+ DEVMETHOD(ifdi_get_counter, em_if_get_counter),
+ DEVMETHOD(ifdi_led_func, em_if_led_func),
+ DEVMETHOD(ifdi_queue_intr_enable, em_if_queue_intr_enable),
+ DEVMETHOD_END
+};
+
+ /*
+ * note that if (adapter->msix_mem) is replaced by:
+ * if (adapter->intr_type == IFLIB_INTR_MSIX)
+ */
+static driver_t em_if_driver = {
+ "em_if", em_if_methods, sizeof(struct adapter)
+};
/*********************************************************************
* Tunable default values.
@@ -365,10 +364,16 @@
#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000)
#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024)
+#define M_TSO_LEN 66
#define MAX_INTS_PER_SEC 8000
#define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256))
+/* Allow common code without TSO */
+#ifndef CSUM_TSO
+#define CSUM_TSO 0
+#endif
+
#define TSO_WORKAROUND 4
static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
@@ -393,39 +398,15 @@
&em_rx_abs_int_delay_dflt, 0,
"Default receive interrupt delay limit in usecs");
-static int em_rxd = EM_DEFAULT_RXD;
-static int em_txd = EM_DEFAULT_TXD;
-SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0,
- "Number of receive descriptors per queue");
-SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0,
- "Number of transmit descriptors per queue");
-
static int em_smart_pwr_down = FALSE;
SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down,
0, "Set to true to leave smart power down enabled on newer adapters");
/* Controls whether promiscuous also shows bad packets */
-static int em_debug_sbp = FALSE;
+static int em_debug_sbp = TRUE;
SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0,
"Show bad packets in promiscuous mode");
-static int em_enable_msix = TRUE;
-SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0,
- "Enable MSI-X interrupts");
-
-#ifdef EM_MULTIQUEUE
-static int em_num_queues = 1;
-SYSCTL_INT(_hw_em, OID_AUTO, num_queues, CTLFLAG_RDTUN, &em_num_queues, 0,
- "82574 only: Number of queues to configure, 0 indicates autoconfigure");
-#endif
-
-/*
-** Global variable to store last used CPU when binding queues
-** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
-** queue is bound to a cpu.
-*/
-static int em_last_bind_cpu = -1;
-
/* How many packets rxeof tries to clean at a time */
static int em_rx_process_limit = 100;
SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
@@ -438,64 +419,258 @@
SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0,
"Enable Energy Efficient Ethernet");
+/*
+** Tuneable Interrupt rate
+*/
+static int em_max_interrupt_rate = 8000;
+SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
+ &em_max_interrupt_rate, 0, "Maximum interrupts per second");
+
+
+
/* Global used in WOL setup with multiport cards */
static int global_quad_port_a = 0;
-#ifdef DEV_NETMAP /* see ixgbe.c for details */
-#include <dev/netmap/if_em_netmap.h>
-#endif /* DEV_NETMAP */
+extern struct if_txrx igb_txrx;
+extern struct if_txrx em_txrx;
+extern struct if_txrx lem_txrx;
+
+static struct if_shared_ctx em_sctx_init = {
+ .isc_magic = IFLIB_MAGIC,
+ .isc_q_align = PAGE_SIZE,
+ .isc_tx_maxsize = EM_TSO_SIZE,
+ .isc_tx_maxsegsize = PAGE_SIZE,
+ .isc_rx_maxsize = MJUM9BYTES,
+ .isc_rx_nsegments = 1,
+ .isc_rx_maxsegsize = MJUM9BYTES,
+ .isc_nfl = 1,
+ .isc_nrxqs = 1,
+ .isc_ntxqs = 1,
+ .isc_admin_intrcnt = 1,
+ .isc_vendor_info = em_vendor_info_array,
+ .isc_driver_version = em_driver_version,
+ .isc_driver = &em_if_driver,
+ .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP,
+
+ .isc_nrxd_min = {EM_MIN_RXD},
+ .isc_ntxd_min = {EM_MIN_TXD},
+ .isc_nrxd_max = {EM_MAX_RXD},
+ .isc_ntxd_max = {EM_MAX_TXD},
+ .isc_nrxd_default = {EM_DEFAULT_RXD},
+ .isc_ntxd_default = {EM_DEFAULT_TXD},
+};
+
+if_shared_ctx_t em_sctx = &em_sctx_init;
-/*********************************************************************
- * Device identification routine
+
+static struct if_shared_ctx igb_sctx_init = {
+ .isc_magic = IFLIB_MAGIC,
+ .isc_q_align = PAGE_SIZE,
+ .isc_tx_maxsize = EM_TSO_SIZE,
+ .isc_tx_maxsegsize = PAGE_SIZE,
+ .isc_rx_maxsize = MJUM9BYTES,
+ .isc_rx_nsegments = 1,
+ .isc_rx_maxsegsize = MJUM9BYTES,
+ .isc_nfl = 1,
+ .isc_nrxqs = 1,
+ .isc_ntxqs = 1,
+ .isc_admin_intrcnt = 1,
+ .isc_vendor_info = igb_vendor_info_array,
+ .isc_driver_version = em_driver_version,
+ .isc_driver = &em_if_driver,
+ .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP,
+
+ .isc_nrxd_min = {EM_MIN_RXD},
+ .isc_ntxd_min = {EM_MIN_TXD},
+ .isc_nrxd_max = {EM_MAX_RXD},
+ .isc_ntxd_max = {EM_MAX_TXD},
+ .isc_nrxd_default = {EM_DEFAULT_RXD},
+ .isc_ntxd_default = {EM_DEFAULT_TXD},
+};
+
+if_shared_ctx_t igb_sctx = &igb_sctx_init;
+
+/*****************************************************************
*
- * em_probe determines if the driver should be loaded on
- * adapter based on PCI vendor/device id of the adapter.
+ * Dump Registers
*
- * return BUS_PROBE_DEFAULT on success, positive on failure
- *********************************************************************/
+ ****************************************************************/
+#define IGB_REGS_LEN 739
-static int
-em_probe(device_t dev)
+static int em_get_regs(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *adapter = (struct adapter *)arg1;
+ struct e1000_hw *hw = &adapter->hw;
+
+ struct sbuf *sb;
+ u32 *regs_buff = (u32 *)malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_NOWAIT);
+ int rc;
+
+ memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32));
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ MPASS(rc == 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req);
+ MPASS(sb != NULL);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ /* General Registers */
+ regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL);
+ regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS);
+ regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT);
+ regs_buff[3] = E1000_READ_REG(hw, E1000_ICR);
+ regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL);
+ regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(0));
+ regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(0));
+ regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(0));
+ regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(0));
+ regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(0));
+ regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(0));
+ regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL);
+ regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(0));
+ regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(0));
+ regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(0));
+ regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(0));
+ regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(0));
+ regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(0));
+ regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH);
+ regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT);
+ regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS);
+ regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC);
+
+ sbuf_printf(sb, "General Registers\n");
+ sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]);
+ sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]);
+ sbuf_printf(sb, "\tCTRL_EXIT\t %08x\n\n", regs_buff[2]);
+
+ sbuf_printf(sb, "Interrupt Registers\n");
+ sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]);
+
+ sbuf_printf(sb, "RX Registers\n");
+ sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]);
+ sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]);
+ sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]);
+ sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]);
+ sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]);
+ sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]);
+ sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]);
+
+ sbuf_printf(sb, "TX Registers\n");
+ sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]);
+ sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]);
+ sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]);
+ sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]);
+ sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]);
+ sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]);
+ sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]);
+ sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]);
+ sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]);
+ sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]);
+ sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]);
+
+#ifdef DUMP_DESCS
+ {
+ if_softc_ctx_t scctx = adapter->shared;
+ struct rx_ring *rxr = &rx_que->rxr;
+ struct tx_ring *txr = &tx_que->txr;
+ int ntxd = scctx->isc_ntxd[0];
+ int nrxd = scctx->isc_nrxd[0];
+ int j;
+
+ for (j = 0; j < nrxd; j++) {
+ u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error);
+ u32 length = le32toh(rxr->rx_base[j].wb.upper.length);
+ sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" PRIx64 " Error:%d Length:%d\n", j, rxr->rx_base[j].read.buffer_addr, staterr, length);
+ }
+
+ for (j = 0; j < min(ntxd, 256); j++) {
+ struct em_txbuffer *buf = &txr->tx_buffers[j];
+ unsigned int *ptr = (unsigned int *)&txr->tx_base[j];
+
+ sbuf_printf(sb, "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x eop: %d DD=%d\n",
+ j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop,
+ buf->eop != -1 ? txr->tx_base[buf->eop].upper.fields.status & E1000_TXD_STAT_DD : 0);
+
+ }
+ }
+#endif
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+ return(rc);
+}
+
+static void *
+em_register(device_t dev)
+{
+ return (em_sctx);
+}
+
+static void *
+igb_register(device_t dev)
{
- char adapter_name[60];
- uint16_t pci_vendor_id = 0;
- uint16_t pci_device_id = 0;
- uint16_t pci_subvendor_id = 0;
- uint16_t pci_subdevice_id = 0;
- em_vendor_info_t *ent;
+ return (igb_sctx);
+}
- INIT_DEBUGOUT("em_probe: begin");
+static void
+em_init_tx_ring(struct em_tx_queue *que)
+{
+ struct adapter *sc = que->adapter;
+ if_softc_ctx_t scctx = sc->shared;
+ struct tx_ring *txr = &que->txr;
+ struct em_txbuffer *tx_buffer;
- pci_vendor_id = pci_get_vendor(dev);
- if (pci_vendor_id != EM_VENDOR_ID)
- return (ENXIO);
+ tx_buffer = txr->tx_buffers;
+ for (int i = 0; i < scctx->isc_ntxd[0]; i++, tx_buffer++) {
+ tx_buffer->eop = -1;
+ }
+}
- pci_device_id = pci_get_device(dev);
- pci_subvendor_id = pci_get_subvendor(dev);
- pci_subdevice_id = pci_get_subdevice(dev);
-
- ent = em_vendor_info_array;
- while (ent->vendor_id != 0) {
- if ((pci_vendor_id == ent->vendor_id) &&
- (pci_device_id == ent->device_id) &&
-
- ((pci_subvendor_id == ent->subvendor_id) ||
- (ent->subvendor_id == PCI_ANY_ID)) &&
-
- ((pci_subdevice_id == ent->subdevice_id) ||
- (ent->subdevice_id == PCI_ANY_ID))) {
- sprintf(adapter_name, "%s %s",
- em_strings[ent->index],
- em_driver_version);
- device_set_desc_copy(dev, adapter_name);
- return (BUS_PROBE_DEFAULT);
- }
- ent++;
+static int
+em_set_num_queues(if_ctx_t ctx)
+{
+ struct adapter *adapter = iflib_get_softc(ctx);
+ int maxqueues;
+
+ /* Sanity check based on HW */
+ switch (adapter->hw.mac.type) {
+ case e1000_82576:
+ case e1000_82580:
+ case e1000_i350:
+ case e1000_i354:
+ maxqueues = 8;
+ break;
+ case e1000_i210:
+ case e1000_82575:
+ maxqueues = 4;
+ break;
+ case e1000_i211:
+ case e1000_82574:
+ maxqueues = 2;
+ break;
+ default:
+ maxqueues = 1;
+ break;
}
- return (ENXIO);
+ return (maxqueues);
}
+
+#define EM_CAPS \
+ IFCAP_TSO4 | IFCAP_TXCSUM | IFCAP_LRO | IFCAP_RXCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL_MAGIC | \
+ IFCAP_WOL_MCAST | IFCAP_WOL | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | \
+ IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU;
+
+#define IGB_CAPS \
+ IFCAP_TSO4 | IFCAP_TXCSUM | IFCAP_LRO | IFCAP_RXCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL_MAGIC | \
+ IFCAP_WOL_MCAST | IFCAP_WOL | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \
+ IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU | IFCAP_TXCSUM_IPV6 | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU;
+
/*********************************************************************
* Device initialization routine
*
@@ -507,23 +682,30 @@
*********************************************************************/
static int
-em_attach(device_t dev)
+em_if_attach_pre(if_ctx_t ctx)
{
- struct adapter *adapter;
+ struct adapter *adapter;
+ if_softc_ctx_t scctx;
+ device_t dev;
struct e1000_hw *hw;
int error = 0;
- INIT_DEBUGOUT("em_attach: begin");
+ INIT_DEBUGOUT("em_if_attach_pre begin");
+ dev = iflib_get_dev(ctx);
+ adapter = iflib_get_softc(ctx);
if (resource_disabled("em", device_get_unit(dev))) {
device_printf(dev, "Disabled by device hint\n");
return (ENXIO);
}
- adapter = device_get_softc(dev);
+ adapter->ctx = ctx;
adapter->dev = adapter->osdep.dev = dev;
- hw = &adapter->hw;
- EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
+ scctx = adapter->shared = iflib_get_softc_ctx(ctx);
+ adapter->media = iflib_get_media(ctx);
+ hw = &adapter->hw;
+
+ adapter->tx_process_limit = scctx->isc_ntxd[0];
/* SYSCTL stuff */
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
@@ -541,13 +723,56 @@
OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
em_set_flowcntl, "I", "Flow Control");
- callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
+ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
+ SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+ OID_AUTO, "reg_dump", CTLTYPE_STRING | CTLFLAG_RD, adapter, 0,
+ em_get_regs, "A", "Dump Registers");
/* Determine hardware and mac info */
- em_identify_hardware(adapter);
+ em_identify_hardware(ctx);
+
+ /* Set isc_msix_bar */
+ scctx->isc_msix_bar = PCIR_BAR(EM_MSIX_BAR);
+ scctx->isc_tx_nsegments = EM_MAX_SCATTER;
+ scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;
+ scctx->isc_tx_tso_size_max = EM_TSO_SIZE;
+ scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE;
+ scctx->isc_nrxqsets_max = scctx->isc_ntxqsets_max = em_set_num_queues(ctx);
+ device_printf(dev, "attach_pre capping queues at %d\n", scctx->isc_ntxqsets_max);
+
+ scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO;
+
+
+ if (adapter->hw.mac.type >= igb_mac_min) {
+ scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN);
+ scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN);
+ scctx->isc_txrx = &igb_txrx;
+ scctx->isc_capenable = IGB_CAPS;
+ scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \
+ | CSUM_IP6_UDP | CSUM_IP6_TCP;
+ if (adapter->hw.mac.type != e1000_82575)
+ scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP;
+
+ } else if (adapter->hw.mac.type >= em_mac_min) {
+ scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]* sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
+ scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN);
+ scctx->isc_txrx = &em_txrx;
+ scctx->isc_capenable = EM_CAPS;
+ scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO;
+ } else {
+ scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
+ scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
+ scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO;
+ scctx->isc_txrx = &lem_txrx;
+ scctx->isc_capenable = EM_CAPS;
+ if (adapter->hw.mac.type < e1000_82543)
+ scctx->isc_capenable &= ~(IFCAP_HWCSUM|IFCAP_VLAN_HWCSUM);
+ scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO;
+ scctx->isc_msix_bar = 0;
+ }
/* Setup PCI resources */
- if (em_allocate_pci_resources(adapter)) {
+ if (em_allocate_pci_resources(ctx)) {
device_printf(dev, "Allocation of PCI resources failed\n");
error = ENXIO;
goto err_pci;
@@ -603,11 +828,7 @@
goto err_pci;
}
- /*
- * Setup MSI/X or MSI if PCI Express
- */
- adapter->msix = em_setup_msix(adapter);
-
+ em_setup_msix(ctx);
e1000_get_bus_info(hw);
/* Set up some sysctls for the tunable interrupt delays */
@@ -637,32 +858,15 @@
em_set_sysctl_value(adapter, "rx_processing_limit",
"max number of rx packets to process", &adapter->rx_process_limit,
em_rx_process_limit);
-
- /*
- * Validate number of transmit and receive descriptors. It
- * must not exceed hardware maximum, and must be multiple
- * of E1000_DBA_ALIGN.
- */
- if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
- (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) {
- device_printf(dev, "Using %d TX descriptors instead of %d!\n",
- EM_DEFAULT_TXD, em_txd);
- adapter->num_tx_desc = EM_DEFAULT_TXD;
- } else
- adapter->num_tx_desc = em_txd;
-
- if (((em_rxd * sizeof(union e1000_rx_desc_extended)) % EM_DBA_ALIGN) != 0 ||
- (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) {
- device_printf(dev, "Using %d RX descriptors instead of %d!\n",
- EM_DEFAULT_RXD, em_rxd);
- adapter->num_rx_desc = EM_DEFAULT_RXD;
- } else
- adapter->num_rx_desc = em_rxd;
-
+
hw->mac.autoneg = DO_AUTO_NEG;
hw->phy.autoneg_wait_to_complete = FALSE;
hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
+ if (adapter->hw.mac.type < em_mac_min) {
+ e1000_init_script_state_82541(&adapter->hw, TRUE);
+ e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
+ }
/* Copper options */
if (hw->phy.media_type == e1000_media_type_copper) {
hw->phy.mdix = AUTO_ALL_MODES;
@@ -683,14 +887,6 @@
*/
hw->mac.report_tx_early = 1;
- /*
- ** Get queue/ring memory
- */
- if (em_allocate_queues(adapter)) {
- error = ENOMEM;
- goto err_pci;
- }
-
/* Allocate multicast array memory. */
adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
@@ -720,7 +916,6 @@
*/
e1000_reset_hw(hw);
-
/* Make sure we have a good EEPROM before we read from it */
if (e1000_validate_nvm_checksum(hw) < 0) {
/*
@@ -753,68 +948,58 @@
/* Disable ULP support */
e1000_disable_ulp_lpt_lp(hw, TRUE);
- /*
- ** Do interrupt configuration
- */
- if (adapter->msix > 1) /* Do MSIX */
- error = em_allocate_msix(adapter);
- else /* MSI or Legacy */
- error = em_allocate_legacy(adapter);
- if (error)
- goto err_late;
-
- /*
+ /*
* Get Wake-on-Lan and Management info for later use
*/
- em_get_wakeup(dev);
+ em_get_wakeup(ctx);
+
+ iflib_set_mac(ctx, hw->mac.addr);
+
+ return (0);
+
+err_late:
+ em_release_hw_control(adapter);
+err_pci:
+ em_free_pci_resources(ctx);
+ free(adapter->mta, M_DEVBUF);
+
+ return (error);
+}
+static int
+em_if_attach_post(if_ctx_t ctx)
+{
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct e1000_hw *hw = &adapter->hw;
+ int error = 0;
+
/* Setup OS specific network interface */
- if (em_setup_interface(dev, adapter) != 0)
+ error = em_setup_interface(ctx);
+ if (error != 0) {
goto err_late;
+ }
- em_reset(adapter);
+ em_reset(ctx);
/* Initialize statistics */
em_update_stats_counters(adapter);
-
hw->mac.get_link_status = 1;
- em_update_link_status(adapter);
-
- /* Register for VLAN events */
- adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
- em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
- adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
- em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
-
+ em_if_update_admin_status(ctx);
em_add_hw_stats(adapter);
/* Non-AMT based hardware can now take control from firmware */
if (adapter->has_manage && !adapter->has_amt)
em_get_hw_control(adapter);
+
+ INIT_DEBUGOUT("em_if_attach_post: end");
- /* Tell the stack that the interface is not active */
- if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
-
- adapter->led_dev = led_create(em_led_func, adapter,
- device_get_nameunit(dev));
-#ifdef DEV_NETMAP
- em_netmap_attach(adapter);
-#endif /* DEV_NETMAP */
-
- INIT_DEBUGOUT("em_attach: end");
-
- return (0);
+ return (error);
err_late:
- em_free_transmit_structures(adapter);
- em_free_receive_structures(adapter);
em_release_hw_control(adapter);
- if (adapter->ifp != (void *)NULL)
- if_free(adapter->ifp);
-err_pci:
- em_free_pci_resources(adapter);
+ em_free_pci_resources(ctx);
+ em_if_queues_free(ctx);
free(adapter->mta, M_DEVBUF);
- EM_CORE_LOCK_DESTROY(adapter);
return (error);
}
@@ -830,60 +1015,17 @@
*********************************************************************/
static int
-em_detach(device_t dev)
+em_if_detach(if_ctx_t ctx)
{
- struct adapter *adapter = device_get_softc(dev);
- if_t ifp = adapter->ifp;
+ struct adapter *adapter = iflib_get_softc(ctx);
INIT_DEBUGOUT("em_detach: begin");
- /* Make sure VLANS are not using driver */
- if (if_vlantrunkinuse(ifp)) {
- device_printf(dev,"Vlan in use, detach first\n");
- return (EBUSY);
- }
-
-#ifdef DEVICE_POLLING
- if (if_getcapenable(ifp) & IFCAP_POLLING)
- ether_poll_deregister(ifp);
-#endif
-
- if (adapter->led_dev != NULL)
- led_destroy(adapter->led_dev);
-
- EM_CORE_LOCK(adapter);
- adapter->in_detach = 1;
- em_stop(adapter);
- EM_CORE_UNLOCK(adapter);
- EM_CORE_LOCK_DESTROY(adapter);
-
e1000_phy_hw_reset(&adapter->hw);
em_release_manageability(adapter);
em_release_hw_control(adapter);
-
- /* Unregister VLAN events */
- if (adapter->vlan_attach != NULL)
- EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
- if (adapter->vlan_detach != NULL)
- EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
-
- ether_ifdetach(adapter->ifp);
- callout_drain(&adapter->timer);
-
-#ifdef DEV_NETMAP
- netmap_detach(ifp);
-#endif /* DEV_NETMAP */
-
- em_free_pci_resources(adapter);
- bus_generic_detach(dev);
- if_free(ifp);
-
- em_free_transmit_structures(adapter);
- em_free_receive_structures(adapter);
-
- em_release_hw_control(adapter);
- free(adapter->mta, M_DEVBUF);
+ em_free_pci_resources(ctx);
return (0);
}
@@ -895,534 +1037,148 @@
**********************************************************************/
static int
-em_shutdown(device_t dev)
+em_if_shutdown(if_ctx_t ctx)
{
- return em_suspend(dev);
+ return em_if_suspend(ctx);
}
/*
* Suspend/resume device methods.
*/
static int
-em_suspend(device_t dev)
+em_if_suspend(if_ctx_t ctx)
{
- struct adapter *adapter = device_get_softc(dev);
-
- EM_CORE_LOCK(adapter);
+ struct adapter *adapter = iflib_get_softc(ctx);
em_release_manageability(adapter);
em_release_hw_control(adapter);
- em_enable_wakeup(dev);
-
- EM_CORE_UNLOCK(adapter);
-
- return bus_generic_suspend(dev);
+ em_enable_wakeup(ctx);
+ return (0);
}
static int
-em_resume(device_t dev)
+em_if_resume(if_ctx_t ctx)
{
- struct adapter *adapter = device_get_softc(dev);
- struct tx_ring *txr = adapter->tx_rings;
- if_t ifp = adapter->ifp;
+ struct adapter *adapter = iflib_get_softc(ctx);
- EM_CORE_LOCK(adapter);
if (adapter->hw.mac.type == e1000_pch2lan)
e1000_resume_workarounds_pchlan(&adapter->hw);
- em_init_locked(adapter);
+ em_if_init(ctx);
em_init_manageability(adapter);
- if ((if_getflags(ifp) & IFF_UP) &&
- (if_getdrvflags(ifp) & IFF_DRV_RUNNING) && adapter->link_active) {
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- EM_TX_LOCK(txr);
-#ifdef EM_MULTIQUEUE
- if (!drbr_empty(ifp, txr->br))
- em_mq_start_locked(ifp, txr);
-#else
- if (!if_sendq_empty(ifp))
- em_start_locked(ifp, txr);
-#endif
- EM_TX_UNLOCK(txr);
- }
- }
- EM_CORE_UNLOCK(adapter);
+ return(0);
+}
- return bus_generic_resume(dev);
+static int
+em_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
+{
+ int max_frame_size;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct ifnet *ifp = iflib_get_ifp(ctx);
+
+ IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
+
+ switch (adapter->hw.mac.type) {
+ case e1000_82571:
+ case e1000_82572:
+ case e1000_ich9lan:
+ case e1000_ich10lan:
+ case e1000_pch2lan:
+ case e1000_pch_lpt:
+ case e1000_pch_spt:
+ case e1000_82574:
+ case e1000_82583:
+ case e1000_80003es2lan: /* 9K Jumbo Frame size */
+ max_frame_size = 9234;
+ break;
+ case e1000_pchlan:
+ max_frame_size = 4096;
+ break;
+ /* Adapters that do not support jumbo frames */
+ case e1000_ich8lan:
+ max_frame_size = ETHER_MAX_LEN;
+ break;
+ default:
+ max_frame_size = MAX_JUMBO_FRAME_SIZE;
+ }
+ if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) {
+ return (EINVAL);
+ }
+
+ adapter->hw.mac.max_frame_size = if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN;
+ return (0);
}
+/*********************************************************************
+ * Init entry point
+ *
+ * This routine is used in two ways. It is used by the stack as
+ * init entry point in network interface structure. It is also used
+ * by the driver as a hw/sw initialization routine to get to a
+ * consistent state.
+ *
+ * return 0 on success, positive on failure
+ **********************************************************************/
-#ifndef EM_MULTIQUEUE
static void
-em_start_locked(if_t ifp, struct tx_ring *txr)
+em_if_init(if_ctx_t ctx)
{
- struct adapter *adapter = if_getsoftc(ifp);
- struct mbuf *m_head;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct ifnet *ifp = iflib_get_ifp(ctx);
- EM_TX_LOCK_ASSERT(txr);
+ INIT_DEBUGOUT("em_if_init: begin");
- if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
- IFF_DRV_RUNNING)
- return;
-
- if (!adapter->link_active)
- return;
-
- while (!if_sendq_empty(ifp)) {
- /* Call cleanup if number of TX descriptors low */
- if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD)
- em_txeof(txr);
- if (txr->tx_avail < EM_MAX_SCATTER) {
- if_setdrvflagbits(ifp,IFF_DRV_OACTIVE, 0);
- break;
- }
- m_head = if_dequeue(ifp);
- if (m_head == NULL)
- break;
- /*
- * Encapsulation can modify our pointer, and or make it
- * NULL on failure. In that event, we can't requeue.
- */
- if (em_xmit(txr, &m_head)) {
- if (m_head == NULL)
- break;
- if_sendq_prepend(ifp, m_head);
- break;
- }
-
- /* Mark the queue as having work */
- if (txr->busy == EM_TX_IDLE)
- txr->busy = EM_TX_BUSY;
+ /* Get the latest mac address, User can use a LAA */
+ bcopy(if_getlladdr(ifp), adapter->hw.mac.addr,
+ ETHER_ADDR_LEN);
- /* Send a copy of the frame to the BPF listener */
- ETHER_BPF_MTAP(ifp, m_head);
+ /* Put the address into the Receive Address Array */
+ e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
+ /*
+ * With the 82571 adapter, RAR[0] may be overwritten
+ * when the other port is reset, we make a duplicate
+ * in RAR[14] for that eventuality, this assures
+ * the interface continues to function.
+ */
+ if (adapter->hw.mac.type == e1000_82571) {
+ e1000_set_laa_state_82571(&adapter->hw, TRUE);
+ e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
+ E1000_RAR_ENTRIES - 1);
}
- return;
-}
+ /* Initialize the hardware */
+ em_reset(ctx);
+ em_if_update_admin_status(ctx);
-static void
-em_start(if_t ifp)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- struct tx_ring *txr = adapter->tx_rings;
+ /* Setup VLAN support, basic and offload if available */
+ E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
- EM_TX_LOCK(txr);
- em_start_locked(ifp, txr);
- EM_TX_UNLOCK(txr);
- }
- return;
-}
-#else /* EM_MULTIQUEUE */
-/*********************************************************************
- * Multiqueue Transmit routines
- *
- * em_mq_start is called by the stack to initiate a transmit.
- * however, if busy the driver can queue the request rather
- * than do an immediate send. It is this that is an advantage
- * in this driver, rather than also having multiple tx queues.
- **********************************************************************/
-/*
-** Multiqueue capable stack interface
-*/
-static int
-em_mq_start(if_t ifp, struct mbuf *m)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- struct tx_ring *txr = adapter->tx_rings;
- unsigned int i, error;
+ /* Clear bad data from Rx FIFOs */
+ if (adapter->hw.mac.type >= igb_mac_min)
+ e1000_rx_fifo_flush_82575(&adapter->hw);
- if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
- i = m->m_pkthdr.flowid % adapter->num_queues;
- else
- i = curcpu % adapter->num_queues;
+ /* Configure for OS presence */
+ em_init_manageability(adapter);
- txr = &adapter->tx_rings[i];
+ /* Prepare transmit descriptors and buffers */
+ em_initialize_transmit_unit(ctx);
- error = drbr_enqueue(ifp, txr->br, m);
- if (error)
- return (error);
+ /* Setup Multicast table */
+ em_if_multi_set(ctx);
- if (EM_TX_TRYLOCK(txr)) {
- em_mq_start_locked(ifp, txr);
- EM_TX_UNLOCK(txr);
- } else
- taskqueue_enqueue(txr->tq, &txr->tx_task);
+ /*
+ ** Figure out the desired mbuf
+ ** pool for doing jumbos
+ */
+ if (adapter->hw.mac.max_frame_size <= 2048)
+ adapter->rx_mbuf_sz = MCLBYTES;
+ else if (adapter->hw.mac.max_frame_size <= 4096)
+ adapter->rx_mbuf_sz = MJUMPAGESIZE;
+ else
+ adapter->rx_mbuf_sz = MJUM9BYTES;
- return (0);
-}
-
-static int
-em_mq_start_locked(if_t ifp, struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
- struct mbuf *next;
- int err = 0, enq = 0;
-
- EM_TX_LOCK_ASSERT(txr);
-
- if (((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) ||
- adapter->link_active == 0) {
- return (ENETDOWN);
- }
-
- /* Process the queue */
- while ((next = drbr_peek(ifp, txr->br)) != NULL) {
- if ((err = em_xmit(txr, &next)) != 0) {
- if (next == NULL) {
- /* It was freed, move forward */
- drbr_advance(ifp, txr->br);
- } else {
- /*
- * Still have one left, it may not be
- * the same since the transmit function
- * may have changed it.
- */
- drbr_putback(ifp, txr->br, next);
- }
- break;
- }
- drbr_advance(ifp, txr->br);
- enq++;
- if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len);
- if (next->m_flags & M_MCAST)
- if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
- ETHER_BPF_MTAP(ifp, next);
- if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
- break;
- }
-
- /* Mark the queue as having work */
- if ((enq > 0) && (txr->busy == EM_TX_IDLE))
- txr->busy = EM_TX_BUSY;
-
- if (txr->tx_avail < EM_MAX_SCATTER)
- em_txeof(txr);
- if (txr->tx_avail < EM_MAX_SCATTER) {
- if_setdrvflagbits(ifp, IFF_DRV_OACTIVE,0);
- }
- return (err);
-}
-
-/*
-** Flush all ring buffers
-*/
-static void
-em_qflush(if_t ifp)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- struct tx_ring *txr = adapter->tx_rings;
- struct mbuf *m;
-
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- EM_TX_LOCK(txr);
- while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
- m_freem(m);
- EM_TX_UNLOCK(txr);
- }
- if_qflush(ifp);
-}
-#endif /* EM_MULTIQUEUE */
-
-/*********************************************************************
- * Ioctl entry point
- *
- * em_ioctl is called when the user wants to configure the
- * interface.
- *
- * return 0 on success, positive on failure
- **********************************************************************/
-
-static int
-em_ioctl(if_t ifp, u_long command, caddr_t data)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- struct ifreq *ifr = (struct ifreq *)data;
-#if defined(INET) || defined(INET6)
- struct ifaddr *ifa = (struct ifaddr *)data;
-#endif
- bool avoid_reset = FALSE;
- int error = 0;
-
- if (adapter->in_detach)
- return (error);
-
- switch (command) {
- case SIOCSIFADDR:
-#ifdef INET
- if (ifa->ifa_addr->sa_family == AF_INET)
- avoid_reset = TRUE;
-#endif
-#ifdef INET6
- if (ifa->ifa_addr->sa_family == AF_INET6)
- avoid_reset = TRUE;
-#endif
- /*
- ** Calling init results in link renegotiation,
- ** so we avoid doing it when possible.
- */
- if (avoid_reset) {
- if_setflagbits(ifp,IFF_UP,0);
- if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING))
- em_init(adapter);
-#ifdef INET
- if (!(if_getflags(ifp) & IFF_NOARP))
- arp_ifinit(ifp, ifa);
-#endif
- } else
- error = ether_ioctl(ifp, command, data);
- break;
- case SIOCSIFMTU:
- {
- int max_frame_size;
-
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
-
- EM_CORE_LOCK(adapter);
- switch (adapter->hw.mac.type) {
- case e1000_82571:
- case e1000_82572:
- case e1000_ich9lan:
- case e1000_ich10lan:
- case e1000_pch2lan:
- case e1000_pch_lpt:
- case e1000_pch_spt:
- case e1000_82574:
- case e1000_82583:
- case e1000_80003es2lan: /* 9K Jumbo Frame size */
- max_frame_size = 9234;
- break;
- case e1000_pchlan:
- max_frame_size = 4096;
- break;
- /* Adapters that do not support jumbo frames */
- case e1000_ich8lan:
- max_frame_size = ETHER_MAX_LEN;
- break;
- default:
- max_frame_size = MAX_JUMBO_FRAME_SIZE;
- }
- if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
- ETHER_CRC_LEN) {
- EM_CORE_UNLOCK(adapter);
- error = EINVAL;
- break;
- }
-
- if_setmtu(ifp, ifr->ifr_mtu);
- adapter->hw.mac.max_frame_size =
- if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN;
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
- em_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
- break;
- }
- case SIOCSIFFLAGS:
- IOCTL_DEBUGOUT("ioctl rcv'd:\
- SIOCSIFFLAGS (Set Interface Flags)");
- EM_CORE_LOCK(adapter);
- if (if_getflags(ifp) & IFF_UP) {
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
- if ((if_getflags(ifp) ^ adapter->if_flags) &
- (IFF_PROMISC | IFF_ALLMULTI)) {
- em_disable_promisc(adapter);
- em_set_promisc(adapter);
- }
- } else
- em_init_locked(adapter);
- } else
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
- em_stop(adapter);
- adapter->if_flags = if_getflags(ifp);
- EM_CORE_UNLOCK(adapter);
- break;
- case SIOCADDMULTI:
- case SIOCDELMULTI:
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
- EM_CORE_LOCK(adapter);
- em_disable_intr(adapter);
- em_set_multi(adapter);
-#ifdef DEVICE_POLLING
- if (!(if_getcapenable(ifp) & IFCAP_POLLING))
-#endif
- em_enable_intr(adapter);
- EM_CORE_UNLOCK(adapter);
- }
- break;
- case SIOCSIFMEDIA:
- /* Check SOL/IDER usage */
- EM_CORE_LOCK(adapter);
- if (e1000_check_reset_block(&adapter->hw)) {
- EM_CORE_UNLOCK(adapter);
- device_printf(adapter->dev, "Media change is"
- " blocked due to SOL/IDER session.\n");
- break;
- }
- EM_CORE_UNLOCK(adapter);
- /* falls thru */
- case SIOCGIFMEDIA:
- IOCTL_DEBUGOUT("ioctl rcv'd: \
- SIOCxIFMEDIA (Get/Set Interface Media)");
- error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
- break;
- case SIOCSIFCAP:
- {
- int mask, reinit;
-
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
- reinit = 0;
- mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
-#ifdef DEVICE_POLLING
- if (mask & IFCAP_POLLING) {
- if (ifr->ifr_reqcap & IFCAP_POLLING) {
- error = ether_poll_register(em_poll, ifp);
- if (error)
- return (error);
- EM_CORE_LOCK(adapter);
- em_disable_intr(adapter);
- if_setcapenablebit(ifp, IFCAP_POLLING, 0);
- EM_CORE_UNLOCK(adapter);
- } else {
- error = ether_poll_deregister(ifp);
- /* Enable interrupt even in error case */
- EM_CORE_LOCK(adapter);
- em_enable_intr(adapter);
- if_setcapenablebit(ifp, 0, IFCAP_POLLING);
- EM_CORE_UNLOCK(adapter);
- }
- }
-#endif
- if (mask & IFCAP_HWCSUM) {
- if_togglecapenable(ifp,IFCAP_HWCSUM);
- reinit = 1;
- }
- if (mask & IFCAP_TSO4) {
- if_togglecapenable(ifp,IFCAP_TSO4);
- reinit = 1;
- }
- if (mask & IFCAP_VLAN_HWTAGGING) {
- if_togglecapenable(ifp,IFCAP_VLAN_HWTAGGING);
- reinit = 1;
- }
- if (mask & IFCAP_VLAN_HWFILTER) {
- if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
- reinit = 1;
- }
- if (mask & IFCAP_VLAN_HWTSO) {
- if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
- reinit = 1;
- }
- if ((mask & IFCAP_WOL) &&
- (if_getcapabilities(ifp) & IFCAP_WOL) != 0) {
- if (mask & IFCAP_WOL_MCAST)
- if_togglecapenable(ifp, IFCAP_WOL_MCAST);
- if (mask & IFCAP_WOL_MAGIC)
- if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
- }
- if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING))
- em_init(adapter);
- if_vlancap(ifp);
- break;
- }
-
- default:
- error = ether_ioctl(ifp, command, data);
- break;
- }
-
- return (error);
-}
-
-
-/*********************************************************************
- * Init entry point
- *
- * This routine is used in two ways. It is used by the stack as
- * init entry point in network interface structure. It is also used
- * by the driver as a hw/sw initialization routine to get to a
- * consistent state.
- *
- * return 0 on success, positive on failure
- **********************************************************************/
-
-static void
-em_init_locked(struct adapter *adapter)
-{
- if_t ifp = adapter->ifp;
- device_t dev = adapter->dev;
-
- INIT_DEBUGOUT("em_init: begin");
-
- EM_CORE_LOCK_ASSERT(adapter);
-
- em_disable_intr(adapter);
- callout_stop(&adapter->timer);
-
- /* Get the latest mac address, User can use a LAA */
- bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr,
- ETHER_ADDR_LEN);
-
- /* Put the address into the Receive Address Array */
- e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
-
- /*
- * With the 82571 adapter, RAR[0] may be overwritten
- * when the other port is reset, we make a duplicate
- * in RAR[14] for that eventuality, this assures
- * the interface continues to function.
- */
- if (adapter->hw.mac.type == e1000_82571) {
- e1000_set_laa_state_82571(&adapter->hw, TRUE);
- e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
- E1000_RAR_ENTRIES - 1);
- }
-
- /* Initialize the hardware */
- em_reset(adapter);
- em_update_link_status(adapter);
-
- /* Setup VLAN support, basic and offload if available */
- E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
-
- /* Set hardware offload abilities */
- if_clearhwassist(ifp);
- if (if_getcapenable(ifp) & IFCAP_TXCSUM)
- if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
-
- if (if_getcapenable(ifp) & IFCAP_TSO4)
- if_sethwassistbits(ifp, CSUM_TSO, 0);
-
- /* Configure for OS presence */
- em_init_manageability(adapter);
-
- /* Prepare transmit descriptors and buffers */
- em_setup_transmit_structures(adapter);
- em_initialize_transmit_unit(adapter);
-
- /* Setup Multicast table */
- em_set_multi(adapter);
-
- /*
- ** Figure out the desired mbuf
- ** pool for doing jumbos
- */
- if (adapter->hw.mac.max_frame_size <= 2048)
- adapter->rx_mbuf_sz = MCLBYTES;
- else if (adapter->hw.mac.max_frame_size <= 4096)
- adapter->rx_mbuf_sz = MJUMPAGESIZE;
- else
- adapter->rx_mbuf_sz = MJUM9BYTES;
-
- /* Prepare receive descriptors and buffers */
- if (em_setup_receive_structures(adapter)) {
- device_printf(dev, "Could not setup receive structures\n");
- em_stop(adapter);
- return;
- }
- em_initialize_receive_unit(adapter);
+ em_initialize_receive_unit(ctx);
/* Use real VLAN Filter support? */
if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
@@ -1438,118 +1194,54 @@
}
/* Don't lose promiscuous settings */
- em_set_promisc(adapter);
-
- /* Set the interface as ACTIVE */
- if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
-
- callout_reset(&adapter->timer, hz, em_local_timer, adapter);
+ em_if_set_promisc(ctx, IFF_PROMISC);
e1000_clear_hw_cntrs_base_generic(&adapter->hw);
/* MSI/X configuration for 82574 */
if (adapter->hw.mac.type == e1000_82574) {
- int tmp;
- tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
+ int tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
+
tmp |= E1000_CTRL_EXT_PBA_CLR;
E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
/* Set the IVAR - interrupt vector routing. */
E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars);
- }
+ } else if (adapter->intr_type == IFLIB_INTR_MSIX) /* Set up queue routing */
+ igb_configure_queues(adapter);
-#ifdef DEVICE_POLLING
- /*
- * Only enable interrupts if we are not polling, make sure
- * they are off otherwise.
- */
- if (if_getcapenable(ifp) & IFCAP_POLLING)
- em_disable_intr(adapter);
- else
-#endif /* DEVICE_POLLING */
- em_enable_intr(adapter);
+ /* this clears any pending interrupts */
+ E1000_READ_REG(&adapter->hw, E1000_ICR);
+ E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
/* AMT based hardware can now take control from firmware */
if (adapter->has_manage && adapter->has_amt)
em_get_hw_control(adapter);
-}
-
-static void
-em_init(void *arg)
-{
- struct adapter *adapter = arg;
-
- EM_CORE_LOCK(adapter);
- em_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
-}
-
-#ifdef DEVICE_POLLING
-/*********************************************************************
- *
- * Legacy polling routine: note this only works with single queue
- *
- *********************************************************************/
-static int
-em_poll(if_t ifp, enum poll_cmd cmd, int count)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- struct tx_ring *txr = adapter->tx_rings;
- struct rx_ring *rxr = adapter->rx_rings;
- u32 reg_icr;
- int rx_done;
-
- EM_CORE_LOCK(adapter);
- if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
- EM_CORE_UNLOCK(adapter);
- return (0);
- }
-
- if (cmd == POLL_AND_CHECK_STATUS) {
- reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
- if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
- callout_stop(&adapter->timer);
- adapter->hw.mac.get_link_status = 1;
- em_update_link_status(adapter);
- callout_reset(&adapter->timer, hz,
- em_local_timer, adapter);
- }
+ /* Set Energy Efficient Ethernet */
+ if (adapter->hw.mac.type >= igb_mac_min &&
+ adapter->hw.phy.media_type == e1000_media_type_copper) {
+ if (adapter->hw.mac.type == e1000_i354)
+ e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
+ else
+ e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
}
- EM_CORE_UNLOCK(adapter);
-
- em_rxeof(rxr, count, &rx_done);
-
- EM_TX_LOCK(txr);
- em_txeof(txr);
-#ifdef EM_MULTIQUEUE
- if (!drbr_empty(ifp, txr->br))
- em_mq_start_locked(ifp, txr);
-#else
- if (!if_sendq_empty(ifp))
- em_start_locked(ifp, txr);
-#endif
- EM_TX_UNLOCK(txr);
-
- return (rx_done);
}
-#endif /* DEVICE_POLLING */
-
/*********************************************************************
*
* Fast Legacy/MSI Combined Interrupt Service routine
*
*********************************************************************/
-static int
-em_irq_fast(void *arg)
+int
+em_intr(void *arg)
{
- struct adapter *adapter = arg;
- if_t ifp;
+ struct adapter *adapter = arg;
+ if_ctx_t ctx = adapter->ctx;
u32 reg_icr;
- ifp = adapter->ifp;
-
reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
+ if (adapter->intr_type != IFLIB_INTR_LEGACY)
+ goto skip_stray;
/* Hot eject? */
if (reg_icr == 0xffffffff)
return FILTER_STRAY;
@@ -1566,80 +1258,42 @@
(reg_icr & E1000_ICR_INT_ASSERTED) == 0)
return FILTER_STRAY;
- em_disable_intr(adapter);
- taskqueue_enqueue(adapter->tq, &adapter->que_task);
-
+skip_stray:
/* Link status change */
if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
adapter->hw.mac.get_link_status = 1;
- taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
+ iflib_admin_intr_deferred(ctx);
}
if (reg_icr & E1000_ICR_RXO)
adapter->rx_overruns++;
- return FILTER_HANDLED;
+
+ return (FILTER_SCHEDULE_THREAD);
}
-/* Combined RX/TX handler, used by Legacy and MSI */
static void
-em_handle_que(void *context, int pending)
+igb_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq)
{
- struct adapter *adapter = context;
- if_t ifp = adapter->ifp;
- struct tx_ring *txr = adapter->tx_rings;
- struct rx_ring *rxr = adapter->rx_rings;
-
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
- bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
-
- EM_TX_LOCK(txr);
- em_txeof(txr);
-#ifdef EM_MULTIQUEUE
- if (!drbr_empty(ifp, txr->br))
- em_mq_start_locked(ifp, txr);
-#else
- if (!if_sendq_empty(ifp))
- em_start_locked(ifp, txr);
-#endif
- EM_TX_UNLOCK(txr);
- if (more) {
- taskqueue_enqueue(adapter->tq, &adapter->que_task);
- return;
- }
- }
-
- em_enable_intr(adapter);
- return;
+ E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxq->eims);
}
-
-/*********************************************************************
- *
- * MSIX Interrupt Service Routines
- *
- **********************************************************************/
static void
-em_msix_tx(void *arg)
+em_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq)
{
- struct tx_ring *txr = arg;
- struct adapter *adapter = txr->adapter;
- if_t ifp = adapter->ifp;
-
- ++txr->tx_irq;
- EM_TX_LOCK(txr);
- em_txeof(txr);
-#ifdef EM_MULTIQUEUE
- if (!drbr_empty(ifp, txr->br))
- em_mq_start_locked(ifp, txr);
-#else
- if (!if_sendq_empty(ifp))
- em_start_locked(ifp, txr);
-#endif
+ E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxq->eims);
+}
- /* Reenable this interrupt */
- E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
- EM_TX_UNLOCK(txr);
- return;
+static int
+em_if_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
+{
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct em_rx_queue *rxq = &adapter->rx_queues[rxqid];
+
+ if (adapter->hw.mac.type >= igb_mac_min)
+ igb_enable_queue(adapter, rxq);
+ else
+ em_enable_queue(adapter, rxq);
+ return (0);
}
/*********************************************************************
@@ -1647,25 +1301,14 @@
* MSIX RX Interrupt Service routine
*
**********************************************************************/
-
-static void
-em_msix_rx(void *arg)
+static int
+em_msix_que(void *arg)
{
- struct rx_ring *rxr = arg;
- struct adapter *adapter = rxr->adapter;
- bool more;
-
- ++rxr->rx_irq;
- if (!(if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING))
- return;
- more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
- if (more)
- taskqueue_enqueue(rxr->tq, &rxr->rx_task);
- else {
- /* Reenable this interrupt */
- E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
- }
- return;
+ struct em_rx_queue *que = arg;
+
+ ++que->irqs;
+
+ return (FILTER_SCHEDULE_THREAD);
}
/*********************************************************************
@@ -1673,103 +1316,51 @@
* MSIX Link Fast Interrupt Service routine
*
**********************************************************************/
-static void
+static int
em_msix_link(void *arg)
{
struct adapter *adapter = arg;
u32 reg_icr;
++adapter->link_irq;
+ MPASS(adapter->hw.back != NULL);
reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
if (reg_icr & E1000_ICR_RXO)
adapter->rx_overruns++;
if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
- adapter->hw.mac.get_link_status = 1;
- em_handle_link(adapter, 0);
- } else
+ em_handle_link(adapter->ctx);
+ } else {
E1000_WRITE_REG(&adapter->hw, E1000_IMS,
- EM_MSIX_LINK | E1000_IMS_LSC);
+ EM_MSIX_LINK | E1000_IMS_LSC);
+ if (adapter->hw.mac.type >= igb_mac_min)
+ E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
+
+ }
+
/*
** Because we must read the ICR for this interrupt
** it may clear other causes using autoclear, for
** this reason we simply create a soft interrupt
** for all these vectors.
*/
- if (reg_icr) {
+ if (reg_icr && adapter->hw.mac.type < igb_mac_min) {
E1000_WRITE_REG(&adapter->hw,
E1000_ICS, adapter->ims);
}
- return;
-}
-
-static void
-em_handle_rx(void *context, int pending)
-{
- struct rx_ring *rxr = context;
- struct adapter *adapter = rxr->adapter;
- bool more;
-
- more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
- if (more)
- taskqueue_enqueue(rxr->tq, &rxr->rx_task);
- else {
- /* Reenable this interrupt */
- E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
- }
-}
-static void
-em_handle_tx(void *context, int pending)
-{
- struct tx_ring *txr = context;
- struct adapter *adapter = txr->adapter;
- if_t ifp = adapter->ifp;
-
- EM_TX_LOCK(txr);
- em_txeof(txr);
-#ifdef EM_MULTIQUEUE
- if (!drbr_empty(ifp, txr->br))
- em_mq_start_locked(ifp, txr);
-#else
- if (!if_sendq_empty(ifp))
- em_start_locked(ifp, txr);
-#endif
- E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
- EM_TX_UNLOCK(txr);
+ return (FILTER_HANDLED);
}
static void
-em_handle_link(void *context, int pending)
+em_handle_link(void *context)
{
- struct adapter *adapter = context;
- struct tx_ring *txr = adapter->tx_rings;
- if_t ifp = adapter->ifp;
-
- if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
- return;
+ if_ctx_t ctx = context;
+ struct adapter *adapter = iflib_get_softc(ctx);
- EM_CORE_LOCK(adapter);
- callout_stop(&adapter->timer);
- em_update_link_status(adapter);
- callout_reset(&adapter->timer, hz, em_local_timer, adapter);
- E1000_WRITE_REG(&adapter->hw, E1000_IMS,
- EM_MSIX_LINK | E1000_IMS_LSC);
- if (adapter->link_active) {
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- EM_TX_LOCK(txr);
-#ifdef EM_MULTIQUEUE
- if (!drbr_empty(ifp, txr->br))
- em_mq_start_locked(ifp, txr);
-#else
- if (if_sendq_empty(ifp))
- em_start_locked(ifp, txr);
-#endif
- EM_TX_UNLOCK(txr);
- }
- }
- EM_CORE_UNLOCK(adapter);
+ adapter->hw.mac.get_link_status = 1;
+ iflib_admin_intr_deferred(ctx);
}
@@ -1782,21 +1373,19 @@
*
**********************************************************************/
static void
-em_media_status(if_t ifp, struct ifmediareq *ifmr)
+em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
{
- struct adapter *adapter = if_getsoftc(ifp);
- u_char fiber_type = IFM_1000_SX;
-
- INIT_DEBUGOUT("em_media_status: begin");
+ struct adapter *adapter = iflib_get_softc(ctx);
+ u_char fiber_type = IFM_1000_SX;
+
+ INIT_DEBUGOUT("em_if_media_status: begin");
- EM_CORE_LOCK(adapter);
- em_update_link_status(adapter);
+ iflib_admin_intr_deferred(ctx);
ifmr->ifm_status = IFM_AVALID;
ifmr->ifm_active = IFM_ETHER;
if (!adapter->link_active) {
- EM_CORE_UNLOCK(adapter);
return;
}
@@ -1804,6 +1393,8 @@
if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
(adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
+ if (adapter->hw.mac.type == e1000_82545)
+ fiber_type = IFM_1000_LX;
ifmr->ifm_active |= fiber_type | IFM_FDX;
} else {
switch (adapter->link_speed) {
@@ -1822,7 +1413,6 @@
else
ifmr->ifm_active |= IFM_HDX;
}
- EM_CORE_UNLOCK(adapter);
}
/*********************************************************************
@@ -1834,17 +1424,16 @@
*
**********************************************************************/
static int
-em_media_change(if_t ifp)
+em_if_media_change(if_ctx_t ctx)
{
- struct adapter *adapter = if_getsoftc(ifp);
- struct ifmedia *ifm = &adapter->media;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct ifmedia *ifm = iflib_get_media(ctx);
- INIT_DEBUGOUT("em_media_change: begin");
+ INIT_DEBUGOUT("em_if_media_change: begin");
if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
return (EINVAL);
- EM_CORE_LOCK(adapter);
switch (IFM_SUBTYPE(ifm->ifm_media)) {
case IFM_AUTO:
adapter->hw.mac.autoneg = DO_AUTO_NEG;
@@ -1876,356 +1465,40 @@
device_printf(adapter->dev, "Unsupported media type\n");
}
- em_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
+ em_if_init(ctx);
return (0);
}
-/*********************************************************************
- *
- * This routine maps the mbufs to tx descriptors.
- *
- * return 0 on success, positive on failure
- **********************************************************************/
-
static int
-em_xmit(struct tx_ring *txr, struct mbuf **m_headp)
+em_if_set_promisc(if_ctx_t ctx, int flags)
{
- struct adapter *adapter = txr->adapter;
- bus_dma_segment_t segs[EM_MAX_SCATTER];
- bus_dmamap_t map;
- struct em_txbuffer *tx_buffer, *tx_buffer_mapped;
- struct e1000_tx_desc *ctxd = NULL;
- struct mbuf *m_head;
- struct ether_header *eh;
- struct ip *ip = NULL;
- struct tcphdr *tp = NULL;
- u32 txd_upper = 0, txd_lower = 0;
- int ip_off, poff;
- int nsegs, i, j, first, last = 0;
- int error;
- bool do_tso, tso_desc, remap = TRUE;
-
- m_head = *m_headp;
- do_tso = (m_head->m_pkthdr.csum_flags & CSUM_TSO);
- tso_desc = FALSE;
- ip_off = poff = 0;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ u32 reg_rctl;
- /*
- * Intel recommends entire IP/TCP header length reside in a single
- * buffer. If multiple descriptors are used to describe the IP and
- * TCP header, each descriptor should describe one or more
- * complete headers; descriptors referencing only parts of headers
- * are not supported. If all layer headers are not coalesced into
- * a single buffer, each buffer should not cross a 4KB boundary,
- * or be larger than the maximum read request size.
- * Controller also requires modifing IP/TCP header to make TSO work
- * so we firstly get a writable mbuf chain then coalesce ethernet/
- * IP/TCP header into a single buffer to meet the requirement of
- * controller. This also simplifies IP/TCP/UDP checksum offloading
- * which also has similar restrictions.
- */
- if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
- if (do_tso || (m_head->m_next != NULL &&
- m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) {
- if (M_WRITABLE(*m_headp) == 0) {
- m_head = m_dup(*m_headp, M_NOWAIT);
- m_freem(*m_headp);
- if (m_head == NULL) {
- *m_headp = NULL;
- return (ENOBUFS);
- }
- *m_headp = m_head;
- }
- }
- /*
- * XXX
- * Assume IPv4, we don't have TSO/checksum offload support
- * for IPv6 yet.
- */
- ip_off = sizeof(struct ether_header);
- if (m_head->m_len < ip_off) {
- m_head = m_pullup(m_head, ip_off);
- if (m_head == NULL) {
- *m_headp = NULL;
- return (ENOBUFS);
- }
- }
- eh = mtod(m_head, struct ether_header *);
- if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
- ip_off = sizeof(struct ether_vlan_header);
- if (m_head->m_len < ip_off) {
- m_head = m_pullup(m_head, ip_off);
- if (m_head == NULL) {
- *m_headp = NULL;
- return (ENOBUFS);
- }
- }
- }
- if (m_head->m_len < ip_off + sizeof(struct ip)) {
- m_head = m_pullup(m_head, ip_off + sizeof(struct ip));
- if (m_head == NULL) {
- *m_headp = NULL;
- return (ENOBUFS);
- }
- }
- ip = (struct ip *)(mtod(m_head, char *) + ip_off);
- poff = ip_off + (ip->ip_hl << 2);
-
- if (do_tso || (m_head->m_pkthdr.csum_flags & CSUM_TCP)) {
- if (m_head->m_len < poff + sizeof(struct tcphdr)) {
- m_head = m_pullup(m_head, poff +
- sizeof(struct tcphdr));
- if (m_head == NULL) {
- *m_headp = NULL;
- return (ENOBUFS);
- }
- }
- tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
- /*
- * TSO workaround:
- * pull 4 more bytes of data into it.
- */
- if (m_head->m_len < poff + (tp->th_off << 2)) {
- m_head = m_pullup(m_head, poff +
- (tp->th_off << 2) +
- TSO_WORKAROUND);
- if (m_head == NULL) {
- *m_headp = NULL;
- return (ENOBUFS);
- }
- }
- ip = (struct ip *)(mtod(m_head, char *) + ip_off);
- tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
- if (do_tso) {
- ip->ip_len = htons(m_head->m_pkthdr.tso_segsz +
- (ip->ip_hl << 2) +
- (tp->th_off << 2));
- ip->ip_sum = 0;
- /*
- * The pseudo TCP checksum does not include TCP
- * payload length so driver should recompute
- * the checksum here what hardware expect to
- * see. This is adherence of Microsoft's Large
- * Send specification.
- */
- tp->th_sum = in_pseudo(ip->ip_src.s_addr,
- ip->ip_dst.s_addr, htons(IPPROTO_TCP));
- }
- } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) {
- if (m_head->m_len < poff + sizeof(struct udphdr)) {
- m_head = m_pullup(m_head, poff +
- sizeof(struct udphdr));
- if (m_head == NULL) {
- *m_headp = NULL;
- return (ENOBUFS);
- }
- }
- ip = (struct ip *)(mtod(m_head, char *) + ip_off);
- }
- *m_headp = m_head;
- }
-
- /*
- * Map the packet for DMA
- *
- * Capture the first descriptor index,
- * this descriptor will have the index
- * of the EOP which is the only one that
- * now gets a DONE bit writeback.
- */
- first = txr->next_avail_desc;
- tx_buffer = &txr->tx_buffers[first];
- tx_buffer_mapped = tx_buffer;
- map = tx_buffer->map;
-
-retry:
- error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
- *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
-
- /*
- * There are two types of errors we can (try) to handle:
- * - EFBIG means the mbuf chain was too long and bus_dma ran
- * out of segments. Defragment the mbuf chain and try again.
- * - ENOMEM means bus_dma could not obtain enough bounce buffers
- * at this point in time. Defer sending and try again later.
- * All other errors, in particular EINVAL, are fatal and prevent the
- * mbuf chain from ever going through. Drop it and report error.
- */
- if (error == EFBIG && remap) {
- struct mbuf *m;
-
- m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER);
- if (m == NULL) {
- adapter->mbuf_defrag_failed++;
- m_freem(*m_headp);
- *m_headp = NULL;
- return (ENOBUFS);
- }
- *m_headp = m;
-
- /* Try it again, but only once */
- remap = FALSE;
- goto retry;
- } else if (error != 0) {
- adapter->no_tx_dma_setup++;
- m_freem(*m_headp);
- *m_headp = NULL;
- return (error);
- }
-
- /*
- * TSO Hardware workaround, if this packet is not
- * TSO, and is only a single descriptor long, and
- * it follows a TSO burst, then we need to add a
- * sentinel descriptor to prevent premature writeback.
- */
- if ((!do_tso) && (txr->tx_tso == TRUE)) {
- if (nsegs == 1)
- tso_desc = TRUE;
- txr->tx_tso = FALSE;
- }
-
- if (txr->tx_avail < (nsegs + EM_MAX_SCATTER)) {
- txr->no_desc_avail++;
- bus_dmamap_unload(txr->txtag, map);
- return (ENOBUFS);
- }
- m_head = *m_headp;
-
- /* Do hardware assists */
- if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
- em_tso_setup(txr, m_head, ip_off, ip, tp,
- &txd_upper, &txd_lower);
- /* we need to make a final sentinel transmit desc */
- tso_desc = TRUE;
- } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
- em_transmit_checksum_setup(txr, m_head,
- ip_off, ip, &txd_upper, &txd_lower);
-
- if (m_head->m_flags & M_VLANTAG) {
- /* Set the vlan id. */
- txd_upper |= htole16(if_getvtag(m_head)) << 16;
- /* Tell hardware to add tag */
- txd_lower |= htole32(E1000_TXD_CMD_VLE);
- }
-
- i = txr->next_avail_desc;
-
- /* Set up our transmit descriptors */
- for (j = 0; j < nsegs; j++) {
- bus_size_t seg_len;
- bus_addr_t seg_addr;
-
- tx_buffer = &txr->tx_buffers[i];
- ctxd = &txr->tx_base[i];
- seg_addr = segs[j].ds_addr;
- seg_len = segs[j].ds_len;
- /*
- ** TSO Workaround:
- ** If this is the last descriptor, we want to
- ** split it so we have a small final sentinel
- */
- if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
- seg_len -= TSO_WORKAROUND;
- ctxd->buffer_addr = htole64(seg_addr);
- ctxd->lower.data = htole32(
- adapter->txd_cmd | txd_lower | seg_len);
- ctxd->upper.data = htole32(txd_upper);
- if (++i == adapter->num_tx_desc)
- i = 0;
-
- /* Now make the sentinel */
- txr->tx_avail--;
- ctxd = &txr->tx_base[i];
- tx_buffer = &txr->tx_buffers[i];
- ctxd->buffer_addr =
- htole64(seg_addr + seg_len);
- ctxd->lower.data = htole32(
- adapter->txd_cmd | txd_lower | TSO_WORKAROUND);
- ctxd->upper.data =
- htole32(txd_upper);
- last = i;
- if (++i == adapter->num_tx_desc)
- i = 0;
- } else {
- ctxd->buffer_addr = htole64(seg_addr);
- ctxd->lower.data = htole32(
- adapter->txd_cmd | txd_lower | seg_len);
- ctxd->upper.data = htole32(txd_upper);
- last = i;
- if (++i == adapter->num_tx_desc)
- i = 0;
- }
- tx_buffer->m_head = NULL;
- tx_buffer->next_eop = -1;
- }
-
- txr->next_avail_desc = i;
- txr->tx_avail -= nsegs;
-
- tx_buffer->m_head = m_head;
- /*
- ** Here we swap the map so the last descriptor,
- ** which gets the completion interrupt has the
- ** real map, and the first descriptor gets the
- ** unused map from this descriptor.
- */
- tx_buffer_mapped->map = tx_buffer->map;
- tx_buffer->map = map;
- bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
-
- /*
- * Last Descriptor of Packet
- * needs End Of Packet (EOP)
- * and Report Status (RS)
- */
- ctxd->lower.data |=
- htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
- /*
- * Keep track in the first buffer which
- * descriptor will be written back
- */
- tx_buffer = &txr->tx_buffers[first];
- tx_buffer->next_eop = last;
-
- /*
- * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
- * that this frame is available to transmit.
- */
- bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
- E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
-
- return (0);
-}
-
-static void
-em_set_promisc(struct adapter *adapter)
-{
- if_t ifp = adapter->ifp;
- u32 reg_rctl;
+ em_disable_promisc(ctx);
reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
- if (if_getflags(ifp) & IFF_PROMISC) {
+ if (flags & IFF_PROMISC) {
reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
/* Turn this on if you want to see bad packets */
if (em_debug_sbp)
reg_rctl |= E1000_RCTL_SBP;
E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
- } else if (if_getflags(ifp) & IFF_ALLMULTI) {
+ } else if (flags & IFF_ALLMULTI) {
reg_rctl |= E1000_RCTL_MPE;
reg_rctl &= ~E1000_RCTL_UPE;
E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
}
+ return (0);
}
static void
-em_disable_promisc(struct adapter *adapter)
+em_disable_promisc(if_ctx_t ctx)
{
- if_t ifp = adapter->ifp;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct ifnet *ifp = iflib_get_ifp(ctx);
u32 reg_rctl;
int mcnt = 0;
@@ -2251,9 +1524,10 @@
**********************************************************************/
static void
-em_set_multi(struct adapter *adapter)
+em_if_multi_set(if_ctx_t ctx)
{
- if_t ifp = adapter->ifp;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct ifnet *ifp = iflib_get_ifp(ctx);
u32 reg_rctl = 0;
u8 *mta; /* Multicast array memory */
int mcnt = 0;
@@ -2302,17 +1576,14 @@
**********************************************************************/
static void
-em_local_timer(void *arg)
+em_if_timer(if_ctx_t ctx, uint16_t qid)
{
- struct adapter *adapter = arg;
- if_t ifp = adapter->ifp;
- struct tx_ring *txr = adapter->tx_rings;
- struct rx_ring *rxr = adapter->rx_rings;
- u32 trigger = 0;
-
- EM_CORE_LOCK_ASSERT(adapter);
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct em_rx_queue *que;
+ int i;
+ int trigger = 0;
- em_update_link_status(adapter);
+ em_if_update_admin_status(ctx);
em_update_stats_counters(adapter);
/* Reset LAA into RAR[0] on 82571 */
@@ -2320,53 +1591,26 @@
e1000_get_laa_state_82571(&adapter->hw))
e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
+ if (adapter->hw.mac.type < em_mac_min)
+ lem_smartspeed(adapter);
+
/* Mask to use in the irq trigger */
- if (adapter->msix_mem) {
- for (int i = 0; i < adapter->num_queues; i++, rxr++)
- trigger |= rxr->ims;
- rxr = adapter->rx_rings;
- } else
+ if (adapter->intr_type == IFLIB_INTR_MSIX) {
+ for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++)
+ trigger |= que->eims;
+ } else {
trigger = E1000_ICS_RXDMT0;
-
- /*
- ** Check on the state of the TX queue(s), this
- ** can be done without the lock because its RO
- ** and the HUNG state will be static if set.
- */
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- if (txr->busy == EM_TX_HUNG)
- goto hung;
- if (txr->busy >= EM_TX_MAXTRIES)
- txr->busy = EM_TX_HUNG;
- /* Schedule a TX tasklet if needed */
- if (txr->tx_avail <= EM_MAX_SCATTER)
- taskqueue_enqueue(txr->tq, &txr->tx_task);
}
-
- callout_reset(&adapter->timer, hz, em_local_timer, adapter);
-#ifndef DEVICE_POLLING
- /* Trigger an RX interrupt to guarantee mbuf refresh */
- E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger);
-#endif
- return;
-hung:
- /* Looks like we're hung */
- device_printf(adapter->dev, "Watchdog timeout Queue[%d]-- resetting\n",
- txr->me);
- em_print_debug_info(adapter);
- if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
- adapter->watchdog_events++;
- em_init_locked(adapter);
}
static void
-em_update_link_status(struct adapter *adapter)
+em_if_update_admin_status(if_ctx_t ctx)
{
+ struct adapter *adapter = iflib_get_softc(ctx);
struct e1000_hw *hw = &adapter->hw;
- if_t ifp = adapter->ifp;
- device_t dev = adapter->dev;
- struct tx_ring *txr = adapter->tx_rings;
+ struct ifnet *ifp = iflib_get_ifp(ctx);
+ device_t dev = iflib_get_dev(ctx);
u32 link_check = 0;
/* Get the cached link value or read phy for real */
@@ -2380,8 +1624,9 @@
link_check = !hw->mac.get_link_status;
if (link_check) /* ESB2 fix */
e1000_cfg_on_link_up(hw);
- } else
+ } else {
link_check = TRUE;
+ }
break;
case e1000_media_type_fiber:
e1000_check_for_link(hw);
@@ -2401,18 +1646,6 @@
if (link_check && (adapter->link_active == 0)) {
e1000_get_speed_and_duplex(hw, &adapter->link_speed,
&adapter->link_duplex);
- /*
- ** There have proven to be problems with TSO when not
- ** at full gigabit speed, so disable the assist automatically
- ** when at lower speeds. -jfv
- */
- if (adapter->link_speed != SPEED_1000) {
- if_sethwassistbits(ifp, 0, CSUM_TSO);
- if_setcapenablebit(ifp, 0, IFCAP_TSO4);
- if_setcapabilitiesbit(ifp, 0, IFCAP_TSO4);
-
- }
-
/* Check if we must disable SPEED_MODE bit on PCI-E */
if ((adapter->link_speed != SPEED_1000) &&
((hw->mac.type == e1000_82571) ||
@@ -2430,7 +1663,8 @@
adapter->link_active = 1;
adapter->smartspeed = 0;
if_setbaudrate(ifp, adapter->link_speed * 1000000);
- if_link_state_change(ifp, LINK_STATE_UP);
+ iflib_link_state_change(ctx, LINK_STATE_UP, ifp->if_baudrate);
+ printf("Link state changed to up\n");
} else if (!link_check && (adapter->link_active == 1)) {
if_setbaudrate(ifp, 0);
adapter->link_speed = 0;
@@ -2438,11 +1672,11 @@
if (bootverbose)
device_printf(dev, "Link is Down\n");
adapter->link_active = 0;
- /* Link down, disable hang detection */
- for (int i = 0; i < adapter->num_queues; i++, txr++)
- txr->busy = EM_TX_IDLE;
- if_link_state_change(ifp, LINK_STATE_DOWN);
+ iflib_link_state_change(ctx, LINK_STATE_DOWN, ifp->if_baudrate);
+ printf("link state changed to down\n");
}
+
+ E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC);
}
/*********************************************************************
@@ -2455,35 +1689,15 @@
**********************************************************************/
static void
-em_stop(void *arg)
+em_if_stop(if_ctx_t ctx)
{
- struct adapter *adapter = arg;
- if_t ifp = adapter->ifp;
- struct tx_ring *txr = adapter->tx_rings;
-
- EM_CORE_LOCK_ASSERT(adapter);
+ struct adapter *adapter = iflib_get_softc(ctx);
INIT_DEBUGOUT("em_stop: begin");
-
- em_disable_intr(adapter);
- callout_stop(&adapter->timer);
-
- /* Tell the stack that the interface is no longer active */
- if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
-
- /* Disarm Hang Detection. */
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- EM_TX_LOCK(txr);
- txr->busy = EM_TX_IDLE;
- EM_TX_UNLOCK(txr);
- }
-
- /* I219 needs some special flushing to avoid hangs */
- if (adapter->hw.mac.type == e1000_pch_spt)
- em_flush_desc_rings(adapter);
-
+
e1000_reset_hw(&adapter->hw);
- E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
+ if (adapter->hw.mac.type >= e1000_82544)
+ E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
e1000_led_off(&adapter->hw);
e1000_cleanup_led(&adapter->hw);
@@ -2496,12 +1710,12 @@
*
**********************************************************************/
static void
-em_identify_hardware(struct adapter *adapter)
+em_identify_hardware(if_ctx_t ctx)
{
- device_t dev = adapter->dev;
-
+ device_t dev = iflib_get_dev(ctx);
+ struct adapter *adapter = iflib_get_softc(ctx);
+
/* Make sure our PCI config space has the necessary stuff set */
- pci_enable_busmaster(dev);
adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
/* Save off the information about this board */
@@ -2521,10 +1735,11 @@
}
static int
-em_allocate_pci_resources(struct adapter *adapter)
+em_allocate_pci_resources(if_ctx_t ctx)
{
- device_t dev = adapter->dev;
- int rid;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ device_t dev = iflib_get_dev(ctx);
+ int rid, val;
rid = PCIR_BAR(0);
adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
@@ -2539,499 +1754,398 @@
rman_get_bushandle(adapter->memory);
adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
- adapter->hw.back = &adapter->osdep;
-
- return (0);
-}
-
-/*********************************************************************
- *
- * Setup the Legacy or MSI Interrupt handler
- *
- **********************************************************************/
-int
-em_allocate_legacy(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- struct tx_ring *txr = adapter->tx_rings;
- int error, rid = 0;
-
- /* Manually turn off all interrupts */
- E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
-
- if (adapter->msix == 1) /* using MSI */
- rid = 1;
- /* We allocate a single interrupt resource */
- adapter->res = bus_alloc_resource_any(dev,
- SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
- if (adapter->res == NULL) {
- device_printf(dev, "Unable to allocate bus resource: "
- "interrupt\n");
- return (ENXIO);
+ /* Only older adapters use IO mapping */
+ if (adapter->hw.mac.type < em_mac_min &&
+ adapter->hw.mac.type > e1000_82543) {
+ /* Figure our where our IO BAR is ? */
+ for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
+ val = pci_read_config(dev, rid, 4);
+ if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
+ adapter->io_rid = rid;
+ break;
+ }
+ rid += 4;
+ /* check for 64bit BAR */
+ if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
+ rid += 4;
+ }
+ if (rid >= PCIR_CIS) {
+ device_printf(dev, "Unable to locate IO BAR\n");
+ return (ENXIO);
+ }
+ adapter->ioport = bus_alloc_resource_any(dev,
+ SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
+ if (adapter->ioport == NULL) {
+ device_printf(dev, "Unable to allocate bus resource: "
+ "ioport\n");
+ return (ENXIO);
+ }
+ adapter->hw.io_base = 0;
+ adapter->osdep.io_bus_space_tag =
+ rman_get_bustag(adapter->ioport);
+ adapter->osdep.io_bus_space_handle =
+ rman_get_bushandle(adapter->ioport);
}
- /*
- * Allocate a fast interrupt and the associated
- * deferred processing contexts.
- */
- TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter);
- adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
- taskqueue_thread_enqueue, &adapter->tq);
- taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que",
- device_get_nameunit(adapter->dev));
- /* Use a TX only tasklet for local timer */
- TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
- txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
- taskqueue_thread_enqueue, &txr->tq);
- taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
- device_get_nameunit(adapter->dev));
- TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
- if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET,
- em_irq_fast, NULL, adapter, &adapter->tag)) != 0) {
- device_printf(dev, "Failed to register fast interrupt "
- "handler: %d\n", error);
- taskqueue_free(adapter->tq);
- adapter->tq = NULL;
- return (error);
- }
-
+ adapter->hw.back = &adapter->osdep;
+
return (0);
}
/*********************************************************************
*
* Setup the MSIX Interrupt handlers
- * This is not really Multiqueue, rather
- * its just separate interrupt vectors
- * for TX, RX, and Link.
*
**********************************************************************/
-int
-em_allocate_msix(struct adapter *adapter)
+static int
+em_if_msix_intr_assign(if_ctx_t ctx, int msix)
{
- device_t dev = adapter->dev;
- struct tx_ring *txr = adapter->tx_rings;
- struct rx_ring *rxr = adapter->rx_rings;
- int error, rid, vector = 0;
- int cpu_id = 0;
-
-
- /* Make sure all interrupts are disabled */
- E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct em_rx_queue *rx_que = adapter->rx_queues;
+ struct em_tx_queue *tx_que = adapter->tx_queues;
+ int error, rid, i, vector = 0;
+ char buf[16];
/* First set up ring resources */
- for (int i = 0; i < adapter->num_queues; i++, rxr++, vector++) {
-
- /* RX ring */
- rid = vector + 1;
-
- rxr->res = bus_alloc_resource_any(dev,
- SYS_RES_IRQ, &rid, RF_ACTIVE);
- if (rxr->res == NULL) {
- device_printf(dev,
- "Unable to allocate bus resource: "
- "RX MSIX Interrupt %d\n", i);
- return (ENXIO);
- }
- if ((error = bus_setup_intr(dev, rxr->res,
- INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx,
- rxr, &rxr->tag)) != 0) {
- device_printf(dev, "Failed to register RX handler");
- return (error);
+ for (i = 0; i < adapter->rx_num_queues; i++, rx_que++, vector++) {
+ rid = vector +1;
+ snprintf(buf, sizeof(buf), "rxq%d", i);
+ error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, IFLIB_INTR_RX, em_msix_que, rx_que, rx_que->me, buf);
+ if (error) {
+ device_printf(iflib_get_dev(ctx), "Failed to allocate que int %d err: %d", i, error);
+ adapter->rx_num_queues = i + 1;
+ goto fail;
}
-#if __FreeBSD_version >= 800504
- bus_describe_intr(dev, rxr->res, rxr->tag, "rx%d", i);
-#endif
- rxr->msix = vector;
- if (em_last_bind_cpu < 0)
- em_last_bind_cpu = CPU_FIRST();
- cpu_id = em_last_bind_cpu;
- bus_bind_intr(dev, rxr->res, cpu_id);
-
- TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr);
- rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT,
- taskqueue_thread_enqueue, &rxr->tq);
- taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq (cpuid %d)",
- device_get_nameunit(adapter->dev), cpu_id);
+ rx_que->msix = vector;
+
/*
** Set the bit to enable interrupt
** in E1000_IMS -- bits 20 and 21
** are for RX0 and RX1, note this has
** NOTHING to do with the MSIX vector
*/
- rxr->ims = 1 << (20 + i);
- adapter->ims |= rxr->ims;
- adapter->ivars |= (8 | rxr->msix) << (i * 4);
-
- em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu);
+ if (adapter->hw.mac.type == e1000_82574) {
+ rx_que->eims = 1 << (20 + i);
+ adapter->ims |= rx_que->eims;
+ adapter->ivars |= (8 | rx_que->msix) << (i * 4);
+ } else if (adapter->hw.mac.type == e1000_82575)
+ rx_que->eims = E1000_EICR_TX_QUEUE0 << vector;
+ else
+ rx_que->eims = 1 << vector;
}
- for (int i = 0; i < adapter->num_queues; i++, txr++, vector++) {
- /* TX ring */
+ for (i = 0; i < adapter->tx_num_queues; i++, tx_que++) {
rid = vector + 1;
- txr->res = bus_alloc_resource_any(dev,
- SYS_RES_IRQ, &rid, RF_ACTIVE);
- if (txr->res == NULL) {
- device_printf(dev,
- "Unable to allocate bus resource: "
- "TX MSIX Interrupt %d\n", i);
- return (ENXIO);
- }
- if ((error = bus_setup_intr(dev, txr->res,
- INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx,
- txr, &txr->tag)) != 0) {
- device_printf(dev, "Failed to register TX handler");
- return (error);
- }
-#if __FreeBSD_version >= 800504
- bus_describe_intr(dev, txr->res, txr->tag, "tx%d", i);
-#endif
- txr->msix = vector;
+ snprintf(buf, sizeof(buf), "txq%d", i);
+ tx_que = &adapter->tx_queues[i];
+ iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_TX, tx_que, tx_que->me, buf);
- if (em_last_bind_cpu < 0)
- em_last_bind_cpu = CPU_FIRST();
- cpu_id = em_last_bind_cpu;
- bus_bind_intr(dev, txr->res, cpu_id);
-
- TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
- txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
- taskqueue_thread_enqueue, &txr->tq);
- taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq (cpuid %d)",
- device_get_nameunit(adapter->dev), cpu_id);
- /*
+ tx_que->msix = vector;
+
+ /*
** Set the bit to enable interrupt
** in E1000_IMS -- bits 22 and 23
** are for TX0 and TX1, note this has
** NOTHING to do with the MSIX vector
*/
- txr->ims = 1 << (22 + i);
- adapter->ims |= txr->ims;
- adapter->ivars |= (8 | txr->msix) << (8 + (i * 4));
-
- em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu);
+ if (adapter->hw.mac.type < igb_mac_min) {
+ tx_que->eims = 1 << (22 + i);
+ adapter->ims |= tx_que->eims;
+ adapter->ivars |= (8 | tx_que->msix) << (8 + (i * 4));
+ } if (adapter->hw.mac.type == e1000_82575)
+ tx_que->eims = E1000_EICR_TX_QUEUE0 << (i % adapter->tx_num_queues);
+ else
+ tx_que->eims = 1 << (i % adapter->tx_num_queues);
}
-
+
/* Link interrupt */
rid = vector + 1;
- adapter->res = bus_alloc_resource_any(dev,
- SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
- if (!adapter->res) {
- device_printf(dev,"Unable to allocate "
- "bus resource: Link interrupt [%d]\n", rid);
- return (ENXIO);
- }
- /* Set the link handler function */
- error = bus_setup_intr(dev, adapter->res,
- INTR_TYPE_NET | INTR_MPSAFE, NULL,
- em_msix_link, adapter, &adapter->tag);
+ error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid, IFLIB_INTR_ADMIN, em_msix_link, adapter, 0, "aq");
+
if (error) {
- adapter->res = NULL;
- device_printf(dev, "Failed to register LINK handler");
- return (error);
+ device_printf(iflib_get_dev(ctx), "Failed to register admin handler");
+ goto fail;
}
-#if __FreeBSD_version >= 800504
- bus_describe_intr(dev, adapter->res, adapter->tag, "link");
-#endif
adapter->linkvec = vector;
- adapter->ivars |= (8 | vector) << 16;
- adapter->ivars |= 0x80000000;
-
+ if (adapter->hw.mac.type < igb_mac_min) {
+ adapter->ivars |= (8 | vector) << 16;
+ adapter->ivars |= 0x80000000;
+ }
return (0);
+ fail:
+ iflib_irq_free(ctx, &adapter->irq);
+ rx_que = adapter->rx_queues;
+ for (int i = 0; i < adapter->rx_num_queues; i++, rx_que++)
+ iflib_irq_free(ctx, &rx_que->que_irq);
+ return (error);
}
-
static void
-em_free_pci_resources(struct adapter *adapter)
+igb_configure_queues(struct adapter *adapter)
{
- device_t dev = adapter->dev;
- struct tx_ring *txr;
- struct rx_ring *rxr;
- int rid;
+ struct e1000_hw *hw = &adapter->hw;
+ struct em_rx_queue *rx_que;
+ struct em_tx_queue *tx_que;
+ u32 tmp, ivar = 0, newitr = 0;
+
+ /* First turn on RSS capability */
+ if (adapter->hw.mac.type != e1000_82575)
+ E1000_WRITE_REG(hw, E1000_GPIE,
+ E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
+ E1000_GPIE_PBA | E1000_GPIE_NSICR);
+ /* Turn on MSIX */
+ switch (adapter->hw.mac.type) {
+ case e1000_82580:
+ case e1000_i350:
+ case e1000_i354:
+ case e1000_i210:
+ case e1000_i211:
+ case e1000_vfadapt:
+ case e1000_vfadapt_i350:
+ /* RX entries */
+ for (int i = 0; i < adapter->rx_num_queues; i++) {
+ u32 index = i >> 1;
+ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ rx_que = &adapter->rx_queues[i];
+ if (i & 1) {
+ ivar &= 0xFF00FFFF;
+ ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16;
+ } else {
+ ivar &= 0xFFFFFF00;
+ ivar |= rx_que->msix | E1000_IVAR_VALID;
+ }
+ E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
+ }
+ /* TX entries */
+ for (int i = 0; i < adapter->tx_num_queues; i++) {
+ u32 index = i >> 1;
+ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ tx_que = &adapter->tx_queues[i];
+ if (i & 1) {
+ ivar &= 0x00FFFFFF;
+ ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24;
+ } else {
+ ivar &= 0xFFFF00FF;
+ ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8;
+ }
+ E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
+ adapter->que_mask |= tx_que->eims;
+ }
- /*
- ** Release all the queue interrupt resources:
- */
- for (int i = 0; i < adapter->num_queues; i++) {
- txr = &adapter->tx_rings[i];
- /* an early abort? */
- if (txr == NULL)
- break;
- rid = txr->msix +1;
- if (txr->tag != NULL) {
- bus_teardown_intr(dev, txr->res, txr->tag);
- txr->tag = NULL;
+ /* And for the link interrupt */
+ ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
+ adapter->link_mask = 1 << adapter->linkvec;
+ E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
+ break;
+ case e1000_82576:
+ /* RX entries */
+ for (int i = 0; i < adapter->rx_num_queues; i++) {
+ u32 index = i & 0x7; /* Each IVAR has two entries */
+ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ rx_que = &adapter->rx_queues[i];
+ if (i < 8) {
+ ivar &= 0xFFFFFF00;
+ ivar |= rx_que->msix | E1000_IVAR_VALID;
+ } else {
+ ivar &= 0xFF00FFFF;
+ ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16;
+ }
+ E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
+ adapter->que_mask |= rx_que->eims;
}
- if (txr->res != NULL)
- bus_release_resource(dev, SYS_RES_IRQ,
- rid, txr->res);
-
- rxr = &adapter->rx_rings[i];
- /* an early abort? */
- if (rxr == NULL)
- break;
- rid = rxr->msix +1;
- if (rxr->tag != NULL) {
- bus_teardown_intr(dev, rxr->res, rxr->tag);
- rxr->tag = NULL;
+ /* TX entries */
+ for (int i = 0; i < adapter->tx_num_queues; i++) {
+ u32 index = i & 0x7; /* Each IVAR has two entries */
+ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ tx_que = &adapter->tx_queues[i];
+ if (i < 8) {
+ ivar &= 0xFFFF00FF;
+ ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8;
+ } else {
+ ivar &= 0x00FFFFFF;
+ ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24;
+ }
+ E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
+ adapter->que_mask |= tx_que->eims;
}
- if (rxr->res != NULL)
- bus_release_resource(dev, SYS_RES_IRQ,
- rid, rxr->res);
+
+ /* And for the link interrupt */
+ ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
+ adapter->link_mask = 1 << adapter->linkvec;
+ E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
+ break;
+
+ case e1000_82575:
+ /* enable MSI-X support*/
+ tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
+ tmp |= E1000_CTRL_EXT_PBA_CLR;
+ /* Auto-Mask interrupts upon ICR read. */
+ tmp |= E1000_CTRL_EXT_EIAME;
+ tmp |= E1000_CTRL_EXT_IRCA;
+ E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
+
+ /* Queues */
+ for (int i = 0; i < adapter->rx_num_queues; i++) {
+ rx_que = &adapter->rx_queues[i];
+ tmp = E1000_EICR_RX_QUEUE0 << i;
+ tmp |= E1000_EICR_TX_QUEUE0 << i;
+ rx_que->eims = tmp;
+ E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
+ i, rx_que->eims);
+ adapter->que_mask |= rx_que->eims;
+ }
+
+ /* Link */
+ E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
+ E1000_EIMS_OTHER);
+ adapter->link_mask |= E1000_EIMS_OTHER;
+ default:
+ break;
}
- if (adapter->linkvec) /* we are doing MSIX */
- rid = adapter->linkvec + 1;
+ /* Set the starting interrupt rate */
+ if (em_max_interrupt_rate > 0)
+ newitr = (4000000 / em_max_interrupt_rate) & 0x7FFC;
+
+ if (hw->mac.type == e1000_82575)
+ newitr |= newitr << 16;
else
- (adapter->msix != 0) ? (rid = 1):(rid = 0);
+ newitr |= E1000_EITR_CNT_IGNR;
- if (adapter->tag != NULL) {
- bus_teardown_intr(dev, adapter->res, adapter->tag);
- adapter->tag = NULL;
+ for (int i = 0; i < adapter->rx_num_queues; i++) {
+ rx_que = &adapter->rx_queues[i];
+ E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), newitr);
}
- if (adapter->res != NULL)
- bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
+ return;
+}
+
+static void
+em_free_pci_resources(if_ctx_t ctx)
+{
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct em_rx_queue *que = adapter->rx_queues;
+ device_t dev = iflib_get_dev(ctx);
+ /* Release all msix queue resources */
+ if (adapter->intr_type == IFLIB_INTR_MSIX)
+ iflib_irq_free(ctx, &adapter->irq);
- if (adapter->msix)
- pci_release_msi(dev);
+ for (int i = 0; i < adapter->rx_num_queues; i++, que++) {
+ iflib_irq_free(ctx, &que->que_irq);
+ }
- if (adapter->msix_mem != NULL)
- bus_release_resource(dev, SYS_RES_MEMORY,
- PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
- if (adapter->memory != NULL)
+ /* First release all the interrupt resources */
+ if (adapter->memory != NULL) {
bus_release_resource(dev, SYS_RES_MEMORY,
- PCIR_BAR(0), adapter->memory);
+ PCIR_BAR(0), adapter->memory);
+ adapter->memory = NULL;
+ }
- if (adapter->flash != NULL)
+ if (adapter->flash != NULL) {
bus_release_resource(dev, SYS_RES_MEMORY,
- EM_FLASH, adapter->flash);
+ EM_FLASH, adapter->flash);
+ adapter->flash = NULL;
+ }
+ if (adapter->ioport != NULL)
+ bus_release_resource(dev, SYS_RES_IOPORT,
+ adapter->io_rid, adapter->ioport);
}
-/*
- * Setup MSI or MSI/X
- */
+/* Setup MSI or MSI/X */
static int
-em_setup_msix(struct adapter *adapter)
+em_setup_msix(if_ctx_t ctx)
{
- device_t dev = adapter->dev;
- int val;
-
- /* Nearly always going to use one queue */
- adapter->num_queues = 1;
-
- /*
- ** Try using MSI-X for Hartwell adapters
- */
- if ((adapter->hw.mac.type == e1000_82574) &&
- (em_enable_msix == TRUE)) {
-#ifdef EM_MULTIQUEUE
- adapter->num_queues = (em_num_queues == 1) ? 1 : 2;
- if (adapter->num_queues > 1)
- em_enable_vectors_82574(adapter);
-#endif
- /* Map the MSIX BAR */
- int rid = PCIR_BAR(EM_MSIX_BAR);
- adapter->msix_mem = bus_alloc_resource_any(dev,
- SYS_RES_MEMORY, &rid, RF_ACTIVE);
- if (adapter->msix_mem == NULL) {
- /* May not be enabled */
- device_printf(adapter->dev,
- "Unable to map MSIX table \n");
- goto msi;
- }
- val = pci_msix_count(dev);
-
-#ifdef EM_MULTIQUEUE
- /* We need 5 vectors in the multiqueue case */
- if (adapter->num_queues > 1 ) {
- if (val >= 5)
- val = 5;
- else {
- adapter->num_queues = 1;
- device_printf(adapter->dev,
- "Insufficient MSIX vectors for >1 queue, "
- "using single queue...\n");
- goto msix_one;
- }
- } else {
-msix_one:
-#endif
- if (val >= 3)
- val = 3;
- else {
- device_printf(adapter->dev,
- "Insufficient MSIX vectors, using MSI\n");
- goto msi;
- }
-#ifdef EM_MULTIQUEUE
- }
-#endif
-
- if ((pci_alloc_msix(dev, &val) == 0)) {
- device_printf(adapter->dev,
- "Using MSIX interrupts "
- "with %d vectors\n", val);
- return (val);
- }
+ struct adapter *adapter = iflib_get_softc(ctx);
- /*
- ** If MSIX alloc failed or provided us with
- ** less than needed, free and fall through to MSI
- */
- pci_release_msi(dev);
- }
-msi:
- if (adapter->msix_mem != NULL) {
- bus_release_resource(dev, SYS_RES_MEMORY,
- PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
- adapter->msix_mem = NULL;
+ if (adapter->hw.mac.type == e1000_82574) {
+ em_enable_vectors_82574(ctx);
}
- val = 1;
- if (pci_alloc_msi(dev, &val) == 0) {
- device_printf(adapter->dev, "Using an MSI interrupt\n");
- return (val);
- }
- /* Should only happen due to manual configuration */
- device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n");
return (0);
}
+/*********************************************************************
+ *
+ * Initialize the hardware to a configuration
+ * as specified by the adapter structure.
+ *
+ **********************************************************************/
-/*
-** The 3 following flush routines are used as a workaround in the
-** I219 client parts and only for them.
-**
-** em_flush_tx_ring - remove all descriptors from the tx_ring
-**
-** We want to clear all pending descriptors from the TX ring.
-** zeroing happens when the HW reads the regs. We assign the ring itself as
-** the data of the next descriptor. We don't care about the data we are about
-** to reset the HW.
-*/
static void
-em_flush_tx_ring(struct adapter *adapter)
+lem_smartspeed(struct adapter *adapter)
{
- struct e1000_hw *hw = &adapter->hw;
- struct tx_ring *txr = adapter->tx_rings;
- struct e1000_tx_desc *txd;
- u32 tctl, txd_lower = E1000_TXD_CMD_IFCS;
- u16 size = 512;
-
- tctl = E1000_READ_REG(hw, E1000_TCTL);
- E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN);
-
- txd = &txr->tx_base[txr->next_avail_desc++];
- if (txr->next_avail_desc == adapter->num_tx_desc)
- txr->next_avail_desc = 0;
-
- /* Just use the ring as a dummy buffer addr */
- txd->buffer_addr = txr->txdma.dma_paddr;
- txd->lower.data = htole32(txd_lower | size);
- txd->upper.data = 0;
-
- /* flush descriptors to memory before notifying the HW */
- wmb();
-
- E1000_WRITE_REG(hw, E1000_TDT(0), txr->next_avail_desc);
- mb();
- usec_delay(250);
+ u16 phy_tmp;
+
+ if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
+ adapter->hw.mac.autoneg == 0 ||
+ (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
+ return;
+
+ if (adapter->smartspeed == 0) {
+ /* If Master/Slave config fault is asserted twice,
+ * we assume back-to-back */
+ e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
+ if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
+ return;
+ e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
+ if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
+ e1000_read_phy_reg(&adapter->hw,
+ PHY_1000T_CTRL, &phy_tmp);
+ if(phy_tmp & CR_1000T_MS_ENABLE) {
+ phy_tmp &= ~CR_1000T_MS_ENABLE;
+ e1000_write_phy_reg(&adapter->hw,
+ PHY_1000T_CTRL, phy_tmp);
+ adapter->smartspeed++;
+ if(adapter->hw.mac.autoneg &&
+ !e1000_copper_link_autoneg(&adapter->hw) &&
+ !e1000_read_phy_reg(&adapter->hw,
+ PHY_CONTROL, &phy_tmp)) {
+ phy_tmp |= (MII_CR_AUTO_NEG_EN |
+ MII_CR_RESTART_AUTO_NEG);
+ e1000_write_phy_reg(&adapter->hw,
+ PHY_CONTROL, phy_tmp);
+ }
+ }
+ }
+ return;
+ } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
+ /* If still no link, perhaps using 2/3 pair cable */
+ e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
+ phy_tmp |= CR_1000T_MS_ENABLE;
+ e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
+ if(adapter->hw.mac.autoneg &&
+ !e1000_copper_link_autoneg(&adapter->hw) &&
+ !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
+ phy_tmp |= (MII_CR_AUTO_NEG_EN |
+ MII_CR_RESTART_AUTO_NEG);
+ e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
+ }
+ }
+ /* Restart process after EM_SMARTSPEED_MAX iterations */
+ if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
+ adapter->smartspeed = 0;
}
-/*
-** em_flush_rx_ring - remove all descriptors from the rx_ring
-**
-** Mark all descriptors in the RX ring as consumed and disable the rx ring
-*/
+
static void
-em_flush_rx_ring(struct adapter *adapter)
+em_reset(if_ctx_t ctx)
{
+ device_t dev = iflib_get_dev(ctx);
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct ifnet *ifp = iflib_get_ifp(ctx);
struct e1000_hw *hw = &adapter->hw;
- u32 rctl, rxdctl;
+ u16 rx_buffer_size;
+ u32 pba;
- rctl = E1000_READ_REG(hw, E1000_RCTL);
- E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
- E1000_WRITE_FLUSH(hw);
- usec_delay(150);
-
- rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
- /* zero the lower 14 bits (prefetch and host thresholds) */
- rxdctl &= 0xffffc000;
- /*
- * update thresholds: prefetch threshold to 31, host threshold to 1
- * and make sure the granularity is "descriptors" and not "cache lines"
- */
- rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC);
- E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl);
+ INIT_DEBUGOUT("em_reset: begin");
- /* momentarily enable the RX ring for the changes to take effect */
- E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN);
- E1000_WRITE_FLUSH(hw);
- usec_delay(150);
- E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
-}
-
-/*
-** em_flush_desc_rings - remove all descriptors from the descriptor rings
-**
-** In i219, the descriptor rings must be emptied before resetting the HW
-** or before changing the device state to D3 during runtime (runtime PM).
-**
-** Failure to do this will cause the HW to enter a unit hang state which can
-** only be released by PCI reset on the device
-**
-*/
-static void
-em_flush_desc_rings(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- device_t dev = adapter->dev;
- u16 hang_state;
- u32 fext_nvm11, tdlen;
-
- /* First, disable MULR fix in FEXTNVM11 */
- fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11);
- fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX;
- E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11);
-
- /* do nothing if we're not in faulty state, or if the queue is empty */
- tdlen = E1000_READ_REG(hw, E1000_TDLEN(0));
- hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2);
- if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen)
- return;
- em_flush_tx_ring(adapter);
-
- /* recheck, maybe the fault is caused by the rx ring */
- hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2);
- if (hang_state & FLUSH_DESC_REQUIRED)
- em_flush_rx_ring(adapter);
-}
-
-
-/*********************************************************************
- *
- * Initialize the hardware to a configuration
- * as specified by the adapter structure.
- *
- **********************************************************************/
-static void
-em_reset(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- if_t ifp = adapter->ifp;
- struct e1000_hw *hw = &adapter->hw;
- u16 rx_buffer_size;
- u32 pba;
-
- INIT_DEBUGOUT("em_reset: begin");
-
- /* Set up smart power down as default off on newer adapters. */
- if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
- hw->mac.type == e1000_82572)) {
- u16 phy_tmp = 0;
+ /* Set up smart power down as default off on newer adapters. */
+ if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
+ hw->mac.type == e1000_82572)) {
+ u16 phy_tmp = 0;
/* Speed up time to link by disabling smart power down. */
e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
@@ -3156,10 +2270,6 @@
break;
}
- /* I219 needs some special flushing to avoid hangs */
- if (hw->mac.type == e1000_pch_spt)
- em_flush_desc_rings(adapter);
-
/* Issue a global reset */
e1000_reset_hw(hw);
E1000_WRITE_REG(hw, E1000_WUC, 0);
@@ -3173,7 +2283,145 @@
E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
e1000_get_phy_info(hw);
e1000_check_for_link(hw);
- return;
+}
+
+#define RSSKEYLEN 10
+static void
+em_initialize_rss_mapping(struct adapter *adapter)
+{
+ uint8_t rss_key[4 * RSSKEYLEN];
+ uint32_t reta = 0;
+ struct e1000_hw *hw = &adapter->hw;
+ int i;
+
+ /*
+ * Configure RSS key
+ */
+ arc4rand(rss_key, sizeof(rss_key), 0);
+ for (i = 0; i < RSSKEYLEN; ++i) {
+ uint32_t rssrk = 0;
+
+ rssrk = EM_RSSRK_VAL(rss_key, i);
+ E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk);
+ }
+
+ /*
+ * Configure RSS redirect table in following fashion:
+ * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
+ */
+ for (i = 0; i < sizeof(reta); ++i) {
+ uint32_t q;
+
+ q = (i % adapter->rx_num_queues) << 7;
+ reta |= q << (8 * i);
+ }
+
+ for (i = 0; i < 32; ++i)
+ E1000_WRITE_REG(hw, E1000_RETA(i), reta);
+
+ E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q |
+ E1000_MRQC_RSS_FIELD_IPV4_TCP |
+ E1000_MRQC_RSS_FIELD_IPV4 |
+ E1000_MRQC_RSS_FIELD_IPV6_TCP_EX |
+ E1000_MRQC_RSS_FIELD_IPV6_EX |
+ E1000_MRQC_RSS_FIELD_IPV6);
+
+}
+
+static void
+igb_initialize_rss_mapping(struct adapter *adapter)
+{
+ struct e1000_hw *hw = &adapter->hw;
+ int i;
+ int queue_id;
+ u32 reta;
+ u32 rss_key[10], mrqc, shift = 0;
+
+ /* XXX? */
+ if (adapter->hw.mac.type == e1000_82575)
+ shift = 6;
+
+ /*
+ * The redirection table controls which destination
+ * queue each bucket redirects traffic to.
+ * Each DWORD represents four queues, with the LSB
+ * being the first queue in the DWORD.
+ *
+ * This just allocates buckets to queues using round-robin
+ * allocation.
+ *
+ * NOTE: It Just Happens to line up with the default
+ * RSS allocation method.
+ */
+
+ /* Warning FM follows */
+ reta = 0;
+ for (i = 0; i < 128; i++) {
+#ifdef RSS
+ queue_id = rss_get_indirection_to_bucket(i);
+ /*
+ * If we have more queues than buckets, we'll
+ * end up mapping buckets to a subset of the
+ * queues.
+ *
+ * If we have more buckets than queues, we'll
+ * end up instead assigning multiple buckets
+ * to queues.
+ *
+ * Both are suboptimal, but we need to handle
+ * the case so we don't go out of bounds
+ * indexing arrays and such.
+ */
+ queue_id = queue_id % adapter->rx_num_queues;
+#else
+ queue_id = (i % adapter->rx_num_queues);
+#endif
+ /* Adjust if required */
+ queue_id = queue_id << shift;
+
+ /*
+ * The low 8 bits are for hash value (n+0);
+ * The next 8 bits are for hash value (n+1), etc.
+ */
+ reta = reta >> 8;
+ reta = reta | ( ((uint32_t) queue_id) << 24);
+ if ((i & 3) == 3) {
+ E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
+ reta = 0;
+ }
+ }
+
+ /* Now fill in hash table */
+
+ /*
+ * MRQC: Multiple Receive Queues Command
+ * Set queuing to RSS control, number depends on the device.
+ */
+ mrqc = E1000_MRQC_ENABLE_RSS_8Q;
+
+#ifdef RSS
+ /* XXX ew typecasting */
+ rss_getkey((uint8_t *) &rss_key);
+#else
+ arc4rand(&rss_key, sizeof(rss_key), 0);
+#endif
+ for (i = 0; i < 10; i++)
+ E1000_WRITE_REG_ARRAY(hw,
+ E1000_RSSRK(0), i, rss_key[i]);
+
+ /*
+ * Configure the RSS fields to hash upon.
+ */
+ mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
+ E1000_MRQC_RSS_FIELD_IPV4_TCP);
+ mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
+ E1000_MRQC_RSS_FIELD_IPV6_TCP);
+ mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
+ E1000_MRQC_RSS_FIELD_IPV6_UDP);
+ mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
+ E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
+
+ E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
}
/*********************************************************************
@@ -3182,57 +2430,36 @@
*
**********************************************************************/
static int
-em_setup_interface(device_t dev, struct adapter *adapter)
+em_setup_interface(if_ctx_t ctx)
{
- if_t ifp;
-
+ struct ifnet *ifp = iflib_get_ifp(ctx);
+ struct adapter *adapter = iflib_get_softc(ctx);
+ if_softc_ctx_t scctx = adapter->shared;
+ uint64_t cap = 0;
+
INIT_DEBUGOUT("em_setup_interface: begin");
- ifp = adapter->ifp = if_gethandle(IFT_ETHER);
- if (ifp == 0) {
- device_printf(dev, "can not allocate ifnet structure\n");
- return (-1);
- }
- if_initname(ifp, device_get_name(dev), device_get_unit(dev));
- if_setdev(ifp, dev);
- if_setinitfn(ifp, em_init);
- if_setsoftc(ifp, adapter);
- if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
- if_setioctlfn(ifp, em_ioctl);
- if_setgetcounterfn(ifp, em_get_counter);
-
/* TSO parameters */
ifp->if_hw_tsomax = IP_MAXPACKET;
/* Take m_pullup(9)'s in em_xmit() w/ TSO into acount. */
ifp->if_hw_tsomaxsegcount = EM_MAX_SCATTER - 5;
ifp->if_hw_tsomaxsegsize = EM_TSO_SEG_SIZE;
-#ifdef EM_MULTIQUEUE
- /* Multiqueue stack interface */
- if_settransmitfn(ifp, em_mq_start);
- if_setqflushfn(ifp, em_qflush);
-#else
- if_setstartfn(ifp, em_start);
- if_setsendqlen(ifp, adapter->num_tx_desc - 1);
- if_setsendqready(ifp);
-#endif
-
- ether_ifattach(ifp, adapter->hw.mac.addr);
-
- if_setcapabilities(ifp, 0);
- if_setcapenable(ifp, 0);
+ /* Single Queue */
+ if (adapter->tx_num_queues == 1) {
+ if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1);
+ if_setsendqready(ifp);
+ }
+ cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4;
+ cap |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU;
- if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM |
- IFCAP_TSO4, 0);
/*
* Tell the upper layer(s) we
* support full VLAN capability
*/
if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
- if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO |
- IFCAP_VLAN_MTU, 0);
- if_setcapenable(ifp, if_getcapabilities(ifp));
+ if_setcapabilitiesbit(ifp, cap, 0);
/*
** Don't turn this on by default, if vlans are
@@ -3244,10 +2471,6 @@
*/
if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER,0);
-#ifdef DEVICE_POLLING
- if_setcapabilitiesbit(ifp, IFCAP_POLLING,0);
-#endif
-
/* Enable only WOL MAGIC by default */
if (adapter->wol) {
if_setcapabilitiesbit(ifp, IFCAP_WOL, 0);
@@ -3258,396 +2481,146 @@
* Specify the media types supported by this adapter and register
* callbacks to update media and link information
*/
- ifmedia_init(&adapter->media, IFM_IMASK,
- em_media_change, em_media_status);
if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
(adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
u_char fiber_type = IFM_1000_SX; /* default type */
- ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
+ if (adapter->hw.mac.type == e1000_82545)
+ fiber_type = IFM_1000_LX;
+ ifmedia_add(adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 0, NULL);
+ ifmedia_add(adapter->media, IFM_ETHER | fiber_type, 0, NULL);
} else {
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
- 0, NULL);
+ ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
+ ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL);
+ ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL);
+ ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL);
if (adapter->hw.phy.type != e1000_phy_ife) {
- ifmedia_add(&adapter->media,
- IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
- ifmedia_add(&adapter->media,
- IFM_ETHER | IFM_1000_T, 0, NULL);
+ ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
+ ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
}
}
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
- ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
+ ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
+ ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO);
return (0);
}
-
-/*
- * Manage DMA'able memory.
- */
-static void
-em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
-{
- if (error)
- return;
- *(bus_addr_t *) arg = segs[0].ds_addr;
-}
-
static int
-em_dma_malloc(struct adapter *adapter, bus_size_t size,
- struct em_dma_alloc *dma, int mapflags)
+em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets)
{
- int error;
-
- error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
- EM_DBA_ALIGN, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- size, /* maxsize */
- 1, /* nsegments */
- size, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockarg */
- &dma->dma_tag);
- if (error) {
- device_printf(adapter->dev,
- "%s: bus_dma_tag_create failed: %d\n",
- __func__, error);
- goto fail_0;
- }
-
- error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
- BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
- if (error) {
- device_printf(adapter->dev,
- "%s: bus_dmamem_alloc(%ju) failed: %d\n",
- __func__, (uintmax_t)size, error);
- goto fail_2;
- }
-
- dma->dma_paddr = 0;
- error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
- size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
- if (error || dma->dma_paddr == 0) {
- device_printf(adapter->dev,
- "%s: bus_dmamap_load failed: %d\n",
- __func__, error);
- goto fail_3;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ if_softc_ctx_t scctx = adapter->shared;
+ int error = E1000_SUCCESS;
+ struct em_tx_queue *que;
+ int i;
+
+ MPASS(adapter->tx_num_queues > 0);
+ MPASS(adapter->tx_num_queues == ntxqsets);
+
+ /* First allocate the top level queue structs */
+ if (!(adapter->tx_queues =
+ (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) *
+ adapter->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
+ device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n");
+ return(ENOMEM);
+ }
+
+ for (i = 0, que = adapter->tx_queues; i < adapter->tx_num_queues; i++, que++) {
+ /* Set up some basics */
+ struct tx_ring *txr = &que->txr;
+ txr->adapter = que->adapter = adapter;
+ txr->que = que;
+ que->me = txr->me = i;
+
+ /* Allocate transmit buffer memory */
+ if (!(txr->tx_buffers = (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) {
+ device_printf(iflib_get_dev(ctx), "failed to allocate tx_buffer memory\n");
+ error = ENOMEM;
+ goto fail;
+ }
+
+ /* get the virtual and physical address of the hardware queues */
+ txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs];
+ txr->tx_paddr = paddrs[i*ntxqs];
+
}
-
+
+ device_printf(iflib_get_dev(ctx), "allocated for %d tx_queues\n", adapter->tx_num_queues);
return (0);
-
-fail_3:
- bus_dmamap_unload(dma->dma_tag, dma->dma_map);
-fail_2:
- bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
- bus_dma_tag_destroy(dma->dma_tag);
-fail_0:
- dma->dma_tag = NULL;
-
+ fail:
+ em_if_queues_free(ctx);
return (error);
}
-static void
-em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
-{
- if (dma->dma_tag == NULL)
- return;
- if (dma->dma_paddr != 0) {
- bus_dmamap_sync(dma->dma_tag, dma->dma_map,
- BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(dma->dma_tag, dma->dma_map);
- dma->dma_paddr = 0;
- }
- if (dma->dma_vaddr != NULL) {
- bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
- dma->dma_vaddr = NULL;
- }
- bus_dma_tag_destroy(dma->dma_tag);
- dma->dma_tag = NULL;
-}
-
-
-/*********************************************************************
- *
- * Allocate memory for the transmit and receive rings, and then
- * the descriptors associated with each, called only once at attach.
- *
- **********************************************************************/
static int
-em_allocate_queues(struct adapter *adapter)
+em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets)
{
- device_t dev = adapter->dev;
- struct tx_ring *txr = NULL;
- struct rx_ring *rxr = NULL;
- int rsize, tsize, error = E1000_SUCCESS;
- int txconf = 0, rxconf = 0;
-
-
- /* Allocate the TX ring struct memory */
- if (!(adapter->tx_rings =
- (struct tx_ring *) malloc(sizeof(struct tx_ring) *
- adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
- device_printf(dev, "Unable to allocate TX ring memory\n");
+ struct adapter *adapter = iflib_get_softc(ctx);
+ int error = E1000_SUCCESS;
+ struct em_rx_queue *que;
+ int i;
+
+ MPASS(adapter->rx_num_queues > 0);
+ MPASS(adapter->rx_num_queues == nrxqsets);
+
+ /* First allocate the top level queue structs */
+ if (!(adapter->rx_queues =
+ (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) *
+ adapter->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
+ device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n");
error = ENOMEM;
- goto fail;
+ goto fail;
}
- /* Now allocate the RX */
- if (!(adapter->rx_rings =
- (struct rx_ring *) malloc(sizeof(struct rx_ring) *
- adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
- device_printf(dev, "Unable to allocate RX ring memory\n");
- error = ENOMEM;
- goto rx_fail;
- }
-
- tsize = roundup2(adapter->num_tx_desc *
- sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
- /*
- * Now set up the TX queues, txconf is needed to handle the
- * possibility that things fail midcourse and we need to
- * undo memory gracefully
- */
- for (int i = 0; i < adapter->num_queues; i++, txconf++) {
+ for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) {
/* Set up some basics */
- txr = &adapter->tx_rings[i];
- txr->adapter = adapter;
- txr->me = i;
-
- /* Initialize the TX lock */
- snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
- device_get_nameunit(dev), txr->me);
- mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
-
- if (em_dma_malloc(adapter, tsize,
- &txr->txdma, BUS_DMA_NOWAIT)) {
- device_printf(dev,
- "Unable to allocate TX Descriptor memory\n");
- error = ENOMEM;
- goto err_tx_desc;
- }
- txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
- bzero((void *)txr->tx_base, tsize);
+ struct rx_ring *rxr = &que->rxr;
+ rxr->adapter = que->adapter = adapter;
+ rxr->que = que;
+ que->me = rxr->me = i;
- if (em_allocate_transmit_buffers(txr)) {
- device_printf(dev,
- "Critical Failure setting up transmit buffers\n");
- error = ENOMEM;
- goto err_tx_desc;
- }
-#if __FreeBSD_version >= 800000
- /* Allocate a buf ring */
- txr->br = buf_ring_alloc(4096, M_DEVBUF,
- M_WAITOK, &txr->tx_mtx);
-#endif
+ /* get the virtual and physical address of the hardware queues */
+ rxr->rx_base = (union e1000_rx_desc_extended *)vaddrs[i*nrxqs];
+ rxr->rx_paddr = paddrs[i*nrxqs];
}
- /*
- * Next the RX queues...
- */
- rsize = roundup2(adapter->num_rx_desc *
- sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN);
- for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
- rxr = &adapter->rx_rings[i];
- rxr->adapter = adapter;
- rxr->me = i;
-
- /* Initialize the RX lock */
- snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
- device_get_nameunit(dev), txr->me);
- mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
-
- if (em_dma_malloc(adapter, rsize,
- &rxr->rxdma, BUS_DMA_NOWAIT)) {
- device_printf(dev,
- "Unable to allocate RxDescriptor memory\n");
- error = ENOMEM;
- goto err_rx_desc;
- }
- rxr->rx_base = (union e1000_rx_desc_extended *)rxr->rxdma.dma_vaddr;
- bzero((void *)rxr->rx_base, rsize);
-
- /* Allocate receive buffers for the ring*/
- if (em_allocate_receive_buffers(rxr)) {
- device_printf(dev,
- "Critical Failure setting up receive buffers\n");
- error = ENOMEM;
- goto err_rx_desc;
- }
- }
+ device_printf(iflib_get_dev(ctx), "allocated for %d rx_queues\n", adapter->rx_num_queues);
return (0);
-
-err_rx_desc:
- for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
- em_dma_free(adapter, &rxr->rxdma);
-err_tx_desc:
- for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
- em_dma_free(adapter, &txr->txdma);
- free(adapter->rx_rings, M_DEVBUF);
-rx_fail:
-#if __FreeBSD_version >= 800000
- buf_ring_free(txr->br, M_DEVBUF);
-#endif
- free(adapter->tx_rings, M_DEVBUF);
fail:
+ em_if_queues_free(ctx);
return (error);
}
-
-/*********************************************************************
- *
- * Allocate memory for tx_buffer structures. The tx_buffer stores all
- * the information needed to transmit a packet on the wire. This is
- * called only once at attach, setup is done every reset.
- *
- **********************************************************************/
-static int
-em_allocate_transmit_buffers(struct tx_ring *txr)
+static void
+em_if_queues_free(if_ctx_t ctx)
{
- struct adapter *adapter = txr->adapter;
- device_t dev = adapter->dev;
- struct em_txbuffer *txbuf;
- int error, i;
-
- /*
- * Setup DMA descriptor areas.
- */
- if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
- 1, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- EM_TSO_SIZE, /* maxsize */
- EM_MAX_SCATTER, /* nsegments */
- PAGE_SIZE, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockfuncarg */
- &txr->txtag))) {
- device_printf(dev,"Unable to allocate TX DMA tag\n");
- goto fail;
- }
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct em_tx_queue *tx_que = adapter->tx_queues;
+ struct em_rx_queue *rx_que = adapter->rx_queues;
+
+ if (tx_que != NULL) {
+ for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) {
+ struct tx_ring *txr = &tx_que->txr;
+ if (txr->tx_buffers == NULL)
+ break;
- if (!(txr->tx_buffers =
- (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) *
- adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
- device_printf(dev, "Unable to allocate tx_buffer memory\n");
- error = ENOMEM;
- goto fail;
+ free(txr->tx_buffers, M_DEVBUF);
+ txr->tx_buffers = NULL;
+ }
+ free(adapter->tx_queues, M_DEVBUF);
+ adapter->tx_queues = NULL;
}
- /* Create the descriptor buffer dma maps */
- txbuf = txr->tx_buffers;
- for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
- error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
- if (error != 0) {
- device_printf(dev, "Unable to create TX DMA map\n");
- goto fail;
- }
+ if (rx_que != NULL) {
+ free(adapter->rx_queues, M_DEVBUF);
+ adapter->rx_queues = NULL;
}
- return 0;
-fail:
- /* We free all, it handles case where we are in the middle */
- em_free_transmit_structures(adapter);
- return (error);
-}
-
-/*********************************************************************
- *
- * Initialize a transmit ring.
- *
- **********************************************************************/
-static void
-em_setup_transmit_ring(struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
- struct em_txbuffer *txbuf;
- int i;
-#ifdef DEV_NETMAP
- struct netmap_slot *slot;
- struct netmap_adapter *na = netmap_getna(adapter->ifp);
-#endif /* DEV_NETMAP */
-
- /* Clear the old descriptor contents */
- EM_TX_LOCK(txr);
-#ifdef DEV_NETMAP
- slot = netmap_reset(na, NR_TX, txr->me, 0);
-#endif /* DEV_NETMAP */
-
- bzero((void *)txr->tx_base,
- (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
- /* Reset indices */
- txr->next_avail_desc = 0;
- txr->next_to_clean = 0;
-
- /* Free any existing tx buffers. */
- txbuf = txr->tx_buffers;
- for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
- if (txbuf->m_head != NULL) {
- bus_dmamap_sync(txr->txtag, txbuf->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(txr->txtag, txbuf->map);
- m_freem(txbuf->m_head);
- txbuf->m_head = NULL;
- }
-#ifdef DEV_NETMAP
- if (slot) {
- int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
- uint64_t paddr;
- void *addr;
-
- addr = PNMB(na, slot + si, &paddr);
- txr->tx_base[i].buffer_addr = htole64(paddr);
- /* reload the map for netmap mode */
- netmap_load_map(na, txr->txtag, txbuf->map, addr);
- }
-#endif /* DEV_NETMAP */
-
- /* clear the watch index */
- txbuf->next_eop = -1;
- }
-
- /* Set number of descriptors available */
- txr->tx_avail = adapter->num_tx_desc;
- txr->busy = EM_TX_IDLE;
-
- /* Clear checksum offload context. */
- txr->last_hw_offload = 0;
- txr->last_hw_ipcss = 0;
- txr->last_hw_ipcso = 0;
- txr->last_hw_tucss = 0;
- txr->last_hw_tucso = 0;
-
- bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
- EM_TX_UNLOCK(txr);
-}
-
-/*********************************************************************
- *
- * Initialize all transmit rings.
- *
- **********************************************************************/
-static void
-em_setup_transmit_structures(struct adapter *adapter)
-{
- struct tx_ring *txr = adapter->tx_rings;
-
- for (int i = 0; i < adapter->num_queues; i++, txr++)
- em_setup_transmit_ring(txr);
+ em_release_hw_control(adapter);
- return;
+ if (adapter->mta != NULL) {
+ free(adapter->mta, M_DEVBUF);
+ }
}
/*********************************************************************
@@ -3656,19 +2629,36 @@
*
**********************************************************************/
static void
-em_initialize_transmit_unit(struct adapter *adapter)
+em_initialize_transmit_unit(if_ctx_t ctx)
{
- struct tx_ring *txr = adapter->tx_rings;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_tx_queue *que;
+ struct tx_ring *txr;
struct e1000_hw *hw = &adapter->hw;
- u32 tctl, txdctl = 0, tarc, tipg = 0;
+ u32 tctl, txdctl = 0, tarc, tipg = 0;
INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- u64 bus_addr = txr->txdma.dma_paddr;
+ for (int i = 0; i < adapter->tx_num_queues; i++, txr++) {
+ u64 bus_addr;
+ caddr_t offp, endp;
+
+ que = &adapter->tx_queues[i];
+ txr = &que->txr;
+ bus_addr = txr->tx_paddr;
+
+ /*Enable all queues */
+ em_init_tx_ring(que);
+
+ /* Clear checksum offload context. */
+ offp = (caddr_t)&txr->csum_flags;
+ endp = (caddr_t)(txr + 1);
+ bzero(offp, endp - offp);
+
/* Base and Len of TX Ring */
E1000_WRITE_REG(hw, E1000_TDLEN(i),
- adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
+ scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc));
E1000_WRITE_REG(hw, E1000_TDBAH(i),
(u32)(bus_addr >> 32));
E1000_WRITE_REG(hw, E1000_TDBAL(i),
@@ -3681,7 +2671,6 @@
E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
- txr->busy = EM_TX_IDLE;
txdctl = 0; /* clear txdctl */
txdctl |= 0x1f; /* PTHRESH */
txdctl |= 1 << 8; /* HTHRESH */
@@ -3700,849 +2689,79 @@
tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
E1000_TIPG_IPGR2_SHIFT;
break;
+ case e1000_82542:
+ tipg = DEFAULT_82542_TIPG_IPGT;
+ tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
+ tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
+ break;
default:
if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
(adapter->hw.phy.media_type ==
e1000_media_type_internal_serdes))
- tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
- else
- tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
- tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
- tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
- }
-
- E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
- E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
-
- if(adapter->hw.mac.type >= e1000_82540)
- E1000_WRITE_REG(&adapter->hw, E1000_TADV,
- adapter->tx_abs_int_delay.value);
-
- if ((adapter->hw.mac.type == e1000_82571) ||
- (adapter->hw.mac.type == e1000_82572)) {
- tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
- tarc |= TARC_SPEED_MODE_BIT;
- E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
- } else if (adapter->hw.mac.type == e1000_80003es2lan) {
- /* errata: program both queues to unweighted RR */
- tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
- tarc |= 1;
- E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
- tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
- tarc |= 1;
- E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
- } else if (adapter->hw.mac.type == e1000_82574) {
- tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
- tarc |= TARC_ERRATA_BIT;
- if ( adapter->num_queues > 1) {
- tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX);
- E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
- E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
- } else
- E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
- }
-
- adapter->txd_cmd = E1000_TXD_CMD_IFCS;
- if (adapter->tx_int_delay.value > 0)
- adapter->txd_cmd |= E1000_TXD_CMD_IDE;
-
- /* Program the Transmit Control Register */
- tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
- tctl &= ~E1000_TCTL_CT;
- tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
- (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
-
- if (adapter->hw.mac.type >= e1000_82571)
- tctl |= E1000_TCTL_MULR;
-
- /* This write will effectively turn on the transmit unit. */
- E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
-
- if (hw->mac.type == e1000_pch_spt) {
- u32 reg;
- reg = E1000_READ_REG(hw, E1000_IOSFPC);
- reg |= E1000_RCTL_RDMTS_HEX;
- E1000_WRITE_REG(hw, E1000_IOSFPC, reg);
- reg = E1000_READ_REG(hw, E1000_TARC(0));
- reg |= E1000_TARC0_CB_MULTIQ_3_REQ;
- E1000_WRITE_REG(hw, E1000_TARC(0), reg);
- }
-}
-
-
-/*********************************************************************
- *
- * Free all transmit rings.
- *
- **********************************************************************/
-static void
-em_free_transmit_structures(struct adapter *adapter)
-{
- struct tx_ring *txr = adapter->tx_rings;
-
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- EM_TX_LOCK(txr);
- em_free_transmit_buffers(txr);
- em_dma_free(adapter, &txr->txdma);
- EM_TX_UNLOCK(txr);
- EM_TX_LOCK_DESTROY(txr);
- }
-
- free(adapter->tx_rings, M_DEVBUF);
-}
-
-/*********************************************************************
- *
- * Free transmit ring related data structures.
- *
- **********************************************************************/
-static void
-em_free_transmit_buffers(struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
- struct em_txbuffer *txbuf;
-
- INIT_DEBUGOUT("free_transmit_ring: begin");
-
- if (txr->tx_buffers == NULL)
- return;
-
- for (int i = 0; i < adapter->num_tx_desc; i++) {
- txbuf = &txr->tx_buffers[i];
- if (txbuf->m_head != NULL) {
- bus_dmamap_sync(txr->txtag, txbuf->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(txr->txtag,
- txbuf->map);
- m_freem(txbuf->m_head);
- txbuf->m_head = NULL;
- if (txbuf->map != NULL) {
- bus_dmamap_destroy(txr->txtag,
- txbuf->map);
- txbuf->map = NULL;
- }
- } else if (txbuf->map != NULL) {
- bus_dmamap_unload(txr->txtag,
- txbuf->map);
- bus_dmamap_destroy(txr->txtag,
- txbuf->map);
- txbuf->map = NULL;
- }
- }
-#if __FreeBSD_version >= 800000
- if (txr->br != NULL)
- buf_ring_free(txr->br, M_DEVBUF);
-#endif
- if (txr->tx_buffers != NULL) {
- free(txr->tx_buffers, M_DEVBUF);
- txr->tx_buffers = NULL;
- }
- if (txr->txtag != NULL) {
- bus_dma_tag_destroy(txr->txtag);
- txr->txtag = NULL;
- }
- return;
-}
-
-
-/*********************************************************************
- * The offload context is protocol specific (TCP/UDP) and thus
- * only needs to be set when the protocol changes. The occasion
- * of a context change can be a performance detriment, and
- * might be better just disabled. The reason arises in the way
- * in which the controller supports pipelined requests from the
- * Tx data DMA. Up to four requests can be pipelined, and they may
- * belong to the same packet or to multiple packets. However all
- * requests for one packet are issued before a request is issued
- * for a subsequent packet and if a request for the next packet
- * requires a context change, that request will be stalled
- * until the previous request completes. This means setting up
- * a new context effectively disables pipelined Tx data DMA which
- * in turn greatly slow down performance to send small sized
- * frames.
- **********************************************************************/
-static void
-em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
- struct ip *ip, u32 *txd_upper, u32 *txd_lower)
-{
- struct adapter *adapter = txr->adapter;
- struct e1000_context_desc *TXD = NULL;
- struct em_txbuffer *tx_buffer;
- int cur, hdr_len;
- u32 cmd = 0;
- u16 offload = 0;
- u8 ipcso, ipcss, tucso, tucss;
-
- ipcss = ipcso = tucss = tucso = 0;
- hdr_len = ip_off + (ip->ip_hl << 2);
- cur = txr->next_avail_desc;
-
- /* Setup of IP header checksum. */
- if (mp->m_pkthdr.csum_flags & CSUM_IP) {
- *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
- offload |= CSUM_IP;
- ipcss = ip_off;
- ipcso = ip_off + offsetof(struct ip, ip_sum);
- /*
- * Start offset for header checksum calculation.
- * End offset for header checksum calculation.
- * Offset of place to put the checksum.
- */
- TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
- TXD->lower_setup.ip_fields.ipcss = ipcss;
- TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
- TXD->lower_setup.ip_fields.ipcso = ipcso;
- cmd |= E1000_TXD_CMD_IP;
- }
-
- if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
- *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
- *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
- offload |= CSUM_TCP;
- tucss = hdr_len;
- tucso = hdr_len + offsetof(struct tcphdr, th_sum);
- /*
- * The 82574L can only remember the *last* context used
- * regardless of queue that it was use for. We cannot reuse
- * contexts on this hardware platform and must generate a new
- * context every time. 82574L hardware spec, section 7.2.6,
- * second note.
- */
- if (adapter->num_queues < 2) {
- /*
- * Setting up new checksum offload context for every
- * frames takes a lot of processing time for hardware.
- * This also reduces performance a lot for small sized
- * frames so avoid it if driver can use previously
- * configured checksum offload context.
- */
- if (txr->last_hw_offload == offload) {
- if (offload & CSUM_IP) {
- if (txr->last_hw_ipcss == ipcss &&
- txr->last_hw_ipcso == ipcso &&
- txr->last_hw_tucss == tucss &&
- txr->last_hw_tucso == tucso)
- return;
- } else {
- if (txr->last_hw_tucss == tucss &&
- txr->last_hw_tucso == tucso)
- return;
- }
- }
- txr->last_hw_offload = offload;
- txr->last_hw_tucss = tucss;
- txr->last_hw_tucso = tucso;
- }
- /*
- * Start offset for payload checksum calculation.
- * End offset for payload checksum calculation.
- * Offset of place to put the checksum.
- */
- TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
- TXD->upper_setup.tcp_fields.tucss = hdr_len;
- TXD->upper_setup.tcp_fields.tucse = htole16(0);
- TXD->upper_setup.tcp_fields.tucso = tucso;
- cmd |= E1000_TXD_CMD_TCP;
- } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
- *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
- *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
- tucss = hdr_len;
- tucso = hdr_len + offsetof(struct udphdr, uh_sum);
- /*
- * The 82574L can only remember the *last* context used
- * regardless of queue that it was use for. We cannot reuse
- * contexts on this hardware platform and must generate a new
- * context every time. 82574L hardware spec, section 7.2.6,
- * second note.
- */
- if (adapter->num_queues < 2) {
- /*
- * Setting up new checksum offload context for every
- * frames takes a lot of processing time for hardware.
- * This also reduces performance a lot for small sized
- * frames so avoid it if driver can use previously
- * configured checksum offload context.
- */
- if (txr->last_hw_offload == offload) {
- if (offload & CSUM_IP) {
- if (txr->last_hw_ipcss == ipcss &&
- txr->last_hw_ipcso == ipcso &&
- txr->last_hw_tucss == tucss &&
- txr->last_hw_tucso == tucso)
- return;
- } else {
- if (txr->last_hw_tucss == tucss &&
- txr->last_hw_tucso == tucso)
- return;
- }
- }
- txr->last_hw_offload = offload;
- txr->last_hw_tucss = tucss;
- txr->last_hw_tucso = tucso;
- }
- /*
- * Start offset for header checksum calculation.
- * End offset for header checksum calculation.
- * Offset of place to put the checksum.
- */
- TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
- TXD->upper_setup.tcp_fields.tucss = tucss;
- TXD->upper_setup.tcp_fields.tucse = htole16(0);
- TXD->upper_setup.tcp_fields.tucso = tucso;
- }
-
- if (offload & CSUM_IP) {
- txr->last_hw_ipcss = ipcss;
- txr->last_hw_ipcso = ipcso;
- }
-
- TXD->tcp_seg_setup.data = htole32(0);
- TXD->cmd_and_length =
- htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
- tx_buffer = &txr->tx_buffers[cur];
- tx_buffer->m_head = NULL;
- tx_buffer->next_eop = -1;
-
- if (++cur == adapter->num_tx_desc)
- cur = 0;
-
- txr->tx_avail--;
- txr->next_avail_desc = cur;
-}
-
-
-/**********************************************************************
- *
- * Setup work for hardware segmentation offload (TSO)
- *
- **********************************************************************/
-static void
-em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
- struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower)
-{
- struct adapter *adapter = txr->adapter;
- struct e1000_context_desc *TXD;
- struct em_txbuffer *tx_buffer;
- int cur, hdr_len;
-
- /*
- * In theory we can use the same TSO context if and only if
- * frame is the same type(IP/TCP) and the same MSS. However
- * checking whether a frame has the same IP/TCP structure is
- * hard thing so just ignore that and always restablish a
- * new TSO context.
- */
- hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2);
- *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
- E1000_TXD_DTYP_D | /* Data descr type */
- E1000_TXD_CMD_TSE); /* Do TSE on this packet */
-
- /* IP and/or TCP header checksum calculation and insertion. */
- *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
-
- cur = txr->next_avail_desc;
- tx_buffer = &txr->tx_buffers[cur];
- TXD = (struct e1000_context_desc *) &txr->tx_base[cur];
-
- /*
- * Start offset for header checksum calculation.
- * End offset for header checksum calculation.
- * Offset of place put the checksum.
- */
- TXD->lower_setup.ip_fields.ipcss = ip_off;
- TXD->lower_setup.ip_fields.ipcse =
- htole16(ip_off + (ip->ip_hl << 2) - 1);
- TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum);
- /*
- * Start offset for payload checksum calculation.
- * End offset for payload checksum calculation.
- * Offset of place to put the checksum.
- */
- TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2);
- TXD->upper_setup.tcp_fields.tucse = 0;
- TXD->upper_setup.tcp_fields.tucso =
- ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum);
- /*
- * Payload size per packet w/o any headers.
- * Length of all headers up to payload.
- */
- TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
- TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
-
- TXD->cmd_and_length = htole32(adapter->txd_cmd |
- E1000_TXD_CMD_DEXT | /* Extended descr */
- E1000_TXD_CMD_TSE | /* TSE context */
- E1000_TXD_CMD_IP | /* Do IP csum */
- E1000_TXD_CMD_TCP | /* Do TCP checksum */
- (mp->m_pkthdr.len - (hdr_len))); /* Total len */
-
- tx_buffer->m_head = NULL;
- tx_buffer->next_eop = -1;
-
- if (++cur == adapter->num_tx_desc)
- cur = 0;
-
- txr->tx_avail--;
- txr->next_avail_desc = cur;
- txr->tx_tso = TRUE;
-}
-
-
-/**********************************************************************
- *
- * Examine each tx_buffer in the used queue. If the hardware is done
- * processing the packet then free associated resources. The
- * tx_buffer is put back on the free queue.
- *
- **********************************************************************/
-static void
-em_txeof(struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
- int first, last, done, processed;
- struct em_txbuffer *tx_buffer;
- struct e1000_tx_desc *tx_desc, *eop_desc;
- if_t ifp = adapter->ifp;
-
- EM_TX_LOCK_ASSERT(txr);
-#ifdef DEV_NETMAP
- if (netmap_tx_irq(ifp, txr->me))
- return;
-#endif /* DEV_NETMAP */
-
- /* No work, make sure hang detection is disabled */
- if (txr->tx_avail == adapter->num_tx_desc) {
- txr->busy = EM_TX_IDLE;
- return;
- }
-
- processed = 0;
- first = txr->next_to_clean;
- tx_desc = &txr->tx_base[first];
- tx_buffer = &txr->tx_buffers[first];
- last = tx_buffer->next_eop;
- eop_desc = &txr->tx_base[last];
-
- /*
- * What this does is get the index of the
- * first descriptor AFTER the EOP of the
- * first packet, that way we can do the
- * simple comparison on the inner while loop.
- */
- if (++last == adapter->num_tx_desc)
- last = 0;
- done = last;
-
- bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
- BUS_DMASYNC_POSTREAD);
-
- while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
- /* We clean the range of the packet */
- while (first != done) {
- tx_desc->upper.data = 0;
- tx_desc->lower.data = 0;
- tx_desc->buffer_addr = 0;
- ++txr->tx_avail;
- ++processed;
-
- if (tx_buffer->m_head) {
- bus_dmamap_sync(txr->txtag,
- tx_buffer->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(txr->txtag,
- tx_buffer->map);
- m_freem(tx_buffer->m_head);
- tx_buffer->m_head = NULL;
- }
- tx_buffer->next_eop = -1;
-
- if (++first == adapter->num_tx_desc)
- first = 0;
-
- tx_buffer = &txr->tx_buffers[first];
- tx_desc = &txr->tx_base[first];
- }
- if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
- /* See if we can continue to the next packet */
- last = tx_buffer->next_eop;
- if (last != -1) {
- eop_desc = &txr->tx_base[last];
- /* Get new done point */
- if (++last == adapter->num_tx_desc) last = 0;
- done = last;
- } else
- break;
- }
- bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
- txr->next_to_clean = first;
-
- /*
- ** Hang detection: we know there's work outstanding
- ** or the entry return would have been taken, so no
- ** descriptor processed here indicates a potential hang.
- ** The local timer will examine this and do a reset if needed.
- */
- if (processed == 0) {
- if (txr->busy != EM_TX_HUNG)
- ++txr->busy;
- } else /* At least one descriptor was cleaned */
- txr->busy = EM_TX_BUSY; /* note this clears HUNG */
-
- /*
- * If we have a minimum free, clear IFF_DRV_OACTIVE
- * to tell the stack that it is OK to send packets.
- * Notice that all writes of OACTIVE happen under the
- * TX lock which, with a single queue, guarantees
- * sanity.
- */
- if (txr->tx_avail >= EM_MAX_SCATTER) {
- if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
- }
-
- /* Disable hang detection if all clean */
- if (txr->tx_avail == adapter->num_tx_desc)
- txr->busy = EM_TX_IDLE;
-}
-
-/*********************************************************************
- *
- * Refresh RX descriptor mbufs from system mbuf buffer pool.
- *
- **********************************************************************/
-static void
-em_refresh_mbufs(struct rx_ring *rxr, int limit)
-{
- struct adapter *adapter = rxr->adapter;
- struct mbuf *m;
- bus_dma_segment_t segs;
- struct em_rxbuffer *rxbuf;
- int i, j, error, nsegs;
- bool cleaned = FALSE;
-
- i = j = rxr->next_to_refresh;
- /*
- ** Get one descriptor beyond
- ** our work mark to control
- ** the loop.
- */
- if (++j == adapter->num_rx_desc)
- j = 0;
-
- while (j != limit) {
- rxbuf = &rxr->rx_buffers[i];
- if (rxbuf->m_head == NULL) {
- m = m_getjcl(M_NOWAIT, MT_DATA,
- M_PKTHDR, adapter->rx_mbuf_sz);
- /*
- ** If we have a temporary resource shortage
- ** that causes a failure, just abort refresh
- ** for now, we will return to this point when
- ** reinvoked from em_rxeof.
- */
- if (m == NULL)
- goto update;
- } else
- m = rxbuf->m_head;
-
- m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz;
- m->m_flags |= M_PKTHDR;
- m->m_data = m->m_ext.ext_buf;
-
- /* Use bus_dma machinery to setup the memory mapping */
- error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map,
- m, &segs, &nsegs, BUS_DMA_NOWAIT);
- if (error != 0) {
- printf("Refresh mbufs: hdr dmamap load"
- " failure - %d\n", error);
- m_free(m);
- rxbuf->m_head = NULL;
- goto update;
- }
- rxbuf->m_head = m;
- rxbuf->paddr = segs.ds_addr;
- bus_dmamap_sync(rxr->rxtag,
- rxbuf->map, BUS_DMASYNC_PREREAD);
- em_setup_rxdesc(&rxr->rx_base[i], rxbuf);
- cleaned = TRUE;
-
- i = j; /* Next is precalulated for us */
- rxr->next_to_refresh = i;
- /* Calculate next controlling index */
- if (++j == adapter->num_rx_desc)
- j = 0;
- }
-update:
- /*
- ** Update the tail pointer only if,
- ** and as far as we have refreshed.
- */
- if (cleaned)
- E1000_WRITE_REG(&adapter->hw,
- E1000_RDT(rxr->me), rxr->next_to_refresh);
-
- return;
-}
-
-
-/*********************************************************************
- *
- * Allocate memory for rx_buffer structures. Since we use one
- * rx_buffer per received packet, the maximum number of rx_buffer's
- * that we'll need is equal to the number of receive descriptors
- * that we've allocated.
- *
- **********************************************************************/
-static int
-em_allocate_receive_buffers(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
- device_t dev = adapter->dev;
- struct em_rxbuffer *rxbuf;
- int error;
-
- rxr->rx_buffers = malloc(sizeof(struct em_rxbuffer) *
- adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
- if (rxr->rx_buffers == NULL) {
- device_printf(dev, "Unable to allocate rx_buffer memory\n");
- return (ENOMEM);
- }
-
- error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
- 1, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- MJUM9BYTES, /* maxsize */
- 1, /* nsegments */
- MJUM9BYTES, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockarg */
- &rxr->rxtag);
- if (error) {
- device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
- __func__, error);
- goto fail;
- }
-
- rxbuf = rxr->rx_buffers;
- for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
- rxbuf = &rxr->rx_buffers[i];
- error = bus_dmamap_create(rxr->rxtag, 0, &rxbuf->map);
- if (error) {
- device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
- __func__, error);
- goto fail;
- }
- }
-
- return (0);
-
-fail:
- em_free_receive_structures(adapter);
- return (error);
-}
-
-
-/*********************************************************************
- *
- * Initialize a receive ring and its buffers.
- *
- **********************************************************************/
-static int
-em_setup_receive_ring(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
- struct em_rxbuffer *rxbuf;
- bus_dma_segment_t seg[1];
- int rsize, nsegs, error = 0;
-#ifdef DEV_NETMAP
- struct netmap_slot *slot;
- struct netmap_adapter *na = netmap_getna(adapter->ifp);
-#endif
-
-
- /* Clear the ring contents */
- EM_RX_LOCK(rxr);
- rsize = roundup2(adapter->num_rx_desc *
- sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN);
- bzero((void *)rxr->rx_base, rsize);
-#ifdef DEV_NETMAP
- slot = netmap_reset(na, NR_RX, rxr->me, 0);
-#endif
-
- /*
- ** Free current RX buffer structs and their mbufs
- */
- for (int i = 0; i < adapter->num_rx_desc; i++) {
- rxbuf = &rxr->rx_buffers[i];
- if (rxbuf->m_head != NULL) {
- bus_dmamap_sync(rxr->rxtag, rxbuf->map,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(rxr->rxtag, rxbuf->map);
- m_freem(rxbuf->m_head);
- rxbuf->m_head = NULL; /* mark as freed */
- }
- }
-
- /* Now replenish the mbufs */
- for (int j = 0; j != adapter->num_rx_desc; ++j) {
- rxbuf = &rxr->rx_buffers[j];
-#ifdef DEV_NETMAP
- if (slot) {
- int si = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
- uint64_t paddr;
- void *addr;
-
- addr = PNMB(na, slot + si, &paddr);
- netmap_load_map(na, rxr->rxtag, rxbuf->map, addr);
- rxbuf->paddr = paddr;
- em_setup_rxdesc(&rxr->rx_base[j], rxbuf);
- continue;
- }
-#endif /* DEV_NETMAP */
- rxbuf->m_head = m_getjcl(M_NOWAIT, MT_DATA,
- M_PKTHDR, adapter->rx_mbuf_sz);
- if (rxbuf->m_head == NULL) {
- error = ENOBUFS;
- goto fail;
- }
- rxbuf->m_head->m_len = adapter->rx_mbuf_sz;
- rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */
- rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz;
-
- /* Get the memory mapping */
- error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
- rxbuf->map, rxbuf->m_head, seg,
- &nsegs, BUS_DMA_NOWAIT);
- if (error != 0) {
- m_freem(rxbuf->m_head);
- rxbuf->m_head = NULL;
- goto fail;
- }
- bus_dmamap_sync(rxr->rxtag,
- rxbuf->map, BUS_DMASYNC_PREREAD);
-
- rxbuf->paddr = seg[0].ds_addr;
- em_setup_rxdesc(&rxr->rx_base[j], rxbuf);
- }
- rxr->next_to_check = 0;
- rxr->next_to_refresh = 0;
- bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
-fail:
- EM_RX_UNLOCK(rxr);
- return (error);
-}
-
-/*********************************************************************
- *
- * Initialize all receive rings.
- *
- **********************************************************************/
-static int
-em_setup_receive_structures(struct adapter *adapter)
-{
- struct rx_ring *rxr = adapter->rx_rings;
- int q;
-
- for (q = 0; q < adapter->num_queues; q++, rxr++)
- if (em_setup_receive_ring(rxr))
- goto fail;
-
- return (0);
-fail:
- /*
- * Free RX buffers allocated so far, we will only handle
- * the rings that completed, the failing case will have
- * cleaned up for itself. 'q' failed, so its the terminus.
- */
- for (int i = 0; i < q; ++i) {
- rxr = &adapter->rx_rings[i];
- for (int n = 0; n < adapter->num_rx_desc; n++) {
- struct em_rxbuffer *rxbuf;
- rxbuf = &rxr->rx_buffers[n];
- if (rxbuf->m_head != NULL) {
- bus_dmamap_sync(rxr->rxtag, rxbuf->map,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(rxr->rxtag, rxbuf->map);
- m_freem(rxbuf->m_head);
- rxbuf->m_head = NULL;
- }
- }
- rxr->next_to_check = 0;
- rxr->next_to_refresh = 0;
+ tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
+ else
+ tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
+ tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
+ tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
}
- return (ENOBUFS);
-}
+ E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
+ E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
-/*********************************************************************
- *
- * Free all receive rings.
- *
- **********************************************************************/
-static void
-em_free_receive_structures(struct adapter *adapter)
-{
- struct rx_ring *rxr = adapter->rx_rings;
+ if(adapter->hw.mac.type >= e1000_82540)
+ E1000_WRITE_REG(&adapter->hw, E1000_TADV,
+ adapter->tx_abs_int_delay.value);
- for (int i = 0; i < adapter->num_queues; i++, rxr++) {
- em_free_receive_buffers(rxr);
- /* Free the ring memory as well */
- em_dma_free(adapter, &rxr->rxdma);
- EM_RX_LOCK_DESTROY(rxr);
+ if ((adapter->hw.mac.type == e1000_82571) ||
+ (adapter->hw.mac.type == e1000_82572)) {
+ tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
+ tarc |= TARC_SPEED_MODE_BIT;
+ E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
+ } else if (adapter->hw.mac.type == e1000_80003es2lan) {
+ /* errata: program both queues to unweighted RR */
+ tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
+ tarc |= 1;
+ E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
+ tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
+ tarc |= 1;
+ E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
+ } else if (adapter->hw.mac.type == e1000_82574) {
+ tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
+ tarc |= TARC_ERRATA_BIT;
+ if ( adapter->tx_num_queues > 1) {
+ tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX);
+ E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
+ E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
+ } else
+ E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
}
- free(adapter->rx_rings, M_DEVBUF);
-}
-
+ if (adapter->tx_int_delay.value > 0)
+ adapter->txd_cmd |= E1000_TXD_CMD_IDE;
-/*********************************************************************
- *
- * Free receive ring data structures
- *
- **********************************************************************/
-static void
-em_free_receive_buffers(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
- struct em_rxbuffer *rxbuf = NULL;
+ /* Program the Transmit Control Register */
+ tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
+ tctl &= ~E1000_TCTL_CT;
+ tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
+ (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
- INIT_DEBUGOUT("free_receive_buffers: begin");
+ if (adapter->hw.mac.type >= e1000_82571)
+ tctl |= E1000_TCTL_MULR;
- if (rxr->rx_buffers != NULL) {
- for (int i = 0; i < adapter->num_rx_desc; i++) {
- rxbuf = &rxr->rx_buffers[i];
- if (rxbuf->map != NULL) {
- bus_dmamap_sync(rxr->rxtag, rxbuf->map,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(rxr->rxtag, rxbuf->map);
- bus_dmamap_destroy(rxr->rxtag, rxbuf->map);
- }
- if (rxbuf->m_head != NULL) {
- m_freem(rxbuf->m_head);
- rxbuf->m_head = NULL;
- }
- }
- free(rxr->rx_buffers, M_DEVBUF);
- rxr->rx_buffers = NULL;
- rxr->next_to_check = 0;
- rxr->next_to_refresh = 0;
- }
+ /* This write will effectively turn on the transmit unit. */
+ E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
- if (rxr->rxtag != NULL) {
- bus_dma_tag_destroy(rxr->rxtag);
- rxr->rxtag = NULL;
+ if (hw->mac.type == e1000_pch_spt) {
+ u32 reg;
+ reg = E1000_READ_REG(hw, E1000_IOSFPC);
+ reg |= E1000_RCTL_RDMTS_HEX;
+ E1000_WRITE_REG(hw, E1000_IOSFPC, reg);
+ reg = E1000_READ_REG(hw, E1000_TARC(0));
+ reg |= E1000_TARC0_CB_MULTIQ_3_REQ;
+ E1000_WRITE_REG(hw, E1000_TARC(0), reg);
}
-
- return;
}
-
/*********************************************************************
*
* Enable receive unit.
@@ -4550,11 +2769,14 @@
**********************************************************************/
static void
-em_initialize_receive_unit(struct adapter *adapter)
+em_initialize_receive_unit(if_ctx_t ctx)
{
- struct rx_ring *rxr = adapter->rx_rings;
- if_t ifp = adapter->ifp;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ if_softc_ctx_t scctx = adapter->shared;
+ struct ifnet *ifp = iflib_get_ifp(ctx);
struct e1000_hw *hw = &adapter->hw;
+ struct em_rx_queue *que;
+ int i;
u32 rctl, rxcsum, rfctl;
INIT_DEBUGOUT("em_initialize_receive_units: begin");
@@ -4587,16 +2809,18 @@
if (!em_disable_crc_stripping)
rctl |= E1000_RCTL_SECRC;
- E1000_WRITE_REG(&adapter->hw, E1000_RADV,
- adapter->rx_abs_int_delay.value);
+ if (adapter->hw.mac.type >= e1000_82540) {
+ E1000_WRITE_REG(&adapter->hw, E1000_RADV,
+ adapter->rx_abs_int_delay.value);
+ /*
+ * Set the interrupt throttling rate. Value is calculated
+ * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
+ */
+ E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
+ }
E1000_WRITE_REG(&adapter->hw, E1000_RDTR,
adapter->rx_int_delay.value);
- /*
- * Set the interrupt throttling rate. Value is calculated
- * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
- */
- E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
/* Use extended rx descriptor formats */
rfctl = E1000_READ_REG(hw, E1000_RFCTL);
@@ -4615,60 +2839,37 @@
E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
- if (if_getcapenable(ifp) & IFCAP_RXCSUM) {
-#ifdef EM_MULTIQUEUE
- rxcsum |= E1000_RXCSUM_TUOFL |
- E1000_RXCSUM_IPOFL |
- E1000_RXCSUM_PCSD;
-#else
- rxcsum |= E1000_RXCSUM_TUOFL;
-#endif
+ if (if_getcapenable(ifp) & IFCAP_RXCSUM &&
+ adapter->hw.mac.type >= e1000_82543) {
+ if (adapter->tx_num_queues > 1) {
+ if (adapter->hw.mac.type >= igb_mac_min) {
+ rxcsum |= E1000_RXCSUM_PCSD;
+ if (hw->mac.type != e1000_82575)
+ rxcsum |= E1000_RXCSUM_CRCOFL;
+ } else
+ rxcsum |= E1000_RXCSUM_TUOFL |
+ E1000_RXCSUM_IPOFL |
+ E1000_RXCSUM_PCSD;
+ } else {
+ if (adapter->hw.mac.type >= igb_mac_min)
+ rxcsum |= E1000_RXCSUM_IPPCSE;
+ else
+ rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL;
+ if (adapter->hw.mac.type > e1000_82575)
+ rxcsum |= E1000_RXCSUM_CRCOFL;
+ }
} else
rxcsum &= ~E1000_RXCSUM_TUOFL;
E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
-#ifdef EM_MULTIQUEUE
-#define RSSKEYLEN 10
- if (adapter->num_queues > 1) {
- uint8_t rss_key[4 * RSSKEYLEN];
- uint32_t reta = 0;
- int i;
-
- /*
- * Configure RSS key
- */
- arc4rand(rss_key, sizeof(rss_key), 0);
- for (i = 0; i < RSSKEYLEN; ++i) {
- uint32_t rssrk = 0;
-
- rssrk = EM_RSSRK_VAL(rss_key, i);
- E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk);
- }
-
- /*
- * Configure RSS redirect table in following fashion:
- * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
- */
- for (i = 0; i < sizeof(reta); ++i) {
- uint32_t q;
-
- q = (i % adapter->num_queues) << 7;
- reta |= q << (8 * i);
- }
-
- for (i = 0; i < 32; ++i) {
- E1000_WRITE_REG(hw, E1000_RETA(i), reta);
- }
-
- E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q |
- E1000_MRQC_RSS_FIELD_IPV4_TCP |
- E1000_MRQC_RSS_FIELD_IPV4 |
- E1000_MRQC_RSS_FIELD_IPV6_TCP_EX |
- E1000_MRQC_RSS_FIELD_IPV6_EX |
- E1000_MRQC_RSS_FIELD_IPV6);
+ if (adapter->rx_num_queues > 1) {
+ if (adapter->hw.mac.type >= igb_mac_min)
+ igb_initialize_rss_mapping(adapter);
+ else
+ em_initialize_rss_mapping(adapter);
}
-#endif
+
/*
** XXX TEMPORARY WORKAROUND: on some systems with 82573
** long latencies are observed, like Lenovo X60. This
@@ -4679,28 +2880,21 @@
if (hw->mac.type == e1000_82573)
E1000_WRITE_REG(hw, E1000_RDTR, 0x20);
- for (int i = 0; i < adapter->num_queues; i++, rxr++) {
+ for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) {
+ struct rx_ring *rxr = &que->rxr;
/* Setup the Base and Length of the Rx Descriptor Ring */
- u64 bus_addr = rxr->rxdma.dma_paddr;
- u32 rdt = adapter->num_rx_desc - 1; /* default */
+ u64 bus_addr = rxr->rx_paddr;
+#if 0
+ u32 rdt = adapter->rx_num_queues -1; /* default */
+#endif
E1000_WRITE_REG(hw, E1000_RDLEN(i),
- adapter->num_rx_desc * sizeof(union e1000_rx_desc_extended));
+ scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended));
E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
/* Setup the Head and Tail Descriptor Pointers */
E1000_WRITE_REG(hw, E1000_RDH(i), 0);
-#ifdef DEV_NETMAP
- /*
- * an init() while a netmap client is active must
- * preserve the rx buffers passed to userspace.
- */
- if (if_getcapenable(ifp) & IFCAP_NETMAP) {
- struct netmap_adapter *na = netmap_getna(adapter->ifp);
- rdt -= nm_kr_rxspace(&na->rx_rings[i]);
- }
-#endif /* DEV_NETMAP */
- E1000_WRITE_REG(hw, E1000_RDT(i), rdt);
+ E1000_WRITE_REG(hw, E1000_RDT(i), 0);
}
/*
@@ -4710,6 +2904,7 @@
* Only write to RXDCTL(1) if there is a need for different
* settings.
*/
+
if (((adapter->hw.mac.type == e1000_ich9lan) ||
(adapter->hw.mac.type == e1000_pch2lan) ||
(adapter->hw.mac.type == e1000_ich10lan)) &&
@@ -4717,17 +2912,82 @@
u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3);
} else if (adapter->hw.mac.type == e1000_82574) {
- for (int i = 0; i < adapter->num_queues; i++) {
+ for (int i = 0; i < adapter->rx_num_queues; i++) {
u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
-
rxdctl |= 0x20; /* PTHRESH */
rxdctl |= 4 << 8; /* HTHRESH */
rxdctl |= 4 << 16;/* WTHRESH */
rxdctl |= 1 << 24; /* Switch to granularity */
E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
}
+ } else if (adapter->hw.mac.type >= igb_mac_min) {
+ u32 psize, srrctl = 0;
+
+ if (ifp->if_mtu > ETHERMTU) {
+ rctl |= E1000_RCTL_LPE;
+
+ /* Set maximum packet len */
+ psize = scctx->isc_max_frame_size;
+ if (psize <= 4096) {
+ srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
+ rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
+ } else if (psize > 4096) {
+ srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
+ rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
+ }
+
+ /* are we on a vlan? */
+ if (ifp->if_vlantrunk != NULL)
+ psize += VLAN_TAG_SIZE;
+ E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
+ } else {
+ rctl &= ~E1000_RCTL_LPE;
+ srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
+ rctl |= E1000_RCTL_SZ_2048;
+ }
+
+ /*
+ * If TX flow control is disabled and there's >1 queue defined,
+ * enable DROP.
+ *
+ * This drops frames rather than hanging the RX MAC for all queues.
+ */
+ if ((adapter->rx_num_queues > 1) &&
+ (adapter->fc == e1000_fc_none ||
+ adapter->fc == e1000_fc_rx_pause)) {
+ srrctl |= E1000_SRRCTL_DROP_EN;
+ }
+ /* Setup the Base and Length of the Rx Descriptor Rings */
+ for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) {
+ struct rx_ring *rxr = &que->rxr;
+ u64 bus_addr = rxr->rx_paddr;
+ u32 rxdctl;
+
+#ifdef notyet
+ /* Configure for header split? -- ignore for now */
+ rxr->hdr_split = igb_header_split;
+#else
+ srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
+#endif
+
+
+ E1000_WRITE_REG(hw, E1000_RDLEN(i),
+ scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc));
+ E1000_WRITE_REG(hw, E1000_RDBAH(i),
+ (uint32_t)(bus_addr >> 32));
+ E1000_WRITE_REG(hw, E1000_RDBAL(i),
+ (uint32_t)bus_addr);
+ E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
+ /* Enable this Queue */
+ rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
+ rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
+ rxdctl &= 0xFFF00000;
+ rxdctl |= IGB_RX_PTHRESH;
+ rxdctl |= IGB_RX_HTHRESH << 8;
+ rxdctl |= IGB_RX_WTHRESH << 16;
+ E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
+ }
}
-
if (adapter->hw.mac.type >= e1000_pch2lan) {
if (if_getmtu(ifp) > ETHERMTU)
e1000_lv_jumbo_workaround_ich8lan(hw, TRUE);
@@ -4753,323 +3013,28 @@
return;
}
-
-/*********************************************************************
- *
- * This routine executes in interrupt context. It replenishes
- * the mbufs in the descriptor and sends data which has been
- * dma'ed into host memory to upper layer.
- *
- * We loop at most count times if count is > 0, or until done if
- * count < 0.
- *
- * For polling we also now return the number of cleaned packets
- *********************************************************************/
-static bool
-em_rxeof(struct rx_ring *rxr, int count, int *done)
-{
- struct adapter *adapter = rxr->adapter;
- if_t ifp = adapter->ifp;
- struct mbuf *mp, *sendmp;
- u32 status = 0;
- u16 len;
- int i, processed, rxdone = 0;
- bool eop;
- union e1000_rx_desc_extended *cur;
-
- EM_RX_LOCK(rxr);
-
- /* Sync the ring */
- bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
- BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
-
-
-#ifdef DEV_NETMAP
- if (netmap_rx_irq(ifp, rxr->me, &processed)) {
- EM_RX_UNLOCK(rxr);
- return (FALSE);
- }
-#endif /* DEV_NETMAP */
-
- for (i = rxr->next_to_check, processed = 0; count != 0;) {
- if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
- break;
-
- cur = &rxr->rx_base[i];
- status = le32toh(cur->wb.upper.status_error);
- mp = sendmp = NULL;
-
- if ((status & E1000_RXD_STAT_DD) == 0)
- break;
-
- len = le16toh(cur->wb.upper.length);
- eop = (status & E1000_RXD_STAT_EOP) != 0;
-
- if ((status & E1000_RXDEXT_ERR_FRAME_ERR_MASK) ||
- (rxr->discard == TRUE)) {
- adapter->dropped_pkts++;
- ++rxr->rx_discarded;
- if (!eop) /* Catch subsequent segs */
- rxr->discard = TRUE;
- else
- rxr->discard = FALSE;
- em_rx_discard(rxr, i);
- goto next_desc;
- }
- bus_dmamap_unload(rxr->rxtag, rxr->rx_buffers[i].map);
-
- /* Assign correct length to the current fragment */
- mp = rxr->rx_buffers[i].m_head;
- mp->m_len = len;
-
- /* Trigger for refresh */
- rxr->rx_buffers[i].m_head = NULL;
-
- /* First segment? */
- if (rxr->fmp == NULL) {
- mp->m_pkthdr.len = len;
- rxr->fmp = rxr->lmp = mp;
- } else {
- /* Chain mbuf's together */
- mp->m_flags &= ~M_PKTHDR;
- rxr->lmp->m_next = mp;
- rxr->lmp = mp;
- rxr->fmp->m_pkthdr.len += len;
- }
-
- if (eop) {
- --count;
- sendmp = rxr->fmp;
- if_setrcvif(sendmp, ifp);
- if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
- em_receive_checksum(status, sendmp);
-#ifndef __NO_STRICT_ALIGNMENT
- if (adapter->hw.mac.max_frame_size >
- (MCLBYTES - ETHER_ALIGN) &&
- em_fixup_rx(rxr) != 0)
- goto skip;
-#endif
- if (status & E1000_RXD_STAT_VP) {
- if_setvtag(sendmp,
- le16toh(cur->wb.upper.vlan));
- sendmp->m_flags |= M_VLANTAG;
- }
-#ifndef __NO_STRICT_ALIGNMENT
-skip:
-#endif
- rxr->fmp = rxr->lmp = NULL;
- }
-next_desc:
- /* Sync the ring */
- bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
- BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
-
- /* Zero out the receive descriptors status. */
- cur->wb.upper.status_error &= htole32(~0xFF);
- ++rxdone; /* cumulative for POLL */
- ++processed;
-
- /* Advance our pointers to the next descriptor. */
- if (++i == adapter->num_rx_desc)
- i = 0;
-
- /* Send to the stack */
- if (sendmp != NULL) {
- rxr->next_to_check = i;
- EM_RX_UNLOCK(rxr);
- if_input(ifp, sendmp);
- EM_RX_LOCK(rxr);
- i = rxr->next_to_check;
- }
-
- /* Only refresh mbufs every 8 descriptors */
- if (processed == 8) {
- em_refresh_mbufs(rxr, i);
- processed = 0;
- }
- }
-
- /* Catch any remaining refresh work */
- if (e1000_rx_unrefreshed(rxr))
- em_refresh_mbufs(rxr, i);
-
- rxr->next_to_check = i;
- if (done != NULL)
- *done = rxdone;
- EM_RX_UNLOCK(rxr);
-
- return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
-}
-
-static __inline void
-em_rx_discard(struct rx_ring *rxr, int i)
-{
- struct em_rxbuffer *rbuf;
-
- rbuf = &rxr->rx_buffers[i];
- bus_dmamap_unload(rxr->rxtag, rbuf->map);
-
- /* Free any previous pieces */
- if (rxr->fmp != NULL) {
- rxr->fmp->m_flags |= M_PKTHDR;
- m_freem(rxr->fmp);
- rxr->fmp = NULL;
- rxr->lmp = NULL;
- }
- /*
- ** Free buffer and allow em_refresh_mbufs()
- ** to clean up and recharge buffer.
- */
- if (rbuf->m_head) {
- m_free(rbuf->m_head);
- rbuf->m_head = NULL;
- }
- return;
-}
-
-#ifndef __NO_STRICT_ALIGNMENT
-/*
- * When jumbo frames are enabled we should realign entire payload on
- * architecures with strict alignment. This is serious design mistake of 8254x
- * as it nullifies DMA operations. 8254x just allows RX buffer size to be
- * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
- * payload. On architecures without strict alignment restrictions 8254x still
- * performs unaligned memory access which would reduce the performance too.
- * To avoid copying over an entire frame to align, we allocate a new mbuf and
- * copy ethernet header to the new mbuf. The new mbuf is prepended into the
- * existing mbuf chain.
- *
- * Be aware, best performance of the 8254x is achived only when jumbo frame is
- * not used at all on architectures with strict alignment.
- */
-static int
-em_fixup_rx(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
- struct mbuf *m, *n;
- int error;
-
- error = 0;
- m = rxr->fmp;
- if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
- bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
- m->m_data += ETHER_HDR_LEN;
- } else {
- MGETHDR(n, M_NOWAIT, MT_DATA);
- if (n != NULL) {
- bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
- m->m_data += ETHER_HDR_LEN;
- m->m_len -= ETHER_HDR_LEN;
- n->m_len = ETHER_HDR_LEN;
- M_MOVE_PKTHDR(n, m);
- n->m_next = m;
- rxr->fmp = n;
- } else {
- adapter->dropped_pkts++;
- m_freem(rxr->fmp);
- rxr->fmp = NULL;
- error = ENOMEM;
- }
- }
-
- return (error);
-}
-#endif
-
-static void
-em_setup_rxdesc(union e1000_rx_desc_extended *rxd, const struct em_rxbuffer *rxbuf)
-{
- rxd->read.buffer_addr = htole64(rxbuf->paddr);
- /* DD bits must be cleared */
- rxd->wb.upper.status_error= 0;
-}
-
-/*********************************************************************
- *
- * Verify that the hardware indicated that the checksum is valid.
- * Inform the stack about the status of checksum so that stack
- * doesn't spend time verifying the checksum.
- *
- *********************************************************************/
-static void
-em_receive_checksum(uint32_t status, struct mbuf *mp)
-{
- mp->m_pkthdr.csum_flags = 0;
-
- /* Ignore Checksum bit is set */
- if (status & E1000_RXD_STAT_IXSM)
- return;
-
- /* If the IP checksum exists and there is no IP Checksum error */
- if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) ==
- E1000_RXD_STAT_IPCS) {
- mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
- }
-
- /* TCP or UDP checksum */
- if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) ==
- E1000_RXD_STAT_TCPCS) {
- mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
- mp->m_pkthdr.csum_data = htons(0xffff);
- }
- if (status & E1000_RXD_STAT_UDPCS) {
- mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
- mp->m_pkthdr.csum_data = htons(0xffff);
- }
-}
-
-/*
- * This routine is run via an vlan
- * config EVENT
- */
static void
-em_register_vlan(void *arg, if_t ifp, u16 vtag)
+em_if_vlan_register(if_ctx_t ctx, u16 vtag)
{
- struct adapter *adapter = if_getsoftc(ifp);
+ struct adapter *adapter = iflib_get_softc(ctx);
u32 index, bit;
- if ((void*)adapter != arg) /* Not our event */
- return;
-
- if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */
- return;
-
- EM_CORE_LOCK(adapter);
index = (vtag >> 5) & 0x7F;
bit = vtag & 0x1F;
adapter->shadow_vfta[index] |= (1 << bit);
++adapter->num_vlans;
- /* Re-init to load the changes */
- if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
- em_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
}
-/*
- * This routine is run via an vlan
- * unconfig EVENT
- */
static void
-em_unregister_vlan(void *arg, if_t ifp, u16 vtag)
+em_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
{
- struct adapter *adapter = if_getsoftc(ifp);
+ struct adapter *adapter = iflib_get_softc(ctx);
u32 index, bit;
- if (adapter != arg)
- return;
-
- if ((vtag == 0) || (vtag > 4095)) /* Invalid */
- return;
-
- EM_CORE_LOCK(adapter);
index = (vtag >> 5) & 0x7F;
bit = vtag & 0x1F;
adapter->shadow_vfta[index] &= ~(1 << bit);
--adapter->num_vlans;
- /* Re-init to load the changes */
- if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
- em_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
}
static void
@@ -5108,25 +3073,38 @@
}
static void
-em_enable_intr(struct adapter *adapter)
+em_if_enable_intr(if_ctx_t ctx)
{
+ struct adapter *adapter = iflib_get_softc(ctx);
struct e1000_hw *hw = &adapter->hw;
u32 ims_mask = IMS_ENABLE_MASK;
if (hw->mac.type == e1000_82574) {
E1000_WRITE_REG(hw, EM_EIAC, adapter->ims);
ims_mask |= adapter->ims;
- }
+ } if (adapter->intr_type == IFLIB_INTR_MSIX && hw->mac.type >= igb_mac_min) {
+ u32 mask = (adapter->que_mask | adapter->link_mask);
+
+ E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
+ E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
+ E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
+ ims_mask = E1000_IMS_LSC;
+ }
+
E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
}
static void
-em_disable_intr(struct adapter *adapter)
+em_if_disable_intr(if_ctx_t ctx)
{
+ struct adapter *adapter = iflib_get_softc(ctx);
struct e1000_hw *hw = &adapter->hw;
- if (hw->mac.type == e1000_82574)
- E1000_WRITE_REG(hw, EM_EIAC, 0);
+ if (adapter->intr_type == IFLIB_INTR_MSIX) {
+ if (hw->mac.type >= igb_mac_min)
+ E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
+ E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
+ }
E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
}
@@ -5245,15 +3223,34 @@
** later use.
*/
static void
-em_get_wakeup(device_t dev)
+em_get_wakeup(if_ctx_t ctx)
{
- struct adapter *adapter = device_get_softc(dev);
+ struct adapter *adapter = iflib_get_softc(ctx);
+ device_t dev = iflib_get_dev(ctx);
u16 eeprom_data = 0, device_id, apme_mask;
adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
apme_mask = EM_EEPROM_APME;
switch (adapter->hw.mac.type) {
+ case e1000_82542:
+ case e1000_82543:
+ break;
+ case e1000_82544:
+ e1000_read_nvm(&adapter->hw,
+ NVM_INIT_CONTROL2_REG, 1, &eeprom_data);
+ apme_mask = EM_82544_APME;
+ break;
+ case e1000_82546:
+ case e1000_82546_rev_3:
+ if (adapter->hw.bus.func == 1) {
+ e1000_read_nvm(&adapter->hw,
+ NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
+ break;
+ } else
+ e1000_read_nvm(&adapter->hw,
+ NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
+ break;
case e1000_82573:
case e1000_82583:
adapter->has_amt = TRUE;
@@ -5274,8 +3271,6 @@
case e1000_ich10lan:
case e1000_pchlan:
case e1000_pch2lan:
- case e1000_pch_lpt:
- case e1000_pch_spt:
apme_mask = E1000_WUC_APME;
adapter->has_amt = TRUE;
eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
@@ -5294,6 +3289,25 @@
*/
device_id = pci_get_device(dev);
switch (device_id) {
+ case E1000_DEV_ID_82546GB_PCIE:
+ adapter->wol = 0;
+ break;
+ case E1000_DEV_ID_82546EB_FIBER:
+ case E1000_DEV_ID_82546GB_FIBER:
+ /* Wake events only supported on port A for dual fiber
+ * regardless of eeprom setting */
+ if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
+ E1000_STATUS_FUNC_1)
+ adapter->wol = 0;
+ break;
+ case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
+ /* if quad port adapter, disable WoL on all but port A */
+ if (global_quad_port_a != 0)
+ adapter->wol = 0;
+ /* Reset for multiple quad port adapters */
+ if (++global_quad_port_a == 4)
+ global_quad_port_a = 0;
+ break;
case E1000_DEV_ID_82571EB_FIBER:
/* Wake events only supported on port A for dual fiber
* regardless of eeprom setting */
@@ -5320,11 +3334,12 @@
* Enable PCI Wake On Lan capability
*/
static void
-em_enable_wakeup(device_t dev)
+em_enable_wakeup(if_ctx_t ctx)
{
- struct adapter *adapter = device_get_softc(dev);
- if_t ifp = adapter->ifp;
- u32 pmc, ctrl, ctrl_ext, rctl, wuc;
+ struct adapter *adapter = iflib_get_softc(ctx);
+ device_t dev = iflib_get_dev(ctx);
+ if_t ifp = iflib_get_ifp(ctx);
+ u32 pmc, ctrl, ctrl_ext, rctl;
u16 status;
if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
@@ -5334,9 +3349,7 @@
ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
- wuc = E1000_READ_REG(&adapter->hw, E1000_WUC);
- wuc |= E1000_WUC_PME_EN;
- E1000_WRITE_REG(&adapter->hw, E1000_WUC, wuc);
+ E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
if ((adapter->hw.mac.type == e1000_ich8lan) ||
(adapter->hw.mac.type == e1000_pchlan) ||
@@ -5367,10 +3380,8 @@
E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
}
- if ((adapter->hw.mac.type == e1000_pchlan) ||
- (adapter->hw.mac.type == e1000_pch2lan) ||
- (adapter->hw.mac.type == e1000_pch_lpt) ||
- (adapter->hw.mac.type == e1000_pch_spt)) {
+ if ((adapter->hw.mac.type == e1000_pchlan) ||
+ (adapter->hw.mac.type == e1000_pch2lan)) {
if (em_enable_phy_wakeup(adapter))
return;
} else {
@@ -5466,11 +3477,10 @@
}
static void
-em_led_func(void *arg, int onoff)
+em_if_led_func(if_ctx_t ctx, int onoff)
{
- struct adapter *adapter = arg;
+ struct adapter *adapter = iflib_get_softc(ctx);
- EM_CORE_LOCK(adapter);
if (onoff) {
e1000_setup_led(&adapter->hw);
e1000_led_on(&adapter->hw);
@@ -5478,7 +3488,6 @@
e1000_led_off(&adapter->hw);
e1000_cleanup_led(&adapter->hw);
}
- EM_CORE_UNLOCK(adapter);
}
/*
@@ -5609,11 +3618,10 @@
}
static uint64_t
-em_get_counter(if_t ifp, ift_counter cnt)
+em_if_get_counter(if_ctx_t ctx, ift_counter cnt)
{
- struct adapter *adapter;
-
- adapter = if_getsoftc(ifp);
+ struct adapter *adapter = iflib_get_softc(ctx);
+ struct ifnet *ifp = iflib_get_ifp(ctx);
switch (cnt) {
case IFCOUNTER_COLLISIONS:
@@ -5649,11 +3657,10 @@
static void
em_add_hw_stats(struct adapter *adapter)
{
- device_t dev = adapter->dev;
-
- struct tx_ring *txr = adapter->tx_rings;
- struct rx_ring *rxr = adapter->rx_rings;
-
+ device_t dev = iflib_get_dev(adapter->ctx);
+ struct em_tx_queue *tx_que = adapter->tx_queues;
+ struct em_rx_queue *rx_que = adapter->rx_queues;
+
struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
struct sysctl_oid *tree = device_get_sysctl_tree(dev);
struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
@@ -5700,7 +3707,8 @@
CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
"Flow Control Low Watermark");
- for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
+ for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) {
+ struct tx_ring *txr = &tx_que->txr;
snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i);
queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
CTLFLAG_RD, NULL, "TX Queue Name");
@@ -5722,8 +3730,11 @@
SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail",
CTLFLAG_RD, &txr->no_desc_avail,
"Queue No Descriptor Available");
+ }
- snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", i);
+ for (int j = 0; j < adapter->rx_num_queues; j++, rx_que++) {
+ struct rx_ring *rxr = &rx_que->rxr;
+ snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j);
queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
CTLFLAG_RD, NULL, "RX Queue Name");
queue_list = SYSCTL_CHILDREN(queue_node);
@@ -6010,7 +4021,6 @@
adapter = info->adapter;
- EM_CORE_LOCK(adapter);
regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
regval = (regval & ~0xffff) | (ticks & 0xffff);
/* Handle a few special cases. */
@@ -6027,7 +4037,6 @@
break;
}
E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
- EM_CORE_UNLOCK(adapter);
return (0);
}
@@ -6112,10 +4121,9 @@
error = sysctl_handle_int(oidp, &value, 0, req);
if (error || req->newptr == NULL)
return (error);
- EM_CORE_LOCK(adapter);
adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0);
- em_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
+ em_if_init(adapter->ctx);
+
return (0);
}
@@ -6148,8 +4156,8 @@
em_print_debug_info(struct adapter *adapter)
{
device_t dev = adapter->dev;
- struct tx_ring *txr = adapter->tx_rings;
- struct rx_ring *rxr = adapter->rx_rings;
+ struct tx_ring *txr = &adapter->tx_queues->txr;
+ struct rx_ring *rxr = &adapter->rx_queues->rxr;
if (if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING)
printf("Interface is RUNNING ");
@@ -6161,38 +4169,33 @@
else
printf("and ACTIVE\n");
- for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
+ for (int i = 0; i < adapter->tx_num_queues; i++, txr++) {
device_printf(dev, "TX Queue %d ------\n", i);
device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
- device_printf(dev, "Tx Queue Status = %d\n", txr->busy);
- device_printf(dev, "TX descriptors avail = %d\n",
- txr->tx_avail);
- device_printf(dev, "Tx Descriptors avail failure = %ld\n",
- txr->no_desc_avail);
- device_printf(dev, "RX Queue %d ------\n", i);
+
+ }
+ for (int j=0; j < adapter->rx_num_queues; j++, rxr++) {
+ device_printf(dev, "RX Queue %d ------\n", j);
device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
- E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
- E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
- device_printf(dev, "RX discarded packets = %ld\n",
- rxr->rx_discarded);
- device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check);
- device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh);
+ E1000_READ_REG(&adapter->hw, E1000_RDH(j)),
+ E1000_READ_REG(&adapter->hw, E1000_RDT(j)));
}
}
-#ifdef EM_MULTIQUEUE
+
/*
* 82574 only:
* Write a new value to the EEPROM increasing the number of MSIX
* vectors from 3 to 5, for proper multiqueue support.
*/
static void
-em_enable_vectors_82574(struct adapter *adapter)
+em_enable_vectors_82574(if_ctx_t ctx)
{
+ struct adapter *adapter = iflib_get_softc(ctx);
struct e1000_hw *hw = &adapter->hw;
- device_t dev = adapter->dev;
+ device_t dev = iflib_get_dev(ctx);
u16 edata;
e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata);
@@ -6207,7 +4210,7 @@
device_printf(dev, "Writing to eeprom: done\n");
}
}
-#endif
+
#ifdef DDB
DB_COMMAND(em_reset_dev, em_ddb_reset_dev)
@@ -6223,9 +4226,7 @@
dev = devclass_get_device(dc, index);
if (device_get_driver(dev) == &em_driver) {
struct adapter *adapter = device_get_softc(dev);
- EM_CORE_LOCK(adapter);
- em_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
+ em_if_init(adapter->ctx);
}
}
}
Index: sys/dev/e1000/if_igb.h
===================================================================
--- sys/dev/e1000/if_igb.h
+++ /dev/null
@@ -1,634 +0,0 @@
-/******************************************************************************
-
- Copyright (c) 2001-2015, Intel Corporation
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
-
-******************************************************************************/
-/*$FreeBSD$*/
-
-#ifndef _IF_IGB_H_
-#define _IF_IGB_H_
-
-#ifdef ALTQ
-#define IGB_LEGACY_TX
-#endif
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#ifndef IGB_LEGACY_TX
-#include <sys/buf_ring.h>
-#endif
-#include <sys/bus.h>
-#include <sys/endian.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/malloc.h>
-#include <sys/mbuf.h>
-#include <sys/module.h>
-#include <sys/rman.h>
-#include <sys/socket.h>
-#include <sys/sockio.h>
-#include <sys/sysctl.h>
-#include <sys/taskqueue.h>
-#include <sys/eventhandler.h>
-#include <sys/pcpu.h>
-#include <sys/smp.h>
-#include <machine/smp.h>
-#include <machine/bus.h>
-#include <machine/resource.h>
-
-#include <net/bpf.h>
-#include <net/ethernet.h>
-#include <net/if.h>
-#include <net/if_var.h>
-#include <net/if_arp.h>
-#include <net/if_dl.h>
-#include <net/if_media.h>
-#ifdef RSS
-#include <net/rss_config.h>
-#include <netinet/in_rss.h>
-#endif
-
-#include <net/if_types.h>
-#include <net/if_vlan_var.h>
-
-#include <netinet/in_systm.h>
-#include <netinet/in.h>
-#include <netinet/if_ether.h>
-#include <netinet/ip.h>
-#include <netinet/ip6.h>
-#include <netinet/tcp.h>
-#include <netinet/tcp_lro.h>
-#include <netinet/udp.h>
-
-#include <machine/in_cksum.h>
-#include <dev/led/led.h>
-#include <dev/pci/pcivar.h>
-#include <dev/pci/pcireg.h>
-
-#include "e1000_api.h"
-#include "e1000_82575.h"
-
-/* Tunables */
-/*
- * IGB_TXD: Maximum number of Transmit Descriptors
- *
- * This value is the number of transmit descriptors allocated by the driver.
- * Increasing this value allows the driver to queue more transmits. Each
- * descriptor is 16 bytes.
- * Since TDLEN should be multiple of 128bytes, the number of transmit
- * desscriptors should meet the following condition.
- * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0
- */
-#define IGB_MIN_TXD 256
-#define IGB_DEFAULT_TXD 1024
-#define IGB_MAX_TXD 4096
-
-/*
- * IGB_RXD: Maximum number of Receive Descriptors
- *
- * This value is the number of receive descriptors allocated by the driver.
- * Increasing this value allows the driver to buffer more incoming packets.
- * Each descriptor is 16 bytes. A receive buffer is also allocated for each
- * descriptor. The maximum MTU size is 16110.
- * Since TDLEN should be multiple of 128bytes, the number of transmit
- * desscriptors should meet the following condition.
- * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0
- */
-#define IGB_MIN_RXD 256
-#define IGB_DEFAULT_RXD 1024
-#define IGB_MAX_RXD 4096
-
-/*
- * IGB_TIDV - Transmit Interrupt Delay Value
- * Valid Range: 0-65535 (0=off)
- * Default Value: 64
- * This value delays the generation of transmit interrupts in units of
- * 1.024 microseconds. Transmit interrupt reduction can improve CPU
- * efficiency if properly tuned for specific network traffic. If the
- * system is reporting dropped transmits, this value may be set too high
- * causing the driver to run out of available transmit descriptors.
- */
-#define IGB_TIDV 64
-
-/*
- * IGB_TADV - Transmit Absolute Interrupt Delay Value
- * Valid Range: 0-65535 (0=off)
- * Default Value: 64
- * This value, in units of 1.024 microseconds, limits the delay in which a
- * transmit interrupt is generated. Useful only if IGB_TIDV is non-zero,
- * this value ensures that an interrupt is generated after the initial
- * packet is sent on the wire within the set amount of time. Proper tuning,
- * along with IGB_TIDV, may improve traffic throughput in specific
- * network conditions.
- */
-#define IGB_TADV 64
-
-/*
- * IGB_RDTR - Receive Interrupt Delay Timer (Packet Timer)
- * Valid Range: 0-65535 (0=off)
- * Default Value: 0
- * This value delays the generation of receive interrupts in units of 1.024
- * microseconds. Receive interrupt reduction can improve CPU efficiency if
- * properly tuned for specific network traffic. Increasing this value adds
- * extra latency to frame reception and can end up decreasing the throughput
- * of TCP traffic. If the system is reporting dropped receives, this value
- * may be set too high, causing the driver to run out of available receive
- * descriptors.
- *
- * CAUTION: When setting IGB_RDTR to a value other than 0, adapters
- * may hang (stop transmitting) under certain network conditions.
- * If this occurs a WATCHDOG message is logged in the system
- * event log. In addition, the controller is automatically reset,
- * restoring the network connection. To eliminate the potential
- * for the hang ensure that IGB_RDTR is set to 0.
- */
-#define IGB_RDTR 0
-
-/*
- * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544)
- * Valid Range: 0-65535 (0=off)
- * Default Value: 64
- * This value, in units of 1.024 microseconds, limits the delay in which a
- * receive interrupt is generated. Useful only if IGB_RDTR is non-zero,
- * this value ensures that an interrupt is generated after the initial
- * packet is received within the set amount of time. Proper tuning,
- * along with IGB_RDTR, may improve traffic throughput in specific network
- * conditions.
- */
-#define IGB_RADV 64
-
-/*
- * This parameter controls the duration of transmit watchdog timer.
- */
-#define IGB_WATCHDOG (10 * hz)
-
-/*
- * This parameter controls when the driver calls the routine to reclaim
- * transmit descriptors. Cleaning earlier seems a win.
- */
-#define IGB_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 2)
-
-/*
- * This parameter controls whether or not autonegotation is enabled.
- * 0 - Disable autonegotiation
- * 1 - Enable autonegotiation
- */
-#define DO_AUTO_NEG 1
-
-/*
- * This parameter control whether or not the driver will wait for
- * autonegotiation to complete.
- * 1 - Wait for autonegotiation to complete
- * 0 - Don't wait for autonegotiation to complete
- */
-#define WAIT_FOR_AUTO_NEG_DEFAULT 0
-
-/* Tunables -- End */
-
-#define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \
- ADVERTISE_100_HALF | ADVERTISE_100_FULL | \
- ADVERTISE_1000_FULL)
-
-#define AUTO_ALL_MODES 0
-
-/* PHY master/slave setting */
-#define IGB_MASTER_SLAVE e1000_ms_hw_default
-
-/* Support AutoMediaDetect for Marvell M88 PHY in i354 */
-#define IGB_MEDIA_RESET (1 << 0)
-
-/*
- * Micellaneous constants
- */
-#define IGB_INTEL_VENDOR_ID 0x8086
-
-#define IGB_JUMBO_PBA 0x00000028
-#define IGB_DEFAULT_PBA 0x00000030
-#define IGB_SMARTSPEED_DOWNSHIFT 3
-#define IGB_SMARTSPEED_MAX 15
-#define IGB_MAX_LOOP 10
-
-#define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \
- ((hw->mac.type <= e1000_82576) ? 16 : 8))
-#define IGB_RX_HTHRESH 8
-#define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \
- adapter->msix_mem) ? 1 : 4)
-
-#define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8)
-#define IGB_TX_HTHRESH 1
-#define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \
- adapter->msix_mem) ? 1 : 16)
-
-#define MAX_NUM_MULTICAST_ADDRESSES 128
-#define PCI_ANY_ID (~0U)
-#define ETHER_ALIGN 2
-#define IGB_TX_BUFFER_SIZE ((uint32_t) 1514)
-#define IGB_FC_PAUSE_TIME 0x0680
-#define IGB_EEPROM_APME 0x400;
-/* Queue minimum free for use */
-#define IGB_QUEUE_THRESHOLD (adapter->num_tx_desc / 8)
-
-/*
- * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be
- * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will
- * also optimize cache line size effect. H/W supports up to cache line size 128.
- */
-#define IGB_DBA_ALIGN 128
-
-#define SPEED_MODE_BIT (1<<21) /* On PCI-E MACs only */
-
-/* PCI Config defines */
-#define IGB_MSIX_BAR 3
-
-/* Defines for printing debug information */
-#define DEBUG_INIT 0
-#define DEBUG_IOCTL 0
-#define DEBUG_HW 0
-
-#define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n")
-#define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A)
-#define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B)
-#define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n")
-#define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A)
-#define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B)
-#define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n")
-#define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A)
-#define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B)
-
-#define IGB_MAX_SCATTER 40
-#define IGB_VFTA_SIZE 128
-#define IGB_BR_SIZE 4096 /* ring buf size */
-#define IGB_TSO_SIZE (65535 + sizeof(struct ether_vlan_header))
-#define IGB_TSO_SEG_SIZE 4096 /* Max dma segment size */
-#define IGB_TXPBSIZE 20408
-#define IGB_HDR_BUF 128
-#define IGB_PKTTYPE_MASK 0x0000FFF0
-#define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */
-#define ETH_ZLEN 60
-#define ETH_ADDR_LEN 6
-
-/* Offload bits in mbuf flag */
-#if __FreeBSD_version >= 1000000
-#define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_IP_TCP|CSUM_IP_UDP|CSUM_IP_SCTP)
-#define CSUM_OFFLOAD_IPV6 (CSUM_IP6_TCP|CSUM_IP6_UDP|CSUM_IP6_SCTP)
-#define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6)
-#elif __FreeBSD_version >= 800000
-#define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP)
-#else
-#define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP)
-#endif
-
-/* Define the starting Interrupt rate per Queue */
-#define IGB_INTS_PER_SEC 8000
-#define IGB_DEFAULT_ITR ((1000000/IGB_INTS_PER_SEC) << 2)
-
-#define IGB_LINK_ITR 2000
-#define I210_LINK_DELAY 1000
-
-/* Precision Time Sync (IEEE 1588) defines */
-#define ETHERTYPE_IEEE1588 0x88F7
-#define PICOSECS_PER_TICK 20833
-#define TSYNC_PORT 319 /* UDP port for the protocol */
-
-/*
- * Bus dma allocation structure used by
- * e1000_dma_malloc and e1000_dma_free.
- */
-struct igb_dma_alloc {
- bus_addr_t dma_paddr;
- caddr_t dma_vaddr;
- bus_dma_tag_t dma_tag;
- bus_dmamap_t dma_map;
- bus_dma_segment_t dma_seg;
- int dma_nseg;
-};
-
-
-/*
-** Driver queue struct: this is the interrupt container
-** for the associated tx and rx ring.
-*/
-struct igb_queue {
- struct adapter *adapter;
- u32 msix; /* This queue's MSIX vector */
- u32 eims; /* This queue's EIMS bit */
- u32 eitr_setting;
- struct resource *res;
- void *tag;
- struct tx_ring *txr;
- struct rx_ring *rxr;
- struct task que_task;
- struct taskqueue *tq;
- u64 irqs;
-};
-
-/*
- * The transmit ring, one per queue
- */
-struct tx_ring {
- struct adapter *adapter;
- struct mtx tx_mtx;
- u32 me;
- int watchdog_time;
- union e1000_adv_tx_desc *tx_base;
- struct igb_tx_buf *tx_buffers;
- struct igb_dma_alloc txdma;
- volatile u16 tx_avail;
- u16 next_avail_desc;
- u16 next_to_clean;
- u16 num_desc;
- enum {
- IGB_QUEUE_IDLE = 1,
- IGB_QUEUE_WORKING = 2,
- IGB_QUEUE_HUNG = 4,
- IGB_QUEUE_DEPLETED = 8,
- } queue_status;
- u32 txd_cmd;
- bus_dma_tag_t txtag;
- char mtx_name[16];
-#ifndef IGB_LEGACY_TX
- struct buf_ring *br;
- struct task txq_task;
-#endif
- u32 bytes; /* used for AIM */
- u32 packets;
- /* Soft Stats */
- unsigned long tso_tx;
- unsigned long no_tx_map_avail;
- unsigned long no_tx_dma_setup;
- u64 no_desc_avail;
- u64 total_packets;
-};
-
-/*
- * Receive ring: one per queue
- */
-struct rx_ring {
- struct adapter *adapter;
- u32 me;
- struct igb_dma_alloc rxdma;
- union e1000_adv_rx_desc *rx_base;
- struct lro_ctrl lro;
- bool lro_enabled;
- bool hdr_split;
- struct mtx rx_mtx;
- char mtx_name[16];
- u32 next_to_refresh;
- u32 next_to_check;
- struct igb_rx_buf *rx_buffers;
- bus_dma_tag_t htag; /* dma tag for rx head */
- bus_dma_tag_t ptag; /* dma tag for rx packet */
- /*
- * First/last mbuf pointers, for
- * collecting multisegment RX packets.
- */
- struct mbuf *fmp;
- struct mbuf *lmp;
-
- u32 bytes;
- u32 packets;
- int rdt;
- int rdh;
-
- /* Soft stats */
- u64 rx_split_packets;
- u64 rx_discarded;
- u64 rx_packets;
- u64 rx_bytes;
-};
-
-struct adapter {
- struct ifnet *ifp;
- struct e1000_hw hw;
-
- struct e1000_osdep osdep;
- device_t dev;
- struct cdev *led_dev;
-
- struct resource *pci_mem;
- struct resource *msix_mem;
- int memrid;
-
- /*
- * Interrupt resources: this set is
- * either used for legacy, or for Link
- * when doing MSIX
- */
- void *tag;
- struct resource *res;
-
- struct ifmedia media;
- struct callout timer;
- int msix;
- int if_flags;
- int pause_frames;
-
- struct mtx core_mtx;
-
- eventhandler_tag vlan_attach;
- eventhandler_tag vlan_detach;
-
- u16 num_vlans;
- u16 num_queues;
-
- /*
- ** Shadow VFTA table, this is needed because
- ** the real vlan filter table gets cleared during
- ** a soft reset and the driver needs to be able
- ** to repopulate it.
- */
- u32 shadow_vfta[IGB_VFTA_SIZE];
-
- /* Info about the interface */
- u32 optics;
- u32 fc; /* local flow ctrl setting */
- int advertise; /* link speeds */
- bool link_active;
- u16 max_frame_size;
- u16 num_segs;
- u16 link_speed;
- bool link_up;
- u32 linkvec;
- u16 link_duplex;
- u32 dmac;
- int link_mask;
-
- /* Flags */
- u32 flags;
-
- /* Mbuf cluster size */
- u32 rx_mbuf_sz;
-
- /* Support for pluggable optics */
- bool sfp_probe;
- struct task link_task; /* Link tasklet */
- struct task mod_task; /* SFP tasklet */
- struct task msf_task; /* Multispeed Fiber */
- struct taskqueue *tq;
-
- /*
- ** Queues:
- ** This is the irq holder, it has
- ** and RX/TX pair or rings associated
- ** with it.
- */
- struct igb_queue *queues;
-
- /*
- * Transmit rings:
- * Allocated at run time, an array of rings.
- */
- struct tx_ring *tx_rings;
- u32 num_tx_desc;
-
- /*
- * Receive rings:
- * Allocated at run time, an array of rings.
- */
- struct rx_ring *rx_rings;
- u64 que_mask;
- u32 num_rx_desc;
-
- /* Multicast array memory */
- u8 *mta;
-
- /* Misc stats maintained by the driver */
- unsigned long device_control;
- unsigned long dropped_pkts;
- unsigned long eint_mask;
- unsigned long int_mask;
- unsigned long link_irq;
- unsigned long mbuf_defrag_failed;
- unsigned long no_tx_dma_setup;
- unsigned long packet_buf_alloc_rx;
- unsigned long packet_buf_alloc_tx;
- unsigned long rx_control;
- unsigned long rx_overruns;
- unsigned long watchdog_events;
-
- /* Used in pf and vf */
- void *stats;
-
- int enable_aim;
- int has_manage;
- int wol;
- int rx_process_limit;
- int tx_process_limit;
- u16 vf_ifp; /* a VF interface */
- bool in_detach; /* Used only in igb_ioctl */
-
-};
-
-/* ******************************************************************************
- * vendor_info_array
- *
- * This array contains the list of Subvendor/Subdevice IDs on which the driver
- * should load.
- *
- * ******************************************************************************/
-typedef struct _igb_vendor_info_t {
- unsigned int vendor_id;
- unsigned int device_id;
- unsigned int subvendor_id;
- unsigned int subdevice_id;
- unsigned int index;
-} igb_vendor_info_t;
-
-struct igb_tx_buf {
- union e1000_adv_tx_desc *eop;
- struct mbuf *m_head;
- bus_dmamap_t map;
-};
-
-struct igb_rx_buf {
- struct mbuf *m_head;
- struct mbuf *m_pack;
- bus_dmamap_t hmap; /* bus_dma map for header */
- bus_dmamap_t pmap; /* bus_dma map for packet */
-};
-
-/*
-** Find the number of unrefreshed RX descriptors
-*/
-static inline u16
-igb_rx_unrefreshed(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
-
- if (rxr->next_to_check > rxr->next_to_refresh)
- return (rxr->next_to_check - rxr->next_to_refresh - 1);
- else
- return ((adapter->num_rx_desc + rxr->next_to_check) -
- rxr->next_to_refresh - 1);
-}
-
-#define IGB_CORE_LOCK_INIT(_sc, _name) \
- mtx_init(&(_sc)->core_mtx, _name, "IGB Core Lock", MTX_DEF)
-#define IGB_CORE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->core_mtx)
-#define IGB_CORE_LOCK(_sc) mtx_lock(&(_sc)->core_mtx)
-#define IGB_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->core_mtx)
-#define IGB_CORE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->core_mtx, MA_OWNED)
-
-#define IGB_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->tx_mtx)
-#define IGB_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_mtx)
-#define IGB_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_mtx)
-#define IGB_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->tx_mtx)
-#define IGB_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_mtx, MA_OWNED)
-
-#define IGB_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_mtx)
-#define IGB_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_mtx)
-#define IGB_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_mtx)
-#define IGB_RX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->rx_mtx, MA_OWNED)
-
-#define UPDATE_VF_REG(reg, last, cur) \
-{ \
- u32 new = E1000_READ_REG(hw, reg); \
- if (new < last) \
- cur += 0x100000000LL; \
- last = new; \
- cur &= 0xFFFFFFFF00000000LL; \
- cur |= new; \
-}
-
-#if __FreeBSD_version >= 800000 && __FreeBSD_version < 800504
-static __inline int
-drbr_needs_enqueue(struct ifnet *ifp, struct buf_ring *br)
-{
-#ifdef ALTQ
- if (ALTQ_IS_ENABLED(&ifp->if_snd))
- return (1);
-#endif
- return (!buf_ring_empty(br));
-}
-#endif
-
-#endif /* _IF_IGB_H_ */
-
-
Index: sys/dev/e1000/if_igb.c
===================================================================
--- sys/dev/e1000/if_igb.c
+++ /dev/null
@@ -1,6450 +0,0 @@
-/******************************************************************************
-
- Copyright (c) 2001-2015, Intel Corporation
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
-
-******************************************************************************/
-/*$FreeBSD$*/
-
-
-#include "opt_inet.h"
-#include "opt_inet6.h"
-#include "opt_rss.h"
-
-#ifdef HAVE_KERNEL_OPTION_HEADERS
-#include "opt_device_polling.h"
-#include "opt_altq.h"
-#endif
-
-#include "if_igb.h"
-
-/*********************************************************************
- * Driver version:
- *********************************************************************/
-char igb_driver_version[] = "2.5.3-k";
-
-
-/*********************************************************************
- * PCI Device ID Table
- *
- * Used by probe to select devices to load on
- * Last field stores an index into e1000_strings
- * Last entry must be all 0s
- *
- * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
- *********************************************************************/
-
-static igb_vendor_info_t igb_vendor_info_array[] =
-{
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0},
- {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0},
- /* required last entry */
- {0, 0, 0, 0, 0}
-};
-
-/*********************************************************************
- * Table of branding strings for all supported NICs.
- *********************************************************************/
-
-static char *igb_strings[] = {
- "Intel(R) PRO/1000 Network Connection"
-};
-
-/*********************************************************************
- * Function prototypes
- *********************************************************************/
-static int igb_probe(device_t);
-static int igb_attach(device_t);
-static int igb_detach(device_t);
-static int igb_shutdown(device_t);
-static int igb_suspend(device_t);
-static int igb_resume(device_t);
-#ifndef IGB_LEGACY_TX
-static int igb_mq_start(struct ifnet *, struct mbuf *);
-static int igb_mq_start_locked(struct ifnet *, struct tx_ring *);
-static void igb_qflush(struct ifnet *);
-static void igb_deferred_mq_start(void *, int);
-#else
-static void igb_start(struct ifnet *);
-static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
-#endif
-static int igb_ioctl(struct ifnet *, u_long, caddr_t);
-static uint64_t igb_get_counter(if_t, ift_counter);
-static void igb_init(void *);
-static void igb_init_locked(struct adapter *);
-static void igb_stop(void *);
-static void igb_media_status(struct ifnet *, struct ifmediareq *);
-static int igb_media_change(struct ifnet *);
-static void igb_identify_hardware(struct adapter *);
-static int igb_allocate_pci_resources(struct adapter *);
-static int igb_allocate_msix(struct adapter *);
-static int igb_allocate_legacy(struct adapter *);
-static int igb_setup_msix(struct adapter *);
-static void igb_free_pci_resources(struct adapter *);
-static void igb_local_timer(void *);
-static void igb_reset(struct adapter *);
-static int igb_setup_interface(device_t, struct adapter *);
-static int igb_allocate_queues(struct adapter *);
-static void igb_configure_queues(struct adapter *);
-
-static int igb_allocate_transmit_buffers(struct tx_ring *);
-static void igb_setup_transmit_structures(struct adapter *);
-static void igb_setup_transmit_ring(struct tx_ring *);
-static void igb_initialize_transmit_units(struct adapter *);
-static void igb_free_transmit_structures(struct adapter *);
-static void igb_free_transmit_buffers(struct tx_ring *);
-
-static int igb_allocate_receive_buffers(struct rx_ring *);
-static int igb_setup_receive_structures(struct adapter *);
-static int igb_setup_receive_ring(struct rx_ring *);
-static void igb_initialize_receive_units(struct adapter *);
-static void igb_free_receive_structures(struct adapter *);
-static void igb_free_receive_buffers(struct rx_ring *);
-static void igb_free_receive_ring(struct rx_ring *);
-
-static void igb_enable_intr(struct adapter *);
-static void igb_disable_intr(struct adapter *);
-static void igb_update_stats_counters(struct adapter *);
-static bool igb_txeof(struct tx_ring *);
-
-static __inline void igb_rx_discard(struct rx_ring *, int);
-static __inline void igb_rx_input(struct rx_ring *,
- struct ifnet *, struct mbuf *, u32);
-
-static bool igb_rxeof(struct igb_queue *, int, int *);
-static void igb_rx_checksum(u32, struct mbuf *, u32);
-static int igb_tx_ctx_setup(struct tx_ring *,
- struct mbuf *, u32 *, u32 *);
-static int igb_tso_setup(struct tx_ring *,
- struct mbuf *, u32 *, u32 *);
-static void igb_set_promisc(struct adapter *);
-static void igb_disable_promisc(struct adapter *);
-static void igb_set_multi(struct adapter *);
-static void igb_update_link_status(struct adapter *);
-static void igb_refresh_mbufs(struct rx_ring *, int);
-
-static void igb_register_vlan(void *, struct ifnet *, u16);
-static void igb_unregister_vlan(void *, struct ifnet *, u16);
-static void igb_setup_vlan_hw_support(struct adapter *);
-
-static int igb_xmit(struct tx_ring *, struct mbuf **);
-static int igb_dma_malloc(struct adapter *, bus_size_t,
- struct igb_dma_alloc *, int);
-static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
-static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
-static void igb_print_nvm_info(struct adapter *);
-static int igb_is_valid_ether_addr(u8 *);
-static void igb_add_hw_stats(struct adapter *);
-
-static void igb_vf_init_stats(struct adapter *);
-static void igb_update_vf_stats_counters(struct adapter *);
-
-/* Management and WOL Support */
-static void igb_init_manageability(struct adapter *);
-static void igb_release_manageability(struct adapter *);
-static void igb_get_hw_control(struct adapter *);
-static void igb_release_hw_control(struct adapter *);
-static void igb_enable_wakeup(device_t);
-static void igb_led_func(void *, int);
-
-static int igb_irq_fast(void *);
-static void igb_msix_que(void *);
-static void igb_msix_link(void *);
-static void igb_handle_que(void *context, int pending);
-static void igb_handle_link(void *context, int pending);
-static void igb_handle_link_locked(struct adapter *);
-
-static void igb_set_sysctl_value(struct adapter *, const char *,
- const char *, int *, int);
-static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
-static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
-static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
-
-#ifdef DEVICE_POLLING
-static poll_handler_t igb_poll;
-#endif /* POLLING */
-
-/*********************************************************************
- * FreeBSD Device Interface Entry Points
- *********************************************************************/
-
-static device_method_t igb_methods[] = {
- /* Device interface */
- DEVMETHOD(device_probe, igb_probe),
- DEVMETHOD(device_attach, igb_attach),
- DEVMETHOD(device_detach, igb_detach),
- DEVMETHOD(device_shutdown, igb_shutdown),
- DEVMETHOD(device_suspend, igb_suspend),
- DEVMETHOD(device_resume, igb_resume),
- DEVMETHOD_END
-};
-
-static driver_t igb_driver = {
- "igb", igb_methods, sizeof(struct adapter),
-};
-
-static devclass_t igb_devclass;
-DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
-MODULE_DEPEND(igb, pci, 1, 1, 1);
-MODULE_DEPEND(igb, ether, 1, 1, 1);
-#ifdef DEV_NETMAP
-MODULE_DEPEND(igb, netmap, 1, 1, 1);
-#endif /* DEV_NETMAP */
-
-/*********************************************************************
- * Tunable default values.
- *********************************************************************/
-
-static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
-
-/* Descriptor defaults */
-static int igb_rxd = IGB_DEFAULT_RXD;
-static int igb_txd = IGB_DEFAULT_TXD;
-SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
- "Number of receive descriptors per queue");
-SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
- "Number of transmit descriptors per queue");
-
-/*
-** AIM: Adaptive Interrupt Moderation
-** which means that the interrupt rate
-** is varied over time based on the
-** traffic for that interrupt vector
-*/
-static int igb_enable_aim = TRUE;
-SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igb_enable_aim, 0,
- "Enable adaptive interrupt moderation");
-
-/*
- * MSIX should be the default for best performance,
- * but this allows it to be forced off for testing.
- */
-static int igb_enable_msix = 1;
-SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
- "Enable MSI-X interrupts");
-
-/*
-** Tuneable Interrupt rate
-*/
-static int igb_max_interrupt_rate = 8000;
-SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
- &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
-
-#ifndef IGB_LEGACY_TX
-/*
-** Tuneable number of buffers in the buf-ring (drbr_xxx)
-*/
-static int igb_buf_ring_size = IGB_BR_SIZE;
-SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
- &igb_buf_ring_size, 0, "Size of the bufring");
-#endif
-
-/*
-** Header split causes the packet header to
-** be dma'd to a separate mbuf from the payload.
-** this can have memory alignment benefits. But
-** another plus is that small packets often fit
-** into the header and thus use no cluster. Its
-** a very workload dependent type feature.
-*/
-static int igb_header_split = FALSE;
-SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
- "Enable receive mbuf header split");
-
-/*
-** This will autoconfigure based on the
-** number of CPUs and max supported
-** MSIX messages if left at 0.
-*/
-static int igb_num_queues = 0;
-SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
- "Number of queues to configure, 0 indicates autoconfigure");
-
-/*
-** Global variable to store last used CPU when binding queues
-** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
-** queue is bound to a cpu.
-*/
-static int igb_last_bind_cpu = -1;
-
-/* How many packets rxeof tries to clean at a time */
-static int igb_rx_process_limit = 100;
-SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
- &igb_rx_process_limit, 0,
- "Maximum number of received packets to process at a time, -1 means unlimited");
-
-/* How many packets txeof tries to clean at a time */
-static int igb_tx_process_limit = -1;
-SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
- &igb_tx_process_limit, 0,
- "Maximum number of sent packets to process at a time, -1 means unlimited");
-
-#ifdef DEV_NETMAP /* see ixgbe.c for details */
-#include <dev/netmap/if_igb_netmap.h>
-#endif /* DEV_NETMAP */
-/*********************************************************************
- * Device identification routine
- *
- * igb_probe determines if the driver should be loaded on
- * adapter based on PCI vendor/device id of the adapter.
- *
- * return BUS_PROBE_DEFAULT on success, positive on failure
- *********************************************************************/
-
-static int
-igb_probe(device_t dev)
-{
- char adapter_name[256];
- uint16_t pci_vendor_id = 0;
- uint16_t pci_device_id = 0;
- uint16_t pci_subvendor_id = 0;
- uint16_t pci_subdevice_id = 0;
- igb_vendor_info_t *ent;
-
- INIT_DEBUGOUT("igb_probe: begin");
-
- pci_vendor_id = pci_get_vendor(dev);
- if (pci_vendor_id != IGB_INTEL_VENDOR_ID)
- return (ENXIO);
-
- pci_device_id = pci_get_device(dev);
- pci_subvendor_id = pci_get_subvendor(dev);
- pci_subdevice_id = pci_get_subdevice(dev);
-
- ent = igb_vendor_info_array;
- while (ent->vendor_id != 0) {
- if ((pci_vendor_id == ent->vendor_id) &&
- (pci_device_id == ent->device_id) &&
-
- ((pci_subvendor_id == ent->subvendor_id) ||
- (ent->subvendor_id == 0)) &&
-
- ((pci_subdevice_id == ent->subdevice_id) ||
- (ent->subdevice_id == 0))) {
- sprintf(adapter_name, "%s, Version - %s",
- igb_strings[ent->index],
- igb_driver_version);
- device_set_desc_copy(dev, adapter_name);
- return (BUS_PROBE_DEFAULT);
- }
- ent++;
- }
- return (ENXIO);
-}
-
-/*********************************************************************
- * Device initialization routine
- *
- * The attach entry point is called when the driver is being loaded.
- * This routine identifies the type of hardware, allocates all resources
- * and initializes the hardware.
- *
- * return 0 on success, positive on failure
- *********************************************************************/
-
-static int
-igb_attach(device_t dev)
-{
- struct adapter *adapter;
- int error = 0;
- u16 eeprom_data;
-
- INIT_DEBUGOUT("igb_attach: begin");
-
- if (resource_disabled("igb", device_get_unit(dev))) {
- device_printf(dev, "Disabled by device hint\n");
- return (ENXIO);
- }
-
- adapter = device_get_softc(dev);
- adapter->dev = adapter->osdep.dev = dev;
- IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
-
- /* SYSCTLs */
- SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
- OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
- igb_sysctl_nvm_info, "I", "NVM Information");
-
- igb_set_sysctl_value(adapter, "enable_aim",
- "Interrupt Moderation", &adapter->enable_aim,
- igb_enable_aim);
-
- SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
- OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
- adapter, 0, igb_set_flowcntl, "I", "Flow Control");
-
- callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
-
- /* Determine hardware and mac info */
- igb_identify_hardware(adapter);
-
- /* Setup PCI resources */
- if (igb_allocate_pci_resources(adapter)) {
- device_printf(dev, "Allocation of PCI resources failed\n");
- error = ENXIO;
- goto err_pci;
- }
-
- /* Do Shared Code initialization */
- if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
- device_printf(dev, "Setup of Shared code failed\n");
- error = ENXIO;
- goto err_pci;
- }
-
- e1000_get_bus_info(&adapter->hw);
-
- /* Sysctls for limiting the amount of work done in the taskqueues */
- igb_set_sysctl_value(adapter, "rx_processing_limit",
- "max number of rx packets to process",
- &adapter->rx_process_limit, igb_rx_process_limit);
-
- igb_set_sysctl_value(adapter, "tx_processing_limit",
- "max number of tx packets to process",
- &adapter->tx_process_limit, igb_tx_process_limit);
-
- /*
- * Validate number of transmit and receive descriptors. It
- * must not exceed hardware maximum, and must be multiple
- * of E1000_DBA_ALIGN.
- */
- if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
- (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
- device_printf(dev, "Using %d TX descriptors instead of %d!\n",
- IGB_DEFAULT_TXD, igb_txd);
- adapter->num_tx_desc = IGB_DEFAULT_TXD;
- } else
- adapter->num_tx_desc = igb_txd;
- if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
- (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
- device_printf(dev, "Using %d RX descriptors instead of %d!\n",
- IGB_DEFAULT_RXD, igb_rxd);
- adapter->num_rx_desc = IGB_DEFAULT_RXD;
- } else
- adapter->num_rx_desc = igb_rxd;
-
- adapter->hw.mac.autoneg = DO_AUTO_NEG;
- adapter->hw.phy.autoneg_wait_to_complete = FALSE;
- adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
-
- /* Copper options */
- if (adapter->hw.phy.media_type == e1000_media_type_copper) {
- adapter->hw.phy.mdix = AUTO_ALL_MODES;
- adapter->hw.phy.disable_polarity_correction = FALSE;
- adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
- }
-
- /*
- * Set the frame limits assuming
- * standard ethernet sized frames.
- */
- adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
-
- /*
- ** Allocate and Setup Queues
- */
- if (igb_allocate_queues(adapter)) {
- error = ENOMEM;
- goto err_pci;
- }
-
- /* Allocate the appropriate stats memory */
- if (adapter->vf_ifp) {
- adapter->stats =
- (struct e1000_vf_stats *)malloc(sizeof \
- (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
- igb_vf_init_stats(adapter);
- } else
- adapter->stats =
- (struct e1000_hw_stats *)malloc(sizeof \
- (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
- if (adapter->stats == NULL) {
- device_printf(dev, "Can not allocate stats memory\n");
- error = ENOMEM;
- goto err_late;
- }
-
- /* Allocate multicast array memory. */
- adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
- MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
- if (adapter->mta == NULL) {
- device_printf(dev, "Can not allocate multicast setup array\n");
- error = ENOMEM;
- goto err_late;
- }
-
- /* Some adapter-specific advanced features */
- if (adapter->hw.mac.type >= e1000_i350) {
- SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
- OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
- adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
- SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
- OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
- adapter, 0, igb_sysctl_eee, "I",
- "Disable Energy Efficient Ethernet");
- if (adapter->hw.phy.media_type == e1000_media_type_copper) {
- if (adapter->hw.mac.type == e1000_i354)
- e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
- else
- e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
- }
- }
-
- /*
- ** Start from a known state, this is
- ** important in reading the nvm and
- ** mac from that.
- */
- e1000_reset_hw(&adapter->hw);
-
- /* Make sure we have a good EEPROM before we read from it */
- if (((adapter->hw.mac.type != e1000_i210) &&
- (adapter->hw.mac.type != e1000_i211)) &&
- (e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
- /*
- ** Some PCI-E parts fail the first check due to
- ** the link being in sleep state, call it again,
- ** if it fails a second time its a real issue.
- */
- if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
- device_printf(dev,
- "The EEPROM Checksum Is Not Valid\n");
- error = EIO;
- goto err_late;
- }
- }
-
- /*
- ** Copy the permanent MAC address out of the EEPROM
- */
- if (e1000_read_mac_addr(&adapter->hw) < 0) {
- device_printf(dev, "EEPROM read error while reading MAC"
- " address\n");
- error = EIO;
- goto err_late;
- }
-
- /* Check its sanity */
- if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
- if (adapter->vf_ifp) {
- u8 addr[ETHER_ADDR_LEN];
- arc4rand(&addr, sizeof(addr), 0);
- addr[0] &= 0xFE;
- addr[0] |= 0x02;
- bcopy(addr, adapter->hw.mac.addr, sizeof(addr));
- } else {
- device_printf(dev, "Invalid MAC address\n");
- error = EIO;
- goto err_late;
- }
- }
-
- /* Setup OS specific network interface */
- if (igb_setup_interface(dev, adapter) != 0)
- goto err_late;
-
- /* Now get a good starting state */
- igb_reset(adapter);
-
- /* Initialize statistics */
- igb_update_stats_counters(adapter);
-
- adapter->hw.mac.get_link_status = 1;
- igb_update_link_status(adapter);
-
- /* Indicate SOL/IDER usage */
- if (e1000_check_reset_block(&adapter->hw))
- device_printf(dev,
- "PHY reset is blocked due to SOL/IDER session.\n");
-
- /* Determine if we have to control management hardware */
- adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
-
- /*
- * Setup Wake-on-Lan
- */
- /* APME bit in EEPROM is mapped to WUC.APME */
- eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
- if (eeprom_data)
- adapter->wol = E1000_WUFC_MAG;
-
- /* Register for VLAN events */
- adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
- igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
- adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
- igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
-
- igb_add_hw_stats(adapter);
-
- /* Tell the stack that the interface is not active */
- adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
- adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-
- adapter->led_dev = led_create(igb_led_func, adapter,
- device_get_nameunit(dev));
-
- /*
- ** Configure Interrupts
- */
- if ((adapter->msix > 1) && (igb_enable_msix))
- error = igb_allocate_msix(adapter);
- else /* MSI or Legacy */
- error = igb_allocate_legacy(adapter);
- if (error)
- goto err_late;
-
-#ifdef DEV_NETMAP
- igb_netmap_attach(adapter);
-#endif /* DEV_NETMAP */
- INIT_DEBUGOUT("igb_attach: end");
-
- return (0);
-
-err_late:
- if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */
- return(error);
- igb_free_transmit_structures(adapter);
- igb_free_receive_structures(adapter);
- igb_release_hw_control(adapter);
-err_pci:
- igb_free_pci_resources(adapter);
- if (adapter->ifp != NULL)
- if_free(adapter->ifp);
- free(adapter->mta, M_DEVBUF);
- IGB_CORE_LOCK_DESTROY(adapter);
-
- return (error);
-}
-
-/*********************************************************************
- * Device removal routine
- *
- * The detach entry point is called when the driver is being removed.
- * This routine stops the adapter and deallocates all the resources
- * that were allocated for driver operation.
- *
- * return 0 on success, positive on failure
- *********************************************************************/
-
-static int
-igb_detach(device_t dev)
-{
- struct adapter *adapter = device_get_softc(dev);
- struct ifnet *ifp = adapter->ifp;
-
- INIT_DEBUGOUT("igb_detach: begin");
-
- /* Make sure VLANS are not using driver */
- if (adapter->ifp->if_vlantrunk != NULL) {
- device_printf(dev,"Vlan in use, detach first\n");
- return (EBUSY);
- }
-
- ether_ifdetach(adapter->ifp);
-
- if (adapter->led_dev != NULL)
- led_destroy(adapter->led_dev);
-
-#ifdef DEVICE_POLLING
- if (ifp->if_capenable & IFCAP_POLLING)
- ether_poll_deregister(ifp);
-#endif
-
- IGB_CORE_LOCK(adapter);
- adapter->in_detach = 1;
- igb_stop(adapter);
- IGB_CORE_UNLOCK(adapter);
-
- e1000_phy_hw_reset(&adapter->hw);
-
- /* Give control back to firmware */
- igb_release_manageability(adapter);
- igb_release_hw_control(adapter);
-
- if (adapter->wol) {
- E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
- E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
- igb_enable_wakeup(dev);
- }
-
- /* Unregister VLAN events */
- if (adapter->vlan_attach != NULL)
- EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
- if (adapter->vlan_detach != NULL)
- EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
-
- callout_drain(&adapter->timer);
-
-#ifdef DEV_NETMAP
- netmap_detach(adapter->ifp);
-#endif /* DEV_NETMAP */
- igb_free_pci_resources(adapter);
- bus_generic_detach(dev);
- if_free(ifp);
-
- igb_free_transmit_structures(adapter);
- igb_free_receive_structures(adapter);
- if (adapter->mta != NULL)
- free(adapter->mta, M_DEVBUF);
-
- IGB_CORE_LOCK_DESTROY(adapter);
-
- return (0);
-}
-
-/*********************************************************************
- *
- * Shutdown entry point
- *
- **********************************************************************/
-
-static int
-igb_shutdown(device_t dev)
-{
- return igb_suspend(dev);
-}
-
-/*
- * Suspend/resume device methods.
- */
-static int
-igb_suspend(device_t dev)
-{
- struct adapter *adapter = device_get_softc(dev);
-
- IGB_CORE_LOCK(adapter);
-
- igb_stop(adapter);
-
- igb_release_manageability(adapter);
- igb_release_hw_control(adapter);
-
- if (adapter->wol) {
- E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
- E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
- igb_enable_wakeup(dev);
- }
-
- IGB_CORE_UNLOCK(adapter);
-
- return bus_generic_suspend(dev);
-}
-
-static int
-igb_resume(device_t dev)
-{
- struct adapter *adapter = device_get_softc(dev);
- struct tx_ring *txr = adapter->tx_rings;
- struct ifnet *ifp = adapter->ifp;
-
- IGB_CORE_LOCK(adapter);
- igb_init_locked(adapter);
- igb_init_manageability(adapter);
-
- if ((ifp->if_flags & IFF_UP) &&
- (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- IGB_TX_LOCK(txr);
-#ifndef IGB_LEGACY_TX
- /* Process the stack queue only if not depleted */
- if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
- !drbr_empty(ifp, txr->br))
- igb_mq_start_locked(ifp, txr);
-#else
- if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
- igb_start_locked(txr, ifp);
-#endif
- IGB_TX_UNLOCK(txr);
- }
- }
- IGB_CORE_UNLOCK(adapter);
-
- return bus_generic_resume(dev);
-}
-
-
-#ifdef IGB_LEGACY_TX
-
-/*********************************************************************
- * Transmit entry point
- *
- * igb_start is called by the stack to initiate a transmit.
- * The driver will remain in this routine as long as there are
- * packets to transmit and transmit resources are available.
- * In case resources are not available stack is notified and
- * the packet is requeued.
- **********************************************************************/
-
-static void
-igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
-{
- struct adapter *adapter = ifp->if_softc;
- struct mbuf *m_head;
-
- IGB_TX_LOCK_ASSERT(txr);
-
- if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
- IFF_DRV_RUNNING)
- return;
- if (!adapter->link_active)
- return;
-
- /* Call cleanup if number of TX descriptors low */
- if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
- igb_txeof(txr);
-
- while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
- if (txr->tx_avail <= IGB_MAX_SCATTER) {
- txr->queue_status |= IGB_QUEUE_DEPLETED;
- break;
- }
- IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
- if (m_head == NULL)
- break;
- /*
- * Encapsulation can modify our pointer, and or make it
- * NULL on failure. In that event, we can't requeue.
- */
- if (igb_xmit(txr, &m_head)) {
- if (m_head != NULL)
- IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
- if (txr->tx_avail <= IGB_MAX_SCATTER)
- txr->queue_status |= IGB_QUEUE_DEPLETED;
- break;
- }
-
- /* Send a copy of the frame to the BPF listener */
- ETHER_BPF_MTAP(ifp, m_head);
-
- /* Set watchdog on */
- txr->watchdog_time = ticks;
- txr->queue_status |= IGB_QUEUE_WORKING;
- }
-}
-
-/*
- * Legacy TX driver routine, called from the
- * stack, always uses tx[0], and spins for it.
- * Should not be used with multiqueue tx
- */
-static void
-igb_start(struct ifnet *ifp)
-{
- struct adapter *adapter = ifp->if_softc;
- struct tx_ring *txr = adapter->tx_rings;
-
- if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
- IGB_TX_LOCK(txr);
- igb_start_locked(txr, ifp);
- IGB_TX_UNLOCK(txr);
- }
- return;
-}
-
-#else /* ~IGB_LEGACY_TX */
-
-/*
-** Multiqueue Transmit Entry:
-** quick turnaround to the stack
-**
-*/
-static int
-igb_mq_start(struct ifnet *ifp, struct mbuf *m)
-{
- struct adapter *adapter = ifp->if_softc;
- struct igb_queue *que;
- struct tx_ring *txr;
- int i, err = 0;
-#ifdef RSS
- uint32_t bucket_id;
-#endif
-
- /* Which queue to use */
- /*
- * When doing RSS, map it to the same outbound queue
- * as the incoming flow would be mapped to.
- *
- * If everything is setup correctly, it should be the
- * same bucket that the current CPU we're on is.
- */
- if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
-#ifdef RSS
- if (rss_hash2bucket(m->m_pkthdr.flowid,
- M_HASHTYPE_GET(m), &bucket_id) == 0) {
- /* XXX TODO: spit out something if bucket_id > num_queues? */
- i = bucket_id % adapter->num_queues;
- } else {
-#endif
- i = m->m_pkthdr.flowid % adapter->num_queues;
-#ifdef RSS
- }
-#endif
- } else {
- i = curcpu % adapter->num_queues;
- }
- txr = &adapter->tx_rings[i];
- que = &adapter->queues[i];
-
- err = drbr_enqueue(ifp, txr->br, m);
- if (err)
- return (err);
- if (IGB_TX_TRYLOCK(txr)) {
- igb_mq_start_locked(ifp, txr);
- IGB_TX_UNLOCK(txr);
- } else
- taskqueue_enqueue(que->tq, &txr->txq_task);
-
- return (0);
-}
-
-static int
-igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
- struct mbuf *next;
- int err = 0, enq = 0;
-
- IGB_TX_LOCK_ASSERT(txr);
-
- if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
- adapter->link_active == 0)
- return (ENETDOWN);
-
- /* Process the queue */
- while ((next = drbr_peek(ifp, txr->br)) != NULL) {
- if ((err = igb_xmit(txr, &next)) != 0) {
- if (next == NULL) {
- /* It was freed, move forward */
- drbr_advance(ifp, txr->br);
- } else {
- /*
- * Still have one left, it may not be
- * the same since the transmit function
- * may have changed it.
- */
- drbr_putback(ifp, txr->br, next);
- }
- break;
- }
- drbr_advance(ifp, txr->br);
- enq++;
- if (next->m_flags & M_MCAST && adapter->vf_ifp)
- if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
- ETHER_BPF_MTAP(ifp, next);
- if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
- break;
- }
- if (enq > 0) {
- /* Set the watchdog */
- txr->queue_status |= IGB_QUEUE_WORKING;
- txr->watchdog_time = ticks;
- }
- if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
- igb_txeof(txr);
- if (txr->tx_avail <= IGB_MAX_SCATTER)
- txr->queue_status |= IGB_QUEUE_DEPLETED;
- return (err);
-}
-
-/*
- * Called from a taskqueue to drain queued transmit packets.
- */
-static void
-igb_deferred_mq_start(void *arg, int pending)
-{
- struct tx_ring *txr = arg;
- struct adapter *adapter = txr->adapter;
- struct ifnet *ifp = adapter->ifp;
-
- IGB_TX_LOCK(txr);
- if (!drbr_empty(ifp, txr->br))
- igb_mq_start_locked(ifp, txr);
- IGB_TX_UNLOCK(txr);
-}
-
-/*
-** Flush all ring buffers
-*/
-static void
-igb_qflush(struct ifnet *ifp)
-{
- struct adapter *adapter = ifp->if_softc;
- struct tx_ring *txr = adapter->tx_rings;
- struct mbuf *m;
-
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- IGB_TX_LOCK(txr);
- while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
- m_freem(m);
- IGB_TX_UNLOCK(txr);
- }
- if_qflush(ifp);
-}
-#endif /* ~IGB_LEGACY_TX */
-
-/*********************************************************************
- * Ioctl entry point
- *
- * igb_ioctl is called when the user wants to configure the
- * interface.
- *
- * return 0 on success, positive on failure
- **********************************************************************/
-
-static int
-igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
-{
- struct adapter *adapter = ifp->if_softc;
- struct ifreq *ifr = (struct ifreq *)data;
-#if defined(INET) || defined(INET6)
- struct ifaddr *ifa = (struct ifaddr *)data;
-#endif
- bool avoid_reset = FALSE;
- int error = 0;
-
- if (adapter->in_detach)
- return (error);
-
- switch (command) {
- case SIOCSIFADDR:
-#ifdef INET
- if (ifa->ifa_addr->sa_family == AF_INET)
- avoid_reset = TRUE;
-#endif
-#ifdef INET6
- if (ifa->ifa_addr->sa_family == AF_INET6)
- avoid_reset = TRUE;
-#endif
- /*
- ** Calling init results in link renegotiation,
- ** so we avoid doing it when possible.
- */
- if (avoid_reset) {
- ifp->if_flags |= IFF_UP;
- if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
- igb_init(adapter);
-#ifdef INET
- if (!(ifp->if_flags & IFF_NOARP))
- arp_ifinit(ifp, ifa);
-#endif
- } else
- error = ether_ioctl(ifp, command, data);
- break;
- case SIOCSIFMTU:
- {
- int max_frame_size;
-
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
-
- IGB_CORE_LOCK(adapter);
- max_frame_size = 9234;
- if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
- ETHER_CRC_LEN) {
- IGB_CORE_UNLOCK(adapter);
- error = EINVAL;
- break;
- }
-
- ifp->if_mtu = ifr->ifr_mtu;
- adapter->max_frame_size =
- ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
- if (ifp->if_drv_flags & IFF_DRV_RUNNING)
- igb_init_locked(adapter);
- IGB_CORE_UNLOCK(adapter);
- break;
- }
- case SIOCSIFFLAGS:
- IOCTL_DEBUGOUT("ioctl rcv'd:\
- SIOCSIFFLAGS (Set Interface Flags)");
- IGB_CORE_LOCK(adapter);
- if (ifp->if_flags & IFF_UP) {
- if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
- if ((ifp->if_flags ^ adapter->if_flags) &
- (IFF_PROMISC | IFF_ALLMULTI)) {
- igb_disable_promisc(adapter);
- igb_set_promisc(adapter);
- }
- } else
- igb_init_locked(adapter);
- } else
- if (ifp->if_drv_flags & IFF_DRV_RUNNING)
- igb_stop(adapter);
- adapter->if_flags = ifp->if_flags;
- IGB_CORE_UNLOCK(adapter);
- break;
- case SIOCADDMULTI:
- case SIOCDELMULTI:
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
- if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
- IGB_CORE_LOCK(adapter);
- igb_disable_intr(adapter);
- igb_set_multi(adapter);
-#ifdef DEVICE_POLLING
- if (!(ifp->if_capenable & IFCAP_POLLING))
-#endif
- igb_enable_intr(adapter);
- IGB_CORE_UNLOCK(adapter);
- }
- break;
- case SIOCSIFMEDIA:
- /* Check SOL/IDER usage */
- IGB_CORE_LOCK(adapter);
- if (e1000_check_reset_block(&adapter->hw)) {
- IGB_CORE_UNLOCK(adapter);
- device_printf(adapter->dev, "Media change is"
- " blocked due to SOL/IDER session.\n");
- break;
- }
- IGB_CORE_UNLOCK(adapter);
- case SIOCGIFMEDIA:
- IOCTL_DEBUGOUT("ioctl rcv'd: \
- SIOCxIFMEDIA (Get/Set Interface Media)");
- error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
- break;
- case SIOCSIFCAP:
- {
- int mask, reinit;
-
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
- reinit = 0;
- mask = ifr->ifr_reqcap ^ ifp->if_capenable;
-#ifdef DEVICE_POLLING
- if (mask & IFCAP_POLLING) {
- if (ifr->ifr_reqcap & IFCAP_POLLING) {
- error = ether_poll_register(igb_poll, ifp);
- if (error)
- return (error);
- IGB_CORE_LOCK(adapter);
- igb_disable_intr(adapter);
- ifp->if_capenable |= IFCAP_POLLING;
- IGB_CORE_UNLOCK(adapter);
- } else {
- error = ether_poll_deregister(ifp);
- /* Enable interrupt even in error case */
- IGB_CORE_LOCK(adapter);
- igb_enable_intr(adapter);
- ifp->if_capenable &= ~IFCAP_POLLING;
- IGB_CORE_UNLOCK(adapter);
- }
- }
-#endif
-#if __FreeBSD_version >= 1000000
- /* HW cannot turn these on/off separately */
- if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
- ifp->if_capenable ^= IFCAP_RXCSUM;
- ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
- reinit = 1;
- }
- if (mask & IFCAP_TXCSUM) {
- ifp->if_capenable ^= IFCAP_TXCSUM;
- reinit = 1;
- }
- if (mask & IFCAP_TXCSUM_IPV6) {
- ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
- reinit = 1;
- }
-#else
- if (mask & IFCAP_HWCSUM) {
- ifp->if_capenable ^= IFCAP_HWCSUM;
- reinit = 1;
- }
-#endif
- if (mask & IFCAP_TSO4) {
- ifp->if_capenable ^= IFCAP_TSO4;
- reinit = 1;
- }
- if (mask & IFCAP_TSO6) {
- ifp->if_capenable ^= IFCAP_TSO6;
- reinit = 1;
- }
- if (mask & IFCAP_VLAN_HWTAGGING) {
- ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
- reinit = 1;
- }
- if (mask & IFCAP_VLAN_HWFILTER) {
- ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
- reinit = 1;
- }
- if (mask & IFCAP_VLAN_HWTSO) {
- ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
- reinit = 1;
- }
- if (mask & IFCAP_LRO) {
- ifp->if_capenable ^= IFCAP_LRO;
- reinit = 1;
- }
- if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
- igb_init(adapter);
- VLAN_CAPABILITIES(ifp);
- break;
- }
-
- default:
- error = ether_ioctl(ifp, command, data);
- break;
- }
-
- return (error);
-}
-
-
-/*********************************************************************
- * Init entry point
- *
- * This routine is used in two ways. It is used by the stack as
- * init entry point in network interface structure. It is also used
- * by the driver as a hw/sw initialization routine to get to a
- * consistent state.
- *
- * return 0 on success, positive on failure
- **********************************************************************/
-
-static void
-igb_init_locked(struct adapter *adapter)
-{
- struct ifnet *ifp = adapter->ifp;
- device_t dev = adapter->dev;
-
- INIT_DEBUGOUT("igb_init: begin");
-
- IGB_CORE_LOCK_ASSERT(adapter);
-
- igb_disable_intr(adapter);
- callout_stop(&adapter->timer);
-
- /* Get the latest mac address, User can use a LAA */
- bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
- ETHER_ADDR_LEN);
-
- /* Put the address into the Receive Address Array */
- e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
-
- igb_reset(adapter);
- igb_update_link_status(adapter);
-
- E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
-
- /* Set hardware offload abilities */
- ifp->if_hwassist = 0;
- if (ifp->if_capenable & IFCAP_TXCSUM) {
-#if __FreeBSD_version >= 1000000
- ifp->if_hwassist |= (CSUM_IP_TCP | CSUM_IP_UDP);
- if (adapter->hw.mac.type != e1000_82575)
- ifp->if_hwassist |= CSUM_IP_SCTP;
-#else
- ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
-#if __FreeBSD_version >= 800000
- if (adapter->hw.mac.type != e1000_82575)
- ifp->if_hwassist |= CSUM_SCTP;
-#endif
-#endif
- }
-
-#if __FreeBSD_version >= 1000000
- if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) {
- ifp->if_hwassist |= (CSUM_IP6_TCP | CSUM_IP6_UDP);
- if (adapter->hw.mac.type != e1000_82575)
- ifp->if_hwassist |= CSUM_IP6_SCTP;
- }
-#endif
- if (ifp->if_capenable & IFCAP_TSO)
- ifp->if_hwassist |= CSUM_TSO;
-
- /* Clear bad data from Rx FIFOs */
- e1000_rx_fifo_flush_82575(&adapter->hw);
-
- /* Configure for OS presence */
- igb_init_manageability(adapter);
-
- /* Prepare transmit descriptors and buffers */
- igb_setup_transmit_structures(adapter);
- igb_initialize_transmit_units(adapter);
-
- /* Setup Multicast table */
- igb_set_multi(adapter);
-
- /*
- ** Figure out the desired mbuf pool
- ** for doing jumbo/packetsplit
- */
- if (adapter->max_frame_size <= 2048)
- adapter->rx_mbuf_sz = MCLBYTES;
- else if (adapter->max_frame_size <= 4096)
- adapter->rx_mbuf_sz = MJUMPAGESIZE;
- else
- adapter->rx_mbuf_sz = MJUM9BYTES;
-
- /* Prepare receive descriptors and buffers */
- if (igb_setup_receive_structures(adapter)) {
- device_printf(dev, "Could not setup receive structures\n");
- return;
- }
- igb_initialize_receive_units(adapter);
-
- /* Enable VLAN support */
- if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
- igb_setup_vlan_hw_support(adapter);
-
- /* Don't lose promiscuous settings */
- igb_set_promisc(adapter);
-
- ifp->if_drv_flags |= IFF_DRV_RUNNING;
- ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
-
- callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
- e1000_clear_hw_cntrs_base_generic(&adapter->hw);
-
- if (adapter->msix > 1) /* Set up queue routing */
- igb_configure_queues(adapter);
-
- /* this clears any pending interrupts */
- E1000_READ_REG(&adapter->hw, E1000_ICR);
-#ifdef DEVICE_POLLING
- /*
- * Only enable interrupts if we are not polling, make sure
- * they are off otherwise.
- */
- if (ifp->if_capenable & IFCAP_POLLING)
- igb_disable_intr(adapter);
- else
-#endif /* DEVICE_POLLING */
- {
- igb_enable_intr(adapter);
- E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
- }
-
- /* Set Energy Efficient Ethernet */
- if (adapter->hw.phy.media_type == e1000_media_type_copper) {
- if (adapter->hw.mac.type == e1000_i354)
- e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
- else
- e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
- }
-}
-
-static void
-igb_init(void *arg)
-{
- struct adapter *adapter = arg;
-
- IGB_CORE_LOCK(adapter);
- igb_init_locked(adapter);
- IGB_CORE_UNLOCK(adapter);
-}
-
-
-static void
-igb_handle_que(void *context, int pending)
-{
- struct igb_queue *que = context;
- struct adapter *adapter = que->adapter;
- struct tx_ring *txr = que->txr;
- struct ifnet *ifp = adapter->ifp;
-
- if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
- bool more;
-
- more = igb_rxeof(que, adapter->rx_process_limit, NULL);
-
- IGB_TX_LOCK(txr);
- igb_txeof(txr);
-#ifndef IGB_LEGACY_TX
- /* Process the stack queue only if not depleted */
- if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
- !drbr_empty(ifp, txr->br))
- igb_mq_start_locked(ifp, txr);
-#else
- if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
- igb_start_locked(txr, ifp);
-#endif
- IGB_TX_UNLOCK(txr);
- /* Do we need another? */
- if (more) {
- taskqueue_enqueue(que->tq, &que->que_task);
- return;
- }
- }
-
-#ifdef DEVICE_POLLING
- if (ifp->if_capenable & IFCAP_POLLING)
- return;
-#endif
- /* Reenable this interrupt */
- if (que->eims)
- E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
- else
- igb_enable_intr(adapter);
-}
-
-/* Deal with link in a sleepable context */
-static void
-igb_handle_link(void *context, int pending)
-{
- struct adapter *adapter = context;
-
- IGB_CORE_LOCK(adapter);
- igb_handle_link_locked(adapter);
- IGB_CORE_UNLOCK(adapter);
-}
-
-static void
-igb_handle_link_locked(struct adapter *adapter)
-{
- struct tx_ring *txr = adapter->tx_rings;
- struct ifnet *ifp = adapter->ifp;
-
- IGB_CORE_LOCK_ASSERT(adapter);
- adapter->hw.mac.get_link_status = 1;
- igb_update_link_status(adapter);
- if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- IGB_TX_LOCK(txr);
-#ifndef IGB_LEGACY_TX
- /* Process the stack queue only if not depleted */
- if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
- !drbr_empty(ifp, txr->br))
- igb_mq_start_locked(ifp, txr);
-#else
- if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
- igb_start_locked(txr, ifp);
-#endif
- IGB_TX_UNLOCK(txr);
- }
- }
-}
-
-/*********************************************************************
- *
- * MSI/Legacy Deferred
- * Interrupt Service routine
- *
- *********************************************************************/
-static int
-igb_irq_fast(void *arg)
-{
- struct adapter *adapter = arg;
- struct igb_queue *que = adapter->queues;
- u32 reg_icr;
-
-
- reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
-
- /* Hot eject? */
- if (reg_icr == 0xffffffff)
- return FILTER_STRAY;
-
- /* Definitely not our interrupt. */
- if (reg_icr == 0x0)
- return FILTER_STRAY;
-
- if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
- return FILTER_STRAY;
-
- /*
- * Mask interrupts until the taskqueue is finished running. This is
- * cheap, just assume that it is needed. This also works around the
- * MSI message reordering errata on certain systems.
- */
- igb_disable_intr(adapter);
- taskqueue_enqueue(que->tq, &que->que_task);
-
- /* Link status change */
- if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
- taskqueue_enqueue(que->tq, &adapter->link_task);
-
- if (reg_icr & E1000_ICR_RXO)
- adapter->rx_overruns++;
- return FILTER_HANDLED;
-}
-
-#ifdef DEVICE_POLLING
-#if __FreeBSD_version >= 800000
-#define POLL_RETURN_COUNT(a) (a)
-static int
-#else
-#define POLL_RETURN_COUNT(a)
-static void
-#endif
-igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
-{
- struct adapter *adapter = ifp->if_softc;
- struct igb_queue *que;
- struct tx_ring *txr;
- u32 reg_icr, rx_done = 0;
- u32 loop = IGB_MAX_LOOP;
- bool more;
-
- IGB_CORE_LOCK(adapter);
- if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
- IGB_CORE_UNLOCK(adapter);
- return POLL_RETURN_COUNT(rx_done);
- }
-
- if (cmd == POLL_AND_CHECK_STATUS) {
- reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
- /* Link status change */
- if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
- igb_handle_link_locked(adapter);
-
- if (reg_icr & E1000_ICR_RXO)
- adapter->rx_overruns++;
- }
- IGB_CORE_UNLOCK(adapter);
-
- for (int i = 0; i < adapter->num_queues; i++) {
- que = &adapter->queues[i];
- txr = que->txr;
-
- igb_rxeof(que, count, &rx_done);
-
- IGB_TX_LOCK(txr);
- do {
- more = igb_txeof(txr);
- } while (loop-- && more);
-#ifndef IGB_LEGACY_TX
- if (!drbr_empty(ifp, txr->br))
- igb_mq_start_locked(ifp, txr);
-#else
- if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
- igb_start_locked(txr, ifp);
-#endif
- IGB_TX_UNLOCK(txr);
- }
-
- return POLL_RETURN_COUNT(rx_done);
-}
-#endif /* DEVICE_POLLING */
-
-/*********************************************************************
- *
- * MSIX Que Interrupt Service routine
- *
- **********************************************************************/
-static void
-igb_msix_que(void *arg)
-{
- struct igb_queue *que = arg;
- struct adapter *adapter = que->adapter;
- struct ifnet *ifp = adapter->ifp;
- struct tx_ring *txr = que->txr;
- struct rx_ring *rxr = que->rxr;
- u32 newitr = 0;
- bool more_rx;
-
- /* Ignore spurious interrupts */
- if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
- return;
-
- E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
- ++que->irqs;
-
- IGB_TX_LOCK(txr);
- igb_txeof(txr);
-#ifndef IGB_LEGACY_TX
- /* Process the stack queue only if not depleted */
- if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
- !drbr_empty(ifp, txr->br))
- igb_mq_start_locked(ifp, txr);
-#else
- if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
- igb_start_locked(txr, ifp);
-#endif
- IGB_TX_UNLOCK(txr);
-
- more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
-
- if (adapter->enable_aim == FALSE)
- goto no_calc;
- /*
- ** Do Adaptive Interrupt Moderation:
- ** - Write out last calculated setting
- ** - Calculate based on average size over
- ** the last interval.
- */
- if (que->eitr_setting)
- E1000_WRITE_REG(&adapter->hw,
- E1000_EITR(que->msix), que->eitr_setting);
-
- que->eitr_setting = 0;
-
- /* Idle, do nothing */
- if ((txr->bytes == 0) && (rxr->bytes == 0))
- goto no_calc;
-
- /* Used half Default if sub-gig */
- if (adapter->link_speed != 1000)
- newitr = IGB_DEFAULT_ITR / 2;
- else {
- if ((txr->bytes) && (txr->packets))
- newitr = txr->bytes/txr->packets;
- if ((rxr->bytes) && (rxr->packets))
- newitr = max(newitr,
- (rxr->bytes / rxr->packets));
- newitr += 24; /* account for hardware frame, crc */
- /* set an upper boundary */
- newitr = min(newitr, 3000);
- /* Be nice to the mid range */
- if ((newitr > 300) && (newitr < 1200))
- newitr = (newitr / 3);
- else
- newitr = (newitr / 2);
- }
- newitr &= 0x7FFC; /* Mask invalid bits */
- if (adapter->hw.mac.type == e1000_82575)
- newitr |= newitr << 16;
- else
- newitr |= E1000_EITR_CNT_IGNR;
-
- /* save for next interrupt */
- que->eitr_setting = newitr;
-
- /* Reset state */
- txr->bytes = 0;
- txr->packets = 0;
- rxr->bytes = 0;
- rxr->packets = 0;
-
-no_calc:
- /* Schedule a clean task if needed*/
- if (more_rx)
- taskqueue_enqueue(que->tq, &que->que_task);
- else
- /* Reenable this interrupt */
- E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
- return;
-}
-
-
-/*********************************************************************
- *
- * MSIX Link Interrupt Service routine
- *
- **********************************************************************/
-
-static void
-igb_msix_link(void *arg)
-{
- struct adapter *adapter = arg;
- u32 icr;
-
- ++adapter->link_irq;
- icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
- if (!(icr & E1000_ICR_LSC))
- goto spurious;
- igb_handle_link(adapter, 0);
-
-spurious:
- /* Rearm */
- E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
- E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
- return;
-}
-
-
-/*********************************************************************
- *
- * Media Ioctl callback
- *
- * This routine is called whenever the user queries the status of
- * the interface using ifconfig.
- *
- **********************************************************************/
-static void
-igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
-{
- struct adapter *adapter = ifp->if_softc;
-
- INIT_DEBUGOUT("igb_media_status: begin");
-
- IGB_CORE_LOCK(adapter);
- igb_update_link_status(adapter);
-
- ifmr->ifm_status = IFM_AVALID;
- ifmr->ifm_active = IFM_ETHER;
-
- if (!adapter->link_active) {
- IGB_CORE_UNLOCK(adapter);
- return;
- }
-
- ifmr->ifm_status |= IFM_ACTIVE;
-
- switch (adapter->link_speed) {
- case 10:
- ifmr->ifm_active |= IFM_10_T;
- break;
- case 100:
- /*
- ** Support for 100Mb SFP - these are Fiber
- ** but the media type appears as serdes
- */
- if (adapter->hw.phy.media_type ==
- e1000_media_type_internal_serdes)
- ifmr->ifm_active |= IFM_100_FX;
- else
- ifmr->ifm_active |= IFM_100_TX;
- break;
- case 1000:
- ifmr->ifm_active |= IFM_1000_T;
- break;
- case 2500:
- ifmr->ifm_active |= IFM_2500_SX;
- break;
- }
-
- if (adapter->link_duplex == FULL_DUPLEX)
- ifmr->ifm_active |= IFM_FDX;
- else
- ifmr->ifm_active |= IFM_HDX;
-
- IGB_CORE_UNLOCK(adapter);
-}
-
-/*********************************************************************
- *
- * Media Ioctl callback
- *
- * This routine is called when the user changes speed/duplex using
- * media/mediopt option with ifconfig.
- *
- **********************************************************************/
-static int
-igb_media_change(struct ifnet *ifp)
-{
- struct adapter *adapter = ifp->if_softc;
- struct ifmedia *ifm = &adapter->media;
-
- INIT_DEBUGOUT("igb_media_change: begin");
-
- if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
- return (EINVAL);
-
- IGB_CORE_LOCK(adapter);
- switch (IFM_SUBTYPE(ifm->ifm_media)) {
- case IFM_AUTO:
- adapter->hw.mac.autoneg = DO_AUTO_NEG;
- adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
- break;
- case IFM_1000_LX:
- case IFM_1000_SX:
- case IFM_1000_T:
- adapter->hw.mac.autoneg = DO_AUTO_NEG;
- adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
- break;
- case IFM_100_TX:
- adapter->hw.mac.autoneg = FALSE;
- adapter->hw.phy.autoneg_advertised = 0;
- if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
- adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
- else
- adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
- break;
- case IFM_10_T:
- adapter->hw.mac.autoneg = FALSE;
- adapter->hw.phy.autoneg_advertised = 0;
- if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
- adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
- else
- adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
- break;
- default:
- device_printf(adapter->dev, "Unsupported media type\n");
- }
-
- igb_init_locked(adapter);
- IGB_CORE_UNLOCK(adapter);
-
- return (0);
-}
-
-
-/*********************************************************************
- *
- * This routine maps the mbufs to Advanced TX descriptors.
- *
- **********************************************************************/
-static int
-igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
-{
- struct adapter *adapter = txr->adapter;
- u32 olinfo_status = 0, cmd_type_len;
- int i, j, error, nsegs;
- int first;
- bool remap = TRUE;
- struct mbuf *m_head;
- bus_dma_segment_t segs[IGB_MAX_SCATTER];
- bus_dmamap_t map;
- struct igb_tx_buf *txbuf;
- union e1000_adv_tx_desc *txd = NULL;
-
- m_head = *m_headp;
-
- /* Basic descriptor defines */
- cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
- E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
-
- if (m_head->m_flags & M_VLANTAG)
- cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
-
- /*
- * Important to capture the first descriptor
- * used because it will contain the index of
- * the one we tell the hardware to report back
- */
- first = txr->next_avail_desc;
- txbuf = &txr->tx_buffers[first];
- map = txbuf->map;
-
- /*
- * Map the packet for DMA.
- */
-retry:
- error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
- *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
-
- if (__predict_false(error)) {
- struct mbuf *m;
-
- switch (error) {
- case EFBIG:
- /* Try it again? - one try */
- if (remap == TRUE) {
- remap = FALSE;
- m = m_collapse(*m_headp, M_NOWAIT,
- IGB_MAX_SCATTER);
- if (m == NULL) {
- adapter->mbuf_defrag_failed++;
- m_freem(*m_headp);
- *m_headp = NULL;
- return (ENOBUFS);
- }
- *m_headp = m;
- goto retry;
- } else
- return (error);
- default:
- txr->no_tx_dma_setup++;
- m_freem(*m_headp);
- *m_headp = NULL;
- return (error);
- }
- }
-
- /* Make certain there are enough descriptors */
- if (txr->tx_avail < (nsegs + 2)) {
- txr->no_desc_avail++;
- bus_dmamap_unload(txr->txtag, map);
- return (ENOBUFS);
- }
- m_head = *m_headp;
-
- /*
- ** Set up the appropriate offload context
- ** this will consume the first descriptor
- */
- error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
- if (__predict_false(error)) {
- m_freem(*m_headp);
- *m_headp = NULL;
- return (error);
- }
-
- /* 82575 needs the queue index added */
- if (adapter->hw.mac.type == e1000_82575)
- olinfo_status |= txr->me << 4;
-
- i = txr->next_avail_desc;
- for (j = 0; j < nsegs; j++) {
- bus_size_t seglen;
- bus_addr_t segaddr;
-
- txbuf = &txr->tx_buffers[i];
- txd = &txr->tx_base[i];
- seglen = segs[j].ds_len;
- segaddr = htole64(segs[j].ds_addr);
-
- txd->read.buffer_addr = segaddr;
- txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
- cmd_type_len | seglen);
- txd->read.olinfo_status = htole32(olinfo_status);
-
- if (++i == txr->num_desc)
- i = 0;
- }
-
- txd->read.cmd_type_len |=
- htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
- txr->tx_avail -= nsegs;
- txr->next_avail_desc = i;
-
- txbuf->m_head = m_head;
- /*
- ** Here we swap the map so the last descriptor,
- ** which gets the completion interrupt has the
- ** real map, and the first descriptor gets the
- ** unused map from this descriptor.
- */
- txr->tx_buffers[first].map = txbuf->map;
- txbuf->map = map;
- bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
-
- /* Set the EOP descriptor that will be marked done */
- txbuf = &txr->tx_buffers[first];
- txbuf->eop = txd;
-
- bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
- /*
- * Advance the Transmit Descriptor Tail (Tdt), this tells the
- * hardware that this frame is available to transmit.
- */
- ++txr->total_packets;
- E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
-
- return (0);
-}
-static void
-igb_set_promisc(struct adapter *adapter)
-{
- struct ifnet *ifp = adapter->ifp;
- struct e1000_hw *hw = &adapter->hw;
- u32 reg;
-
- if (adapter->vf_ifp) {
- e1000_promisc_set_vf(hw, e1000_promisc_enabled);
- return;
- }
-
- reg = E1000_READ_REG(hw, E1000_RCTL);
- if (ifp->if_flags & IFF_PROMISC) {
- reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
- E1000_WRITE_REG(hw, E1000_RCTL, reg);
- } else if (ifp->if_flags & IFF_ALLMULTI) {
- reg |= E1000_RCTL_MPE;
- reg &= ~E1000_RCTL_UPE;
- E1000_WRITE_REG(hw, E1000_RCTL, reg);
- }
-}
-
-static void
-igb_disable_promisc(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- struct ifnet *ifp = adapter->ifp;
- u32 reg;
- int mcnt = 0;
-
- if (adapter->vf_ifp) {
- e1000_promisc_set_vf(hw, e1000_promisc_disabled);
- return;
- }
- reg = E1000_READ_REG(hw, E1000_RCTL);
- reg &= (~E1000_RCTL_UPE);
- if (ifp->if_flags & IFF_ALLMULTI)
- mcnt = MAX_NUM_MULTICAST_ADDRESSES;
- else {
- struct ifmultiaddr *ifma;
-#if __FreeBSD_version < 800000
- IF_ADDR_LOCK(ifp);
-#else
- if_maddr_rlock(ifp);
-#endif
- TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
- if (ifma->ifma_addr->sa_family != AF_LINK)
- continue;
- if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
- break;
- mcnt++;
- }
-#if __FreeBSD_version < 800000
- IF_ADDR_UNLOCK(ifp);
-#else
- if_maddr_runlock(ifp);
-#endif
- }
- /* Don't disable if in MAX groups */
- if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
- reg &= (~E1000_RCTL_MPE);
- E1000_WRITE_REG(hw, E1000_RCTL, reg);
-}
-
-
-/*********************************************************************
- * Multicast Update
- *
- * This routine is called whenever multicast address list is updated.
- *
- **********************************************************************/
-
-static void
-igb_set_multi(struct adapter *adapter)
-{
- struct ifnet *ifp = adapter->ifp;
- struct ifmultiaddr *ifma;
- u32 reg_rctl = 0;
- u8 *mta;
-
- int mcnt = 0;
-
- IOCTL_DEBUGOUT("igb_set_multi: begin");
-
- mta = adapter->mta;
- bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
- MAX_NUM_MULTICAST_ADDRESSES);
-
-#if __FreeBSD_version < 800000
- IF_ADDR_LOCK(ifp);
-#else
- if_maddr_rlock(ifp);
-#endif
- TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
- if (ifma->ifma_addr->sa_family != AF_LINK)
- continue;
-
- if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
- break;
-
- bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
- &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
- mcnt++;
- }
-#if __FreeBSD_version < 800000
- IF_ADDR_UNLOCK(ifp);
-#else
- if_maddr_runlock(ifp);
-#endif
-
- if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
- reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
- reg_rctl |= E1000_RCTL_MPE;
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
- } else
- e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
-}
-
-
-/*********************************************************************
- * Timer routine:
- * This routine checks for link status,
- * updates statistics, and does the watchdog.
- *
- **********************************************************************/
-
-static void
-igb_local_timer(void *arg)
-{
- struct adapter *adapter = arg;
- device_t dev = adapter->dev;
- struct ifnet *ifp = adapter->ifp;
- struct tx_ring *txr = adapter->tx_rings;
- struct igb_queue *que = adapter->queues;
- int hung = 0, busy = 0;
-
-
- IGB_CORE_LOCK_ASSERT(adapter);
-
- igb_update_link_status(adapter);
- igb_update_stats_counters(adapter);
-
- /*
- ** Check the TX queues status
- ** - central locked handling of OACTIVE
- ** - watchdog only if all queues show hung
- */
- for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
- if ((txr->queue_status & IGB_QUEUE_HUNG) &&
- (adapter->pause_frames == 0))
- ++hung;
- if (txr->queue_status & IGB_QUEUE_DEPLETED)
- ++busy;
- if ((txr->queue_status & IGB_QUEUE_IDLE) == 0)
- taskqueue_enqueue(que->tq, &que->que_task);
- }
- if (hung == adapter->num_queues)
- goto timeout;
- if (busy == adapter->num_queues)
- ifp->if_drv_flags |= IFF_DRV_OACTIVE;
- else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
- (busy < adapter->num_queues))
- ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
-
- adapter->pause_frames = 0;
- callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
-#ifndef DEVICE_POLLING
- /* Schedule all queue interrupts - deadlock protection */
- E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
-#endif
- return;
-
-timeout:
- device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
- device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
- E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
- E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
- device_printf(dev,"TX(%d) desc avail = %d,"
- "Next TX to Clean = %d\n",
- txr->me, txr->tx_avail, txr->next_to_clean);
- adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
- adapter->watchdog_events++;
- igb_init_locked(adapter);
-}
-
-static void
-igb_update_link_status(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- struct e1000_fc_info *fc = &hw->fc;
- struct ifnet *ifp = adapter->ifp;
- device_t dev = adapter->dev;
- struct tx_ring *txr = adapter->tx_rings;
- u32 link_check, thstat, ctrl;
- char *flowctl = NULL;
-
- link_check = thstat = ctrl = 0;
-
- /* Get the cached link value or read for real */
- switch (hw->phy.media_type) {
- case e1000_media_type_copper:
- if (hw->mac.get_link_status) {
- /* Do the work to read phy */
- e1000_check_for_link(hw);
- link_check = !hw->mac.get_link_status;
- } else
- link_check = TRUE;
- break;
- case e1000_media_type_fiber:
- e1000_check_for_link(hw);
- link_check = (E1000_READ_REG(hw, E1000_STATUS) &
- E1000_STATUS_LU);
- break;
- case e1000_media_type_internal_serdes:
- e1000_check_for_link(hw);
- link_check = adapter->hw.mac.serdes_has_link;
- break;
- /* VF device is type_unknown */
- case e1000_media_type_unknown:
- e1000_check_for_link(hw);
- link_check = !hw->mac.get_link_status;
- /* Fall thru */
- default:
- break;
- }
-
- /* Check for thermal downshift or shutdown */
- if (hw->mac.type == e1000_i350) {
- thstat = E1000_READ_REG(hw, E1000_THSTAT);
- ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
- }
-
- /* Get the flow control for display */
- switch (fc->current_mode) {
- case e1000_fc_rx_pause:
- flowctl = "RX";
- break;
- case e1000_fc_tx_pause:
- flowctl = "TX";
- break;
- case e1000_fc_full:
- flowctl = "Full";
- break;
- case e1000_fc_none:
- default:
- flowctl = "None";
- break;
- }
-
- /* Now we check if a transition has happened */
- if (link_check && (adapter->link_active == 0)) {
- e1000_get_speed_and_duplex(&adapter->hw,
- &adapter->link_speed, &adapter->link_duplex);
- if (bootverbose)
- device_printf(dev, "Link is up %d Mbps %s,"
- " Flow Control: %s\n",
- adapter->link_speed,
- ((adapter->link_duplex == FULL_DUPLEX) ?
- "Full Duplex" : "Half Duplex"), flowctl);
- adapter->link_active = 1;
- ifp->if_baudrate = adapter->link_speed * 1000000;
- if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
- (thstat & E1000_THSTAT_LINK_THROTTLE))
- device_printf(dev, "Link: thermal downshift\n");
- /* Delay Link Up for Phy update */
- if (((hw->mac.type == e1000_i210) ||
- (hw->mac.type == e1000_i211)) &&
- (hw->phy.id == I210_I_PHY_ID))
- msec_delay(I210_LINK_DELAY);
- /* Reset if the media type changed. */
- if (hw->dev_spec._82575.media_changed) {
- hw->dev_spec._82575.media_changed = false;
- adapter->flags |= IGB_MEDIA_RESET;
- igb_reset(adapter);
- }
- /* This can sleep */
- if_link_state_change(ifp, LINK_STATE_UP);
- } else if (!link_check && (adapter->link_active == 1)) {
- ifp->if_baudrate = adapter->link_speed = 0;
- adapter->link_duplex = 0;
- if (bootverbose)
- device_printf(dev, "Link is Down\n");
- if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
- (thstat & E1000_THSTAT_PWR_DOWN))
- device_printf(dev, "Link: thermal shutdown\n");
- adapter->link_active = 0;
- /* This can sleep */
- if_link_state_change(ifp, LINK_STATE_DOWN);
- /* Reset queue state */
- for (int i = 0; i < adapter->num_queues; i++, txr++)
- txr->queue_status = IGB_QUEUE_IDLE;
- }
-}
-
-/*********************************************************************
- *
- * This routine disables all traffic on the adapter by issuing a
- * global reset on the MAC and deallocates TX/RX buffers.
- *
- **********************************************************************/
-
-static void
-igb_stop(void *arg)
-{
- struct adapter *adapter = arg;
- struct ifnet *ifp = adapter->ifp;
- struct tx_ring *txr = adapter->tx_rings;
-
- IGB_CORE_LOCK_ASSERT(adapter);
-
- INIT_DEBUGOUT("igb_stop: begin");
-
- igb_disable_intr(adapter);
-
- callout_stop(&adapter->timer);
-
- /* Tell the stack that the interface is no longer active */
- ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
- ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-
- /* Disarm watchdog timer. */
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- IGB_TX_LOCK(txr);
- txr->queue_status = IGB_QUEUE_IDLE;
- IGB_TX_UNLOCK(txr);
- }
-
- e1000_reset_hw(&adapter->hw);
- E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
-
- e1000_led_off(&adapter->hw);
- e1000_cleanup_led(&adapter->hw);
-}
-
-
-/*********************************************************************
- *
- * Determine hardware revision.
- *
- **********************************************************************/
-static void
-igb_identify_hardware(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
-
- /* Make sure our PCI config space has the necessary stuff set */
- pci_enable_busmaster(dev);
- adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
-
- /* Save off the information about this board */
- adapter->hw.vendor_id = pci_get_vendor(dev);
- adapter->hw.device_id = pci_get_device(dev);
- adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
- adapter->hw.subsystem_vendor_id =
- pci_read_config(dev, PCIR_SUBVEND_0, 2);
- adapter->hw.subsystem_device_id =
- pci_read_config(dev, PCIR_SUBDEV_0, 2);
-
- /* Set MAC type early for PCI setup */
- e1000_set_mac_type(&adapter->hw);
-
- /* Are we a VF device? */
- if ((adapter->hw.mac.type == e1000_vfadapt) ||
- (adapter->hw.mac.type == e1000_vfadapt_i350))
- adapter->vf_ifp = 1;
- else
- adapter->vf_ifp = 0;
-}
-
-static int
-igb_allocate_pci_resources(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- int rid;
-
- rid = PCIR_BAR(0);
- adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
- &rid, RF_ACTIVE);
- if (adapter->pci_mem == NULL) {
- device_printf(dev, "Unable to allocate bus resource: memory\n");
- return (ENXIO);
- }
- adapter->osdep.mem_bus_space_tag =
- rman_get_bustag(adapter->pci_mem);
- adapter->osdep.mem_bus_space_handle =
- rman_get_bushandle(adapter->pci_mem);
- adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
-
- adapter->num_queues = 1; /* Defaults for Legacy or MSI */
-
- /* This will setup either MSI/X or MSI */
- adapter->msix = igb_setup_msix(adapter);
- adapter->hw.back = &adapter->osdep;
-
- return (0);
-}
-
-/*********************************************************************
- *
- * Setup the Legacy or MSI Interrupt handler
- *
- **********************************************************************/
-static int
-igb_allocate_legacy(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- struct igb_queue *que = adapter->queues;
-#ifndef IGB_LEGACY_TX
- struct tx_ring *txr = adapter->tx_rings;
-#endif
- int error, rid = 0;
-
- /* Turn off all interrupts */
- E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
-
- /* MSI RID is 1 */
- if (adapter->msix == 1)
- rid = 1;
-
- /* We allocate a single interrupt resource */
- adapter->res = bus_alloc_resource_any(dev,
- SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
- if (adapter->res == NULL) {
- device_printf(dev, "Unable to allocate bus resource: "
- "interrupt\n");
- return (ENXIO);
- }
-
-#ifndef IGB_LEGACY_TX
- TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
-#endif
-
- /*
- * Try allocating a fast interrupt and the associated deferred
- * processing contexts.
- */
- TASK_INIT(&que->que_task, 0, igb_handle_que, que);
- /* Make tasklet for deferred link handling */
- TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
- que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
- taskqueue_thread_enqueue, &que->tq);
- taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
- device_get_nameunit(adapter->dev));
- if ((error = bus_setup_intr(dev, adapter->res,
- INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
- adapter, &adapter->tag)) != 0) {
- device_printf(dev, "Failed to register fast interrupt "
- "handler: %d\n", error);
- taskqueue_free(que->tq);
- que->tq = NULL;
- return (error);
- }
-
- return (0);
-}
-
-
-/*********************************************************************
- *
- * Setup the MSIX Queue Interrupt handlers:
- *
- **********************************************************************/
-static int
-igb_allocate_msix(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- struct igb_queue *que = adapter->queues;
- int error, rid, vector = 0;
- int cpu_id = 0;
-#ifdef RSS
- cpuset_t cpu_mask;
-#endif
-
- /* Be sure to start with all interrupts disabled */
- E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
- E1000_WRITE_FLUSH(&adapter->hw);
-
-#ifdef RSS
- /*
- * If we're doing RSS, the number of queues needs to
- * match the number of RSS buckets that are configured.
- *
- * + If there's more queues than RSS buckets, we'll end
- * up with queues that get no traffic.
- *
- * + If there's more RSS buckets than queues, we'll end
- * up having multiple RSS buckets map to the same queue,
- * so there'll be some contention.
- */
- if (adapter->num_queues != rss_getnumbuckets()) {
- device_printf(dev,
- "%s: number of queues (%d) != number of RSS buckets (%d)"
- "; performance will be impacted.\n",
- __func__,
- adapter->num_queues,
- rss_getnumbuckets());
- }
-#endif
-
- for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
- rid = vector +1;
- que->res = bus_alloc_resource_any(dev,
- SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
- if (que->res == NULL) {
- device_printf(dev,
- "Unable to allocate bus resource: "
- "MSIX Queue Interrupt\n");
- return (ENXIO);
- }
- error = bus_setup_intr(dev, que->res,
- INTR_TYPE_NET | INTR_MPSAFE, NULL,
- igb_msix_que, que, &que->tag);
- if (error) {
- que->res = NULL;
- device_printf(dev, "Failed to register Queue handler");
- return (error);
- }
-#if __FreeBSD_version >= 800504
- bus_describe_intr(dev, que->res, que->tag, "que %d", i);
-#endif
- que->msix = vector;
- if (adapter->hw.mac.type == e1000_82575)
- que->eims = E1000_EICR_TX_QUEUE0 << i;
- else
- que->eims = 1 << vector;
-
-#ifdef RSS
- /*
- * The queue ID is used as the RSS layer bucket ID.
- * We look up the queue ID -> RSS CPU ID and select
- * that.
- */
- cpu_id = rss_getcpu(i % rss_getnumbuckets());
-#else
- /*
- * Bind the msix vector, and thus the
- * rings to the corresponding cpu.
- *
- * This just happens to match the default RSS round-robin
- * bucket -> queue -> CPU allocation.
- */
- if (adapter->num_queues > 1) {
- if (igb_last_bind_cpu < 0)
- igb_last_bind_cpu = CPU_FIRST();
- cpu_id = igb_last_bind_cpu;
- }
-#endif
-
- if (adapter->num_queues > 1) {
- bus_bind_intr(dev, que->res, cpu_id);
-#ifdef RSS
- device_printf(dev,
- "Bound queue %d to RSS bucket %d\n",
- i, cpu_id);
-#else
- device_printf(dev,
- "Bound queue %d to cpu %d\n",
- i, cpu_id);
-#endif
- }
-
-#ifndef IGB_LEGACY_TX
- TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
- que->txr);
-#endif
- /* Make tasklet for deferred handling */
- TASK_INIT(&que->que_task, 0, igb_handle_que, que);
- que->tq = taskqueue_create("igb_que", M_NOWAIT,
- taskqueue_thread_enqueue, &que->tq);
- if (adapter->num_queues > 1) {
- /*
- * Only pin the taskqueue thread to a CPU if
- * RSS is in use.
- *
- * This again just happens to match the default RSS
- * round-robin bucket -> queue -> CPU allocation.
- */
-#ifdef RSS
- CPU_SETOF(cpu_id, &cpu_mask);
- taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
- &cpu_mask,
- "%s que (bucket %d)",
- device_get_nameunit(adapter->dev),
- cpu_id);
-#else
- taskqueue_start_threads(&que->tq, 1, PI_NET,
- "%s que (qid %d)",
- device_get_nameunit(adapter->dev),
- cpu_id);
-#endif
- } else {
- taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
- device_get_nameunit(adapter->dev));
- }
-
- /* Finally update the last bound CPU id */
- if (adapter->num_queues > 1)
- igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu);
- }
-
- /* And Link */
- rid = vector + 1;
- adapter->res = bus_alloc_resource_any(dev,
- SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
- if (adapter->res == NULL) {
- device_printf(dev,
- "Unable to allocate bus resource: "
- "MSIX Link Interrupt\n");
- return (ENXIO);
- }
- if ((error = bus_setup_intr(dev, adapter->res,
- INTR_TYPE_NET | INTR_MPSAFE, NULL,
- igb_msix_link, adapter, &adapter->tag)) != 0) {
- device_printf(dev, "Failed to register Link handler");
- return (error);
- }
-#if __FreeBSD_version >= 800504
- bus_describe_intr(dev, adapter->res, adapter->tag, "link");
-#endif
- adapter->linkvec = vector;
-
- return (0);
-}
-
-
-static void
-igb_configure_queues(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- struct igb_queue *que;
- u32 tmp, ivar = 0, newitr = 0;
-
- /* First turn on RSS capability */
- if (adapter->hw.mac.type != e1000_82575)
- E1000_WRITE_REG(hw, E1000_GPIE,
- E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
- E1000_GPIE_PBA | E1000_GPIE_NSICR);
-
- /* Turn on MSIX */
- switch (adapter->hw.mac.type) {
- case e1000_82580:
- case e1000_i350:
- case e1000_i354:
- case e1000_i210:
- case e1000_i211:
- case e1000_vfadapt:
- case e1000_vfadapt_i350:
- /* RX entries */
- for (int i = 0; i < adapter->num_queues; i++) {
- u32 index = i >> 1;
- ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
- que = &adapter->queues[i];
- if (i & 1) {
- ivar &= 0xFF00FFFF;
- ivar |= (que->msix | E1000_IVAR_VALID) << 16;
- } else {
- ivar &= 0xFFFFFF00;
- ivar |= que->msix | E1000_IVAR_VALID;
- }
- E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
- }
- /* TX entries */
- for (int i = 0; i < adapter->num_queues; i++) {
- u32 index = i >> 1;
- ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
- que = &adapter->queues[i];
- if (i & 1) {
- ivar &= 0x00FFFFFF;
- ivar |= (que->msix | E1000_IVAR_VALID) << 24;
- } else {
- ivar &= 0xFFFF00FF;
- ivar |= (que->msix | E1000_IVAR_VALID) << 8;
- }
- E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
- adapter->que_mask |= que->eims;
- }
-
- /* And for the link interrupt */
- ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
- adapter->link_mask = 1 << adapter->linkvec;
- E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
- break;
- case e1000_82576:
- /* RX entries */
- for (int i = 0; i < adapter->num_queues; i++) {
- u32 index = i & 0x7; /* Each IVAR has two entries */
- ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
- que = &adapter->queues[i];
- if (i < 8) {
- ivar &= 0xFFFFFF00;
- ivar |= que->msix | E1000_IVAR_VALID;
- } else {
- ivar &= 0xFF00FFFF;
- ivar |= (que->msix | E1000_IVAR_VALID) << 16;
- }
- E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
- adapter->que_mask |= que->eims;
- }
- /* TX entries */
- for (int i = 0; i < adapter->num_queues; i++) {
- u32 index = i & 0x7; /* Each IVAR has two entries */
- ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
- que = &adapter->queues[i];
- if (i < 8) {
- ivar &= 0xFFFF00FF;
- ivar |= (que->msix | E1000_IVAR_VALID) << 8;
- } else {
- ivar &= 0x00FFFFFF;
- ivar |= (que->msix | E1000_IVAR_VALID) << 24;
- }
- E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
- adapter->que_mask |= que->eims;
- }
-
- /* And for the link interrupt */
- ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
- adapter->link_mask = 1 << adapter->linkvec;
- E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
- break;
-
- case e1000_82575:
- /* enable MSI-X support*/
- tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
- tmp |= E1000_CTRL_EXT_PBA_CLR;
- /* Auto-Mask interrupts upon ICR read. */
- tmp |= E1000_CTRL_EXT_EIAME;
- tmp |= E1000_CTRL_EXT_IRCA;
- E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
-
- /* Queues */
- for (int i = 0; i < adapter->num_queues; i++) {
- que = &adapter->queues[i];
- tmp = E1000_EICR_RX_QUEUE0 << i;
- tmp |= E1000_EICR_TX_QUEUE0 << i;
- que->eims = tmp;
- E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
- i, que->eims);
- adapter->que_mask |= que->eims;
- }
-
- /* Link */
- E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
- E1000_EIMS_OTHER);
- adapter->link_mask |= E1000_EIMS_OTHER;
- default:
- break;
- }
-
- /* Set the starting interrupt rate */
- if (igb_max_interrupt_rate > 0)
- newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
-
- if (hw->mac.type == e1000_82575)
- newitr |= newitr << 16;
- else
- newitr |= E1000_EITR_CNT_IGNR;
-
- for (int i = 0; i < adapter->num_queues; i++) {
- que = &adapter->queues[i];
- E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
- }
-
- return;
-}
-
-
-static void
-igb_free_pci_resources(struct adapter *adapter)
-{
- struct igb_queue *que = adapter->queues;
- device_t dev = adapter->dev;
- int rid;
-
- /*
- ** There is a slight possibility of a failure mode
- ** in attach that will result in entering this function
- ** before interrupt resources have been initialized, and
- ** in that case we do not want to execute the loops below
- ** We can detect this reliably by the state of the adapter
- ** res pointer.
- */
- if (adapter->res == NULL)
- goto mem;
-
- /*
- * First release all the interrupt resources:
- */
- for (int i = 0; i < adapter->num_queues; i++, que++) {
- rid = que->msix + 1;
- if (que->tag != NULL) {
- bus_teardown_intr(dev, que->res, que->tag);
- que->tag = NULL;
- }
- if (que->res != NULL)
- bus_release_resource(dev,
- SYS_RES_IRQ, rid, que->res);
- }
-
- /* Clean the Legacy or Link interrupt last */
- if (adapter->linkvec) /* we are doing MSIX */
- rid = adapter->linkvec + 1;
- else
- (adapter->msix != 0) ? (rid = 1):(rid = 0);
-
- que = adapter->queues;
- if (adapter->tag != NULL) {
- taskqueue_drain(que->tq, &adapter->link_task);
- bus_teardown_intr(dev, adapter->res, adapter->tag);
- adapter->tag = NULL;
- }
- if (adapter->res != NULL)
- bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
-
- for (int i = 0; i < adapter->num_queues; i++, que++) {
- if (que->tq != NULL) {
-#ifndef IGB_LEGACY_TX
- taskqueue_drain(que->tq, &que->txr->txq_task);
-#endif
- taskqueue_drain(que->tq, &que->que_task);
- taskqueue_free(que->tq);
- }
- }
-mem:
- if (adapter->msix)
- pci_release_msi(dev);
-
- if (adapter->msix_mem != NULL)
- bus_release_resource(dev, SYS_RES_MEMORY,
- adapter->memrid, adapter->msix_mem);
-
- if (adapter->pci_mem != NULL)
- bus_release_resource(dev, SYS_RES_MEMORY,
- PCIR_BAR(0), adapter->pci_mem);
-
-}
-
-/*
- * Setup Either MSI/X or MSI
- */
-static int
-igb_setup_msix(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- int bar, want, queues, msgs, maxqueues;
-
- /* tuneable override */
- if (igb_enable_msix == 0)
- goto msi;
-
- /* First try MSI/X */
- msgs = pci_msix_count(dev);
- if (msgs == 0)
- goto msi;
- /*
- ** Some new devices, as with ixgbe, now may
- ** use a different BAR, so we need to keep
- ** track of which is used.
- */
- adapter->memrid = PCIR_BAR(IGB_MSIX_BAR);
- bar = pci_read_config(dev, adapter->memrid, 4);
- if (bar == 0) /* use next bar */
- adapter->memrid += 4;
- adapter->msix_mem = bus_alloc_resource_any(dev,
- SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE);
- if (adapter->msix_mem == NULL) {
- /* May not be enabled */
- device_printf(adapter->dev,
- "Unable to map MSIX table \n");
- goto msi;
- }
-
- queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
-
- /* Override via tuneable */
- if (igb_num_queues != 0)
- queues = igb_num_queues;
-
-#ifdef RSS
- /* If we're doing RSS, clamp at the number of RSS buckets */
- if (queues > rss_getnumbuckets())
- queues = rss_getnumbuckets();
-#endif
-
-
- /* Sanity check based on HW */
- switch (adapter->hw.mac.type) {
- case e1000_82575:
- maxqueues = 4;
- break;
- case e1000_82576:
- case e1000_82580:
- case e1000_i350:
- case e1000_i354:
- maxqueues = 8;
- break;
- case e1000_i210:
- maxqueues = 4;
- break;
- case e1000_i211:
- maxqueues = 2;
- break;
- default: /* VF interfaces */
- maxqueues = 1;
- break;
- }
-
- /* Final clamp on the actual hardware capability */
- if (queues > maxqueues)
- queues = maxqueues;
-
- /*
- ** One vector (RX/TX pair) per queue
- ** plus an additional for Link interrupt
- */
- want = queues + 1;
- if (msgs >= want)
- msgs = want;
- else {
- device_printf(adapter->dev,
- "MSIX Configuration Problem, "
- "%d vectors configured, but %d queues wanted!\n",
- msgs, want);
- goto msi;
- }
- if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
- device_printf(adapter->dev,
- "Using MSIX interrupts with %d vectors\n", msgs);
- adapter->num_queues = queues;
- return (msgs);
- }
- /*
- ** If MSIX alloc failed or provided us with
- ** less than needed, free and fall through to MSI
- */
- pci_release_msi(dev);
-
-msi:
- if (adapter->msix_mem != NULL) {
- bus_release_resource(dev, SYS_RES_MEMORY,
- PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
- adapter->msix_mem = NULL;
- }
- msgs = 1;
- if (pci_alloc_msi(dev, &msgs) == 0) {
- device_printf(adapter->dev," Using an MSI interrupt\n");
- return (msgs);
- }
- device_printf(adapter->dev," Using a Legacy interrupt\n");
- return (0);
-}
-
-/*********************************************************************
- *
- * Initialize the DMA Coalescing feature
- *
- **********************************************************************/
-static void
-igb_init_dmac(struct adapter *adapter, u32 pba)
-{
- device_t dev = adapter->dev;
- struct e1000_hw *hw = &adapter->hw;
- u32 dmac, reg = ~E1000_DMACR_DMAC_EN;
- u16 hwm;
-
- if (hw->mac.type == e1000_i211)
- return;
-
- if (hw->mac.type > e1000_82580) {
-
- if (adapter->dmac == 0) { /* Disabling it */
- E1000_WRITE_REG(hw, E1000_DMACR, reg);
- return;
- } else
- device_printf(dev, "DMA Coalescing enabled\n");
-
- /* Set starting threshold */
- E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
-
- hwm = 64 * pba - adapter->max_frame_size / 16;
- if (hwm < 64 * (pba - 6))
- hwm = 64 * (pba - 6);
- reg = E1000_READ_REG(hw, E1000_FCRTC);
- reg &= ~E1000_FCRTC_RTH_COAL_MASK;
- reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
- & E1000_FCRTC_RTH_COAL_MASK);
- E1000_WRITE_REG(hw, E1000_FCRTC, reg);
-
-
- dmac = pba - adapter->max_frame_size / 512;
- if (dmac < pba - 10)
- dmac = pba - 10;
- reg = E1000_READ_REG(hw, E1000_DMACR);
- reg &= ~E1000_DMACR_DMACTHR_MASK;
- reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT)
- & E1000_DMACR_DMACTHR_MASK);
-
- /* transition to L0x or L1 if available..*/
- reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
-
- /* Check if status is 2.5Gb backplane connection
- * before configuration of watchdog timer, which is
- * in msec values in 12.8usec intervals
- * watchdog timer= msec values in 32usec intervals
- * for non 2.5Gb connection
- */
- if (hw->mac.type == e1000_i354) {
- int status = E1000_READ_REG(hw, E1000_STATUS);
- if ((status & E1000_STATUS_2P5_SKU) &&
- (!(status & E1000_STATUS_2P5_SKU_OVER)))
- reg |= ((adapter->dmac * 5) >> 6);
- else
- reg |= (adapter->dmac >> 5);
- } else {
- reg |= (adapter->dmac >> 5);
- }
-
- E1000_WRITE_REG(hw, E1000_DMACR, reg);
-
- E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
-
- /* Set the interval before transition */
- reg = E1000_READ_REG(hw, E1000_DMCTLX);
- if (hw->mac.type == e1000_i350)
- reg |= IGB_DMCTLX_DCFLUSH_DIS;
- /*
- ** in 2.5Gb connection, TTLX unit is 0.4 usec
- ** which is 0x4*2 = 0xA. But delay is still 4 usec
- */
- if (hw->mac.type == e1000_i354) {
- int status = E1000_READ_REG(hw, E1000_STATUS);
- if ((status & E1000_STATUS_2P5_SKU) &&
- (!(status & E1000_STATUS_2P5_SKU_OVER)))
- reg |= 0xA;
- else
- reg |= 0x4;
- } else {
- reg |= 0x4;
- }
-
- E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
-
- /* free space in tx packet buffer to wake from DMA coal */
- E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
- (2 * adapter->max_frame_size)) >> 6);
-
- /* make low power state decision controlled by DMA coal */
- reg = E1000_READ_REG(hw, E1000_PCIEMISC);
- reg &= ~E1000_PCIEMISC_LX_DECISION;
- E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
-
- } else if (hw->mac.type == e1000_82580) {
- u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
- E1000_WRITE_REG(hw, E1000_PCIEMISC,
- reg & ~E1000_PCIEMISC_LX_DECISION);
- E1000_WRITE_REG(hw, E1000_DMACR, 0);
- }
-}
-
-
-/*********************************************************************
- *
- * Set up an fresh starting state
- *
- **********************************************************************/
-static void
-igb_reset(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- struct e1000_hw *hw = &adapter->hw;
- struct e1000_fc_info *fc = &hw->fc;
- struct ifnet *ifp = adapter->ifp;
- u32 pba = 0;
- u16 hwm;
-
- INIT_DEBUGOUT("igb_reset: begin");
-
- /* Let the firmware know the OS is in control */
- igb_get_hw_control(adapter);
-
- /*
- * Packet Buffer Allocation (PBA)
- * Writing PBA sets the receive portion of the buffer
- * the remainder is used for the transmit buffer.
- */
- switch (hw->mac.type) {
- case e1000_82575:
- pba = E1000_PBA_32K;
- break;
- case e1000_82576:
- case e1000_vfadapt:
- pba = E1000_READ_REG(hw, E1000_RXPBS);
- pba &= E1000_RXPBS_SIZE_MASK_82576;
- break;
- case e1000_82580:
- case e1000_i350:
- case e1000_i354:
- case e1000_vfadapt_i350:
- pba = E1000_READ_REG(hw, E1000_RXPBS);
- pba = e1000_rxpbs_adjust_82580(pba);
- break;
- case e1000_i210:
- case e1000_i211:
- pba = E1000_PBA_34K;
- default:
- break;
- }
-
- /* Special needs in case of Jumbo frames */
- if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
- u32 tx_space, min_tx, min_rx;
- pba = E1000_READ_REG(hw, E1000_PBA);
- tx_space = pba >> 16;
- pba &= 0xffff;
- min_tx = (adapter->max_frame_size +
- sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
- min_tx = roundup2(min_tx, 1024);
- min_tx >>= 10;
- min_rx = adapter->max_frame_size;
- min_rx = roundup2(min_rx, 1024);
- min_rx >>= 10;
- if (tx_space < min_tx &&
- ((min_tx - tx_space) < pba)) {
- pba = pba - (min_tx - tx_space);
- /*
- * if short on rx space, rx wins
- * and must trump tx adjustment
- */
- if (pba < min_rx)
- pba = min_rx;
- }
- E1000_WRITE_REG(hw, E1000_PBA, pba);
- }
-
- INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
-
- /*
- * These parameters control the automatic generation (Tx) and
- * response (Rx) to Ethernet PAUSE frames.
- * - High water mark should allow for at least two frames to be
- * received after sending an XOFF.
- * - Low water mark works best when it is very near the high water mark.
- * This allows the receiver to restart by sending XON when it has
- * drained a bit.
- */
- hwm = min(((pba << 10) * 9 / 10),
- ((pba << 10) - 2 * adapter->max_frame_size));
-
- if (hw->mac.type < e1000_82576) {
- fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
- fc->low_water = fc->high_water - 8;
- } else {
- fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
- fc->low_water = fc->high_water - 16;
- }
-
- fc->pause_time = IGB_FC_PAUSE_TIME;
- fc->send_xon = TRUE;
- if (adapter->fc)
- fc->requested_mode = adapter->fc;
- else
- fc->requested_mode = e1000_fc_default;
-
- /* Issue a global reset */
- e1000_reset_hw(hw);
- E1000_WRITE_REG(hw, E1000_WUC, 0);
-
- /* Reset for AutoMediaDetect */
- if (adapter->flags & IGB_MEDIA_RESET) {
- e1000_setup_init_funcs(hw, TRUE);
- e1000_get_bus_info(hw);
- adapter->flags &= ~IGB_MEDIA_RESET;
- }
-
- if (e1000_init_hw(hw) < 0)
- device_printf(dev, "Hardware Initialization Failed\n");
-
- /* Setup DMA Coalescing */
- igb_init_dmac(adapter, pba);
-
- E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
- e1000_get_phy_info(hw);
- e1000_check_for_link(hw);
- return;
-}
-
-/*********************************************************************
- *
- * Setup networking device structure and register an interface.
- *
- **********************************************************************/
-static int
-igb_setup_interface(device_t dev, struct adapter *adapter)
-{
- struct ifnet *ifp;
-
- INIT_DEBUGOUT("igb_setup_interface: begin");
-
- ifp = adapter->ifp = if_alloc(IFT_ETHER);
- if (ifp == NULL) {
- device_printf(dev, "can not allocate ifnet structure\n");
- return (-1);
- }
- if_initname(ifp, device_get_name(dev), device_get_unit(dev));
- ifp->if_init = igb_init;
- ifp->if_softc = adapter;
- ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
- ifp->if_ioctl = igb_ioctl;
- ifp->if_get_counter = igb_get_counter;
-
- /* TSO parameters */
- ifp->if_hw_tsomax = IP_MAXPACKET;
- ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER;
- ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE;
-
-#ifndef IGB_LEGACY_TX
- ifp->if_transmit = igb_mq_start;
- ifp->if_qflush = igb_qflush;
-#else
- ifp->if_start = igb_start;
- IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
- ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
- IFQ_SET_READY(&ifp->if_snd);
-#endif
-
- ether_ifattach(ifp, adapter->hw.mac.addr);
-
- ifp->if_capabilities = ifp->if_capenable = 0;
-
- ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
-#if __FreeBSD_version >= 1000000
- ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
-#endif
- ifp->if_capabilities |= IFCAP_TSO;
- ifp->if_capabilities |= IFCAP_JUMBO_MTU;
- ifp->if_capenable = ifp->if_capabilities;
-
- /* Don't enable LRO by default */
- ifp->if_capabilities |= IFCAP_LRO;
-
-#ifdef DEVICE_POLLING
- ifp->if_capabilities |= IFCAP_POLLING;
-#endif
-
- /*
- * Tell the upper layer(s) we
- * support full VLAN capability.
- */
- ifp->if_hdrlen = sizeof(struct ether_vlan_header);
- ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
- | IFCAP_VLAN_HWTSO
- | IFCAP_VLAN_MTU;
- ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
- | IFCAP_VLAN_HWTSO
- | IFCAP_VLAN_MTU;
-
- /*
- ** Don't turn this on by default, if vlans are
- ** created on another pseudo device (eg. lagg)
- ** then vlan events are not passed thru, breaking
- ** operation, but with HW FILTER off it works. If
- ** using vlans directly on the igb driver you can
- ** enable this and get full hardware tag filtering.
- */
- ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
-
- /*
- * Specify the media types supported by this adapter and register
- * callbacks to update media and link information
- */
- ifmedia_init(&adapter->media, IFM_IMASK,
- igb_media_change, igb_media_status);
- if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
- (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
- } else {
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
- 0, NULL);
- if (adapter->hw.phy.type != e1000_phy_ife) {
- ifmedia_add(&adapter->media,
- IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
- ifmedia_add(&adapter->media,
- IFM_ETHER | IFM_1000_T, 0, NULL);
- }
- }
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
- ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
- return (0);
-}
-
-
-/*
- * Manage DMA'able memory.
- */
-static void
-igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
-{
- if (error)
- return;
- *(bus_addr_t *) arg = segs[0].ds_addr;
-}
-
-static int
-igb_dma_malloc(struct adapter *adapter, bus_size_t size,
- struct igb_dma_alloc *dma, int mapflags)
-{
- int error;
-
- error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
- IGB_DBA_ALIGN, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- size, /* maxsize */
- 1, /* nsegments */
- size, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockarg */
- &dma->dma_tag);
- if (error) {
- device_printf(adapter->dev,
- "%s: bus_dma_tag_create failed: %d\n",
- __func__, error);
- goto fail_0;
- }
-
- error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
- BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
- if (error) {
- device_printf(adapter->dev,
- "%s: bus_dmamem_alloc(%ju) failed: %d\n",
- __func__, (uintmax_t)size, error);
- goto fail_2;
- }
-
- dma->dma_paddr = 0;
- error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
- size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
- if (error || dma->dma_paddr == 0) {
- device_printf(adapter->dev,
- "%s: bus_dmamap_load failed: %d\n",
- __func__, error);
- goto fail_3;
- }
-
- return (0);
-
-fail_3:
- bus_dmamap_unload(dma->dma_tag, dma->dma_map);
-fail_2:
- bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
- bus_dma_tag_destroy(dma->dma_tag);
-fail_0:
- dma->dma_tag = NULL;
-
- return (error);
-}
-
-static void
-igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
-{
- if (dma->dma_tag == NULL)
- return;
- if (dma->dma_paddr != 0) {
- bus_dmamap_sync(dma->dma_tag, dma->dma_map,
- BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(dma->dma_tag, dma->dma_map);
- dma->dma_paddr = 0;
- }
- if (dma->dma_vaddr != NULL) {
- bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
- dma->dma_vaddr = NULL;
- }
- bus_dma_tag_destroy(dma->dma_tag);
- dma->dma_tag = NULL;
-}
-
-
-/*********************************************************************
- *
- * Allocate memory for the transmit and receive rings, and then
- * the descriptors associated with each, called only once at attach.
- *
- **********************************************************************/
-static int
-igb_allocate_queues(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- struct igb_queue *que = NULL;
- struct tx_ring *txr = NULL;
- struct rx_ring *rxr = NULL;
- int rsize, tsize, error = E1000_SUCCESS;
- int txconf = 0, rxconf = 0;
-
- /* First allocate the top level queue structs */
- if (!(adapter->queues =
- (struct igb_queue *) malloc(sizeof(struct igb_queue) *
- adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
- device_printf(dev, "Unable to allocate queue memory\n");
- error = ENOMEM;
- goto fail;
- }
-
- /* Next allocate the TX ring struct memory */
- if (!(adapter->tx_rings =
- (struct tx_ring *) malloc(sizeof(struct tx_ring) *
- adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
- device_printf(dev, "Unable to allocate TX ring memory\n");
- error = ENOMEM;
- goto tx_fail;
- }
-
- /* Now allocate the RX */
- if (!(adapter->rx_rings =
- (struct rx_ring *) malloc(sizeof(struct rx_ring) *
- adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
- device_printf(dev, "Unable to allocate RX ring memory\n");
- error = ENOMEM;
- goto rx_fail;
- }
-
- tsize = roundup2(adapter->num_tx_desc *
- sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
- /*
- * Now set up the TX queues, txconf is needed to handle the
- * possibility that things fail midcourse and we need to
- * undo memory gracefully
- */
- for (int i = 0; i < adapter->num_queues; i++, txconf++) {
- /* Set up some basics */
- txr = &adapter->tx_rings[i];
- txr->adapter = adapter;
- txr->me = i;
- txr->num_desc = adapter->num_tx_desc;
-
- /* Initialize the TX lock */
- snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
- device_get_nameunit(dev), txr->me);
- mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
-
- if (igb_dma_malloc(adapter, tsize,
- &txr->txdma, BUS_DMA_NOWAIT)) {
- device_printf(dev,
- "Unable to allocate TX Descriptor memory\n");
- error = ENOMEM;
- goto err_tx_desc;
- }
- txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr;
- bzero((void *)txr->tx_base, tsize);
-
- /* Now allocate transmit buffers for the ring */
- if (igb_allocate_transmit_buffers(txr)) {
- device_printf(dev,
- "Critical Failure setting up transmit buffers\n");
- error = ENOMEM;
- goto err_tx_desc;
- }
-#ifndef IGB_LEGACY_TX
- /* Allocate a buf ring */
- txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
- M_WAITOK, &txr->tx_mtx);
-#endif
- }
-
- /*
- * Next the RX queues...
- */
- rsize = roundup2(adapter->num_rx_desc *
- sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
- for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
- rxr = &adapter->rx_rings[i];
- rxr->adapter = adapter;
- rxr->me = i;
-
- /* Initialize the RX lock */
- snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
- device_get_nameunit(dev), txr->me);
- mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
-
- if (igb_dma_malloc(adapter, rsize,
- &rxr->rxdma, BUS_DMA_NOWAIT)) {
- device_printf(dev,
- "Unable to allocate RxDescriptor memory\n");
- error = ENOMEM;
- goto err_rx_desc;
- }
- rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
- bzero((void *)rxr->rx_base, rsize);
-
- /* Allocate receive buffers for the ring*/
- if (igb_allocate_receive_buffers(rxr)) {
- device_printf(dev,
- "Critical Failure setting up receive buffers\n");
- error = ENOMEM;
- goto err_rx_desc;
- }
- }
-
- /*
- ** Finally set up the queue holding structs
- */
- for (int i = 0; i < adapter->num_queues; i++) {
- que = &adapter->queues[i];
- que->adapter = adapter;
- que->txr = &adapter->tx_rings[i];
- que->rxr = &adapter->rx_rings[i];
- }
-
- return (0);
-
-err_rx_desc:
- for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
- igb_dma_free(adapter, &rxr->rxdma);
-err_tx_desc:
- for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
- igb_dma_free(adapter, &txr->txdma);
- free(adapter->rx_rings, M_DEVBUF);
-rx_fail:
-#ifndef IGB_LEGACY_TX
- buf_ring_free(txr->br, M_DEVBUF);
-#endif
- free(adapter->tx_rings, M_DEVBUF);
-tx_fail:
- free(adapter->queues, M_DEVBUF);
-fail:
- return (error);
-}
-
-/*********************************************************************
- *
- * Allocate memory for tx_buffer structures. The tx_buffer stores all
- * the information needed to transmit a packet on the wire. This is
- * called only once at attach, setup is done every reset.
- *
- **********************************************************************/
-static int
-igb_allocate_transmit_buffers(struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
- device_t dev = adapter->dev;
- struct igb_tx_buf *txbuf;
- int error, i;
-
- /*
- * Setup DMA descriptor areas.
- */
- if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
- 1, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- IGB_TSO_SIZE, /* maxsize */
- IGB_MAX_SCATTER, /* nsegments */
- PAGE_SIZE, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockfuncarg */
- &txr->txtag))) {
- device_printf(dev,"Unable to allocate TX DMA tag\n");
- goto fail;
- }
-
- if (!(txr->tx_buffers =
- (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) *
- adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
- device_printf(dev, "Unable to allocate tx_buffer memory\n");
- error = ENOMEM;
- goto fail;
- }
-
- /* Create the descriptor buffer dma maps */
- txbuf = txr->tx_buffers;
- for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
- error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
- if (error != 0) {
- device_printf(dev, "Unable to create TX DMA map\n");
- goto fail;
- }
- }
-
- return 0;
-fail:
- /* We free all, it handles case where we are in the middle */
- igb_free_transmit_structures(adapter);
- return (error);
-}
-
-/*********************************************************************
- *
- * Initialize a transmit ring.
- *
- **********************************************************************/
-static void
-igb_setup_transmit_ring(struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
- struct igb_tx_buf *txbuf;
- int i;
-#ifdef DEV_NETMAP
- struct netmap_adapter *na = NA(adapter->ifp);
- struct netmap_slot *slot;
-#endif /* DEV_NETMAP */
-
- /* Clear the old descriptor contents */
- IGB_TX_LOCK(txr);
-#ifdef DEV_NETMAP
- slot = netmap_reset(na, NR_TX, txr->me, 0);
-#endif /* DEV_NETMAP */
- bzero((void *)txr->tx_base,
- (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
- /* Reset indices */
- txr->next_avail_desc = 0;
- txr->next_to_clean = 0;
-
- /* Free any existing tx buffers. */
- txbuf = txr->tx_buffers;
- for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
- if (txbuf->m_head != NULL) {
- bus_dmamap_sync(txr->txtag, txbuf->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(txr->txtag, txbuf->map);
- m_freem(txbuf->m_head);
- txbuf->m_head = NULL;
- }
-#ifdef DEV_NETMAP
- if (slot) {
- int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
- /* no need to set the address */
- netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
- }
-#endif /* DEV_NETMAP */
- /* clear the watch index */
- txbuf->eop = NULL;
- }
-
- /* Set number of descriptors available */
- txr->tx_avail = adapter->num_tx_desc;
-
- bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
- IGB_TX_UNLOCK(txr);
-}
-
-/*********************************************************************
- *
- * Initialize all transmit rings.
- *
- **********************************************************************/
-static void
-igb_setup_transmit_structures(struct adapter *adapter)
-{
- struct tx_ring *txr = adapter->tx_rings;
-
- for (int i = 0; i < adapter->num_queues; i++, txr++)
- igb_setup_transmit_ring(txr);
-
- return;
-}
-
-/*********************************************************************
- *
- * Enable transmit unit.
- *
- **********************************************************************/
-static void
-igb_initialize_transmit_units(struct adapter *adapter)
-{
- struct tx_ring *txr = adapter->tx_rings;
- struct e1000_hw *hw = &adapter->hw;
- u32 tctl, txdctl;
-
- INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
- tctl = txdctl = 0;
-
- /* Setup the Tx Descriptor Rings */
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- u64 bus_addr = txr->txdma.dma_paddr;
-
- E1000_WRITE_REG(hw, E1000_TDLEN(i),
- adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
- E1000_WRITE_REG(hw, E1000_TDBAH(i),
- (uint32_t)(bus_addr >> 32));
- E1000_WRITE_REG(hw, E1000_TDBAL(i),
- (uint32_t)bus_addr);
-
- /* Setup the HW Tx Head and Tail descriptor pointers */
- E1000_WRITE_REG(hw, E1000_TDT(i), 0);
- E1000_WRITE_REG(hw, E1000_TDH(i), 0);
-
- HW_DEBUGOUT2("Base = %x, Length = %x\n",
- E1000_READ_REG(hw, E1000_TDBAL(i)),
- E1000_READ_REG(hw, E1000_TDLEN(i)));
-
- txr->queue_status = IGB_QUEUE_IDLE;
-
- txdctl |= IGB_TX_PTHRESH;
- txdctl |= IGB_TX_HTHRESH << 8;
- txdctl |= IGB_TX_WTHRESH << 16;
- txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
- E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
- }
-
- if (adapter->vf_ifp)
- return;
-
- e1000_config_collision_dist(hw);
-
- /* Program the Transmit Control Register */
- tctl = E1000_READ_REG(hw, E1000_TCTL);
- tctl &= ~E1000_TCTL_CT;
- tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
- (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
-
- /* This write will effectively turn on the transmit unit. */
- E1000_WRITE_REG(hw, E1000_TCTL, tctl);
-}
-
-/*********************************************************************
- *
- * Free all transmit rings.
- *
- **********************************************************************/
-static void
-igb_free_transmit_structures(struct adapter *adapter)
-{
- struct tx_ring *txr = adapter->tx_rings;
-
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- IGB_TX_LOCK(txr);
- igb_free_transmit_buffers(txr);
- igb_dma_free(adapter, &txr->txdma);
- IGB_TX_UNLOCK(txr);
- IGB_TX_LOCK_DESTROY(txr);
- }
- free(adapter->tx_rings, M_DEVBUF);
-}
-
-/*********************************************************************
- *
- * Free transmit ring related data structures.
- *
- **********************************************************************/
-static void
-igb_free_transmit_buffers(struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
- struct igb_tx_buf *tx_buffer;
- int i;
-
- INIT_DEBUGOUT("free_transmit_ring: begin");
-
- if (txr->tx_buffers == NULL)
- return;
-
- tx_buffer = txr->tx_buffers;
- for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
- if (tx_buffer->m_head != NULL) {
- bus_dmamap_sync(txr->txtag, tx_buffer->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(txr->txtag,
- tx_buffer->map);
- m_freem(tx_buffer->m_head);
- tx_buffer->m_head = NULL;
- if (tx_buffer->map != NULL) {
- bus_dmamap_destroy(txr->txtag,
- tx_buffer->map);
- tx_buffer->map = NULL;
- }
- } else if (tx_buffer->map != NULL) {
- bus_dmamap_unload(txr->txtag,
- tx_buffer->map);
- bus_dmamap_destroy(txr->txtag,
- tx_buffer->map);
- tx_buffer->map = NULL;
- }
- }
-#ifndef IGB_LEGACY_TX
- if (txr->br != NULL)
- buf_ring_free(txr->br, M_DEVBUF);
-#endif
- if (txr->tx_buffers != NULL) {
- free(txr->tx_buffers, M_DEVBUF);
- txr->tx_buffers = NULL;
- }
- if (txr->txtag != NULL) {
- bus_dma_tag_destroy(txr->txtag);
- txr->txtag = NULL;
- }
- return;
-}
-
-/**********************************************************************
- *
- * Setup work for hardware segmentation offload (TSO) on
- * adapters using advanced tx descriptors
- *
- **********************************************************************/
-static int
-igb_tso_setup(struct tx_ring *txr, struct mbuf *mp,
- u32 *cmd_type_len, u32 *olinfo_status)
-{
- struct adapter *adapter = txr->adapter;
- struct e1000_adv_tx_context_desc *TXD;
- u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
- u32 mss_l4len_idx = 0, paylen;
- u16 vtag = 0, eh_type;
- int ctxd, ehdrlen, ip_hlen, tcp_hlen;
- struct ether_vlan_header *eh;
-#ifdef INET6
- struct ip6_hdr *ip6;
-#endif
-#ifdef INET
- struct ip *ip;
-#endif
- struct tcphdr *th;
-
-
- /*
- * Determine where frame payload starts.
- * Jump over vlan headers if already present
- */
- eh = mtod(mp, struct ether_vlan_header *);
- if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
- ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
- eh_type = eh->evl_proto;
- } else {
- ehdrlen = ETHER_HDR_LEN;
- eh_type = eh->evl_encap_proto;
- }
-
- switch (ntohs(eh_type)) {
-#ifdef INET6
- case ETHERTYPE_IPV6:
- ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
- /* XXX-BZ For now we do not pretend to support ext. hdrs. */
- if (ip6->ip6_nxt != IPPROTO_TCP)
- return (ENXIO);
- ip_hlen = sizeof(struct ip6_hdr);
- ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
- th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
- th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
- type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
- break;
-#endif
-#ifdef INET
- case ETHERTYPE_IP:
- ip = (struct ip *)(mp->m_data + ehdrlen);
- if (ip->ip_p != IPPROTO_TCP)
- return (ENXIO);
- ip->ip_sum = 0;
- ip_hlen = ip->ip_hl << 2;
- th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
- th->th_sum = in_pseudo(ip->ip_src.s_addr,
- ip->ip_dst.s_addr, htons(IPPROTO_TCP));
- type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
- /* Tell transmit desc to also do IPv4 checksum. */
- *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
- break;
-#endif
- default:
- panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
- __func__, ntohs(eh_type));
- break;
- }
-
- ctxd = txr->next_avail_desc;
- TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
-
- tcp_hlen = th->th_off << 2;
-
- /* This is used in the transmit desc in encap */
- paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
-
- /* VLAN MACLEN IPLEN */
- if (mp->m_flags & M_VLANTAG) {
- vtag = htole16(mp->m_pkthdr.ether_vtag);
- vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
- }
-
- vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
- vlan_macip_lens |= ip_hlen;
- TXD->vlan_macip_lens = htole32(vlan_macip_lens);
-
- /* ADV DTYPE TUCMD */
- type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
- type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
- TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
-
- /* MSS L4LEN IDX */
- mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
- mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
- /* 82575 needs the queue index added */
- if (adapter->hw.mac.type == e1000_82575)
- mss_l4len_idx |= txr->me << 4;
- TXD->mss_l4len_idx = htole32(mss_l4len_idx);
-
- TXD->seqnum_seed = htole32(0);
-
- if (++ctxd == txr->num_desc)
- ctxd = 0;
-
- txr->tx_avail--;
- txr->next_avail_desc = ctxd;
- *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
- *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
- *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
- ++txr->tso_tx;
- return (0);
-}
-
-/*********************************************************************
- *
- * Advanced Context Descriptor setup for VLAN, CSUM or TSO
- *
- **********************************************************************/
-
-static int
-igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
- u32 *cmd_type_len, u32 *olinfo_status)
-{
- struct e1000_adv_tx_context_desc *TXD;
- struct adapter *adapter = txr->adapter;
- struct ether_vlan_header *eh;
- struct ip *ip;
- struct ip6_hdr *ip6;
- u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
- int ehdrlen, ip_hlen = 0;
- u16 etype;
- u8 ipproto = 0;
- int offload = TRUE;
- int ctxd = txr->next_avail_desc;
- u16 vtag = 0;
-
- /* First check if TSO is to be used */
- if (mp->m_pkthdr.csum_flags & CSUM_TSO)
- return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status));
-
- if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
- offload = FALSE;
-
- /* Indicate the whole packet as payload when not doing TSO */
- *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
-
- /* Now ready a context descriptor */
- TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
-
- /*
- ** In advanced descriptors the vlan tag must
- ** be placed into the context descriptor. Hence
- ** we need to make one even if not doing offloads.
- */
- if (mp->m_flags & M_VLANTAG) {
- vtag = htole16(mp->m_pkthdr.ether_vtag);
- vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
- } else if (offload == FALSE) /* ... no offload to do */
- return (0);
-
- /*
- * Determine where frame payload starts.
- * Jump over vlan headers if already present,
- * helpful for QinQ too.
- */
- eh = mtod(mp, struct ether_vlan_header *);
- if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
- etype = ntohs(eh->evl_proto);
- ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
- } else {
- etype = ntohs(eh->evl_encap_proto);
- ehdrlen = ETHER_HDR_LEN;
- }
-
- /* Set the ether header length */
- vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
-
- switch (etype) {
- case ETHERTYPE_IP:
- ip = (struct ip *)(mp->m_data + ehdrlen);
- ip_hlen = ip->ip_hl << 2;
- ipproto = ip->ip_p;
- type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
- break;
- case ETHERTYPE_IPV6:
- ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
- ip_hlen = sizeof(struct ip6_hdr);
- /* XXX-BZ this will go badly in case of ext hdrs. */
- ipproto = ip6->ip6_nxt;
- type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
- break;
- default:
- offload = FALSE;
- break;
- }
-
- vlan_macip_lens |= ip_hlen;
- type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
-
- switch (ipproto) {
- case IPPROTO_TCP:
-#if __FreeBSD_version >= 1000000
- if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))
-#else
- if (mp->m_pkthdr.csum_flags & CSUM_TCP)
-#endif
- type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
- break;
- case IPPROTO_UDP:
-#if __FreeBSD_version >= 1000000
- if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP))
-#else
- if (mp->m_pkthdr.csum_flags & CSUM_UDP)
-#endif
- type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
- break;
-
-#if __FreeBSD_version >= 800000
- case IPPROTO_SCTP:
-#if __FreeBSD_version >= 1000000
- if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP))
-#else
- if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
-#endif
- type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
- break;
-#endif
- default:
- offload = FALSE;
- break;
- }
-
- if (offload) /* For the TX descriptor setup */
- *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
-
- /* 82575 needs the queue index added */
- if (adapter->hw.mac.type == e1000_82575)
- mss_l4len_idx = txr->me << 4;
-
- /* Now copy bits into descriptor */
- TXD->vlan_macip_lens = htole32(vlan_macip_lens);
- TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
- TXD->seqnum_seed = htole32(0);
- TXD->mss_l4len_idx = htole32(mss_l4len_idx);
-
- /* We've consumed the first desc, adjust counters */
- if (++ctxd == txr->num_desc)
- ctxd = 0;
- txr->next_avail_desc = ctxd;
- --txr->tx_avail;
-
- return (0);
-}
-
-/**********************************************************************
- *
- * Examine each tx_buffer in the used queue. If the hardware is done
- * processing the packet then free associated resources. The
- * tx_buffer is put back on the free queue.
- *
- * TRUE return means there's work in the ring to clean, FALSE its empty.
- **********************************************************************/
-static bool
-igb_txeof(struct tx_ring *txr)
-{
- struct adapter *adapter = txr->adapter;
-#ifdef DEV_NETMAP
- struct ifnet *ifp = adapter->ifp;
-#endif /* DEV_NETMAP */
- u32 work, processed = 0;
- int limit = adapter->tx_process_limit;
- struct igb_tx_buf *buf;
- union e1000_adv_tx_desc *txd;
-
- mtx_assert(&txr->tx_mtx, MA_OWNED);
-
-#ifdef DEV_NETMAP
- if (netmap_tx_irq(ifp, txr->me))
- return (FALSE);
-#endif /* DEV_NETMAP */
-
- if (txr->tx_avail == txr->num_desc) {
- txr->queue_status = IGB_QUEUE_IDLE;
- return FALSE;
- }
-
- /* Get work starting point */
- work = txr->next_to_clean;
- buf = &txr->tx_buffers[work];
- txd = &txr->tx_base[work];
- work -= txr->num_desc; /* The distance to ring end */
- bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
- BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
- do {
- union e1000_adv_tx_desc *eop = buf->eop;
- if (eop == NULL) /* No work */
- break;
-
- if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
- break; /* I/O not complete */
-
- if (buf->m_head) {
- txr->bytes +=
- buf->m_head->m_pkthdr.len;
- bus_dmamap_sync(txr->txtag,
- buf->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(txr->txtag,
- buf->map);
- m_freem(buf->m_head);
- buf->m_head = NULL;
- }
- buf->eop = NULL;
- ++txr->tx_avail;
-
- /* We clean the range if multi segment */
- while (txd != eop) {
- ++txd;
- ++buf;
- ++work;
- /* wrap the ring? */
- if (__predict_false(!work)) {
- work -= txr->num_desc;
- buf = txr->tx_buffers;
- txd = txr->tx_base;
- }
- if (buf->m_head) {
- txr->bytes +=
- buf->m_head->m_pkthdr.len;
- bus_dmamap_sync(txr->txtag,
- buf->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(txr->txtag,
- buf->map);
- m_freem(buf->m_head);
- buf->m_head = NULL;
- }
- ++txr->tx_avail;
- buf->eop = NULL;
-
- }
- ++txr->packets;
- ++processed;
- txr->watchdog_time = ticks;
-
- /* Try the next packet */
- ++txd;
- ++buf;
- ++work;
- /* reset with a wrap */
- if (__predict_false(!work)) {
- work -= txr->num_desc;
- buf = txr->tx_buffers;
- txd = txr->tx_base;
- }
- prefetch(txd);
- } while (__predict_true(--limit));
-
- bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
- work += txr->num_desc;
- txr->next_to_clean = work;
-
- /*
- ** Watchdog calculation, we know there's
- ** work outstanding or the first return
- ** would have been taken, so none processed
- ** for too long indicates a hang.
- */
- if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
- txr->queue_status |= IGB_QUEUE_HUNG;
-
- if (txr->tx_avail >= IGB_QUEUE_THRESHOLD)
- txr->queue_status &= ~IGB_QUEUE_DEPLETED;
-
- if (txr->tx_avail == txr->num_desc) {
- txr->queue_status = IGB_QUEUE_IDLE;
- return (FALSE);
- }
-
- return (TRUE);
-}
-
-/*********************************************************************
- *
- * Refresh mbuf buffers for RX descriptor rings
- * - now keeps its own state so discards due to resource
- * exhaustion are unnecessary, if an mbuf cannot be obtained
- * it just returns, keeping its placeholder, thus it can simply
- * be recalled to try again.
- *
- **********************************************************************/
-static void
-igb_refresh_mbufs(struct rx_ring *rxr, int limit)
-{
- struct adapter *adapter = rxr->adapter;
- bus_dma_segment_t hseg[1];
- bus_dma_segment_t pseg[1];
- struct igb_rx_buf *rxbuf;
- struct mbuf *mh, *mp;
- int i, j, nsegs, error;
- bool refreshed = FALSE;
-
- i = j = rxr->next_to_refresh;
- /*
- ** Get one descriptor beyond
- ** our work mark to control
- ** the loop.
- */
- if (++j == adapter->num_rx_desc)
- j = 0;
-
- while (j != limit) {
- rxbuf = &rxr->rx_buffers[i];
- /* No hdr mbuf used with header split off */
- if (rxr->hdr_split == FALSE)
- goto no_split;
- if (rxbuf->m_head == NULL) {
- mh = m_gethdr(M_NOWAIT, MT_DATA);
- if (mh == NULL)
- goto update;
- } else
- mh = rxbuf->m_head;
-
- mh->m_pkthdr.len = mh->m_len = MHLEN;
- mh->m_len = MHLEN;
- mh->m_flags |= M_PKTHDR;
- /* Get the memory mapping */
- error = bus_dmamap_load_mbuf_sg(rxr->htag,
- rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
- if (error != 0) {
- printf("Refresh mbufs: hdr dmamap load"
- " failure - %d\n", error);
- m_free(mh);
- rxbuf->m_head = NULL;
- goto update;
- }
- rxbuf->m_head = mh;
- bus_dmamap_sync(rxr->htag, rxbuf->hmap,
- BUS_DMASYNC_PREREAD);
- rxr->rx_base[i].read.hdr_addr =
- htole64(hseg[0].ds_addr);
-no_split:
- if (rxbuf->m_pack == NULL) {
- mp = m_getjcl(M_NOWAIT, MT_DATA,
- M_PKTHDR, adapter->rx_mbuf_sz);
- if (mp == NULL)
- goto update;
- } else
- mp = rxbuf->m_pack;
-
- mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
- /* Get the memory mapping */
- error = bus_dmamap_load_mbuf_sg(rxr->ptag,
- rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
- if (error != 0) {
- printf("Refresh mbufs: payload dmamap load"
- " failure - %d\n", error);
- m_free(mp);
- rxbuf->m_pack = NULL;
- goto update;
- }
- rxbuf->m_pack = mp;
- bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
- BUS_DMASYNC_PREREAD);
- rxr->rx_base[i].read.pkt_addr =
- htole64(pseg[0].ds_addr);
- refreshed = TRUE; /* I feel wefreshed :) */
-
- i = j; /* our next is precalculated */
- rxr->next_to_refresh = i;
- if (++j == adapter->num_rx_desc)
- j = 0;
- }
-update:
- if (refreshed) /* update tail */
- E1000_WRITE_REG(&adapter->hw,
- E1000_RDT(rxr->me), rxr->next_to_refresh);
- return;
-}
-
-
-/*********************************************************************
- *
- * Allocate memory for rx_buffer structures. Since we use one
- * rx_buffer per received packet, the maximum number of rx_buffer's
- * that we'll need is equal to the number of receive descriptors
- * that we've allocated.
- *
- **********************************************************************/
-static int
-igb_allocate_receive_buffers(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
- device_t dev = adapter->dev;
- struct igb_rx_buf *rxbuf;
- int i, bsize, error;
-
- bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
- if (!(rxr->rx_buffers =
- (struct igb_rx_buf *) malloc(bsize,
- M_DEVBUF, M_NOWAIT | M_ZERO))) {
- device_printf(dev, "Unable to allocate rx_buffer memory\n");
- error = ENOMEM;
- goto fail;
- }
-
- if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
- 1, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- MSIZE, /* maxsize */
- 1, /* nsegments */
- MSIZE, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockfuncarg */
- &rxr->htag))) {
- device_printf(dev, "Unable to create RX DMA tag\n");
- goto fail;
- }
-
- if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
- 1, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- MJUM9BYTES, /* maxsize */
- 1, /* nsegments */
- MJUM9BYTES, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockfuncarg */
- &rxr->ptag))) {
- device_printf(dev, "Unable to create RX payload DMA tag\n");
- goto fail;
- }
-
- for (i = 0; i < adapter->num_rx_desc; i++) {
- rxbuf = &rxr->rx_buffers[i];
- error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
- if (error) {
- device_printf(dev,
- "Unable to create RX head DMA maps\n");
- goto fail;
- }
- error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
- if (error) {
- device_printf(dev,
- "Unable to create RX packet DMA maps\n");
- goto fail;
- }
- }
-
- return (0);
-
-fail:
- /* Frees all, but can handle partial completion */
- igb_free_receive_structures(adapter);
- return (error);
-}
-
-
-static void
-igb_free_receive_ring(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
- struct igb_rx_buf *rxbuf;
-
-
- for (int i = 0; i < adapter->num_rx_desc; i++) {
- rxbuf = &rxr->rx_buffers[i];
- if (rxbuf->m_head != NULL) {
- bus_dmamap_sync(rxr->htag, rxbuf->hmap,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(rxr->htag, rxbuf->hmap);
- rxbuf->m_head->m_flags |= M_PKTHDR;
- m_freem(rxbuf->m_head);
- }
- if (rxbuf->m_pack != NULL) {
- bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
- rxbuf->m_pack->m_flags |= M_PKTHDR;
- m_freem(rxbuf->m_pack);
- }
- rxbuf->m_head = NULL;
- rxbuf->m_pack = NULL;
- }
-}
-
-
-/*********************************************************************
- *
- * Initialize a receive ring and its buffers.
- *
- **********************************************************************/
-static int
-igb_setup_receive_ring(struct rx_ring *rxr)
-{
- struct adapter *adapter;
- struct ifnet *ifp;
- device_t dev;
- struct igb_rx_buf *rxbuf;
- bus_dma_segment_t pseg[1], hseg[1];
- struct lro_ctrl *lro = &rxr->lro;
- int rsize, nsegs, error = 0;
-#ifdef DEV_NETMAP
- struct netmap_adapter *na = NA(rxr->adapter->ifp);
- struct netmap_slot *slot;
-#endif /* DEV_NETMAP */
-
- adapter = rxr->adapter;
- dev = adapter->dev;
- ifp = adapter->ifp;
-
- /* Clear the ring contents */
- IGB_RX_LOCK(rxr);
-#ifdef DEV_NETMAP
- slot = netmap_reset(na, NR_RX, rxr->me, 0);
-#endif /* DEV_NETMAP */
- rsize = roundup2(adapter->num_rx_desc *
- sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
- bzero((void *)rxr->rx_base, rsize);
-
- /*
- ** Free current RX buffer structures and their mbufs
- */
- igb_free_receive_ring(rxr);
-
- /* Configure for header split? */
- if (igb_header_split)
- rxr->hdr_split = TRUE;
-
- /* Now replenish the ring mbufs */
- for (int j = 0; j < adapter->num_rx_desc; ++j) {
- struct mbuf *mh, *mp;
-
- rxbuf = &rxr->rx_buffers[j];
-#ifdef DEV_NETMAP
- if (slot) {
- /* slot sj is mapped to the j-th NIC-ring entry */
- int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
- uint64_t paddr;
- void *addr;
-
- addr = PNMB(na, slot + sj, &paddr);
- netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
- /* Update descriptor */
- rxr->rx_base[j].read.pkt_addr = htole64(paddr);
- continue;
- }
-#endif /* DEV_NETMAP */
- if (rxr->hdr_split == FALSE)
- goto skip_head;
-
- /* First the header */
- rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
- if (rxbuf->m_head == NULL) {
- error = ENOBUFS;
- goto fail;
- }
- m_adj(rxbuf->m_head, ETHER_ALIGN);
- mh = rxbuf->m_head;
- mh->m_len = mh->m_pkthdr.len = MHLEN;
- mh->m_flags |= M_PKTHDR;
- /* Get the memory mapping */
- error = bus_dmamap_load_mbuf_sg(rxr->htag,
- rxbuf->hmap, rxbuf->m_head, hseg,
- &nsegs, BUS_DMA_NOWAIT);
- if (error != 0) /* Nothing elegant to do here */
- goto fail;
- bus_dmamap_sync(rxr->htag,
- rxbuf->hmap, BUS_DMASYNC_PREREAD);
- /* Update descriptor */
- rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
-
-skip_head:
- /* Now the payload cluster */
- rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
- M_PKTHDR, adapter->rx_mbuf_sz);
- if (rxbuf->m_pack == NULL) {
- error = ENOBUFS;
- goto fail;
- }
- mp = rxbuf->m_pack;
- mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
- /* Get the memory mapping */
- error = bus_dmamap_load_mbuf_sg(rxr->ptag,
- rxbuf->pmap, mp, pseg,
- &nsegs, BUS_DMA_NOWAIT);
- if (error != 0)
- goto fail;
- bus_dmamap_sync(rxr->ptag,
- rxbuf->pmap, BUS_DMASYNC_PREREAD);
- /* Update descriptor */
- rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
- }
-
- /* Setup our descriptor indices */
- rxr->next_to_check = 0;
- rxr->next_to_refresh = adapter->num_rx_desc - 1;
- rxr->lro_enabled = FALSE;
- rxr->rx_split_packets = 0;
- rxr->rx_bytes = 0;
-
- rxr->fmp = NULL;
- rxr->lmp = NULL;
-
- bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
- /*
- ** Now set up the LRO interface, we
- ** also only do head split when LRO
- ** is enabled, since so often they
- ** are undesirable in similar setups.
- */
- if (ifp->if_capenable & IFCAP_LRO) {
- error = tcp_lro_init(lro);
- if (error) {
- device_printf(dev, "LRO Initialization failed!\n");
- goto fail;
- }
- INIT_DEBUGOUT("RX LRO Initialized\n");
- rxr->lro_enabled = TRUE;
- lro->ifp = adapter->ifp;
- }
-
- IGB_RX_UNLOCK(rxr);
- return (0);
-
-fail:
- igb_free_receive_ring(rxr);
- IGB_RX_UNLOCK(rxr);
- return (error);
-}
-
-
-/*********************************************************************
- *
- * Initialize all receive rings.
- *
- **********************************************************************/
-static int
-igb_setup_receive_structures(struct adapter *adapter)
-{
- struct rx_ring *rxr = adapter->rx_rings;
- int i;
-
- for (i = 0; i < adapter->num_queues; i++, rxr++)
- if (igb_setup_receive_ring(rxr))
- goto fail;
-
- return (0);
-fail:
- /*
- * Free RX buffers allocated so far, we will only handle
- * the rings that completed, the failing case will have
- * cleaned up for itself. 'i' is the endpoint.
- */
- for (int j = 0; j < i; ++j) {
- rxr = &adapter->rx_rings[j];
- IGB_RX_LOCK(rxr);
- igb_free_receive_ring(rxr);
- IGB_RX_UNLOCK(rxr);
- }
-
- return (ENOBUFS);
-}
-
-/*
- * Initialise the RSS mapping for NICs that support multiple transmit/
- * receive rings.
- */
-static void
-igb_initialise_rss_mapping(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- int i;
- int queue_id;
- u32 reta;
- u32 rss_key[10], mrqc, shift = 0;
-
- /* XXX? */
- if (adapter->hw.mac.type == e1000_82575)
- shift = 6;
-
- /*
- * The redirection table controls which destination
- * queue each bucket redirects traffic to.
- * Each DWORD represents four queues, with the LSB
- * being the first queue in the DWORD.
- *
- * This just allocates buckets to queues using round-robin
- * allocation.
- *
- * NOTE: It Just Happens to line up with the default
- * RSS allocation method.
- */
-
- /* Warning FM follows */
- reta = 0;
- for (i = 0; i < 128; i++) {
-#ifdef RSS
- queue_id = rss_get_indirection_to_bucket(i);
- /*
- * If we have more queues than buckets, we'll
- * end up mapping buckets to a subset of the
- * queues.
- *
- * If we have more buckets than queues, we'll
- * end up instead assigning multiple buckets
- * to queues.
- *
- * Both are suboptimal, but we need to handle
- * the case so we don't go out of bounds
- * indexing arrays and such.
- */
- queue_id = queue_id % adapter->num_queues;
-#else
- queue_id = (i % adapter->num_queues);
-#endif
- /* Adjust if required */
- queue_id = queue_id << shift;
-
- /*
- * The low 8 bits are for hash value (n+0);
- * The next 8 bits are for hash value (n+1), etc.
- */
- reta = reta >> 8;
- reta = reta | ( ((uint32_t) queue_id) << 24);
- if ((i & 3) == 3) {
- E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
- reta = 0;
- }
- }
-
- /* Now fill in hash table */
-
- /*
- * MRQC: Multiple Receive Queues Command
- * Set queuing to RSS control, number depends on the device.
- */
- mrqc = E1000_MRQC_ENABLE_RSS_8Q;
-
-#ifdef RSS
- /* XXX ew typecasting */
- rss_getkey((uint8_t *) &rss_key);
-#else
- arc4rand(&rss_key, sizeof(rss_key), 0);
-#endif
- for (i = 0; i < 10; i++)
- E1000_WRITE_REG_ARRAY(hw,
- E1000_RSSRK(0), i, rss_key[i]);
-
- /*
- * Configure the RSS fields to hash upon.
- */
- mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
- E1000_MRQC_RSS_FIELD_IPV4_TCP);
- mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
- E1000_MRQC_RSS_FIELD_IPV6_TCP);
- mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
- E1000_MRQC_RSS_FIELD_IPV6_UDP);
- mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
- E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
-
- E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
-}
-
-/*********************************************************************
- *
- * Enable receive unit.
- *
- **********************************************************************/
-static void
-igb_initialize_receive_units(struct adapter *adapter)
-{
- struct rx_ring *rxr = adapter->rx_rings;
- struct ifnet *ifp = adapter->ifp;
- struct e1000_hw *hw = &adapter->hw;
- u32 rctl, rxcsum, psize, srrctl = 0;
-
- INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
-
- /*
- * Make sure receives are disabled while setting
- * up the descriptor ring
- */
- rctl = E1000_READ_REG(hw, E1000_RCTL);
- E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
-
- /*
- ** Set up for header split
- */
- if (igb_header_split) {
- /* Use a standard mbuf for the header */
- srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
- srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
- } else
- srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
-
- /*
- ** Set up for jumbo frames
- */
- if (ifp->if_mtu > ETHERMTU) {
- rctl |= E1000_RCTL_LPE;
- if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
- srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
- rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
- } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
- srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
- rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
- }
- /* Set maximum packet len */
- psize = adapter->max_frame_size;
- /* are we on a vlan? */
- if (adapter->ifp->if_vlantrunk != NULL)
- psize += VLAN_TAG_SIZE;
- E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
- } else {
- rctl &= ~E1000_RCTL_LPE;
- srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
- rctl |= E1000_RCTL_SZ_2048;
- }
-
- /*
- * If TX flow control is disabled and there's >1 queue defined,
- * enable DROP.
- *
- * This drops frames rather than hanging the RX MAC for all queues.
- */
- if ((adapter->num_queues > 1) &&
- (adapter->fc == e1000_fc_none ||
- adapter->fc == e1000_fc_rx_pause)) {
- srrctl |= E1000_SRRCTL_DROP_EN;
- }
-
- /* Setup the Base and Length of the Rx Descriptor Rings */
- for (int i = 0; i < adapter->num_queues; i++, rxr++) {
- u64 bus_addr = rxr->rxdma.dma_paddr;
- u32 rxdctl;
-
- E1000_WRITE_REG(hw, E1000_RDLEN(i),
- adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
- E1000_WRITE_REG(hw, E1000_RDBAH(i),
- (uint32_t)(bus_addr >> 32));
- E1000_WRITE_REG(hw, E1000_RDBAL(i),
- (uint32_t)bus_addr);
- E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
- /* Enable this Queue */
- rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
- rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
- rxdctl &= 0xFFF00000;
- rxdctl |= IGB_RX_PTHRESH;
- rxdctl |= IGB_RX_HTHRESH << 8;
- rxdctl |= IGB_RX_WTHRESH << 16;
- E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
- }
-
- /*
- ** Setup for RX MultiQueue
- */
- rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
- if (adapter->num_queues >1) {
-
- /* rss setup */
- igb_initialise_rss_mapping(adapter);
-
- /*
- ** NOTE: Receive Full-Packet Checksum Offload
- ** is mutually exclusive with Multiqueue. However
- ** this is not the same as TCP/IP checksums which
- ** still work.
- */
- rxcsum |= E1000_RXCSUM_PCSD;
-#if __FreeBSD_version >= 800000
- /* For SCTP Offload */
- if ((hw->mac.type != e1000_82575) &&
- (ifp->if_capenable & IFCAP_RXCSUM))
- rxcsum |= E1000_RXCSUM_CRCOFL;
-#endif
- } else {
- /* Non RSS setup */
- if (ifp->if_capenable & IFCAP_RXCSUM) {
- rxcsum |= E1000_RXCSUM_IPPCSE;
-#if __FreeBSD_version >= 800000
- if (adapter->hw.mac.type != e1000_82575)
- rxcsum |= E1000_RXCSUM_CRCOFL;
-#endif
- } else
- rxcsum &= ~E1000_RXCSUM_TUOFL;
- }
- E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
-
- /* Setup the Receive Control Register */
- rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
- rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
- E1000_RCTL_RDMTS_HALF |
- (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
- /* Strip CRC bytes. */
- rctl |= E1000_RCTL_SECRC;
- /* Make sure VLAN Filters are off */
- rctl &= ~E1000_RCTL_VFE;
- /* Don't store bad packets */
- rctl &= ~E1000_RCTL_SBP;
-
- /* Enable Receives */
- E1000_WRITE_REG(hw, E1000_RCTL, rctl);
-
- /*
- * Setup the HW Rx Head and Tail Descriptor Pointers
- * - needs to be after enable
- */
- for (int i = 0; i < adapter->num_queues; i++) {
- rxr = &adapter->rx_rings[i];
- E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
-#ifdef DEV_NETMAP
- /*
- * an init() while a netmap client is active must
- * preserve the rx buffers passed to userspace.
- * In this driver it means we adjust RDT to
- * something different from next_to_refresh
- * (which is not used in netmap mode).
- */
- if (ifp->if_capenable & IFCAP_NETMAP) {
- struct netmap_adapter *na = NA(adapter->ifp);
- struct netmap_kring *kring = &na->rx_rings[i];
- int t = rxr->next_to_refresh - nm_kr_rxspace(kring);
-
- if (t >= adapter->num_rx_desc)
- t -= adapter->num_rx_desc;
- else if (t < 0)
- t += adapter->num_rx_desc;
- E1000_WRITE_REG(hw, E1000_RDT(i), t);
- } else
-#endif /* DEV_NETMAP */
- E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
- }
- return;
-}
-
-/*********************************************************************
- *
- * Free receive rings.
- *
- **********************************************************************/
-static void
-igb_free_receive_structures(struct adapter *adapter)
-{
- struct rx_ring *rxr = adapter->rx_rings;
-
- for (int i = 0; i < adapter->num_queues; i++, rxr++) {
- struct lro_ctrl *lro = &rxr->lro;
- igb_free_receive_buffers(rxr);
- tcp_lro_free(lro);
- igb_dma_free(adapter, &rxr->rxdma);
- }
-
- free(adapter->rx_rings, M_DEVBUF);
-}
-
-/*********************************************************************
- *
- * Free receive ring data structures.
- *
- **********************************************************************/
-static void
-igb_free_receive_buffers(struct rx_ring *rxr)
-{
- struct adapter *adapter = rxr->adapter;
- struct igb_rx_buf *rxbuf;
- int i;
-
- INIT_DEBUGOUT("free_receive_structures: begin");
-
- /* Cleanup any existing buffers */
- if (rxr->rx_buffers != NULL) {
- for (i = 0; i < adapter->num_rx_desc; i++) {
- rxbuf = &rxr->rx_buffers[i];
- if (rxbuf->m_head != NULL) {
- bus_dmamap_sync(rxr->htag, rxbuf->hmap,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(rxr->htag, rxbuf->hmap);
- rxbuf->m_head->m_flags |= M_PKTHDR;
- m_freem(rxbuf->m_head);
- }
- if (rxbuf->m_pack != NULL) {
- bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
- rxbuf->m_pack->m_flags |= M_PKTHDR;
- m_freem(rxbuf->m_pack);
- }
- rxbuf->m_head = NULL;
- rxbuf->m_pack = NULL;
- if (rxbuf->hmap != NULL) {
- bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
- rxbuf->hmap = NULL;
- }
- if (rxbuf->pmap != NULL) {
- bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
- rxbuf->pmap = NULL;
- }
- }
- if (rxr->rx_buffers != NULL) {
- free(rxr->rx_buffers, M_DEVBUF);
- rxr->rx_buffers = NULL;
- }
- }
-
- if (rxr->htag != NULL) {
- bus_dma_tag_destroy(rxr->htag);
- rxr->htag = NULL;
- }
- if (rxr->ptag != NULL) {
- bus_dma_tag_destroy(rxr->ptag);
- rxr->ptag = NULL;
- }
-}
-
-static __inline void
-igb_rx_discard(struct rx_ring *rxr, int i)
-{
- struct igb_rx_buf *rbuf;
-
- rbuf = &rxr->rx_buffers[i];
-
- /* Partially received? Free the chain */
- if (rxr->fmp != NULL) {
- rxr->fmp->m_flags |= M_PKTHDR;
- m_freem(rxr->fmp);
- rxr->fmp = NULL;
- rxr->lmp = NULL;
- }
-
- /*
- ** With advanced descriptors the writeback
- ** clobbers the buffer addrs, so its easier
- ** to just free the existing mbufs and take
- ** the normal refresh path to get new buffers
- ** and mapping.
- */
- if (rbuf->m_head) {
- m_free(rbuf->m_head);
- rbuf->m_head = NULL;
- bus_dmamap_unload(rxr->htag, rbuf->hmap);
- }
-
- if (rbuf->m_pack) {
- m_free(rbuf->m_pack);
- rbuf->m_pack = NULL;
- bus_dmamap_unload(rxr->ptag, rbuf->pmap);
- }
-
- return;
-}
-
-static __inline void
-igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
-{
-
- /*
- * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
- * should be computed by hardware. Also it should not have VLAN tag in
- * ethernet header.
- */
- if (rxr->lro_enabled &&
- (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
- (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
- (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
- (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
- (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
- (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
- /*
- * Send to the stack if:
- ** - LRO not enabled, or
- ** - no LRO resources, or
- ** - lro enqueue fails
- */
- if (rxr->lro.lro_cnt != 0)
- if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
- return;
- }
- IGB_RX_UNLOCK(rxr);
- (*ifp->if_input)(ifp, m);
- IGB_RX_LOCK(rxr);
-}
-
-/*********************************************************************
- *
- * This routine executes in interrupt context. It replenishes
- * the mbufs in the descriptor and sends data which has been
- * dma'ed into host memory to upper layer.
- *
- * We loop at most count times if count is > 0, or until done if
- * count < 0.
- *
- * Return TRUE if more to clean, FALSE otherwise
- *********************************************************************/
-static bool
-igb_rxeof(struct igb_queue *que, int count, int *done)
-{
- struct adapter *adapter = que->adapter;
- struct rx_ring *rxr = que->rxr;
- struct ifnet *ifp = adapter->ifp;
- struct lro_ctrl *lro = &rxr->lro;
- int i, processed = 0, rxdone = 0;
- u32 ptype, staterr = 0;
- union e1000_adv_rx_desc *cur;
-
- IGB_RX_LOCK(rxr);
- /* Sync the ring. */
- bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
- BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
-
-#ifdef DEV_NETMAP
- if (netmap_rx_irq(ifp, rxr->me, &processed)) {
- IGB_RX_UNLOCK(rxr);
- return (FALSE);
- }
-#endif /* DEV_NETMAP */
-
- /* Main clean loop */
- for (i = rxr->next_to_check; count != 0;) {
- struct mbuf *sendmp, *mh, *mp;
- struct igb_rx_buf *rxbuf;
- u16 hlen, plen, hdr, vtag, pkt_info;
- bool eop = FALSE;
-
- cur = &rxr->rx_base[i];
- staterr = le32toh(cur->wb.upper.status_error);
- if ((staterr & E1000_RXD_STAT_DD) == 0)
- break;
- if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
- break;
- count--;
- sendmp = mh = mp = NULL;
- cur->wb.upper.status_error = 0;
- rxbuf = &rxr->rx_buffers[i];
- plen = le16toh(cur->wb.upper.length);
- ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
- if (((adapter->hw.mac.type == e1000_i350) ||
- (adapter->hw.mac.type == e1000_i354)) &&
- (staterr & E1000_RXDEXT_STATERR_LB))
- vtag = be16toh(cur->wb.upper.vlan);
- else
- vtag = le16toh(cur->wb.upper.vlan);
- hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
- pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info);
- eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
-
- /*
- * Free the frame (all segments) if we're at EOP and
- * it's an error.
- *
- * The datasheet states that EOP + status is only valid for
- * the final segment in a multi-segment frame.
- */
- if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) {
- adapter->dropped_pkts++;
- ++rxr->rx_discarded;
- igb_rx_discard(rxr, i);
- goto next_desc;
- }
-
- /*
- ** The way the hardware is configured to
- ** split, it will ONLY use the header buffer
- ** when header split is enabled, otherwise we
- ** get normal behavior, ie, both header and
- ** payload are DMA'd into the payload buffer.
- **
- ** The fmp test is to catch the case where a
- ** packet spans multiple descriptors, in that
- ** case only the first header is valid.
- */
- if (rxr->hdr_split && rxr->fmp == NULL) {
- bus_dmamap_unload(rxr->htag, rxbuf->hmap);
- hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
- E1000_RXDADV_HDRBUFLEN_SHIFT;
- if (hlen > IGB_HDR_BUF)
- hlen = IGB_HDR_BUF;
- mh = rxr->rx_buffers[i].m_head;
- mh->m_len = hlen;
- /* clear buf pointer for refresh */
- rxbuf->m_head = NULL;
- /*
- ** Get the payload length, this
- ** could be zero if its a small
- ** packet.
- */
- if (plen > 0) {
- mp = rxr->rx_buffers[i].m_pack;
- mp->m_len = plen;
- mh->m_next = mp;
- /* clear buf pointer */
- rxbuf->m_pack = NULL;
- rxr->rx_split_packets++;
- }
- } else {
- /*
- ** Either no header split, or a
- ** secondary piece of a fragmented
- ** split packet.
- */
- mh = rxr->rx_buffers[i].m_pack;
- mh->m_len = plen;
- /* clear buf info for refresh */
- rxbuf->m_pack = NULL;
- }
- bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
-
- ++processed; /* So we know when to refresh */
-
- /* Initial frame - setup */
- if (rxr->fmp == NULL) {
- mh->m_pkthdr.len = mh->m_len;
- /* Save the head of the chain */
- rxr->fmp = mh;
- rxr->lmp = mh;
- if (mp != NULL) {
- /* Add payload if split */
- mh->m_pkthdr.len += mp->m_len;
- rxr->lmp = mh->m_next;
- }
- } else {
- /* Chain mbuf's together */
- rxr->lmp->m_next = mh;
- rxr->lmp = rxr->lmp->m_next;
- rxr->fmp->m_pkthdr.len += mh->m_len;
- }
-
- if (eop) {
- rxr->fmp->m_pkthdr.rcvif = ifp;
- rxr->rx_packets++;
- /* capture data for AIM */
- rxr->packets++;
- rxr->bytes += rxr->fmp->m_pkthdr.len;
- rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
-
- if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
- igb_rx_checksum(staterr, rxr->fmp, ptype);
-
- if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
- (staterr & E1000_RXD_STAT_VP) != 0) {
- rxr->fmp->m_pkthdr.ether_vtag = vtag;
- rxr->fmp->m_flags |= M_VLANTAG;
- }
-
- /*
- * In case of multiqueue, we have RXCSUM.PCSD bit set
- * and never cleared. This means we have RSS hash
- * available to be used.
- */
- if (adapter->num_queues > 1) {
- rxr->fmp->m_pkthdr.flowid =
- le32toh(cur->wb.lower.hi_dword.rss);
- switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
- case E1000_RXDADV_RSSTYPE_IPV4_TCP:
- M_HASHTYPE_SET(rxr->fmp,
- M_HASHTYPE_RSS_TCP_IPV4);
- break;
- case E1000_RXDADV_RSSTYPE_IPV4:
- M_HASHTYPE_SET(rxr->fmp,
- M_HASHTYPE_RSS_IPV4);
- break;
- case E1000_RXDADV_RSSTYPE_IPV6_TCP:
- M_HASHTYPE_SET(rxr->fmp,
- M_HASHTYPE_RSS_TCP_IPV6);
- break;
- case E1000_RXDADV_RSSTYPE_IPV6_EX:
- M_HASHTYPE_SET(rxr->fmp,
- M_HASHTYPE_RSS_IPV6_EX);
- break;
- case E1000_RXDADV_RSSTYPE_IPV6:
- M_HASHTYPE_SET(rxr->fmp,
- M_HASHTYPE_RSS_IPV6);
- break;
- case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
- M_HASHTYPE_SET(rxr->fmp,
- M_HASHTYPE_RSS_TCP_IPV6_EX);
- break;
- default:
- /* XXX fallthrough */
- M_HASHTYPE_SET(rxr->fmp,
- M_HASHTYPE_OPAQUE_HASH);
- }
- } else {
-#ifndef IGB_LEGACY_TX
- rxr->fmp->m_pkthdr.flowid = que->msix;
- M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE);
-#endif
- }
- sendmp = rxr->fmp;
- /* Make sure to set M_PKTHDR. */
- sendmp->m_flags |= M_PKTHDR;
- rxr->fmp = NULL;
- rxr->lmp = NULL;
- }
-
-next_desc:
- bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
- /* Advance our pointers to the next descriptor. */
- if (++i == adapter->num_rx_desc)
- i = 0;
- /*
- ** Send to the stack or LRO
- */
- if (sendmp != NULL) {
- rxr->next_to_check = i;
- igb_rx_input(rxr, ifp, sendmp, ptype);
- i = rxr->next_to_check;
- rxdone++;
- }
-
- /* Every 8 descriptors we go to refresh mbufs */
- if (processed == 8) {
- igb_refresh_mbufs(rxr, i);
- processed = 0;
- }
- }
-
- /* Catch any remainders */
- if (igb_rx_unrefreshed(rxr))
- igb_refresh_mbufs(rxr, i);
-
- rxr->next_to_check = i;
-
- /*
- * Flush any outstanding LRO work
- */
- tcp_lro_flush_all(lro);
-
- if (done != NULL)
- *done += rxdone;
-
- IGB_RX_UNLOCK(rxr);
- return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
-}
-
-/*********************************************************************
- *
- * Verify that the hardware indicated that the checksum is valid.
- * Inform the stack about the status of checksum so that stack
- * doesn't spend time verifying the checksum.
- *
- *********************************************************************/
-static void
-igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
-{
- u16 status = (u16)staterr;
- u8 errors = (u8) (staterr >> 24);
- int sctp;
-
- /* Ignore Checksum bit is set */
- if (status & E1000_RXD_STAT_IXSM) {
- mp->m_pkthdr.csum_flags = 0;
- return;
- }
-
- if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
- (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
- sctp = 1;
- else
- sctp = 0;
- if (status & E1000_RXD_STAT_IPCS) {
- /* Did it pass? */
- if (!(errors & E1000_RXD_ERR_IPE)) {
- /* IP Checksum Good */
- mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
- mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
- } else
- mp->m_pkthdr.csum_flags = 0;
- }
-
- if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
- u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
-#if __FreeBSD_version >= 800000
- if (sctp) /* reassign */
- type = CSUM_SCTP_VALID;
-#endif
- /* Did it pass? */
- if (!(errors & E1000_RXD_ERR_TCPE)) {
- mp->m_pkthdr.csum_flags |= type;
- if (sctp == 0)
- mp->m_pkthdr.csum_data = htons(0xffff);
- }
- }
- return;
-}
-
-/*
- * This routine is run via an vlan
- * config EVENT
- */
-static void
-igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
-{
- struct adapter *adapter = ifp->if_softc;
- u32 index, bit;
-
- if (ifp->if_softc != arg) /* Not our event */
- return;
-
- if ((vtag == 0) || (vtag > 4095)) /* Invalid */
- return;
-
- IGB_CORE_LOCK(adapter);
- index = (vtag >> 5) & 0x7F;
- bit = vtag & 0x1F;
- adapter->shadow_vfta[index] |= (1 << bit);
- ++adapter->num_vlans;
- /* Change hw filter setting */
- if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
- igb_setup_vlan_hw_support(adapter);
- IGB_CORE_UNLOCK(adapter);
-}
-
-/*
- * This routine is run via an vlan
- * unconfig EVENT
- */
-static void
-igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
-{
- struct adapter *adapter = ifp->if_softc;
- u32 index, bit;
-
- if (ifp->if_softc != arg)
- return;
-
- if ((vtag == 0) || (vtag > 4095)) /* Invalid */
- return;
-
- IGB_CORE_LOCK(adapter);
- index = (vtag >> 5) & 0x7F;
- bit = vtag & 0x1F;
- adapter->shadow_vfta[index] &= ~(1 << bit);
- --adapter->num_vlans;
- /* Change hw filter setting */
- if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
- igb_setup_vlan_hw_support(adapter);
- IGB_CORE_UNLOCK(adapter);
-}
-
-static void
-igb_setup_vlan_hw_support(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- struct ifnet *ifp = adapter->ifp;
- u32 reg;
-
- if (adapter->vf_ifp) {
- e1000_rlpml_set_vf(hw,
- adapter->max_frame_size + VLAN_TAG_SIZE);
- return;
- }
-
- reg = E1000_READ_REG(hw, E1000_CTRL);
- reg |= E1000_CTRL_VME;
- E1000_WRITE_REG(hw, E1000_CTRL, reg);
-
- /* Enable the Filter Table */
- if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
- reg = E1000_READ_REG(hw, E1000_RCTL);
- reg &= ~E1000_RCTL_CFIEN;
- reg |= E1000_RCTL_VFE;
- E1000_WRITE_REG(hw, E1000_RCTL, reg);
- }
-
- /* Update the frame size */
- E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
- adapter->max_frame_size + VLAN_TAG_SIZE);
-
- /* Don't bother with table if no vlans */
- if ((adapter->num_vlans == 0) ||
- ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
- return;
- /*
- ** A soft reset zero's out the VFTA, so
- ** we need to repopulate it now.
- */
- for (int i = 0; i < IGB_VFTA_SIZE; i++)
- if (adapter->shadow_vfta[i] != 0) {
- if (adapter->vf_ifp)
- e1000_vfta_set_vf(hw,
- adapter->shadow_vfta[i], TRUE);
- else
- e1000_write_vfta(hw,
- i, adapter->shadow_vfta[i]);
- }
-}
-
-static void
-igb_enable_intr(struct adapter *adapter)
-{
- /* With RSS set up what to auto clear */
- if (adapter->msix_mem) {
- u32 mask = (adapter->que_mask | adapter->link_mask);
- E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
- E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
- E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
- E1000_WRITE_REG(&adapter->hw, E1000_IMS,
- E1000_IMS_LSC);
- } else {
- E1000_WRITE_REG(&adapter->hw, E1000_IMS,
- IMS_ENABLE_MASK);
- }
- E1000_WRITE_FLUSH(&adapter->hw);
-
- return;
-}
-
-static void
-igb_disable_intr(struct adapter *adapter)
-{
- if (adapter->msix_mem) {
- E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
- E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
- }
- E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
- E1000_WRITE_FLUSH(&adapter->hw);
- return;
-}
-
-/*
- * Bit of a misnomer, what this really means is
- * to enable OS management of the system... aka
- * to disable special hardware management features
- */
-static void
-igb_init_manageability(struct adapter *adapter)
-{
- if (adapter->has_manage) {
- int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
- int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
-
- /* disable hardware interception of ARP */
- manc &= ~(E1000_MANC_ARP_EN);
-
- /* enable receiving management packets to the host */
- manc |= E1000_MANC_EN_MNG2HOST;
- manc2h |= 1 << 5; /* Mng Port 623 */
- manc2h |= 1 << 6; /* Mng Port 664 */
- E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
- E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
- }
-}
-
-/*
- * Give control back to hardware management
- * controller if there is one.
- */
-static void
-igb_release_manageability(struct adapter *adapter)
-{
- if (adapter->has_manage) {
- int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
-
- /* re-enable hardware interception of ARP */
- manc |= E1000_MANC_ARP_EN;
- manc &= ~E1000_MANC_EN_MNG2HOST;
-
- E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
- }
-}
-
-/*
- * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
- * For ASF and Pass Through versions of f/w this means that
- * the driver is loaded.
- *
- */
-static void
-igb_get_hw_control(struct adapter *adapter)
-{
- u32 ctrl_ext;
-
- if (adapter->vf_ifp)
- return;
-
- /* Let firmware know the driver has taken over */
- ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
- E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
- ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
-}
-
-/*
- * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
- * For ASF and Pass Through versions of f/w this means that the
- * driver is no longer loaded.
- *
- */
-static void
-igb_release_hw_control(struct adapter *adapter)
-{
- u32 ctrl_ext;
-
- if (adapter->vf_ifp)
- return;
-
- /* Let firmware taken over control of h/w */
- ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
- E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
- ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
-}
-
-static int
-igb_is_valid_ether_addr(uint8_t *addr)
-{
- char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
-
- if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
- return (FALSE);
- }
-
- return (TRUE);
-}
-
-
-/*
- * Enable PCI Wake On Lan capability
- */
-static void
-igb_enable_wakeup(device_t dev)
-{
- u16 cap, status;
- u8 id;
-
- /* First find the capabilities pointer*/
- cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
- /* Read the PM Capabilities */
- id = pci_read_config(dev, cap, 1);
- if (id != PCIY_PMG) /* Something wrong */
- return;
- /* OK, we have the power capabilities, so
- now get the status register */
- cap += PCIR_POWER_STATUS;
- status = pci_read_config(dev, cap, 2);
- status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
- pci_write_config(dev, cap, status, 2);
- return;
-}
-
-static void
-igb_led_func(void *arg, int onoff)
-{
- struct adapter *adapter = arg;
-
- IGB_CORE_LOCK(adapter);
- if (onoff) {
- e1000_setup_led(&adapter->hw);
- e1000_led_on(&adapter->hw);
- } else {
- e1000_led_off(&adapter->hw);
- e1000_cleanup_led(&adapter->hw);
- }
- IGB_CORE_UNLOCK(adapter);
-}
-
-static uint64_t
-igb_get_vf_counter(if_t ifp, ift_counter cnt)
-{
- struct adapter *adapter;
- struct e1000_vf_stats *stats;
-#ifndef IGB_LEGACY_TX
- struct tx_ring *txr;
- uint64_t rv;
-#endif
-
- adapter = if_getsoftc(ifp);
- stats = (struct e1000_vf_stats *)adapter->stats;
-
- switch (cnt) {
- case IFCOUNTER_IPACKETS:
- return (stats->gprc);
- case IFCOUNTER_OPACKETS:
- return (stats->gptc);
- case IFCOUNTER_IBYTES:
- return (stats->gorc);
- case IFCOUNTER_OBYTES:
- return (stats->gotc);
- case IFCOUNTER_IMCASTS:
- return (stats->mprc);
- case IFCOUNTER_IERRORS:
- return (adapter->dropped_pkts);
- case IFCOUNTER_OERRORS:
- return (adapter->watchdog_events);
-#ifndef IGB_LEGACY_TX
- case IFCOUNTER_OQDROPS:
- rv = 0;
- txr = adapter->tx_rings;
- for (int i = 0; i < adapter->num_queues; i++, txr++)
- rv += txr->br->br_drops;
- return (rv);
-#endif
- default:
- return (if_get_counter_default(ifp, cnt));
- }
-}
-
-static uint64_t
-igb_get_counter(if_t ifp, ift_counter cnt)
-{
- struct adapter *adapter;
- struct e1000_hw_stats *stats;
-#ifndef IGB_LEGACY_TX
- struct tx_ring *txr;
- uint64_t rv;
-#endif
-
- adapter = if_getsoftc(ifp);
- if (adapter->vf_ifp)
- return (igb_get_vf_counter(ifp, cnt));
-
- stats = (struct e1000_hw_stats *)adapter->stats;
-
- switch (cnt) {
- case IFCOUNTER_IPACKETS:
- return (stats->gprc);
- case IFCOUNTER_OPACKETS:
- return (stats->gptc);
- case IFCOUNTER_IBYTES:
- return (stats->gorc);
- case IFCOUNTER_OBYTES:
- return (stats->gotc);
- case IFCOUNTER_IMCASTS:
- return (stats->mprc);
- case IFCOUNTER_OMCASTS:
- return (stats->mptc);
- case IFCOUNTER_IERRORS:
- return (adapter->dropped_pkts + stats->rxerrc +
- stats->crcerrs + stats->algnerrc +
- stats->ruc + stats->roc + stats->cexterr);
- case IFCOUNTER_OERRORS:
- return (stats->ecol + stats->latecol +
- adapter->watchdog_events);
- case IFCOUNTER_COLLISIONS:
- return (stats->colc);
- case IFCOUNTER_IQDROPS:
- return (stats->mpc);
-#ifndef IGB_LEGACY_TX
- case IFCOUNTER_OQDROPS:
- rv = 0;
- txr = adapter->tx_rings;
- for (int i = 0; i < adapter->num_queues; i++, txr++)
- rv += txr->br->br_drops;
- return (rv);
-#endif
- default:
- return (if_get_counter_default(ifp, cnt));
- }
-}
-
-/**********************************************************************
- *
- * Update the board statistics counters.
- *
- **********************************************************************/
-static void
-igb_update_stats_counters(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- struct e1000_hw_stats *stats;
-
- /*
- ** The virtual function adapter has only a
- ** small controlled set of stats, do only
- ** those and return.
- */
- if (adapter->vf_ifp) {
- igb_update_vf_stats_counters(adapter);
- return;
- }
-
- stats = (struct e1000_hw_stats *)adapter->stats;
-
- if (adapter->hw.phy.media_type == e1000_media_type_copper ||
- (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
- stats->symerrs +=
- E1000_READ_REG(hw,E1000_SYMERRS);
- stats->sec += E1000_READ_REG(hw, E1000_SEC);
- }
-
- stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
- stats->mpc += E1000_READ_REG(hw, E1000_MPC);
- stats->scc += E1000_READ_REG(hw, E1000_SCC);
- stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
-
- stats->mcc += E1000_READ_REG(hw, E1000_MCC);
- stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
- stats->colc += E1000_READ_REG(hw, E1000_COLC);
- stats->dc += E1000_READ_REG(hw, E1000_DC);
- stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
- stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
- stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
- /*
- ** For watchdog management we need to know if we have been
- ** paused during the last interval, so capture that here.
- */
- adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
- stats->xoffrxc += adapter->pause_frames;
- stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
- stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
- stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
- stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
- stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
- stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
- stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
- stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
- stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
- stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
- stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
- stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
-
- /* For the 64-bit byte counters the low dword must be read first. */
- /* Both registers clear on the read of the high dword */
-
- stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
- ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
- stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
- ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
-
- stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
- stats->ruc += E1000_READ_REG(hw, E1000_RUC);
- stats->rfc += E1000_READ_REG(hw, E1000_RFC);
- stats->roc += E1000_READ_REG(hw, E1000_ROC);
- stats->rjc += E1000_READ_REG(hw, E1000_RJC);
-
- stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
- stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC);
- stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
-
- stats->tor += E1000_READ_REG(hw, E1000_TORL) +
- ((u64)E1000_READ_REG(hw, E1000_TORH) << 32);
- stats->tot += E1000_READ_REG(hw, E1000_TOTL) +
- ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32);
-
- stats->tpr += E1000_READ_REG(hw, E1000_TPR);
- stats->tpt += E1000_READ_REG(hw, E1000_TPT);
- stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
- stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
- stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
- stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
- stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
- stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
- stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
- stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
-
- /* Interrupt Counts */
-
- stats->iac += E1000_READ_REG(hw, E1000_IAC);
- stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
- stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
- stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
- stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
- stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
- stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
- stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
- stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
-
- /* Host to Card Statistics */
-
- stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
- stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
- stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
- stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
- stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
- stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
- stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
- stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
- ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
- stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
- ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
- stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
- stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
- stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
-
- stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
- stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
- stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
- stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
- stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
- stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
-
- /* Driver specific counters */
- adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
- adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
- adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
- adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
- adapter->packet_buf_alloc_tx =
- ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
- adapter->packet_buf_alloc_rx =
- (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
-}
-
-
-/**********************************************************************
- *
- * Initialize the VF board statistics counters.
- *
- **********************************************************************/
-static void
-igb_vf_init_stats(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- struct e1000_vf_stats *stats;
-
- stats = (struct e1000_vf_stats *)adapter->stats;
- if (stats == NULL)
- return;
- stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
- stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
- stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
- stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
- stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
-}
-
-/**********************************************************************
- *
- * Update the VF board statistics counters.
- *
- **********************************************************************/
-static void
-igb_update_vf_stats_counters(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- struct e1000_vf_stats *stats;
-
- if (adapter->link_speed == 0)
- return;
-
- stats = (struct e1000_vf_stats *)adapter->stats;
-
- UPDATE_VF_REG(E1000_VFGPRC,
- stats->last_gprc, stats->gprc);
- UPDATE_VF_REG(E1000_VFGORC,
- stats->last_gorc, stats->gorc);
- UPDATE_VF_REG(E1000_VFGPTC,
- stats->last_gptc, stats->gptc);
- UPDATE_VF_REG(E1000_VFGOTC,
- stats->last_gotc, stats->gotc);
- UPDATE_VF_REG(E1000_VFMPRC,
- stats->last_mprc, stats->mprc);
-}
-
-/* Export a single 32-bit register via a read-only sysctl. */
-static int
-igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
-{
- struct adapter *adapter;
- u_int val;
-
- adapter = oidp->oid_arg1;
- val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
- return (sysctl_handle_int(oidp, &val, 0, req));
-}
-
-/*
-** Tuneable interrupt rate handler
-*/
-static int
-igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
-{
- struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1);
- int error;
- u32 reg, usec, rate;
-
- reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
- usec = ((reg & 0x7FFC) >> 2);
- if (usec > 0)
- rate = 1000000 / usec;
- else
- rate = 0;
- error = sysctl_handle_int(oidp, &rate, 0, req);
- if (error || !req->newptr)
- return error;
- return 0;
-}
-
-/*
- * Add sysctl variables, one per statistic, to the system.
- */
-static void
-igb_add_hw_stats(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
-
- struct tx_ring *txr = adapter->tx_rings;
- struct rx_ring *rxr = adapter->rx_rings;
-
- struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
- struct sysctl_oid *tree = device_get_sysctl_tree(dev);
- struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
- struct e1000_hw_stats *stats = adapter->stats;
-
- struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
- struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
-
-#define QUEUE_NAME_LEN 32
- char namebuf[QUEUE_NAME_LEN];
-
- /* Driver Statistics */
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
- CTLFLAG_RD, &adapter->dropped_pkts,
- "Driver dropped packets");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
- CTLFLAG_RD, &adapter->link_irq,
- "Link MSIX IRQ Handled");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
- CTLFLAG_RD, &adapter->mbuf_defrag_failed,
- "Defragmenting mbuf chain failed");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
- CTLFLAG_RD, &adapter->no_tx_dma_setup,
- "Driver tx dma failure in xmit");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
- CTLFLAG_RD, &adapter->rx_overruns,
- "RX overruns");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
- CTLFLAG_RD, &adapter->watchdog_events,
- "Watchdog timeouts");
-
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control",
- CTLFLAG_RD, &adapter->device_control,
- "Device Control Register");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control",
- CTLFLAG_RD, &adapter->rx_control,
- "Receiver Control Register");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask",
- CTLFLAG_RD, &adapter->int_mask,
- "Interrupt Mask");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask",
- CTLFLAG_RD, &adapter->eint_mask,
- "Extended Interrupt Mask");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc",
- CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
- "Transmit Buffer Packet Allocation");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc",
- CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
- "Receive Buffer Packet Allocation");
- SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
- CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
- "Flow Control High Watermark");
- SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
- CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
- "Flow Control Low Watermark");
-
- for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
- struct lro_ctrl *lro = &rxr->lro;
-
- snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
- queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
- CTLFLAG_RD, NULL, "Queue Name");
- queue_list = SYSCTL_CHILDREN(queue_node);
-
- SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
- CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i],
- sizeof(&adapter->queues[i]),
- igb_sysctl_interrupt_rate_handler,
- "IU", "Interrupt Rate");
-
- SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me),
- igb_sysctl_reg_handler, "IU",
- "Transmit Descriptor Head");
- SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me),
- igb_sysctl_reg_handler, "IU",
- "Transmit Descriptor Tail");
- SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
- CTLFLAG_RD, &txr->no_desc_avail,
- "Queue Descriptors Unavailable");
- SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
- CTLFLAG_RD, &txr->total_packets,
- "Queue Packets Transmitted");
-
- SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
- igb_sysctl_reg_handler, "IU",
- "Receive Descriptor Head");
- SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
- igb_sysctl_reg_handler, "IU",
- "Receive Descriptor Tail");
- SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
- CTLFLAG_RD, &rxr->rx_packets,
- "Queue Packets Received");
- SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
- CTLFLAG_RD, &rxr->rx_bytes,
- "Queue Bytes Received");
- SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_queued",
- CTLFLAG_RD, &lro->lro_queued, 0,
- "LRO Queued");
- SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_flushed",
- CTLFLAG_RD, &lro->lro_flushed, 0,
- "LRO Flushed");
- }
-
- /* MAC stats get their own sub node */
-
- stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
- CTLFLAG_RD, NULL, "MAC Statistics");
- stat_list = SYSCTL_CHILDREN(stat_node);
-
- /*
- ** VF adapter has a very limited set of stats
- ** since its not managing the metal, so to speak.
- */
- if (adapter->vf_ifp) {
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
- CTLFLAG_RD, &stats->gprc,
- "Good Packets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
- CTLFLAG_RD, &stats->gptc,
- "Good Packets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
- CTLFLAG_RD, &stats->gorc,
- "Good Octets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
- CTLFLAG_RD, &stats->gotc,
- "Good Octets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
- CTLFLAG_RD, &stats->mprc,
- "Multicast Packets Received");
- return;
- }
-
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll",
- CTLFLAG_RD, &stats->ecol,
- "Excessive collisions");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll",
- CTLFLAG_RD, &stats->scc,
- "Single collisions");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
- CTLFLAG_RD, &stats->mcc,
- "Multiple collisions");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll",
- CTLFLAG_RD, &stats->latecol,
- "Late collisions");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count",
- CTLFLAG_RD, &stats->colc,
- "Collision Count");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
- CTLFLAG_RD, &stats->symerrs,
- "Symbol Errors");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
- CTLFLAG_RD, &stats->sec,
- "Sequence Errors");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
- CTLFLAG_RD, &stats->dc,
- "Defer Count");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
- CTLFLAG_RD, &stats->mpc,
- "Missed Packets");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors",
- CTLFLAG_RD, &stats->rlec,
- "Receive Length Errors");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
- CTLFLAG_RD, &stats->rnbc,
- "Receive No Buffers");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
- CTLFLAG_RD, &stats->ruc,
- "Receive Undersize");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
- CTLFLAG_RD, &stats->rfc,
- "Fragmented Packets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
- CTLFLAG_RD, &stats->roc,
- "Oversized Packets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
- CTLFLAG_RD, &stats->rjc,
- "Recevied Jabber");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
- CTLFLAG_RD, &stats->rxerrc,
- "Receive Errors");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
- CTLFLAG_RD, &stats->crcerrs,
- "CRC errors");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
- CTLFLAG_RD, &stats->algnerrc,
- "Alignment Errors");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs",
- CTLFLAG_RD, &stats->tncrs,
- "Transmit with No CRS");
- /* On 82575 these are collision counts */
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
- CTLFLAG_RD, &stats->cexterr,
- "Collision/Carrier extension errors");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
- CTLFLAG_RD, &stats->xonrxc,
- "XON Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
- CTLFLAG_RD, &stats->xontxc,
- "XON Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
- CTLFLAG_RD, &stats->xoffrxc,
- "XOFF Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
- CTLFLAG_RD, &stats->xofftxc,
- "XOFF Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd",
- CTLFLAG_RD, &stats->fcruc,
- "Unsupported Flow Control Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd",
- CTLFLAG_RD, &stats->mgprc,
- "Management Packets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop",
- CTLFLAG_RD, &stats->mgpdc,
- "Management Packets Dropped");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd",
- CTLFLAG_RD, &stats->mgptc,
- "Management Packets Transmitted");
- /* Packet Reception Stats */
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
- CTLFLAG_RD, &stats->tpr,
- "Total Packets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
- CTLFLAG_RD, &stats->gprc,
- "Good Packets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
- CTLFLAG_RD, &stats->bprc,
- "Broadcast Packets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
- CTLFLAG_RD, &stats->mprc,
- "Multicast Packets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
- CTLFLAG_RD, &stats->prc64,
- "64 byte frames received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
- CTLFLAG_RD, &stats->prc127,
- "65-127 byte frames received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
- CTLFLAG_RD, &stats->prc255,
- "128-255 byte frames received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
- CTLFLAG_RD, &stats->prc511,
- "256-511 byte frames received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
- CTLFLAG_RD, &stats->prc1023,
- "512-1023 byte frames received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
- CTLFLAG_RD, &stats->prc1522,
- "1023-1522 byte frames received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
- CTLFLAG_RD, &stats->gorc,
- "Good Octets Received");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd",
- CTLFLAG_RD, &stats->tor,
- "Total Octets Received");
-
- /* Packet Transmission Stats */
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
- CTLFLAG_RD, &stats->gotc,
- "Good Octets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd",
- CTLFLAG_RD, &stats->tot,
- "Total Octets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
- CTLFLAG_RD, &stats->tpt,
- "Total Packets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
- CTLFLAG_RD, &stats->gptc,
- "Good Packets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
- CTLFLAG_RD, &stats->bptc,
- "Broadcast Packets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
- CTLFLAG_RD, &stats->mptc,
- "Multicast Packets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
- CTLFLAG_RD, &stats->ptc64,
- "64 byte frames transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
- CTLFLAG_RD, &stats->ptc127,
- "65-127 byte frames transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
- CTLFLAG_RD, &stats->ptc255,
- "128-255 byte frames transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
- CTLFLAG_RD, &stats->ptc511,
- "256-511 byte frames transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
- CTLFLAG_RD, &stats->ptc1023,
- "512-1023 byte frames transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
- CTLFLAG_RD, &stats->ptc1522,
- "1024-1522 byte frames transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
- CTLFLAG_RD, &stats->tsctc,
- "TSO Contexts Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
- CTLFLAG_RD, &stats->tsctfc,
- "TSO Contexts Failed");
-
-
- /* Interrupt Stats */
-
- int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
- CTLFLAG_RD, NULL, "Interrupt Statistics");
- int_list = SYSCTL_CHILDREN(int_node);
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
- CTLFLAG_RD, &stats->iac,
- "Interrupt Assertion Count");
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
- CTLFLAG_RD, &stats->icrxptc,
- "Interrupt Cause Rx Pkt Timer Expire Count");
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
- CTLFLAG_RD, &stats->icrxatc,
- "Interrupt Cause Rx Abs Timer Expire Count");
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
- CTLFLAG_RD, &stats->ictxptc,
- "Interrupt Cause Tx Pkt Timer Expire Count");
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
- CTLFLAG_RD, &stats->ictxatc,
- "Interrupt Cause Tx Abs Timer Expire Count");
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
- CTLFLAG_RD, &stats->ictxqec,
- "Interrupt Cause Tx Queue Empty Count");
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
- CTLFLAG_RD, &stats->ictxqmtc,
- "Interrupt Cause Tx Queue Min Thresh Count");
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
- CTLFLAG_RD, &stats->icrxdmtc,
- "Interrupt Cause Rx Desc Min Thresh Count");
-
- SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
- CTLFLAG_RD, &stats->icrxoc,
- "Interrupt Cause Receiver Overrun Count");
-
- /* Host to Card Stats */
-
- host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host",
- CTLFLAG_RD, NULL,
- "Host to Card Statistics");
-
- host_list = SYSCTL_CHILDREN(host_node);
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
- CTLFLAG_RD, &stats->cbtmpc,
- "Circuit Breaker Tx Packet Count");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
- CTLFLAG_RD, &stats->htdpmc,
- "Host Transmit Discarded Packets");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
- CTLFLAG_RD, &stats->rpthc,
- "Rx Packets To Host");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
- CTLFLAG_RD, &stats->cbrmpc,
- "Circuit Breaker Rx Packet Count");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
- CTLFLAG_RD, &stats->cbrdpc,
- "Circuit Breaker Rx Dropped Count");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
- CTLFLAG_RD, &stats->hgptc,
- "Host Good Packets Tx Count");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
- CTLFLAG_RD, &stats->htcbdpc,
- "Host Tx Circuit Breaker Dropped Count");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
- CTLFLAG_RD, &stats->hgorc,
- "Host Good Octets Received Count");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
- CTLFLAG_RD, &stats->hgotc,
- "Host Good Octets Transmit Count");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
- CTLFLAG_RD, &stats->lenerrs,
- "Length Errors");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
- CTLFLAG_RD, &stats->scvpc,
- "SerDes/SGMII Code Violation Pkt Count");
-
- SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
- CTLFLAG_RD, &stats->hrmpc,
- "Header Redirection Missed Packet Count");
-}
-
-
-/**********************************************************************
- *
- * This routine provides a way to dump out the adapter eeprom,
- * often a useful debug/service tool. This only dumps the first
- * 32 words, stuff that matters is in that extent.
- *
- **********************************************************************/
-static int
-igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
-{
- struct adapter *adapter;
- int error;
- int result;
-
- result = -1;
- error = sysctl_handle_int(oidp, &result, 0, req);
-
- if (error || !req->newptr)
- return (error);
-
- /*
- * This value will cause a hex dump of the
- * first 32 16-bit words of the EEPROM to
- * the screen.
- */
- if (result == 1) {
- adapter = (struct adapter *)arg1;
- igb_print_nvm_info(adapter);
- }
-
- return (error);
-}
-
-static void
-igb_print_nvm_info(struct adapter *adapter)
-{
- u16 eeprom_data;
- int i, j, row = 0;
-
- /* Its a bit crude, but it gets the job done */
- printf("\nInterface EEPROM Dump:\n");
- printf("Offset\n0x0000 ");
- for (i = 0, j = 0; i < 32; i++, j++) {
- if (j == 8) { /* Make the offset block */
- j = 0; ++row;
- printf("\n0x00%x0 ",row);
- }
- e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
- printf("%04x ", eeprom_data);
- }
- printf("\n");
-}
-
-static void
-igb_set_sysctl_value(struct adapter *adapter, const char *name,
- const char *description, int *limit, int value)
-{
- *limit = value;
- SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
- OID_AUTO, name, CTLFLAG_RW, limit, value, description);
-}
-
-/*
-** Set flow control using sysctl:
-** Flow control values:
-** 0 - off
-** 1 - rx pause
-** 2 - tx pause
-** 3 - full
-*/
-static int
-igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
-{
- int error;
- static int input = 3; /* default is full */
- struct adapter *adapter = (struct adapter *) arg1;
-
- error = sysctl_handle_int(oidp, &input, 0, req);
-
- if ((error) || (req->newptr == NULL))
- return (error);
-
- switch (input) {
- case e1000_fc_rx_pause:
- case e1000_fc_tx_pause:
- case e1000_fc_full:
- case e1000_fc_none:
- adapter->hw.fc.requested_mode = input;
- adapter->fc = input;
- break;
- default:
- /* Do nothing */
- return (error);
- }
-
- adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
- e1000_force_mac_fc(&adapter->hw);
- /* XXX TODO: update DROP_EN on each RX queue if appropriate */
- return (error);
-}
-
-/*
-** Manage DMA Coalesce:
-** Control values:
-** 0/1 - off/on
-** Legal timer values are:
-** 250,500,1000-10000 in thousands
-*/
-static int
-igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
-{
- struct adapter *adapter = (struct adapter *) arg1;
- int error;
-
- error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
-
- if ((error) || (req->newptr == NULL))
- return (error);
-
- switch (adapter->dmac) {
- case 0:
- /* Disabling */
- break;
- case 1: /* Just enable and use default */
- adapter->dmac = 1000;
- break;
- case 250:
- case 500:
- case 1000:
- case 2000:
- case 3000:
- case 4000:
- case 5000:
- case 6000:
- case 7000:
- case 8000:
- case 9000:
- case 10000:
- /* Legal values - allow */
- break;
- default:
- /* Do nothing, illegal value */
- adapter->dmac = 0;
- return (EINVAL);
- }
- /* Reinit the interface */
- igb_init(adapter);
- return (error);
-}
-
-/*
-** Manage Energy Efficient Ethernet:
-** Control values:
-** 0/1 - enabled/disabled
-*/
-static int
-igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
-{
- struct adapter *adapter = (struct adapter *) arg1;
- int error, value;
-
- value = adapter->hw.dev_spec._82575.eee_disable;
- error = sysctl_handle_int(oidp, &value, 0, req);
- if (error || req->newptr == NULL)
- return (error);
- IGB_CORE_LOCK(adapter);
- adapter->hw.dev_spec._82575.eee_disable = (value != 0);
- igb_init_locked(adapter);
- IGB_CORE_UNLOCK(adapter);
- return (0);
-}
Index: sys/dev/e1000/if_lem.h
===================================================================
--- sys/dev/e1000/if_lem.h
+++ /dev/null
@@ -1,519 +0,0 @@
-/******************************************************************************
-
- Copyright (c) 2001-2015, Intel Corporation
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
-
-******************************************************************************/
-/*$FreeBSD$*/
-
-
-#ifndef _LEM_H_DEFINED_
-#define _LEM_H_DEFINED_
-
-
-/* Tunables */
-
-/*
- * EM_TXD: Maximum number of Transmit Descriptors
- * Valid Range: 80-256 for 82542 and 82543-based adapters
- * 80-4096 for others
- * Default Value: 256
- * This value is the number of transmit descriptors allocated by the driver.
- * Increasing this value allows the driver to queue more transmits. Each
- * descriptor is 16 bytes.
- * Since TDLEN should be multiple of 128bytes, the number of transmit
- * desscriptors should meet the following condition.
- * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0
- */
-#define EM_MIN_TXD 80
-#define EM_MAX_TXD_82543 256
-#define EM_MAX_TXD 4096
-#define EM_DEFAULT_TXD EM_MAX_TXD_82543
-
-/*
- * EM_RXD - Maximum number of receive Descriptors
- * Valid Range: 80-256 for 82542 and 82543-based adapters
- * 80-4096 for others
- * Default Value: 256
- * This value is the number of receive descriptors allocated by the driver.
- * Increasing this value allows the driver to buffer more incoming packets.
- * Each descriptor is 16 bytes. A receive buffer is also allocated for each
- * descriptor. The maximum MTU size is 16110.
- * Since TDLEN should be multiple of 128bytes, the number of transmit
- * desscriptors should meet the following condition.
- * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0
- */
-#define EM_MIN_RXD 80
-#define EM_MAX_RXD_82543 256
-#define EM_MAX_RXD 4096
-#define EM_DEFAULT_RXD EM_MAX_RXD_82543
-
-/*
- * EM_TIDV - Transmit Interrupt Delay Value
- * Valid Range: 0-65535 (0=off)
- * Default Value: 64
- * This value delays the generation of transmit interrupts in units of
- * 1.024 microseconds. Transmit interrupt reduction can improve CPU
- * efficiency if properly tuned for specific network traffic. If the
- * system is reporting dropped transmits, this value may be set too high
- * causing the driver to run out of available transmit descriptors.
- */
-#define EM_TIDV 64
-
-/*
- * EM_TADV - Transmit Absolute Interrupt Delay Value
- * (Not valid for 82542/82543/82544)
- * Valid Range: 0-65535 (0=off)
- * Default Value: 64
- * This value, in units of 1.024 microseconds, limits the delay in which a
- * transmit interrupt is generated. Useful only if EM_TIDV is non-zero,
- * this value ensures that an interrupt is generated after the initial
- * packet is sent on the wire within the set amount of time. Proper tuning,
- * along with EM_TIDV, may improve traffic throughput in specific
- * network conditions.
- */
-#define EM_TADV 64
-
-/*
- * EM_RDTR - Receive Interrupt Delay Timer (Packet Timer)
- * Valid Range: 0-65535 (0=off)
- * Default Value: 0
- * This value delays the generation of receive interrupts in units of 1.024
- * microseconds. Receive interrupt reduction can improve CPU efficiency if
- * properly tuned for specific network traffic. Increasing this value adds
- * extra latency to frame reception and can end up decreasing the throughput
- * of TCP traffic. If the system is reporting dropped receives, this value
- * may be set too high, causing the driver to run out of available receive
- * descriptors.
- *
- * CAUTION: When setting EM_RDTR to a value other than 0, adapters
- * may hang (stop transmitting) under certain network conditions.
- * If this occurs a WATCHDOG message is logged in the system
- * event log. In addition, the controller is automatically reset,
- * restoring the network connection. To eliminate the potential
- * for the hang ensure that EM_RDTR is set to 0.
- */
-#define EM_RDTR 0
-
-/*
- * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544)
- * Valid Range: 0-65535 (0=off)
- * Default Value: 64
- * This value, in units of 1.024 microseconds, limits the delay in which a
- * receive interrupt is generated. Useful only if EM_RDTR is non-zero,
- * this value ensures that an interrupt is generated after the initial
- * packet is received within the set amount of time. Proper tuning,
- * along with EM_RDTR, may improve traffic throughput in specific network
- * conditions.
- */
-#define EM_RADV 64
-
-/*
- * This parameter controls the max duration of transmit watchdog.
- */
-#define EM_WATCHDOG (10 * hz)
-
-/*
- * This parameter controls when the driver calls the routine to reclaim
- * transmit descriptors.
- */
-#define EM_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 8)
-#define EM_TX_OP_THRESHOLD (adapter->num_tx_desc / 32)
-
-/*
- * This parameter controls whether or not autonegotation is enabled.
- * 0 - Disable autonegotiation
- * 1 - Enable autonegotiation
- */
-#define DO_AUTO_NEG 1
-
-/*
- * This parameter control whether or not the driver will wait for
- * autonegotiation to complete.
- * 1 - Wait for autonegotiation to complete
- * 0 - Don't wait for autonegotiation to complete
- */
-#define WAIT_FOR_AUTO_NEG_DEFAULT 0
-
-/* Tunables -- End */
-
-#define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \
- ADVERTISE_100_HALF | ADVERTISE_100_FULL | \
- ADVERTISE_1000_FULL)
-
-#define AUTO_ALL_MODES 0
-
-/* PHY master/slave setting */
-#define EM_MASTER_SLAVE e1000_ms_hw_default
-
-/*
- * Micellaneous constants
- */
-#define EM_VENDOR_ID 0x8086
-#define EM_FLASH 0x0014
-
-#define EM_JUMBO_PBA 0x00000028
-#define EM_DEFAULT_PBA 0x00000030
-#define EM_SMARTSPEED_DOWNSHIFT 3
-#define EM_SMARTSPEED_MAX 15
-#define EM_MAX_LOOP 10
-
-#define MAX_NUM_MULTICAST_ADDRESSES 128
-#define PCI_ANY_ID (~0U)
-#define ETHER_ALIGN 2
-#define EM_FC_PAUSE_TIME 0x0680
-#define EM_EEPROM_APME 0x400;
-#define EM_82544_APME 0x0004;
-
-/* Code compatilbility between 6 and 7 */
-#ifndef ETHER_BPF_MTAP
-#define ETHER_BPF_MTAP BPF_MTAP
-#endif
-
-/*
- * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be
- * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will
- * also optimize cache line size effect. H/W supports up to cache line size 128.
- */
-#define EM_DBA_ALIGN 128
-
-#define SPEED_MODE_BIT (1<<21) /* On PCI-E MACs only */
-
-/* PCI Config defines */
-#define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK)
-#define EM_BAR_TYPE_MASK 0x00000001
-#define EM_BAR_TYPE_MMEM 0x00000000
-#define EM_BAR_TYPE_IO 0x00000001
-#define EM_BAR_TYPE_FLASH 0x0014
-#define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK)
-#define EM_BAR_MEM_TYPE_MASK 0x00000006
-#define EM_BAR_MEM_TYPE_32BIT 0x00000000
-#define EM_BAR_MEM_TYPE_64BIT 0x00000004
-#define EM_MSIX_BAR 3 /* On 82575 */
-
-#if __FreeBSD_version < 900000
-#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD
-#endif
-
-/* Defines for printing debug information */
-#define DEBUG_INIT 0
-#define DEBUG_IOCTL 0
-#define DEBUG_HW 0
-
-#define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n")
-#define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A)
-#define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B)
-#define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n")
-#define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A)
-#define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B)
-#define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n")
-#define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A)
-#define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B)
-
-#define EM_MAX_SCATTER 40
-#define EM_VFTA_SIZE 128
-#define EM_MSIX_MASK 0x01F00000 /* For 82574 use */
-#define ETH_ZLEN 60
-#define ETH_ADDR_LEN 6
-#define CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */
-
-/*
- * 82574 has a nonstandard address for EIAC
- * and since its only used in MSIX, and in
- * the em driver only 82574 uses MSIX we can
- * solve it just using this define.
- */
-#define EM_EIAC 0x000DC
-
-/* Used in for 82547 10Mb Half workaround */
-#define EM_PBA_BYTES_SHIFT 0xA
-#define EM_TX_HEAD_ADDR_SHIFT 7
-#define EM_PBA_TX_MASK 0xFFFF0000
-#define EM_FIFO_HDR 0x10
-#define EM_82547_PKT_THRESH 0x3e0
-
-/* Precision Time Sync (IEEE 1588) defines */
-#define ETHERTYPE_IEEE1588 0x88F7
-#define PICOSECS_PER_TICK 20833
-#define TSYNC_PORT 319 /* UDP port for the protocol */
-
-#ifdef NIC_PARAVIRT
-#define E1000_PARA_SUBDEV 0x1101 /* special id */
-#define E1000_CSBAL 0x02830 /* csb phys. addr. low */
-#define E1000_CSBAH 0x02834 /* csb phys. addr. hi */
-#include <net/paravirt.h>
-#endif /* NIC_PARAVIRT */
-
-/*
- * Bus dma allocation structure used by
- * e1000_dma_malloc and e1000_dma_free.
- */
-struct em_dma_alloc {
- bus_addr_t dma_paddr;
- caddr_t dma_vaddr;
- bus_dma_tag_t dma_tag;
- bus_dmamap_t dma_map;
- bus_dma_segment_t dma_seg;
- int dma_nseg;
-};
-
-struct adapter;
-
-struct em_int_delay_info {
- struct adapter *adapter; /* Back-pointer to the adapter struct */
- int offset; /* Register offset to read/write */
- int value; /* Current value in usecs */
-};
-
-/* Our adapter structure */
-struct adapter {
- if_t ifp;
- struct e1000_hw hw;
-
- /* FreeBSD operating-system-specific structures. */
- struct e1000_osdep osdep;
- device_t dev;
- struct cdev *led_dev;
-
- struct resource *memory;
- struct resource *flash;
- struct resource *msix;
-
- struct resource *ioport;
- int io_rid;
-
- /* 82574 may use 3 int vectors */
- struct resource *res[3];
- void *tag[3];
- int rid[3];
-
- struct ifmedia media;
- struct callout timer;
- struct callout tx_fifo_timer;
- bool watchdog_check;
- int watchdog_time;
- int msi;
- int if_flags;
- int max_frame_size;
- int min_frame_size;
- struct mtx core_mtx;
- struct mtx tx_mtx;
- struct mtx rx_mtx;
- int em_insert_vlan_header;
-
- /* Task for FAST handling */
- struct task link_task;
- struct task rxtx_task;
- struct task rx_task;
- struct task tx_task;
- struct taskqueue *tq; /* private task queue */
-
- eventhandler_tag vlan_attach;
- eventhandler_tag vlan_detach;
- u32 num_vlans;
-
- /* Management and WOL features */
- u32 wol;
- bool has_manage;
- bool has_amt;
-
- /* Multicast array memory */
- u8 *mta;
-
- /*
- ** Shadow VFTA table, this is needed because
- ** the real vlan filter table gets cleared during
- ** a soft reset and the driver needs to be able
- ** to repopulate it.
- */
- u32 shadow_vfta[EM_VFTA_SIZE];
-
- /* Info about the interface */
- uint8_t link_active;
- uint16_t link_speed;
- uint16_t link_duplex;
- uint32_t smartspeed;
- uint32_t fc_setting;
-
- struct em_int_delay_info tx_int_delay;
- struct em_int_delay_info tx_abs_int_delay;
- struct em_int_delay_info rx_int_delay;
- struct em_int_delay_info rx_abs_int_delay;
- struct em_int_delay_info tx_itr;
-
- /*
- * Transmit definitions
- *
- * We have an array of num_tx_desc descriptors (handled
- * by the controller) paired with an array of tx_buffers
- * (at tx_buffer_area).
- * The index of the next available descriptor is next_avail_tx_desc.
- * The number of remaining tx_desc is num_tx_desc_avail.
- */
- struct em_dma_alloc txdma; /* bus_dma glue for tx desc */
- struct e1000_tx_desc *tx_desc_base;
- uint32_t next_avail_tx_desc;
- uint32_t next_tx_to_clean;
- volatile uint16_t num_tx_desc_avail;
- uint16_t num_tx_desc;
- uint16_t last_hw_offload;
- uint32_t txd_cmd;
- struct em_buffer *tx_buffer_area;
- bus_dma_tag_t txtag; /* dma tag for tx */
- uint32_t tx_tso; /* last tx was tso */
-
- /*
- * Receive definitions
- *
- * we have an array of num_rx_desc rx_desc (handled by the
- * controller), and paired with an array of rx_buffers
- * (at rx_buffer_area).
- * The next pair to check on receive is at offset next_rx_desc_to_check
- */
- struct em_dma_alloc rxdma; /* bus_dma glue for rx desc */
- struct e1000_rx_desc *rx_desc_base;
- uint32_t next_rx_desc_to_check;
- uint32_t rx_buffer_len;
- uint16_t num_rx_desc;
- int rx_process_limit;
- struct em_buffer *rx_buffer_area;
- bus_dma_tag_t rxtag;
- bus_dmamap_t rx_sparemap;
-
- /*
- * First/last mbuf pointers, for
- * collecting multisegment RX packets.
- */
- struct mbuf *fmp;
- struct mbuf *lmp;
-
- /* Misc stats maintained by the driver */
- unsigned long dropped_pkts;
- unsigned long link_irq;
- unsigned long mbuf_cluster_failed;
- unsigned long mbuf_defrag_failed;
- unsigned long no_tx_desc_avail1;
- unsigned long no_tx_desc_avail2;
- unsigned long no_tx_dma_setup;
- unsigned long no_tx_map_avail;
- unsigned long watchdog_events;
- unsigned long rx_irq;
- unsigned long rx_overruns;
- unsigned long tx_irq;
-
- /* 82547 workaround */
- uint32_t tx_fifo_size;
- uint32_t tx_fifo_head;
- uint32_t tx_fifo_head_addr;
- uint64_t tx_fifo_reset_cnt;
- uint64_t tx_fifo_wrk_cnt;
- uint32_t tx_head_addr;
-
- /* For 82544 PCIX Workaround */
- boolean_t pcix_82544;
- boolean_t in_detach;
-
-#ifdef NIC_SEND_COMBINING
- /* 0 = idle; 1xxxx int-pending; 3xxxx int + d pending + tdt */
-#define MIT_PENDING_INT 0x10000 /* pending interrupt */
-#define MIT_PENDING_TDT 0x30000 /* both intr and tdt write are pending */
- uint32_t shadow_tdt;
- uint32_t sc_enable;
-#endif /* NIC_SEND_COMBINING */
-#ifdef BATCH_DISPATCH
- uint32_t batch_enable;
-#endif /* BATCH_DISPATCH */
-
-#ifdef NIC_PARAVIRT
- struct em_dma_alloc csb_mem; /* phys address */
- struct paravirt_csb *csb; /* virtual addr */
- uint32_t rx_retries; /* optimize rx loop */
- uint32_t tdt_csb_count;// XXX stat
- uint32_t tdt_reg_count;// XXX stat
- uint32_t tdt_int_count;// XXX stat
- uint32_t guest_need_kick_count;// XXX stat
-#endif /* NIC_PARAVIRT */
-
- struct e1000_hw_stats stats;
-};
-
-/* ******************************************************************************
- * vendor_info_array
- *
- * This array contains the list of Subvendor/Subdevice IDs on which the driver
- * should load.
- *
- * ******************************************************************************/
-typedef struct _em_vendor_info_t {
- unsigned int vendor_id;
- unsigned int device_id;
- unsigned int subvendor_id;
- unsigned int subdevice_id;
- unsigned int index;
-} em_vendor_info_t;
-
-struct em_buffer {
- int next_eop; /* Index of the desc to watch */
- struct mbuf *m_head;
- bus_dmamap_t map; /* bus_dma map for packet */
-};
-
-/* For 82544 PCIX Workaround */
-typedef struct _ADDRESS_LENGTH_PAIR
-{
- uint64_t address;
- uint32_t length;
-} ADDRESS_LENGTH_PAIR, *PADDRESS_LENGTH_PAIR;
-
-typedef struct _DESCRIPTOR_PAIR
-{
- ADDRESS_LENGTH_PAIR descriptor[4];
- uint32_t elements;
-} DESC_ARRAY, *PDESC_ARRAY;
-
-#define EM_CORE_LOCK_INIT(_sc, _name) \
- mtx_init(&(_sc)->core_mtx, _name, "EM Core Lock", MTX_DEF)
-#define EM_TX_LOCK_INIT(_sc, _name) \
- mtx_init(&(_sc)->tx_mtx, _name, "EM TX Lock", MTX_DEF)
-#define EM_RX_LOCK_INIT(_sc, _name) \
- mtx_init(&(_sc)->rx_mtx, _name, "EM RX Lock", MTX_DEF)
-#define EM_CORE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->core_mtx)
-#define EM_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->tx_mtx)
-#define EM_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_mtx)
-#define EM_CORE_LOCK(_sc) mtx_lock(&(_sc)->core_mtx)
-#define EM_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_mtx)
-#define EM_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->tx_mtx)
-#define EM_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_mtx)
-#define EM_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->core_mtx)
-#define EM_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_mtx)
-#define EM_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_mtx)
-#define EM_CORE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->core_mtx, MA_OWNED)
-#define EM_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_mtx, MA_OWNED)
-
-#endif /* _LEM_H_DEFINED_ */
Index: sys/dev/e1000/if_lem.c
===================================================================
--- sys/dev/e1000/if_lem.c
+++ /dev/null
@@ -1,4730 +0,0 @@
-/******************************************************************************
-
- Copyright (c) 2001-2015, Intel Corporation
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
-
-******************************************************************************/
-/*$FreeBSD$*/
-
-/*
- * Uncomment the following extensions for better performance in a VM,
- * especially if you have support in the hypervisor.
- * See http://info.iet.unipi.it/~luigi/netmap/
- */
-// #define BATCH_DISPATCH
-// #define NIC_SEND_COMBINING
-
-#include "opt_inet.h"
-#include "opt_inet6.h"
-
-#ifdef HAVE_KERNEL_OPTION_HEADERS
-#include "opt_device_polling.h"
-#endif
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/buf_ring.h>
-#include <sys/bus.h>
-#include <sys/endian.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/malloc.h>
-#include <sys/mbuf.h>
-#include <sys/module.h>
-#include <sys/rman.h>
-#include <sys/socket.h>
-#include <sys/sockio.h>
-#include <sys/sysctl.h>
-#include <sys/taskqueue.h>
-#include <sys/eventhandler.h>
-#include <machine/bus.h>
-#include <machine/resource.h>
-
-#include <net/bpf.h>
-#include <net/ethernet.h>
-#include <net/if.h>
-#include <net/if_var.h>
-#include <net/if_arp.h>
-#include <net/if_dl.h>
-#include <net/if_media.h>
-
-#include <net/if_types.h>
-#include <net/if_vlan_var.h>
-
-#include <netinet/in_systm.h>
-#include <netinet/in.h>
-#include <netinet/if_ether.h>
-#include <netinet/ip.h>
-#include <netinet/ip6.h>
-#include <netinet/tcp.h>
-#include <netinet/udp.h>
-
-#include <machine/in_cksum.h>
-#include <dev/led/led.h>
-#include <dev/pci/pcivar.h>
-#include <dev/pci/pcireg.h>
-
-#include "e1000_api.h"
-#include "if_lem.h"
-
-/*********************************************************************
- * Legacy Em Driver version:
- *********************************************************************/
-char lem_driver_version[] = "1.1.0";
-
-/*********************************************************************
- * PCI Device ID Table
- *
- * Used by probe to select devices to load on
- * Last field stores an index into e1000_strings
- * Last entry must be all 0s
- *
- * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
- *********************************************************************/
-
-static em_vendor_info_t lem_vendor_info_array[] =
-{
- /* Intel(R) PRO/1000 Network Connection */
- { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0},
-
- { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
-
- { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0},
-
- { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
-
- { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
-
- { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
-
- { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
- PCI_ANY_ID, PCI_ANY_ID, 0},
-
- { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
- { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0},
- /* required last entry */
- { 0, 0, 0, 0, 0}
-};
-
-/*********************************************************************
- * Table of branding strings for all supported NICs.
- *********************************************************************/
-
-static char *lem_strings[] = {
- "Intel(R) PRO/1000 Legacy Network Connection"
-};
-
-/*********************************************************************
- * Function prototypes
- *********************************************************************/
-static int lem_probe(device_t);
-static int lem_attach(device_t);
-static int lem_detach(device_t);
-static int lem_shutdown(device_t);
-static int lem_suspend(device_t);
-static int lem_resume(device_t);
-static void lem_start(if_t);
-static void lem_start_locked(if_t ifp);
-static int lem_ioctl(if_t, u_long, caddr_t);
-static uint64_t lem_get_counter(if_t, ift_counter);
-static void lem_init(void *);
-static void lem_init_locked(struct adapter *);
-static void lem_stop(void *);
-static void lem_media_status(if_t, struct ifmediareq *);
-static int lem_media_change(if_t);
-static void lem_identify_hardware(struct adapter *);
-static int lem_allocate_pci_resources(struct adapter *);
-static int lem_allocate_irq(struct adapter *adapter);
-static void lem_free_pci_resources(struct adapter *);
-static void lem_local_timer(void *);
-static int lem_hardware_init(struct adapter *);
-static int lem_setup_interface(device_t, struct adapter *);
-static void lem_setup_transmit_structures(struct adapter *);
-static void lem_initialize_transmit_unit(struct adapter *);
-static int lem_setup_receive_structures(struct adapter *);
-static void lem_initialize_receive_unit(struct adapter *);
-static void lem_enable_intr(struct adapter *);
-static void lem_disable_intr(struct adapter *);
-static void lem_free_transmit_structures(struct adapter *);
-static void lem_free_receive_structures(struct adapter *);
-static void lem_update_stats_counters(struct adapter *);
-static void lem_add_hw_stats(struct adapter *adapter);
-static void lem_txeof(struct adapter *);
-static void lem_tx_purge(struct adapter *);
-static int lem_allocate_receive_structures(struct adapter *);
-static int lem_allocate_transmit_structures(struct adapter *);
-static bool lem_rxeof(struct adapter *, int, int *);
-#ifndef __NO_STRICT_ALIGNMENT
-static int lem_fixup_rx(struct adapter *);
-#endif
-static void lem_receive_checksum(struct adapter *, struct e1000_rx_desc *,
- struct mbuf *);
-static void lem_transmit_checksum_setup(struct adapter *, struct mbuf *,
- u32 *, u32 *);
-static void lem_set_promisc(struct adapter *);
-static void lem_disable_promisc(struct adapter *);
-static void lem_set_multi(struct adapter *);
-static void lem_update_link_status(struct adapter *);
-static int lem_get_buf(struct adapter *, int);
-static void lem_register_vlan(void *, if_t, u16);
-static void lem_unregister_vlan(void *, if_t, u16);
-static void lem_setup_vlan_hw_support(struct adapter *);
-static int lem_xmit(struct adapter *, struct mbuf **);
-static void lem_smartspeed(struct adapter *);
-static int lem_82547_fifo_workaround(struct adapter *, int);
-static void lem_82547_update_fifo_head(struct adapter *, int);
-static int lem_82547_tx_fifo_reset(struct adapter *);
-static void lem_82547_move_tail(void *);
-static int lem_dma_malloc(struct adapter *, bus_size_t,
- struct em_dma_alloc *, int);
-static void lem_dma_free(struct adapter *, struct em_dma_alloc *);
-static int lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
-static void lem_print_nvm_info(struct adapter *);
-static int lem_is_valid_ether_addr(u8 *);
-static u32 lem_fill_descriptors (bus_addr_t address, u32 length,
- PDESC_ARRAY desc_array);
-static int lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
-static void lem_add_int_delay_sysctl(struct adapter *, const char *,
- const char *, struct em_int_delay_info *, int, int);
-static void lem_set_flow_cntrl(struct adapter *, const char *,
- const char *, int *, int);
-/* Management and WOL Support */
-static void lem_init_manageability(struct adapter *);
-static void lem_release_manageability(struct adapter *);
-static void lem_get_hw_control(struct adapter *);
-static void lem_release_hw_control(struct adapter *);
-static void lem_get_wakeup(device_t);
-static void lem_enable_wakeup(device_t);
-static int lem_enable_phy_wakeup(struct adapter *);
-static void lem_led_func(void *, int);
-
-static void lem_intr(void *);
-static int lem_irq_fast(void *);
-static void lem_handle_rxtx(void *context, int pending);
-static void lem_handle_link(void *context, int pending);
-static void lem_add_rx_process_limit(struct adapter *, const char *,
- const char *, int *, int);
-
-#ifdef DEVICE_POLLING
-static poll_handler_t lem_poll;
-#endif /* POLLING */
-
-/*********************************************************************
- * FreeBSD Device Interface Entry Points
- *********************************************************************/
-
-static device_method_t lem_methods[] = {
- /* Device interface */
- DEVMETHOD(device_probe, lem_probe),
- DEVMETHOD(device_attach, lem_attach),
- DEVMETHOD(device_detach, lem_detach),
- DEVMETHOD(device_shutdown, lem_shutdown),
- DEVMETHOD(device_suspend, lem_suspend),
- DEVMETHOD(device_resume, lem_resume),
- DEVMETHOD_END
-};
-
-static driver_t lem_driver = {
- "em", lem_methods, sizeof(struct adapter),
-};
-
-extern devclass_t em_devclass;
-DRIVER_MODULE(lem, pci, lem_driver, em_devclass, 0, 0);
-MODULE_DEPEND(lem, pci, 1, 1, 1);
-MODULE_DEPEND(lem, ether, 1, 1, 1);
-#ifdef DEV_NETMAP
-MODULE_DEPEND(lem, netmap, 1, 1, 1);
-#endif /* DEV_NETMAP */
-
-/*********************************************************************
- * Tunable default values.
- *********************************************************************/
-
-#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000)
-#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024)
-
-#define MAX_INTS_PER_SEC 8000
-#define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256))
-
-static int lem_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
-static int lem_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
-static int lem_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
-static int lem_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
-/*
- * increase lem_rxd and lem_txd to at least 2048 in netmap mode
- * for better performance.
- */
-static int lem_rxd = EM_DEFAULT_RXD;
-static int lem_txd = EM_DEFAULT_TXD;
-static int lem_smart_pwr_down = FALSE;
-
-/* Controls whether promiscuous also shows bad packets */
-static int lem_debug_sbp = FALSE;
-
-TUNABLE_INT("hw.em.tx_int_delay", &lem_tx_int_delay_dflt);
-TUNABLE_INT("hw.em.rx_int_delay", &lem_rx_int_delay_dflt);
-TUNABLE_INT("hw.em.tx_abs_int_delay", &lem_tx_abs_int_delay_dflt);
-TUNABLE_INT("hw.em.rx_abs_int_delay", &lem_rx_abs_int_delay_dflt);
-TUNABLE_INT("hw.em.rxd", &lem_rxd);
-TUNABLE_INT("hw.em.txd", &lem_txd);
-TUNABLE_INT("hw.em.smart_pwr_down", &lem_smart_pwr_down);
-TUNABLE_INT("hw.em.sbp", &lem_debug_sbp);
-
-/* Interrupt style - default to fast */
-static int lem_use_legacy_irq = 0;
-TUNABLE_INT("hw.em.use_legacy_irq", &lem_use_legacy_irq);
-
-/* How many packets rxeof tries to clean at a time */
-static int lem_rx_process_limit = 100;
-TUNABLE_INT("hw.em.rx_process_limit", &lem_rx_process_limit);
-
-/* Flow control setting - default to FULL */
-static int lem_fc_setting = e1000_fc_full;
-TUNABLE_INT("hw.em.fc_setting", &lem_fc_setting);
-
-/* Global used in WOL setup with multiport cards */
-static int global_quad_port_a = 0;
-
-#ifdef DEV_NETMAP /* see ixgbe.c for details */
-#include <dev/netmap/if_lem_netmap.h>
-#endif /* DEV_NETMAP */
-
-/*********************************************************************
- * Device identification routine
- *
- * em_probe determines if the driver should be loaded on
- * adapter based on PCI vendor/device id of the adapter.
- *
- * return BUS_PROBE_DEFAULT on success, positive on failure
- *********************************************************************/
-
-static int
-lem_probe(device_t dev)
-{
- char adapter_name[60];
- u16 pci_vendor_id = 0;
- u16 pci_device_id = 0;
- u16 pci_subvendor_id = 0;
- u16 pci_subdevice_id = 0;
- em_vendor_info_t *ent;
-
- INIT_DEBUGOUT("em_probe: begin");
-
- pci_vendor_id = pci_get_vendor(dev);
- if (pci_vendor_id != EM_VENDOR_ID)
- return (ENXIO);
-
- pci_device_id = pci_get_device(dev);
- pci_subvendor_id = pci_get_subvendor(dev);
- pci_subdevice_id = pci_get_subdevice(dev);
-
- ent = lem_vendor_info_array;
- while (ent->vendor_id != 0) {
- if ((pci_vendor_id == ent->vendor_id) &&
- (pci_device_id == ent->device_id) &&
-
- ((pci_subvendor_id == ent->subvendor_id) ||
- (ent->subvendor_id == PCI_ANY_ID)) &&
-
- ((pci_subdevice_id == ent->subdevice_id) ||
- (ent->subdevice_id == PCI_ANY_ID))) {
- sprintf(adapter_name, "%s %s",
- lem_strings[ent->index],
- lem_driver_version);
- device_set_desc_copy(dev, adapter_name);
- return (BUS_PROBE_DEFAULT);
- }
- ent++;
- }
-
- return (ENXIO);
-}
-
-/*********************************************************************
- * Device initialization routine
- *
- * The attach entry point is called when the driver is being loaded.
- * This routine identifies the type of hardware, allocates all resources
- * and initializes the hardware.
- *
- * return 0 on success, positive on failure
- *********************************************************************/
-
-static int
-lem_attach(device_t dev)
-{
- struct adapter *adapter;
- int tsize, rsize;
- int error = 0;
-
- INIT_DEBUGOUT("lem_attach: begin");
-
- adapter = device_get_softc(dev);
- adapter->dev = adapter->osdep.dev = dev;
- EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
- EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
- EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
-
- /* SYSCTL stuff */
- SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
- OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
- lem_sysctl_nvm_info, "I", "NVM Information");
-
- callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
- callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0);
-
- /* Determine hardware and mac info */
- lem_identify_hardware(adapter);
-
- /* Setup PCI resources */
- if (lem_allocate_pci_resources(adapter)) {
- device_printf(dev, "Allocation of PCI resources failed\n");
- error = ENXIO;
- goto err_pci;
- }
-
- /* Do Shared Code initialization */
- if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
- device_printf(dev, "Setup of Shared code failed\n");
- error = ENXIO;
- goto err_pci;
- }
-
- e1000_get_bus_info(&adapter->hw);
-
- /* Set up some sysctls for the tunable interrupt delays */
- lem_add_int_delay_sysctl(adapter, "rx_int_delay",
- "receive interrupt delay in usecs", &adapter->rx_int_delay,
- E1000_REGISTER(&adapter->hw, E1000_RDTR), lem_rx_int_delay_dflt);
- lem_add_int_delay_sysctl(adapter, "tx_int_delay",
- "transmit interrupt delay in usecs", &adapter->tx_int_delay,
- E1000_REGISTER(&adapter->hw, E1000_TIDV), lem_tx_int_delay_dflt);
- if (adapter->hw.mac.type >= e1000_82540) {
- lem_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
- "receive interrupt delay limit in usecs",
- &adapter->rx_abs_int_delay,
- E1000_REGISTER(&adapter->hw, E1000_RADV),
- lem_rx_abs_int_delay_dflt);
- lem_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
- "transmit interrupt delay limit in usecs",
- &adapter->tx_abs_int_delay,
- E1000_REGISTER(&adapter->hw, E1000_TADV),
- lem_tx_abs_int_delay_dflt);
- lem_add_int_delay_sysctl(adapter, "itr",
- "interrupt delay limit in usecs/4",
- &adapter->tx_itr,
- E1000_REGISTER(&adapter->hw, E1000_ITR),
- DEFAULT_ITR);
- }
-
- /* Sysctls for limiting the amount of work done in the taskqueue */
- lem_add_rx_process_limit(adapter, "rx_processing_limit",
- "max number of rx packets to process", &adapter->rx_process_limit,
- lem_rx_process_limit);
-
-#ifdef NIC_SEND_COMBINING
- /* Sysctls to control mitigation */
- lem_add_rx_process_limit(adapter, "sc_enable",
- "driver TDT mitigation", &adapter->sc_enable, 0);
-#endif /* NIC_SEND_COMBINING */
-#ifdef BATCH_DISPATCH
- lem_add_rx_process_limit(adapter, "batch_enable",
- "driver rx batch", &adapter->batch_enable, 0);
-#endif /* BATCH_DISPATCH */
-
- /* Sysctl for setting the interface flow control */
- lem_set_flow_cntrl(adapter, "flow_control",
- "flow control setting",
- &adapter->fc_setting, lem_fc_setting);
-
- /*
- * Validate number of transmit and receive descriptors. It
- * must not exceed hardware maximum, and must be multiple
- * of E1000_DBA_ALIGN.
- */
- if (((lem_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
- (adapter->hw.mac.type >= e1000_82544 && lem_txd > EM_MAX_TXD) ||
- (adapter->hw.mac.type < e1000_82544 && lem_txd > EM_MAX_TXD_82543) ||
- (lem_txd < EM_MIN_TXD)) {
- device_printf(dev, "Using %d TX descriptors instead of %d!\n",
- EM_DEFAULT_TXD, lem_txd);
- adapter->num_tx_desc = EM_DEFAULT_TXD;
- } else
- adapter->num_tx_desc = lem_txd;
- if (((lem_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
- (adapter->hw.mac.type >= e1000_82544 && lem_rxd > EM_MAX_RXD) ||
- (adapter->hw.mac.type < e1000_82544 && lem_rxd > EM_MAX_RXD_82543) ||
- (lem_rxd < EM_MIN_RXD)) {
- device_printf(dev, "Using %d RX descriptors instead of %d!\n",
- EM_DEFAULT_RXD, lem_rxd);
- adapter->num_rx_desc = EM_DEFAULT_RXD;
- } else
- adapter->num_rx_desc = lem_rxd;
-
- adapter->hw.mac.autoneg = DO_AUTO_NEG;
- adapter->hw.phy.autoneg_wait_to_complete = FALSE;
- adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
- adapter->rx_buffer_len = 2048;
-
- e1000_init_script_state_82541(&adapter->hw, TRUE);
- e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
-
- /* Copper options */
- if (adapter->hw.phy.media_type == e1000_media_type_copper) {
- adapter->hw.phy.mdix = AUTO_ALL_MODES;
- adapter->hw.phy.disable_polarity_correction = FALSE;
- adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
- }
-
- /*
- * Set the frame limits assuming
- * standard ethernet sized frames.
- */
- adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
- adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
-
- /*
- * This controls when hardware reports transmit completion
- * status.
- */
- adapter->hw.mac.report_tx_early = 1;
-
- /*
- * It seems that the descriptor DMA engine on some PCI cards
- * fetches memory past the end of the last descriptor in the
- * ring. These reads are problematic when VT-d (DMAR) busdma
- * is used. Allocate the scratch space to avoid getting
- * faults from DMAR, by requesting scratch memory for one more
- * descriptor.
- */
- tsize = roundup2((adapter->num_tx_desc + 1) *
- sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
-
- /* Allocate Transmit Descriptor ring */
- if (lem_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
- device_printf(dev, "Unable to allocate tx_desc memory\n");
- error = ENOMEM;
- goto err_tx_desc;
- }
- adapter->tx_desc_base =
- (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
-
- /*
- * See comment above txdma allocation for rationale behind +1.
- */
- rsize = roundup2((adapter->num_rx_desc + 1) *
- sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
-
- /* Allocate Receive Descriptor ring */
- if (lem_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
- device_printf(dev, "Unable to allocate rx_desc memory\n");
- error = ENOMEM;
- goto err_rx_desc;
- }
- adapter->rx_desc_base =
- (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
-
- /* Allocate multicast array memory. */
- adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
- MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
- if (adapter->mta == NULL) {
- device_printf(dev, "Can not allocate multicast setup array\n");
- error = ENOMEM;
- goto err_hw_init;
- }
-
- /*
- ** Start from a known state, this is
- ** important in reading the nvm and
- ** mac from that.
- */
- e1000_reset_hw(&adapter->hw);
-
- /* Make sure we have a good EEPROM before we read from it */
- if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
- /*
- ** Some PCI-E parts fail the first check due to
- ** the link being in sleep state, call it again,
- ** if it fails a second time its a real issue.
- */
- if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
- device_printf(dev,
- "The EEPROM Checksum Is Not Valid\n");
- error = EIO;
- goto err_hw_init;
- }
- }
-
- /* Copy the permanent MAC address out of the EEPROM */
- if (e1000_read_mac_addr(&adapter->hw) < 0) {
- device_printf(dev, "EEPROM read error while reading MAC"
- " address\n");
- error = EIO;
- goto err_hw_init;
- }
-
- if (!lem_is_valid_ether_addr(adapter->hw.mac.addr)) {
- device_printf(dev, "Invalid MAC address\n");
- error = EIO;
- goto err_hw_init;
- }
-
- /* Initialize the hardware */
- if (lem_hardware_init(adapter)) {
- device_printf(dev, "Unable to initialize the hardware\n");
- error = EIO;
- goto err_hw_init;
- }
-
- /* Allocate transmit descriptors and buffers */
- if (lem_allocate_transmit_structures(adapter)) {
- device_printf(dev, "Could not setup transmit structures\n");
- error = ENOMEM;
- goto err_tx_struct;
- }
-
- /* Allocate receive descriptors and buffers */
- if (lem_allocate_receive_structures(adapter)) {
- device_printf(dev, "Could not setup receive structures\n");
- error = ENOMEM;
- goto err_rx_struct;
- }
-
- /*
- ** Do interrupt configuration
- */
- error = lem_allocate_irq(adapter);
- if (error)
- goto err_rx_struct;
-
- /*
- * Get Wake-on-Lan and Management info for later use
- */
- lem_get_wakeup(dev);
-
- /* Setup OS specific network interface */
- if (lem_setup_interface(dev, adapter) != 0)
- goto err_rx_struct;
-
- /* Initialize statistics */
- lem_update_stats_counters(adapter);
-
- adapter->hw.mac.get_link_status = 1;
- lem_update_link_status(adapter);
-
- /* Indicate SOL/IDER usage */
- if (e1000_check_reset_block(&adapter->hw))
- device_printf(dev,
- "PHY reset is blocked due to SOL/IDER session.\n");
-
- /* Do we need workaround for 82544 PCI-X adapter? */
- if (adapter->hw.bus.type == e1000_bus_type_pcix &&
- adapter->hw.mac.type == e1000_82544)
- adapter->pcix_82544 = TRUE;
- else
- adapter->pcix_82544 = FALSE;
-
- /* Register for VLAN events */
- adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
- lem_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
- adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
- lem_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
-
- lem_add_hw_stats(adapter);
-
- /* Non-AMT based hardware can now take control from firmware */
- if (adapter->has_manage && !adapter->has_amt)
- lem_get_hw_control(adapter);
-
- /* Tell the stack that the interface is not active */
- if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
-
- adapter->led_dev = led_create(lem_led_func, adapter,
- device_get_nameunit(dev));
-
-#ifdef DEV_NETMAP
- lem_netmap_attach(adapter);
-#endif /* DEV_NETMAP */
- INIT_DEBUGOUT("lem_attach: end");
-
- return (0);
-
-err_rx_struct:
- lem_free_transmit_structures(adapter);
-err_tx_struct:
-err_hw_init:
- lem_release_hw_control(adapter);
- lem_dma_free(adapter, &adapter->rxdma);
-err_rx_desc:
- lem_dma_free(adapter, &adapter->txdma);
-err_tx_desc:
-err_pci:
- if (adapter->ifp != (void *)NULL)
- if_free(adapter->ifp);
- lem_free_pci_resources(adapter);
- free(adapter->mta, M_DEVBUF);
- EM_TX_LOCK_DESTROY(adapter);
- EM_RX_LOCK_DESTROY(adapter);
- EM_CORE_LOCK_DESTROY(adapter);
-
- return (error);
-}
-
-/*********************************************************************
- * Device removal routine
- *
- * The detach entry point is called when the driver is being removed.
- * This routine stops the adapter and deallocates all the resources
- * that were allocated for driver operation.
- *
- * return 0 on success, positive on failure
- *********************************************************************/
-
-static int
-lem_detach(device_t dev)
-{
- struct adapter *adapter = device_get_softc(dev);
- if_t ifp = adapter->ifp;
-
- INIT_DEBUGOUT("em_detach: begin");
-
- /* Make sure VLANS are not using driver */
- if (if_vlantrunkinuse(ifp)) {
- device_printf(dev,"Vlan in use, detach first\n");
- return (EBUSY);
- }
-
-#ifdef DEVICE_POLLING
- if (if_getcapenable(ifp) & IFCAP_POLLING)
- ether_poll_deregister(ifp);
-#endif
-
- if (adapter->led_dev != NULL)
- led_destroy(adapter->led_dev);
-
- EM_CORE_LOCK(adapter);
- EM_TX_LOCK(adapter);
- adapter->in_detach = 1;
- lem_stop(adapter);
- e1000_phy_hw_reset(&adapter->hw);
-
- lem_release_manageability(adapter);
-
- EM_TX_UNLOCK(adapter);
- EM_CORE_UNLOCK(adapter);
-
- /* Unregister VLAN events */
- if (adapter->vlan_attach != NULL)
- EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
- if (adapter->vlan_detach != NULL)
- EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
-
- ether_ifdetach(adapter->ifp);
- callout_drain(&adapter->timer);
- callout_drain(&adapter->tx_fifo_timer);
-
-#ifdef DEV_NETMAP
- netmap_detach(ifp);
-#endif /* DEV_NETMAP */
- lem_free_pci_resources(adapter);
- bus_generic_detach(dev);
- if_free(ifp);
-
- lem_free_transmit_structures(adapter);
- lem_free_receive_structures(adapter);
-
- /* Free Transmit Descriptor ring */
- if (adapter->tx_desc_base) {
- lem_dma_free(adapter, &adapter->txdma);
- adapter->tx_desc_base = NULL;
- }
-
- /* Free Receive Descriptor ring */
- if (adapter->rx_desc_base) {
- lem_dma_free(adapter, &adapter->rxdma);
- adapter->rx_desc_base = NULL;
- }
-
- lem_release_hw_control(adapter);
- free(adapter->mta, M_DEVBUF);
- EM_TX_LOCK_DESTROY(adapter);
- EM_RX_LOCK_DESTROY(adapter);
- EM_CORE_LOCK_DESTROY(adapter);
-
- return (0);
-}
-
-/*********************************************************************
- *
- * Shutdown entry point
- *
- **********************************************************************/
-
-static int
-lem_shutdown(device_t dev)
-{
- return lem_suspend(dev);
-}
-
-/*
- * Suspend/resume device methods.
- */
-static int
-lem_suspend(device_t dev)
-{
- struct adapter *adapter = device_get_softc(dev);
-
- EM_CORE_LOCK(adapter);
-
- lem_release_manageability(adapter);
- lem_release_hw_control(adapter);
- lem_enable_wakeup(dev);
-
- EM_CORE_UNLOCK(adapter);
-
- return bus_generic_suspend(dev);
-}
-
-static int
-lem_resume(device_t dev)
-{
- struct adapter *adapter = device_get_softc(dev);
- if_t ifp = adapter->ifp;
-
- EM_CORE_LOCK(adapter);
- lem_init_locked(adapter);
- lem_init_manageability(adapter);
- EM_CORE_UNLOCK(adapter);
- lem_start(ifp);
-
- return bus_generic_resume(dev);
-}
-
-
-static void
-lem_start_locked(if_t ifp)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- struct mbuf *m_head;
-
- EM_TX_LOCK_ASSERT(adapter);
-
- if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
- IFF_DRV_RUNNING)
- return;
- if (!adapter->link_active)
- return;
-
- /*
- * Force a cleanup if number of TX descriptors
- * available hits the threshold
- */
- if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
- lem_txeof(adapter);
- /* Now do we at least have a minimal? */
- if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
- adapter->no_tx_desc_avail1++;
- return;
- }
- }
-
- while (!if_sendq_empty(ifp)) {
- m_head = if_dequeue(ifp);
-
- if (m_head == NULL)
- break;
- /*
- * Encapsulation can modify our pointer, and or make it
- * NULL on failure. In that event, we can't requeue.
- */
- if (lem_xmit(adapter, &m_head)) {
- if (m_head == NULL)
- break;
- if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
- if_sendq_prepend(ifp, m_head);
- break;
- }
-
- /* Send a copy of the frame to the BPF listener */
- if_etherbpfmtap(ifp, m_head);
-
- /* Set timeout in case hardware has problems transmitting. */
- adapter->watchdog_check = TRUE;
- adapter->watchdog_time = ticks;
- }
- if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
- if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
-
- return;
-}
-
-static void
-lem_start(if_t ifp)
-{
- struct adapter *adapter = if_getsoftc(ifp);
-
- EM_TX_LOCK(adapter);
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
- lem_start_locked(ifp);
- EM_TX_UNLOCK(adapter);
-}
-
-/*********************************************************************
- * Ioctl entry point
- *
- * em_ioctl is called when the user wants to configure the
- * interface.
- *
- * return 0 on success, positive on failure
- **********************************************************************/
-
-static int
-lem_ioctl(if_t ifp, u_long command, caddr_t data)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- struct ifreq *ifr = (struct ifreq *)data;
-#if defined(INET) || defined(INET6)
- struct ifaddr *ifa = (struct ifaddr *)data;
-#endif
- bool avoid_reset = FALSE;
- int error = 0;
-
- if (adapter->in_detach)
- return (error);
-
- switch (command) {
- case SIOCSIFADDR:
-#ifdef INET
- if (ifa->ifa_addr->sa_family == AF_INET)
- avoid_reset = TRUE;
-#endif
-#ifdef INET6
- if (ifa->ifa_addr->sa_family == AF_INET6)
- avoid_reset = TRUE;
-#endif
- /*
- ** Calling init results in link renegotiation,
- ** so we avoid doing it when possible.
- */
- if (avoid_reset) {
- if_setflagbits(ifp, IFF_UP, 0);
- if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
- lem_init(adapter);
-#ifdef INET
- if (!(if_getflags(ifp) & IFF_NOARP))
- arp_ifinit(ifp, ifa);
-#endif
- } else
- error = ether_ioctl(ifp, command, data);
- break;
- case SIOCSIFMTU:
- {
- int max_frame_size;
-
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
-
- EM_CORE_LOCK(adapter);
- switch (adapter->hw.mac.type) {
- case e1000_82542:
- max_frame_size = ETHER_MAX_LEN;
- break;
- default:
- max_frame_size = MAX_JUMBO_FRAME_SIZE;
- }
- if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
- ETHER_CRC_LEN) {
- EM_CORE_UNLOCK(adapter);
- error = EINVAL;
- break;
- }
-
- if_setmtu(ifp, ifr->ifr_mtu);
- adapter->max_frame_size =
- if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN;
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
- lem_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
- break;
- }
- case SIOCSIFFLAGS:
- IOCTL_DEBUGOUT("ioctl rcv'd:\
- SIOCSIFFLAGS (Set Interface Flags)");
- EM_CORE_LOCK(adapter);
- if (if_getflags(ifp) & IFF_UP) {
- if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
- if ((if_getflags(ifp) ^ adapter->if_flags) &
- (IFF_PROMISC | IFF_ALLMULTI)) {
- lem_disable_promisc(adapter);
- lem_set_promisc(adapter);
- }
- } else
- lem_init_locked(adapter);
- } else
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
- EM_TX_LOCK(adapter);
- lem_stop(adapter);
- EM_TX_UNLOCK(adapter);
- }
- adapter->if_flags = if_getflags(ifp);
- EM_CORE_UNLOCK(adapter);
- break;
- case SIOCADDMULTI:
- case SIOCDELMULTI:
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
- EM_CORE_LOCK(adapter);
- lem_disable_intr(adapter);
- lem_set_multi(adapter);
- if (adapter->hw.mac.type == e1000_82542 &&
- adapter->hw.revision_id == E1000_REVISION_2) {
- lem_initialize_receive_unit(adapter);
- }
-#ifdef DEVICE_POLLING
- if (!(if_getcapenable(ifp) & IFCAP_POLLING))
-#endif
- lem_enable_intr(adapter);
- EM_CORE_UNLOCK(adapter);
- }
- break;
- case SIOCSIFMEDIA:
- /* Check SOL/IDER usage */
- EM_CORE_LOCK(adapter);
- if (e1000_check_reset_block(&adapter->hw)) {
- EM_CORE_UNLOCK(adapter);
- device_printf(adapter->dev, "Media change is"
- " blocked due to SOL/IDER session.\n");
- break;
- }
- EM_CORE_UNLOCK(adapter);
- case SIOCGIFMEDIA:
- IOCTL_DEBUGOUT("ioctl rcv'd: \
- SIOCxIFMEDIA (Get/Set Interface Media)");
- error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
- break;
- case SIOCSIFCAP:
- {
- int mask, reinit;
-
- IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
- reinit = 0;
- mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
-#ifdef DEVICE_POLLING
- if (mask & IFCAP_POLLING) {
- if (ifr->ifr_reqcap & IFCAP_POLLING) {
- error = ether_poll_register(lem_poll, ifp);
- if (error)
- return (error);
- EM_CORE_LOCK(adapter);
- lem_disable_intr(adapter);
- if_setcapenablebit(ifp, IFCAP_POLLING, 0);
- EM_CORE_UNLOCK(adapter);
- } else {
- error = ether_poll_deregister(ifp);
- /* Enable interrupt even in error case */
- EM_CORE_LOCK(adapter);
- lem_enable_intr(adapter);
- if_setcapenablebit(ifp, 0, IFCAP_POLLING);
- EM_CORE_UNLOCK(adapter);
- }
- }
-#endif
- if (mask & IFCAP_HWCSUM) {
- if_togglecapenable(ifp, IFCAP_HWCSUM);
- reinit = 1;
- }
- if (mask & IFCAP_VLAN_HWTAGGING) {
- if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
- reinit = 1;
- }
- if ((mask & IFCAP_WOL) &&
- (if_getcapabilities(ifp) & IFCAP_WOL) != 0) {
- if (mask & IFCAP_WOL_MCAST)
- if_togglecapenable(ifp, IFCAP_WOL_MCAST);
- if (mask & IFCAP_WOL_MAGIC)
- if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
- }
- if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING))
- lem_init(adapter);
- if_vlancap(ifp);
- break;
- }
-
- default:
- error = ether_ioctl(ifp, command, data);
- break;
- }
-
- return (error);
-}
-
-
-/*********************************************************************
- * Init entry point
- *
- * This routine is used in two ways. It is used by the stack as
- * init entry point in network interface structure. It is also used
- * by the driver as a hw/sw initialization routine to get to a
- * consistent state.
- *
- * return 0 on success, positive on failure
- **********************************************************************/
-
-static void
-lem_init_locked(struct adapter *adapter)
-{
- if_t ifp = adapter->ifp;
- device_t dev = adapter->dev;
- u32 pba;
-
- INIT_DEBUGOUT("lem_init: begin");
-
- EM_CORE_LOCK_ASSERT(adapter);
-
- EM_TX_LOCK(adapter);
- lem_stop(adapter);
- EM_TX_UNLOCK(adapter);
-
- /*
- * Packet Buffer Allocation (PBA)
- * Writing PBA sets the receive portion of the buffer
- * the remainder is used for the transmit buffer.
- *
- * Devices before the 82547 had a Packet Buffer of 64K.
- * Default allocation: PBA=48K for Rx, leaving 16K for Tx.
- * After the 82547 the buffer was reduced to 40K.
- * Default allocation: PBA=30K for Rx, leaving 10K for Tx.
- * Note: default does not leave enough room for Jumbo Frame >10k.
- */
- switch (adapter->hw.mac.type) {
- case e1000_82547:
- case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
- if (adapter->max_frame_size > 8192)
- pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
- else
- pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
- adapter->tx_fifo_head = 0;
- adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
- adapter->tx_fifo_size =
- (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
- break;
- default:
- /* Devices before 82547 had a Packet Buffer of 64K. */
- if (adapter->max_frame_size > 8192)
- pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
- else
- pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
- }
-
- INIT_DEBUGOUT1("lem_init: pba=%dK",pba);
- E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
-
- /* Get the latest mac address, User can use a LAA */
- bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr,
- ETHER_ADDR_LEN);
-
- /* Put the address into the Receive Address Array */
- e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
-
- /* Initialize the hardware */
- if (lem_hardware_init(adapter)) {
- device_printf(dev, "Unable to initialize the hardware\n");
- return;
- }
- lem_update_link_status(adapter);
-
- /* Setup VLAN support, basic and offload if available */
- E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
-
- /* Set hardware offload abilities */
- if_clearhwassist(ifp);
- if (adapter->hw.mac.type >= e1000_82543) {
- if (if_getcapenable(ifp) & IFCAP_TXCSUM)
- if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
- }
-
- /* Configure for OS presence */
- lem_init_manageability(adapter);
-
- /* Prepare transmit descriptors and buffers */
- lem_setup_transmit_structures(adapter);
- lem_initialize_transmit_unit(adapter);
-
- /* Setup Multicast table */
- lem_set_multi(adapter);
-
- /* Prepare receive descriptors and buffers */
- if (lem_setup_receive_structures(adapter)) {
- device_printf(dev, "Could not setup receive structures\n");
- EM_TX_LOCK(adapter);
- lem_stop(adapter);
- EM_TX_UNLOCK(adapter);
- return;
- }
- lem_initialize_receive_unit(adapter);
-
- /* Use real VLAN Filter support? */
- if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
- if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
- /* Use real VLAN Filter support */
- lem_setup_vlan_hw_support(adapter);
- else {
- u32 ctrl;
- ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
- ctrl |= E1000_CTRL_VME;
- E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
- }
- }
-
- /* Don't lose promiscuous settings */
- lem_set_promisc(adapter);
-
- if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
-
- callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
- e1000_clear_hw_cntrs_base_generic(&adapter->hw);
-
-#ifdef DEVICE_POLLING
- /*
- * Only enable interrupts if we are not polling, make sure
- * they are off otherwise.
- */
- if (if_getcapenable(ifp) & IFCAP_POLLING)
- lem_disable_intr(adapter);
- else
-#endif /* DEVICE_POLLING */
- lem_enable_intr(adapter);
-
- /* AMT based hardware can now take control from firmware */
- if (adapter->has_manage && adapter->has_amt)
- lem_get_hw_control(adapter);
-}
-
-static void
-lem_init(void *arg)
-{
- struct adapter *adapter = arg;
-
- EM_CORE_LOCK(adapter);
- lem_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
-}
-
-
-#ifdef DEVICE_POLLING
-/*********************************************************************
- *
- * Legacy polling routine
- *
- *********************************************************************/
-static int
-lem_poll(if_t ifp, enum poll_cmd cmd, int count)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- u32 reg_icr, rx_done = 0;
-
- EM_CORE_LOCK(adapter);
- if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
- EM_CORE_UNLOCK(adapter);
- return (rx_done);
- }
-
- if (cmd == POLL_AND_CHECK_STATUS) {
- reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
- if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
- callout_stop(&adapter->timer);
- adapter->hw.mac.get_link_status = 1;
- lem_update_link_status(adapter);
- callout_reset(&adapter->timer, hz,
- lem_local_timer, adapter);
- }
- }
- EM_CORE_UNLOCK(adapter);
-
- lem_rxeof(adapter, count, &rx_done);
-
- EM_TX_LOCK(adapter);
- lem_txeof(adapter);
- if(!if_sendq_empty(ifp))
- lem_start_locked(ifp);
- EM_TX_UNLOCK(adapter);
- return (rx_done);
-}
-#endif /* DEVICE_POLLING */
-
-/*********************************************************************
- *
- * Legacy Interrupt Service routine
- *
- *********************************************************************/
-static void
-lem_intr(void *arg)
-{
- struct adapter *adapter = arg;
- if_t ifp = adapter->ifp;
- u32 reg_icr;
-
-
- if ((if_getcapenable(ifp) & IFCAP_POLLING) ||
- ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
- return;
-
- EM_CORE_LOCK(adapter);
- reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
- if (reg_icr & E1000_ICR_RXO)
- adapter->rx_overruns++;
-
- if ((reg_icr == 0xffffffff) || (reg_icr == 0)) {
- EM_CORE_UNLOCK(adapter);
- return;
- }
-
- if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
- callout_stop(&adapter->timer);
- adapter->hw.mac.get_link_status = 1;
- lem_update_link_status(adapter);
- /* Deal with TX cruft when link lost */
- lem_tx_purge(adapter);
- callout_reset(&adapter->timer, hz,
- lem_local_timer, adapter);
- EM_CORE_UNLOCK(adapter);
- return;
- }
-
- EM_CORE_UNLOCK(adapter);
- lem_rxeof(adapter, -1, NULL);
-
- EM_TX_LOCK(adapter);
- lem_txeof(adapter);
- if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) &&
- (!if_sendq_empty(ifp)))
- lem_start_locked(ifp);
- EM_TX_UNLOCK(adapter);
- return;
-}
-
-
-static void
-lem_handle_link(void *context, int pending)
-{
- struct adapter *adapter = context;
- if_t ifp = adapter->ifp;
-
- if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
- return;
-
- EM_CORE_LOCK(adapter);
- callout_stop(&adapter->timer);
- lem_update_link_status(adapter);
- /* Deal with TX cruft when link lost */
- lem_tx_purge(adapter);
- callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
- EM_CORE_UNLOCK(adapter);
-}
-
-
-/* Combined RX/TX handler, used by Legacy and MSI */
-static void
-lem_handle_rxtx(void *context, int pending)
-{
- struct adapter *adapter = context;
- if_t ifp = adapter->ifp;
-
-
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
- bool more = lem_rxeof(adapter, adapter->rx_process_limit, NULL);
- EM_TX_LOCK(adapter);
- lem_txeof(adapter);
- if(!if_sendq_empty(ifp))
- lem_start_locked(ifp);
- EM_TX_UNLOCK(adapter);
- if (more) {
- taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
- return;
- }
- }
-
- if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
- lem_enable_intr(adapter);
-}
-
-/*********************************************************************
- *
- * Fast Legacy/MSI Combined Interrupt Service routine
- *
- *********************************************************************/
-static int
-lem_irq_fast(void *arg)
-{
- struct adapter *adapter = arg;
- if_t ifp;
- u32 reg_icr;
-
- ifp = adapter->ifp;
-
- reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
-
- /* Hot eject? */
- if (reg_icr == 0xffffffff)
- return FILTER_STRAY;
-
- /* Definitely not our interrupt. */
- if (reg_icr == 0x0)
- return FILTER_STRAY;
-
- /*
- * Mask interrupts until the taskqueue is finished running. This is
- * cheap, just assume that it is needed. This also works around the
- * MSI message reordering errata on certain systems.
- */
- lem_disable_intr(adapter);
- taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
-
- /* Link status change */
- if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
- adapter->hw.mac.get_link_status = 1;
- taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
- }
-
- if (reg_icr & E1000_ICR_RXO)
- adapter->rx_overruns++;
- return FILTER_HANDLED;
-}
-
-
-/*********************************************************************
- *
- * Media Ioctl callback
- *
- * This routine is called whenever the user queries the status of
- * the interface using ifconfig.
- *
- **********************************************************************/
-static void
-lem_media_status(if_t ifp, struct ifmediareq *ifmr)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- u_char fiber_type = IFM_1000_SX;
-
- INIT_DEBUGOUT("lem_media_status: begin");
-
- EM_CORE_LOCK(adapter);
- lem_update_link_status(adapter);
-
- ifmr->ifm_status = IFM_AVALID;
- ifmr->ifm_active = IFM_ETHER;
-
- if (!adapter->link_active) {
- EM_CORE_UNLOCK(adapter);
- return;
- }
-
- ifmr->ifm_status |= IFM_ACTIVE;
-
- if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
- (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
- if (adapter->hw.mac.type == e1000_82545)
- fiber_type = IFM_1000_LX;
- ifmr->ifm_active |= fiber_type | IFM_FDX;
- } else {
- switch (adapter->link_speed) {
- case 10:
- ifmr->ifm_active |= IFM_10_T;
- break;
- case 100:
- ifmr->ifm_active |= IFM_100_TX;
- break;
- case 1000:
- ifmr->ifm_active |= IFM_1000_T;
- break;
- }
- if (adapter->link_duplex == FULL_DUPLEX)
- ifmr->ifm_active |= IFM_FDX;
- else
- ifmr->ifm_active |= IFM_HDX;
- }
- EM_CORE_UNLOCK(adapter);
-}
-
-/*********************************************************************
- *
- * Media Ioctl callback
- *
- * This routine is called when the user changes speed/duplex using
- * media/mediopt option with ifconfig.
- *
- **********************************************************************/
-static int
-lem_media_change(if_t ifp)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- struct ifmedia *ifm = &adapter->media;
-
- INIT_DEBUGOUT("lem_media_change: begin");
-
- if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
- return (EINVAL);
-
- EM_CORE_LOCK(adapter);
- switch (IFM_SUBTYPE(ifm->ifm_media)) {
- case IFM_AUTO:
- adapter->hw.mac.autoneg = DO_AUTO_NEG;
- adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
- break;
- case IFM_1000_LX:
- case IFM_1000_SX:
- case IFM_1000_T:
- adapter->hw.mac.autoneg = DO_AUTO_NEG;
- adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
- break;
- case IFM_100_TX:
- adapter->hw.mac.autoneg = FALSE;
- adapter->hw.phy.autoneg_advertised = 0;
- if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
- adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
- else
- adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
- break;
- case IFM_10_T:
- adapter->hw.mac.autoneg = FALSE;
- adapter->hw.phy.autoneg_advertised = 0;
- if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
- adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
- else
- adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
- break;
- default:
- device_printf(adapter->dev, "Unsupported media type\n");
- }
-
- lem_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
-
- return (0);
-}
-
-/*********************************************************************
- *
- * This routine maps the mbufs to tx descriptors.
- *
- * return 0 on success, positive on failure
- **********************************************************************/
-
-static int
-lem_xmit(struct adapter *adapter, struct mbuf **m_headp)
-{
- bus_dma_segment_t segs[EM_MAX_SCATTER];
- bus_dmamap_t map;
- struct em_buffer *tx_buffer, *tx_buffer_mapped;
- struct e1000_tx_desc *ctxd = NULL;
- struct mbuf *m_head;
- u32 txd_upper, txd_lower, txd_used, txd_saved;
- int error, nsegs, i, j, first, last = 0;
-
- m_head = *m_headp;
- txd_upper = txd_lower = txd_used = txd_saved = 0;
-
- /*
- ** When doing checksum offload, it is critical to
- ** make sure the first mbuf has more than header,
- ** because that routine expects data to be present.
- */
- if ((m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) &&
- (m_head->m_len < ETHER_HDR_LEN + sizeof(struct ip))) {
- m_head = m_pullup(m_head, ETHER_HDR_LEN + sizeof(struct ip));
- *m_headp = m_head;
- if (m_head == NULL)
- return (ENOBUFS);
- }
-
- /*
- * Map the packet for DMA
- *
- * Capture the first descriptor index,
- * this descriptor will have the index
- * of the EOP which is the only one that
- * now gets a DONE bit writeback.
- */
- first = adapter->next_avail_tx_desc;
- tx_buffer = &adapter->tx_buffer_area[first];
- tx_buffer_mapped = tx_buffer;
- map = tx_buffer->map;
-
- error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
- *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
-
- /*
- * There are two types of errors we can (try) to handle:
- * - EFBIG means the mbuf chain was too long and bus_dma ran
- * out of segments. Defragment the mbuf chain and try again.
- * - ENOMEM means bus_dma could not obtain enough bounce buffers
- * at this point in time. Defer sending and try again later.
- * All other errors, in particular EINVAL, are fatal and prevent the
- * mbuf chain from ever going through. Drop it and report error.
- */
- if (error == EFBIG) {
- struct mbuf *m;
-
- m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER);
- if (m == NULL) {
- adapter->mbuf_defrag_failed++;
- m_freem(*m_headp);
- *m_headp = NULL;
- return (ENOBUFS);
- }
- *m_headp = m;
-
- /* Try it again */
- error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
- *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
-
- if (error) {
- adapter->no_tx_dma_setup++;
- m_freem(*m_headp);
- *m_headp = NULL;
- return (error);
- }
- } else if (error != 0) {
- adapter->no_tx_dma_setup++;
- return (error);
- }
-
- if (adapter->num_tx_desc_avail < (nsegs + 2)) {
- adapter->no_tx_desc_avail2++;
- bus_dmamap_unload(adapter->txtag, map);
- return (ENOBUFS);
- }
- m_head = *m_headp;
-
- /* Do hardware assists */
- if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
- lem_transmit_checksum_setup(adapter, m_head,
- &txd_upper, &txd_lower);
-
- i = adapter->next_avail_tx_desc;
- if (adapter->pcix_82544)
- txd_saved = i;
-
- /* Set up our transmit descriptors */
- for (j = 0; j < nsegs; j++) {
- bus_size_t seg_len;
- bus_addr_t seg_addr;
- /* If adapter is 82544 and on PCIX bus */
- if(adapter->pcix_82544) {
- DESC_ARRAY desc_array;
- u32 array_elements, counter;
- /*
- * Check the Address and Length combination and
- * split the data accordingly
- */
- array_elements = lem_fill_descriptors(segs[j].ds_addr,
- segs[j].ds_len, &desc_array);
- for (counter = 0; counter < array_elements; counter++) {
- if (txd_used == adapter->num_tx_desc_avail) {
- adapter->next_avail_tx_desc = txd_saved;
- adapter->no_tx_desc_avail2++;
- bus_dmamap_unload(adapter->txtag, map);
- return (ENOBUFS);
- }
- tx_buffer = &adapter->tx_buffer_area[i];
- ctxd = &adapter->tx_desc_base[i];
- ctxd->buffer_addr = htole64(
- desc_array.descriptor[counter].address);
- ctxd->lower.data = htole32(
- (adapter->txd_cmd | txd_lower | (u16)
- desc_array.descriptor[counter].length));
- ctxd->upper.data =
- htole32((txd_upper));
- last = i;
- if (++i == adapter->num_tx_desc)
- i = 0;
- tx_buffer->m_head = NULL;
- tx_buffer->next_eop = -1;
- txd_used++;
- }
- } else {
- tx_buffer = &adapter->tx_buffer_area[i];
- ctxd = &adapter->tx_desc_base[i];
- seg_addr = segs[j].ds_addr;
- seg_len = segs[j].ds_len;
- ctxd->buffer_addr = htole64(seg_addr);
- ctxd->lower.data = htole32(
- adapter->txd_cmd | txd_lower | seg_len);
- ctxd->upper.data =
- htole32(txd_upper);
- last = i;
- if (++i == adapter->num_tx_desc)
- i = 0;
- tx_buffer->m_head = NULL;
- tx_buffer->next_eop = -1;
- }
- }
-
- adapter->next_avail_tx_desc = i;
-
- if (adapter->pcix_82544)
- adapter->num_tx_desc_avail -= txd_used;
- else
- adapter->num_tx_desc_avail -= nsegs;
-
- if (m_head->m_flags & M_VLANTAG) {
- /* Set the vlan id. */
- ctxd->upper.fields.special =
- htole16(m_head->m_pkthdr.ether_vtag);
- /* Tell hardware to add tag */
- ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
- }
-
- tx_buffer->m_head = m_head;
- tx_buffer_mapped->map = tx_buffer->map;
- tx_buffer->map = map;
- bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
-
- /*
- * Last Descriptor of Packet
- * needs End Of Packet (EOP)
- * and Report Status (RS)
- */
- ctxd->lower.data |=
- htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
- /*
- * Keep track in the first buffer which
- * descriptor will be written back
- */
- tx_buffer = &adapter->tx_buffer_area[first];
- tx_buffer->next_eop = last;
- adapter->watchdog_time = ticks;
-
- /*
- * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
- * that this frame is available to transmit.
- */
- bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
-#ifdef NIC_SEND_COMBINING
- if (adapter->sc_enable) {
- if (adapter->shadow_tdt & MIT_PENDING_INT) {
- /* signal intr and data pending */
- adapter->shadow_tdt = MIT_PENDING_TDT | (i & 0xffff);
- return (0);
- } else {
- adapter->shadow_tdt = MIT_PENDING_INT;
- }
- }
-#endif /* NIC_SEND_COMBINING */
-
- if (adapter->hw.mac.type == e1000_82547 &&
- adapter->link_duplex == HALF_DUPLEX)
- lem_82547_move_tail(adapter);
- else {
- E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
- if (adapter->hw.mac.type == e1000_82547)
- lem_82547_update_fifo_head(adapter,
- m_head->m_pkthdr.len);
- }
-
- return (0);
-}
-
-/*********************************************************************
- *
- * 82547 workaround to avoid controller hang in half-duplex environment.
- * The workaround is to avoid queuing a large packet that would span
- * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
- * in this case. We do that only when FIFO is quiescent.
- *
- **********************************************************************/
-static void
-lem_82547_move_tail(void *arg)
-{
- struct adapter *adapter = arg;
- struct e1000_tx_desc *tx_desc;
- u16 hw_tdt, sw_tdt, length = 0;
- bool eop = 0;
-
- EM_TX_LOCK_ASSERT(adapter);
-
- hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
- sw_tdt = adapter->next_avail_tx_desc;
-
- while (hw_tdt != sw_tdt) {
- tx_desc = &adapter->tx_desc_base[hw_tdt];
- length += tx_desc->lower.flags.length;
- eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
- if (++hw_tdt == adapter->num_tx_desc)
- hw_tdt = 0;
-
- if (eop) {
- if (lem_82547_fifo_workaround(adapter, length)) {
- adapter->tx_fifo_wrk_cnt++;
- callout_reset(&adapter->tx_fifo_timer, 1,
- lem_82547_move_tail, adapter);
- break;
- }
- E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
- lem_82547_update_fifo_head(adapter, length);
- length = 0;
- }
- }
-}
-
-static int
-lem_82547_fifo_workaround(struct adapter *adapter, int len)
-{
- int fifo_space, fifo_pkt_len;
-
- fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
-
- if (adapter->link_duplex == HALF_DUPLEX) {
- fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
-
- if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
- if (lem_82547_tx_fifo_reset(adapter))
- return (0);
- else
- return (1);
- }
- }
-
- return (0);
-}
-
-static void
-lem_82547_update_fifo_head(struct adapter *adapter, int len)
-{
- int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
-
- /* tx_fifo_head is always 16 byte aligned */
- adapter->tx_fifo_head += fifo_pkt_len;
- if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
- adapter->tx_fifo_head -= adapter->tx_fifo_size;
- }
-}
-
-
-static int
-lem_82547_tx_fifo_reset(struct adapter *adapter)
-{
- u32 tctl;
-
- if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
- E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
- (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
- E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
- (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
- E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
- (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
- /* Disable TX unit */
- tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
- E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
- tctl & ~E1000_TCTL_EN);
-
- /* Reset FIFO pointers */
- E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
- adapter->tx_head_addr);
- E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
- adapter->tx_head_addr);
- E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
- adapter->tx_head_addr);
- E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
- adapter->tx_head_addr);
-
- /* Re-enable TX unit */
- E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
- E1000_WRITE_FLUSH(&adapter->hw);
-
- adapter->tx_fifo_head = 0;
- adapter->tx_fifo_reset_cnt++;
-
- return (TRUE);
- }
- else {
- return (FALSE);
- }
-}
-
-static void
-lem_set_promisc(struct adapter *adapter)
-{
- if_t ifp = adapter->ifp;
- u32 reg_rctl;
-
- reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
-
- if (if_getflags(ifp) & IFF_PROMISC) {
- reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
- /* Turn this on if you want to see bad packets */
- if (lem_debug_sbp)
- reg_rctl |= E1000_RCTL_SBP;
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
- } else if (if_getflags(ifp) & IFF_ALLMULTI) {
- reg_rctl |= E1000_RCTL_MPE;
- reg_rctl &= ~E1000_RCTL_UPE;
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
- }
-}
-
-static void
-lem_disable_promisc(struct adapter *adapter)
-{
- if_t ifp = adapter->ifp;
- u32 reg_rctl;
- int mcnt = 0;
-
- reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
- reg_rctl &= (~E1000_RCTL_UPE);
- if (if_getflags(ifp) & IFF_ALLMULTI)
- mcnt = MAX_NUM_MULTICAST_ADDRESSES;
- else
- mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES);
-
- /* Don't disable if in MAX groups */
- if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
- reg_rctl &= (~E1000_RCTL_MPE);
- reg_rctl &= (~E1000_RCTL_SBP);
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
-}
-
-
-/*********************************************************************
- * Multicast Update
- *
- * This routine is called whenever multicast address list is updated.
- *
- **********************************************************************/
-
-static void
-lem_set_multi(struct adapter *adapter)
-{
- if_t ifp = adapter->ifp;
- u32 reg_rctl = 0;
- u8 *mta; /* Multicast array memory */
- int mcnt = 0;
-
- IOCTL_DEBUGOUT("lem_set_multi: begin");
-
- mta = adapter->mta;
- bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
-
- if (adapter->hw.mac.type == e1000_82542 &&
- adapter->hw.revision_id == E1000_REVISION_2) {
- reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
- if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
- e1000_pci_clear_mwi(&adapter->hw);
- reg_rctl |= E1000_RCTL_RST;
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
- msec_delay(5);
- }
-
- if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES);
-
- if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
- reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
- reg_rctl |= E1000_RCTL_MPE;
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
- } else
- e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
-
- if (adapter->hw.mac.type == e1000_82542 &&
- adapter->hw.revision_id == E1000_REVISION_2) {
- reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
- reg_rctl &= ~E1000_RCTL_RST;
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
- msec_delay(5);
- if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
- e1000_pci_set_mwi(&adapter->hw);
- }
-}
-
-
-/*********************************************************************
- * Timer routine
- *
- * This routine checks for link status and updates statistics.
- *
- **********************************************************************/
-
-static void
-lem_local_timer(void *arg)
-{
- struct adapter *adapter = arg;
-
- EM_CORE_LOCK_ASSERT(adapter);
-
- lem_update_link_status(adapter);
- lem_update_stats_counters(adapter);
-
- lem_smartspeed(adapter);
-
- /*
- * We check the watchdog: the time since
- * the last TX descriptor was cleaned.
- * This implies a functional TX engine.
- */
- if ((adapter->watchdog_check == TRUE) &&
- (ticks - adapter->watchdog_time > EM_WATCHDOG))
- goto hung;
-
- callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
- return;
-hung:
- device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
- if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_RUNNING);
- adapter->watchdog_events++;
- lem_init_locked(adapter);
-}
-
-static void
-lem_update_link_status(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- if_t ifp = adapter->ifp;
- device_t dev = adapter->dev;
- u32 link_check = 0;
-
- /* Get the cached link value or read phy for real */
- switch (hw->phy.media_type) {
- case e1000_media_type_copper:
- if (hw->mac.get_link_status) {
- /* Do the work to read phy */
- e1000_check_for_link(hw);
- link_check = !hw->mac.get_link_status;
- if (link_check) /* ESB2 fix */
- e1000_cfg_on_link_up(hw);
- } else
- link_check = TRUE;
- break;
- case e1000_media_type_fiber:
- e1000_check_for_link(hw);
- link_check = (E1000_READ_REG(hw, E1000_STATUS) &
- E1000_STATUS_LU);
- break;
- case e1000_media_type_internal_serdes:
- e1000_check_for_link(hw);
- link_check = adapter->hw.mac.serdes_has_link;
- break;
- default:
- case e1000_media_type_unknown:
- break;
- }
-
- /* Now check for a transition */
- if (link_check && (adapter->link_active == 0)) {
- e1000_get_speed_and_duplex(hw, &adapter->link_speed,
- &adapter->link_duplex);
- if (bootverbose)
- device_printf(dev, "Link is up %d Mbps %s\n",
- adapter->link_speed,
- ((adapter->link_duplex == FULL_DUPLEX) ?
- "Full Duplex" : "Half Duplex"));
- adapter->link_active = 1;
- adapter->smartspeed = 0;
- if_setbaudrate(ifp, adapter->link_speed * 1000000);
- if_link_state_change(ifp, LINK_STATE_UP);
- } else if (!link_check && (adapter->link_active == 1)) {
- if_setbaudrate(ifp, 0);
- adapter->link_speed = 0;
- adapter->link_duplex = 0;
- if (bootverbose)
- device_printf(dev, "Link is Down\n");
- adapter->link_active = 0;
- /* Link down, disable watchdog */
- adapter->watchdog_check = FALSE;
- if_link_state_change(ifp, LINK_STATE_DOWN);
- }
-}
-
-/*********************************************************************
- *
- * This routine disables all traffic on the adapter by issuing a
- * global reset on the MAC and deallocates TX/RX buffers.
- *
- * This routine should always be called with BOTH the CORE
- * and TX locks.
- **********************************************************************/
-
-static void
-lem_stop(void *arg)
-{
- struct adapter *adapter = arg;
- if_t ifp = adapter->ifp;
-
- EM_CORE_LOCK_ASSERT(adapter);
- EM_TX_LOCK_ASSERT(adapter);
-
- INIT_DEBUGOUT("lem_stop: begin");
-
- lem_disable_intr(adapter);
- callout_stop(&adapter->timer);
- callout_stop(&adapter->tx_fifo_timer);
-
- /* Tell the stack that the interface is no longer active */
- if_setdrvflagbits(ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE));
-
- e1000_reset_hw(&adapter->hw);
- if (adapter->hw.mac.type >= e1000_82544)
- E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
-
- e1000_led_off(&adapter->hw);
- e1000_cleanup_led(&adapter->hw);
-}
-
-
-/*********************************************************************
- *
- * Determine hardware revision.
- *
- **********************************************************************/
-static void
-lem_identify_hardware(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
-
- /* Make sure our PCI config space has the necessary stuff set */
- pci_enable_busmaster(dev);
- adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
-
- /* Save off the information about this board */
- adapter->hw.vendor_id = pci_get_vendor(dev);
- adapter->hw.device_id = pci_get_device(dev);
- adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
- adapter->hw.subsystem_vendor_id =
- pci_read_config(dev, PCIR_SUBVEND_0, 2);
- adapter->hw.subsystem_device_id =
- pci_read_config(dev, PCIR_SUBDEV_0, 2);
-
- /* Do Shared Code Init and Setup */
- if (e1000_set_mac_type(&adapter->hw)) {
- device_printf(dev, "Setup init failure\n");
- return;
- }
-}
-
-static int
-lem_allocate_pci_resources(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- int val, rid, error = E1000_SUCCESS;
-
- rid = PCIR_BAR(0);
- adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
- &rid, RF_ACTIVE);
- if (adapter->memory == NULL) {
- device_printf(dev, "Unable to allocate bus resource: memory\n");
- return (ENXIO);
- }
- adapter->osdep.mem_bus_space_tag =
- rman_get_bustag(adapter->memory);
- adapter->osdep.mem_bus_space_handle =
- rman_get_bushandle(adapter->memory);
- adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
-
- /* Only older adapters use IO mapping */
- if (adapter->hw.mac.type > e1000_82543) {
- /* Figure our where our IO BAR is ? */
- for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
- val = pci_read_config(dev, rid, 4);
- if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
- adapter->io_rid = rid;
- break;
- }
- rid += 4;
- /* check for 64bit BAR */
- if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
- rid += 4;
- }
- if (rid >= PCIR_CIS) {
- device_printf(dev, "Unable to locate IO BAR\n");
- return (ENXIO);
- }
- adapter->ioport = bus_alloc_resource_any(dev,
- SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
- if (adapter->ioport == NULL) {
- device_printf(dev, "Unable to allocate bus resource: "
- "ioport\n");
- return (ENXIO);
- }
- adapter->hw.io_base = 0;
- adapter->osdep.io_bus_space_tag =
- rman_get_bustag(adapter->ioport);
- adapter->osdep.io_bus_space_handle =
- rman_get_bushandle(adapter->ioport);
- }
-
- adapter->hw.back = &adapter->osdep;
-
- return (error);
-}
-
-/*********************************************************************
- *
- * Setup the Legacy or MSI Interrupt handler
- *
- **********************************************************************/
-int
-lem_allocate_irq(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- int error, rid = 0;
-
- /* Manually turn off all interrupts */
- E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
-
- /* We allocate a single interrupt resource */
- adapter->res[0] = bus_alloc_resource_any(dev,
- SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
- if (adapter->res[0] == NULL) {
- device_printf(dev, "Unable to allocate bus resource: "
- "interrupt\n");
- return (ENXIO);
- }
-
- /* Do Legacy setup? */
- if (lem_use_legacy_irq) {
- if ((error = bus_setup_intr(dev, adapter->res[0],
- INTR_TYPE_NET | INTR_MPSAFE, NULL, lem_intr, adapter,
- &adapter->tag[0])) != 0) {
- device_printf(dev,
- "Failed to register interrupt handler");
- return (error);
- }
- return (0);
- }
-
- /*
- * Use a Fast interrupt and the associated
- * deferred processing contexts.
- */
- TASK_INIT(&adapter->rxtx_task, 0, lem_handle_rxtx, adapter);
- TASK_INIT(&adapter->link_task, 0, lem_handle_link, adapter);
- adapter->tq = taskqueue_create_fast("lem_taskq", M_NOWAIT,
- taskqueue_thread_enqueue, &adapter->tq);
- taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
- device_get_nameunit(adapter->dev));
- if ((error = bus_setup_intr(dev, adapter->res[0],
- INTR_TYPE_NET, lem_irq_fast, NULL, adapter,
- &adapter->tag[0])) != 0) {
- device_printf(dev, "Failed to register fast interrupt "
- "handler: %d\n", error);
- taskqueue_free(adapter->tq);
- adapter->tq = NULL;
- return (error);
- }
-
- return (0);
-}
-
-
-static void
-lem_free_pci_resources(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
-
-
- if (adapter->tag[0] != NULL) {
- bus_teardown_intr(dev, adapter->res[0],
- adapter->tag[0]);
- adapter->tag[0] = NULL;
- }
-
- if (adapter->res[0] != NULL) {
- bus_release_resource(dev, SYS_RES_IRQ,
- 0, adapter->res[0]);
- }
-
- if (adapter->memory != NULL)
- bus_release_resource(dev, SYS_RES_MEMORY,
- PCIR_BAR(0), adapter->memory);
-
- if (adapter->ioport != NULL)
- bus_release_resource(dev, SYS_RES_IOPORT,
- adapter->io_rid, adapter->ioport);
-}
-
-
-/*********************************************************************
- *
- * Initialize the hardware to a configuration
- * as specified by the adapter structure.
- *
- **********************************************************************/
-static int
-lem_hardware_init(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- u16 rx_buffer_size;
-
- INIT_DEBUGOUT("lem_hardware_init: begin");
-
- /* Issue a global reset */
- e1000_reset_hw(&adapter->hw);
-
- /* When hardware is reset, fifo_head is also reset */
- adapter->tx_fifo_head = 0;
-
- /*
- * These parameters control the automatic generation (Tx) and
- * response (Rx) to Ethernet PAUSE frames.
- * - High water mark should allow for at least two frames to be
- * received after sending an XOFF.
- * - Low water mark works best when it is very near the high water mark.
- * This allows the receiver to restart by sending XON when it has
- * drained a bit. Here we use an arbitrary value of 1500 which will
- * restart after one full frame is pulled from the buffer. There
- * could be several smaller frames in the buffer and if so they will
- * not trigger the XON until their total number reduces the buffer
- * by 1500.
- * - The pause time is fairly large at 1000 x 512ns = 512 usec.
- */
- rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
- 0xffff) << 10 );
-
- adapter->hw.fc.high_water = rx_buffer_size -
- roundup2(adapter->max_frame_size, 1024);
- adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
-
- adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
- adapter->hw.fc.send_xon = TRUE;
-
- /* Set Flow control, use the tunable location if sane */
- if ((lem_fc_setting >= 0) && (lem_fc_setting < 4))
- adapter->hw.fc.requested_mode = lem_fc_setting;
- else
- adapter->hw.fc.requested_mode = e1000_fc_none;
-
- if (e1000_init_hw(&adapter->hw) < 0) {
- device_printf(dev, "Hardware Initialization Failed\n");
- return (EIO);
- }
-
- e1000_check_for_link(&adapter->hw);
-
- return (0);
-}
-
-/*********************************************************************
- *
- * Setup networking device structure and register an interface.
- *
- **********************************************************************/
-static int
-lem_setup_interface(device_t dev, struct adapter *adapter)
-{
- if_t ifp;
-
- INIT_DEBUGOUT("lem_setup_interface: begin");
-
- ifp = adapter->ifp = if_gethandle(IFT_ETHER);
- if (ifp == (void *)NULL) {
- device_printf(dev, "can not allocate ifnet structure\n");
- return (-1);
- }
- if_initname(ifp, device_get_name(dev), device_get_unit(dev));
- if_setinitfn(ifp, lem_init);
- if_setsoftc(ifp, adapter);
- if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
- if_setioctlfn(ifp, lem_ioctl);
- if_setstartfn(ifp, lem_start);
- if_setgetcounterfn(ifp, lem_get_counter);
- if_setsendqlen(ifp, adapter->num_tx_desc - 1);
- if_setsendqready(ifp);
-
- ether_ifattach(ifp, adapter->hw.mac.addr);
-
- if_setcapabilities(ifp, 0);
-
- if (adapter->hw.mac.type >= e1000_82543) {
- if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0);
- if_setcapenablebit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0);
- }
-
- /*
- * Tell the upper layer(s) we support long frames.
- */
- if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
- if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0);
- if_setcapenablebit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0);
-
- /*
- ** Dont turn this on by default, if vlans are
- ** created on another pseudo device (eg. lagg)
- ** then vlan events are not passed thru, breaking
- ** operation, but with HW FILTER off it works. If
- ** using vlans directly on the em driver you can
- ** enable this and get full hardware tag filtering.
- */
- if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0);
-
-#ifdef DEVICE_POLLING
- if_setcapabilitiesbit(ifp, IFCAP_POLLING, 0);
-#endif
-
- /* Enable only WOL MAGIC by default */
- if (adapter->wol) {
- if_setcapabilitiesbit(ifp, IFCAP_WOL, 0);
- if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0);
- }
-
- /*
- * Specify the media types supported by this adapter and register
- * callbacks to update media and link information
- */
- ifmedia_init(&adapter->media, IFM_IMASK,
- lem_media_change, lem_media_status);
- if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
- (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
- u_char fiber_type = IFM_1000_SX; /* default type */
-
- if (adapter->hw.mac.type == e1000_82545)
- fiber_type = IFM_1000_LX;
- ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
- } else {
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
- 0, NULL);
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
- 0, NULL);
- if (adapter->hw.phy.type != e1000_phy_ife) {
- ifmedia_add(&adapter->media,
- IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
- ifmedia_add(&adapter->media,
- IFM_ETHER | IFM_1000_T, 0, NULL);
- }
- }
- ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
- ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
- return (0);
-}
-
-
-/*********************************************************************
- *
- * Workaround for SmartSpeed on 82541 and 82547 controllers
- *
- **********************************************************************/
-static void
-lem_smartspeed(struct adapter *adapter)
-{
- u16 phy_tmp;
-
- if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
- adapter->hw.mac.autoneg == 0 ||
- (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
- return;
-
- if (adapter->smartspeed == 0) {
- /* If Master/Slave config fault is asserted twice,
- * we assume back-to-back */
- e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
- if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
- return;
- e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
- if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
- e1000_read_phy_reg(&adapter->hw,
- PHY_1000T_CTRL, &phy_tmp);
- if(phy_tmp & CR_1000T_MS_ENABLE) {
- phy_tmp &= ~CR_1000T_MS_ENABLE;
- e1000_write_phy_reg(&adapter->hw,
- PHY_1000T_CTRL, phy_tmp);
- adapter->smartspeed++;
- if(adapter->hw.mac.autoneg &&
- !e1000_copper_link_autoneg(&adapter->hw) &&
- !e1000_read_phy_reg(&adapter->hw,
- PHY_CONTROL, &phy_tmp)) {
- phy_tmp |= (MII_CR_AUTO_NEG_EN |
- MII_CR_RESTART_AUTO_NEG);
- e1000_write_phy_reg(&adapter->hw,
- PHY_CONTROL, phy_tmp);
- }
- }
- }
- return;
- } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
- /* If still no link, perhaps using 2/3 pair cable */
- e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
- phy_tmp |= CR_1000T_MS_ENABLE;
- e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
- if(adapter->hw.mac.autoneg &&
- !e1000_copper_link_autoneg(&adapter->hw) &&
- !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
- phy_tmp |= (MII_CR_AUTO_NEG_EN |
- MII_CR_RESTART_AUTO_NEG);
- e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
- }
- }
- /* Restart process after EM_SMARTSPEED_MAX iterations */
- if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
- adapter->smartspeed = 0;
-}
-
-
-/*
- * Manage DMA'able memory.
- */
-static void
-lem_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
-{
- if (error)
- return;
- *(bus_addr_t *) arg = segs[0].ds_addr;
-}
-
-static int
-lem_dma_malloc(struct adapter *adapter, bus_size_t size,
- struct em_dma_alloc *dma, int mapflags)
-{
- int error;
-
- error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
- EM_DBA_ALIGN, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- size, /* maxsize */
- 1, /* nsegments */
- size, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockarg */
- &dma->dma_tag);
- if (error) {
- device_printf(adapter->dev,
- "%s: bus_dma_tag_create failed: %d\n",
- __func__, error);
- goto fail_0;
- }
-
- error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
- BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
- if (error) {
- device_printf(adapter->dev,
- "%s: bus_dmamem_alloc(%ju) failed: %d\n",
- __func__, (uintmax_t)size, error);
- goto fail_2;
- }
-
- dma->dma_paddr = 0;
- error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
- size, lem_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
- if (error || dma->dma_paddr == 0) {
- device_printf(adapter->dev,
- "%s: bus_dmamap_load failed: %d\n",
- __func__, error);
- goto fail_3;
- }
-
- return (0);
-
-fail_3:
- bus_dmamap_unload(dma->dma_tag, dma->dma_map);
-fail_2:
- bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
- bus_dma_tag_destroy(dma->dma_tag);
-fail_0:
- dma->dma_tag = NULL;
-
- return (error);
-}
-
-static void
-lem_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
-{
- if (dma->dma_tag == NULL)
- return;
- if (dma->dma_paddr != 0) {
- bus_dmamap_sync(dma->dma_tag, dma->dma_map,
- BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(dma->dma_tag, dma->dma_map);
- dma->dma_paddr = 0;
- }
- if (dma->dma_vaddr != NULL) {
- bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
- dma->dma_vaddr = NULL;
- }
- bus_dma_tag_destroy(dma->dma_tag);
- dma->dma_tag = NULL;
-}
-
-
-/*********************************************************************
- *
- * Allocate memory for tx_buffer structures. The tx_buffer stores all
- * the information needed to transmit a packet on the wire.
- *
- **********************************************************************/
-static int
-lem_allocate_transmit_structures(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- struct em_buffer *tx_buffer;
- int error;
-
- /*
- * Create DMA tags for tx descriptors
- */
- if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
- 1, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- MCLBYTES * EM_MAX_SCATTER, /* maxsize */
- EM_MAX_SCATTER, /* nsegments */
- MCLBYTES, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockarg */
- &adapter->txtag)) != 0) {
- device_printf(dev, "Unable to allocate TX DMA tag\n");
- goto fail;
- }
-
- adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
- adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
- if (adapter->tx_buffer_area == NULL) {
- device_printf(dev, "Unable to allocate tx_buffer memory\n");
- error = ENOMEM;
- goto fail;
- }
-
- /* Create the descriptor buffer dma maps */
- for (int i = 0; i < adapter->num_tx_desc; i++) {
- tx_buffer = &adapter->tx_buffer_area[i];
- error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
- if (error != 0) {
- device_printf(dev, "Unable to create TX DMA map\n");
- goto fail;
- }
- tx_buffer->next_eop = -1;
- }
-
- return (0);
-fail:
- lem_free_transmit_structures(adapter);
- return (error);
-}
-
-/*********************************************************************
- *
- * (Re)Initialize transmit structures.
- *
- **********************************************************************/
-static void
-lem_setup_transmit_structures(struct adapter *adapter)
-{
- struct em_buffer *tx_buffer;
-#ifdef DEV_NETMAP
- /* we are already locked */
- struct netmap_adapter *na = netmap_getna(adapter->ifp);
- struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0);
-#endif /* DEV_NETMAP */
-
- /* Clear the old ring contents */
- bzero(adapter->tx_desc_base,
- (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
-
- /* Free any existing TX buffers */
- for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
- tx_buffer = &adapter->tx_buffer_area[i];
- bus_dmamap_sync(adapter->txtag, tx_buffer->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->txtag, tx_buffer->map);
- m_freem(tx_buffer->m_head);
- tx_buffer->m_head = NULL;
-#ifdef DEV_NETMAP
- if (slot) {
- /* the i-th NIC entry goes to slot si */
- int si = netmap_idx_n2k(&na->tx_rings[0], i);
- uint64_t paddr;
- void *addr;
-
- addr = PNMB(na, slot + si, &paddr);
- adapter->tx_desc_base[i].buffer_addr = htole64(paddr);
- /* reload the map for netmap mode */
- netmap_load_map(na, adapter->txtag, tx_buffer->map, addr);
- }
-#endif /* DEV_NETMAP */
- tx_buffer->next_eop = -1;
- }
-
- /* Reset state */
- adapter->last_hw_offload = 0;
- adapter->next_avail_tx_desc = 0;
- adapter->next_tx_to_clean = 0;
- adapter->num_tx_desc_avail = adapter->num_tx_desc;
-
- bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
- return;
-}
-
-/*********************************************************************
- *
- * Enable transmit unit.
- *
- **********************************************************************/
-static void
-lem_initialize_transmit_unit(struct adapter *adapter)
-{
- u32 tctl, tipg = 0;
- u64 bus_addr;
-
- INIT_DEBUGOUT("lem_initialize_transmit_unit: begin");
- /* Setup the Base and Length of the Tx Descriptor Ring */
- bus_addr = adapter->txdma.dma_paddr;
- E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
- adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
- E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
- (u32)(bus_addr >> 32));
- E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
- (u32)bus_addr);
- /* Setup the HW Tx Head and Tail descriptor pointers */
- E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
- E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
-
- HW_DEBUGOUT2("Base = %x, Length = %x\n",
- E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
- E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
-
- /* Set the default values for the Tx Inter Packet Gap timer */
- switch (adapter->hw.mac.type) {
- case e1000_82542:
- tipg = DEFAULT_82542_TIPG_IPGT;
- tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
- tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
- break;
- default:
- if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
- (adapter->hw.phy.media_type ==
- e1000_media_type_internal_serdes))
- tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
- else
- tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
- tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
- tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
- }
-
- E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
- E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
- if(adapter->hw.mac.type >= e1000_82540)
- E1000_WRITE_REG(&adapter->hw, E1000_TADV,
- adapter->tx_abs_int_delay.value);
-
- /* Program the Transmit Control Register */
- tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
- tctl &= ~E1000_TCTL_CT;
- tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
- (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
-
- /* This write will effectively turn on the transmit unit. */
- E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
-
- /* Setup Transmit Descriptor Base Settings */
- adapter->txd_cmd = E1000_TXD_CMD_IFCS;
-
- if (adapter->tx_int_delay.value > 0)
- adapter->txd_cmd |= E1000_TXD_CMD_IDE;
-}
-
-/*********************************************************************
- *
- * Free all transmit related data structures.
- *
- **********************************************************************/
-static void
-lem_free_transmit_structures(struct adapter *adapter)
-{
- struct em_buffer *tx_buffer;
-
- INIT_DEBUGOUT("free_transmit_structures: begin");
-
- if (adapter->tx_buffer_area != NULL) {
- for (int i = 0; i < adapter->num_tx_desc; i++) {
- tx_buffer = &adapter->tx_buffer_area[i];
- if (tx_buffer->m_head != NULL) {
- bus_dmamap_sync(adapter->txtag, tx_buffer->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->txtag,
- tx_buffer->map);
- m_freem(tx_buffer->m_head);
- tx_buffer->m_head = NULL;
- } else if (tx_buffer->map != NULL)
- bus_dmamap_unload(adapter->txtag,
- tx_buffer->map);
- if (tx_buffer->map != NULL) {
- bus_dmamap_destroy(adapter->txtag,
- tx_buffer->map);
- tx_buffer->map = NULL;
- }
- }
- }
- if (adapter->tx_buffer_area != NULL) {
- free(adapter->tx_buffer_area, M_DEVBUF);
- adapter->tx_buffer_area = NULL;
- }
- if (adapter->txtag != NULL) {
- bus_dma_tag_destroy(adapter->txtag);
- adapter->txtag = NULL;
- }
-}
-
-/*********************************************************************
- *
- * The offload context needs to be set when we transfer the first
- * packet of a particular protocol (TCP/UDP). This routine has been
- * enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
- *
- * Added back the old method of keeping the current context type
- * and not setting if unnecessary, as this is reported to be a
- * big performance win. -jfv
- **********************************************************************/
-static void
-lem_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
- u32 *txd_upper, u32 *txd_lower)
-{
- struct e1000_context_desc *TXD = NULL;
- struct em_buffer *tx_buffer;
- struct ether_vlan_header *eh;
- struct ip *ip = NULL;
- struct ip6_hdr *ip6;
- int curr_txd, ehdrlen;
- u32 cmd, hdr_len, ip_hlen;
- u16 etype;
- u8 ipproto;
-
-
- cmd = hdr_len = ipproto = 0;
- *txd_upper = *txd_lower = 0;
- curr_txd = adapter->next_avail_tx_desc;
-
- /*
- * Determine where frame payload starts.
- * Jump over vlan headers if already present,
- * helpful for QinQ too.
- */
- eh = mtod(mp, struct ether_vlan_header *);
- if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
- etype = ntohs(eh->evl_proto);
- ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
- } else {
- etype = ntohs(eh->evl_encap_proto);
- ehdrlen = ETHER_HDR_LEN;
- }
-
- /*
- * We only support TCP/UDP for IPv4 and IPv6 for the moment.
- * TODO: Support SCTP too when it hits the tree.
- */
- switch (etype) {
- case ETHERTYPE_IP:
- ip = (struct ip *)(mp->m_data + ehdrlen);
- ip_hlen = ip->ip_hl << 2;
-
- /* Setup of IP header checksum. */
- if (mp->m_pkthdr.csum_flags & CSUM_IP) {
- /*
- * Start offset for header checksum calculation.
- * End offset for header checksum calculation.
- * Offset of place to put the checksum.
- */
- TXD = (struct e1000_context_desc *)
- &adapter->tx_desc_base[curr_txd];
- TXD->lower_setup.ip_fields.ipcss = ehdrlen;
- TXD->lower_setup.ip_fields.ipcse =
- htole16(ehdrlen + ip_hlen);
- TXD->lower_setup.ip_fields.ipcso =
- ehdrlen + offsetof(struct ip, ip_sum);
- cmd |= E1000_TXD_CMD_IP;
- *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
- }
-
- hdr_len = ehdrlen + ip_hlen;
- ipproto = ip->ip_p;
-
- break;
- case ETHERTYPE_IPV6:
- ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
- ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
-
- /* IPv6 doesn't have a header checksum. */
-
- hdr_len = ehdrlen + ip_hlen;
- ipproto = ip6->ip6_nxt;
- break;
-
- default:
- return;
- }
-
- switch (ipproto) {
- case IPPROTO_TCP:
- if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
- *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
- *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
- /* no need for context if already set */
- if (adapter->last_hw_offload == CSUM_TCP)
- return;
- adapter->last_hw_offload = CSUM_TCP;
- /*
- * Start offset for payload checksum calculation.
- * End offset for payload checksum calculation.
- * Offset of place to put the checksum.
- */
- TXD = (struct e1000_context_desc *)
- &adapter->tx_desc_base[curr_txd];
- TXD->upper_setup.tcp_fields.tucss = hdr_len;
- TXD->upper_setup.tcp_fields.tucse = htole16(0);
- TXD->upper_setup.tcp_fields.tucso =
- hdr_len + offsetof(struct tcphdr, th_sum);
- cmd |= E1000_TXD_CMD_TCP;
- }
- break;
- case IPPROTO_UDP:
- {
- if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
- *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
- *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
- /* no need for context if already set */
- if (adapter->last_hw_offload == CSUM_UDP)
- return;
- adapter->last_hw_offload = CSUM_UDP;
- /*
- * Start offset for header checksum calculation.
- * End offset for header checksum calculation.
- * Offset of place to put the checksum.
- */
- TXD = (struct e1000_context_desc *)
- &adapter->tx_desc_base[curr_txd];
- TXD->upper_setup.tcp_fields.tucss = hdr_len;
- TXD->upper_setup.tcp_fields.tucse = htole16(0);
- TXD->upper_setup.tcp_fields.tucso =
- hdr_len + offsetof(struct udphdr, uh_sum);
- }
- /* Fall Thru */
- }
- default:
- break;
- }
-
- if (TXD == NULL)
- return;
- TXD->tcp_seg_setup.data = htole32(0);
- TXD->cmd_and_length =
- htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
- tx_buffer = &adapter->tx_buffer_area[curr_txd];
- tx_buffer->m_head = NULL;
- tx_buffer->next_eop = -1;
-
- if (++curr_txd == adapter->num_tx_desc)
- curr_txd = 0;
-
- adapter->num_tx_desc_avail--;
- adapter->next_avail_tx_desc = curr_txd;
-}
-
-
-/**********************************************************************
- *
- * Examine each tx_buffer in the used queue. If the hardware is done
- * processing the packet then free associated resources. The
- * tx_buffer is put back on the free queue.
- *
- **********************************************************************/
-static void
-lem_txeof(struct adapter *adapter)
-{
- int first, last, done, num_avail;
- struct em_buffer *tx_buffer;
- struct e1000_tx_desc *tx_desc, *eop_desc;
- if_t ifp = adapter->ifp;
-
- EM_TX_LOCK_ASSERT(adapter);
-
-#ifdef DEV_NETMAP
- if (netmap_tx_irq(ifp, 0))
- return;
-#endif /* DEV_NETMAP */
- if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
- return;
-
- num_avail = adapter->num_tx_desc_avail;
- first = adapter->next_tx_to_clean;
- tx_desc = &adapter->tx_desc_base[first];
- tx_buffer = &adapter->tx_buffer_area[first];
- last = tx_buffer->next_eop;
- eop_desc = &adapter->tx_desc_base[last];
-
- /*
- * What this does is get the index of the
- * first descriptor AFTER the EOP of the
- * first packet, that way we can do the
- * simple comparison on the inner while loop.
- */
- if (++last == adapter->num_tx_desc)
- last = 0;
- done = last;
-
- bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
- BUS_DMASYNC_POSTREAD);
-
- while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
- /* We clean the range of the packet */
- while (first != done) {
- tx_desc->upper.data = 0;
- tx_desc->lower.data = 0;
- tx_desc->buffer_addr = 0;
- ++num_avail;
-
- if (tx_buffer->m_head) {
- if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
- bus_dmamap_sync(adapter->txtag,
- tx_buffer->map,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->txtag,
- tx_buffer->map);
-
- m_freem(tx_buffer->m_head);
- tx_buffer->m_head = NULL;
- }
- tx_buffer->next_eop = -1;
- adapter->watchdog_time = ticks;
-
- if (++first == adapter->num_tx_desc)
- first = 0;
-
- tx_buffer = &adapter->tx_buffer_area[first];
- tx_desc = &adapter->tx_desc_base[first];
- }
- /* See if we can continue to the next packet */
- last = tx_buffer->next_eop;
- if (last != -1) {
- eop_desc = &adapter->tx_desc_base[last];
- /* Get new done point */
- if (++last == adapter->num_tx_desc) last = 0;
- done = last;
- } else
- break;
- }
- bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
- adapter->next_tx_to_clean = first;
- adapter->num_tx_desc_avail = num_avail;
-
-#ifdef NIC_SEND_COMBINING
- if ((adapter->shadow_tdt & MIT_PENDING_TDT) == MIT_PENDING_TDT) {
- /* a tdt write is pending, do it */
- E1000_WRITE_REG(&adapter->hw, E1000_TDT(0),
- 0xffff & adapter->shadow_tdt);
- adapter->shadow_tdt = MIT_PENDING_INT;
- } else {
- adapter->shadow_tdt = 0; // disable
- }
-#endif /* NIC_SEND_COMBINING */
- /*
- * If we have enough room, clear IFF_DRV_OACTIVE to
- * tell the stack that it is OK to send packets.
- * If there are no pending descriptors, clear the watchdog.
- */
- if (adapter->num_tx_desc_avail > EM_TX_CLEANUP_THRESHOLD) {
- if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
- if (adapter->num_tx_desc_avail == adapter->num_tx_desc) {
- adapter->watchdog_check = FALSE;
- return;
- }
- }
-}
-
-/*********************************************************************
- *
- * When Link is lost sometimes there is work still in the TX ring
- * which may result in a watchdog, rather than allow that we do an
- * attempted cleanup and then reinit here. Note that this has been
- * seens mostly with fiber adapters.
- *
- **********************************************************************/
-static void
-lem_tx_purge(struct adapter *adapter)
-{
- if ((!adapter->link_active) && (adapter->watchdog_check)) {
- EM_TX_LOCK(adapter);
- lem_txeof(adapter);
- EM_TX_UNLOCK(adapter);
- if (adapter->watchdog_check) /* Still outstanding? */
- lem_init_locked(adapter);
- }
-}
-
-/*********************************************************************
- *
- * Get a buffer from system mbuf buffer pool.
- *
- **********************************************************************/
-static int
-lem_get_buf(struct adapter *adapter, int i)
-{
- struct mbuf *m;
- bus_dma_segment_t segs[1];
- bus_dmamap_t map;
- struct em_buffer *rx_buffer;
- int error, nsegs;
-
- m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
- if (m == NULL) {
- adapter->mbuf_cluster_failed++;
- return (ENOBUFS);
- }
- m->m_len = m->m_pkthdr.len = MCLBYTES;
-
- if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
- m_adj(m, ETHER_ALIGN);
-
- /*
- * Using memory from the mbuf cluster pool, invoke the
- * bus_dma machinery to arrange the memory mapping.
- */
- error = bus_dmamap_load_mbuf_sg(adapter->rxtag,
- adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
- if (error != 0) {
- m_free(m);
- return (error);
- }
-
- /* If nsegs is wrong then the stack is corrupt. */
- KASSERT(nsegs == 1, ("Too many segments returned!"));
-
- rx_buffer = &adapter->rx_buffer_area[i];
- if (rx_buffer->m_head != NULL)
- bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
-
- map = rx_buffer->map;
- rx_buffer->map = adapter->rx_sparemap;
- adapter->rx_sparemap = map;
- bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
- rx_buffer->m_head = m;
-
- adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
- return (0);
-}
-
-/*********************************************************************
- *
- * Allocate memory for rx_buffer structures. Since we use one
- * rx_buffer per received packet, the maximum number of rx_buffer's
- * that we'll need is equal to the number of receive descriptors
- * that we've allocated.
- *
- **********************************************************************/
-static int
-lem_allocate_receive_structures(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
- struct em_buffer *rx_buffer;
- int i, error;
-
- adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
- adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
- if (adapter->rx_buffer_area == NULL) {
- device_printf(dev, "Unable to allocate rx_buffer memory\n");
- return (ENOMEM);
- }
-
- error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
- 1, 0, /* alignment, bounds */
- BUS_SPACE_MAXADDR, /* lowaddr */
- BUS_SPACE_MAXADDR, /* highaddr */
- NULL, NULL, /* filter, filterarg */
- MCLBYTES, /* maxsize */
- 1, /* nsegments */
- MCLBYTES, /* maxsegsize */
- 0, /* flags */
- NULL, /* lockfunc */
- NULL, /* lockarg */
- &adapter->rxtag);
- if (error) {
- device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
- __func__, error);
- goto fail;
- }
-
- /* Create the spare map (used by getbuf) */
- error = bus_dmamap_create(adapter->rxtag, 0, &adapter->rx_sparemap);
- if (error) {
- device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
- __func__, error);
- goto fail;
- }
-
- rx_buffer = adapter->rx_buffer_area;
- for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
- error = bus_dmamap_create(adapter->rxtag, 0, &rx_buffer->map);
- if (error) {
- device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
- __func__, error);
- goto fail;
- }
- }
-
- return (0);
-
-fail:
- lem_free_receive_structures(adapter);
- return (error);
-}
-
-/*********************************************************************
- *
- * (Re)initialize receive structures.
- *
- **********************************************************************/
-static int
-lem_setup_receive_structures(struct adapter *adapter)
-{
- struct em_buffer *rx_buffer;
- int i, error;
-#ifdef DEV_NETMAP
- /* we are already under lock */
- struct netmap_adapter *na = netmap_getna(adapter->ifp);
- struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0);
-#endif
-
- /* Reset descriptor ring */
- bzero(adapter->rx_desc_base,
- (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
-
- /* Free current RX buffers. */
- rx_buffer = adapter->rx_buffer_area;
- for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
- if (rx_buffer->m_head != NULL) {
- bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
- m_freem(rx_buffer->m_head);
- rx_buffer->m_head = NULL;
- }
- }
-
- /* Allocate new ones. */
- for (i = 0; i < adapter->num_rx_desc; i++) {
-#ifdef DEV_NETMAP
- if (slot) {
- /* the i-th NIC entry goes to slot si */
- int si = netmap_idx_n2k(&na->rx_rings[0], i);
- uint64_t paddr;
- void *addr;
-
- addr = PNMB(na, slot + si, &paddr);
- netmap_load_map(na, adapter->rxtag, rx_buffer->map, addr);
- /* Update descriptor */
- adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
- continue;
- }
-#endif /* DEV_NETMAP */
- error = lem_get_buf(adapter, i);
- if (error)
- return (error);
- }
-
- /* Setup our descriptor pointers */
- adapter->next_rx_desc_to_check = 0;
- bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
- return (0);
-}
-
-/*********************************************************************
- *
- * Enable receive unit.
- *
- **********************************************************************/
-
-static void
-lem_initialize_receive_unit(struct adapter *adapter)
-{
- if_t ifp = adapter->ifp;
- u64 bus_addr;
- u32 rctl, rxcsum;
-
- INIT_DEBUGOUT("lem_initialize_receive_unit: begin");
-
- /*
- * Make sure receives are disabled while setting
- * up the descriptor ring
- */
- rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
-
- if (adapter->hw.mac.type >= e1000_82540) {
- E1000_WRITE_REG(&adapter->hw, E1000_RADV,
- adapter->rx_abs_int_delay.value);
- /*
- * Set the interrupt throttling rate. Value is calculated
- * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
- */
- E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR);
- }
-
- /* Setup the Base and Length of the Rx Descriptor Ring */
- bus_addr = adapter->rxdma.dma_paddr;
- E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
- adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
- E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
- (u32)(bus_addr >> 32));
- E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
- (u32)bus_addr);
-
- /* Setup the Receive Control Register */
- rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
- rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
- E1000_RCTL_RDMTS_HALF |
- (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
-
- /* Make sure VLAN Filters are off */
- rctl &= ~E1000_RCTL_VFE;
-
- if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
- rctl |= E1000_RCTL_SBP;
- else
- rctl &= ~E1000_RCTL_SBP;
-
- switch (adapter->rx_buffer_len) {
- default:
- case 2048:
- rctl |= E1000_RCTL_SZ_2048;
- break;
- case 4096:
- rctl |= E1000_RCTL_SZ_4096 |
- E1000_RCTL_BSEX | E1000_RCTL_LPE;
- break;
- case 8192:
- rctl |= E1000_RCTL_SZ_8192 |
- E1000_RCTL_BSEX | E1000_RCTL_LPE;
- break;
- case 16384:
- rctl |= E1000_RCTL_SZ_16384 |
- E1000_RCTL_BSEX | E1000_RCTL_LPE;
- break;
- }
-
- if (if_getmtu(ifp) > ETHERMTU)
- rctl |= E1000_RCTL_LPE;
- else
- rctl &= ~E1000_RCTL_LPE;
-
- /* Enable 82543 Receive Checksum Offload for TCP and UDP */
- if ((adapter->hw.mac.type >= e1000_82543) &&
- (if_getcapenable(ifp) & IFCAP_RXCSUM)) {
- rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
- rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
- E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
- }
-
- /* Enable Receives */
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
-
- /*
- * Setup the HW Rx Head and
- * Tail Descriptor Pointers
- */
- E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
- rctl = adapter->num_rx_desc - 1; /* default RDT value */
-#ifdef DEV_NETMAP
- /* preserve buffers already made available to clients */
- if (if_getcapenable(ifp) & IFCAP_NETMAP) {
- struct netmap_adapter *na = netmap_getna(adapter->ifp);
- rctl -= nm_kr_rxspace(&na->rx_rings[0]);
- }
-#endif /* DEV_NETMAP */
- E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), rctl);
-
- return;
-}
-
-/*********************************************************************
- *
- * Free receive related data structures.
- *
- **********************************************************************/
-static void
-lem_free_receive_structures(struct adapter *adapter)
-{
- struct em_buffer *rx_buffer;
- int i;
-
- INIT_DEBUGOUT("free_receive_structures: begin");
-
- if (adapter->rx_sparemap) {
- bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
- adapter->rx_sparemap = NULL;
- }
-
- /* Cleanup any existing buffers */
- if (adapter->rx_buffer_area != NULL) {
- rx_buffer = adapter->rx_buffer_area;
- for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
- if (rx_buffer->m_head != NULL) {
- bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
- BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(adapter->rxtag,
- rx_buffer->map);
- m_freem(rx_buffer->m_head);
- rx_buffer->m_head = NULL;
- } else if (rx_buffer->map != NULL)
- bus_dmamap_unload(adapter->rxtag,
- rx_buffer->map);
- if (rx_buffer->map != NULL) {
- bus_dmamap_destroy(adapter->rxtag,
- rx_buffer->map);
- rx_buffer->map = NULL;
- }
- }
- }
-
- if (adapter->rx_buffer_area != NULL) {
- free(adapter->rx_buffer_area, M_DEVBUF);
- adapter->rx_buffer_area = NULL;
- }
-
- if (adapter->rxtag != NULL) {
- bus_dma_tag_destroy(adapter->rxtag);
- adapter->rxtag = NULL;
- }
-}
-
-/*********************************************************************
- *
- * This routine executes in interrupt context. It replenishes
- * the mbufs in the descriptor and sends data which has been
- * dma'ed into host memory to upper layer.
- *
- * We loop at most count times if count is > 0, or until done if
- * count < 0.
- *
- * For polling we also now return the number of cleaned packets
- *********************************************************************/
-static bool
-lem_rxeof(struct adapter *adapter, int count, int *done)
-{
- if_t ifp = adapter->ifp;
- struct mbuf *mp;
- u8 status = 0, accept_frame = 0, eop = 0;
- u16 len, desc_len, prev_len_adj;
- int i, rx_sent = 0;
- struct e1000_rx_desc *current_desc;
-
-#ifdef BATCH_DISPATCH
- struct mbuf *mh = NULL, *mt = NULL;
-#endif /* BATCH_DISPATCH */
- EM_RX_LOCK(adapter);
-
-#ifdef BATCH_DISPATCH
- batch_again:
-#endif /* BATCH_DISPATCH */
- i = adapter->next_rx_desc_to_check;
- current_desc = &adapter->rx_desc_base[i];
- bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
- BUS_DMASYNC_POSTREAD);
-
-#ifdef DEV_NETMAP
- if (netmap_rx_irq(ifp, 0, &rx_sent)) {
- EM_RX_UNLOCK(adapter);
- return (FALSE);
- }
-#endif /* DEV_NETMAP */
-
- if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
- if (done != NULL)
- *done = rx_sent;
- EM_RX_UNLOCK(adapter);
- return (FALSE);
- }
-
- while (count != 0 && if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
- struct mbuf *m = NULL;
-
- status = current_desc->status;
- if ((status & E1000_RXD_STAT_DD) == 0) {
- break;
- }
-
- mp = adapter->rx_buffer_area[i].m_head;
- /*
- * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
- * needs to access the last received byte in the mbuf.
- */
- bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
- BUS_DMASYNC_POSTREAD);
-
- accept_frame = 1;
- prev_len_adj = 0;
- desc_len = le16toh(current_desc->length);
- if (status & E1000_RXD_STAT_EOP) {
- count--;
- eop = 1;
- if (desc_len < ETHER_CRC_LEN) {
- len = 0;
- prev_len_adj = ETHER_CRC_LEN - desc_len;
- } else
- len = desc_len - ETHER_CRC_LEN;
- } else {
- eop = 0;
- len = desc_len;
- }
-
- if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
- u8 last_byte;
- u32 pkt_len = desc_len;
-
- if (adapter->fmp != NULL)
- pkt_len += adapter->fmp->m_pkthdr.len;
-
- last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
- if (TBI_ACCEPT(&adapter->hw, status,
- current_desc->errors, pkt_len, last_byte,
- adapter->min_frame_size, adapter->max_frame_size)) {
- e1000_tbi_adjust_stats_82543(&adapter->hw,
- &adapter->stats, pkt_len,
- adapter->hw.mac.addr,
- adapter->max_frame_size);
- if (len > 0)
- len--;
- } else
- accept_frame = 0;
- }
-
- if (accept_frame) {
- if (lem_get_buf(adapter, i) != 0) {
- if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
- goto discard;
- }
-
- /* Assign correct length to the current fragment */
- mp->m_len = len;
-
- if (adapter->fmp == NULL) {
- mp->m_pkthdr.len = len;
- adapter->fmp = mp; /* Store the first mbuf */
- adapter->lmp = mp;
- } else {
- /* Chain mbuf's together */
- mp->m_flags &= ~M_PKTHDR;
- /*
- * Adjust length of previous mbuf in chain if
- * we received less than 4 bytes in the last
- * descriptor.
- */
- if (prev_len_adj > 0) {
- adapter->lmp->m_len -= prev_len_adj;
- adapter->fmp->m_pkthdr.len -=
- prev_len_adj;
- }
- adapter->lmp->m_next = mp;
- adapter->lmp = adapter->lmp->m_next;
- adapter->fmp->m_pkthdr.len += len;
- }
-
- if (eop) {
- if_setrcvif(adapter->fmp, ifp);
- if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
- lem_receive_checksum(adapter, current_desc,
- adapter->fmp);
-#ifndef __NO_STRICT_ALIGNMENT
- if (adapter->max_frame_size >
- (MCLBYTES - ETHER_ALIGN) &&
- lem_fixup_rx(adapter) != 0)
- goto skip;
-#endif
- if (status & E1000_RXD_STAT_VP) {
- adapter->fmp->m_pkthdr.ether_vtag =
- le16toh(current_desc->special);
- adapter->fmp->m_flags |= M_VLANTAG;
- }
-#ifndef __NO_STRICT_ALIGNMENT
-skip:
-#endif
- m = adapter->fmp;
- adapter->fmp = NULL;
- adapter->lmp = NULL;
- }
- } else {
- adapter->dropped_pkts++;
-discard:
- /* Reuse loaded DMA map and just update mbuf chain */
- mp = adapter->rx_buffer_area[i].m_head;
- mp->m_len = mp->m_pkthdr.len = MCLBYTES;
- mp->m_data = mp->m_ext.ext_buf;
- mp->m_next = NULL;
- if (adapter->max_frame_size <=
- (MCLBYTES - ETHER_ALIGN))
- m_adj(mp, ETHER_ALIGN);
- if (adapter->fmp != NULL) {
- m_freem(adapter->fmp);
- adapter->fmp = NULL;
- adapter->lmp = NULL;
- }
- m = NULL;
- }
-
- /* Zero out the receive descriptors status. */
- current_desc->status = 0;
- bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
- /* Advance our pointers to the next descriptor. */
- if (++i == adapter->num_rx_desc)
- i = 0;
- /* Call into the stack */
- if (m != NULL) {
-#ifdef BATCH_DISPATCH
- if (adapter->batch_enable) {
- if (mh == NULL)
- mh = mt = m;
- else
- mt->m_nextpkt = m;
- mt = m;
- m->m_nextpkt = NULL;
- rx_sent++;
- current_desc = &adapter->rx_desc_base[i];
- continue;
- }
-#endif /* BATCH_DISPATCH */
- adapter->next_rx_desc_to_check = i;
- EM_RX_UNLOCK(adapter);
- if_input(ifp, m);
- EM_RX_LOCK(adapter);
- rx_sent++;
- i = adapter->next_rx_desc_to_check;
- }
- current_desc = &adapter->rx_desc_base[i];
- }
- adapter->next_rx_desc_to_check = i;
-#ifdef BATCH_DISPATCH
- if (mh) {
- EM_RX_UNLOCK(adapter);
- while ( (mt = mh) != NULL) {
- mh = mh->m_nextpkt;
- mt->m_nextpkt = NULL;
- if_input(ifp, mt);
- }
- EM_RX_LOCK(adapter);
- i = adapter->next_rx_desc_to_check; /* in case of interrupts */
- if (count > 0)
- goto batch_again;
- }
-#endif /* BATCH_DISPATCH */
-
- /* Advance the E1000's Receive Queue #0 "Tail Pointer". */
- if (--i < 0)
- i = adapter->num_rx_desc - 1;
- E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
- if (done != NULL)
- *done = rx_sent;
- EM_RX_UNLOCK(adapter);
- return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
-}
-
-#ifndef __NO_STRICT_ALIGNMENT
-/*
- * When jumbo frames are enabled we should realign entire payload on
- * architecures with strict alignment. This is serious design mistake of 8254x
- * as it nullifies DMA operations. 8254x just allows RX buffer size to be
- * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
- * payload. On architecures without strict alignment restrictions 8254x still
- * performs unaligned memory access which would reduce the performance too.
- * To avoid copying over an entire frame to align, we allocate a new mbuf and
- * copy ethernet header to the new mbuf. The new mbuf is prepended into the
- * existing mbuf chain.
- *
- * Be aware, best performance of the 8254x is achieved only when jumbo frame is
- * not used at all on architectures with strict alignment.
- */
-static int
-lem_fixup_rx(struct adapter *adapter)
-{
- struct mbuf *m, *n;
- int error;
-
- error = 0;
- m = adapter->fmp;
- if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
- bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
- m->m_data += ETHER_HDR_LEN;
- } else {
- MGETHDR(n, M_NOWAIT, MT_DATA);
- if (n != NULL) {
- bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
- m->m_data += ETHER_HDR_LEN;
- m->m_len -= ETHER_HDR_LEN;
- n->m_len = ETHER_HDR_LEN;
- M_MOVE_PKTHDR(n, m);
- n->m_next = m;
- adapter->fmp = n;
- } else {
- adapter->dropped_pkts++;
- m_freem(adapter->fmp);
- adapter->fmp = NULL;
- error = ENOMEM;
- }
- }
-
- return (error);
-}
-#endif
-
-/*********************************************************************
- *
- * Verify that the hardware indicated that the checksum is valid.
- * Inform the stack about the status of checksum so that stack
- * doesn't spend time verifying the checksum.
- *
- *********************************************************************/
-static void
-lem_receive_checksum(struct adapter *adapter,
- struct e1000_rx_desc *rx_desc, struct mbuf *mp)
-{
- /* 82543 or newer only */
- if ((adapter->hw.mac.type < e1000_82543) ||
- /* Ignore Checksum bit is set */
- (rx_desc->status & E1000_RXD_STAT_IXSM)) {
- mp->m_pkthdr.csum_flags = 0;
- return;
- }
-
- if (rx_desc->status & E1000_RXD_STAT_IPCS) {
- /* Did it pass? */
- if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
- /* IP Checksum Good */
- mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
- mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
-
- } else {
- mp->m_pkthdr.csum_flags = 0;
- }
- }
-
- if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
- /* Did it pass? */
- if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
- mp->m_pkthdr.csum_flags |=
- (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
- mp->m_pkthdr.csum_data = htons(0xffff);
- }
- }
-}
-
-/*
- * This routine is run via an vlan
- * config EVENT
- */
-static void
-lem_register_vlan(void *arg, if_t ifp, u16 vtag)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- u32 index, bit;
-
- if (if_getsoftc(ifp) != arg) /* Not our event */
- return;
-
- if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */
- return;
-
- EM_CORE_LOCK(adapter);
- index = (vtag >> 5) & 0x7F;
- bit = vtag & 0x1F;
- adapter->shadow_vfta[index] |= (1 << bit);
- ++adapter->num_vlans;
- /* Re-init to load the changes */
- if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
- lem_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
-}
-
-/*
- * This routine is run via an vlan
- * unconfig EVENT
- */
-static void
-lem_unregister_vlan(void *arg, if_t ifp, u16 vtag)
-{
- struct adapter *adapter = if_getsoftc(ifp);
- u32 index, bit;
-
- if (if_getsoftc(ifp) != arg)
- return;
-
- if ((vtag == 0) || (vtag > 4095)) /* Invalid */
- return;
-
- EM_CORE_LOCK(adapter);
- index = (vtag >> 5) & 0x7F;
- bit = vtag & 0x1F;
- adapter->shadow_vfta[index] &= ~(1 << bit);
- --adapter->num_vlans;
- /* Re-init to load the changes */
- if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
- lem_init_locked(adapter);
- EM_CORE_UNLOCK(adapter);
-}
-
-static void
-lem_setup_vlan_hw_support(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- u32 reg;
-
- /*
- ** We get here thru init_locked, meaning
- ** a soft reset, this has already cleared
- ** the VFTA and other state, so if there
- ** have been no vlan's registered do nothing.
- */
- if (adapter->num_vlans == 0)
- return;
-
- /*
- ** A soft reset zero's out the VFTA, so
- ** we need to repopulate it now.
- */
- for (int i = 0; i < EM_VFTA_SIZE; i++)
- if (adapter->shadow_vfta[i] != 0)
- E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
- i, adapter->shadow_vfta[i]);
-
- reg = E1000_READ_REG(hw, E1000_CTRL);
- reg |= E1000_CTRL_VME;
- E1000_WRITE_REG(hw, E1000_CTRL, reg);
-
- /* Enable the Filter Table */
- reg = E1000_READ_REG(hw, E1000_RCTL);
- reg &= ~E1000_RCTL_CFIEN;
- reg |= E1000_RCTL_VFE;
- E1000_WRITE_REG(hw, E1000_RCTL, reg);
-}
-
-static void
-lem_enable_intr(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- u32 ims_mask = IMS_ENABLE_MASK;
-
- E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
-}
-
-static void
-lem_disable_intr(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
-
- E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff);
-}
-
-/*
- * Bit of a misnomer, what this really means is
- * to enable OS management of the system... aka
- * to disable special hardware management features
- */
-static void
-lem_init_manageability(struct adapter *adapter)
-{
- /* A shared code workaround */
- if (adapter->has_manage) {
- int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
- /* disable hardware interception of ARP */
- manc &= ~(E1000_MANC_ARP_EN);
- E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
- }
-}
-
-/*
- * Give control back to hardware management
- * controller if there is one.
- */
-static void
-lem_release_manageability(struct adapter *adapter)
-{
- if (adapter->has_manage) {
- int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
-
- /* re-enable hardware interception of ARP */
- manc |= E1000_MANC_ARP_EN;
- E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
- }
-}
-
-/*
- * lem_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
- * For ASF and Pass Through versions of f/w this means
- * that the driver is loaded. For AMT version type f/w
- * this means that the network i/f is open.
- */
-static void
-lem_get_hw_control(struct adapter *adapter)
-{
- u32 ctrl_ext;
-
- ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
- E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
- ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
- return;
-}
-
-/*
- * lem_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
- * For ASF and Pass Through versions of f/w this means that
- * the driver is no longer loaded. For AMT versions of the
- * f/w this means that the network i/f is closed.
- */
-static void
-lem_release_hw_control(struct adapter *adapter)
-{
- u32 ctrl_ext;
-
- if (!adapter->has_manage)
- return;
-
- ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
- E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
- ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
- return;
-}
-
-static int
-lem_is_valid_ether_addr(u8 *addr)
-{
- char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
-
- if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
- return (FALSE);
- }
-
- return (TRUE);
-}
-
-/*
-** Parse the interface capabilities with regard
-** to both system management and wake-on-lan for
-** later use.
-*/
-static void
-lem_get_wakeup(device_t dev)
-{
- struct adapter *adapter = device_get_softc(dev);
- u16 eeprom_data = 0, device_id, apme_mask;
-
- adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
- apme_mask = EM_EEPROM_APME;
-
- switch (adapter->hw.mac.type) {
- case e1000_82542:
- case e1000_82543:
- break;
- case e1000_82544:
- e1000_read_nvm(&adapter->hw,
- NVM_INIT_CONTROL2_REG, 1, &eeprom_data);
- apme_mask = EM_82544_APME;
- break;
- case e1000_82546:
- case e1000_82546_rev_3:
- if (adapter->hw.bus.func == 1) {
- e1000_read_nvm(&adapter->hw,
- NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
- break;
- } else
- e1000_read_nvm(&adapter->hw,
- NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
- break;
- default:
- e1000_read_nvm(&adapter->hw,
- NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
- break;
- }
- if (eeprom_data & apme_mask)
- adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
- /*
- * We have the eeprom settings, now apply the special cases
- * where the eeprom may be wrong or the board won't support
- * wake on lan on a particular port
- */
- device_id = pci_get_device(dev);
- switch (device_id) {
- case E1000_DEV_ID_82546GB_PCIE:
- adapter->wol = 0;
- break;
- case E1000_DEV_ID_82546EB_FIBER:
- case E1000_DEV_ID_82546GB_FIBER:
- /* Wake events only supported on port A for dual fiber
- * regardless of eeprom setting */
- if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
- E1000_STATUS_FUNC_1)
- adapter->wol = 0;
- break;
- case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
- /* if quad port adapter, disable WoL on all but port A */
- if (global_quad_port_a != 0)
- adapter->wol = 0;
- /* Reset for multiple quad port adapters */
- if (++global_quad_port_a == 4)
- global_quad_port_a = 0;
- break;
- }
- return;
-}
-
-
-/*
- * Enable PCI Wake On Lan capability
- */
-static void
-lem_enable_wakeup(device_t dev)
-{
- struct adapter *adapter = device_get_softc(dev);
- if_t ifp = adapter->ifp;
- u32 pmc, ctrl, ctrl_ext, rctl;
- u16 status;
-
- if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
- return;
-
- /* Advertise the wakeup capability */
- ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
- ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
- E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
- E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
-
- /* Keep the laser running on Fiber adapters */
- if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
- adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
- ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
- ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
- E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
- }
-
- /*
- ** Determine type of Wakeup: note that wol
- ** is set with all bits on by default.
- */
- if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0)
- adapter->wol &= ~E1000_WUFC_MAG;
-
- if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0)
- adapter->wol &= ~E1000_WUFC_MC;
- else {
- rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
- rctl |= E1000_RCTL_MPE;
- E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
- }
-
- if (adapter->hw.mac.type == e1000_pchlan) {
- if (lem_enable_phy_wakeup(adapter))
- return;
- } else {
- E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
- E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
- }
-
-
- /* Request PME */
- status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
- status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
- if (if_getcapenable(ifp) & IFCAP_WOL)
- status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
- pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
-
- return;
-}
-
-/*
-** WOL in the newer chipset interfaces (pchlan)
-** require thing to be copied into the phy
-*/
-static int
-lem_enable_phy_wakeup(struct adapter *adapter)
-{
- struct e1000_hw *hw = &adapter->hw;
- u32 mreg, ret = 0;
- u16 preg;
-
- /* copy MAC RARs to PHY RARs */
- for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) {
- mreg = E1000_READ_REG(hw, E1000_RAL(i));
- e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF));
- e1000_write_phy_reg(hw, BM_RAR_M(i),
- (u16)((mreg >> 16) & 0xFFFF));
- mreg = E1000_READ_REG(hw, E1000_RAH(i));
- e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF));
- e1000_write_phy_reg(hw, BM_RAR_CTRL(i),
- (u16)((mreg >> 16) & 0xFFFF));
- }
-
- /* copy MAC MTA to PHY MTA */
- for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
- mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
- e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
- e1000_write_phy_reg(hw, BM_MTA(i) + 1,
- (u16)((mreg >> 16) & 0xFFFF));
- }
-
- /* configure PHY Rx Control register */
- e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
- mreg = E1000_READ_REG(hw, E1000_RCTL);
- if (mreg & E1000_RCTL_UPE)
- preg |= BM_RCTL_UPE;
- if (mreg & E1000_RCTL_MPE)
- preg |= BM_RCTL_MPE;
- preg &= ~(BM_RCTL_MO_MASK);
- if (mreg & E1000_RCTL_MO_3)
- preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
- << BM_RCTL_MO_SHIFT);
- if (mreg & E1000_RCTL_BAM)
- preg |= BM_RCTL_BAM;
- if (mreg & E1000_RCTL_PMCF)
- preg |= BM_RCTL_PMCF;
- mreg = E1000_READ_REG(hw, E1000_CTRL);
- if (mreg & E1000_CTRL_RFCE)
- preg |= BM_RCTL_RFCE;
- e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
-
- /* enable PHY wakeup in MAC register */
- E1000_WRITE_REG(hw, E1000_WUC,
- E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
- E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
-
- /* configure and enable PHY wakeup in PHY registers */
- e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
- e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
-
- /* activate PHY wakeup */
- ret = hw->phy.ops.acquire(hw);
- if (ret) {
- printf("Could not acquire PHY\n");
- return ret;
- }
- e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
- (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
- ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
- if (ret) {
- printf("Could not read PHY page 769\n");
- goto out;
- }
- preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
- ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
- if (ret)
- printf("Could not set PHY Host Wakeup bit\n");
-out:
- hw->phy.ops.release(hw);
-
- return ret;
-}
-
-static void
-lem_led_func(void *arg, int onoff)
-{
- struct adapter *adapter = arg;
-
- EM_CORE_LOCK(adapter);
- if (onoff) {
- e1000_setup_led(&adapter->hw);
- e1000_led_on(&adapter->hw);
- } else {
- e1000_led_off(&adapter->hw);
- e1000_cleanup_led(&adapter->hw);
- }
- EM_CORE_UNLOCK(adapter);
-}
-
-/*********************************************************************
-* 82544 Coexistence issue workaround.
-* There are 2 issues.
-* 1. Transmit Hang issue.
-* To detect this issue, following equation can be used...
-* SIZE[3:0] + ADDR[2:0] = SUM[3:0].
-* If SUM[3:0] is in between 1 to 4, we will have this issue.
-*
-* 2. DAC issue.
-* To detect this issue, following equation can be used...
-* SIZE[3:0] + ADDR[2:0] = SUM[3:0].
-* If SUM[3:0] is in between 9 to c, we will have this issue.
-*
-*
-* WORKAROUND:
-* Make sure we do not have ending address
-* as 1,2,3,4(Hang) or 9,a,b,c (DAC)
-*
-*************************************************************************/
-static u32
-lem_fill_descriptors (bus_addr_t address, u32 length,
- PDESC_ARRAY desc_array)
-{
- u32 safe_terminator;
-
- /* Since issue is sensitive to length and address.*/
- /* Let us first check the address...*/
- if (length <= 4) {
- desc_array->descriptor[0].address = address;
- desc_array->descriptor[0].length = length;
- desc_array->elements = 1;
- return (desc_array->elements);
- }
- safe_terminator = (u32)((((u32)address & 0x7) +
- (length & 0xF)) & 0xF);
- /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
- if (safe_terminator == 0 ||
- (safe_terminator > 4 &&
- safe_terminator < 9) ||
- (safe_terminator > 0xC &&
- safe_terminator <= 0xF)) {
- desc_array->descriptor[0].address = address;
- desc_array->descriptor[0].length = length;
- desc_array->elements = 1;
- return (desc_array->elements);
- }
-
- desc_array->descriptor[0].address = address;
- desc_array->descriptor[0].length = length - 4;
- desc_array->descriptor[1].address = address + (length - 4);
- desc_array->descriptor[1].length = 4;
- desc_array->elements = 2;
- return (desc_array->elements);
-}
-
-/**********************************************************************
- *
- * Update the board statistics counters.
- *
- **********************************************************************/
-static void
-lem_update_stats_counters(struct adapter *adapter)
-{
-
- if(adapter->hw.phy.media_type == e1000_media_type_copper ||
- (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
- adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
- adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
- }
- adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
- adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
- adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
- adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
-
- adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
- adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
- adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
- adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
- adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
- adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
- adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
- adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
- adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
- adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
- adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
- adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
- adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
- adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
- adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
- adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
- adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
- adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
- adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
- adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
-
- /* For the 64-bit byte counters the low dword must be read first. */
- /* Both registers clear on the read of the high dword */
-
- adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
- ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
- adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
- ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
-
- adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
- adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
- adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
- adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
- adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
-
- adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
- adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
-
- adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
- adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
- adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
- adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
- adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
- adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
- adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
- adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
- adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
- adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
-
- if (adapter->hw.mac.type >= e1000_82543) {
- adapter->stats.algnerrc +=
- E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
- adapter->stats.rxerrc +=
- E1000_READ_REG(&adapter->hw, E1000_RXERRC);
- adapter->stats.tncrs +=
- E1000_READ_REG(&adapter->hw, E1000_TNCRS);
- adapter->stats.cexterr +=
- E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
- adapter->stats.tsctc +=
- E1000_READ_REG(&adapter->hw, E1000_TSCTC);
- adapter->stats.tsctfc +=
- E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
- }
-}
-
-static uint64_t
-lem_get_counter(if_t ifp, ift_counter cnt)
-{
- struct adapter *adapter;
-
- adapter = if_getsoftc(ifp);
-
- switch (cnt) {
- case IFCOUNTER_COLLISIONS:
- return (adapter->stats.colc);
- case IFCOUNTER_IERRORS:
- return (adapter->dropped_pkts + adapter->stats.rxerrc +
- adapter->stats.crcerrs + adapter->stats.algnerrc +
- adapter->stats.ruc + adapter->stats.roc +
- adapter->stats.mpc + adapter->stats.cexterr);
- case IFCOUNTER_OERRORS:
- return (adapter->stats.ecol + adapter->stats.latecol +
- adapter->watchdog_events);
- default:
- return (if_get_counter_default(ifp, cnt));
- }
-}
-
-/* Export a single 32-bit register via a read-only sysctl. */
-static int
-lem_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
-{
- struct adapter *adapter;
- u_int val;
-
- adapter = oidp->oid_arg1;
- val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
- return (sysctl_handle_int(oidp, &val, 0, req));
-}
-
-/*
- * Add sysctl variables, one per statistic, to the system.
- */
-static void
-lem_add_hw_stats(struct adapter *adapter)
-{
- device_t dev = adapter->dev;
-
- struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
- struct sysctl_oid *tree = device_get_sysctl_tree(dev);
- struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
- struct e1000_hw_stats *stats = &adapter->stats;
-
- struct sysctl_oid *stat_node;
- struct sysctl_oid_list *stat_list;
-
- /* Driver Statistics */
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail",
- CTLFLAG_RD, &adapter->mbuf_cluster_failed,
- "Std mbuf cluster failed");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
- CTLFLAG_RD, &adapter->mbuf_defrag_failed,
- "Defragmenting mbuf chain failed");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
- CTLFLAG_RD, &adapter->dropped_pkts,
- "Driver dropped packets");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
- CTLFLAG_RD, &adapter->no_tx_dma_setup,
- "Driver tx dma failure in xmit");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail1",
- CTLFLAG_RD, &adapter->no_tx_desc_avail1,
- "Not enough tx descriptors failure in xmit");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail2",
- CTLFLAG_RD, &adapter->no_tx_desc_avail2,
- "Not enough tx descriptors failure in xmit");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
- CTLFLAG_RD, &adapter->rx_overruns,
- "RX overruns");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
- CTLFLAG_RD, &adapter->watchdog_events,
- "Watchdog timeouts");
-
- SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
- lem_sysctl_reg_handler, "IU",
- "Device Control Register");
- SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
- lem_sysctl_reg_handler, "IU",
- "Receiver Control Register");
- SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
- CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
- "Flow Control High Watermark");
- SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
- CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
- "Flow Control Low Watermark");
- SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_workaround",
- CTLFLAG_RD, &adapter->tx_fifo_wrk_cnt,
- "TX FIFO workaround events");
- SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_reset",
- CTLFLAG_RD, &adapter->tx_fifo_reset_cnt,
- "TX FIFO resets");
-
- SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_head",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(0),
- lem_sysctl_reg_handler, "IU",
- "Transmit Descriptor Head");
- SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_tail",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(0),
- lem_sysctl_reg_handler, "IU",
- "Transmit Descriptor Tail");
- SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_head",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(0),
- lem_sysctl_reg_handler, "IU",
- "Receive Descriptor Head");
- SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_tail",
- CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(0),
- lem_sysctl_reg_handler, "IU",
- "Receive Descriptor Tail");
-
-
- /* MAC stats get their own sub node */
-
- stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
- CTLFLAG_RD, NULL, "Statistics");
- stat_list = SYSCTL_CHILDREN(stat_node);
-
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
- CTLFLAG_RD, &stats->ecol,
- "Excessive collisions");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
- CTLFLAG_RD, &stats->scc,
- "Single collisions");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
- CTLFLAG_RD, &stats->mcc,
- "Multiple collisions");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
- CTLFLAG_RD, &stats->latecol,
- "Late collisions");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
- CTLFLAG_RD, &stats->colc,
- "Collision Count");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
- CTLFLAG_RD, &adapter->stats.symerrs,
- "Symbol Errors");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
- CTLFLAG_RD, &adapter->stats.sec,
- "Sequence Errors");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
- CTLFLAG_RD, &adapter->stats.dc,
- "Defer Count");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
- CTLFLAG_RD, &adapter->stats.mpc,
- "Missed Packets");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
- CTLFLAG_RD, &adapter->stats.rnbc,
- "Receive No Buffers");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
- CTLFLAG_RD, &adapter->stats.ruc,
- "Receive Undersize");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
- CTLFLAG_RD, &adapter->stats.rfc,
- "Fragmented Packets Received ");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
- CTLFLAG_RD, &adapter->stats.roc,
- "Oversized Packets Received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
- CTLFLAG_RD, &adapter->stats.rjc,
- "Recevied Jabber");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
- CTLFLAG_RD, &adapter->stats.rxerrc,
- "Receive Errors");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
- CTLFLAG_RD, &adapter->stats.crcerrs,
- "CRC errors");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
- CTLFLAG_RD, &adapter->stats.algnerrc,
- "Alignment Errors");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
- CTLFLAG_RD, &adapter->stats.cexterr,
- "Collision/Carrier extension errors");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
- CTLFLAG_RD, &adapter->stats.xonrxc,
- "XON Received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
- CTLFLAG_RD, &adapter->stats.xontxc,
- "XON Transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
- CTLFLAG_RD, &adapter->stats.xoffrxc,
- "XOFF Received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
- CTLFLAG_RD, &adapter->stats.xofftxc,
- "XOFF Transmitted");
-
- /* Packet Reception Stats */
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
- CTLFLAG_RD, &adapter->stats.tpr,
- "Total Packets Received ");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
- CTLFLAG_RD, &adapter->stats.gprc,
- "Good Packets Received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
- CTLFLAG_RD, &adapter->stats.bprc,
- "Broadcast Packets Received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
- CTLFLAG_RD, &adapter->stats.mprc,
- "Multicast Packets Received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
- CTLFLAG_RD, &adapter->stats.prc64,
- "64 byte frames received ");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
- CTLFLAG_RD, &adapter->stats.prc127,
- "65-127 byte frames received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
- CTLFLAG_RD, &adapter->stats.prc255,
- "128-255 byte frames received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
- CTLFLAG_RD, &adapter->stats.prc511,
- "256-511 byte frames received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
- CTLFLAG_RD, &adapter->stats.prc1023,
- "512-1023 byte frames received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
- CTLFLAG_RD, &adapter->stats.prc1522,
- "1023-1522 byte frames received");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
- CTLFLAG_RD, &adapter->stats.gorc,
- "Good Octets Received");
-
- /* Packet Transmission Stats */
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
- CTLFLAG_RD, &adapter->stats.gotc,
- "Good Octets Transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
- CTLFLAG_RD, &adapter->stats.tpt,
- "Total Packets Transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
- CTLFLAG_RD, &adapter->stats.gptc,
- "Good Packets Transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
- CTLFLAG_RD, &adapter->stats.bptc,
- "Broadcast Packets Transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
- CTLFLAG_RD, &adapter->stats.mptc,
- "Multicast Packets Transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
- CTLFLAG_RD, &adapter->stats.ptc64,
- "64 byte frames transmitted ");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
- CTLFLAG_RD, &adapter->stats.ptc127,
- "65-127 byte frames transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
- CTLFLAG_RD, &adapter->stats.ptc255,
- "128-255 byte frames transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
- CTLFLAG_RD, &adapter->stats.ptc511,
- "256-511 byte frames transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
- CTLFLAG_RD, &adapter->stats.ptc1023,
- "512-1023 byte frames transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
- CTLFLAG_RD, &adapter->stats.ptc1522,
- "1024-1522 byte frames transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
- CTLFLAG_RD, &adapter->stats.tsctc,
- "TSO Contexts Transmitted");
- SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
- CTLFLAG_RD, &adapter->stats.tsctfc,
- "TSO Contexts Failed");
-}
-
-/**********************************************************************
- *
- * This routine provides a way to dump out the adapter eeprom,
- * often a useful debug/service tool. This only dumps the first
- * 32 words, stuff that matters is in that extent.
- *
- **********************************************************************/
-
-static int
-lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
-{
- struct adapter *adapter;
- int error;
- int result;
-
- result = -1;
- error = sysctl_handle_int(oidp, &result, 0, req);
-
- if (error || !req->newptr)
- return (error);
-
- /*
- * This value will cause a hex dump of the
- * first 32 16-bit words of the EEPROM to
- * the screen.
- */
- if (result == 1) {
- adapter = (struct adapter *)arg1;
- lem_print_nvm_info(adapter);
- }
-
- return (error);
-}
-
-static void
-lem_print_nvm_info(struct adapter *adapter)
-{
- u16 eeprom_data;
- int i, j, row = 0;
-
- /* Its a bit crude, but it gets the job done */
- printf("\nInterface EEPROM Dump:\n");
- printf("Offset\n0x0000 ");
- for (i = 0, j = 0; i < 32; i++, j++) {
- if (j == 8) { /* Make the offset block */
- j = 0; ++row;
- printf("\n0x00%x0 ",row);
- }
- e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
- printf("%04x ", eeprom_data);
- }
- printf("\n");
-}
-
-static int
-lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
-{
- struct em_int_delay_info *info;
- struct adapter *adapter;
- u32 regval;
- int error;
- int usecs;
- int ticks;
-
- info = (struct em_int_delay_info *)arg1;
- usecs = info->value;
- error = sysctl_handle_int(oidp, &usecs, 0, req);
- if (error != 0 || req->newptr == NULL)
- return (error);
- if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
- return (EINVAL);
- info->value = usecs;
- ticks = EM_USECS_TO_TICKS(usecs);
- if (info->offset == E1000_ITR) /* units are 256ns here */
- ticks *= 4;
-
- adapter = info->adapter;
-
- EM_CORE_LOCK(adapter);
- regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
- regval = (regval & ~0xffff) | (ticks & 0xffff);
- /* Handle a few special cases. */
- switch (info->offset) {
- case E1000_RDTR:
- break;
- case E1000_TIDV:
- if (ticks == 0) {
- adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
- /* Don't write 0 into the TIDV register. */
- regval++;
- } else
- adapter->txd_cmd |= E1000_TXD_CMD_IDE;
- break;
- }
- E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
- EM_CORE_UNLOCK(adapter);
- return (0);
-}
-
-static void
-lem_add_int_delay_sysctl(struct adapter *adapter, const char *name,
- const char *description, struct em_int_delay_info *info,
- int offset, int value)
-{
- info->adapter = adapter;
- info->offset = offset;
- info->value = value;
- SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
- OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
- info, 0, lem_sysctl_int_delay, "I", description);
-}
-
-static void
-lem_set_flow_cntrl(struct adapter *adapter, const char *name,
- const char *description, int *limit, int value)
-{
- *limit = value;
- SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
- OID_AUTO, name, CTLFLAG_RW, limit, value, description);
-}
-
-static void
-lem_add_rx_process_limit(struct adapter *adapter, const char *name,
- const char *description, int *limit, int value)
-{
- *limit = value;
- SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
- SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
- OID_AUTO, name, CTLFLAG_RW, limit, value, description);
-}
Index: sys/dev/e1000/igb_txrx.c
===================================================================
--- /dev/null
+++ sys/dev/e1000/igb_txrx.c
@@ -0,0 +1,593 @@
+#include "if_em.h"
+
+#ifdef RSS
+#include <net/rss_config.h>
+#include <netinet/in_rss.h>
+#endif
+
+#ifdef VERBOSE_DEBUG
+#define DPRINTF device_printf
+#else
+#define DPRINTF(...)
+#endif
+
+/*********************************************************************
+ * Local Function prototypes
+ *********************************************************************/
+static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi);
+static void igb_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx);
+static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx, bool clear);
+
+static void igb_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
+ uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buf_len __unused);
+static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx);
+static int igb_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx,
+ int budget);
+static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
+
+static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status);
+static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status);
+
+static void igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype);
+static int igb_determine_rsstype(u16 pkt_info);
+
+extern void igb_if_enable_intr(if_ctx_t ctx);
+extern int em_intr(void *arg);
+
+struct if_txrx igb_txrx = {
+ igb_isc_txd_encap,
+ igb_isc_txd_flush,
+ igb_isc_txd_credits_update,
+ igb_isc_rxd_available,
+ igb_isc_rxd_pkt_get,
+ igb_isc_rxd_refill,
+ igb_isc_rxd_flush,
+ em_intr
+};
+
+extern if_shared_ctx_t em_sctx;
+
+/**********************************************************************
+ *
+ * Setup work for hardware segmentation offload (TSO) on
+ * adapters using advanced tx descriptors
+ *
+ **********************************************************************/
+static int
+igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status)
+{
+ struct e1000_adv_tx_context_desc *TXD;
+ struct adapter *adapter = txr->adapter;
+ u32 type_tucmd_mlhl = 0, vlan_macip_lens = 0;
+ u32 mss_l4len_idx = 0;
+ u32 paylen;
+
+ switch(pi->ipi_etype) {
+ case ETHERTYPE_IPV6:
+ type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
+ break;
+ case ETHERTYPE_IP:
+ type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
+ /* Tell transmit desc to also do IPv4 checksum. */
+ *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
+ break;
+ default:
+ panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
+ __func__, ntohs(pi->ipi_etype));
+ break;
+ }
+
+ TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx];
+
+ /* This is used in the transmit desc in encap */
+ paylen = pi->ipi_len - pi->ipi_ehdrlen - pi->ipi_ip_hlen - pi->ipi_tcp_hlen;
+
+ /* VLAN MACLEN IPLEN */
+ if (pi->ipi_mflags & M_VLANTAG) {
+ vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT);
+ }
+
+ vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
+ vlan_macip_lens |= pi->ipi_ip_hlen;
+ TXD->vlan_macip_lens = htole32(vlan_macip_lens);
+
+ /* ADV DTYPE TUCMD */
+ type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
+ type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
+ TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
+
+ /* MSS L4LEN IDX */
+ mss_l4len_idx |= (pi->ipi_tso_segsz << E1000_ADVTXD_MSS_SHIFT);
+ mss_l4len_idx |= (pi->ipi_tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
+ /* 82575 needs the queue index added */
+ if (adapter->hw.mac.type == e1000_82575)
+ mss_l4len_idx |= txr->me << 4;
+ TXD->mss_l4len_idx = htole32(mss_l4len_idx);
+
+ TXD->seqnum_seed = htole32(0);
+ *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
+ *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
+ *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
+
+ return (1);
+}
+
+/*********************************************************************
+ *
+ * Advanced Context Descriptor setup for VLAN, CSUM or TSO
+ *
+ **********************************************************************/
+static int
+igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status)
+{
+ struct e1000_adv_tx_context_desc *TXD;
+ struct adapter *adapter = txr->adapter;
+ u32 vlan_macip_lens, type_tucmd_mlhl;
+ u32 mss_l4len_idx;
+ mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0;
+ int offload = TRUE;
+
+ /* First check if TSO is to be used */
+ if (pi->ipi_csum_flags & CSUM_TSO)
+ return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status));
+
+ /* Indicate the whole packet as payload when not doing TSO */
+ *olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT;
+
+ /* Now ready a context descriptor */
+ TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx];
+
+ /*
+ ** In advanced descriptors the vlan tag must
+ ** be placed into the context descriptor. Hence
+ ** we need to make one even if not doing offloads.
+ */
+ if (pi->ipi_mflags & M_VLANTAG) {
+ vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT);
+ } else if ((pi->ipi_csum_flags & CSUM_OFFLOAD) == 0) {
+ return (0);
+ }
+
+ /* Set the ether header length */
+ vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
+
+ switch(pi->ipi_etype) {
+ case ETHERTYPE_IP:
+ type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
+ break;
+ case ETHERTYPE_IPV6:
+ type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
+ break;
+ default:
+ offload = FALSE;
+ break;
+ }
+
+ vlan_macip_lens |= pi->ipi_ip_hlen;
+ type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
+
+ switch (pi->ipi_ipproto) {
+ case IPPROTO_TCP:
+ #if __FreeBSD_version >= 1000000
+ if (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))
+#else
+ if (pi->ipi_csum_flags & CSUM_TCP)
+#endif
+ type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
+ break;
+ case IPPROTO_UDP:
+#if __FreeBSD_version >= 1000000
+ if (pi->ipi_csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP))
+#else
+ if (pi->ipi_csum_flags & CSUM_UDP)
+#endif
+ type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
+ break;
+
+#if __FreeBSD_version >= 800000
+ case IPPROTO_SCTP:
+#if __FreeBSD_version >= 1000000
+ if (pi->ipi_csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP))
+#else
+ if (pi->ipi_csum_flags & CSUM_SCTP)
+#endif
+ type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
+ break;
+#endif
+ default:
+ offload = FALSE;
+ break;
+ }
+
+ if (offload) /* For the TX descriptor setup */
+ *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
+
+ /* 82575 needs the queue index added */
+ if (adapter->hw.mac.type == e1000_82575)
+ mss_l4len_idx = txr->me << 4;
+
+ /* Now copy bits into descriptor */
+ TXD->vlan_macip_lens = htole32(vlan_macip_lens);
+ TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
+ TXD->seqnum_seed = htole32(0);
+ TXD->mss_l4len_idx = htole32(mss_l4len_idx);
+
+ return (1);
+}
+
+static int
+igb_isc_txd_encap(void *arg, if_pkt_info_t pi)
+{
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
+ struct tx_ring *txr = &que->txr;
+ int nsegs = pi->ipi_nsegs;
+ bus_dma_segment_t *segs = pi->ipi_segs;
+ struct em_txbuffer *txbuf;
+ union e1000_adv_tx_desc *txd = NULL;
+
+ int i, j, first, pidx_last;
+ u32 olinfo_status, cmd_type_len;
+
+ pidx_last = olinfo_status = 0;
+ /* Basic descriptor defines */
+ cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
+ E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
+
+ if (pi->ipi_mflags & M_VLANTAG)
+ cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
+
+ first = i = pi->ipi_pidx;
+
+ /* Consume the first descriptor */
+ i += igb_tx_ctx_setup(txr, pi, &cmd_type_len, &olinfo_status);
+ if (i == scctx->isc_ntxd[0])
+ i = 0;
+
+ /* 82575 needs the queue index added */
+ if (sc->hw.mac.type == e1000_82575)
+ olinfo_status |= txr->me << 4;
+
+ for (j = 0; j < nsegs; j++) {
+ bus_size_t seglen;
+ bus_addr_t segaddr;
+
+ txbuf = &txr->tx_buffers[i];
+ txd = (union e1000_adv_tx_desc *)&txr->tx_base[i];
+ seglen = segs[j].ds_len;
+ segaddr = htole64(segs[j].ds_addr);
+
+ txd->read.buffer_addr = segaddr;
+ txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
+ cmd_type_len | seglen);
+ txd->read.olinfo_status = htole32(olinfo_status);
+ pidx_last = i;
+ if (++i == scctx->isc_ntxd[0]) {
+ i = 0;
+ }
+ }
+
+ txd->read.cmd_type_len |=
+ htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
+
+ /* Set the EOP descriptor that will be marked done */
+ txbuf = &txr->tx_buffers[first];
+ txbuf->eop = pidx_last;
+
+ pi->ipi_new_pidx = i;
+
+ return (0);
+}
+
+static void
+igb_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx)
+{
+ struct adapter *adapter = arg;
+ struct em_tx_queue *que = &adapter->tx_queues[txqid];
+ struct tx_ring *txr = &que->txr;
+
+ E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx);
+}
+
+static int
+igb_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear)
+{
+ struct adapter *adapter = arg;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_tx_queue *que = &adapter->tx_queues[txqid];
+ struct tx_ring *txr = &que->txr;
+
+ u32 cidx, ntxd, processed = 0;
+
+ struct em_txbuffer *buf;
+ union e1000_adv_tx_desc *txd, *eop;
+ int limit;
+
+ cidx = cidx_init;
+
+ buf = &txr->tx_buffers[cidx];
+ txd = (union e1000_adv_tx_desc *)&txr->tx_base[cidx];
+ ntxd = scctx->isc_ntxd[0];
+ limit = adapter->tx_process_limit;
+
+ do {
+ if (buf->eop == -1) /* No work */
+ break;
+
+ eop = (union e1000_adv_tx_desc *)&txr->tx_base[buf->eop];
+ if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
+ break; /* I/O not complete */
+
+ if (clear)
+ buf->eop = -1; /* clear indicate processed */
+
+ /* We clean the range if multi segment */
+ while (txd != eop) {
+ ++txd;
+ ++buf;
+ /* wrap the ring? */
+ if (++cidx == scctx->isc_ntxd[0]) {
+ cidx = 0;
+ buf = txr->tx_buffers;
+ txd = (union e1000_adv_tx_desc *)txr->tx_base;
+ }
+
+ buf = &txr->tx_buffers[cidx];
+ if (clear)
+ buf->eop = -1;
+ processed++;
+ }
+ processed++;
+
+ /* Try the next packet */
+ txd++;
+ buf++;
+
+ /* reset with a wrap */
+ if (++cidx == scctx->isc_ntxd[0]) {
+ cidx = 0;
+ buf = txr->tx_buffers;
+ txd = (union e1000_adv_tx_desc *)txr->tx_base;
+ }
+ prefetch(txd);
+ prefetch(txd+1);
+ } while (__predict_true(--limit) && cidx != cidx_init);
+
+ return (processed);
+}
+
+static void
+igb_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
+ uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused,
+ uint16_t count, uint16_t buf_len __unused)
+{
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ union e1000_adv_rx_desc *rxd;
+ struct rx_ring *rxr = &que->rxr;
+ int i;
+ uint32_t next_pidx;
+
+ for (i = 0, next_pidx = pidx; i < count; i++) {
+ rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[next_pidx];
+
+ rxd->read.pkt_addr = htole64(paddrs[i]);
+ if (++next_pidx == scctx->isc_nrxd[0])
+ next_pidx = 0;
+ }
+}
+
+static void
+igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx)
+{
+ struct adapter *sc = arg;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
+
+ E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
+}
+
+static int
+igb_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget)
+{
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
+ union e1000_adv_rx_desc *rxd;
+ u32 staterr = 0;
+ int cnt, i, iter;
+
+ for (iter = cnt = 0, i = idx; iter < scctx->isc_nrxd[0] && iter <= budget;) {
+ rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i];
+ staterr = le32toh(rxd->wb.upper.status_error);
+
+ if ((staterr & E1000_RXD_STAT_DD) == 0)
+ break;
+
+ if (++i == scctx->isc_nrxd[0]) {
+ i = 0;
+ }
+
+ if (staterr & E1000_RXD_STAT_EOP)
+ cnt++;
+ iter++;
+ }
+ {
+ struct e1000_hw *hw = &sc->hw;
+ int rdt, rdh;
+ rdt = E1000_READ_REG(hw, E1000_RDT(rxr->me));
+ rdh = E1000_READ_REG(hw, E1000_RDH(rxr->me));
+ DPRINTF(iflib_get_dev(sc->ctx), "sidx:%d eidx:%d iter=%d pktcnt=%d RDT=%d RDH=%d\n", idx, i, iter, cnt, rdt, rdh);
+ }
+ return (cnt);
+}
+
+/****************************************************************
+ * Routine sends data which has been dma'ed into host memory
+ * to upper layer. Initialize ri structure.
+ *
+ * Returns 0 upon success, errno on failure
+ ***************************************************************/
+
+static int
+igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
+{
+ struct adapter *adapter = arg;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
+ struct rx_ring *rxr = &que->rxr;
+ struct ifnet *ifp = iflib_get_ifp(adapter->ctx);
+ union e1000_adv_rx_desc *rxd;
+
+ u16 pkt_info, len;
+ u16 vtag = 0;
+ u32 ptype;
+ u32 staterr = 0;
+ bool eop;
+ int i = 0;
+ int cidx = ri->iri_cidx;
+
+ do {
+ rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx];
+ staterr = le32toh(rxd->wb.upper.status_error);
+ pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info);
+
+ MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
+
+ len = le16toh(rxd->wb.upper.length);
+ ptype = le32toh(rxd->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
+
+ ri->iri_len += len;
+ rxr->rx_bytes += ri->iri_len;
+
+ rxd->wb.upper.status_error = 0;
+ eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
+
+ if (((adapter->hw.mac.type == e1000_i350) ||
+ (adapter->hw.mac.type == e1000_i354)) &&
+ (staterr & E1000_RXDEXT_STATERR_LB))
+ vtag = be16toh(rxd->wb.upper.vlan);
+ else
+ vtag = le16toh(rxd->wb.upper.vlan);
+
+ /* Make sure bad packets are discarded */
+ if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) {
+ adapter->dropped_pkts++;
+ ++rxr->rx_discarded;
+ return (EBADMSG);
+ }
+ ri->iri_frags[i].irf_flid = 0;
+ ri->iri_frags[i].irf_idx = cidx;
+ ri->iri_frags[i].irf_len = len;
+
+ if (++cidx == scctx->isc_nrxd[0])
+ cidx = 0;
+#ifdef notyet
+ if (rxr->hdr_split == TRUE) {
+ ri->iri_frags[i].irf_flid = 1;
+ ri->iri_frags[i].irf_idx = cidx;
+ if (++cidx == scctx->isc_nrxd[0])
+ cidx = 0;
+ }
+#endif
+ i++;
+ } while (!eop);
+
+ rxr->rx_packets++;
+
+ if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
+ igb_rx_checksum(staterr, ri, ptype);
+
+ if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
+ (staterr & E1000_RXD_STAT_VP) != 0) {
+ ri->iri_vtag = vtag;
+ ri->iri_flags |= M_VLANTAG;
+ }
+ ri->iri_flowid =
+ le32toh(rxd->wb.lower.hi_dword.rss);
+ ri->iri_rsstype = igb_determine_rsstype(pkt_info);
+ ri->iri_nfrags = i;
+
+ return (0);
+}
+
+/*********************************************************************
+ *
+ * Verify that the hardware indicated that the checksum is valid.
+ * Inform the stack about the status of checksum so that stack
+ * doesn't spend time verifying the checksum.
+ *
+ *********************************************************************/
+static void
+igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype)
+{
+ u16 status = (u16)staterr;
+ u8 errors = (u8) (staterr >> 24);
+ bool sctp = FALSE;
+
+ /* Ignore Checksum bit is set */
+ if (status & E1000_RXD_STAT_IXSM) {
+ ri->iri_csum_flags = 0;
+ return;
+ }
+
+ if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
+ (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
+ sctp = 1;
+ else
+ sctp = 0;
+
+ if (status & E1000_RXD_STAT_IPCS) {
+ /* Did it pass? */
+ if (!(errors & E1000_RXD_ERR_IPE)) {
+ /* IP Checksum Good */
+ ri->iri_csum_flags = CSUM_IP_CHECKED;
+ ri->iri_csum_flags |= CSUM_IP_VALID;
+ } else
+ ri->iri_csum_flags = 0;
+ }
+
+ if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
+ u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
+#if __FreeBSD_version >= 800000
+ if (sctp) /* reassign */
+ type = CSUM_SCTP_VALID;
+#endif
+ /* Did it pass? */
+ if (!(errors & E1000_RXD_ERR_TCPE)) {
+ ri->iri_csum_flags |= type;
+ if (sctp == 0)
+ ri->iri_csum_data = htons(0xffff);
+ }
+ }
+ return;
+}
+
+/********************************************************************
+ *
+ * Parse the packet type to determine the appropriate hash
+ *
+ ******************************************************************/
+static int
+igb_determine_rsstype(u16 pkt_info)
+{
+ switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
+ case E1000_RXDADV_RSSTYPE_IPV4_TCP:
+ return M_HASHTYPE_RSS_TCP_IPV4;
+ case E1000_RXDADV_RSSTYPE_IPV4:
+ return M_HASHTYPE_RSS_IPV4;
+ case E1000_RXDADV_RSSTYPE_IPV6_TCP:
+ return M_HASHTYPE_RSS_TCP_IPV6;
+ case E1000_RXDADV_RSSTYPE_IPV6_EX:
+ return M_HASHTYPE_RSS_IPV6_EX;
+ case E1000_RXDADV_RSSTYPE_IPV6:
+ return M_HASHTYPE_RSS_IPV6;
+ case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
+ return M_HASHTYPE_RSS_TCP_IPV6_EX;
+ default:
+ return M_HASHTYPE_OPAQUE;
+ }
+}
Index: sys/modules/Makefile
===================================================================
--- sys/modules/Makefile
+++ sys/modules/Makefile
@@ -163,7 +163,6 @@
if_tun \
if_vlan \
if_vxlan \
- ${_igb} \
${_iir} \
imgact_binmisc \
${_intelspi} \
@@ -544,7 +543,6 @@
.if ${MACHINE_CPUARCH} == "aarch64"
_armv8crypto= armv8crypto
_em= em
-_igb= igb
.endif
.if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64"
@@ -572,7 +570,6 @@
_ibcore= ibcore
.endif
_if_ndis= if_ndis
-_igb= igb
_io= io
.if ${MK_OFED} != "no" || defined(ALL_MODULES)
_ipoib= ipoib
@@ -779,7 +776,6 @@
_auxio= auxio
_em= em
_epic= epic
-_igb= igb
.endif
.if (${MACHINE_CPUARCH} == "amd64" || ${MACHINE_ARCH} == "armv6" || \
Index: sys/modules/em/Makefile
===================================================================
--- sys/modules/em/Makefile
+++ sys/modules/em/Makefile
@@ -1,23 +1,28 @@
+
# $FreeBSD$
.PATH: ${.CURDIR}/../../dev/e1000
KMOD = if_em
SRCS = device_if.h bus_if.h pci_if.h opt_ddb.h opt_em.h opt_inet.h \
- opt_inet6.h
+ opt_inet6.h ifdi_if.h
SRCS += $(CORE_SRC) $(LEGACY_SRC)
SRCS += $(COMMON_SHARED) $(LEGACY_SHARED) $(PCIE_SHARED)
-CORE_SRC = if_em.c e1000_osdep.c
+CORE_SRC = if_em.c em_txrx.c e1000_osdep.c
+CORE_SRC += igb_txrx.c
# This is the Legacy, pre-PCIE source, it can be
# undefined when using modular driver if not needed
-LEGACY_SRC += if_lem.c
COMMON_SHARED = e1000_api.c e1000_phy.c e1000_nvm.c e1000_mac.c \
e1000_manage.c e1000_vf.c e1000_mbx.c e1000_i210.c
PCIE_SHARED = e1000_80003es2lan.c e1000_ich8lan.c e1000_82571.c e1000_82575.c
LEGACY_SHARED = e1000_82540.c e1000_82542.c e1000_82541.c e1000_82543.c
-CFLAGS += -I${.CURDIR}/../../dev/e1000
+
+CFLAGS += -I${.CURDIR}/../../../dev/e1000
# DEVICE_POLLING for a non-interrupt-driven method
#CFLAGS += -DDEVICE_POLLING
+afterinstall:
+ ln -sf ${DESTDIR}${KMODDIR}/${KMOD}.ko ${DESTDIR}${KMODDIR}/if_igb.ko
+
.include <bsd.kmod.mk>
Index: sys/modules/igb/Makefile
===================================================================
--- sys/modules/igb/Makefile
+++ /dev/null
@@ -1,24 +0,0 @@
-#$FreeBSD$
-
-.PATH: ${.CURDIR}/../../dev/e1000
-KMOD = if_igb
-SRCS = device_if.h bus_if.h pci_if.h opt_inet.h opt_inet6.h opt_rss.h
-SRCS += if_igb.c $(SHARED_SRCS)
-SHARED_SRCS = e1000_api.c e1000_phy.c e1000_nvm.c e1000_mac.c e1000_manage.c
-SHARED_SRCS += e1000_80003es2lan.c e1000_82542.c e1000_82541.c e1000_82543.c
-SHARED_SRCS += e1000_82540.c e1000_ich8lan.c e1000_82571.c e1000_osdep.c
-SHARED_SRCS += e1000_82575.c e1000_vf.c e1000_mbx.c e1000_i210.c
-
-CFLAGS += -I${.CURDIR}/../../dev/e1000 -DSMP
-
-# DEVICE_POLLING gives you non-interrupt handling
-# not advisable since MSIX gives better results
-#CFLAGS += -DDEVICE_POLLING
-
-# IGB_LEGACY_TX will override the stack if_transmit path and
-# instead use the older if_start non-multiqueue capable interface.
-# This might be desirable for testing, or to enable the use of
-# ALTQ.
-#CFLAGS += -DIGB_LEGACY_TX
-
-.include <bsd.kmod.mk>

File Metadata

Mime Type
text/plain
Expires
Thu, Apr 2, 1:39 PM (15 h, 35 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
30710952
Default Alt Text
D8299.id23800.diff (622 KB)

Event Timeline