Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/e1000/if_em.c
Show First 20 Lines • Show All 254 Lines • ▼ Show 20 Lines | |||||
static void em_disable_intr(struct adapter *); | static void em_disable_intr(struct adapter *); | ||||
static void em_update_stats_counters(struct adapter *); | static void em_update_stats_counters(struct adapter *); | ||||
static void em_add_hw_stats(struct adapter *adapter); | static void em_add_hw_stats(struct adapter *adapter); | ||||
static void em_txeof(struct tx_ring *); | static void em_txeof(struct tx_ring *); | ||||
static bool em_rxeof(struct rx_ring *, int, int *); | static bool em_rxeof(struct rx_ring *, int, int *); | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
static int em_fixup_rx(struct rx_ring *); | static int em_fixup_rx(struct rx_ring *); | ||||
#endif | #endif | ||||
static void em_receive_checksum(struct e1000_rx_desc *, struct mbuf *); | static void em_setup_rxdesc(union e1000_rx_desc_extended *, | ||||
const struct em_rxbuffer *rxbuf); | |||||
static void em_receive_checksum(uint32_t status, struct mbuf *); | |||||
static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int, | static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int, | ||||
struct ip *, u32 *, u32 *); | struct ip *, u32 *, u32 *); | ||||
static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *, | static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *, | ||||
struct tcphdr *, u32 *, u32 *); | struct tcphdr *, u32 *, u32 *); | ||||
static void em_set_promisc(struct adapter *); | static void em_set_promisc(struct adapter *); | ||||
static void em_disable_promisc(struct adapter *); | static void em_disable_promisc(struct adapter *); | ||||
static void em_set_multi(struct adapter *); | static void em_set_multi(struct adapter *); | ||||
static void em_update_link_status(struct adapter *); | static void em_update_link_status(struct adapter *); | ||||
▲ Show 20 Lines • Show All 354 Lines • ▼ Show 20 Lines | em_attach(device_t dev) | ||||
if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || | if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || | ||||
(em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { | (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { | ||||
device_printf(dev, "Using %d TX descriptors instead of %d!\n", | device_printf(dev, "Using %d TX descriptors instead of %d!\n", | ||||
EM_DEFAULT_TXD, em_txd); | EM_DEFAULT_TXD, em_txd); | ||||
adapter->num_tx_desc = EM_DEFAULT_TXD; | adapter->num_tx_desc = EM_DEFAULT_TXD; | ||||
} else | } else | ||||
adapter->num_tx_desc = em_txd; | adapter->num_tx_desc = em_txd; | ||||
if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || | if (((em_rxd * sizeof(union e1000_rx_desc_extended)) % EM_DBA_ALIGN) != 0 || | ||||
(em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { | (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { | ||||
device_printf(dev, "Using %d RX descriptors instead of %d!\n", | device_printf(dev, "Using %d RX descriptors instead of %d!\n", | ||||
EM_DEFAULT_RXD, em_rxd); | EM_DEFAULT_RXD, em_rxd); | ||||
adapter->num_rx_desc = EM_DEFAULT_RXD; | adapter->num_rx_desc = EM_DEFAULT_RXD; | ||||
} else | } else | ||||
adapter->num_rx_desc = em_rxd; | adapter->num_rx_desc = em_rxd; | ||||
hw->mac.autoneg = DO_AUTO_NEG; | hw->mac.autoneg = DO_AUTO_NEG; | ||||
▲ Show 20 Lines • Show All 1,224 Lines • ▼ Show 20 Lines | |||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
em_xmit(struct tx_ring *txr, struct mbuf **m_headp) | em_xmit(struct tx_ring *txr, struct mbuf **m_headp) | ||||
{ | { | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
bus_dma_segment_t segs[EM_MAX_SCATTER]; | bus_dma_segment_t segs[EM_MAX_SCATTER]; | ||||
bus_dmamap_t map; | bus_dmamap_t map; | ||||
struct em_buffer *tx_buffer, *tx_buffer_mapped; | struct em_txbuffer *tx_buffer, *tx_buffer_mapped; | ||||
struct e1000_tx_desc *ctxd = NULL; | struct e1000_tx_desc *ctxd = NULL; | ||||
struct mbuf *m_head; | struct mbuf *m_head; | ||||
struct ether_header *eh; | struct ether_header *eh; | ||||
struct ip *ip = NULL; | struct ip *ip = NULL; | ||||
struct tcphdr *tp = NULL; | struct tcphdr *tp = NULL; | ||||
u32 txd_upper = 0, txd_lower = 0; | u32 txd_upper = 0, txd_lower = 0; | ||||
int ip_off, poff; | int ip_off, poff; | ||||
int nsegs, i, j, first, last = 0; | int nsegs, i, j, first, last = 0; | ||||
▲ Show 20 Lines • Show All 1,407 Lines • ▼ Show 20 Lines | txr->br = buf_ring_alloc(4096, M_DEVBUF, | ||||
M_WAITOK, &txr->tx_mtx); | M_WAITOK, &txr->tx_mtx); | ||||
#endif | #endif | ||||
} | } | ||||
/* | /* | ||||
* Next the RX queues... | * Next the RX queues... | ||||
*/ | */ | ||||
rsize = roundup2(adapter->num_rx_desc * | rsize = roundup2(adapter->num_rx_desc * | ||||
sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); | sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); | ||||
for (int i = 0; i < adapter->num_queues; i++, rxconf++) { | for (int i = 0; i < adapter->num_queues; i++, rxconf++) { | ||||
rxr = &adapter->rx_rings[i]; | rxr = &adapter->rx_rings[i]; | ||||
rxr->adapter = adapter; | rxr->adapter = adapter; | ||||
rxr->me = i; | rxr->me = i; | ||||
/* Initialize the RX lock */ | /* Initialize the RX lock */ | ||||
snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", | snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", | ||||
device_get_nameunit(dev), txr->me); | device_get_nameunit(dev), txr->me); | ||||
mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); | mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); | ||||
if (em_dma_malloc(adapter, rsize, | if (em_dma_malloc(adapter, rsize, | ||||
&rxr->rxdma, BUS_DMA_NOWAIT)) { | &rxr->rxdma, BUS_DMA_NOWAIT)) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"Unable to allocate RxDescriptor memory\n"); | "Unable to allocate RxDescriptor memory\n"); | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto err_rx_desc; | goto err_rx_desc; | ||||
} | } | ||||
rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr; | rxr->rx_base = (union e1000_rx_desc_extended *)rxr->rxdma.dma_vaddr; | ||||
bzero((void *)rxr->rx_base, rsize); | bzero((void *)rxr->rx_base, rsize); | ||||
/* Allocate receive buffers for the ring*/ | /* Allocate receive buffers for the ring*/ | ||||
if (em_allocate_receive_buffers(rxr)) { | if (em_allocate_receive_buffers(rxr)) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"Critical Failure setting up receive buffers\n"); | "Critical Failure setting up receive buffers\n"); | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto err_rx_desc; | goto err_rx_desc; | ||||
Show All 26 Lines | |||||
* called only once at attach, setup is done every reset. | * called only once at attach, setup is done every reset. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
em_allocate_transmit_buffers(struct tx_ring *txr) | em_allocate_transmit_buffers(struct tx_ring *txr) | ||||
{ | { | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
device_t dev = adapter->dev; | device_t dev = adapter->dev; | ||||
struct em_buffer *txbuf; | struct em_txbuffer *txbuf; | ||||
int error, i; | int error, i; | ||||
/* | /* | ||||
* Setup DMA descriptor areas. | * Setup DMA descriptor areas. | ||||
*/ | */ | ||||
if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), | if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), | ||||
1, 0, /* alignment, bounds */ | 1, 0, /* alignment, bounds */ | ||||
BUS_SPACE_MAXADDR, /* lowaddr */ | BUS_SPACE_MAXADDR, /* lowaddr */ | ||||
BUS_SPACE_MAXADDR, /* highaddr */ | BUS_SPACE_MAXADDR, /* highaddr */ | ||||
NULL, NULL, /* filter, filterarg */ | NULL, NULL, /* filter, filterarg */ | ||||
EM_TSO_SIZE, /* maxsize */ | EM_TSO_SIZE, /* maxsize */ | ||||
EM_MAX_SCATTER, /* nsegments */ | EM_MAX_SCATTER, /* nsegments */ | ||||
PAGE_SIZE, /* maxsegsize */ | PAGE_SIZE, /* maxsegsize */ | ||||
0, /* flags */ | 0, /* flags */ | ||||
NULL, /* lockfunc */ | NULL, /* lockfunc */ | ||||
NULL, /* lockfuncarg */ | NULL, /* lockfuncarg */ | ||||
&txr->txtag))) { | &txr->txtag))) { | ||||
device_printf(dev,"Unable to allocate TX DMA tag\n"); | device_printf(dev,"Unable to allocate TX DMA tag\n"); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
if (!(txr->tx_buffers = | if (!(txr->tx_buffers = | ||||
(struct em_buffer *) malloc(sizeof(struct em_buffer) * | (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) * | ||||
adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { | adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | device_printf(dev, "Unable to allocate tx_buffer memory\n"); | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Create the descriptor buffer dma maps */ | /* Create the descriptor buffer dma maps */ | ||||
txbuf = txr->tx_buffers; | txbuf = txr->tx_buffers; | ||||
Show All 16 Lines | |||||
* | * | ||||
* Initialize a transmit ring. | * Initialize a transmit ring. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_setup_transmit_ring(struct tx_ring *txr) | em_setup_transmit_ring(struct tx_ring *txr) | ||||
{ | { | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
struct em_buffer *txbuf; | struct em_txbuffer *txbuf; | ||||
int i; | int i; | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
struct netmap_slot *slot; | struct netmap_slot *slot; | ||||
struct netmap_adapter *na = netmap_getna(adapter->ifp); | struct netmap_adapter *na = netmap_getna(adapter->ifp); | ||||
#endif /* DEV_NETMAP */ | #endif /* DEV_NETMAP */ | ||||
/* Clear the old descriptor contents */ | /* Clear the old descriptor contents */ | ||||
EM_TX_LOCK(txr); | EM_TX_LOCK(txr); | ||||
▲ Show 20 Lines • Show All 202 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* Free transmit ring related data structures. | * Free transmit ring related data structures. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_free_transmit_buffers(struct tx_ring *txr) | em_free_transmit_buffers(struct tx_ring *txr) | ||||
{ | { | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
struct em_buffer *txbuf; | struct em_txbuffer *txbuf; | ||||
INIT_DEBUGOUT("free_transmit_ring: begin"); | INIT_DEBUGOUT("free_transmit_ring: begin"); | ||||
if (txr->tx_buffers == NULL) | if (txr->tx_buffers == NULL) | ||||
return; | return; | ||||
for (int i = 0; i < adapter->num_tx_desc; i++) { | for (int i = 0; i < adapter->num_tx_desc; i++) { | ||||
txbuf = &txr->tx_buffers[i]; | txbuf = &txr->tx_buffers[i]; | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | |||||
* frames. | * frames. | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, | em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, | ||||
struct ip *ip, u32 *txd_upper, u32 *txd_lower) | struct ip *ip, u32 *txd_upper, u32 *txd_lower) | ||||
{ | { | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
struct e1000_context_desc *TXD = NULL; | struct e1000_context_desc *TXD = NULL; | ||||
struct em_buffer *tx_buffer; | struct em_txbuffer *tx_buffer; | ||||
int cur, hdr_len; | int cur, hdr_len; | ||||
u32 cmd = 0; | u32 cmd = 0; | ||||
u16 offload = 0; | u16 offload = 0; | ||||
u8 ipcso, ipcss, tucso, tucss; | u8 ipcso, ipcss, tucso, tucss; | ||||
ipcss = ipcso = tucss = tucso = 0; | ipcss = ipcso = tucss = tucso = 0; | ||||
hdr_len = ip_off + (ip->ip_hl << 2); | hdr_len = ip_off + (ip->ip_hl << 2); | ||||
cur = txr->next_avail_desc; | cur = txr->next_avail_desc; | ||||
▲ Show 20 Lines • Show All 120 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, | em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, | ||||
struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower) | struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower) | ||||
{ | { | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
struct e1000_context_desc *TXD; | struct e1000_context_desc *TXD; | ||||
struct em_buffer *tx_buffer; | struct em_txbuffer *tx_buffer; | ||||
int cur, hdr_len; | int cur, hdr_len; | ||||
/* | /* | ||||
* In theory we can use the same TSO context if and only if | * In theory we can use the same TSO context if and only if | ||||
* frame is the same type(IP/TCP) and the same MSS. However | * frame is the same type(IP/TCP) and the same MSS. However | ||||
* checking whether a frame has the same IP/TCP structure is | * checking whether a frame has the same IP/TCP structure is | ||||
* hard thing so just ignore that and always restablish a | * hard thing so just ignore that and always restablish a | ||||
* new TSO context. | * new TSO context. | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | |||||
* tx_buffer is put back on the free queue. | * tx_buffer is put back on the free queue. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_txeof(struct tx_ring *txr) | em_txeof(struct tx_ring *txr) | ||||
{ | { | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
int first, last, done, processed; | int first, last, done, processed; | ||||
struct em_buffer *tx_buffer; | struct em_txbuffer *tx_buffer; | ||||
struct e1000_tx_desc *tx_desc, *eop_desc; | struct e1000_tx_desc *tx_desc, *eop_desc; | ||||
if_t ifp = adapter->ifp; | if_t ifp = adapter->ifp; | ||||
EM_TX_LOCK_ASSERT(txr); | EM_TX_LOCK_ASSERT(txr); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
if (netmap_tx_irq(ifp, txr->me)) | if (netmap_tx_irq(ifp, txr->me)) | ||||
return; | return; | ||||
#endif /* DEV_NETMAP */ | #endif /* DEV_NETMAP */ | ||||
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | * sanity. | ||||
if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); | if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); | ||||
} | } | ||||
/* Disable hang detection if all clean */ | /* Disable hang detection if all clean */ | ||||
if (txr->tx_avail == adapter->num_tx_desc) | if (txr->tx_avail == adapter->num_tx_desc) | ||||
txr->busy = EM_TX_IDLE; | txr->busy = EM_TX_IDLE; | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Refresh RX descriptor mbufs from system mbuf buffer pool. | * Refresh RX descriptor mbufs from system mbuf buffer pool. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_refresh_mbufs(struct rx_ring *rxr, int limit) | em_refresh_mbufs(struct rx_ring *rxr, int limit) | ||||
{ | { | ||||
struct adapter *adapter = rxr->adapter; | struct adapter *adapter = rxr->adapter; | ||||
struct mbuf *m; | struct mbuf *m; | ||||
bus_dma_segment_t segs[1]; | bus_dma_segment_t segs; | ||||
struct em_buffer *rxbuf; | struct em_rxbuffer *rxbuf; | ||||
int i, j, error, nsegs; | int i, j, error, nsegs; | ||||
bool cleaned = FALSE; | bool cleaned = FALSE; | ||||
i = j = rxr->next_to_refresh; | i = j = rxr->next_to_refresh; | ||||
/* | /* | ||||
** Get one descriptor beyond | ** Get one descriptor beyond | ||||
** our work mark to control | ** our work mark to control | ||||
** the loop. | ** the loop. | ||||
Show All 18 Lines | if (rxbuf->m_head == NULL) { | ||||
m = rxbuf->m_head; | m = rxbuf->m_head; | ||||
m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz; | m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz; | ||||
m->m_flags |= M_PKTHDR; | m->m_flags |= M_PKTHDR; | ||||
m->m_data = m->m_ext.ext_buf; | m->m_data = m->m_ext.ext_buf; | ||||
/* Use bus_dma machinery to setup the memory mapping */ | /* Use bus_dma machinery to setup the memory mapping */ | ||||
error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, | error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, | ||||
m, segs, &nsegs, BUS_DMA_NOWAIT); | m, &segs, &nsegs, BUS_DMA_NOWAIT); | ||||
if (error != 0) { | if (error != 0) { | ||||
printf("Refresh mbufs: hdr dmamap load" | printf("Refresh mbufs: hdr dmamap load" | ||||
" failure - %d\n", error); | " failure - %d\n", error); | ||||
m_free(m); | m_free(m); | ||||
rxbuf->m_head = NULL; | rxbuf->m_head = NULL; | ||||
goto update; | goto update; | ||||
} | } | ||||
rxbuf->m_head = m; | rxbuf->m_head = m; | ||||
rxbuf->paddr = segs.ds_addr; | |||||
bus_dmamap_sync(rxr->rxtag, | bus_dmamap_sync(rxr->rxtag, | ||||
rxbuf->map, BUS_DMASYNC_PREREAD); | rxbuf->map, BUS_DMASYNC_PREREAD); | ||||
rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr); | em_setup_rxdesc(&rxr->rx_base[i], rxbuf); | ||||
cleaned = TRUE; | cleaned = TRUE; | ||||
i = j; /* Next is precalulated for us */ | i = j; /* Next is precalulated for us */ | ||||
rxr->next_to_refresh = i; | rxr->next_to_refresh = i; | ||||
/* Calculate next controlling index */ | /* Calculate next controlling index */ | ||||
if (++j == adapter->num_rx_desc) | if (++j == adapter->num_rx_desc) | ||||
j = 0; | j = 0; | ||||
} | } | ||||
Show All 18 Lines | |||||
* that we've allocated. | * that we've allocated. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
em_allocate_receive_buffers(struct rx_ring *rxr) | em_allocate_receive_buffers(struct rx_ring *rxr) | ||||
{ | { | ||||
struct adapter *adapter = rxr->adapter; | struct adapter *adapter = rxr->adapter; | ||||
device_t dev = adapter->dev; | device_t dev = adapter->dev; | ||||
struct em_buffer *rxbuf; | struct em_rxbuffer *rxbuf; | ||||
int error; | int error; | ||||
rxr->rx_buffers = malloc(sizeof(struct em_buffer) * | rxr->rx_buffers = malloc(sizeof(struct em_rxbuffer) * | ||||
adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); | adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); | ||||
if (rxr->rx_buffers == NULL) { | if (rxr->rx_buffers == NULL) { | ||||
device_printf(dev, "Unable to allocate rx_buffer memory\n"); | device_printf(dev, "Unable to allocate rx_buffer memory\n"); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | ||||
1, 0, /* alignment, bounds */ | 1, 0, /* alignment, bounds */ | ||||
Show All 36 Lines | |||||
* | * | ||||
* Initialize a receive ring and its buffers. | * Initialize a receive ring and its buffers. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
em_setup_receive_ring(struct rx_ring *rxr) | em_setup_receive_ring(struct rx_ring *rxr) | ||||
{ | { | ||||
struct adapter *adapter = rxr->adapter; | struct adapter *adapter = rxr->adapter; | ||||
struct em_buffer *rxbuf; | struct em_rxbuffer *rxbuf; | ||||
bus_dma_segment_t seg[1]; | bus_dma_segment_t seg[1]; | ||||
int rsize, nsegs, error = 0; | int rsize, nsegs, error = 0; | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
struct netmap_slot *slot; | struct netmap_slot *slot; | ||||
struct netmap_adapter *na = netmap_getna(adapter->ifp); | struct netmap_adapter *na = netmap_getna(adapter->ifp); | ||||
#endif | #endif | ||||
/* Clear the ring contents */ | /* Clear the ring contents */ | ||||
EM_RX_LOCK(rxr); | EM_RX_LOCK(rxr); | ||||
rsize = roundup2(adapter->num_rx_desc * | rsize = roundup2(adapter->num_rx_desc * | ||||
sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); | sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); | ||||
bzero((void *)rxr->rx_base, rsize); | bzero((void *)rxr->rx_base, rsize); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
slot = netmap_reset(na, NR_RX, rxr->me, 0); | slot = netmap_reset(na, NR_RX, rxr->me, 0); | ||||
#endif | #endif | ||||
/* | /* | ||||
** Free current RX buffer structs and their mbufs | ** Free current RX buffer structs and their mbufs | ||||
*/ | */ | ||||
Show All 14 Lines | |||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
if (slot) { | if (slot) { | ||||
int si = netmap_idx_n2k(&na->rx_rings[rxr->me], j); | int si = netmap_idx_n2k(&na->rx_rings[rxr->me], j); | ||||
uint64_t paddr; | uint64_t paddr; | ||||
void *addr; | void *addr; | ||||
addr = PNMB(na, slot + si, &paddr); | addr = PNMB(na, slot + si, &paddr); | ||||
netmap_load_map(na, rxr->rxtag, rxbuf->map, addr); | netmap_load_map(na, rxr->rxtag, rxbuf->map, addr); | ||||
/* Update descriptor */ | em_setup_rxdesc(&rxr->rx_base[j], rxbuf); | ||||
rxr->rx_base[j].buffer_addr = htole64(paddr); | |||||
continue; | continue; | ||||
} | } | ||||
#endif /* DEV_NETMAP */ | #endif /* DEV_NETMAP */ | ||||
rxbuf->m_head = m_getjcl(M_NOWAIT, MT_DATA, | rxbuf->m_head = m_getjcl(M_NOWAIT, MT_DATA, | ||||
M_PKTHDR, adapter->rx_mbuf_sz); | M_PKTHDR, adapter->rx_mbuf_sz); | ||||
if (rxbuf->m_head == NULL) { | if (rxbuf->m_head == NULL) { | ||||
error = ENOBUFS; | error = ENOBUFS; | ||||
goto fail; | goto fail; | ||||
Show All 9 Lines | #endif /* DEV_NETMAP */ | ||||
if (error != 0) { | if (error != 0) { | ||||
m_freem(rxbuf->m_head); | m_freem(rxbuf->m_head); | ||||
rxbuf->m_head = NULL; | rxbuf->m_head = NULL; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
bus_dmamap_sync(rxr->rxtag, | bus_dmamap_sync(rxr->rxtag, | ||||
rxbuf->map, BUS_DMASYNC_PREREAD); | rxbuf->map, BUS_DMASYNC_PREREAD); | ||||
/* Update descriptor */ | rxbuf->paddr = seg[0].ds_addr; | ||||
rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr); | em_setup_rxdesc(&rxr->rx_base[j], rxbuf); | ||||
} | } | ||||
rxr->next_to_check = 0; | rxr->next_to_check = 0; | ||||
rxr->next_to_refresh = 0; | rxr->next_to_refresh = 0; | ||||
bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
fail: | fail: | ||||
EM_RX_UNLOCK(rxr); | EM_RX_UNLOCK(rxr); | ||||
Show All 20 Lines | fail: | ||||
/* | /* | ||||
* Free RX buffers allocated so far, we will only handle | * Free RX buffers allocated so far, we will only handle | ||||
* the rings that completed, the failing case will have | * the rings that completed, the failing case will have | ||||
* cleaned up for itself. 'q' failed, so its the terminus. | * cleaned up for itself. 'q' failed, so its the terminus. | ||||
*/ | */ | ||||
for (int i = 0; i < q; ++i) { | for (int i = 0; i < q; ++i) { | ||||
rxr = &adapter->rx_rings[i]; | rxr = &adapter->rx_rings[i]; | ||||
for (int n = 0; n < adapter->num_rx_desc; n++) { | for (int n = 0; n < adapter->num_rx_desc; n++) { | ||||
struct em_buffer *rxbuf; | struct em_rxbuffer *rxbuf; | ||||
rxbuf = &rxr->rx_buffers[n]; | rxbuf = &rxr->rx_buffers[n]; | ||||
if (rxbuf->m_head != NULL) { | if (rxbuf->m_head != NULL) { | ||||
bus_dmamap_sync(rxr->rxtag, rxbuf->map, | bus_dmamap_sync(rxr->rxtag, rxbuf->map, | ||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
bus_dmamap_unload(rxr->rxtag, rxbuf->map); | bus_dmamap_unload(rxr->rxtag, rxbuf->map); | ||||
m_freem(rxbuf->m_head); | m_freem(rxbuf->m_head); | ||||
rxbuf->m_head = NULL; | rxbuf->m_head = NULL; | ||||
} | } | ||||
Show All 30 Lines | |||||
* | * | ||||
* Free receive ring data structures | * Free receive ring data structures | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_free_receive_buffers(struct rx_ring *rxr) | em_free_receive_buffers(struct rx_ring *rxr) | ||||
{ | { | ||||
struct adapter *adapter = rxr->adapter; | struct adapter *adapter = rxr->adapter; | ||||
struct em_buffer *rxbuf = NULL; | struct em_rxbuffer *rxbuf = NULL; | ||||
INIT_DEBUGOUT("free_receive_buffers: begin"); | INIT_DEBUGOUT("free_receive_buffers: begin"); | ||||
if (rxr->rx_buffers != NULL) { | if (rxr->rx_buffers != NULL) { | ||||
for (int i = 0; i < adapter->num_rx_desc; i++) { | for (int i = 0; i < adapter->num_rx_desc; i++) { | ||||
rxbuf = &rxr->rx_buffers[i]; | rxbuf = &rxr->rx_buffers[i]; | ||||
if (rxbuf->map != NULL) { | if (rxbuf->map != NULL) { | ||||
bus_dmamap_sync(rxr->rxtag, rxbuf->map, | bus_dmamap_sync(rxr->rxtag, rxbuf->map, | ||||
Show All 25 Lines | |||||
* | * | ||||
* Enable receive unit. | * Enable receive unit. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_initialize_receive_unit(struct adapter *adapter) | em_initialize_receive_unit(struct adapter *adapter) | ||||
{ | { | ||||
struct rx_ring *rxr = adapter->rx_rings; | struct rx_ring *rxr = adapter->rx_rings; | ||||
if_t ifp = adapter->ifp; | if_t ifp = adapter->ifp; | ||||
struct e1000_hw *hw = &adapter->hw; | struct e1000_hw *hw = &adapter->hw; | ||||
u64 bus_addr; | u32 rctl, rxcsum, rfctl; | ||||
u32 rctl, rxcsum; | |||||
INIT_DEBUGOUT("em_initialize_receive_units: begin"); | INIT_DEBUGOUT("em_initialize_receive_units: begin"); | ||||
/* | /* | ||||
* Make sure receives are disabled while setting | * Make sure receives are disabled while setting | ||||
* up the descriptor ring | * up the descriptor ring | ||||
*/ | */ | ||||
rctl = E1000_READ_REG(hw, E1000_RCTL); | rctl = E1000_READ_REG(hw, E1000_RCTL); | ||||
/* Do not disable if ever enabled on this hardware */ | /* Do not disable if ever enabled on this hardware */ | ||||
if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583)) | if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583)) | ||||
E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); | E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); | ||||
/* Setup the Receive Control Register */ | |||||
rctl &= ~(3 << E1000_RCTL_MO_SHIFT); | |||||
rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | | |||||
E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | | |||||
(hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); | |||||
/* Do not store bad packets */ | |||||
rctl &= ~E1000_RCTL_SBP; | |||||
/* Enable Long Packet receive */ | |||||
if (if_getmtu(ifp) > ETHERMTU) | |||||
rctl |= E1000_RCTL_LPE; | |||||
else | |||||
rctl &= ~E1000_RCTL_LPE; | |||||
/* Strip the CRC */ | |||||
if (!em_disable_crc_stripping) | |||||
rctl |= E1000_RCTL_SECRC; | |||||
E1000_WRITE_REG(&adapter->hw, E1000_RADV, | E1000_WRITE_REG(&adapter->hw, E1000_RADV, | ||||
adapter->rx_abs_int_delay.value); | adapter->rx_abs_int_delay.value); | ||||
E1000_WRITE_REG(&adapter->hw, E1000_RDTR, | E1000_WRITE_REG(&adapter->hw, E1000_RDTR, | ||||
adapter->rx_int_delay.value); | adapter->rx_int_delay.value); | ||||
/* | /* | ||||
* Set the interrupt throttling rate. Value is calculated | * Set the interrupt throttling rate. Value is calculated | ||||
* as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) | * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) | ||||
*/ | */ | ||||
E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); | E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); | ||||
/* Use extended rx descriptor formats */ | |||||
rfctl = E1000_READ_REG(hw, E1000_RFCTL); | |||||
rfctl |= E1000_RFCTL_EXTEN; | |||||
/* | /* | ||||
** When using MSIX interrupts we need to throttle | ** When using MSIX interrupts we need to throttle | ||||
** using the EITR register (82574 only) | ** using the EITR register (82574 only) | ||||
*/ | */ | ||||
if (hw->mac.type == e1000_82574) { | if (hw->mac.type == e1000_82574) { | ||||
u32 rfctl; | |||||
for (int i = 0; i < 4; i++) | for (int i = 0; i < 4; i++) | ||||
E1000_WRITE_REG(hw, E1000_EITR_82574(i), | E1000_WRITE_REG(hw, E1000_EITR_82574(i), | ||||
DEFAULT_ITR); | DEFAULT_ITR); | ||||
/* Disable accelerated acknowledge */ | /* Disable accelerated acknowledge */ | ||||
rfctl = E1000_READ_REG(hw, E1000_RFCTL); | |||||
rfctl |= E1000_RFCTL_ACK_DIS; | rfctl |= E1000_RFCTL_ACK_DIS; | ||||
E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); | |||||
} | } | ||||
E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); | |||||
rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); | rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); | ||||
if (if_getcapenable(ifp) & IFCAP_RXCSUM) { | if (if_getcapenable(ifp) & IFCAP_RXCSUM) { | ||||
#ifdef EM_MULTIQUEUE | #ifdef EM_MULTIQUEUE | ||||
rxcsum |= E1000_RXCSUM_TUOFL | | rxcsum |= E1000_RXCSUM_TUOFL | | ||||
E1000_RXCSUM_IPOFL | | E1000_RXCSUM_IPOFL | | ||||
E1000_RXCSUM_PCSD; | E1000_RXCSUM_PCSD; | ||||
#else | #else | ||||
rxcsum |= E1000_RXCSUM_TUOFL; | rxcsum |= E1000_RXCSUM_TUOFL; | ||||
#endif | #endif | ||||
} else | } else | ||||
rxcsum &= ~E1000_RXCSUM_TUOFL; | rxcsum &= ~E1000_RXCSUM_TUOFL; | ||||
E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); | E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); | ||||
#ifdef EM_MULTIQUEUE | #ifdef EM_MULTIQUEUE | ||||
#define RSSKEYLEN 10 | |||||
if (adapter->num_queues > 1) { | if (adapter->num_queues > 1) { | ||||
uint32_t rss_key[10]; | uint8_t rss_key[4 * RSSKEYLEN]; | ||||
uint32_t reta; | uint32_t reta = 0; | ||||
int i; | int i; | ||||
/* | /* | ||||
* Configure RSS key | * Configure RSS key | ||||
*/ | */ | ||||
arc4rand(rss_key, sizeof(rss_key), 0); | arc4rand(rss_key, sizeof(rss_key), 0); | ||||
for (i = 0; i < 10; ++i) | for (i = 0; i < RSSKEYLEN; ++i) { | ||||
E1000_WRITE_REG_ARRAY(hw,E1000_RSSRK(0), i, rss_key[i]); | uint32_t rssrk = 0; | ||||
rssrk = EM_RSSRK_VAL(rss_key, i); | |||||
E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); | |||||
} | |||||
/* | /* | ||||
* Configure RSS redirect table in following fashion: | * Configure RSS redirect table in following fashion: | ||||
* (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] | * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] | ||||
*/ | */ | ||||
reta = 0; | for (i = 0; i < sizeof(reta); ++i) { | ||||
for (i = 0; i < 4; ++i) { | |||||
uint32_t q; | uint32_t q; | ||||
q = (i % adapter->num_queues) << 7; | q = (i % adapter->num_queues) << 7; | ||||
reta |= q << (8 * i); | reta |= q << (8 * i); | ||||
} | } | ||||
for (i = 0; i < 32; ++i) | |||||
for (i = 0; i < 32; ++i) { | |||||
E1000_WRITE_REG(hw, E1000_RETA(i), reta); | E1000_WRITE_REG(hw, E1000_RETA(i), reta); | ||||
} | |||||
E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | | E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | | ||||
E1000_MRQC_RSS_FIELD_IPV4_TCP | | E1000_MRQC_RSS_FIELD_IPV4_TCP | | ||||
E1000_MRQC_RSS_FIELD_IPV4 | | E1000_MRQC_RSS_FIELD_IPV4 | | ||||
E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | | E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | | ||||
E1000_MRQC_RSS_FIELD_IPV6_EX | | E1000_MRQC_RSS_FIELD_IPV6_EX | | ||||
E1000_MRQC_RSS_FIELD_IPV6 | | E1000_MRQC_RSS_FIELD_IPV6); | ||||
E1000_MRQC_RSS_FIELD_IPV6_TCP); | |||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
** XXX TEMPORARY WORKAROUND: on some systems with 82573 | ** XXX TEMPORARY WORKAROUND: on some systems with 82573 | ||||
** long latencies are observed, like Lenovo X60. This | ** long latencies are observed, like Lenovo X60. This | ||||
** change eliminates the problem, but since having positive | ** change eliminates the problem, but since having positive | ||||
** values in RDTR is a known source of problems on other | ** values in RDTR is a known source of problems on other | ||||
** platforms another solution is being sought. | ** platforms another solution is being sought. | ||||
*/ | */ | ||||
if (hw->mac.type == e1000_82573) | if (hw->mac.type == e1000_82573) | ||||
E1000_WRITE_REG(hw, E1000_RDTR, 0x20); | E1000_WRITE_REG(hw, E1000_RDTR, 0x20); | ||||
for (int i = 0; i < adapter->num_queues; i++, rxr++) { | for (int i = 0; i < adapter->num_queues; i++, rxr++) { | ||||
/* Setup the Base and Length of the Rx Descriptor Ring */ | /* Setup the Base and Length of the Rx Descriptor Ring */ | ||||
u64 bus_addr = rxr->rxdma.dma_paddr; | |||||
u32 rdt = adapter->num_rx_desc - 1; /* default */ | u32 rdt = adapter->num_rx_desc - 1; /* default */ | ||||
bus_addr = rxr->rxdma.dma_paddr; | |||||
E1000_WRITE_REG(hw, E1000_RDLEN(i), | E1000_WRITE_REG(hw, E1000_RDLEN(i), | ||||
adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); | adapter->num_rx_desc * sizeof(union e1000_rx_desc_extended)); | ||||
E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); | E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); | ||||
E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); | E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); | ||||
/* Setup the Head and Tail Descriptor Pointers */ | /* Setup the Head and Tail Descriptor Pointers */ | ||||
E1000_WRITE_REG(hw, E1000_RDH(i), 0); | E1000_WRITE_REG(hw, E1000_RDH(i), 0); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
/* | /* | ||||
* an init() while a netmap client is active must | * an init() while a netmap client is active must | ||||
* preserve the rx buffers passed to userspace. | * preserve the rx buffers passed to userspace. | ||||
Show All 14 Lines | #endif /* DEV_NETMAP */ | ||||
* settings. | * settings. | ||||
*/ | */ | ||||
if (((adapter->hw.mac.type == e1000_ich9lan) || | if (((adapter->hw.mac.type == e1000_ich9lan) || | ||||
(adapter->hw.mac.type == e1000_pch2lan) || | (adapter->hw.mac.type == e1000_pch2lan) || | ||||
(adapter->hw.mac.type == e1000_ich10lan)) && | (adapter->hw.mac.type == e1000_ich10lan)) && | ||||
(if_getmtu(ifp) > ETHERMTU)) { | (if_getmtu(ifp) > ETHERMTU)) { | ||||
u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); | u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); | ||||
E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); | E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); | ||||
} else if ((adapter->hw.mac.type == e1000_82574) && | } else if (adapter->hw.mac.type == e1000_82574) { | ||||
(if_getmtu(ifp) > ETHERMTU)) { | |||||
for (int i = 0; i < adapter->num_queues; i++) { | for (int i = 0; i < adapter->num_queues; i++) { | ||||
u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); | u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); | ||||
rxdctl |= 0x20; /* PTHRESH */ | rxdctl |= 0x20; /* PTHRESH */ | ||||
rxdctl |= 4 << 8; /* HTHRESH */ | rxdctl |= 4 << 8; /* HTHRESH */ | ||||
rxdctl |= 4 << 16;/* WTHRESH */ | rxdctl |= 4 << 16;/* WTHRESH */ | ||||
rxdctl |= 1 << 24; /* Switch to granularity */ | rxdctl |= 1 << 24; /* Switch to granularity */ | ||||
E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); | E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); | ||||
} | } | ||||
} | } | ||||
if (adapter->hw.mac.type >= e1000_pch2lan) { | if (adapter->hw.mac.type >= e1000_pch2lan) { | ||||
if (if_getmtu(ifp) > ETHERMTU) | if (if_getmtu(ifp) > ETHERMTU) | ||||
e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); | e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); | ||||
else | else | ||||
e1000_lv_jumbo_workaround_ich8lan(hw, FALSE); | e1000_lv_jumbo_workaround_ich8lan(hw, FALSE); | ||||
} | } | ||||
/* Setup the Receive Control Register */ | |||||
rctl &= ~(3 << E1000_RCTL_MO_SHIFT); | |||||
rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | | |||||
E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | | |||||
(hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); | |||||
/* Strip the CRC */ | |||||
if (!em_disable_crc_stripping) | |||||
rctl |= E1000_RCTL_SECRC; | |||||
/* Make sure VLAN Filters are off */ | /* Make sure VLAN Filters are off */ | ||||
rctl &= ~E1000_RCTL_VFE; | rctl &= ~E1000_RCTL_VFE; | ||||
rctl &= ~E1000_RCTL_SBP; | |||||
if (adapter->rx_mbuf_sz == MCLBYTES) | if (adapter->rx_mbuf_sz == MCLBYTES) | ||||
rctl |= E1000_RCTL_SZ_2048; | rctl |= E1000_RCTL_SZ_2048; | ||||
else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) | else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) | ||||
rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; | rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; | ||||
else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) | else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) | ||||
rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; | rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; | ||||
if (if_getmtu(ifp) > ETHERMTU) | /* ensure we clear use DTYPE of 00 here */ | ||||
rctl |= E1000_RCTL_LPE; | rctl &= ~0x00000C00; | ||||
else | |||||
rctl &= ~E1000_RCTL_LPE; | |||||
/* Write out the settings */ | /* Write out the settings */ | ||||
E1000_WRITE_REG(hw, E1000_RCTL, rctl); | E1000_WRITE_REG(hw, E1000_RCTL, rctl); | ||||
return; | return; | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* This routine executes in interrupt context. It replenishes | * This routine executes in interrupt context. It replenishes | ||||
* the mbufs in the descriptor and sends data which has been | * the mbufs in the descriptor and sends data which has been | ||||
* dma'ed into host memory to upper layer. | * dma'ed into host memory to upper layer. | ||||
* | * | ||||
* We loop at most count times if count is > 0, or until done if | * We loop at most count times if count is > 0, or until done if | ||||
* count < 0. | * count < 0. | ||||
* | * | ||||
* For polling we also now return the number of cleaned packets | * For polling we also now return the number of cleaned packets | ||||
*********************************************************************/ | *********************************************************************/ | ||||
static bool | static bool | ||||
em_rxeof(struct rx_ring *rxr, int count, int *done) | em_rxeof(struct rx_ring *rxr, int count, int *done) | ||||
{ | { | ||||
struct adapter *adapter = rxr->adapter; | struct adapter *adapter = rxr->adapter; | ||||
if_t ifp = adapter->ifp; | if_t ifp = adapter->ifp; | ||||
struct mbuf *mp, *sendmp; | struct mbuf *mp, *sendmp; | ||||
u8 status = 0; | u32 status = 0; | ||||
u16 len; | u16 len; | ||||
int i, processed, rxdone = 0; | int i, processed, rxdone = 0; | ||||
bool eop; | bool eop; | ||||
struct e1000_rx_desc *cur; | union e1000_rx_desc_extended *cur; | ||||
EM_RX_LOCK(rxr); | EM_RX_LOCK(rxr); | ||||
/* Sync the ring */ | /* Sync the ring */ | ||||
bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
if (netmap_rx_irq(ifp, rxr->me, &processed)) { | if (netmap_rx_irq(ifp, rxr->me, &processed)) { | ||||
EM_RX_UNLOCK(rxr); | EM_RX_UNLOCK(rxr); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
#endif /* DEV_NETMAP */ | #endif /* DEV_NETMAP */ | ||||
for (i = rxr->next_to_check, processed = 0; count != 0;) { | for (i = rxr->next_to_check, processed = 0; count != 0;) { | ||||
if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) | if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) | ||||
break; | break; | ||||
cur = &rxr->rx_base[i]; | cur = &rxr->rx_base[i]; | ||||
status = cur->status; | status = le32toh(cur->wb.upper.status_error); | ||||
mp = sendmp = NULL; | mp = sendmp = NULL; | ||||
if ((status & E1000_RXD_STAT_DD) == 0) | if ((status & E1000_RXD_STAT_DD) == 0) | ||||
break; | break; | ||||
len = le16toh(cur->length); | len = le16toh(cur->wb.upper.length); | ||||
eop = (status & E1000_RXD_STAT_EOP) != 0; | eop = (status & E1000_RXD_STAT_EOP) != 0; | ||||
if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) || | if ((status & E1000_RXDEXT_ERR_FRAME_ERR_MASK) || | ||||
(rxr->discard == TRUE)) { | (rxr->discard == TRUE)) { | ||||
adapter->dropped_pkts++; | adapter->dropped_pkts++; | ||||
++rxr->rx_discarded; | ++rxr->rx_discarded; | ||||
if (!eop) /* Catch subsequent segs */ | if (!eop) /* Catch subsequent segs */ | ||||
rxr->discard = TRUE; | rxr->discard = TRUE; | ||||
else | else | ||||
rxr->discard = FALSE; | rxr->discard = FALSE; | ||||
em_rx_discard(rxr, i); | em_rx_discard(rxr, i); | ||||
Show All 20 Lines | if (rxr->fmp == NULL) { | ||||
rxr->fmp->m_pkthdr.len += len; | rxr->fmp->m_pkthdr.len += len; | ||||
} | } | ||||
if (eop) { | if (eop) { | ||||
--count; | --count; | ||||
sendmp = rxr->fmp; | sendmp = rxr->fmp; | ||||
if_setrcvif(sendmp, ifp); | if_setrcvif(sendmp, ifp); | ||||
if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); | if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); | ||||
em_receive_checksum(cur, sendmp); | em_receive_checksum(status, sendmp); | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
if (adapter->hw.mac.max_frame_size > | if (adapter->hw.mac.max_frame_size > | ||||
(MCLBYTES - ETHER_ALIGN) && | (MCLBYTES - ETHER_ALIGN) && | ||||
em_fixup_rx(rxr) != 0) | em_fixup_rx(rxr) != 0) | ||||
goto skip; | goto skip; | ||||
#endif | #endif | ||||
if (status & E1000_RXD_STAT_VP) { | if (status & E1000_RXD_STAT_VP) { | ||||
if_setvtag(sendmp, | if_setvtag(sendmp, | ||||
le16toh(cur->special)); | le16toh(cur->wb.upper.vlan)); | ||||
sendmp->m_flags |= M_VLANTAG; | sendmp->m_flags |= M_VLANTAG; | ||||
} | } | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
skip: | skip: | ||||
#endif | #endif | ||||
rxr->fmp = rxr->lmp = NULL; | rxr->fmp = rxr->lmp = NULL; | ||||
} | } | ||||
next_desc: | next_desc: | ||||
/* Sync the ring */ | /* Sync the ring */ | ||||
bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
/* Zero out the receive descriptors status. */ | /* Zero out the receive descriptors status. */ | ||||
cur->status = 0; | cur->wb.upper.status_error &= htole32(~0xFF); | ||||
++rxdone; /* cumulative for POLL */ | ++rxdone; /* cumulative for POLL */ | ||||
++processed; | ++processed; | ||||
/* Advance our pointers to the next descriptor. */ | /* Advance our pointers to the next descriptor. */ | ||||
if (++i == adapter->num_rx_desc) | if (++i == adapter->num_rx_desc) | ||||
i = 0; | i = 0; | ||||
/* Send to the stack */ | /* Send to the stack */ | ||||
Show All 22 Lines | next_desc: | ||||
EM_RX_UNLOCK(rxr); | EM_RX_UNLOCK(rxr); | ||||
return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); | return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
em_rx_discard(struct rx_ring *rxr, int i) | em_rx_discard(struct rx_ring *rxr, int i) | ||||
{ | { | ||||
struct em_buffer *rbuf; | struct em_rxbuffer *rbuf; | ||||
rbuf = &rxr->rx_buffers[i]; | rbuf = &rxr->rx_buffers[i]; | ||||
bus_dmamap_unload(rxr->rxtag, rbuf->map); | bus_dmamap_unload(rxr->rxtag, rbuf->map); | ||||
/* Free any previous pieces */ | /* Free any previous pieces */ | ||||
if (rxr->fmp != NULL) { | if (rxr->fmp != NULL) { | ||||
rxr->fmp->m_flags |= M_PKTHDR; | rxr->fmp->m_flags |= M_PKTHDR; | ||||
m_freem(rxr->fmp); | m_freem(rxr->fmp); | ||||
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | if (n != NULL) { | ||||
error = ENOMEM; | error = ENOMEM; | ||||
} | } | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
#endif | #endif | ||||
static void | |||||
em_setup_rxdesc(union e1000_rx_desc_extended *rxd, const struct em_rxbuffer *rxbuf) | |||||
{ | |||||
rxd->read.buffer_addr = htole64(rxbuf->paddr); | |||||
/* DD bits must be cleared */ | |||||
rxd->wb.upper.status_error= 0; | |||||
} | |||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Verify that the hardware indicated that the checksum is valid. | * Verify that the hardware indicated that the checksum is valid. | ||||
* Inform the stack about the status of checksum so that stack | * Inform the stack about the status of checksum so that stack | ||||
* doesn't spend time verifying the checksum. | * doesn't spend time verifying the checksum. | ||||
* | * | ||||
*********************************************************************/ | *********************************************************************/ | ||||
static void | static void | ||||
em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp) | em_receive_checksum(uint32_t status, struct mbuf *mp) | ||||
{ | { | ||||
mp->m_pkthdr.csum_flags = 0; | mp->m_pkthdr.csum_flags = 0; | ||||
/* Ignore Checksum bit is set */ | /* Ignore Checksum bit is set */ | ||||
if (rx_desc->status & E1000_RXD_STAT_IXSM) | if (status & E1000_RXD_STAT_IXSM) | ||||
return; | return; | ||||
if (rx_desc->errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) | /* If the IP checksum exists and there is no IP Checksum error */ | ||||
return; | if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == | ||||
E1000_RXD_STAT_IPCS) { | |||||
/* IP Checksum Good? */ | |||||
if (rx_desc->status & E1000_RXD_STAT_IPCS) | |||||
mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); | mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); | ||||
} | |||||
/* TCP or UDP checksum */ | /* TCP or UDP checksum */ | ||||
if (rx_desc->status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { | if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == | ||||
E1000_RXD_STAT_TCPCS) { | |||||
mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); | |||||
mp->m_pkthdr.csum_data = htons(0xffff); | |||||
} | |||||
if (status & E1000_RXD_STAT_UDPCS) { | |||||
mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); | mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); | ||||
mp->m_pkthdr.csum_data = htons(0xffff); | mp->m_pkthdr.csum_data = htons(0xffff); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* This routine is run via an vlan | * This routine is run via an vlan | ||||
* config EVENT | * config EVENT | ||||
▲ Show 20 Lines • Show All 1,222 Lines • Show Last 20 Lines |