Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/e1000/if_em.c
Show All 26 Lines | /****************************************************************************** | ||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||||
POSSIBILITY OF SUCH DAMAGE. | POSSIBILITY OF SUCH DAMAGE. | ||||
******************************************************************************/ | ******************************************************************************/ | ||||
/*$FreeBSD$*/ | /*$FreeBSD$*/ | ||||
#include "opt_em.h" | |||||
#include "opt_inet.h" | #include "opt_inet.h" | ||||
#include "opt_inet6.h" | #include "opt_inet6.h" | ||||
#ifdef HAVE_KERNEL_OPTION_HEADERS | #ifdef HAVE_KERNEL_OPTION_HEADERS | ||||
#include "opt_device_polling.h" | #include "opt_device_polling.h" | ||||
#endif | #endif | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
▲ Show 20 Lines • Show All 251 Lines • ▼ Show 20 Lines | |||||
/* MSIX handlers */ | /* MSIX handlers */ | ||||
static void em_msix_tx(void *); | static void em_msix_tx(void *); | ||||
static void em_msix_rx(void *); | static void em_msix_rx(void *); | ||||
static void em_msix_link(void *); | static void em_msix_link(void *); | ||||
static void em_handle_tx(void *context, int pending); | static void em_handle_tx(void *context, int pending); | ||||
static void em_handle_rx(void *context, int pending); | static void em_handle_rx(void *context, int pending); | ||||
static void em_handle_link(void *context, int pending); | static void em_handle_link(void *context, int pending); | ||||
#ifdef EM_MULTIQUEUE | |||||
static void em_enable_vectors_82574(struct adapter *); | |||||
#endif | |||||
static void em_set_sysctl_value(struct adapter *, const char *, | static void em_set_sysctl_value(struct adapter *, const char *, | ||||
const char *, int *, int); | const char *, int *, int); | ||||
static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); | static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); | ||||
static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); | static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); | ||||
static __inline void em_rx_discard(struct rx_ring *, int); | static __inline void em_rx_discard(struct rx_ring *, int); | ||||
#ifdef DEVICE_POLLING | #ifdef DEVICE_POLLING | ||||
▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines | |||||
static int em_debug_sbp = FALSE; | static int em_debug_sbp = FALSE; | ||||
SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, | SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, | ||||
"Show bad packets in promiscuous mode"); | "Show bad packets in promiscuous mode"); | ||||
static int em_enable_msix = TRUE; | static int em_enable_msix = TRUE; | ||||
SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0, | SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0, | ||||
"Enable MSI-X interrupts"); | "Enable MSI-X interrupts"); | ||||
#ifdef EM_MULTIQUEUE | |||||
static int em_num_tx_queues = 1; | |||||
SYSCTL_INT(_hw_em, OID_AUTO, num_tx_queues, CTLFLAG_RDTUN, &em_num_tx_queues, 0, | |||||
"82574 only: Number of tx queues to configure, 0 indicates autoconfigure"); | |||||
static int em_num_rx_queues = 1; | |||||
SYSCTL_INT(_hw_em, OID_AUTO, num_rx_queues, CTLFLAG_RDTUN, &em_num_rx_queues, 0, | |||||
"82574 only: Number of rx queues to configure, 0 indicates autoconfigure"); | |||||
#endif | |||||
/* How many packets rxeof tries to clean at a time */ | /* How many packets rxeof tries to clean at a time */ | ||||
static int em_rx_process_limit = 100; | static int em_rx_process_limit = 100; | ||||
SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, | SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, | ||||
&em_rx_process_limit, 0, | &em_rx_process_limit, 0, | ||||
"Maximum number of received packets to process " | "Maximum number of received packets to process " | ||||
"at a time, -1 means unlimited"); | "at a time, -1 means unlimited"); | ||||
/* Energy efficient ethernet - default to OFF */ | /* Energy efficient ethernet - default to OFF */ | ||||
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | if ((pci_vendor_id == ent->vendor_id) && | ||||
return (BUS_PROBE_DEFAULT); | return (BUS_PROBE_DEFAULT); | ||||
} | } | ||||
ent++; | ent++; | ||||
} | } | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
#ifdef EM_MULTIQUEUE | |||||
/* | |||||
* 82574 only: | |||||
* Write a new value to the EEPROM increasing the number of MSIX | |||||
* vectors from 3 to 5, for proper multiqueue support. | |||||
*/ | |||||
static void | |||||
em_enable_vectors_82574(struct adapter *adapter) | |||||
{ | |||||
struct e1000_hw *hw = &adapter->hw; | |||||
device_t dev = adapter->dev; | |||||
u16 edata; | |||||
e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); | |||||
printf("Current cap: %#06x\n", edata); | |||||
if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) { | |||||
device_printf(dev, "Writing to eeprom: increasing " | |||||
"reported MSIX vectors from 3 to 5...\n"); | |||||
edata &= ~(EM_NVM_MSIX_N_MASK); | |||||
edata |= 4 << EM_NVM_MSIX_N_SHIFT; | |||||
e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); | |||||
e1000_update_nvm_checksum(hw); | |||||
device_printf(dev, "Writing to eeprom: done\n"); | |||||
} | |||||
} | |||||
#endif | |||||
/********************************************************************* | /********************************************************************* | ||||
* Device initialization routine | * Device initialization routine | ||||
* | * | ||||
* The attach entry point is called when the driver is being loaded. | * The attach entry point is called when the driver is being loaded. | ||||
* This routine identifies the type of hardware, allocates all resources | * This routine identifies the type of hardware, allocates all resources | ||||
* and initializes the hardware. | * and initializes the hardware. | ||||
* | * | ||||
* return 0 on success, positive on failure | * return 0 on success, positive on failure | ||||
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | em_attach(device_t dev) | ||||
/* Do Shared Code initialization */ | /* Do Shared Code initialization */ | ||||
if (e1000_setup_init_funcs(hw, TRUE)) { | if (e1000_setup_init_funcs(hw, TRUE)) { | ||||
device_printf(dev, "Setup of Shared code failed\n"); | device_printf(dev, "Setup of Shared code failed\n"); | ||||
error = ENXIO; | error = ENXIO; | ||||
goto err_pci; | goto err_pci; | ||||
} | } | ||||
/* | |||||
* Setup MSI/X or MSI if PCI Express | |||||
*/ | |||||
adapter->msix = em_setup_msix(adapter); | |||||
e1000_get_bus_info(hw); | e1000_get_bus_info(hw); | ||||
/* Set up some sysctls for the tunable interrupt delays */ | /* Set up some sysctls for the tunable interrupt delays */ | ||||
em_add_int_delay_sysctl(adapter, "rx_int_delay", | em_add_int_delay_sysctl(adapter, "rx_int_delay", | ||||
"receive interrupt delay in usecs", &adapter->rx_int_delay, | "receive interrupt delay in usecs", &adapter->rx_int_delay, | ||||
E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt); | E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt); | ||||
em_add_int_delay_sysctl(adapter, "tx_int_delay", | em_add_int_delay_sysctl(adapter, "tx_int_delay", | ||||
"transmit interrupt delay in usecs", &adapter->tx_int_delay, | "transmit interrupt delay in usecs", &adapter->tx_int_delay, | ||||
▲ Show 20 Lines • Show All 310 Lines • ▼ Show 20 Lines | em_resume(device_t dev) | ||||
EM_CORE_LOCK(adapter); | EM_CORE_LOCK(adapter); | ||||
if (adapter->hw.mac.type == e1000_pch2lan) | if (adapter->hw.mac.type == e1000_pch2lan) | ||||
e1000_resume_workarounds_pchlan(&adapter->hw); | e1000_resume_workarounds_pchlan(&adapter->hw); | ||||
em_init_locked(adapter); | em_init_locked(adapter); | ||||
em_init_manageability(adapter); | em_init_manageability(adapter); | ||||
if ((if_getflags(ifp) & IFF_UP) && | if ((if_getflags(ifp) & IFF_UP) && | ||||
(if_getdrvflags(ifp) & IFF_DRV_RUNNING) && adapter->link_active) { | (if_getdrvflags(ifp) & IFF_DRV_RUNNING) && adapter->link_active) { | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | ||||
EM_TX_LOCK(txr); | EM_TX_LOCK(txr); | ||||
#ifdef EM_MULTIQUEUE | #ifdef EM_MULTIQUEUE | ||||
if (!drbr_empty(ifp, txr->br)) | if (!drbr_empty(ifp, txr->br)) | ||||
em_mq_start_locked(ifp, txr, NULL); | em_mq_start_locked(ifp, txr, NULL); | ||||
#else | #else | ||||
if (!if_sendq_empty(ifp)) | if (!if_sendq_empty(ifp)) | ||||
em_start_locked(ifp, txr); | em_start_locked(ifp, txr); | ||||
#endif | #endif | ||||
Show All 17 Lines | |||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
em_mq_start_locked(if_t ifp, struct tx_ring *txr, struct mbuf *m) | em_mq_start_locked(if_t ifp, struct tx_ring *txr, struct mbuf *m) | ||||
{ | { | ||||
struct adapter *adapter = txr->adapter; | struct adapter *adapter = txr->adapter; | ||||
struct mbuf *next; | struct mbuf *next; | ||||
int err = 0, enq = 0; | int err = 0, enq = 0; | ||||
EM_TX_LOCK_ASSERT(txr); | |||||
if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != | if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != | ||||
IFF_DRV_RUNNING || adapter->link_active == 0) { | IFF_DRV_RUNNING || adapter->link_active == 0) { | ||||
if (m != NULL) | if (m != NULL) | ||||
err = drbr_enqueue(ifp, txr->br, m); | err = drbr_enqueue(ifp, txr->br, m); | ||||
return (err); | return (err); | ||||
} | } | ||||
enq = 0; | enq = 0; | ||||
Show All 12 Lines | if ((err = em_xmit(txr, &next)) != 0) { | ||||
drbr_putback(ifp, txr->br, next); | drbr_putback(ifp, txr->br, next); | ||||
break; | break; | ||||
} | } | ||||
drbr_advance(ifp, txr->br); | drbr_advance(ifp, txr->br); | ||||
enq++; | enq++; | ||||
if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); | if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); | ||||
if (next->m_flags & M_MCAST) | if (next->m_flags & M_MCAST) | ||||
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | ||||
if_etherbpfmtap(ifp, next); | ETHER_BPF_MTAP(ifp, next); | ||||
if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) | if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) | ||||
break; | break; | ||||
} | } | ||||
if (enq > 0) { | if (enq > 0) { | ||||
/* Set the watchdog */ | /* Set the watchdog */ | ||||
txr->queue_status = EM_QUEUE_WORKING; | txr->queue_status = EM_QUEUE_WORKING; | ||||
txr->watchdog_time = ticks; | txr->watchdog_time = ticks; | ||||
Show All 9 Lines | |||||
/* | /* | ||||
** Multiqueue capable stack interface | ** Multiqueue capable stack interface | ||||
*/ | */ | ||||
static int | static int | ||||
em_mq_start(if_t ifp, struct mbuf *m) | em_mq_start(if_t ifp, struct mbuf *m) | ||||
{ | { | ||||
struct adapter *adapter = if_getsoftc(ifp); | struct adapter *adapter = if_getsoftc(ifp); | ||||
struct tx_ring *txr = adapter->tx_rings; | struct tx_ring *txr = adapter->tx_rings; | ||||
int error; | int i, error; | ||||
if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) | |||||
i = m->m_pkthdr.flowid % adapter->num_tx_queues; | |||||
else | |||||
i = curcpu % adapter->num_tx_queues; | |||||
txr = &adapter->tx_rings[i]; | |||||
if (EM_TX_TRYLOCK(txr)) { | if (EM_TX_TRYLOCK(txr)) { | ||||
error = em_mq_start_locked(ifp, txr, m); | error = em_mq_start_locked(ifp, txr, m); | ||||
EM_TX_UNLOCK(txr); | EM_TX_UNLOCK(txr); | ||||
} else | } else | ||||
error = drbr_enqueue(ifp, txr->br, m); | error = drbr_enqueue(ifp, txr->br, m); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
** Flush all ring buffers | ** Flush all ring buffers | ||||
*/ | */ | ||||
static void | static void | ||||
em_qflush(if_t ifp) | em_qflush(if_t ifp) | ||||
{ | { | ||||
struct adapter *adapter = if_getsoftc(ifp); | struct adapter *adapter = if_getsoftc(ifp); | ||||
struct tx_ring *txr = adapter->tx_rings; | struct tx_ring *txr = adapter->tx_rings; | ||||
struct mbuf *m; | struct mbuf *m; | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | ||||
EM_TX_LOCK(txr); | EM_TX_LOCK(txr); | ||||
while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) | while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) | ||||
m_freem(m); | m_freem(m); | ||||
EM_TX_UNLOCK(txr); | EM_TX_UNLOCK(txr); | ||||
} | } | ||||
if_qflush(ifp); | if_qflush(ifp); | ||||
} | } | ||||
#else /* !EM_MULTIQUEUE */ | #else /* !EM_MULTIQUEUE */ | ||||
Show All 31 Lines | while (!if_sendq_empty(ifp)) { | ||||
if (em_xmit(txr, &m_head)) { | if (em_xmit(txr, &m_head)) { | ||||
if (m_head == NULL) | if (m_head == NULL) | ||||
break; | break; | ||||
if_sendq_prepend(ifp, m_head); | if_sendq_prepend(ifp, m_head); | ||||
break; | break; | ||||
} | } | ||||
/* Send a copy of the frame to the BPF listener */ | /* Send a copy of the frame to the BPF listener */ | ||||
if_etherbpfmtap(ifp, m_head); | ETHER_BPF_MTAP(ifp, m_head); | ||||
/* Set timeout in case hardware has problems transmitting. */ | /* Set timeout in case hardware has problems transmitting. */ | ||||
txr->watchdog_time = ticks; | txr->watchdog_time = ticks; | ||||
txr->queue_status = EM_QUEUE_WORKING; | txr->queue_status = EM_QUEUE_WORKING; | ||||
} | } | ||||
return; | return; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 620 Lines • ▼ Show 20 Lines | em_handle_link(void *context, int pending) | ||||
EM_CORE_LOCK(adapter); | EM_CORE_LOCK(adapter); | ||||
callout_stop(&adapter->timer); | callout_stop(&adapter->timer); | ||||
em_update_link_status(adapter); | em_update_link_status(adapter); | ||||
callout_reset(&adapter->timer, hz, em_local_timer, adapter); | callout_reset(&adapter->timer, hz, em_local_timer, adapter); | ||||
E1000_WRITE_REG(&adapter->hw, E1000_IMS, | E1000_WRITE_REG(&adapter->hw, E1000_IMS, | ||||
EM_MSIX_LINK | E1000_IMS_LSC); | EM_MSIX_LINK | E1000_IMS_LSC); | ||||
if (adapter->link_active) { | if (adapter->link_active) { | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | ||||
EM_TX_LOCK(txr); | EM_TX_LOCK(txr); | ||||
#ifdef EM_MULTIQUEUE | #ifdef EM_MULTIQUEUE | ||||
if (!drbr_empty(ifp, txr->br)) | if (!drbr_empty(ifp, txr->br)) | ||||
em_mq_start_locked(ifp, txr, NULL); | em_mq_start_locked(ifp, txr, NULL); | ||||
#else | #else | ||||
if (if_sendq_empty(ifp)) | if (if_sendq_empty(ifp)) | ||||
em_start_locked(ifp, txr); | em_start_locked(ifp, txr); | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 556 Lines • ▼ Show 20 Lines | em_local_timer(void *arg) | ||||
else | else | ||||
trigger = E1000_ICS_RXDMT0; | trigger = E1000_ICS_RXDMT0; | ||||
/* | /* | ||||
** Check on the state of the TX queue(s), this | ** Check on the state of the TX queue(s), this | ||||
** can be done without the lock because its RO | ** can be done without the lock because its RO | ||||
** and the HUNG state will be static if set. | ** and the HUNG state will be static if set. | ||||
*/ | */ | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | ||||
if ((txr->queue_status == EM_QUEUE_HUNG) && | if ((txr->queue_status == EM_QUEUE_HUNG) && | ||||
(adapter->pause_frames == 0)) | (adapter->pause_frames == 0)) | ||||
goto hung; | goto hung; | ||||
/* Schedule a TX tasklet if needed */ | /* Schedule a TX tasklet if needed */ | ||||
if (txr->tx_avail <= EM_MAX_SCATTER) | if (txr->tx_avail <= EM_MAX_SCATTER) | ||||
taskqueue_enqueue(txr->tq, &txr->tx_task); | taskqueue_enqueue(txr->tq, &txr->tx_task); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | em_update_link_status(struct adapter *adapter) | ||||
} else if (!link_check && (adapter->link_active == 1)) { | } else if (!link_check && (adapter->link_active == 1)) { | ||||
if_setbaudrate(ifp, 0); | if_setbaudrate(ifp, 0); | ||||
adapter->link_speed = 0; | adapter->link_speed = 0; | ||||
adapter->link_duplex = 0; | adapter->link_duplex = 0; | ||||
if (bootverbose) | if (bootverbose) | ||||
device_printf(dev, "Link is Down\n"); | device_printf(dev, "Link is Down\n"); | ||||
adapter->link_active = 0; | adapter->link_active = 0; | ||||
/* Link down, disable watchdog */ | /* Link down, disable watchdog */ | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) | ||||
txr->queue_status = EM_QUEUE_IDLE; | txr->queue_status = EM_QUEUE_IDLE; | ||||
if_link_state_change(ifp, LINK_STATE_DOWN); | if_link_state_change(ifp, LINK_STATE_DOWN); | ||||
} | } | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* This routine disables all traffic on the adapter by issuing a | * This routine disables all traffic on the adapter by issuing a | ||||
Show All 16 Lines | em_stop(void *arg) | ||||
em_disable_intr(adapter); | em_disable_intr(adapter); | ||||
callout_stop(&adapter->timer); | callout_stop(&adapter->timer); | ||||
/* Tell the stack that the interface is no longer active */ | /* Tell the stack that the interface is no longer active */ | ||||
if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | ||||
/* Unarm watchdog timer. */ | /* Unarm watchdog timer. */ | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | ||||
EM_TX_LOCK(txr); | EM_TX_LOCK(txr); | ||||
txr->queue_status = EM_QUEUE_IDLE; | txr->queue_status = EM_QUEUE_IDLE; | ||||
EM_TX_UNLOCK(txr); | EM_TX_UNLOCK(txr); | ||||
} | } | ||||
e1000_reset_hw(&adapter->hw); | e1000_reset_hw(&adapter->hw); | ||||
E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); | E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | if (adapter->memory == NULL) { | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
adapter->osdep.mem_bus_space_tag = | adapter->osdep.mem_bus_space_tag = | ||||
rman_get_bustag(adapter->memory); | rman_get_bustag(adapter->memory); | ||||
adapter->osdep.mem_bus_space_handle = | adapter->osdep.mem_bus_space_handle = | ||||
rman_get_bushandle(adapter->memory); | rman_get_bushandle(adapter->memory); | ||||
adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; | adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; | ||||
/* Default to a single queue */ | |||||
adapter->num_queues = 1; | |||||
/* | |||||
* Setup MSI/X or MSI if PCI Express | |||||
*/ | |||||
adapter->msix = em_setup_msix(adapter); | |||||
adapter->hw.back = &adapter->osdep; | adapter->hw.back = &adapter->osdep; | ||||
return (0); | return (0); | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Setup the Legacy or MSI Interrupt handler | * Setup the Legacy or MSI Interrupt handler | ||||
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | em_allocate_msix(struct adapter *adapter) | ||||
struct rx_ring *rxr = adapter->rx_rings; | struct rx_ring *rxr = adapter->rx_rings; | ||||
int error, rid, vector = 0; | int error, rid, vector = 0; | ||||
/* Make sure all interrupts are disabled */ | /* Make sure all interrupts are disabled */ | ||||
E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); | E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); | ||||
/* First set up ring resources */ | /* First set up ring resources */ | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { | for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { | ||||
/* RX ring */ | /* RX ring */ | ||||
rid = vector + 1; | rid = vector + 1; | ||||
rxr->res = bus_alloc_resource_any(dev, | rxr->res = bus_alloc_resource_any(dev, | ||||
SYS_RES_IRQ, &rid, RF_ACTIVE); | SYS_RES_IRQ, &rid, RF_ACTIVE); | ||||
if (rxr->res == NULL) { | if (rxr->res == NULL) { | ||||
device_printf(dev, | device_printf(dev, | ||||
Show All 9 Lines | for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { | ||||
} | } | ||||
#if __FreeBSD_version >= 800504 | #if __FreeBSD_version >= 800504 | ||||
bus_describe_intr(dev, rxr->res, rxr->tag, "rx %d", i); | bus_describe_intr(dev, rxr->res, rxr->tag, "rx %d", i); | ||||
#endif | #endif | ||||
rxr->msix = vector++; /* NOTE increment vector for TX */ | rxr->msix = vector++; /* NOTE increment vector for TX */ | ||||
TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); | TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); | ||||
rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, | rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, | ||||
taskqueue_thread_enqueue, &rxr->tq); | taskqueue_thread_enqueue, &rxr->tq); | ||||
taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", | taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq (qid %d)", | ||||
device_get_nameunit(adapter->dev)); | device_get_nameunit(adapter->dev), i); | ||||
/* | /* | ||||
** Set the bit to enable interrupt | ** Set the bit to enable interrupt | ||||
** in E1000_IMS -- bits 20 and 21 | ** in E1000_IMS -- bits 20 and 21 | ||||
** are for RX0 and RX1, note this has | ** are for RX0 and RX1, note this has | ||||
** NOTHING to do with the MSIX vector | ** NOTHING to do with the MSIX vector | ||||
*/ | */ | ||||
rxr->ims = 1 << (20 + i); | rxr->ims = 1 << (20 + i); | ||||
adapter->ivars |= (8 | rxr->msix) << (i * 4); | adapter->ivars |= (8 | rxr->msix) << (i * 4); | ||||
} | |||||
for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | |||||
/* TX ring */ | /* TX ring */ | ||||
rid = vector + 1; | rid = vector + 1; | ||||
txr->res = bus_alloc_resource_any(dev, | txr->res = bus_alloc_resource_any(dev, | ||||
SYS_RES_IRQ, &rid, RF_ACTIVE); | SYS_RES_IRQ, &rid, RF_ACTIVE); | ||||
if (txr->res == NULL) { | if (txr->res == NULL) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"Unable to allocate bus resource: " | "Unable to allocate bus resource: " | ||||
"TX MSIX Interrupt %d\n", i); | "TX MSIX Interrupt %d\n", i); | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
if ((error = bus_setup_intr(dev, txr->res, | if ((error = bus_setup_intr(dev, txr->res, | ||||
INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, | INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, | ||||
txr, &txr->tag)) != 0) { | txr, &txr->tag)) != 0) { | ||||
device_printf(dev, "Failed to register TX handler"); | device_printf(dev, "Failed to register TX handler"); | ||||
return (error); | return (error); | ||||
} | } | ||||
#if __FreeBSD_version >= 800504 | #if __FreeBSD_version >= 800504 | ||||
bus_describe_intr(dev, txr->res, txr->tag, "tx %d", i); | bus_describe_intr(dev, txr->res, txr->tag, "tx %d", i); | ||||
#endif | #endif | ||||
txr->msix = vector++; /* Increment vector for next pass */ | txr->msix = vector++; /* Increment vector for next pass */ | ||||
TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); | TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); | ||||
txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, | txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, | ||||
taskqueue_thread_enqueue, &txr->tq); | taskqueue_thread_enqueue, &txr->tq); | ||||
taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", | taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq (qid %d)", | ||||
device_get_nameunit(adapter->dev)); | device_get_nameunit(adapter->dev), i); | ||||
/* | /* | ||||
** Set the bit to enable interrupt | ** Set the bit to enable interrupt | ||||
** in E1000_IMS -- bits 22 and 23 | ** in E1000_IMS -- bits 22 and 23 | ||||
** are for TX0 and TX1, note this has | ** are for TX0 and TX1, note this has | ||||
** NOTHING to do with the MSIX vector | ** NOTHING to do with the MSIX vector | ||||
*/ | */ | ||||
txr->ims = 1 << (22 + i); | txr->ims = 1 << (22 + i); | ||||
adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); | adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); | ||||
Show All 35 Lines | em_free_pci_resources(struct adapter *adapter) | ||||
struct tx_ring *txr; | struct tx_ring *txr; | ||||
struct rx_ring *rxr; | struct rx_ring *rxr; | ||||
int rid; | int rid; | ||||
/* | /* | ||||
** Release all the queue interrupt resources: | ** Release all the queue interrupt resources: | ||||
*/ | */ | ||||
for (int i = 0; i < adapter->num_queues; i++) { | for (int i = 0; i < adapter->num_tx_queues; i++) { | ||||
txr = &adapter->tx_rings[i]; | txr = &adapter->tx_rings[i]; | ||||
rxr = &adapter->rx_rings[i]; | |||||
/* an early abort? */ | /* an early abort? */ | ||||
if ((txr == NULL) || (rxr == NULL)) | if (txr == NULL) | ||||
break; | break; | ||||
rid = txr->msix +1; | rid = txr->msix +1; | ||||
if (txr->tag != NULL) { | if (txr->tag != NULL) { | ||||
bus_teardown_intr(dev, txr->res, txr->tag); | bus_teardown_intr(dev, txr->res, txr->tag); | ||||
txr->tag = NULL; | txr->tag = NULL; | ||||
} | } | ||||
if (txr->res != NULL) | if (txr->res != NULL) | ||||
bus_release_resource(dev, SYS_RES_IRQ, | bus_release_resource(dev, SYS_RES_IRQ, | ||||
rid, txr->res); | rid, txr->res); | ||||
} | |||||
for (int i = 0; i < adapter->num_tx_queues; i++) { | |||||
rxr = &adapter->rx_rings[i]; | |||||
/* an early abort? */ | |||||
if (rxr == NULL) | |||||
break; | |||||
rid = rxr->msix +1; | rid = rxr->msix +1; | ||||
if (rxr->tag != NULL) { | if (rxr->tag != NULL) { | ||||
bus_teardown_intr(dev, rxr->res, rxr->tag); | bus_teardown_intr(dev, rxr->res, rxr->tag); | ||||
rxr->tag = NULL; | rxr->tag = NULL; | ||||
} | } | ||||
if (rxr->res != NULL) | if (rxr->res != NULL) | ||||
bus_release_resource(dev, SYS_RES_IRQ, | bus_release_resource(dev, SYS_RES_IRQ, | ||||
rid, rxr->res); | rid, rxr->res); | ||||
Show All 33 Lines | |||||
* Setup MSI or MSI/X | * Setup MSI or MSI/X | ||||
*/ | */ | ||||
static int | static int | ||||
em_setup_msix(struct adapter *adapter) | em_setup_msix(struct adapter *adapter) | ||||
{ | { | ||||
device_t dev = adapter->dev; | device_t dev = adapter->dev; | ||||
int val; | int val; | ||||
/* Nearly always going to use one queue */ | |||||
adapter->num_rx_queues = 1; | |||||
adapter->num_tx_queues = 1; | |||||
/* | /* | ||||
** Setup MSI/X for Hartwell: tests have shown | ** Try using MSI-X for Hartwell adapters | ||||
** use of two queues to be unstable, and to | |||||
** provide no great gain anyway, so we simply | |||||
** seperate the interrupts and use a single queue. | |||||
*/ | */ | ||||
if ((adapter->hw.mac.type == e1000_82574) && | if ((adapter->hw.mac.type == e1000_82574) && | ||||
(em_enable_msix == TRUE)) { | (em_enable_msix == TRUE)) { | ||||
#ifdef EM_MULTIQUEUE | |||||
adapter->num_tx_queues = (em_num_tx_queues == 1) ? 1 : 2; | |||||
adapter->num_rx_queues = (em_num_rx_queues == 1) ? 1 : 2; | |||||
if (adapter->num_rx_queues > 1 || adapter->num_tx_queues > 1) | |||||
em_enable_vectors_82574(adapter); | |||||
#endif | |||||
/* Map the MSIX BAR */ | /* Map the MSIX BAR */ | ||||
int rid = PCIR_BAR(EM_MSIX_BAR); | int rid = PCIR_BAR(EM_MSIX_BAR); | ||||
adapter->msix_mem = bus_alloc_resource_any(dev, | adapter->msix_mem = bus_alloc_resource_any(dev, | ||||
SYS_RES_MEMORY, &rid, RF_ACTIVE); | SYS_RES_MEMORY, &rid, RF_ACTIVE); | ||||
if (adapter->msix_mem == NULL) { | if (adapter->msix_mem == NULL) { | ||||
/* May not be enabled */ | /* May not be enabled */ | ||||
device_printf(adapter->dev, | device_printf(adapter->dev, | ||||
"Unable to map MSIX table \n"); | "Unable to map MSIX table \n"); | ||||
goto msi; | goto msi; | ||||
} | } | ||||
val = pci_msix_count(dev); | val = pci_msix_count(dev); | ||||
/* We only need/want 3 vectors */ | |||||
#ifdef EM_MULTIQUEUE | |||||
/* We need 5 vectors in the multiqueue case */ | |||||
if (adapter->num_rx_queues > 1 || adapter->num_tx_queues > 1) { | |||||
if (val >= 5) | |||||
val = 5; | |||||
else { | |||||
adapter->num_tx_queues = 1; | |||||
adapter->num_rx_queues = 1; | |||||
device_printf(adapter->dev, | |||||
"Insufficient MSIX vectors for >1 queue, " | |||||
"using single queue...\n"); | |||||
goto msix_one; | |||||
} | |||||
} else { | |||||
msix_one: | |||||
#endif | |||||
if (val >= 3) | if (val >= 3) | ||||
val = 3; | val = 3; | ||||
else { | else { | ||||
device_printf(adapter->dev, | device_printf(adapter->dev, | ||||
"MSIX: insufficient vectors, using MSI\n"); | "Insufficient MSIX vectors, using MSI\n"); | ||||
goto msi; | goto msi; | ||||
} | } | ||||
#ifdef EM_MULTIQUEUE | |||||
} | |||||
#endif | |||||
if ((pci_alloc_msix(dev, &val) == 0) && (val == 3)) { | if ((pci_alloc_msix(dev, &val) == 0)) { | ||||
device_printf(adapter->dev, | device_printf(adapter->dev, | ||||
"Using MSIX interrupts " | "Using MSIX interrupts " | ||||
"with %d vectors\n", val); | "with %d vectors\n", val); | ||||
return (val); | return (val); | ||||
} | } | ||||
/* | /* | ||||
** If MSIX alloc failed or provided us with | ** If MSIX alloc failed or provided us with | ||||
** less than needed, free and fall through to MSI | ** less than needed, free and fall through to MSI | ||||
*/ | */ | ||||
pci_release_msi(dev); | pci_release_msi(dev); | ||||
} | } | ||||
msi: | msi: | ||||
if (adapter->msix_mem != NULL) { | if (adapter->msix_mem != NULL) { | ||||
bus_release_resource(dev, SYS_RES_MEMORY, | bus_release_resource(dev, SYS_RES_MEMORY, | ||||
PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); | PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); | ||||
adapter->msix_mem = NULL; | adapter->msix_mem = NULL; | ||||
} | } | ||||
val = 1; | val = 1; | ||||
if (pci_alloc_msi(dev, &val) == 0) { | if (pci_alloc_msi(dev, &val) == 0) { | ||||
device_printf(adapter->dev,"Using an MSI interrupt\n"); | device_printf(adapter->dev, "Using an MSI interrupt\n"); | ||||
return (val); | return (val); | ||||
} | } | ||||
/* Should only happen due to manual configuration */ | /* Should only happen due to manual configuration */ | ||||
device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n"); | device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n"); | ||||
return (0); | return (0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 363 Lines • ▼ Show 20 Lines | em_allocate_queues(struct adapter *adapter) | ||||
struct rx_ring *rxr = NULL; | struct rx_ring *rxr = NULL; | ||||
int rsize, tsize, error = E1000_SUCCESS; | int rsize, tsize, error = E1000_SUCCESS; | ||||
int txconf = 0, rxconf = 0; | int txconf = 0, rxconf = 0; | ||||
/* Allocate the TX ring struct memory */ | /* Allocate the TX ring struct memory */ | ||||
if (!(adapter->tx_rings = | if (!(adapter->tx_rings = | ||||
(struct tx_ring *) malloc(sizeof(struct tx_ring) * | (struct tx_ring *) malloc(sizeof(struct tx_ring) * | ||||
adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { | adapter->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate TX ring memory\n"); | device_printf(dev, "Unable to allocate TX ring memory\n"); | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Now allocate the RX */ | /* Now allocate the RX */ | ||||
if (!(adapter->rx_rings = | if (!(adapter->rx_rings = | ||||
(struct rx_ring *) malloc(sizeof(struct rx_ring) * | (struct rx_ring *) malloc(sizeof(struct rx_ring) * | ||||
adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { | adapter->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate RX ring memory\n"); | device_printf(dev, "Unable to allocate RX ring memory\n"); | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto rx_fail; | goto rx_fail; | ||||
} | } | ||||
tsize = roundup2(adapter->num_tx_desc * | tsize = roundup2(adapter->num_tx_desc * | ||||
sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); | sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); | ||||
/* | /* | ||||
* Now set up the TX queues, txconf is needed to handle the | * Now set up the TX queues, txconf is needed to handle the | ||||
* possibility that things fail midcourse and we need to | * possibility that things fail midcourse and we need to | ||||
* undo memory gracefully | * undo memory gracefully | ||||
*/ | */ | ||||
for (int i = 0; i < adapter->num_queues; i++, txconf++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txconf++) { | ||||
/* Set up some basics */ | /* Set up some basics */ | ||||
txr = &adapter->tx_rings[i]; | txr = &adapter->tx_rings[i]; | ||||
txr->adapter = adapter; | txr->adapter = adapter; | ||||
txr->me = i; | txr->me = i; | ||||
/* Initialize the TX lock */ | /* Initialize the TX lock */ | ||||
snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", | snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", | ||||
device_get_nameunit(dev), txr->me); | device_get_nameunit(dev), txr->me); | ||||
Show All 22 Lines | |||||
#endif | #endif | ||||
} | } | ||||
/* | /* | ||||
* Next the RX queues... | * Next the RX queues... | ||||
*/ | */ | ||||
rsize = roundup2(adapter->num_rx_desc * | rsize = roundup2(adapter->num_rx_desc * | ||||
sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); | sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); | ||||
for (int i = 0; i < adapter->num_queues; i++, rxconf++) { | for (int i = 0; i < adapter->num_rx_queues; i++, rxconf++) { | ||||
rxr = &adapter->rx_rings[i]; | rxr = &adapter->rx_rings[i]; | ||||
rxr->adapter = adapter; | rxr->adapter = adapter; | ||||
rxr->me = i; | rxr->me = i; | ||||
/* Initialize the RX lock */ | /* Initialize the RX lock */ | ||||
snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", | snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", | ||||
device_get_nameunit(dev), txr->me); | device_get_nameunit(dev), txr->me); | ||||
mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); | mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); | ||||
▲ Show 20 Lines • Show All 171 Lines • ▼ Show 20 Lines | |||||
* Initialize all transmit rings. | * Initialize all transmit rings. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_setup_transmit_structures(struct adapter *adapter) | em_setup_transmit_structures(struct adapter *adapter) | ||||
{ | { | ||||
struct tx_ring *txr = adapter->tx_rings; | struct tx_ring *txr = adapter->tx_rings; | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) | ||||
em_setup_transmit_ring(txr); | em_setup_transmit_ring(txr); | ||||
return; | return; | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Enable transmit unit. | * Enable transmit unit. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_initialize_transmit_unit(struct adapter *adapter) | em_initialize_transmit_unit(struct adapter *adapter) | ||||
{ | { | ||||
struct tx_ring *txr = adapter->tx_rings; | struct tx_ring *txr = adapter->tx_rings; | ||||
struct e1000_hw *hw = &adapter->hw; | struct e1000_hw *hw = &adapter->hw; | ||||
u32 tctl, tarc, tipg = 0; | u32 tctl, tarc, tipg = 0; | ||||
INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); | INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | ||||
u64 bus_addr = txr->txdma.dma_paddr; | u64 bus_addr = txr->txdma.dma_paddr; | ||||
/* Base and Len of TX Ring */ | /* Base and Len of TX Ring */ | ||||
E1000_WRITE_REG(hw, E1000_TDLEN(i), | E1000_WRITE_REG(hw, E1000_TDLEN(i), | ||||
adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); | adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); | ||||
E1000_WRITE_REG(hw, E1000_TDBAH(i), | E1000_WRITE_REG(hw, E1000_TDBAH(i), | ||||
(u32)(bus_addr >> 32)); | (u32)(bus_addr >> 32)); | ||||
E1000_WRITE_REG(hw, E1000_TDBAL(i), | E1000_WRITE_REG(hw, E1000_TDBAL(i), | ||||
(u32)bus_addr); | (u32)bus_addr); | ||||
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | |||||
* Free all transmit rings. | * Free all transmit rings. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_free_transmit_structures(struct adapter *adapter) | em_free_transmit_structures(struct adapter *adapter) | ||||
{ | { | ||||
struct tx_ring *txr = adapter->tx_rings; | struct tx_ring *txr = adapter->tx_rings; | ||||
for (int i = 0; i < adapter->num_queues; i++, txr++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | ||||
EM_TX_LOCK(txr); | EM_TX_LOCK(txr); | ||||
em_free_transmit_buffers(txr); | em_free_transmit_buffers(txr); | ||||
em_dma_free(adapter, &txr->txdma); | em_dma_free(adapter, &txr->txdma); | ||||
EM_TX_UNLOCK(txr); | EM_TX_UNLOCK(txr); | ||||
EM_TX_LOCK_DESTROY(txr); | EM_TX_LOCK_DESTROY(txr); | ||||
} | } | ||||
free(adapter->tx_rings, M_DEVBUF); | free(adapter->tx_rings, M_DEVBUF); | ||||
▲ Show 20 Lines • Show All 633 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
em_setup_receive_structures(struct adapter *adapter) | em_setup_receive_structures(struct adapter *adapter) | ||||
{ | { | ||||
struct rx_ring *rxr = adapter->rx_rings; | struct rx_ring *rxr = adapter->rx_rings; | ||||
int q; | int q; | ||||
for (q = 0; q < adapter->num_queues; q++, rxr++) | for (q = 0; q < adapter->num_rx_queues; q++, rxr++) | ||||
if (em_setup_receive_ring(rxr)) | if (em_setup_receive_ring(rxr)) | ||||
goto fail; | goto fail; | ||||
return (0); | return (0); | ||||
fail: | fail: | ||||
/* | /* | ||||
* Free RX buffers allocated so far, we will only handle | * Free RX buffers allocated so far, we will only handle | ||||
* the rings that completed, the failing case will have | * the rings that completed, the failing case will have | ||||
Show All 24 Lines | |||||
* Free all receive rings. | * Free all receive rings. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
em_free_receive_structures(struct adapter *adapter) | em_free_receive_structures(struct adapter *adapter) | ||||
{ | { | ||||
struct rx_ring *rxr = adapter->rx_rings; | struct rx_ring *rxr = adapter->rx_rings; | ||||
for (int i = 0; i < adapter->num_queues; i++, rxr++) { | for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { | ||||
em_free_receive_buffers(rxr); | em_free_receive_buffers(rxr); | ||||
/* Free the ring memory as well */ | /* Free the ring memory as well */ | ||||
em_dma_free(adapter, &rxr->rxdma); | em_dma_free(adapter, &rxr->rxdma); | ||||
EM_RX_LOCK_DESTROY(rxr); | EM_RX_LOCK_DESTROY(rxr); | ||||
} | } | ||||
free(adapter->rx_rings, M_DEVBUF); | free(adapter->rx_rings, M_DEVBUF); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | if (hw->mac.type == e1000_82574) { | ||||
for (int i = 0; i < 4; i++) | for (int i = 0; i < 4; i++) | ||||
E1000_WRITE_REG(hw, E1000_EITR_82574(i), | E1000_WRITE_REG(hw, E1000_EITR_82574(i), | ||||
DEFAULT_ITR); | DEFAULT_ITR); | ||||
/* Disable accelerated acknowledge */ | /* Disable accelerated acknowledge */ | ||||
E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS); | E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS); | ||||
} | } | ||||
rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); | rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); | ||||
if (if_getcapenable(ifp) & IFCAP_RXCSUM) | if (if_getcapenable(ifp) & IFCAP_RXCSUM) { | ||||
#ifdef EM_MULTIQUEUE | |||||
rxcsum |= E1000_RXCSUM_TUOFL | | |||||
E1000_RXCSUM_IPOFL | | |||||
E1000_RXCSUM_PCSD; | |||||
#else | |||||
rxcsum |= E1000_RXCSUM_TUOFL; | rxcsum |= E1000_RXCSUM_TUOFL; | ||||
else | #endif | ||||
} else | |||||
rxcsum &= ~E1000_RXCSUM_TUOFL; | rxcsum &= ~E1000_RXCSUM_TUOFL; | ||||
E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); | E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); | ||||
#ifdef EM_MULTIQUEUE | |||||
if (adapter->num_rx_queues > 1) { | |||||
uint32_t rss_key[10]; | |||||
uint32_t reta; | |||||
int i; | |||||
/* | /* | ||||
* Configure RSS key | |||||
*/ | |||||
arc4rand(rss_key, sizeof(rss_key), 0); | |||||
for (i = 0; i < 10; ++i) | |||||
E1000_WRITE_REG_ARRAY(hw,E1000_RSSRK(0), i, rss_key[i]); | |||||
/* | |||||
* Configure RSS redirect table in following fashion: | |||||
* (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] | |||||
*/ | |||||
reta = 0; | |||||
for (i = 0; i < 4; ++i) { | |||||
uint32_t q; | |||||
q = (i % 2) << 7; | |||||
reta |= q << (8 * i); | |||||
} | |||||
for (i = 0; i < 32; ++i) | |||||
E1000_WRITE_REG(hw, E1000_RETA(i), reta); | |||||
E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | | |||||
E1000_MRQC_RSS_FIELD_IPV4_TCP | | |||||
E1000_MRQC_RSS_FIELD_IPV4 | | |||||
E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | | |||||
E1000_MRQC_RSS_FIELD_IPV6_EX | | |||||
E1000_MRQC_RSS_FIELD_IPV6 | | |||||
E1000_MRQC_RSS_FIELD_IPV6_TCP); | |||||
} | |||||
#endif | |||||
/* | |||||
** XXX TEMPORARY WORKAROUND: on some systems with 82573 | ** XXX TEMPORARY WORKAROUND: on some systems with 82573 | ||||
** long latencies are observed, like Lenovo X60. This | ** long latencies are observed, like Lenovo X60. This | ||||
** change eliminates the problem, but since having positive | ** change eliminates the problem, but since having positive | ||||
** values in RDTR is a known source of problems on other | ** values in RDTR is a known source of problems on other | ||||
** platforms another solution is being sought. | ** platforms another solution is being sought. | ||||
*/ | */ | ||||
if (hw->mac.type == e1000_82573) | if (hw->mac.type == e1000_82573) | ||||
E1000_WRITE_REG(hw, E1000_RDTR, 0x20); | E1000_WRITE_REG(hw, E1000_RDTR, 0x20); | ||||
for (int i = 0; i < adapter->num_queues; i++, rxr++) { | for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { | ||||
/* Setup the Base and Length of the Rx Descriptor Ring */ | /* Setup the Base and Length of the Rx Descriptor Ring */ | ||||
u32 rdt = adapter->num_rx_desc - 1; /* default */ | u32 rdt = adapter->num_rx_desc - 1; /* default */ | ||||
bus_addr = rxr->rxdma.dma_paddr; | bus_addr = rxr->rxdma.dma_paddr; | ||||
E1000_WRITE_REG(hw, E1000_RDLEN(i), | E1000_WRITE_REG(hw, E1000_RDLEN(i), | ||||
adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); | adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); | ||||
E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); | E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); | ||||
E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); | E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); | ||||
/* Setup the Head and Tail Descriptor Pointers */ | /* Setup the Head and Tail Descriptor Pointers */ | ||||
E1000_WRITE_REG(hw, E1000_RDH(i), 0); | E1000_WRITE_REG(hw, E1000_RDH(i), 0); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
/* | /* | ||||
* an init() while a netmap client is active must | * an init() while a netmap client is active must | ||||
* preserve the rx buffers passed to userspace. | * preserve the rx buffers passed to userspace. | ||||
*/ | */ | ||||
if (if_getcapenable(ifp) & IFCAP_NETMAP) { | if (if_getcapenable(ifp) & IFCAP_NETMAP) { | ||||
struct netmap_adapter *na = netmap_getna(adapter->ifp); | struct netmap_adapter *na = netmap_getna(adapter->ifp); | ||||
rdt -= nm_kr_rxspace(&na->rx_rings[i]); | rdt -= nm_kr_rxspace(&na->rx_rings[i]); | ||||
} | } | ||||
#endif /* DEV_NETMAP */ | #endif /* DEV_NETMAP */ | ||||
E1000_WRITE_REG(hw, E1000_RDT(i), rdt); | E1000_WRITE_REG(hw, E1000_RDT(i), rdt); | ||||
} | } | ||||
/* Set PTHRESH for improved jumbo performance */ | /* Set PTHRESH for improved jumbo performance */ | ||||
if (((adapter->hw.mac.type == e1000_ich9lan) || | if (((adapter->hw.mac.type == e1000_ich9lan) || | ||||
(adapter->hw.mac.type == e1000_pch2lan) || | (adapter->hw.mac.type == e1000_pch2lan) || | ||||
(adapter->hw.mac.type == e1000_ich10lan)) && | (adapter->hw.mac.type == e1000_ich10lan)) && | ||||
(if_getmtu(ifp) > ETHERMTU)) { | (if_getmtu(ifp) > ETHERMTU)) { | ||||
u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); | u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); | ||||
E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); | E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 963 Lines • ▼ Show 20 Lines | SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", | ||||
"Receiver Control Register"); | "Receiver Control Register"); | ||||
SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", | SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", | ||||
CTLFLAG_RD, &adapter->hw.fc.high_water, 0, | CTLFLAG_RD, &adapter->hw.fc.high_water, 0, | ||||
"Flow Control High Watermark"); | "Flow Control High Watermark"); | ||||
SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", | SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", | ||||
CTLFLAG_RD, &adapter->hw.fc.low_water, 0, | CTLFLAG_RD, &adapter->hw.fc.low_water, 0, | ||||
"Flow Control Low Watermark"); | "Flow Control Low Watermark"); | ||||
for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { | for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { | ||||
snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); | snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); | ||||
queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, | queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, | ||||
CTLFLAG_RD, NULL, "Queue Name"); | CTLFLAG_RD, NULL, "TX Queue Name"); | ||||
queue_list = SYSCTL_CHILDREN(queue_node); | queue_list = SYSCTL_CHILDREN(queue_node); | ||||
SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", | SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", | ||||
CTLTYPE_UINT | CTLFLAG_RD, adapter, | CTLTYPE_UINT | CTLFLAG_RD, adapter, | ||||
E1000_TDH(txr->me), | E1000_TDH(txr->me), | ||||
em_sysctl_reg_handler, "IU", | em_sysctl_reg_handler, "IU", | ||||
"Transmit Descriptor Head"); | "Transmit Descriptor Head"); | ||||
SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", | SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", | ||||
CTLTYPE_UINT | CTLFLAG_RD, adapter, | CTLTYPE_UINT | CTLFLAG_RD, adapter, | ||||
E1000_TDT(txr->me), | E1000_TDT(txr->me), | ||||
em_sysctl_reg_handler, "IU", | em_sysctl_reg_handler, "IU", | ||||
"Transmit Descriptor Tail"); | "Transmit Descriptor Tail"); | ||||
SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", | SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", | ||||
CTLFLAG_RD, &txr->tx_irq, | CTLFLAG_RD, &txr->tx_irq, | ||||
"Queue MSI-X Transmit Interrupts"); | "Queue MSI-X Transmit Interrupts"); | ||||
SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", | SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", | ||||
CTLFLAG_RD, &txr->no_desc_avail, | CTLFLAG_RD, &txr->no_desc_avail, | ||||
"Queue No Descriptor Available"); | "Queue No Descriptor Available"); | ||||
} | |||||
for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { | |||||
snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", i); | |||||
queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, | |||||
CTLFLAG_RD, NULL, "RX Queue Name"); | |||||
queue_list = SYSCTL_CHILDREN(queue_node); | |||||
SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", | SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", | ||||
CTLTYPE_UINT | CTLFLAG_RD, adapter, | CTLTYPE_UINT | CTLFLAG_RD, adapter, | ||||
E1000_RDH(rxr->me), | E1000_RDH(rxr->me), | ||||
em_sysctl_reg_handler, "IU", | em_sysctl_reg_handler, "IU", | ||||
"Receive Descriptor Head"); | "Receive Descriptor Head"); | ||||
SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", | SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", | ||||
CTLTYPE_UINT | CTLFLAG_RD, adapter, | CTLTYPE_UINT | CTLFLAG_RD, adapter, | ||||
E1000_RDT(rxr->me), | E1000_RDT(rxr->me), | ||||
▲ Show 20 Lines • Show All 441 Lines • Show Last 20 Lines |