Changeset View
Changeset View
Standalone View
Standalone View
head/sys/dev/ena/ena.c
Show First 20 Lines • Show All 135 Lines • ▼ Show 20 Lines | |||||
static int ena_request_mgmnt_irq(struct ena_adapter *); | static int ena_request_mgmnt_irq(struct ena_adapter *); | ||||
static int ena_request_io_irq(struct ena_adapter *); | static int ena_request_io_irq(struct ena_adapter *); | ||||
static void ena_free_mgmnt_irq(struct ena_adapter *); | static void ena_free_mgmnt_irq(struct ena_adapter *); | ||||
static void ena_free_io_irq(struct ena_adapter *); | static void ena_free_io_irq(struct ena_adapter *); | ||||
static void ena_free_irqs(struct ena_adapter*); | static void ena_free_irqs(struct ena_adapter*); | ||||
static void ena_disable_msix(struct ena_adapter *); | static void ena_disable_msix(struct ena_adapter *); | ||||
static void ena_unmask_all_io_irqs(struct ena_adapter *); | static void ena_unmask_all_io_irqs(struct ena_adapter *); | ||||
static int ena_rss_configure(struct ena_adapter *); | static int ena_rss_configure(struct ena_adapter *); | ||||
static void ena_update_hw_stats(void *, int); | |||||
static int ena_up_complete(struct ena_adapter *); | static int ena_up_complete(struct ena_adapter *); | ||||
static int ena_up(struct ena_adapter *); | static int ena_up(struct ena_adapter *); | ||||
static void ena_down(struct ena_adapter *); | static void ena_down(struct ena_adapter *); | ||||
static uint64_t ena_get_counter(if_t, ift_counter); | static uint64_t ena_get_counter(if_t, ift_counter); | ||||
static int ena_media_change(if_t); | static int ena_media_change(if_t); | ||||
static void ena_media_status(if_t, struct ifmediareq *); | static void ena_media_status(if_t, struct ifmediareq *); | ||||
static void ena_init(void *); | static void ena_init(void *); | ||||
static int ena_ioctl(if_t, u_long, caddr_t); | static int ena_ioctl(if_t, u_long, caddr_t); | ||||
▲ Show 20 Lines • Show All 1,424 Lines • ▼ Show 20 Lines | do { | ||||
ena_trace(ENA_DBG | ENA_RXPTH, "Rx: %d bytes", | ena_trace(ENA_DBG | ENA_RXPTH, "Rx: %d bytes", | ||||
mbuf->m_pkthdr.len); | mbuf->m_pkthdr.len); | ||||
if ((ifp->if_capenable & IFCAP_RXCSUM) || | if ((ifp->if_capenable & IFCAP_RXCSUM) || | ||||
(ifp->if_capenable & IFCAP_RXCSUM_IPV6)) { | (ifp->if_capenable & IFCAP_RXCSUM_IPV6)) { | ||||
ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf); | ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf); | ||||
} | } | ||||
counter_u64_add(rx_ring->rx_stats.bytes, mbuf->m_pkthdr.len); | counter_enter(); | ||||
counter_u64_add_protected(rx_ring->rx_stats.bytes, | |||||
mbuf->m_pkthdr.len); | |||||
counter_u64_add_protected(adapter->hw_stats.rx_bytes, | |||||
mbuf->m_pkthdr.len); | |||||
counter_exit(); | |||||
/* | /* | ||||
* LRO is only for IP/TCP packets and TCP checksum of the packet | * LRO is only for IP/TCP packets and TCP checksum of the packet | ||||
* should be computed by hardware. | * should be computed by hardware. | ||||
*/ | */ | ||||
do_if_input = 1; | do_if_input = 1; | ||||
if ((ifp->if_capenable & IFCAP_LRO) && | if ((ifp->if_capenable & IFCAP_LRO) && | ||||
(mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) && | (mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) && | ||||
ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP) { | ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP) { | ||||
/* | /* | ||||
* Send to the stack if: | * Send to the stack if: | ||||
* - LRO not enabled, or | * - LRO not enabled, or | ||||
* - no LRO resources, or | * - no LRO resources, or | ||||
* - lro enqueue fails | * - lro enqueue fails | ||||
*/ | */ | ||||
if (rx_ring->lro.lro_cnt != 0 && | if (rx_ring->lro.lro_cnt != 0 && | ||||
tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0) | tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0) | ||||
do_if_input = 0; | do_if_input = 0; | ||||
} | } | ||||
if (do_if_input) { | if (do_if_input) { | ||||
ena_trace(ENA_DBG | ENA_RXPTH, "calling if_input() with mbuf %p", | ena_trace(ENA_DBG | ENA_RXPTH, "calling if_input() with mbuf %p", | ||||
mbuf); | mbuf); | ||||
(*ifp->if_input)(ifp, mbuf); | (*ifp->if_input)(ifp, mbuf); | ||||
} | } | ||||
counter_u64_add(rx_ring->rx_stats.cnt, 1); | counter_enter(); | ||||
counter_u64_add_protected(rx_ring->rx_stats.cnt, 1); | |||||
counter_u64_add_protected(adapter->hw_stats.rx_packets, 1); | |||||
counter_exit(); | |||||
} while (--budget); | } while (--budget); | ||||
rx_ring->next_to_clean = next_to_clean; | rx_ring->next_to_clean = next_to_clean; | ||||
refill_required = ena_com_free_desc(io_sq); | refill_required = ena_com_free_desc(io_sq); | ||||
refill_threshold = rx_ring->ring_size / ENA_RX_REFILL_THRESH_DEVIDER; | refill_threshold = rx_ring->ring_size / ENA_RX_REFILL_THRESH_DEVIDER; | ||||
if (refill_required > refill_threshold) { | if (refill_required > refill_threshold) { | ||||
▲ Show 20 Lines • Show All 439 Lines • ▼ Show 20 Lines | static int ena_rss_configure(struct ena_adapter *adapter) | ||||
/* Configure hash inputs (if supported) */ | /* Configure hash inputs (if supported) */ | ||||
rc = ena_com_set_hash_ctrl(ena_dev); | rc = ena_com_set_hash_ctrl(ena_dev); | ||||
if (unlikely(rc && (rc != EOPNOTSUPP))) | if (unlikely(rc && (rc != EOPNOTSUPP))) | ||||
return rc; | return rc; | ||||
return 0; | return 0; | ||||
} | } | ||||
static void | |||||
ena_update_hw_stats(void *arg, int pending) | |||||
{ | |||||
struct ena_adapter *adapter = arg; | |||||
int rc; | |||||
for (;;) { | |||||
if (!adapter->up) | |||||
return; | |||||
rc = ena_update_stats_counters(adapter); | |||||
if (rc) | |||||
ena_trace(ENA_WARNING, | |||||
"Error updating stats counters, rc = %d", rc); | |||||
pause("ena update hw stats", hz); | |||||
} | |||||
} | |||||
static int | static int | ||||
ena_up_complete(struct ena_adapter *adapter) | ena_up_complete(struct ena_adapter *adapter) | ||||
{ | { | ||||
int rc; | int rc; | ||||
if (adapter->rss_support) { | if (adapter->rss_support) { | ||||
rc = ena_rss_configure(adapter); | rc = ena_rss_configure(adapter); | ||||
if (rc) | if (rc) | ||||
return (rc); | return (rc); | ||||
} | } | ||||
ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu); | ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu); | ||||
ena_refill_all_rx_bufs(adapter); | ena_refill_all_rx_bufs(adapter); | ||||
ena_reset_counters((counter_u64_t *)&adapter->hw_stats, | |||||
sizeof(adapter->hw_stats)); | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
ena_up(struct ena_adapter *adapter) | ena_up(struct ena_adapter *adapter) | ||||
{ | { | ||||
int rc = 0; | int rc = 0; | ||||
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | if (!adapter->up) { | ||||
ena_update_hwassist(adapter); | ena_update_hwassist(adapter); | ||||
if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, | if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, | ||||
IFF_DRV_OACTIVE); | IFF_DRV_OACTIVE); | ||||
callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S, | callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S, | ||||
ena_timer_service, (void *)adapter, 0); | ena_timer_service, (void *)adapter, 0); | ||||
taskqueue_enqueue(adapter->stats_tq, &adapter->stats_task); | |||||
adapter->up = true; | adapter->up = true; | ||||
ena_unmask_all_io_irqs(adapter); | ena_unmask_all_io_irqs(adapter); | ||||
} | } | ||||
return (0); | return (0); | ||||
err_up_complete: | err_up_complete: | ||||
ena_destroy_all_io_queues(adapter); | ena_destroy_all_io_queues(adapter); | ||||
err_io_que: | err_io_que: | ||||
ena_free_all_rx_resources(adapter); | ena_free_all_rx_resources(adapter); | ||||
err_setup_rx: | err_setup_rx: | ||||
ena_free_all_tx_resources(adapter); | ena_free_all_tx_resources(adapter); | ||||
err_setup_tx: | err_setup_tx: | ||||
ena_free_io_irq(adapter); | ena_free_io_irq(adapter); | ||||
err_req_irq: | err_req_irq: | ||||
return (rc); | return (rc); | ||||
} | } | ||||
int | |||||
ena_update_stats_counters(struct ena_adapter *adapter) | |||||
{ | |||||
struct ena_admin_basic_stats ena_stats; | |||||
struct ena_hw_stats *stats = &adapter->hw_stats; | |||||
int rc = 0; | |||||
if (!adapter->up) | |||||
return (rc); | |||||
rc = ena_com_get_dev_basic_stats(adapter->ena_dev, &ena_stats); | |||||
if (rc) | |||||
return (rc); | |||||
stats->tx_bytes = ((uint64_t)ena_stats.tx_bytes_high << 32) | | |||||
ena_stats.tx_bytes_low; | |||||
stats->rx_bytes = ((uint64_t)ena_stats.rx_bytes_high << 32) | | |||||
ena_stats.rx_bytes_low; | |||||
stats->rx_packets = ((uint64_t)ena_stats.rx_pkts_high << 32) | | |||||
ena_stats.rx_pkts_low; | |||||
stats->tx_packets = ((uint64_t)ena_stats.tx_pkts_high << 32) | | |||||
ena_stats.tx_pkts_low; | |||||
stats->rx_drops = ((uint64_t)ena_stats.rx_drops_high << 32) | | |||||
ena_stats.rx_drops_low; | |||||
return (0); | |||||
} | |||||
static uint64_t | static uint64_t | ||||
ena_get_counter(if_t ifp, ift_counter cnt) | ena_get_counter(if_t ifp, ift_counter cnt) | ||||
{ | { | ||||
struct ena_adapter *adapter; | struct ena_adapter *adapter; | ||||
struct ena_hw_stats *stats; | struct ena_hw_stats *stats; | ||||
adapter = if_getsoftc(ifp); | adapter = if_getsoftc(ifp); | ||||
stats = &adapter->hw_stats; | stats = &adapter->hw_stats; | ||||
switch (cnt) { | switch (cnt) { | ||||
case IFCOUNTER_IPACKETS: | case IFCOUNTER_IPACKETS: | ||||
return (stats->rx_packets); | return (counter_u64_fetch(stats->rx_packets)); | ||||
case IFCOUNTER_OPACKETS: | case IFCOUNTER_OPACKETS: | ||||
return (stats->tx_packets); | return (counter_u64_fetch(stats->tx_packets)); | ||||
case IFCOUNTER_IBYTES: | case IFCOUNTER_IBYTES: | ||||
return (stats->rx_bytes); | return (counter_u64_fetch(stats->rx_bytes)); | ||||
case IFCOUNTER_OBYTES: | case IFCOUNTER_OBYTES: | ||||
return (stats->tx_bytes); | return (counter_u64_fetch(stats->tx_bytes)); | ||||
case IFCOUNTER_IQDROPS: | case IFCOUNTER_IQDROPS: | ||||
return (stats->rx_drops); | return (counter_u64_fetch(stats->rx_drops)); | ||||
default: | default: | ||||
return (if_get_counter_default(ifp, cnt)); | return (if_get_counter_default(ifp, cnt)); | ||||
} | } | ||||
} | } | ||||
static int | static int | ||||
ena_media_change(if_t ifp) | ena_media_change(if_t ifp) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 266 Lines • ▼ Show 20 Lines | if (adapter->up) { | ||||
device_printf(adapter->pdev, "device is going DOWN\n"); | device_printf(adapter->pdev, "device is going DOWN\n"); | ||||
callout_drain(&adapter->timer_service); | callout_drain(&adapter->timer_service); | ||||
adapter->up = false; | adapter->up = false; | ||||
if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, | if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, | ||||
IFF_DRV_RUNNING); | IFF_DRV_RUNNING); | ||||
/* Drain task responsible for updating hw stats */ | |||||
while (taskqueue_cancel(adapter->stats_tq, &adapter->stats_task, NULL)) | |||||
taskqueue_drain(adapter->stats_tq, &adapter->stats_task); | |||||
ena_free_io_irq(adapter); | ena_free_io_irq(adapter); | ||||
if (adapter->trigger_reset) { | if (adapter->trigger_reset) { | ||||
rc = ena_com_dev_reset(adapter->ena_dev, | rc = ena_com_dev_reset(adapter->ena_dev, | ||||
adapter->reset_reason); | adapter->reset_reason); | ||||
if (rc) | if (rc) | ||||
device_printf(adapter->pdev, | device_printf(adapter->pdev, | ||||
"Device reset failed\n"); | "Device reset failed\n"); | ||||
▲ Show 20 Lines • Show All 217 Lines • ▼ Show 20 Lines | if (rc != 0) { | ||||
counter_u64_add_protected(tx_ring->tx_stats.prepare_ctx_err, 1); | counter_u64_add_protected(tx_ring->tx_stats.prepare_ctx_err, 1); | ||||
counter_exit(); | counter_exit(); | ||||
goto dma_error; | goto dma_error; | ||||
} | } | ||||
counter_enter(); | counter_enter(); | ||||
counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); | counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); | ||||
counter_u64_add_protected(tx_ring->tx_stats.bytes, (*mbuf)->m_pkthdr.len); | counter_u64_add_protected(tx_ring->tx_stats.bytes, (*mbuf)->m_pkthdr.len); | ||||
counter_u64_add_protected(adapter->hw_stats.tx_packets, 1); | |||||
counter_u64_add_protected(adapter->hw_stats.tx_bytes, | |||||
(*mbuf)->m_pkthdr.len); | |||||
counter_exit(); | counter_exit(); | ||||
tx_info->tx_descs = nb_hw_desc; | tx_info->tx_descs = nb_hw_desc; | ||||
getbinuptime(&tx_info->timestamp); | getbinuptime(&tx_info->timestamp); | ||||
tx_info->print_once = true; | tx_info->print_once = true; | ||||
tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, | tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, | ||||
tx_ring->ring_size); | tx_ring->ring_size); | ||||
▲ Show 20 Lines • Show All 473 Lines • ▼ Show 20 Lines | err_disable_msix: | ||||
return rc; | return rc; | ||||
} | } | ||||
/* Function called on ENA_ADMIN_KEEP_ALIVE event */ | /* Function called on ENA_ADMIN_KEEP_ALIVE event */ | ||||
static void ena_keep_alive_wd(void *adapter_data, | static void ena_keep_alive_wd(void *adapter_data, | ||||
struct ena_admin_aenq_entry *aenq_e) | struct ena_admin_aenq_entry *aenq_e) | ||||
{ | { | ||||
struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; | struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; | ||||
struct ena_admin_aenq_keep_alive_desc *desc; | |||||
sbintime_t stime; | sbintime_t stime; | ||||
uint64_t rx_drops; | |||||
desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; | |||||
rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; | |||||
counter_u64_zero(adapter->hw_stats.rx_drops); | |||||
counter_u64_add(adapter->hw_stats.rx_drops, rx_drops); | |||||
stime = getsbinuptime(); | stime = getsbinuptime(); | ||||
atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); | atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); | ||||
} | } | ||||
/* Check for keep alive expiration */ | /* Check for keep alive expiration */ | ||||
static void check_for_missing_keep_alive(struct ena_adapter *adapter) | static void check_for_missing_keep_alive(struct ena_adapter *adapter) | ||||
{ | { | ||||
sbintime_t timestamp, time; | sbintime_t timestamp, time; | ||||
▲ Show 20 Lines • Show All 392 Lines • ▼ Show 20 Lines | ena_attach(device_t pdev) | ||||
if (adapter->reset_tq == NULL) { | if (adapter->reset_tq == NULL) { | ||||
device_printf(adapter->pdev, | device_printf(adapter->pdev, | ||||
"Unable to create reset task queue\n"); | "Unable to create reset task queue\n"); | ||||
goto err_reset_tq; | goto err_reset_tq; | ||||
} | } | ||||
taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, | taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, | ||||
"%s rstq", device_get_nameunit(adapter->pdev)); | "%s rstq", device_get_nameunit(adapter->pdev)); | ||||
/* Initialize task queue responsible for updating hw stats */ | |||||
TASK_INIT(&adapter->stats_task, 0, ena_update_hw_stats, adapter); | |||||
adapter->stats_tq = taskqueue_create_fast("ena_stats_update", | |||||
M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->stats_tq); | |||||
if (adapter->stats_tq == NULL) { | |||||
device_printf(adapter->pdev, | |||||
"Unable to create taskqueue for updating hw stats\n"); | |||||
goto err_stats_tq; | |||||
} | |||||
taskqueue_start_threads(&adapter->stats_tq, 1, PI_REALTIME, | |||||
"%s stats tq", device_get_nameunit(adapter->pdev)); | |||||
/* Initialize statistics */ | /* Initialize statistics */ | ||||
ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, | ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, | ||||
sizeof(struct ena_stats_dev)); | sizeof(struct ena_stats_dev)); | ||||
ena_update_stats_counters(adapter); | ena_alloc_counters((counter_u64_t *)&adapter->hw_stats, | ||||
sizeof(struct ena_hw_stats)); | |||||
ena_sysctl_add_nodes(adapter); | ena_sysctl_add_nodes(adapter); | ||||
/* Tell the stack that the interface is not active */ | /* Tell the stack that the interface is not active */ | ||||
if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | ||||
adapter->running = true; | adapter->running = true; | ||||
return (0); | return (0); | ||||
err_stats_tq: | |||||
taskqueue_free(adapter->reset_tq); | |||||
err_reset_tq: | err_reset_tq: | ||||
ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR); | ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR); | ||||
ena_free_mgmnt_irq(adapter); | ena_free_mgmnt_irq(adapter); | ||||
ena_disable_msix(adapter); | ena_disable_msix(adapter); | ||||
err_ifp_free: | err_ifp_free: | ||||
if_detach(adapter->ifp); | if_detach(adapter->ifp); | ||||
if_free(adapter->ifp); | if_free(adapter->ifp); | ||||
err_com_free: | err_com_free: | ||||
Show All 40 Lines | ena_detach(device_t pdev) | ||||
while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) | while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) | ||||
taskqueue_drain(adapter->reset_tq, &adapter->reset_task); | taskqueue_drain(adapter->reset_tq, &adapter->reset_task); | ||||
taskqueue_free(adapter->reset_tq); | taskqueue_free(adapter->reset_tq); | ||||
sx_xlock(&adapter->ioctl_sx); | sx_xlock(&adapter->ioctl_sx); | ||||
ena_down(adapter); | ena_down(adapter); | ||||
sx_unlock(&adapter->ioctl_sx); | sx_unlock(&adapter->ioctl_sx); | ||||
taskqueue_free(adapter->stats_tq); | |||||
if (adapter->ifp != NULL) { | if (adapter->ifp != NULL) { | ||||
ether_ifdetach(adapter->ifp); | ether_ifdetach(adapter->ifp); | ||||
if_free(adapter->ifp); | if_free(adapter->ifp); | ||||
} | } | ||||
ena_free_all_io_rings_resources(adapter); | ena_free_all_io_rings_resources(adapter); | ||||
ena_free_counters((counter_u64_t *)&adapter->hw_stats, | |||||
sizeof(struct ena_hw_stats)); | |||||
ena_free_counters((counter_u64_t *)&adapter->dev_stats, | ena_free_counters((counter_u64_t *)&adapter->dev_stats, | ||||
sizeof(struct ena_stats_dev)); | sizeof(struct ena_stats_dev)); | ||||
if (adapter->rss_support) | if (adapter->rss_support) | ||||
ena_com_rss_destroy(ena_dev); | ena_com_rss_destroy(ena_dev); | ||||
rc = ena_free_rx_dma_tag(adapter); | rc = ena_free_rx_dma_tag(adapter); | ||||
if (rc) | if (rc) | ||||
▲ Show 20 Lines • Show All 111 Lines • Show Last 20 Lines |