Changeset View
Changeset View
Standalone View
Standalone View
head/sys/dev/ena/ena.c
Show First 20 Lines • Show All 149 Lines • ▼ Show 20 Lines | |||||
static void ena_init(void *); | static void ena_init(void *); | ||||
static int ena_ioctl(if_t, u_long, caddr_t); | static int ena_ioctl(if_t, u_long, caddr_t); | ||||
static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *); | static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *); | ||||
static void ena_update_host_info(struct ena_admin_host_info *, if_t); | static void ena_update_host_info(struct ena_admin_host_info *, if_t); | ||||
static void ena_update_hwassist(struct ena_adapter *); | static void ena_update_hwassist(struct ena_adapter *); | ||||
static int ena_setup_ifnet(device_t, struct ena_adapter *, | static int ena_setup_ifnet(device_t, struct ena_adapter *, | ||||
struct ena_com_dev_get_features_ctx *); | struct ena_com_dev_get_features_ctx *); | ||||
static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *); | static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *); | ||||
static int ena_xmit_mbuf(struct ena_ring *, struct mbuf *); | static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **); | ||||
static void ena_start_xmit(struct ena_ring *); | static void ena_start_xmit(struct ena_ring *); | ||||
static int ena_mq_start(if_t, struct mbuf *); | static int ena_mq_start(if_t, struct mbuf *); | ||||
static void ena_deferred_mq_start(void *, int); | static void ena_deferred_mq_start(void *, int); | ||||
static void ena_qflush(if_t); | static void ena_qflush(if_t); | ||||
static int ena_calc_io_queue_num(struct ena_adapter *, | static int ena_calc_io_queue_num(struct ena_adapter *, | ||||
struct ena_com_dev_get_features_ctx *); | struct ena_com_dev_get_features_ctx *); | ||||
static int ena_calc_queue_size(struct ena_adapter *, uint16_t *, | static int ena_calc_queue_size(struct ena_adapter *, uint16_t *, | ||||
uint16_t *, struct ena_com_dev_get_features_ctx *); | uint16_t *, struct ena_com_dev_get_features_ctx *); | ||||
▲ Show 20 Lines • Show All 2,429 Lines • ▼ Show 20 Lines | ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf) | ||||
ena_meta->mss = mss; | ena_meta->mss = mss; | ||||
ena_meta->l3_hdr_len = iphlen; | ena_meta->l3_hdr_len = iphlen; | ||||
ena_meta->l3_hdr_offset = ehdrlen; | ena_meta->l3_hdr_offset = ehdrlen; | ||||
ena_tx_ctx->meta_valid = 1; | ena_tx_ctx->meta_valid = 1; | ||||
} | } | ||||
static int | static int | ||||
ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf *mbuf) | ena_check_and_defragment_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) | ||||
{ | { | ||||
struct ena_adapter *adapter; | struct ena_adapter *adapter; | ||||
struct mbuf *defrag_mbuf; | |||||
int num_frags; | |||||
adapter = tx_ring->adapter; | |||||
num_frags = ena_mbuf_count(*mbuf); | |||||
/* One segment must be reserved for configuration descriptor. */ | |||||
if (num_frags < adapter->max_tx_sgl_size) | |||||
return (0); | |||||
counter_u64_add(tx_ring->tx_stats.defragment, 1); | |||||
defrag_mbuf = m_defrag(*mbuf, M_NOWAIT); | |||||
if (defrag_mbuf == NULL) { | |||||
counter_u64_add(tx_ring->tx_stats.defragment_err, 1); | |||||
return (ENOMEM); | |||||
} | |||||
/* If mbuf was defragmented succesfully, original mbuf is released. */ | |||||
*mbuf = defrag_mbuf; | |||||
return (0); | |||||
} | |||||
static int | |||||
ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) | |||||
{ | |||||
struct ena_adapter *adapter; | |||||
struct ena_tx_buffer *tx_info; | struct ena_tx_buffer *tx_info; | ||||
struct ena_com_tx_ctx ena_tx_ctx; | struct ena_com_tx_ctx ena_tx_ctx; | ||||
struct ena_com_dev *ena_dev; | struct ena_com_dev *ena_dev; | ||||
struct ena_com_buf *ena_buf; | struct ena_com_buf *ena_buf; | ||||
struct ena_com_io_sq* io_sq; | struct ena_com_io_sq* io_sq; | ||||
bus_dma_segment_t segs[ENA_BUS_DMA_SEGS]; | bus_dma_segment_t segs[ENA_BUS_DMA_SEGS]; | ||||
void *push_hdr; | void *push_hdr; | ||||
uint16_t next_to_use; | uint16_t next_to_use; | ||||
uint16_t req_id; | uint16_t req_id; | ||||
uint16_t push_len; | uint16_t push_len; | ||||
uint16_t ena_qid; | uint16_t ena_qid; | ||||
uint32_t len, nsegs, header_len; | uint32_t len, nsegs, header_len; | ||||
int i, rc; | int i, rc; | ||||
int nb_hw_desc, num_frags; | int nb_hw_desc; | ||||
ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); | ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); | ||||
adapter = tx_ring->que->adapter; | adapter = tx_ring->que->adapter; | ||||
ena_dev = adapter->ena_dev; | ena_dev = adapter->ena_dev; | ||||
io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; | io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; | ||||
ENA_ASSERT(mbuf, "mbuf is NULL\n"); | ENA_ASSERT(*mbuf, "mbuf is NULL\n"); | ||||
num_frags = ena_mbuf_count(mbuf); | rc = ena_check_and_defragment_mbuf(tx_ring, mbuf); | ||||
if (num_frags > (adapter->max_tx_sgl_size - 2)) { | if (rc) { | ||||
device_printf(adapter->pdev, | ena_trace(ENA_WARNING, | ||||
"too many fragments. Last fragment: %d!\n", num_frags); | "Failed to defragment mbuf! err: %d", rc); | ||||
return (ENA_COM_INVAL); | return (rc); | ||||
} | } | ||||
next_to_use = tx_ring->next_to_use; | next_to_use = tx_ring->next_to_use; | ||||
req_id = tx_ring->free_tx_ids[next_to_use]; | req_id = tx_ring->free_tx_ids[next_to_use]; | ||||
tx_info = &tx_ring->tx_buffer_info[req_id]; | tx_info = &tx_ring->tx_buffer_info[req_id]; | ||||
tx_info->mbuf = mbuf; | tx_info->mbuf = *mbuf; | ||||
tx_info->num_of_bufs = 0; | tx_info->num_of_bufs = 0; | ||||
ena_buf = tx_info->bufs; | ena_buf = tx_info->bufs; | ||||
len = mbuf->m_len; | len = (*mbuf)->m_len; | ||||
ena_trace(ENA_DBG | ENA_TXPTH, "Tx: %d bytes", mbuf->m_pkthdr.len); | ena_trace(ENA_DBG | ENA_TXPTH, "Tx: %d bytes", (*mbuf)->m_pkthdr.len); | ||||
push_len = 0; | push_len = 0; | ||||
header_len = min_t(uint32_t, len, tx_ring->tx_max_header_size); | header_len = min_t(uint32_t, len, tx_ring->tx_max_header_size); | ||||
push_hdr = NULL; | push_hdr = NULL; | ||||
rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, tx_info->map, | rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, tx_info->map, | ||||
mbuf, segs, &nsegs, BUS_DMA_NOWAIT); | *mbuf, segs, &nsegs, BUS_DMA_NOWAIT); | ||||
if (rc || (nsegs == 0)) { | if (rc || (nsegs == 0)) { | ||||
ena_trace(ENA_WARNING, | ena_trace(ENA_WARNING, | ||||
"dmamap load failed! err: %d nsegs: %d", rc, nsegs); | "dmamap load failed! err: %d nsegs: %d", rc, nsegs); | ||||
counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1); | counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1); | ||||
tx_info->mbuf = NULL; | tx_info->mbuf = NULL; | ||||
if (rc == ENOMEM) | if (rc == ENOMEM) | ||||
return (ENA_COM_NO_MEM); | return (ENA_COM_NO_MEM); | ||||
Show All 11 Lines | ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) | ||||
memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); | memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); | ||||
ena_tx_ctx.ena_bufs = tx_info->bufs; | ena_tx_ctx.ena_bufs = tx_info->bufs; | ||||
ena_tx_ctx.push_header = push_hdr; | ena_tx_ctx.push_header = push_hdr; | ||||
ena_tx_ctx.num_bufs = tx_info->num_of_bufs; | ena_tx_ctx.num_bufs = tx_info->num_of_bufs; | ||||
ena_tx_ctx.req_id = req_id; | ena_tx_ctx.req_id = req_id; | ||||
ena_tx_ctx.header_len = header_len; | ena_tx_ctx.header_len = header_len; | ||||
/* Set flags and meta data */ | /* Set flags and meta data */ | ||||
ena_tx_csum(&ena_tx_ctx, mbuf); | ena_tx_csum(&ena_tx_ctx, *mbuf); | ||||
/* Prepare the packet's descriptors and send them to device */ | /* Prepare the packet's descriptors and send them to device */ | ||||
rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc); | rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc); | ||||
if (rc != 0) { | if (rc != 0) { | ||||
ena_trace(ENA_WARNING, "failed to prepare tx bufs\n"); | ena_trace(ENA_WARNING, "failed to prepare tx bufs\n"); | ||||
counter_enter(); | counter_enter(); | ||||
counter_u64_add_protected(tx_ring->tx_stats.queue_stop, 1); | counter_u64_add_protected(tx_ring->tx_stats.queue_stop, 1); | ||||
counter_u64_add_protected(tx_ring->tx_stats.prepare_ctx_err, 1); | counter_u64_add_protected(tx_ring->tx_stats.prepare_ctx_err, 1); | ||||
counter_exit(); | counter_exit(); | ||||
goto dma_error; | goto dma_error; | ||||
} | } | ||||
counter_enter(); | counter_enter(); | ||||
counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); | counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); | ||||
counter_u64_add_protected(tx_ring->tx_stats.bytes, mbuf->m_pkthdr.len); | counter_u64_add_protected(tx_ring->tx_stats.bytes, (*mbuf)->m_pkthdr.len); | ||||
counter_exit(); | counter_exit(); | ||||
tx_info->tx_descs = nb_hw_desc; | tx_info->tx_descs = nb_hw_desc; | ||||
getbinuptime(&tx_info->timestamp); | getbinuptime(&tx_info->timestamp); | ||||
tx_info->print_once = true; | tx_info->print_once = true; | ||||
tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, | tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, | ||||
tx_ring->ring_size); | tx_ring->ring_size); | ||||
Show All 31 Lines | ena_start_xmit(struct ena_ring *tx_ring) | ||||
while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) { | while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) { | ||||
ena_trace(ENA_DBG | ENA_TXPTH, "\ndequeued mbuf %p with flags %#x and" | ena_trace(ENA_DBG | ENA_TXPTH, "\ndequeued mbuf %p with flags %#x and" | ||||
" header csum flags %#jx", | " header csum flags %#jx", | ||||
mbuf, mbuf->m_flags, mbuf->m_pkthdr.csum_flags); | mbuf, mbuf->m_flags, mbuf->m_pkthdr.csum_flags); | ||||
if (ena_com_sq_empty_space(io_sq) < ENA_TX_CLEANUP_TRESHOLD) | if (ena_com_sq_empty_space(io_sq) < ENA_TX_CLEANUP_TRESHOLD) | ||||
ena_tx_cleanup(tx_ring); | ena_tx_cleanup(tx_ring); | ||||
if ((ret = ena_xmit_mbuf(tx_ring, mbuf)) != 0) { | if ((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0) { | ||||
if (ret == ENA_COM_NO_MEM) { | if (ret == ENA_COM_NO_MEM) { | ||||
drbr_putback(adapter->ifp, tx_ring->br, mbuf); | drbr_putback(adapter->ifp, tx_ring->br, mbuf); | ||||
} else if (ret == ENA_COM_NO_SPACE) { | } else if (ret == ENA_COM_NO_SPACE) { | ||||
drbr_putback(adapter->ifp, tx_ring->br, mbuf); | drbr_putback(adapter->ifp, tx_ring->br, mbuf); | ||||
} else { | } else { | ||||
m_freem(mbuf); | m_freem(mbuf); | ||||
drbr_advance(adapter->ifp, tx_ring->br); | drbr_advance(adapter->ifp, tx_ring->br); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 1,018 Lines • Show Last 20 Lines |