Changeset View
Changeset View
Standalone View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 185 Lines • ▼ Show 20 Lines | struct iflib_ctx { | ||||
struct sysctl_oid *ifc_sysctl_node; | struct sysctl_oid *ifc_sysctl_node; | ||||
uint16_t ifc_sysctl_ntxqs; | uint16_t ifc_sysctl_ntxqs; | ||||
uint16_t ifc_sysctl_nrxqs; | uint16_t ifc_sysctl_nrxqs; | ||||
uint16_t ifc_sysctl_qs_eq_override; | uint16_t ifc_sysctl_qs_eq_override; | ||||
uint16_t ifc_sysctl_rx_budget; | uint16_t ifc_sysctl_rx_budget; | ||||
uint16_t ifc_sysctl_tx_abdicate; | uint16_t ifc_sysctl_tx_abdicate; | ||||
uint16_t ifc_sysctl_core_offset; | uint16_t ifc_sysctl_core_offset; | ||||
#define CORE_OFFSET_UNSPECIFIED 0xffff | #define CORE_OFFSET_UNSPECIFIED 0xffff | ||||
int ifc_sysctl_bypassmp; | |||||
uint8_t ifc_sysctl_separate_txrx; | uint8_t ifc_sysctl_separate_txrx; | ||||
qidx_t ifc_sysctl_ntxds[8]; | qidx_t ifc_sysctl_ntxds[8]; | ||||
qidx_t ifc_sysctl_nrxds[8]; | qidx_t ifc_sysctl_nrxds[8]; | ||||
struct if_txrx ifc_txrx; | struct if_txrx ifc_txrx; | ||||
#define isc_txd_encap ifc_txrx.ift_txd_encap | #define isc_txd_encap ifc_txrx.ift_txd_encap | ||||
#define isc_txd_flush ifc_txrx.ift_txd_flush | #define isc_txd_flush ifc_txrx.ift_txd_flush | ||||
#define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | #define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | ||||
▲ Show 20 Lines • Show All 3,304 Lines • ▼ Show 20 Lines | iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh) | ||||
int reclaim; | int reclaim; | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
KASSERT(thresh >= 0, ("invalid threshold to reclaim")); | KASSERT(thresh >= 0, ("invalid threshold to reclaim")); | ||||
MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size); | MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size); | ||||
/* | /* | ||||
* Need a rate-limiting check so that this isn't called every time | * Need a rate-limiting check so that this isn't called every time | ||||
* But this is terrible - discuss | |||||
*/ | */ | ||||
if (txq->ift_in_use < thresh) { | |||||
return 0; | |||||
} | |||||
iflib_tx_credits_update(ctx, txq); | iflib_tx_credits_update(ctx, txq); | ||||
reclaim = DESC_RECLAIMABLE(txq); | reclaim = DESC_RECLAIMABLE(txq); | ||||
if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) { | if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (iflib_verbose_debug) { | if (iflib_verbose_debug) { | ||||
printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __FUNCTION__, | printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __FUNCTION__, | ||||
txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments, | txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments, | ||||
reclaim, thresh); | reclaim, thresh); | ||||
} | } | ||||
#endif | #endif | ||||
return (0); | return (0); | ||||
} | } | ||||
iflib_tx_desc_free(txq, reclaim); | iflib_tx_desc_free(txq, reclaim); | ||||
txq->ift_cleaned += reclaim; | txq->ift_cleaned += reclaim; | ||||
txq->ift_in_use -= reclaim; | txq->ift_in_use -= reclaim; | ||||
return (reclaim); | return (reclaim); | ||||
} | } | ||||
static struct mbuf ** | static struct mbuf ** | ||||
_ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining) | _ring_peek_one(struct ifmp_ring *r, void *volatile *itms, int cidx, int offset, int remaining) | ||||
{ | { | ||||
int next, size; | int next, size; | ||||
struct mbuf **items; | struct mbuf **items; | ||||
size = r->size; | size = r->size; | ||||
next = (cidx + CACHE_PTR_INCREMENT) & (size-1); | next = (cidx + CACHE_PTR_INCREMENT) & (size-1); | ||||
items = __DEVOLATILE(struct mbuf **, &r->items[0]); | items = __DEVOLATILE(struct mbuf **, &itms[0]); | ||||
prefetch(items[(cidx + offset) & (size-1)]); | prefetch(items[(cidx + offset) & (size-1)]); | ||||
if (remaining > 1) { | if (remaining > 1) { | ||||
prefetch2cachelines(&items[next]); | prefetch2cachelines(&items[next]); | ||||
prefetch2cachelines(items[(cidx + offset + 1) & (size-1)]); | prefetch2cachelines(items[(cidx + offset + 1) & (size-1)]); | ||||
prefetch2cachelines(items[(cidx + offset + 2) & (size-1)]); | prefetch2cachelines(items[(cidx + offset + 2) & (size-1)]); | ||||
prefetch2cachelines(items[(cidx + offset + 3) & (size-1)]); | prefetch2cachelines(items[(cidx + offset + 3) & (size-1)]); | ||||
} | } | ||||
return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)])); | return (__DEVOLATILE(struct mbuf **, &itms[(cidx + offset) & (size-1)])); | ||||
} | } | ||||
static void | static void | ||||
iflib_txq_check_drain(iflib_txq_t txq, int budget) | iflib_txq_check_drain(iflib_txq_t txq, int budget) | ||||
{ | { | ||||
if_ctx_t ctx = txq->ift_ctx; | |||||
if (!ctx->ifc_sysctl_bypassmp) | |||||
ifmp_ring_check_drainage(txq->ift_br, budget); | ifmp_ring_check_drainage(txq->ift_br, budget); | ||||
} | } | ||||
static uint32_t | static uint32_t | ||||
iflib_txq_can_drain(struct ifmp_ring *r) | iflib_txq_can_drain(struct ifmp_ring *r) | ||||
{ | { | ||||
iflib_txq_t txq = r->cookie; | iflib_txq_t txq = r->cookie; | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) | if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) | ||||
return (1); | return (1); | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, | return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, | ||||
false)); | false)); | ||||
} | } | ||||
static uint32_t | static uint32_t | ||||
iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx) | iflib_txq_drain(struct ifmp_ring *r, void *volatile *items, uint32_t cidx, uint32_t pidx) | ||||
{ | { | ||||
iflib_txq_t txq = r->cookie; | iflib_txq_t txq = r->cookie; | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
struct mbuf *m, **mp; | struct mbuf *m, **mp; | ||||
int avail, bytes_sent, consumed, count, err, i, in_use_prev; | int avail, bytes_sent, consumed, count, err, i, in_use_prev; | ||||
int mcast_sent, pkt_sent, reclaimed, txq_avail; | int mcast_sent, pkt_sent, reclaimed, txq_avail; | ||||
bool do_prefetch, rang, ring; | bool do_prefetch, rang, ring; | ||||
if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) || | if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) || | ||||
!LINK_ACTIVE(ctx))) { | !LINK_ACTIVE(ctx))) { | ||||
DBG_COUNTER_INC(txq_drain_notready); | DBG_COUNTER_INC(txq_drain_notready); | ||||
return (0); | return (0); | ||||
} | } | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_LOCK(txq); | |||||
if (ctx->ifc_sysctl_bypassmp == 0) | |||||
reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx)); | reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx)); | ||||
else if (cidx == pidx) | |||||
reclaimed = iflib_completed_tx_reclaim(txq, 0); | |||||
else | |||||
reclaimed = iflib_completed_tx_reclaim(txq, txq->ift_size/2); | |||||
rang = iflib_txd_db_check(ctx, txq, reclaimed, txq->ift_in_use); | rang = iflib_txd_db_check(ctx, txq, reclaimed, txq->ift_in_use); | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_UNLOCK(txq); | |||||
avail = IDXDIFF(pidx, cidx, r->size); | avail = IDXDIFF(pidx, cidx, r->size); | ||||
if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) { | if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) { | ||||
DBG_COUNTER_INC(txq_drain_flushing); | DBG_COUNTER_INC(txq_drain_flushing); | ||||
for (i = 0; i < avail; i++) { | for (i = 0; i < avail; i++) { | ||||
if (__predict_true(r->items[(cidx + i) & (r->size-1)] != (void *)txq)) | if (__predict_true(items[(cidx + i) & (r->size-1)] != (void *)txq)) | ||||
m_free(r->items[(cidx + i) & (r->size-1)]); | m_free(items[(cidx + i) & (r->size-1)]); | ||||
r->items[(cidx + i) & (r->size-1)] = NULL; | items[(cidx + i) & (r->size-1)] = NULL; | ||||
} | } | ||||
return (avail); | return (avail); | ||||
} | } | ||||
if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) { | if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) { | ||||
txq->ift_qstatus = IFLIB_QUEUE_IDLE; | txq->ift_qstatus = IFLIB_QUEUE_IDLE; | ||||
CALLOUT_LOCK(txq); | CALLOUT_LOCK(txq); | ||||
callout_stop(&txq->ift_timer); | callout_stop(&txq->ift_timer); | ||||
CALLOUT_UNLOCK(txq); | CALLOUT_UNLOCK(txq); | ||||
DBG_COUNTER_INC(txq_drain_oactive); | DBG_COUNTER_INC(txq_drain_oactive); | ||||
return (0); | return (0); | ||||
} | } | ||||
if (reclaimed) | if (reclaimed) { | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_LOCK(txq); | |||||
txq->ift_qstatus = IFLIB_QUEUE_IDLE; | txq->ift_qstatus = IFLIB_QUEUE_IDLE; | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_UNLOCK(txq); | |||||
} | |||||
consumed = mcast_sent = bytes_sent = pkt_sent = 0; | consumed = mcast_sent = bytes_sent = pkt_sent = 0; | ||||
count = MIN(avail, TX_BATCH_SIZE); | count = MIN(avail, TX_BATCH_SIZE); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (iflib_verbose_debug) | if (iflib_verbose_debug) | ||||
printf("%s avail=%d ifc_flags=%x txq_avail=%d ", __FUNCTION__, | printf("%s avail=%d ifc_flags=%x txq_avail=%d ", __FUNCTION__, | ||||
avail, ctx->ifc_flags, TXQ_AVAIL(txq)); | avail, ctx->ifc_flags, TXQ_AVAIL(txq)); | ||||
#endif | #endif | ||||
do_prefetch = (ctx->ifc_flags & IFC_PREFETCH); | do_prefetch = (ctx->ifc_flags & IFC_PREFETCH); | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_LOCK(txq); | |||||
txq_avail = TXQ_AVAIL(txq); | txq_avail = TXQ_AVAIL(txq); | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_UNLOCK(txq); | |||||
err = 0; | err = 0; | ||||
// The use of txq_avail outside the lock is suspect here... | |||||
for (i = 0; i < count && txq_avail > MAX_TX_DESC(ctx) + 2; i++) { | for (i = 0; i < count && txq_avail > MAX_TX_DESC(ctx) + 2; i++) { | ||||
int rem = do_prefetch ? count - i : 0; | int rem = do_prefetch ? count - i : 0; | ||||
mp = _ring_peek_one(r, cidx, i, rem); | mp = _ring_peek_one(r, items, cidx, i, rem); | ||||
MPASS(mp != NULL && *mp != NULL); | MPASS(mp != NULL && *mp != NULL); | ||||
if (__predict_false(*mp == (struct mbuf *)txq)) { | if (__predict_false(*mp == (struct mbuf *)txq)) { | ||||
consumed++; | consumed++; | ||||
continue; | continue; | ||||
} | } | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_LOCK(txq); | |||||
in_use_prev = txq->ift_in_use; | in_use_prev = txq->ift_in_use; | ||||
err = iflib_encap(txq, mp); | err = iflib_encap(txq, mp); | ||||
if (__predict_false(err)) { | if (__predict_false(err)) { | ||||
/* no room - bail out */ | /* no room - bail out */ | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_UNLOCK(txq); | |||||
if (err == ENOBUFS) | if (err == ENOBUFS) | ||||
break; | break; | ||||
consumed++; | consumed++; | ||||
/* we can't send this packet - skip it */ | /* we can't send this packet - skip it */ | ||||
continue; | continue; | ||||
} | } | ||||
txq_avail = TXQ_AVAIL(txq); | |||||
txq->ift_db_pending += (txq->ift_in_use - in_use_prev); | |||||
rang = iflib_txd_db_check(ctx, txq, false, in_use_prev); | |||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_UNLOCK(txq); | |||||
consumed++; | consumed++; | ||||
pkt_sent++; | pkt_sent++; | ||||
m = *mp; | m = *mp; | ||||
DBG_COUNTER_INC(tx_sent); | DBG_COUNTER_INC(tx_sent); | ||||
bytes_sent += m->m_pkthdr.len; | bytes_sent += m->m_pkthdr.len; | ||||
mcast_sent += !!(m->m_flags & M_MCAST); | mcast_sent += !!(m->m_flags & M_MCAST); | ||||
txq_avail = TXQ_AVAIL(txq); | |||||
txq->ift_db_pending += (txq->ift_in_use - in_use_prev); | |||||
ETHER_BPF_MTAP(ifp, m); | ETHER_BPF_MTAP(ifp, m); | ||||
if (__predict_false(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) | if (__predict_false(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) | ||||
break; | break; | ||||
rang = iflib_txd_db_check(ctx, txq, false, in_use_prev); | |||||
} | } | ||||
/* deliberate use of bitwise or to avoid gratuitous short-circuit */ | /* deliberate use of bitwise or to avoid gratuitous short-circuit */ | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_LOCK(txq); | |||||
ring = rang ? false : (iflib_min_tx_latency | err) || (TXQ_AVAIL(txq) < MAX_TX_DESC(ctx)); | ring = rang ? false : (iflib_min_tx_latency | err) || (TXQ_AVAIL(txq) < MAX_TX_DESC(ctx)); | ||||
iflib_txd_db_check(ctx, txq, ring, txq->ift_in_use); | iflib_txd_db_check(ctx, txq, ring, txq->ift_in_use); | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
CALLOUT_UNLOCK(txq); | |||||
if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent); | if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent); | ||||
if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent); | if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent); | ||||
if (mcast_sent) | if (mcast_sent) | ||||
if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent); | if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (iflib_verbose_debug) | if (iflib_verbose_debug) | ||||
printf("consumed=%d\n", consumed); | printf("consumed=%d\n", consumed); | ||||
#endif | #endif | ||||
return (consumed); | return (consumed); | ||||
} | } | ||||
static uint32_t | static uint32_t | ||||
iflib_txq_drain_always(struct ifmp_ring *r) | iflib_txq_drain_always(struct ifmp_ring *r) | ||||
{ | { | ||||
return (1); | return (1); | ||||
} | } | ||||
static uint32_t | static uint32_t | ||||
iflib_txq_drain_free(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx) | iflib_txq_drain_free(struct ifmp_ring *r, void *volatile *items, uint32_t cidx, uint32_t pidx) | ||||
{ | { | ||||
int i, avail; | int i, avail; | ||||
struct mbuf **mp; | struct mbuf **mp; | ||||
iflib_txq_t txq; | iflib_txq_t txq; | ||||
txq = r->cookie; | txq = r->cookie; | ||||
txq->ift_qstatus = IFLIB_QUEUE_IDLE; | txq->ift_qstatus = IFLIB_QUEUE_IDLE; | ||||
CALLOUT_LOCK(txq); | CALLOUT_LOCK(txq); | ||||
callout_stop(&txq->ift_timer); | callout_stop(&txq->ift_timer); | ||||
CALLOUT_UNLOCK(txq); | CALLOUT_UNLOCK(txq); | ||||
avail = IDXDIFF(pidx, cidx, r->size); | avail = IDXDIFF(pidx, cidx, r->size); | ||||
for (i = 0; i < avail; i++) { | for (i = 0; i < avail; i++) { | ||||
mp = _ring_peek_one(r, cidx, i, avail - i); | mp = _ring_peek_one(r, items, cidx, i, avail - i); | ||||
if (__predict_false(*mp == (struct mbuf *)txq)) | if (__predict_false(*mp == (struct mbuf *)txq)) | ||||
continue; | continue; | ||||
m_freem(*mp); | m_freem(*mp); | ||||
DBG_COUNTER_INC(tx_frees); | DBG_COUNTER_INC(tx_frees); | ||||
} | } | ||||
MPASS(ifmp_ring_is_stalled(r) == 0); | MPASS(ifmp_ring_is_stalled(r) == 0); | ||||
return (avail); | return (avail); | ||||
} | } | ||||
static void | static void | ||||
iflib_ifmp_purge(iflib_txq_t txq) | iflib_ifmp_purge(iflib_txq_t txq) | ||||
{ | { | ||||
if_ctx_t ctx = txq->ift_ctx; | |||||
struct ifmp_ring *r; | struct ifmp_ring *r; | ||||
r = txq->ift_br; | r = txq->ift_br; | ||||
r->drain = iflib_txq_drain_free; | r->drain = iflib_txq_drain_free; | ||||
r->can_drain = iflib_txq_drain_always; | r->can_drain = iflib_txq_drain_always; | ||||
if (!ctx->ifc_sysctl_bypassmp) | |||||
ifmp_ring_check_drainage(r, r->size); | ifmp_ring_check_drainage(r, r->size); | ||||
r->drain = iflib_txq_drain; | r->drain = iflib_txq_drain; | ||||
r->can_drain = iflib_txq_can_drain; | r->can_drain = iflib_txq_can_drain; | ||||
} | } | ||||
static void | static void | ||||
_task_fn_tx(void *context) | _task_fn_tx(void *context) | ||||
{ | { | ||||
Show All 21 Lines | else | ||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | ||||
return; | return; | ||||
} | } | ||||
#endif | #endif | ||||
#ifdef ALTQ | #ifdef ALTQ | ||||
if (ALTQ_IS_ENABLED(&ifp->if_snd)) | if (ALTQ_IS_ENABLED(&ifp->if_snd)) | ||||
iflib_altq_if_start(ifp); | iflib_altq_if_start(ifp); | ||||
#endif | #endif | ||||
if (ctx->ifc_sysctl_bypassmp) { | |||||
CALLOUT_LOCK(txq); | |||||
if (txq->ift_db_pending) | if (txq->ift_db_pending) | ||||
iflib_txd_db_check(ctx, txq, true, txq->ift_in_use); | |||||
if (txq->ift_in_use) | |||||
txq->ift_br->drain(txq->ift_br, NULL, 0, 0); | |||||
CALLOUT_UNLOCK(txq); | |||||
} else { | |||||
if (txq->ift_db_pending) | |||||
ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); | ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); | ||||
else if (!abdicate) | else if (!abdicate) | ||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ||||
/* | /* | ||||
* When abdicating, we always need to check drainage, not just when we don't enqueue | * When abdicating, we always need to check drainage, not just when we don't enqueue | ||||
*/ | */ | ||||
if (abdicate) | if (abdicate) | ||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ||||
} | |||||
if (ctx->ifc_flags & IFC_LEGACY) | if (ctx->ifc_flags & IFC_LEGACY) | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
else | else | ||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | ||||
} | } | ||||
static void | static void | ||||
_task_fn_rx(void *context) | _task_fn_rx(void *context) | ||||
▲ Show 20 Lines • Show All 219 Lines • ▼ Show 20 Lines | #endif | ||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ||||
m_freem(m); | m_freem(m); | ||||
DBG_COUNTER_INC(tx_frees); | DBG_COUNTER_INC(tx_frees); | ||||
} | } | ||||
return (err); | return (err); | ||||
} | } | ||||
static int | |||||
iflib_if_transmit_bypass(if_t ifp, struct mbuf *m) | |||||
{ | |||||
if_ctx_t ctx = if_getsoftc(ifp); | |||||
iflib_txq_t txq; | |||||
int err, qidx; | |||||
if (__predict_false((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || !LINK_ACTIVE(ctx))) { | |||||
DBG_COUNTER_INC(tx_frees); | |||||
m_freem(m); | |||||
return (ENETDOWN); | |||||
} | |||||
MPASS(m->m_nextpkt == NULL); | |||||
/* ALTQ-enabled interfaces always use queue 0. */ | |||||
qidx = 0; | |||||
if ((NTXQSETS(ctx) > 1) && M_HASHTYPE_GET(m) && !ALTQ_IS_ENABLED(&ifp->if_snd)) | |||||
qidx = QIDX(ctx, m); | |||||
/* | |||||
* XXX calculate buf_ring based on flowid (divvy up bits?) | |||||
*/ | |||||
txq = &ctx->ifc_txqs[qidx]; | |||||
DBG_COUNTER_INC(tx_seen); | |||||
err = (txq->ift_br->drain(txq->ift_br, (void *volatile *)&m, 0, 1) != 1); | |||||
if (err) { | |||||
m_freem(m); | |||||
DBG_COUNTER_INC(tx_frees); | |||||
} | |||||
return (err ? EAGAIN : 0); | |||||
} | |||||
#ifdef ALTQ | #ifdef ALTQ | ||||
/* | /* | ||||
* The overall approach to integrating iflib with ALTQ is to continue to use | * The overall approach to integrating iflib with ALTQ is to continue to use | ||||
* the iflib mp_ring machinery between the ALTQ queue(s) and the hardware | * the iflib mp_ring machinery between the ALTQ queue(s) and the hardware | ||||
* ring. Technically, when using ALTQ, queueing to an intermediate mp_ring | * ring. Technically, when using ALTQ, queueing to an intermediate mp_ring | ||||
* is redundant/unnecessary, but doing so minimizes the amount of | * is redundant/unnecessary, but doing so minimizes the amount of | ||||
* ALTQ-specific code required in iflib. It is assumed that the overhead of | * ALTQ-specific code required in iflib. It is assumed that the overhead of | ||||
* redundantly queueing to an intermediate mp_ring is swamped by the | * redundantly queueing to an intermediate mp_ring is swamped by the | ||||
Show All 11 Lines | |||||
* need to trigger queue drains on a scheduled basis. | * need to trigger queue drains on a scheduled basis. | ||||
* | * | ||||
*/ | */ | ||||
static void | static void | ||||
iflib_altq_if_start(if_t ifp) | iflib_altq_if_start(if_t ifp) | ||||
{ | { | ||||
struct ifaltq *ifq = &ifp->if_snd; | struct ifaltq *ifq = &ifp->if_snd; | ||||
struct mbuf *m; | struct mbuf *m; | ||||
if_ctx_t ctx = if_getsoftc(ifp); | |||||
IFQ_LOCK(ifq); | IFQ_LOCK(ifq); | ||||
IFQ_DEQUEUE_NOLOCK(ifq, m); | IFQ_DEQUEUE_NOLOCK(ifq, m); | ||||
while (m != NULL) { | while (m != NULL) { | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
iflib_if_transmit_bypass(ifp, m); | |||||
else | |||||
iflib_if_transmit(ifp, m); | iflib_if_transmit(ifp, m); | ||||
IFQ_DEQUEUE_NOLOCK(ifq, m); | IFQ_DEQUEUE_NOLOCK(ifq, m); | ||||
} | } | ||||
IFQ_UNLOCK(ifq); | IFQ_UNLOCK(ifq); | ||||
} | } | ||||
static int | static int | ||||
iflib_altq_if_transmit(if_t ifp, struct mbuf *m) | iflib_altq_if_transmit(if_t ifp, struct mbuf *m) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | |||||
int err; | int err; | ||||
if (ALTQ_IS_ENABLED(&ifp->if_snd)) { | if (ALTQ_IS_ENABLED(&ifp->if_snd)) { | ||||
IFQ_ENQUEUE(&ifp->if_snd, m, err); | IFQ_ENQUEUE(&ifp->if_snd, m, err); | ||||
if (err == 0) | if (err == 0) | ||||
iflib_altq_if_start(ifp); | iflib_altq_if_start(ifp); | ||||
} else | } else { | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
err = iflib_if_transmit_bypass(ifp, m); | |||||
else | |||||
err = iflib_if_transmit(ifp, m); | err = iflib_if_transmit(ifp, m); | ||||
} | |||||
return (err); | return (err); | ||||
} | } | ||||
#endif /* ALTQ */ | #endif /* ALTQ */ | ||||
static void | static void | ||||
iflib_if_qflush(if_t ifp) | iflib_if_qflush(if_t ifp) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
iflib_txq_t txq = ctx->ifc_txqs; | iflib_txq_t txq = ctx->ifc_txqs; | ||||
int i; | int i; | ||||
STATE_LOCK(ctx); | STATE_LOCK(ctx); | ||||
ctx->ifc_flags |= IFC_QFLUSH; | ctx->ifc_flags |= IFC_QFLUSH; | ||||
STATE_UNLOCK(ctx); | STATE_UNLOCK(ctx); | ||||
if (!ctx->ifc_sysctl_bypassmp) { | |||||
for (i = 0; i < NTXQSETS(ctx); i++, txq++) | for (i = 0; i < NTXQSETS(ctx); i++, txq++) | ||||
while (!(ifmp_ring_is_idle(txq->ift_br) || ifmp_ring_is_stalled(txq->ift_br))) | while (!(ifmp_ring_is_idle(txq->ift_br) || ifmp_ring_is_stalled(txq->ift_br))) | ||||
iflib_txq_check_drain(txq, 0); | iflib_txq_check_drain(txq, 0); | ||||
} | |||||
STATE_LOCK(ctx); | STATE_LOCK(ctx); | ||||
ctx->ifc_flags &= ~IFC_QFLUSH; | ctx->ifc_flags &= ~IFC_QFLUSH; | ||||
STATE_UNLOCK(ctx); | STATE_UNLOCK(ctx); | ||||
/* | /* | ||||
* When ALTQ is enabled, this will also take care of purging the | * When ALTQ is enabled, this will also take care of purging the | ||||
* ALTQ queue(s). | * ALTQ queue(s). | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 495 Lines • ▼ Show 20 Lines | iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | ||||
ctx->ifc_dev = dev; | ctx->ifc_dev = dev; | ||||
ctx->ifc_softc = sc; | ctx->ifc_softc = sc; | ||||
if ((err = iflib_register(ctx)) != 0) { | if ((err = iflib_register(ctx)) != 0) { | ||||
device_printf(dev, "iflib_register failed %d\n", err); | device_printf(dev, "iflib_register failed %d\n", err); | ||||
goto fail_ctx_free; | goto fail_ctx_free; | ||||
} | } | ||||
iflib_add_device_sysctl_pre(ctx); | iflib_add_device_sysctl_pre(ctx); | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
if_settransmitfn(ctx->ifc_ifp, iflib_if_transmit_bypass); | |||||
scctx = &ctx->ifc_softc_ctx; | scctx = &ctx->ifc_softc_ctx; | ||||
ifp = ctx->ifc_ifp; | ifp = ctx->ifc_ifp; | ||||
iflib_reset_qvalues(ctx); | iflib_reset_qvalues(ctx); | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
if ((err = IFDI_ATTACH_PRE(ctx)) != 0) { | if ((err = IFDI_ATTACH_PRE(ctx)) != 0) { | ||||
device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err); | device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err); | ||||
▲ Show 20 Lines • Show All 233 Lines • ▼ Show 20 Lines | iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, | ||||
ctx->ifc_softc = sc; | ctx->ifc_softc = sc; | ||||
ctx->ifc_dev = dev; | ctx->ifc_dev = dev; | ||||
if ((err = iflib_register(ctx)) != 0) { | if ((err = iflib_register(ctx)) != 0) { | ||||
device_printf(dev, "%s: iflib_register failed %d\n", __func__, err); | device_printf(dev, "%s: iflib_register failed %d\n", __func__, err); | ||||
goto fail_ctx_free; | goto fail_ctx_free; | ||||
} | } | ||||
iflib_add_device_sysctl_pre(ctx); | iflib_add_device_sysctl_pre(ctx); | ||||
if (ctx->ifc_sysctl_bypassmp) | |||||
if_settransmitfn(ctx->ifc_ifp, iflib_if_transmit_bypass); | |||||
scctx = &ctx->ifc_softc_ctx; | scctx = &ctx->ifc_softc_ctx; | ||||
ifp = ctx->ifc_ifp; | ifp = ctx->ifc_ifp; | ||||
iflib_reset_qvalues(ctx); | iflib_reset_qvalues(ctx); | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
if ((err = IFDI_ATTACH_PRE(ctx)) != 0) { | if ((err = IFDI_ATTACH_PRE(ctx)) != 0) { | ||||
device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err); | device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err); | ||||
▲ Show 20 Lines • Show All 490 Lines • ▼ Show 20 Lines | |||||
static int | static int | ||||
iflib_register(if_ctx_t ctx) | iflib_register(if_ctx_t ctx) | ||||
{ | { | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
driver_t *driver = sctx->isc_driver; | driver_t *driver = sctx->isc_driver; | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
if_t ifp; | if_t ifp; | ||||
char path[64]; | |||||
_iflib_assert(sctx); | _iflib_assert(sctx); | ||||
CTX_LOCK_INIT(ctx); | CTX_LOCK_INIT(ctx); | ||||
STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev)); | STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev)); | ||||
ifp = ctx->ifc_ifp = if_alloc(IFT_ETHER); | ifp = ctx->ifc_ifp = if_alloc(IFT_ETHER); | ||||
if (ifp == NULL) { | if (ifp == NULL) { | ||||
device_printf(dev, "can not allocate ifnet structure\n"); | device_printf(dev, "can not allocate ifnet structure\n"); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
/* | /* | ||||
* Initialize our context's device specific methods | * Initialize our context's device specific methods | ||||
*/ | */ | ||||
kobj_init((kobj_t) ctx, (kobj_class_t) driver); | kobj_init((kobj_t) ctx, (kobj_class_t) driver); | ||||
kobj_class_compile((kobj_class_t) driver); | kobj_class_compile((kobj_class_t) driver); | ||||
driver->refs++; | driver->refs++; | ||||
if_initname(ifp, device_get_name(dev), device_get_unit(dev)); | if_initname(ifp, device_get_name(dev), device_get_unit(dev)); | ||||
if_setsoftc(ifp, ctx); | if_setsoftc(ifp, ctx); | ||||
if_setdev(ifp, dev); | if_setdev(ifp, dev); | ||||
if_setinitfn(ifp, iflib_if_init); | if_setinitfn(ifp, iflib_if_init); | ||||
if_setioctlfn(ifp, iflib_if_ioctl); | if_setioctlfn(ifp, iflib_if_ioctl); | ||||
snprintf(path, sizeof(path), "dev.%s.%d.iflib.bypass_mpring", device_get_name(ctx->ifc_dev), device_get_unit(ctx->ifc_dev)); | |||||
TUNABLE_INT_FETCH(path, &ctx->ifc_sysctl_bypassmp); | |||||
#ifdef ALTQ | #ifdef ALTQ | ||||
if_setstartfn(ifp, iflib_altq_if_start); | if_setstartfn(ifp, iflib_altq_if_start); | ||||
if_settransmitfn(ifp, iflib_altq_if_transmit); | if_settransmitfn(ifp, iflib_altq_if_transmit); | ||||
if_setsendqready(ifp); | if_setsendqready(ifp); | ||||
#else | #else | ||||
if (ctx->ifc_sysctl_bypassmp) { | |||||
// This doesn't work since the sysctl isn't set up yet. | |||||
if_settransmitfn(ifp, iflib_if_transmit_bypass); | |||||
} | |||||
else | |||||
if_settransmitfn(ifp, iflib_if_transmit); | if_settransmitfn(ifp, iflib_if_transmit); | ||||
#endif | #endif | ||||
if_setqflushfn(ifp, iflib_if_qflush); | if_setqflushfn(ifp, iflib_if_qflush); | ||||
if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); | if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); | ||||
ctx->ifc_vlan_attach_event = | ctx->ifc_vlan_attach_event = | ||||
EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx, | EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx, | ||||
EVENTHANDLER_PRI_FIRST); | EVENTHANDLER_PRI_FIRST); | ||||
ctx->ifc_vlan_detach_event = | ctx->ifc_vlan_detach_event = | ||||
▲ Show 20 Lines • Show All 1,407 Lines • Show Last 20 Lines |