Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/iflib.c
Show First 20 Lines • Show All 839 Lines • ▼ Show 20 Lines | for (int tmp_pidx = 0; tmp_pidx < IFLIB_MAX_RX_REFRESH && nm_i != head; tmp_pidx++) { | ||||
nic_i = netmap_idx_k2n(kring, nm_i); | nic_i = netmap_idx_k2n(kring, nm_i); | ||||
MPASS(tmp_pidx < IFLIB_MAX_RX_REFRESH); | MPASS(tmp_pidx < IFLIB_MAX_RX_REFRESH); | ||||
if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ | if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ | ||||
return netmap_ring_reinit(kring); | return netmap_ring_reinit(kring); | ||||
fl->ifl_vm_addrs[tmp_pidx] = addr; | fl->ifl_vm_addrs[tmp_pidx] = addr; | ||||
if (__predict_false(init) && map) { | if (__predict_false(init)) { | ||||
netmap_load_map(na, fl->ifl_ifdi->idi_tag, map[nic_i], addr); | netmap_load_map(na, fl->ifl_buf_tag, | ||||
} else if (map && (slot->flags & NS_BUF_CHANGED)) { | map[nic_i], addr); | ||||
} else if (slot->flags & NS_BUF_CHANGED) { | |||||
/* buffer has changed, reload map */ | /* buffer has changed, reload map */ | ||||
netmap_reload_map(na, fl->ifl_ifdi->idi_tag, map[nic_i], addr); | netmap_reload_map(na, fl->ifl_buf_tag, | ||||
map[nic_i], addr); | |||||
} | } | ||||
slot->flags &= ~NS_BUF_CHANGED; | slot->flags &= ~NS_BUF_CHANGED; | ||||
nm_i = nm_next(nm_i, lim); | nm_i = nm_next(nm_i, lim); | ||||
fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim); | fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim); | ||||
if (nm_i != head && tmp_pidx < IFLIB_MAX_RX_REFRESH-1) | if (nm_i != head && tmp_pidx < IFLIB_MAX_RX_REFRESH-1) | ||||
continue; | continue; | ||||
iru.iru_pidx = refill_pidx; | iru.iru_pidx = refill_pidx; | ||||
iru.iru_count = tmp_pidx+1; | iru.iru_count = tmp_pidx+1; | ||||
ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | ||||
refill_pidx = nic_i; | refill_pidx = nic_i; | ||||
if (map == NULL) | |||||
continue; | |||||
for (int n = 0; n < iru.iru_count; n++) { | for (int n = 0; n < iru.iru_count; n++) { | ||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, map[nic_i_dma], | bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i_dma], | ||||
BUS_DMASYNC_PREREAD); | BUS_DMASYNC_PREREAD); | ||||
/* XXX - change this to not use the netmap func*/ | /* XXX - change this to not use the netmap func*/ | ||||
nic_i_dma = nm_next(nic_i_dma, lim); | nic_i_dma = nm_next(nic_i_dma, lim); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
kring->nr_hwcur = head; | kring->nr_hwcur = head; | ||||
if (map) | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
if (__predict_true(nic_i != UINT_MAX)) { | if (__predict_true(nic_i != UINT_MAX)) { | ||||
ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i); | ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i); | ||||
DBG_COUNTER_INC(rxd_flush); | DBG_COUNTER_INC(rxd_flush); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
Show All 27 Lines | iflib_netmap_txsync(struct netmap_kring *kring, int flags) | ||||
* interrupts on every tx packet are expensive so request | * interrupts on every tx packet are expensive so request | ||||
* them every half ring, or where NS_REPORT is set | * them every half ring, or where NS_REPORT is set | ||||
*/ | */ | ||||
u_int report_frequency = kring->nkr_num_slots >> 1; | u_int report_frequency = kring->nkr_num_slots >> 1; | ||||
/* device-specific */ | /* device-specific */ | ||||
if_ctx_t ctx = ifp->if_softc; | if_ctx_t ctx = ifp->if_softc; | ||||
iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id]; | iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id]; | ||||
bus_dmamap_sync(txq->ift_buf_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
/* | /* | ||||
* First part: process new packets to send. | * First part: process new packets to send. | ||||
* nm_i is the current index in the netmap kring, | * nm_i is the current index in the netmap kring, | ||||
* nic_i is the corresponding index in the NIC ring. | * nic_i is the corresponding index in the NIC ring. | ||||
* | * | ||||
* If we have packets to send (nm_i != head) | * If we have packets to send (nm_i != head) | ||||
Show All 17 Lines | iflib_netmap_txsync(struct netmap_kring *kring, int flags) | ||||
if (nm_i != head) { /* we have new packets to send */ | if (nm_i != head) { /* we have new packets to send */ | ||||
pkt_info_zero(&pi); | pkt_info_zero(&pi); | ||||
pi.ipi_segs = txq->ift_segs; | pi.ipi_segs = txq->ift_segs; | ||||
pi.ipi_qsidx = kring->ring_id; | pi.ipi_qsidx = kring->ring_id; | ||||
nic_i = netmap_idx_k2n(kring, nm_i); | nic_i = netmap_idx_k2n(kring, nm_i); | ||||
__builtin_prefetch(&ring->slot[nm_i]); | __builtin_prefetch(&ring->slot[nm_i]); | ||||
__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]); | __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]); | ||||
if (txq->ift_sds.ifsd_map) | |||||
__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]); | __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]); | ||||
for (n = 0; nm_i != head; n++) { | for (n = 0; nm_i != head; n++) { | ||||
struct netmap_slot *slot = &ring->slot[nm_i]; | struct netmap_slot *slot = &ring->slot[nm_i]; | ||||
u_int len = slot->len; | u_int len = slot->len; | ||||
uint64_t paddr; | uint64_t paddr; | ||||
void *addr = PNMB(na, slot, &paddr); | void *addr = PNMB(na, slot, &paddr); | ||||
int flags = (slot->flags & NS_REPORT || | int flags = (slot->flags & NS_REPORT || | ||||
nic_i == 0 || nic_i == report_frequency) ? | nic_i == 0 || nic_i == report_frequency) ? | ||||
Show All 10 Lines | for (n = 0; nm_i != head; n++) { | ||||
/* Fill the slot in the NIC ring. */ | /* Fill the slot in the NIC ring. */ | ||||
ctx->isc_txd_encap(ctx->ifc_softc, &pi); | ctx->isc_txd_encap(ctx->ifc_softc, &pi); | ||||
DBG_COUNTER_INC(tx_encap); | DBG_COUNTER_INC(tx_encap); | ||||
/* prefetch for next round */ | /* prefetch for next round */ | ||||
__builtin_prefetch(&ring->slot[nm_i + 1]); | __builtin_prefetch(&ring->slot[nm_i + 1]); | ||||
__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]); | __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]); | ||||
if (txq->ift_sds.ifsd_map) { | |||||
__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]); | __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]); | ||||
NM_CHECK_ADDR_LEN(na, addr, len); | NM_CHECK_ADDR_LEN(na, addr, len); | ||||
if (slot->flags & NS_BUF_CHANGED) { | if (slot->flags & NS_BUF_CHANGED) { | ||||
/* buffer has changed, reload map */ | /* buffer has changed, reload map */ | ||||
netmap_reload_map(na, txq->ift_buf_tag, | netmap_reload_map(na, txq->ift_buf_tag, | ||||
txq->ift_sds.ifsd_map[nic_i], addr); | txq->ift_sds.ifsd_map[nic_i], addr); | ||||
} | } | ||||
/* make sure changes to the buffer are synced */ | /* make sure changes to the buffer are synced */ | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds.ifsd_map[nic_i], | bus_dmamap_sync(txq->ift_buf_tag, | ||||
txq->ift_sds.ifsd_map[nic_i], | |||||
BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREWRITE); | ||||
} | |||||
slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); | slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); | ||||
nm_i = nm_next(nm_i, lim); | nm_i = nm_next(nm_i, lim); | ||||
nic_i = nm_next(nic_i, lim); | nic_i = nm_next(nic_i, lim); | ||||
} | } | ||||
kring->nr_hwcur = nm_i; | kring->nr_hwcur = nm_i; | ||||
/* synchronize the NIC ring */ | /* synchronize the NIC ring */ | ||||
bus_dmamap_sync(txq->ift_buf_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
/* (re)start the tx unit up to slot nic_i (excluded) */ | /* (re)start the tx unit up to slot nic_i (excluded) */ | ||||
ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i); | ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i); | ||||
} | } | ||||
/* | /* | ||||
* Second part: reclaim buffers for completed transmissions. | * Second part: reclaim buffers for completed transmissions. | ||||
Show All 31 Lines | |||||
* If (flags & NAF_FORCE_READ) also check for incoming packets irrespective | * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective | ||||
* of whether or not we received an interrupt. | * of whether or not we received an interrupt. | ||||
*/ | */ | ||||
static int | static int | ||||
iflib_netmap_rxsync(struct netmap_kring *kring, int flags) | iflib_netmap_rxsync(struct netmap_kring *kring, int flags) | ||||
{ | { | ||||
struct netmap_adapter *na = kring->na; | struct netmap_adapter *na = kring->na; | ||||
struct netmap_ring *ring = kring->ring; | struct netmap_ring *ring = kring->ring; | ||||
iflib_fl_t fl; | |||||
uint32_t nm_i; /* index into the netmap ring */ | uint32_t nm_i; /* index into the netmap ring */ | ||||
uint32_t nic_i; /* index into the NIC ring */ | uint32_t nic_i; /* index into the NIC ring */ | ||||
u_int i, n; | u_int i, n; | ||||
u_int const lim = kring->nkr_num_slots - 1; | u_int const lim = kring->nkr_num_slots - 1; | ||||
u_int const head = kring->rhead; | u_int const head = kring->rhead; | ||||
int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; | int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; | ||||
struct if_rxd_info ri; | struct if_rxd_info ri; | ||||
struct ifnet *ifp = na->ifp; | struct ifnet *ifp = na->ifp; | ||||
if_ctx_t ctx = ifp->if_softc; | if_ctx_t ctx = ifp->if_softc; | ||||
iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id]; | iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id]; | ||||
iflib_fl_t fl = rxq->ifr_fl; | |||||
if (head > lim) | if (head > lim) | ||||
return netmap_ring_reinit(kring); | return netmap_ring_reinit(kring); | ||||
/* XXX check sync modes */ | /* | ||||
* XXX netmap_fl_refill() only ever (re)fills free list 0 so far. | |||||
*/ | |||||
for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) { | for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) { | ||||
if (fl->ifl_sds.ifsd_map == NULL) | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | ||||
continue; | |||||
bus_dmamap_sync(rxq->ifr_fl[i].ifl_buf_tag, | |||||
fl->ifl_ifdi->idi_map, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
} | } | ||||
/* | /* | ||||
* First part: import newly received packets. | * First part: import newly received packets. | ||||
* | * | ||||
* nm_i is the index of the next free slot in the netmap ring, | * nm_i is the index of the next free slot in the netmap ring, | ||||
* nic_i is the index of the next received packet in the NIC ring, | * nic_i is the index of the next received packet in the NIC ring, | ||||
* and they may differ in case if_init() has been called while | * and they may differ in case if_init() has been called while | ||||
* in netmap mode. For the receive ring we have | * in netmap mode. For the receive ring we have | ||||
* | * | ||||
* nic_i = rxr->next_check; | * nic_i = rxr->next_check; | ||||
* nm_i = kring->nr_hwtail (previous) | * nm_i = kring->nr_hwtail (previous) | ||||
* and | * and | ||||
* nm_i == (nic_i + kring->nkr_hwofs) % ring_size | * nm_i == (nic_i + kring->nkr_hwofs) % ring_size | ||||
* | * | ||||
* rxr->next_check is set to 0 on a ring reinit | * rxr->next_check is set to 0 on a ring reinit | ||||
*/ | */ | ||||
if (netmap_no_pendintr || force_update) { | if (netmap_no_pendintr || force_update) { | ||||
int crclen = iflib_crcstrip ? 0 : 4; | int crclen = iflib_crcstrip ? 0 : 4; | ||||
int error, avail; | int error, avail; | ||||
for (i = 0; i < rxq->ifr_nfl; i++) { | for (i = 0; i < rxq->ifr_nfl; i++) { | ||||
fl = &rxq->ifr_fl[i]; | fl = &rxq->ifr_fl[i]; | ||||
nic_i = fl->ifl_cidx; | nic_i = fl->ifl_cidx; | ||||
nm_i = netmap_idx_n2k(kring, nic_i); | nm_i = netmap_idx_n2k(kring, nic_i); | ||||
avail = iflib_rxd_avail(ctx, rxq, nic_i, USHRT_MAX); | avail = ctx->isc_rxd_available(ctx->ifc_softc, | ||||
rxq->ifr_id, nic_i, USHRT_MAX); | |||||
for (n = 0; avail > 0; n++, avail--) { | for (n = 0; avail > 0; n++, avail--) { | ||||
rxd_info_zero(&ri); | rxd_info_zero(&ri); | ||||
ri.iri_frags = rxq->ifr_frags; | ri.iri_frags = rxq->ifr_frags; | ||||
ri.iri_qsidx = kring->ring_id; | ri.iri_qsidx = kring->ring_id; | ||||
ri.iri_ifp = ctx->ifc_ifp; | ri.iri_ifp = ctx->ifc_ifp; | ||||
ri.iri_cidx = nic_i; | ri.iri_cidx = nic_i; | ||||
error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | ||||
ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen; | ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen; | ||||
ring->slot[nm_i].flags = 0; | ring->slot[nm_i].flags = 0; | ||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, | bus_dmamap_sync(fl->ifl_buf_tag, | ||||
fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); | fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); | ||||
nm_i = nm_next(nm_i, lim); | nm_i = nm_next(nm_i, lim); | ||||
nic_i = nm_next(nic_i, lim); | nic_i = nm_next(nic_i, lim); | ||||
} | } | ||||
if (n) { /* update the state variables */ | if (n) { /* update the state variables */ | ||||
if (netmap_no_pendintr && !force_update) { | if (netmap_no_pendintr && !force_update) { | ||||
/* diagnostics */ | /* diagnostics */ | ||||
iflib_rx_miss ++; | iflib_rx_miss ++; | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq) | ||||
slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0); | slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0); | ||||
if (slot == NULL) | if (slot == NULL) | ||||
return; | return; | ||||
nm_i = netmap_idx_n2k(kring, 0); | nm_i = netmap_idx_n2k(kring, 0); | ||||
netmap_fl_refill(rxq, kring, nm_i, true); | netmap_fl_refill(rxq, kring, nm_i, true); | ||||
} | } | ||||
static void | static void | ||||
iflib_netmap_timer_adjust(if_ctx_t ctx, uint16_t txqid, uint32_t *reset_on) | iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on) | ||||
{ | { | ||||
struct netmap_kring *kring; | struct netmap_kring *kring; | ||||
uint16_t txqid; | |||||
txqid = txq->ift_id; | |||||
kring = NA(ctx->ifc_ifp)->tx_rings[txqid]; | kring = NA(ctx->ifc_ifp)->tx_rings[txqid]; | ||||
if (kring->nr_hwcur != nm_next(kring->nr_hwtail, kring->nkr_num_slots - 1)) { | if (kring->nr_hwcur != nm_next(kring->nr_hwtail, kring->nkr_num_slots - 1)) { | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | |||||
BUS_DMASYNC_POSTREAD); | |||||
if (ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) | if (ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) | ||||
netmap_tx_irq(ctx->ifc_ifp, txqid); | netmap_tx_irq(ctx->ifc_ifp, txqid); | ||||
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) { | if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) { | ||||
if (hz < 2000) | if (hz < 2000) | ||||
*reset_on = 1; | *reset_on = 1; | ||||
else | else | ||||
*reset_on = hz / 1000; | *reset_on = hz / 1000; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
#define iflib_netmap_detach(ifp) netmap_detach(ifp) | #define iflib_netmap_detach(ifp) netmap_detach(ifp) | ||||
#else | #else | ||||
#define iflib_netmap_txq_init(ctx, txq) | #define iflib_netmap_txq_init(ctx, txq) | ||||
#define iflib_netmap_rxq_init(ctx, rxq) | #define iflib_netmap_rxq_init(ctx, rxq) | ||||
#define iflib_netmap_detach(ifp) | #define iflib_netmap_detach(ifp) | ||||
#define iflib_netmap_attach(ctx) (0) | #define iflib_netmap_attach(ctx) (0) | ||||
#define netmap_rx_irq(ifp, qid, budget) (0) | #define netmap_rx_irq(ifp, qid, budget) (0) | ||||
#define netmap_tx_irq(ifp, qid) do {} while (0) | #define netmap_tx_irq(ifp, qid) do {} while (0) | ||||
#define iflib_netmap_timer_adjust(ctx, txqid, reset_on) | #define iflib_netmap_timer_adjust(ctx, txq, reset_on) | ||||
#endif | #endif | ||||
#if defined(__i386__) || defined(__amd64__) | #if defined(__i386__) || defined(__amd64__) | ||||
static __inline void | static __inline void | ||||
prefetch(void *x) | prefetch(void *x) | ||||
{ | { | ||||
__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); | __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); | ||||
▲ Show 20 Lines • Show All 216 Lines • ▼ Show 20 Lines | iflib_fast_intr(void *arg) | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
} | } | ||||
static int | static int | ||||
iflib_fast_intr_rxtx(void *arg) | iflib_fast_intr_rxtx(void *arg) | ||||
{ | { | ||||
iflib_filter_info_t info = arg; | iflib_filter_info_t info = arg; | ||||
struct grouptask *gtask = info->ifi_task; | struct grouptask *gtask = info->ifi_task; | ||||
if_ctx_t ctx; | |||||
iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; | iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; | ||||
if_ctx_t ctx = NULL;; | iflib_txq_t txq; | ||||
void *sc; | |||||
int i, cidx; | int i, cidx; | ||||
qidx_t txqid; | |||||
if (!iflib_started) | if (!iflib_started) | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
DBG_COUNTER_INC(fast_intrs); | DBG_COUNTER_INC(fast_intrs); | ||||
if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED) | if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED) | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
ctx = rxq->ifr_ctx; | |||||
sc = ctx->ifc_softc; | |||||
MPASS(rxq->ifr_ntxqirq); | MPASS(rxq->ifr_ntxqirq); | ||||
for (i = 0; i < rxq->ifr_ntxqirq; i++) { | for (i = 0; i < rxq->ifr_ntxqirq; i++) { | ||||
qidx_t txqid = rxq->ifr_txqid[i]; | txqid = rxq->ifr_txqid[i]; | ||||
txq = &ctx->ifc_txqs[txqid]; | |||||
ctx = rxq->ifr_ctx; | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
bus_dmamap_sync(rxq->ifr_ifdi->idi_tag, rxq->ifr_ifdi->idi_map, | |||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
if (!ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) { | if (!ctx->isc_txd_credits_update(sc, txqid, false)) { | ||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid); | ||||
continue; | continue; | ||||
} | } | ||||
GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
} | } | ||||
if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) | if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) | ||||
cidx = rxq->ifr_cq_cidx; | cidx = rxq->ifr_cq_cidx; | ||||
else | else | ||||
cidx = rxq->ifr_fl[0].ifl_cidx; | cidx = rxq->ifr_fl[0].ifl_cidx; | ||||
if (iflib_rxd_avail(ctx, rxq, cidx, 1)) | if (iflib_rxd_avail(ctx, rxq, cidx, 1)) | ||||
GROUPTASK_ENQUEUE(gtask); | GROUPTASK_ENQUEUE(gtask); | ||||
else { | else { | ||||
▲ Show 20 Lines • Show All 282 Lines • ▼ Show 20 Lines | iflib_txq_setup(iflib_txq_t txq) | ||||
txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset]; | txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset]; | ||||
for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) | for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) | ||||
bzero((void *)di->idi_vaddr, di->idi_size); | bzero((void *)di->idi_vaddr, di->idi_size); | ||||
IFDI_TXQ_SETUP(ctx, txq->ift_id); | IFDI_TXQ_SETUP(ctx, txq->ift_id); | ||||
for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) | for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) | ||||
bus_dmamap_sync(di->idi_tag, di->idi_map, | bus_dmamap_sync(di->idi_tag, di->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
return (0); | return (0); | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Allocate DMA resources for RX buffers as well as memory for the RX | * Allocate DMA resources for RX buffers as well as memory for the RX | ||||
* mbuf map, direct RX cluster pointer map and RX cluster bus address | * mbuf map, direct RX cluster pointer map and RX cluster bus address | ||||
* map. RX DMA map, RX mbuf map, direct RX cluster pointer map and | * map. RX DMA map, RX mbuf map, direct RX cluster pointer map and | ||||
▲ Show 20 Lines • Show All 187 Lines • ▼ Show 20 Lines | if ((cl = sd_cl[frag_idx]) == NULL) { | ||||
/* | /* | ||||
* !zone_pack ? | * !zone_pack ? | ||||
*/ | */ | ||||
if (fl->ifl_zone == zone_pack) | if (fl->ifl_zone == zone_pack) | ||||
uma_zfree(fl->ifl_zone, cl); | uma_zfree(fl->ifl_zone, cl); | ||||
break; | break; | ||||
} | } | ||||
bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], | |||||
BUS_DMASYNC_PREREAD); | |||||
sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; | sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; | ||||
sd_cl[frag_idx] = cl; | sd_cl[frag_idx] = cl; | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
fl->ifl_cl_enqueued++; | fl->ifl_cl_enqueued++; | ||||
#endif | #endif | ||||
} else { | } else { | ||||
bus_addr = sd_ba[frag_idx]; | bus_addr = sd_ba[frag_idx]; | ||||
} | } | ||||
bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], | |||||
BUS_DMASYNC_PREREAD); | |||||
MPASS(sd_m[frag_idx] == NULL); | MPASS(sd_m[frag_idx] == NULL); | ||||
if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { | if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { | ||||
break; | break; | ||||
} | } | ||||
sd_m[frag_idx] = m; | sd_m[frag_idx] = m; | ||||
bit_set(fl->ifl_rx_bitmap, frag_idx); | bit_set(fl->ifl_rx_bitmap, frag_idx); | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
▲ Show 20 Lines • Show All 251 Lines • ▼ Show 20 Lines | if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) && | ||||
goto hung; | goto hung; | ||||
if (ifmp_ring_is_stalled(txq->ift_br)) | if (ifmp_ring_is_stalled(txq->ift_br)) | ||||
txq->ift_qstatus = IFLIB_QUEUE_HUNG; | txq->ift_qstatus = IFLIB_QUEUE_HUNG; | ||||
txq->ift_cleaned_prev = txq->ift_cleaned; | txq->ift_cleaned_prev = txq->ift_cleaned; | ||||
} | } | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) | if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) | ||||
iflib_netmap_timer_adjust(ctx, txq->ift_id, &reset_on); | iflib_netmap_timer_adjust(ctx, txq, &reset_on); | ||||
#endif | #endif | ||||
/* handle any laggards */ | /* handle any laggards */ | ||||
if (txq->ift_db_pending) | if (txq->ift_db_pending) | ||||
GROUPTASK_ENQUEUE(&txq->ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
sctx->isc_pause_frames = 0; | sctx->isc_pause_frames = 0; | ||||
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) | if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) | ||||
callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); | callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); | ||||
▲ Show 20 Lines • Show All 192 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static void | static void | ||||
rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int unload, if_rxsd_t sd) | rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int unload, if_rxsd_t sd) | ||||
{ | { | ||||
int flid, cidx; | int flid, cidx; | ||||
bus_dmamap_t map; | bus_dmamap_t map; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
iflib_dma_info_t di; | |||||
int next; | int next; | ||||
map = NULL; | map = NULL; | ||||
flid = irf->irf_flid; | flid = irf->irf_flid; | ||||
cidx = irf->irf_idx; | cidx = irf->irf_idx; | ||||
fl = &rxq->ifr_fl[flid]; | fl = &rxq->ifr_fl[flid]; | ||||
sd->ifsd_fl = fl; | sd->ifsd_fl = fl; | ||||
sd->ifsd_cidx = cidx; | sd->ifsd_cidx = cidx; | ||||
sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx]; | sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx]; | ||||
sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; | sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; | ||||
fl->ifl_credits--; | fl->ifl_credits--; | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
fl->ifl_m_dequeued++; | fl->ifl_m_dequeued++; | ||||
#endif | #endif | ||||
if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) | if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) | ||||
prefetch_pkts(fl, cidx); | prefetch_pkts(fl, cidx); | ||||
next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); | next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); | ||||
prefetch(&fl->ifl_sds.ifsd_map[next]); | prefetch(&fl->ifl_sds.ifsd_map[next]); | ||||
map = fl->ifl_sds.ifsd_map[cidx]; | map = fl->ifl_sds.ifsd_map[cidx]; | ||||
di = fl->ifl_ifdi; | |||||
next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); | next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); | ||||
/* not valid assert if bxe really does SGE from non-contiguous elements */ | /* not valid assert if bxe really does SGE from non-contiguous elements */ | ||||
MPASS(fl->ifl_cidx == cidx); | MPASS(fl->ifl_cidx == cidx); | ||||
bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); | bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); | ||||
if (unload) | if (unload) | ||||
bus_dmamap_unload(fl->ifl_buf_tag, map); | bus_dmamap_unload(fl->ifl_buf_tag, map); | ||||
fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | ||||
if (__predict_false(fl->ifl_cidx == 0)) | if (__predict_false(fl->ifl_cidx == 0)) | ||||
fl->ifl_gen = 0; | fl->ifl_gen = 0; | ||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
bit_clear(fl->ifl_rx_bitmap, cidx); | bit_clear(fl->ifl_rx_bitmap, cidx); | ||||
} | } | ||||
static struct mbuf * | static struct mbuf * | ||||
assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd) | assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd) | ||||
{ | { | ||||
int i, padlen , flags; | int i, padlen , flags; | ||||
struct mbuf *m, *mh, *mt; | struct mbuf *m, *mh, *mt; | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | if (ri->iri_nfrags == 1 && | ||||
m = *sd.ifsd_m; | m = *sd.ifsd_m; | ||||
*sd.ifsd_m = NULL; | *sd.ifsd_m = NULL; | ||||
m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR); | m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR); | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
if (!IP_ALIGNED(m)) | if (!IP_ALIGNED(m)) | ||||
m->m_data += 2; | m->m_data += 2; | ||||
#endif | #endif | ||||
memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); | memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); | ||||
bus_dmamap_sync(rxq->ifr_fl->ifl_buf_tag, | |||||
rxq->ifr_fl->ifl_sds.ifsd_map[ri->iri_frags[0].irf_idx], | |||||
BUS_DMASYNC_PREREAD); | |||||
m->m_len = ri->iri_frags[0].irf_len; | m->m_len = ri->iri_frags[0].irf_len; | ||||
} else { | } else { | ||||
m = assemble_segments(rxq, ri, &sd); | m = assemble_segments(rxq, ri, &sd); | ||||
} | } | ||||
m->m_pkthdr.len = ri->iri_len; | m->m_pkthdr.len = ri->iri_len; | ||||
m->m_pkthdr.rcvif = ri->iri_ifp; | m->m_pkthdr.rcvif = ri->iri_ifp; | ||||
m->m_flags |= ri->iri_flags; | m->m_flags |= ri->iri_flags; | ||||
m->m_pkthdr.ether_vtag = ri->iri_vtag; | m->m_pkthdr.ether_vtag = ri->iri_vtag; | ||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
static bool | static bool | ||||
iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) | iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) | ||||
{ | { | ||||
if_ctx_t ctx = rxq->ifr_ctx; | if_ctx_t ctx = rxq->ifr_ctx; | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
iflib_dma_info_t di; | |||||
int avail, i; | int avail, i; | ||||
qidx_t *cidxp; | qidx_t *cidxp; | ||||
struct if_rxd_info ri; | struct if_rxd_info ri; | ||||
int err, budget_left, rx_bytes, rx_pkts; | int err, budget_left, rx_bytes, rx_pkts; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
int lro_enabled; | int lro_enabled; | ||||
bool v4_forwarding, v6_forwarding, lro_possible; | bool v4_forwarding, v6_forwarding, lro_possible; | ||||
Show All 28 Lines | for (budget_left = budget; budget_left > 0 && avail > 0;) { | ||||
/* | /* | ||||
* Reset client set fields to their default values | * Reset client set fields to their default values | ||||
*/ | */ | ||||
rxd_info_zero(&ri); | rxd_info_zero(&ri); | ||||
ri.iri_qsidx = rxq->ifr_id; | ri.iri_qsidx = rxq->ifr_id; | ||||
ri.iri_cidx = *cidxp; | ri.iri_cidx = *cidxp; | ||||
ri.iri_ifp = ifp; | ri.iri_ifp = ifp; | ||||
ri.iri_frags = rxq->ifr_frags; | ri.iri_frags = rxq->ifr_frags; | ||||
di = rxq->ifr_fl[rxq->ifr_frags[0].irf_flid].ifl_ifdi; | |||||
bus_dmamap_sync(di->idi_tag, di->idi_map, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | ||||
if (err) | if (err) | ||||
goto err; | goto err; | ||||
if (sctx->isc_flags & IFLIB_HAS_RXCQ) { | if (sctx->isc_flags & IFLIB_HAS_RXCQ) { | ||||
*cidxp = ri.iri_cidx; | *cidxp = ri.iri_cidx; | ||||
/* Update our consumer index */ | /* Update our consumer index */ | ||||
/* XXX NB: shurd - check if this is still safe */ | /* XXX NB: shurd - check if this is still safe */ | ||||
▲ Show 20 Lines • Show All 152 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
qidx_t dbval, max; | qidx_t dbval, max; | ||||
bool rang; | bool rang; | ||||
rang = false; | rang = false; | ||||
max = TXQ_MAX_DB_DEFERRED(txq, in_use); | max = TXQ_MAX_DB_DEFERRED(txq, in_use); | ||||
if (ring || txq->ift_db_pending >= max) { | if (ring || txq->ift_db_pending >= max) { | ||||
dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx; | dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx; | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval); | ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval); | ||||
txq->ift_db_pending = txq->ift_npending = 0; | txq->ift_db_pending = txq->ift_npending = 0; | ||||
rang = true; | rang = true; | ||||
} | } | ||||
return (rang); | return (rang); | ||||
} | } | ||||
#ifdef PKT_DEBUG | #ifdef PKT_DEBUG | ||||
▲ Show 20 Lines • Show All 426 Lines • ▼ Show 20 Lines | defrag: | ||||
pi.ipi_segs = segs; | pi.ipi_segs = segs; | ||||
pi.ipi_nsegs = nsegs; | pi.ipi_nsegs = nsegs; | ||||
MPASS(pidx >= 0 && pidx < txq->ift_size); | MPASS(pidx >= 0 && pidx < txq->ift_size); | ||||
#ifdef PKT_DEBUG | #ifdef PKT_DEBUG | ||||
print_pkt(&pi); | print_pkt(&pi); | ||||
#endif | #endif | ||||
bus_dmamap_sync(buf_tag, map, BUS_DMASYNC_PREWRITE); | |||||
if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) { | if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) { | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(buf_tag, map, BUS_DMASYNC_PREWRITE); | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
DBG_COUNTER_INC(tx_encap); | DBG_COUNTER_INC(tx_encap); | ||||
MPASS(pi.ipi_new_pidx < txq->ift_size); | MPASS(pi.ipi_new_pidx < txq->ift_size); | ||||
ndesc = pi.ipi_new_pidx - pi.ipi_pidx; | ndesc = pi.ipi_new_pidx - pi.ipi_pidx; | ||||
if (pi.ipi_new_pidx < pi.ipi_pidx) { | if (pi.ipi_new_pidx < pi.ipi_pidx) { | ||||
ndesc += txq->ift_size; | ndesc += txq->ift_size; | ||||
txq->ift_gen = 1; | txq->ift_gen = 1; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 152 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static uint32_t | static uint32_t | ||||
iflib_txq_can_drain(struct ifmp_ring *r) | iflib_txq_can_drain(struct ifmp_ring *r) | ||||
{ | { | ||||
iflib_txq_t txq = r->cookie; | iflib_txq_t txq = r->cookie; | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) | |||||
return (1); | |||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
return ((TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) || | return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, | ||||
ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)); | false)); | ||||
} | } | ||||
static uint32_t | static uint32_t | ||||
iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx) | iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx) | ||||
{ | { | ||||
iflib_txq_t txq = r->cookie; | iflib_txq_t txq = r->cookie; | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
struct ifnet *ifp = ctx->ifc_ifp; | struct ifnet *ifp = ctx->ifc_ifp; | ||||
▲ Show 20 Lines • Show All 137 Lines • ▼ Show 20 Lines | iflib_ifmp_purge(iflib_txq_t txq) | ||||
r->can_drain = iflib_txq_can_drain; | r->can_drain = iflib_txq_can_drain; | ||||
} | } | ||||
static void | static void | ||||
_task_fn_tx(void *context) | _task_fn_tx(void *context) | ||||
{ | { | ||||
iflib_txq_t txq = context; | iflib_txq_t txq = context; | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
struct ifnet *ifp = ctx->ifc_ifp; | |||||
int abdicate = ctx->ifc_sysctl_tx_abdicate; | int abdicate = ctx->ifc_sysctl_tx_abdicate; | ||||
#ifdef IFLIB_DIAGNOSTICS | #ifdef IFLIB_DIAGNOSTICS | ||||
txq->ift_cpu_exec_count[curcpu]++; | txq->ift_cpu_exec_count[curcpu]++; | ||||
#endif | #endif | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | ||||
return; | return; | ||||
if (if_getcapenable(ifp) & IFCAP_NETMAP) { | #ifdef DEV_NETMAP | ||||
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) { | |||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)) | if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)) | ||||
netmap_tx_irq(ifp, txq->ift_id); | netmap_tx_irq(ctx->ifc_ifp, txq->ift_id); | ||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | ||||
return; | return; | ||||
} | } | ||||
#endif | |||||
#ifdef ALTQ | #ifdef ALTQ | ||||
if (ALTQ_IS_ENABLED(&ifp->if_snd)) | if (ALTQ_IS_ENABLED(&ifp->if_snd)) | ||||
iflib_altq_if_start(ifp); | iflib_altq_if_start(ifp); | ||||
#endif | #endif | ||||
if (txq->ift_db_pending) | if (txq->ift_db_pending) | ||||
ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); | ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); | ||||
else if (!abdicate) | else if (!abdicate) | ||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ||||
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | if (do_watchdog) { | ||||
ctx->ifc_watchdog_events++; | ctx->ifc_watchdog_events++; | ||||
IFDI_WATCHDOG_RESET(ctx); | IFDI_WATCHDOG_RESET(ctx); | ||||
} | } | ||||
IFDI_UPDATE_ADMIN_STATUS(ctx); | IFDI_UPDATE_ADMIN_STATUS(ctx); | ||||
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { | for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
reset_on = hz / 2; | reset_on = hz / 2; | ||||
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) | if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) | ||||
iflib_netmap_timer_adjust(ctx, txq->ift_id, &reset_on); | iflib_netmap_timer_adjust(ctx, txq, &reset_on); | ||||
#endif | #endif | ||||
callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); | callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); | ||||
} | } | ||||
IFDI_LINK_INTR_ENABLE(ctx); | IFDI_LINK_INTR_ENABLE(ctx); | ||||
if (do_reset) | if (do_reset) | ||||
iflib_if_init_locked(ctx); | iflib_if_init_locked(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
▲ Show 20 Lines • Show All 2,151 Lines • ▼ Show 20 Lines | #endif | ||||
if (txq->ift_cidx_processed >= txq->ift_size) | if (txq->ift_cidx_processed >= txq->ift_size) | ||||
txq->ift_cidx_processed -= txq->ift_size; | txq->ift_cidx_processed -= txq->ift_size; | ||||
return (credits); | return (credits); | ||||
} | } | ||||
static int | static int | ||||
iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget) | iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget) | ||||
{ | { | ||||
iflib_fl_t fl; | |||||
u_int i; | |||||
for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++) | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx, | return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx, | ||||
budget)); | budget)); | ||||
} | } | ||||
void | void | ||||
iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name, | iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name, | ||||
const char *description, if_int_delay_info_t info, | const char *description, if_int_delay_info_t info, | ||||
int offset, int value) | int offset, int value) | ||||
▲ Show 20 Lines • Show All 559 Lines • Show Last 20 Lines |