Differential D54274 Diff 168280 FreeBSD/sys/net/iflib.c

Changeset View

Standalone View

View Options

FreeBSD/sys/net/iflib.c

Show First 20 Lines • Show All 370 Lines • ▼ Show 20 Lines

#endif

struct mtx ift_mtx;

struct mtx ift_db_mtx;

/* constant values */

if_ctx_t ift_ctx;

struct ifmp_ring *ift_br;

struct grouptask ift_task;

qidx_t ift_size;

qidx_t ift_pad;

uint16_t ift_id;

struct callout ift_timer;

#ifdef DEV_NETMAP

struct callout ift_netmap_timer;

#endif /* DEV_NETMAP */

if_txsd_vec_t ift_sds;

uint8_t ift_qstatus;

▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines

get_inuse(int size, qidx_t cidx, qidx_t pidx, uint8_t gen)

else if (gen == 1 && pidx == cidx)

used = size;

else

panic("bad state");

return (used);

}

#define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))

#define TXQ_AVAIL(txq) ((txq->ift_size - txq->ift_pad) -\

get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))

#define IDXDIFF(head, tail, wrap) \

((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))

struct iflib_rxq {

if_ctx_t ifr_ctx;

iflib_fl_t ifr_fl;

uint64_t ifr_rx_irq;

Show All 23 Lines

typedef struct if_rxsd {

caddr_t *ifsd_cl;

iflib_fl_t ifsd_fl;

} *if_rxsd_t;

/* multiple of word size */

#ifdef __LP64__

#define PKT_INFO_SIZE 6

#define PKT_INFO_SIZE 7

#define RXD_INFO_SIZE 5

#define PKT_TYPE uint64_t

#else

#define PKT_INFO_SIZE 11

#define PKT_INFO_SIZE 12

#define RXD_INFO_SIZE 8

#define PKT_TYPE uint32_t

#endif

#define PKT_LOOP_BOUND ((PKT_INFO_SIZE / 3) * 3)

#define RXD_LOOP_BOUND ((RXD_INFO_SIZE / 4) * 4)

typedef struct if_pkt_info_pad {

PKT_TYPE pkt_val[PKT_INFO_SIZE];

▲ Show 20 Lines • Show All 1,427 Lines • ▼ Show 20 Lines

iflib_txq_setup(iflib_txq_t txq)

txq->ift_qstatus = IFLIB_QUEUE_IDLE;

/* XXX make configurable */

txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ;

/* Reset indices */

txq->ift_cidx_processed = 0;

txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;

txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset];

txq->ift_pad = scctx->isc_tx_pad;

for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)

bzero((void *)di->idi_vaddr, di->idi_size);

IFDI_TXQ_SETUP(ctx, txq->ift_id);

for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)

bus_dmamap_sync(di->idi_tag, di->idi_map,

BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

▲ Show 20 Lines • Show All 1,152 Lines • ▼ Show 20 Lines

iflib_txd_db_check(iflib_txq_t txq, int ring)

{

if_ctx_t ctx = txq->ift_ctx;

qidx_t dbval, max;

max = TXQ_MAX_DB_DEFERRED(txq, txq->ift_in_use);

/* force || threshold exceeded || at the edge of the ring */

if (ring || (txq->ift_db_pending >= max) || (TXQ_AVAIL(txq) <= MAX_TX_DESC(ctx) + 2)) {

if (ring || (txq->ift_db_pending >= max) || (TXQ_AVAIL(txq) <= MAX_TX_DESC(ctx))) {

* 'npending' is used if the card's doorbell is in terms of the number of descriptors

* pending flush (BRCM). 'pidx' is used in cases where the card's doorbeel uses the

* producer index explicitly (INTC).

dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;

bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,

▲ Show 20 Lines • Show All 487 Lines • ▼ Show 20 Lines

default:

break;

}

txq->ift_map_failed++;

DBG_COUNTER_INC(encap_load_mbuf_fail);

DBG_COUNTER_INC(encap_txd_encap_fail);

return (err);

}

ifsd_m[pidx] = m_head;

if (m_head->m_pkthdr.csum_flags & CSUM_SND_TAG)

pi.ipi_mbuf = m_head;

else

pi.ipi_mbuf = NULL;

* XXX assumes a 1 to 1 relationship between segments and

* descriptors - this does not hold true on all drivers, e.g.

* cxgb

if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {

if (__predict_false(nsegs > TXQ_AVAIL(txq))) {

(void)iflib_completed_tx_reclaim(txq);

if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {

if (__predict_false(nsegs > TXQ_AVAIL(txq))) {

txq->ift_no_desc_avail++;

bus_dmamap_unload(buf_tag, map);

DBG_COUNTER_INC(encap_txq_avail_fail);

DBG_COUNTER_INC(encap_txd_encap_fail);

if (ctx->ifc_sysctl_simple_tx) {

*m_headp = m_head = iflib_remove_mbuf(txq);

m_freem(*m_headp);

DBG_COUNTER_INC(tx_frees);

*m_headp = NULL;

}

if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0)

GROUPTASK_ENQUEUE(&txq->ift_task);

return (ENOBUFS);

}

* On Intel cards we can greatly reduce the number of TX interrupts

* we see by only setting report status on every Nth descriptor.

* However, this also means that the driver will need to keep track

* of the descriptors that RS was set on to check them for the DD bit.

txq->ift_rs_pending += nsegs + 1;

if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) ||

iflib_no_tx_batch || (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx) + 2) {

iflib_no_tx_batch || (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx)) {

pi.ipi_flags |= IPI_TX_INTR;

txq->ift_rs_pending = 0;

}

pi.ipi_segs = segs;

pi.ipi_nsegs = nsegs;

MPASS(pidx >= 0 && pidx < txq->ift_size);

#ifdef PKT_DEBUG

print_pkt(&pi);

#endif

if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) {

bus_dmamap_sync(buf_tag, map, BUS_DMASYNC_PREWRITE);

DBG_COUNTER_INC(tx_encap);

MPASS(pi.ipi_new_pidx < txq->ift_size);

ndesc = pi.ipi_new_pidx - pi.ipi_pidx;

if (pi.ipi_new_pidx < pi.ipi_pidx) {

ndesc += txq->ift_size;

txq->ift_gen = 1;

}

* drivers can need as many as

* drivers can need up to ift_pad sentinals

kgalazkaUnsubmitted

Done

txq->ift_gen = 1;

}

- * drivers can need up to ift_pad sentinals

+ * drivers can need up to ift_pad sentinels

MPASS(ndesc <= pi.ipi_nsegs + txq->ift_pad);

kgalazka:

gallatinAuthorUnsubmitted

Done

Thank you.. Fixed this in my tree..

gallatin: Thank you.. Fixed this in my tree..

* two sentinels

MPASS(ndesc <= pi.ipi_nsegs + 2);

MPASS(ndesc <= pi.ipi_nsegs + txq->ift_pad);

MPASS(pi.ipi_new_pidx != pidx);

MPASS(ndesc > 0);

txq->ift_in_use += ndesc;

txq->ift_db_pending += ndesc;

* We update the last software descriptor again here because there may

* be a sentinel and/or there may be more mbufs than segments

▲ Show 20 Lines • Show All 138 Lines • ▼ Show 20 Lines

}

static uint32_t

iflib_txq_can_drain(struct ifmp_ring *r)

{

iflib_txq_t txq = r->cookie;

if_ctx_t ctx = txq->ift_ctx;

if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2)

if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx))

return (1);

bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,

BUS_DMASYNC_POSTREAD);

return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id,

false));

}

static uint32_t

▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines

iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)

count = MIN(avail, TX_BATCH_SIZE);

#ifdef INVARIANTS

if (iflib_verbose_debug)

printf("%s avail=%d ifc_flags=%x txq_avail=%d ", __func__,

avail, ctx->ifc_flags, TXQ_AVAIL(txq));

#endif

do_prefetch = (ctx->ifc_flags & IFC_PREFETCH);

err = 0;

for (i = 0; i < count && TXQ_AVAIL(txq) >= MAX_TX_DESC(ctx) + 2; i++) {

for (i = 0; i < count && TXQ_AVAIL(txq) >= MAX_TX_DESC(ctx); i++) {

int rem = do_prefetch ? count - i : 0;

mp = _ring_peek_one(r, cidx, i, rem);

MPASS(mp != NULL && *mp != NULL);

* Completion interrupts will use the address of the txq

* as a sentinel to enqueue _something_ in order to acquire

▲ Show 20 Lines • Show All 844 Lines • ▼ Show 20 Lines

if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) {

scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i];

}

if (!powerof2(scctx->isc_ntxd[i])) {

device_printf(dev, "ntxd%d: %d is not a power of 2 - using default value of %d\n",

i, scctx->isc_ntxd[i], sctx->isc_ntxd_default[i]);

scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i];

}

scctx->isc_tx_pad = 2;

}

static void

iflib_add_pfil(if_ctx_t ctx)

{

struct pfil_head *pfil;

struct pfil_head_args pa;

iflib_rxq_t rxq;

▲ Show 20 Lines • Show All 2,474 Lines • Show Last 20 Lines