Changeset View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | |||||
#include <sys/syslog.h> | #include <sys/syslog.h> | ||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/limits.h> | #include <sys/limits.h> | ||||
#include <net/if.h> | #include <net/if.h> | ||||
#include <net/if_var.h> | #include <net/if_var.h> | ||||
#include <net/if_types.h> | #include <net/if_types.h> | ||||
#include <net/if_media.h> | #include <net/if_media.h> | ||||
#include <net/pfil.h> | |||||
marius: Is there a particular reason to not sort this header alphabetically between <net/mp_ring.h> and… | |||||
#include <net/bpf.h> | #include <net/bpf.h> | ||||
#include <net/ethernet.h> | #include <net/ethernet.h> | ||||
#include <net/mp_ring.h> | #include <net/mp_ring.h> | ||||
#include <net/vnet.h> | #include <net/vnet.h> | ||||
#include <netinet/in.h> | #include <netinet/in.h> | ||||
#include <netinet/in_pcb.h> | #include <netinet/in_pcb.h> | ||||
#include <netinet/tcp_lro.h> | #include <netinet/tcp_lro.h> | ||||
▲ Show 20 Lines • Show All 359 Lines • ▼ Show 20 Lines | struct iflib_rxq { | ||||
qidx_t ifr_cq_cidx; | qidx_t ifr_cq_cidx; | ||||
qidx_t ifr_cq_pidx; | qidx_t ifr_cq_pidx; | ||||
uint8_t ifr_cq_gen; | uint8_t ifr_cq_gen; | ||||
uint8_t ifr_fl_offset; | uint8_t ifr_fl_offset; | ||||
if_ctx_t ifr_ctx; | if_ctx_t ifr_ctx; | ||||
iflib_fl_t ifr_fl; | iflib_fl_t ifr_fl; | ||||
uint64_t ifr_rx_irq; | uint64_t ifr_rx_irq; | ||||
struct pfil_head *pfil; | |||||
uint16_t ifr_id; | uint16_t ifr_id; | ||||
uint8_t ifr_lro_enabled; | uint8_t ifr_lro_enabled; | ||||
uint8_t ifr_nfl; | uint8_t ifr_nfl; | ||||
uint8_t ifr_ntxqirq; | uint8_t ifr_ntxqirq; | ||||
uint8_t ifr_txqid[IFLIB_MAX_TX_SHARED_INTR]; | uint8_t ifr_txqid[IFLIB_MAX_TX_SHARED_INTR]; | ||||
struct lro_ctrl ifr_lc; | struct lro_ctrl ifr_lc; | ||||
struct grouptask ifr_task; | struct grouptask ifr_task; | ||||
struct iflib_filter_info ifr_filter_info; | struct iflib_filter_info ifr_filter_info; | ||||
iflib_dma_info_t ifr_ifdi; | iflib_dma_info_t ifr_ifdi; | ||||
/* dynamically allocate if any drivers need a value substantially larger than this */ | /* dynamically allocate if any drivers need a value substantially larger than this */ | ||||
struct if_rxd_frag ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE); | struct if_rxd_frag ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE); | ||||
#ifdef IFLIB_DIAGNOSTICS | #ifdef IFLIB_DIAGNOSTICS | ||||
uint64_t ifr_cpu_exec_count[256]; | uint64_t ifr_cpu_exec_count[256]; | ||||
#endif | #endif | ||||
} __aligned(CACHE_LINE_SIZE); | } __aligned(CACHE_LINE_SIZE); | ||||
typedef struct if_rxsd { | typedef struct if_rxsd { | ||||
caddr_t *ifsd_cl; | caddr_t *ifsd_cl; | ||||
struct mbuf **ifsd_m; | |||||
iflib_fl_t ifsd_fl; | iflib_fl_t ifsd_fl; | ||||
qidx_t ifsd_cidx; | qidx_t ifsd_cidx; | ||||
} *if_rxsd_t; | } *if_rxsd_t; | ||||
/* multiple of word size */ | /* multiple of word size */ | ||||
#ifdef __LP64__ | #ifdef __LP64__ | ||||
#define PKT_INFO_SIZE 6 | #define PKT_INFO_SIZE 6 | ||||
#define RXD_INFO_SIZE 5 | #define RXD_INFO_SIZE 5 | ||||
▲ Show 20 Lines • Show All 1,566 Lines • ▼ Show 20 Lines | #if MEMORY_LOGGING | ||||
fl->ifl_cl_enqueued++; | fl->ifl_cl_enqueued++; | ||||
#endif | #endif | ||||
} else { | } else { | ||||
bus_addr = sd_ba[frag_idx]; | bus_addr = sd_ba[frag_idx]; | ||||
} | } | ||||
bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], | bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], | ||||
BUS_DMASYNC_PREREAD); | BUS_DMASYNC_PREREAD); | ||||
MPASS(sd_m[frag_idx] == NULL); | if (sd_m[frag_idx] == NULL) { | ||||
if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { | if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { | ||||
break; | break; | ||||
} | } | ||||
sd_m[frag_idx] = m; | sd_m[frag_idx] = m; | ||||
} | |||||
bit_set(fl->ifl_rx_bitmap, frag_idx); | bit_set(fl->ifl_rx_bitmap, frag_idx); | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
fl->ifl_m_enqueued++; | fl->ifl_m_enqueued++; | ||||
#endif | #endif | ||||
DBG_COUNTER_INC(rx_allocs); | DBG_COUNTER_INC(rx_allocs); | ||||
fl->ifl_rxd_idxs[i] = frag_idx; | fl->ifl_rxd_idxs[i] = frag_idx; | ||||
fl->ifl_bus_addrs[i] = bus_addr; | fl->ifl_bus_addrs[i] = bus_addr; | ||||
▲ Show 20 Lines • Show All 439 Lines • ▼ Show 20 Lines | prefetch_pkts(iflib_fl_t fl, int cidx) | ||||
prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]); | prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]); | ||||
prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]); | prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]); | ||||
prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]); | prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]); | ||||
prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]); | prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]); | ||||
prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]); | prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]); | ||||
prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]); | prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]); | ||||
} | } | ||||
static void | static struct mbuf * | ||||
rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int unload, if_rxsd_t sd) | rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, bool unload, if_rxsd_t sd, | ||||
int *pf_rv, if_rxd_info_t ri) | |||||
{ | { | ||||
int flid, cidx; | int flid, cidx; | ||||
bus_dmamap_t map; | bus_dmamap_t map; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
int next; | caddr_t payload; | ||||
struct mbuf *m; | |||||
int next, len; | |||||
Done Inline ActionsThe "cidx" and "flid" from above should be merged with into this line and the resulting line sorted alphabetically, getting the variable declaration of rxd_frag_to_sd() close to style(9). marius: The "cidx" and "flid" from above should be merged with into this line and the resulting line… | |||||
map = NULL; | map = NULL; | ||||
flid = irf->irf_flid; | flid = irf->irf_flid; | ||||
cidx = irf->irf_idx; | cidx = irf->irf_idx; | ||||
fl = &rxq->ifr_fl[flid]; | fl = &rxq->ifr_fl[flid]; | ||||
sd->ifsd_fl = fl; | sd->ifsd_fl = fl; | ||||
sd->ifsd_cidx = cidx; | sd->ifsd_cidx = cidx; | ||||
sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx]; | m = fl->ifl_sds.ifsd_m[cidx]; | ||||
sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; | sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; | ||||
fl->ifl_credits--; | fl->ifl_credits--; | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
fl->ifl_m_dequeued++; | fl->ifl_m_dequeued++; | ||||
#endif | #endif | ||||
if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) | if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) | ||||
prefetch_pkts(fl, cidx); | prefetch_pkts(fl, cidx); | ||||
next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); | next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); | ||||
prefetch(&fl->ifl_sds.ifsd_map[next]); | prefetch(&fl->ifl_sds.ifsd_map[next]); | ||||
map = fl->ifl_sds.ifsd_map[cidx]; | map = fl->ifl_sds.ifsd_map[cidx]; | ||||
next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); | next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); | ||||
/* not valid assert if bxe really does SGE from non-contiguous elements */ | /* not valid assert if bxe really does SGE from non-contiguous elements */ | ||||
MPASS(fl->ifl_cidx == cidx); | MPASS(fl->ifl_cidx == cidx); | ||||
bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); | bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); | ||||
if (rxq->pfil != NULL && PFIL_HOOKED_IN(rxq->pfil) && pf_rv != NULL) { | |||||
payload = *sd->ifsd_cl; | |||||
payload += ri->iri_pad; | |||||
len = ri->iri_len - ri->iri_pad; | |||||
*pf_rv = pfil_run_hooks(rxq->pfil, payload, ri->iri_ifp, | |||||
len | PFIL_MEMPTR | PFIL_IN, NULL); | |||||
switch (*pf_rv) { | |||||
case PFIL_DROPPED: | |||||
case PFIL_CONSUMED: | |||||
/* | |||||
* The filter ate it. Everything is recycled. | |||||
*/ | |||||
m = NULL; | |||||
unload = 0; | |||||
break; | |||||
case PFIL_REALLOCED: | |||||
/* | |||||
* The ilter copied it. Everything is recycled. | |||||
erjUnsubmitted Done Inline ActionsYou dropped an 'f' here erj: You dropped an 'f' here | |||||
Done Inline ActionsThere should be two spaces after the dot in this comment, too. marius: There should be two spaces after the dot in this comment, too. | |||||
*/ | |||||
m = pfil_mem2mbuf(payload); | |||||
unload = 0; | |||||
break; | |||||
default: | |||||
Not Done Inline ActionsI'm not really crazy about "default is OK" here. If pf_rv was a pfil_return_t, and this was "case PFIL_PASS:", the warning causing the build to fail would be better than relying on the MPASS() wouldn't it? shurd: I'm not really crazy about "default is OK" here.
If pf_rv was a pfil_return_t, and this was… | |||||
Done Inline ActionsI've moved it out of the default. However, I'd prefer to not touch pf_rv, as there are other reviews outstanding for other drivers. gallatin: I've moved it out of the default. However, I'd prefer to not touch pf_rv, as there are other… | |||||
/* | |||||
* Filter said it was OK, so receive like | |||||
* normal | |||||
*/ | |||||
MPASS(*pf_rv == PFIL_PASS); | |||||
fl->ifl_sds.ifsd_m[cidx] = NULL; | |||||
} | |||||
} else { | |||||
fl->ifl_sds.ifsd_m[cidx] = NULL; | |||||
*pf_rv = PFIL_PASS; | |||||
} | |||||
if (unload) | if (unload) | ||||
bus_dmamap_unload(fl->ifl_buf_tag, map); | bus_dmamap_unload(fl->ifl_buf_tag, map); | ||||
fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | ||||
if (__predict_false(fl->ifl_cidx == 0)) | if (__predict_false(fl->ifl_cidx == 0)) | ||||
fl->ifl_gen = 0; | fl->ifl_gen = 0; | ||||
bit_clear(fl->ifl_rx_bitmap, cidx); | bit_clear(fl->ifl_rx_bitmap, cidx); | ||||
return (m); | |||||
} | } | ||||
static struct mbuf * | static struct mbuf * | ||||
assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd) | assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd, int *pf_rv) | ||||
{ | { | ||||
int i, padlen , flags; | int i, padlen , flags; | ||||
struct mbuf *m, *mh, *mt; | struct mbuf *m, *mh, *mt; | ||||
caddr_t cl; | caddr_t cl; | ||||
int *pf_rv_ptr; | |||||
Done Inline ActionsSimilarly, "i", "flags" and "padlen" should be merged here. marius: Similarly, "i", "flags" and "padlen" should be merged here. | |||||
bool consumed; | |||||
i = 0; | i = 0; | ||||
mh = NULL; | mh = NULL; | ||||
consumed = false; | |||||
*pf_rv = PFIL_PASS; | |||||
pf_rv_ptr = pf_rv; | |||||
do { | do { | ||||
rxd_frag_to_sd(rxq, &ri->iri_frags[i], TRUE, sd); | m = rxd_frag_to_sd(rxq, &ri->iri_frags[i], !consumed, sd, | ||||
pf_rv_ptr, ri); | |||||
MPASS(*sd->ifsd_cl != NULL); | MPASS(*sd->ifsd_cl != NULL); | ||||
MPASS(*sd->ifsd_m != NULL); | |||||
/* Don't include zero-length frags */ | /* Don't include zero-length frags */ | ||||
erjUnsubmitted Done Inline ActionsYou should update this comment since you added new checks to the if statement below. erj: You should update this comment since you added new checks to the if statement below. | |||||
if (ri->iri_frags[i].irf_len == 0) { | if (ri->iri_frags[i].irf_len == 0 || consumed || | ||||
*pf_rv == PFIL_CONSUMED || *pf_rv == PFIL_DROPPED) { | |||||
if (mh == NULL) { | |||||
/* everything saved here */ | |||||
consumed = true; | |||||
pf_rv_ptr = NULL; | |||||
continue; | |||||
} | |||||
/* XXX we can save the cluster here, but not the mbuf */ | /* XXX we can save the cluster here, but not the mbuf */ | ||||
m_init(*sd->ifsd_m, M_NOWAIT, MT_DATA, 0); | m_init(m, M_NOWAIT, MT_DATA, 0); | ||||
m_free(*sd->ifsd_m); | m_free(m); | ||||
*sd->ifsd_m = NULL; | |||||
continue; | continue; | ||||
} | } | ||||
m = *sd->ifsd_m; | |||||
*sd->ifsd_m = NULL; | |||||
if (mh == NULL) { | if (mh == NULL) { | ||||
flags = M_PKTHDR|M_EXT; | flags = M_PKTHDR|M_EXT; | ||||
mh = mt = m; | mh = mt = m; | ||||
padlen = ri->iri_pad; | padlen = ri->iri_pad; | ||||
} else { | } else { | ||||
flags = M_EXT; | flags = M_EXT; | ||||
mt->m_next = m; | mt->m_next = m; | ||||
mt = m; | mt = m; | ||||
/* assuming padding is only on the first fragment */ | /* assuming padding is only on the first fragment */ | ||||
padlen = 0; | padlen = 0; | ||||
} | } | ||||
if (*pf_rv == PFIL_CONSUMED) { | |||||
/* mbuf is already in a good state */ | |||||
continue; | |||||
} | |||||
cl = *sd->ifsd_cl; | cl = *sd->ifsd_cl; | ||||
*sd->ifsd_cl = NULL; | *sd->ifsd_cl = NULL; | ||||
/* Can these two be made one ? */ | /* Can these two be made one ? */ | ||||
m_init(m, M_NOWAIT, MT_DATA, flags); | m_init(m, M_NOWAIT, MT_DATA, flags); | ||||
m_cljset(m, cl, sd->ifsd_fl->ifl_cltype); | m_cljset(m, cl, sd->ifsd_fl->ifl_cltype); | ||||
/* | /* | ||||
* These must follow m_init and m_cljset | * These must follow m_init and m_cljset | ||||
Show All 9 Lines | |||||
/* | /* | ||||
* Process one software descriptor | * Process one software descriptor | ||||
*/ | */ | ||||
static struct mbuf * | static struct mbuf * | ||||
iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri) | iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri) | ||||
{ | { | ||||
struct if_rxsd sd; | struct if_rxsd sd; | ||||
struct mbuf *m; | struct mbuf *m; | ||||
int pf_rv; | |||||
/* should I merge this back in now that the two paths are basically duplicated? */ | /* should I merge this back in now that the two paths are basically duplicated? */ | ||||
if (ri->iri_nfrags == 1 && | if (ri->iri_nfrags == 1 && | ||||
ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) { | ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) { | ||||
rxd_frag_to_sd(rxq, &ri->iri_frags[0], FALSE, &sd); | m = rxd_frag_to_sd(rxq, &ri->iri_frags[0], FALSE, &sd, | ||||
m = *sd.ifsd_m; | &pf_rv, ri); | ||||
*sd.ifsd_m = NULL; | if (pf_rv != PFIL_PASS && pf_rv != PFIL_REALLOCED) | ||||
return (m); | |||||
if (pf_rv == PFIL_PASS) { | |||||
m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR); | m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR); | ||||
Done Inline ActionsThis should be "false" now. marius: This should be "false" now. | |||||
Done Inline Actionsnice catch gallatin: nice catch | |||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
if (!IP_ALIGNED(m)) | if (!IP_ALIGNED(m)) | ||||
m->m_data += 2; | m->m_data += 2; | ||||
#endif | #endif | ||||
memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); | memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); | ||||
m->m_len = ri->iri_frags[0].irf_len; | m->m_len = ri->iri_frags[0].irf_len; | ||||
} | |||||
} else { | } else { | ||||
m = assemble_segments(rxq, ri, &sd); | m = assemble_segments(rxq, ri, &sd, &pf_rv); | ||||
if (pf_rv != PFIL_PASS && pf_rv != PFIL_REALLOCED) | |||||
return (m); | |||||
} | } | ||||
m->m_pkthdr.len = ri->iri_len; | m->m_pkthdr.len = ri->iri_len; | ||||
m->m_pkthdr.rcvif = ri->iri_ifp; | m->m_pkthdr.rcvif = ri->iri_ifp; | ||||
m->m_flags |= ri->iri_flags; | m->m_flags |= ri->iri_flags; | ||||
m->m_pkthdr.ether_vtag = ri->iri_vtag; | m->m_pkthdr.ether_vtag = ri->iri_vtag; | ||||
m->m_pkthdr.flowid = ri->iri_flowid; | m->m_pkthdr.flowid = ri->iri_flowid; | ||||
M_HASHTYPE_SET(m, ri->iri_rsstype); | M_HASHTYPE_SET(m, ri->iri_rsstype); | ||||
m->m_pkthdr.csum_flags = ri->iri_csum_flags; | m->m_pkthdr.csum_flags = ri->iri_csum_flags; | ||||
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines | else | ||||
cidxp = &rxq->ifr_fl[0].ifl_cidx; | cidxp = &rxq->ifr_fl[0].ifl_cidx; | ||||
if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) { | if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) { | ||||
for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) | for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) | ||||
__iflib_fl_refill_lt(ctx, fl, budget + 8); | __iflib_fl_refill_lt(ctx, fl, budget + 8); | ||||
DBG_COUNTER_INC(rx_unavail); | DBG_COUNTER_INC(rx_unavail); | ||||
return (false); | return (false); | ||||
} | } | ||||
/* pfil needs the vnet to be set */ | |||||
CURVNET_SET_QUIET(ifp->if_vnet); | |||||
for (budget_left = budget; budget_left > 0 && avail > 0;) { | for (budget_left = budget; budget_left > 0 && avail > 0;) { | ||||
if (__predict_false(!CTX_ACTIVE(ctx))) { | if (__predict_false(!CTX_ACTIVE(ctx))) { | ||||
DBG_COUNTER_INC(rx_ctx_inactive); | DBG_COUNTER_INC(rx_ctx_inactive); | ||||
break; | break; | ||||
} | } | ||||
/* | /* | ||||
* Reset client set fields to their default values | * Reset client set fields to their default values | ||||
*/ | */ | ||||
rxd_info_zero(&ri); | rxd_info_zero(&ri); | ||||
ri.iri_qsidx = rxq->ifr_id; | ri.iri_qsidx = rxq->ifr_id; | ||||
ri.iri_cidx = *cidxp; | ri.iri_cidx = *cidxp; | ||||
ri.iri_ifp = ifp; | ri.iri_ifp = ifp; | ||||
ri.iri_frags = rxq->ifr_frags; | ri.iri_frags = rxq->ifr_frags; | ||||
err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | ||||
if (err) | if (err) | ||||
goto err; | goto err; | ||||
rx_pkts += 1; | |||||
rx_bytes += ri.iri_len; | |||||
if (sctx->isc_flags & IFLIB_HAS_RXCQ) { | if (sctx->isc_flags & IFLIB_HAS_RXCQ) { | ||||
*cidxp = ri.iri_cidx; | *cidxp = ri.iri_cidx; | ||||
/* Update our consumer index */ | /* Update our consumer index */ | ||||
/* XXX NB: shurd - check if this is still safe */ | /* XXX NB: shurd - check if this is still safe */ | ||||
while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) { | while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) { | ||||
rxq->ifr_cq_cidx -= scctx->isc_nrxd[0]; | rxq->ifr_cq_cidx -= scctx->isc_nrxd[0]; | ||||
rxq->ifr_cq_gen = 0; | rxq->ifr_cq_gen = 0; | ||||
} | } | ||||
/* was this only a completion queue message? */ | /* was this only a completion queue message? */ | ||||
if (__predict_false(ri.iri_nfrags == 0)) | if (__predict_false(ri.iri_nfrags == 0)) | ||||
continue; | continue; | ||||
} | } | ||||
MPASS(ri.iri_nfrags != 0); | MPASS(ri.iri_nfrags != 0); | ||||
MPASS(ri.iri_len != 0); | MPASS(ri.iri_len != 0); | ||||
/* will advance the cidx on the corresponding free lists */ | /* will advance the cidx on the corresponding free lists */ | ||||
m = iflib_rxd_pkt_get(rxq, &ri); | m = iflib_rxd_pkt_get(rxq, &ri); | ||||
avail--; | avail--; | ||||
budget_left--; | budget_left--; | ||||
if (avail == 0 && budget_left) | if (avail == 0 && budget_left) | ||||
avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget_left); | avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget_left); | ||||
if (__predict_false(m == NULL)) { | if (__predict_false(m == NULL)) { | ||||
/* XXX no longer makes sense */ | |||||
DBG_COUNTER_INC(rx_mbuf_null); | DBG_COUNTER_INC(rx_mbuf_null); | ||||
Done Inline ActionsThe "rx_mbuf_null" counter should just go now. marius: The "rx_mbuf_null" counter should just go now. | |||||
Done Inline ActionsThanks for saying that. gallatin: Thanks for saying that. | |||||
continue; | continue; | ||||
} | } | ||||
/* imm_pkt: -- cxgb */ | /* imm_pkt: -- cxgb */ | ||||
if (mh == NULL) | if (mh == NULL) | ||||
mh = mt = m; | mh = mt = m; | ||||
else { | else { | ||||
mt->m_nextpkt = m; | mt->m_nextpkt = m; | ||||
mt = m; | mt = m; | ||||
} | } | ||||
} | } | ||||
CURVNET_RESTORE(); | |||||
/* make sure that we can refill faster than drain */ | /* make sure that we can refill faster than drain */ | ||||
for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) | for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) | ||||
__iflib_fl_refill_lt(ctx, fl, budget + 8); | __iflib_fl_refill_lt(ctx, fl, budget + 8); | ||||
lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO); | lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO); | ||||
if (lro_enabled) | if (lro_enabled) | ||||
iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding); | iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding); | ||||
mt = mf = NULL; | mt = mf = NULL; | ||||
▲ Show 20 Lines • Show All 1,600 Lines • ▼ Show 20 Lines | for (i = 0; i < sctx->isc_ntxqs; i++) { | ||||
if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) { | if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) { | ||||
device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n", | device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n", | ||||
i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]); | i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]); | ||||
scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i]; | scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i]; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
static void | |||||
iflib_add_pfil(if_ctx_t ctx) | |||||
{ | |||||
struct pfil_head *pfil; | |||||
struct pfil_head_args pa; | |||||
iflib_rxq_t rxq; | |||||
int i; | |||||
pa.pa_version = PFIL_VERSION; | |||||
pa.pa_flags = PFIL_IN; | |||||
pa.pa_type = PFIL_TYPE_ETHERNET; | |||||
pa.pa_headname = ctx->ifc_ifp->if_xname; | |||||
pfil = pfil_head_register(&pa); | |||||
for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { | |||||
rxq->pfil = pfil; | |||||
} | |||||
} | |||||
static void | |||||
iflib_rem_pfil(if_ctx_t ctx) | |||||
{ | |||||
struct pfil_head *pfil; | |||||
iflib_rxq_t rxq; | |||||
int i; | |||||
Done Inline ActionsSuperfluous empty line marius: Superfluous empty line | |||||
rxq = ctx->ifc_rxqs; | |||||
pfil = rxq->pfil; | |||||
for (i = 0; i < NRXQSETS(ctx); i++, rxq++) { | |||||
rxq->pfil = NULL; | |||||
} | |||||
pfil_head_unregister(pfil); | |||||
Done Inline ActionsDitto marius: Ditto | |||||
} | |||||
int | int | ||||
iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | ||||
{ | { | ||||
int err, rid, msix; | int err, rid, msix; | ||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
if_t ifp; | if_t ifp; | ||||
if_softc_ctx_t scctx; | if_softc_ctx_t scctx; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 187 Lines • ▼ Show 20 Lines | if ((err = iflib_netmap_attach(ctx))) { | ||||
goto fail_detach; | goto fail_detach; | ||||
} | } | ||||
*ctxp = ctx; | *ctxp = ctx; | ||||
NETDUMP_SET(ctx->ifc_ifp, iflib); | NETDUMP_SET(ctx->ifc_ifp, iflib); | ||||
if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | ||||
iflib_add_device_sysctl_post(ctx); | iflib_add_device_sysctl_post(ctx); | ||||
iflib_add_pfil(ctx); | |||||
ctx->ifc_flags |= IFC_INIT_DONE; | ctx->ifc_flags |= IFC_INIT_DONE; | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
return (0); | return (0); | ||||
fail_detach: | fail_detach: | ||||
ether_ifdetach(ctx->ifc_ifp); | ether_ifdetach(ctx->ifc_ifp); | ||||
fail_intr_free: | fail_intr_free: | ||||
iflib_free_intr_mem(ctx); | iflib_free_intr_mem(ctx); | ||||
▲ Show 20 Lines • Show All 315 Lines • ▼ Show 20 Lines | #endif | ||||
/* Unregister VLAN events */ | /* Unregister VLAN events */ | ||||
if (ctx->ifc_vlan_attach_event != NULL) | if (ctx->ifc_vlan_attach_event != NULL) | ||||
EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); | EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); | ||||
if (ctx->ifc_vlan_detach_event != NULL) | if (ctx->ifc_vlan_detach_event != NULL) | ||||
EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); | EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); | ||||
iflib_netmap_detach(ifp); | iflib_netmap_detach(ifp); | ||||
ether_ifdetach(ifp); | ether_ifdetach(ifp); | ||||
iflib_rem_pfil(ctx); | |||||
if (ctx->ifc_led_dev != NULL) | if (ctx->ifc_led_dev != NULL) | ||||
led_destroy(ctx->ifc_led_dev); | led_destroy(ctx->ifc_led_dev); | ||||
/* XXX drain any dependent tasks */ | /* XXX drain any dependent tasks */ | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io_tqg; | ||||
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { | for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { | ||||
callout_drain(&txq->ift_timer); | callout_drain(&txq->ift_timer); | ||||
if (txq->ift_task.gt_uniq != NULL) | if (txq->ift_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &txq->ift_task); | taskqgroup_detach(tqg, &txq->ift_task); | ||||
▲ Show 20 Lines • Show All 1,624 Lines • Show Last 20 Lines |
Is there a particular reason to not sort this header alphabetically between <net/mp_ring.h> and <net/vnet.h>?