Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/iflib.c
Show First 20 Lines • Show All 87 Lines • ▼ Show 20 Lines | |||||
#include <sys/memdesc.h> | #include <sys/memdesc.h> | ||||
#include <machine/bus.h> | #include <machine/bus.h> | ||||
#include <machine/md_var.h> | #include <machine/md_var.h> | ||||
#include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
#include <x86/include/busdma_impl.h> | #include <x86/include/busdma_impl.h> | ||||
#include <x86/iommu/busdma_dmar.h> | #include <x86/iommu/busdma_dmar.h> | ||||
#endif | #endif | ||||
#include <sys/bitstring.h> | |||||
/* | /* | ||||
* enable accounting of every mbuf as it comes in to and goes out of | * enable accounting of every mbuf as it comes in to and goes out of | ||||
* iflib's software descriptor references | * iflib's software descriptor references | ||||
*/ | */ | ||||
#define MEMORY_LOGGING 0 | #define MEMORY_LOGGING 0 | ||||
/* | /* | ||||
* Enable mbuf vectors for compressing long mbuf chains | * Enable mbuf vectors for compressing long mbuf chains | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 272 Lines • ▼ Show 20 Lines | |||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
uint64_t ifl_m_enqueued; | uint64_t ifl_m_enqueued; | ||||
uint64_t ifl_m_dequeued; | uint64_t ifl_m_dequeued; | ||||
uint64_t ifl_cl_enqueued; | uint64_t ifl_cl_enqueued; | ||||
uint64_t ifl_cl_dequeued; | uint64_t ifl_cl_dequeued; | ||||
#endif | #endif | ||||
/* implicit pad */ | /* implicit pad */ | ||||
bitstr_t *ifl_rx_bitmap; | |||||
qidx_t ifl_fragidx; | |||||
/* constant */ | /* constant */ | ||||
qidx_t ifl_size; | qidx_t ifl_size; | ||||
uint16_t ifl_buf_size; | uint16_t ifl_buf_size; | ||||
uint16_t ifl_cltype; | uint16_t ifl_cltype; | ||||
uma_zone_t ifl_zone; | uma_zone_t ifl_zone; | ||||
iflib_rxsd_array_t ifl_sds; | iflib_rxsd_array_t ifl_sds; | ||||
iflib_rxq_t ifl_rxq; | iflib_rxq_t ifl_rxq; | ||||
uint8_t ifl_id; | uint8_t ifl_id; | ||||
▲ Show 20 Lines • Show All 1,400 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* (Re)populate an rxq free-buffer list with up to @n new packet buffers. | * (Re)populate an rxq free-buffer list with up to @n new packet buffers. | ||||
* The caller must assure that @n does not exceed the queue's capacity. | * The caller must assure that @n does not exceed the queue's capacity. | ||||
*/ | */ | ||||
static void | static void | ||||
_iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) | _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) | ||||
{ | { | ||||
struct mbuf *m; | struct mbuf *m; | ||||
int idx, pidx = fl->ifl_pidx; | int idx, frag_idx = fl->ifl_fragidx; | ||||
int pidx = fl->ifl_pidx; | |||||
caddr_t cl, *sd_cl; | caddr_t cl, *sd_cl; | ||||
struct mbuf **sd_m; | struct mbuf **sd_m; | ||||
uint8_t *sd_flags; | uint8_t *sd_flags; | ||||
struct if_rxd_update iru; | struct if_rxd_update iru; | ||||
bus_dmamap_t *sd_map; | bus_dmamap_t *sd_map; | ||||
int n, i = 0; | int n, i = 0; | ||||
uint64_t bus_addr; | uint64_t bus_addr; | ||||
int err; | int err; | ||||
Show All 26 Lines | _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) | ||||
iru.iru_flidx = fl->ifl_id; | iru.iru_flidx = fl->ifl_id; | ||||
while (n--) { | while (n--) { | ||||
/* | /* | ||||
* We allocate an uninitialized mbuf + cluster, mbuf is | * We allocate an uninitialized mbuf + cluster, mbuf is | ||||
* initialized after rx. | * initialized after rx. | ||||
* | * | ||||
* If the cluster is still set then we know a minimum sized packet was received | * If the cluster is still set then we know a minimum sized packet was received | ||||
*/ | */ | ||||
if ((cl = sd_cl[idx]) == NULL) { | bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, &frag_idx); | ||||
if ((cl = sd_cl[idx] = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL) | if ((frag_idx < 0) || (frag_idx >= fl->ifl_size)) | ||||
bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); | |||||
if ((cl = sd_cl[frag_idx]) == NULL) { | |||||
if ((cl = sd_cl[frag_idx] = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL) | |||||
break; | break; | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
fl->ifl_cl_enqueued++; | fl->ifl_cl_enqueued++; | ||||
#endif | #endif | ||||
} | } | ||||
if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { | if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { | ||||
break; | break; | ||||
} | } | ||||
Show All 9 Lines | |||||
#endif | #endif | ||||
{ | { | ||||
struct rxq_refill_cb_arg cb_arg; | struct rxq_refill_cb_arg cb_arg; | ||||
iflib_rxq_t q; | iflib_rxq_t q; | ||||
cb_arg.error = 0; | cb_arg.error = 0; | ||||
q = fl->ifl_rxq; | q = fl->ifl_rxq; | ||||
MPASS(sd_map != NULL); | MPASS(sd_map != NULL); | ||||
MPASS(sd_map[idx] != NULL); | MPASS(sd_map[frag_idx] != NULL); | ||||
err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[idx], | err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[frag_idx], | ||||
cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0); | cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0); | ||||
bus_dmamap_sync(fl->ifl_desc_tag, sd_map[idx], BUS_DMASYNC_PREREAD); | bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx], | ||||
BUS_DMASYNC_PREREAD); | |||||
if (err != 0 || cb_arg.error) { | if (err != 0 || cb_arg.error) { | ||||
/* | /* | ||||
* !zone_pack ? | * !zone_pack ? | ||||
*/ | */ | ||||
if (fl->ifl_zone == zone_pack) | if (fl->ifl_zone == zone_pack) | ||||
uma_zfree(fl->ifl_zone, cl); | uma_zfree(fl->ifl_zone, cl); | ||||
m_free(m); | m_free(m); | ||||
n = 0; | n = 0; | ||||
goto done; | goto done; | ||||
} | } | ||||
bus_addr = cb_arg.seg.ds_addr; | bus_addr = cb_arg.seg.ds_addr; | ||||
} | } | ||||
sd_flags[idx] |= RX_SW_DESC_INUSE; | bit_set(fl->ifl_rx_bitmap, frag_idx); | ||||
sd_flags[frag_idx] |= RX_SW_DESC_INUSE; | |||||
MPASS(sd_m[idx] == NULL); | MPASS(sd_m[frag_idx] == NULL); | ||||
sd_cl[idx] = cl; | sd_cl[frag_idx] = cl; | ||||
sd_m[idx] = m; | sd_m[frag_idx] = m; | ||||
fl->ifl_rxd_idxs[i] = idx; | fl->ifl_rxd_idxs[i] = frag_idx; | ||||
fl->ifl_bus_addrs[i] = bus_addr; | fl->ifl_bus_addrs[i] = bus_addr; | ||||
fl->ifl_vm_addrs[i] = cl; | fl->ifl_vm_addrs[i] = cl; | ||||
fl->ifl_credits++; | fl->ifl_credits++; | ||||
i++; | i++; | ||||
MPASS(fl->ifl_credits <= fl->ifl_size); | MPASS(fl->ifl_credits <= fl->ifl_size); | ||||
if (++idx == fl->ifl_size) { | if (++idx == fl->ifl_size) { | ||||
fl->ifl_gen = 1; | fl->ifl_gen = 1; | ||||
idx = 0; | idx = 0; | ||||
} | } | ||||
if (n == 0 || i == IFLIB_MAX_RX_REFRESH) { | if (n == 0 || i == IFLIB_MAX_RX_REFRESH) { | ||||
iru.iru_pidx = pidx; | iru.iru_pidx = pidx; | ||||
iru.iru_count = i; | iru.iru_count = i; | ||||
ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | ||||
i = 0; | i = 0; | ||||
pidx = idx; | pidx = idx; | ||||
} | |||||
fl->ifl_pidx = idx; | fl->ifl_pidx = idx; | ||||
} | |||||
} | } | ||||
done: | done: | ||||
DBG_COUNTER_INC(rxd_flush); | DBG_COUNTER_INC(rxd_flush); | ||||
if (fl->ifl_pidx == 0) | if (fl->ifl_pidx == 0) | ||||
pidx = fl->ifl_size - 1; | pidx = fl->ifl_size - 1; | ||||
else | else | ||||
pidx = fl->ifl_pidx - 1; | pidx = fl->ifl_pidx - 1; | ||||
if (sd_map) | if (sd_map) | ||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx); | ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx); | ||||
fl->ifl_fragidx = frag_idx; | |||||
} | } | ||||
static __inline void | static __inline void | ||||
__iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max) | __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max) | ||||
{ | { | ||||
/* we avoid allowing pidx to catch up with cidx as it confuses ixl */ | /* we avoid allowing pidx to catch up with cidx as it confuses ixl */ | ||||
int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; | int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | for (i = 0; i < fl->ifl_size; i++) { | ||||
MPASS(fl->ifl_sds.ifsd_flags[i] == 0); | MPASS(fl->ifl_sds.ifsd_flags[i] == 0); | ||||
MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); | MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); | ||||
MPASS(fl->ifl_sds.ifsd_m[i] == NULL); | MPASS(fl->ifl_sds.ifsd_m[i] == NULL); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Reset free list values | * Reset free list values | ||||
*/ | */ | ||||
fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = 0;; | fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; | ||||
bzero(idi->idi_vaddr, idi->idi_size); | bzero(idi->idi_vaddr, idi->idi_size); | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Initialize a receive ring and its buffers. | * Initialize a receive ring and its buffers. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
iflib_fl_setup(iflib_fl_t fl) | iflib_fl_setup(iflib_fl_t fl) | ||||
{ | { | ||||
iflib_rxq_t rxq = fl->ifl_rxq; | iflib_rxq_t rxq = fl->ifl_rxq; | ||||
if_ctx_t ctx = rxq->ifr_ctx; | if_ctx_t ctx = rxq->ifr_ctx; | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | ||||
bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size); | |||||
/* | /* | ||||
** Free current RX buffer structs and their mbufs | ** Free current RX buffer structs and their mbufs | ||||
*/ | */ | ||||
iflib_fl_bufs_free(fl); | iflib_fl_bufs_free(fl); | ||||
/* Now replenish the mbufs */ | /* Now replenish the mbufs */ | ||||
MPASS(fl->ifl_credits == 0); | MPASS(fl->ifl_credits == 0); | ||||
/* | /* | ||||
* XXX don't set the max_frame_size to larger | * XXX don't set the max_frame_size to larger | ||||
▲ Show 20 Lines • Show All 333 Lines • ▼ Show 20 Lines | if (unload) | ||||
bus_dmamap_unload(fl->ifl_desc_tag, map); | bus_dmamap_unload(fl->ifl_desc_tag, map); | ||||
} | } | ||||
fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | ||||
if (__predict_false(fl->ifl_cidx == 0)) | if (__predict_false(fl->ifl_cidx == 0)) | ||||
fl->ifl_gen = 0; | fl->ifl_gen = 0; | ||||
if (map != NULL) | if (map != NULL) | ||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
bit_clear(fl->ifl_rx_bitmap, cidx); | |||||
} | } | ||||
static struct mbuf * | static struct mbuf * | ||||
assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd) | assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd) | ||||
{ | { | ||||
int i, padlen , flags; | int i, padlen , flags; | ||||
struct mbuf *m, *mh, *mt; | struct mbuf *m, *mh, *mt; | ||||
caddr_t cl; | caddr_t cl; | ||||
▲ Show 20 Lines • Show All 1,879 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
iflib_device_deregister(if_ctx_t ctx) | iflib_device_deregister(if_ctx_t ctx) | ||||
{ | { | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
iflib_txq_t txq; | iflib_txq_t txq; | ||||
iflib_rxq_t rxq; | iflib_rxq_t rxq; | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
int i; | int i, j; | ||||
struct taskqgroup *tqg; | struct taskqgroup *tqg; | ||||
iflib_fl_t fl; | |||||
/* Make sure VLANS are not using driver */ | /* Make sure VLANS are not using driver */ | ||||
if (if_vlantrunkinuse(ifp)) { | if (if_vlantrunkinuse(ifp)) { | ||||
device_printf(dev,"Vlan in use, detach first\n"); | device_printf(dev,"Vlan in use, detach first\n"); | ||||
return (EBUSY); | return (EBUSY); | ||||
} | } | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
Show All 18 Lines | iflib_device_deregister(if_ctx_t ctx) | ||||
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { | for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { | ||||
callout_drain(&txq->ift_timer); | callout_drain(&txq->ift_timer); | ||||
if (txq->ift_task.gt_uniq != NULL) | if (txq->ift_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &txq->ift_task); | taskqgroup_detach(tqg, &txq->ift_task); | ||||
} | } | ||||
for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { | for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { | ||||
if (rxq->ifr_task.gt_uniq != NULL) | if (rxq->ifr_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &rxq->ifr_task); | taskqgroup_detach(tqg, &rxq->ifr_task); | ||||
for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) | |||||
free(fl->ifl_rx_bitmap, M_IFLIB); | |||||
} | } | ||||
tqg = qgroup_if_config_tqg; | tqg = qgroup_if_config_tqg; | ||||
if (ctx->ifc_admin_task.gt_uniq != NULL) | if (ctx->ifc_admin_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &ctx->ifc_admin_task); | taskqgroup_detach(tqg, &ctx->ifc_admin_task); | ||||
if (ctx->ifc_vflr_task.gt_uniq != NULL) | if (ctx->ifc_vflr_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &ctx->ifc_vflr_task); | taskqgroup_detach(tqg, &ctx->ifc_vflr_task); | ||||
IFDI_DETACH(ctx); | IFDI_DETACH(ctx); | ||||
▲ Show 20 Lines • Show All 377 Lines • ▼ Show 20 Lines | for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) { | ||||
} | } | ||||
/* Allocate receive buffers for the ring*/ | /* Allocate receive buffers for the ring*/ | ||||
if (iflib_rxsd_alloc(rxq)) { | if (iflib_rxsd_alloc(rxq)) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"Critical Failure setting up receive buffers\n"); | "Critical Failure setting up receive buffers\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto err_rx_desc; | goto err_rx_desc; | ||||
} | } | ||||
for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) | |||||
fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, M_WAITOK|M_ZERO); | |||||
} | } | ||||
/* TXQs */ | /* TXQs */ | ||||
vaddrs = malloc(sizeof(caddr_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK); | vaddrs = malloc(sizeof(caddr_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK); | ||||
paddrs = malloc(sizeof(uint64_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK); | paddrs = malloc(sizeof(uint64_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK); | ||||
for (i = 0; i < ntxqsets; i++) { | for (i = 0; i < ntxqsets; i++) { | ||||
iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi; | iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi; | ||||
▲ Show 20 Lines • Show All 944 Lines • Show Last 20 Lines |