Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/cxgbe/t4_sge.c
Show First 20 Lines • Show All 264 Lines • ▼ Show 20 Lines | |||||
static int alloc_eq(struct adapter *, struct vi_info *, struct sge_eq *); | static int alloc_eq(struct adapter *, struct vi_info *, struct sge_eq *); | ||||
static int free_eq(struct adapter *, struct sge_eq *); | static int free_eq(struct adapter *, struct sge_eq *); | ||||
static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, | static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, | ||||
struct sysctl_oid *); | struct sysctl_oid *); | ||||
static int free_wrq(struct adapter *, struct sge_wrq *); | static int free_wrq(struct adapter *, struct sge_wrq *); | ||||
static int alloc_txq(struct vi_info *, struct sge_txq *, int, | static int alloc_txq(struct vi_info *, struct sge_txq *, int, | ||||
struct sysctl_oid *); | struct sysctl_oid *); | ||||
static int free_txq(struct vi_info *, struct sge_txq *); | static int free_txq(struct vi_info *, struct sge_txq *); | ||||
#if defined(TCP_OFFLOAD) || defined(RATELIMIT) | |||||
static int alloc_ofld_txq(struct vi_info *, struct sge_ofld_txq *, int, | |||||
struct sysctl_oid *); | |||||
static int free_ofld_txq(struct vi_info *, struct sge_ofld_txq *); | |||||
#endif | |||||
static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int); | static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int); | ||||
static inline void ring_fl_db(struct adapter *, struct sge_fl *); | static inline void ring_fl_db(struct adapter *, struct sge_fl *); | ||||
static int refill_fl(struct adapter *, struct sge_fl *, int); | static int refill_fl(struct adapter *, struct sge_fl *, int); | ||||
static void refill_sfl(void *); | static void refill_sfl(void *); | ||||
static int alloc_fl_sdesc(struct sge_fl *); | static int alloc_fl_sdesc(struct sge_fl *); | ||||
static void free_fl_sdesc(struct adapter *, struct sge_fl *); | static void free_fl_sdesc(struct adapter *, struct sge_fl *); | ||||
static int find_refill_source(struct adapter *, int, bool); | static int find_refill_source(struct adapter *, int, bool); | ||||
static void add_fl_to_sfl(struct adapter *, struct sge_fl *); | static void add_fl_to_sfl(struct adapter *, struct sge_fl *); | ||||
▲ Show 20 Lines • Show All 823 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
int rc = 0, i, intr_idx, iqidx; | int rc = 0, i, intr_idx, iqidx; | ||||
struct sge_rxq *rxq; | struct sge_rxq *rxq; | ||||
struct sge_txq *txq; | struct sge_txq *txq; | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
struct sge_ofld_rxq *ofld_rxq; | struct sge_ofld_rxq *ofld_rxq; | ||||
#endif | #endif | ||||
#if defined(TCP_OFFLOAD) || defined(RATELIMIT) | #if defined(TCP_OFFLOAD) || defined(RATELIMIT) | ||||
struct sge_wrq *ofld_txq; | struct sge_ofld_txq *ofld_txq; | ||||
#endif | #endif | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
int saved_idx; | int saved_idx; | ||||
struct sge_nm_rxq *nm_rxq; | struct sge_nm_rxq *nm_rxq; | ||||
struct sge_nm_txq *nm_txq; | struct sge_nm_txq *nm_txq; | ||||
#endif | #endif | ||||
char name[16]; | char name[16]; | ||||
struct port_info *pi = vi->pi; | struct port_info *pi = vi->pi; | ||||
▲ Show 20 Lines • Show All 102 Lines • ▼ Show 20 Lines | for_each_txq(vi, i, txq) { | ||||
rc = alloc_txq(vi, txq, i, oid); | rc = alloc_txq(vi, txq, i, oid); | ||||
if (rc != 0) | if (rc != 0) | ||||
goto done; | goto done; | ||||
} | } | ||||
#if defined(TCP_OFFLOAD) || defined(RATELIMIT) | #if defined(TCP_OFFLOAD) || defined(RATELIMIT) | ||||
oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq", | oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq", | ||||
CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queues for TOE/ETHOFLD"); | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queues for TOE/ETHOFLD"); | ||||
for_each_ofld_txq(vi, i, ofld_txq) { | for_each_ofld_txq(vi, i, ofld_txq) { | ||||
struct sysctl_oid *oid2; | |||||
snprintf(name, sizeof(name), "%s ofld_txq%d", | snprintf(name, sizeof(name), "%s ofld_txq%d", | ||||
device_get_nameunit(vi->dev), i); | device_get_nameunit(vi->dev), i); | ||||
if (vi->nofldrxq > 0) { | if (vi->nofldrxq > 0) { | ||||
iqidx = vi->first_ofld_rxq + (i % vi->nofldrxq); | iqidx = vi->first_ofld_rxq + (i % vi->nofldrxq); | ||||
init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, | init_eq(sc, &ofld_txq->wrq.eq, EQ_OFLD, vi->qsize_txq, | ||||
pi->tx_chan, sc->sge.ofld_rxq[iqidx].iq.cntxt_id, | pi->tx_chan, sc->sge.ofld_rxq[iqidx].iq.cntxt_id, | ||||
name); | name); | ||||
} else { | } else { | ||||
iqidx = vi->first_rxq + (i % vi->nrxq); | iqidx = vi->first_rxq + (i % vi->nrxq); | ||||
init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, | init_eq(sc, &ofld_txq->wrq.eq, EQ_OFLD, vi->qsize_txq, | ||||
pi->tx_chan, sc->sge.rxq[iqidx].iq.cntxt_id, name); | pi->tx_chan, sc->sge.rxq[iqidx].iq.cntxt_id, name); | ||||
} | } | ||||
snprintf(name, sizeof(name), "%d", i); | rc = alloc_ofld_txq(vi, ofld_txq, i, oid); | ||||
oid2 = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO, | |||||
name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue"); | |||||
rc = alloc_wrq(sc, vi, ofld_txq, oid2); | |||||
if (rc != 0) | if (rc != 0) | ||||
goto done; | goto done; | ||||
} | } | ||||
#endif | #endif | ||||
done: | done: | ||||
if (rc) | if (rc) | ||||
t4_teardown_vi_queues(vi); | t4_teardown_vi_queues(vi); | ||||
return (rc); | return (rc); | ||||
} | } | ||||
/* | /* | ||||
* Idempotent | * Idempotent | ||||
*/ | */ | ||||
int | int | ||||
t4_teardown_vi_queues(struct vi_info *vi) | t4_teardown_vi_queues(struct vi_info *vi) | ||||
{ | { | ||||
int i; | int i; | ||||
struct sge_rxq *rxq; | struct sge_rxq *rxq; | ||||
struct sge_txq *txq; | struct sge_txq *txq; | ||||
#if defined(TCP_OFFLOAD) || defined(RATELIMIT) | #if defined(TCP_OFFLOAD) || defined(RATELIMIT) | ||||
struct port_info *pi = vi->pi; | struct sge_ofld_txq *ofld_txq; | ||||
struct adapter *sc = pi->adapter; | |||||
struct sge_wrq *ofld_txq; | |||||
#endif | #endif | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
struct sge_ofld_rxq *ofld_rxq; | struct sge_ofld_rxq *ofld_rxq; | ||||
#endif | #endif | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
struct sge_nm_rxq *nm_rxq; | struct sge_nm_rxq *nm_rxq; | ||||
struct sge_nm_txq *nm_txq; | struct sge_nm_txq *nm_txq; | ||||
#endif | #endif | ||||
Show All 21 Lines | #endif | ||||
* (for egress updates, etc.). | * (for egress updates, etc.). | ||||
*/ | */ | ||||
for_each_txq(vi, i, txq) { | for_each_txq(vi, i, txq) { | ||||
free_txq(vi, txq); | free_txq(vi, txq); | ||||
} | } | ||||
#if defined(TCP_OFFLOAD) || defined(RATELIMIT) | #if defined(TCP_OFFLOAD) || defined(RATELIMIT) | ||||
for_each_ofld_txq(vi, i, ofld_txq) { | for_each_ofld_txq(vi, i, ofld_txq) { | ||||
free_wrq(sc, ofld_txq); | free_ofld_txq(vi, ofld_txq); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Then take down the rx queues. | * Then take down the rx queues. | ||||
*/ | */ | ||||
for_each_rxq(vi, i, rxq) { | for_each_rxq(vi, i, rxq) { | ||||
▲ Show 20 Lines • Show All 3,155 Lines • ▼ Show 20 Lines | free_txq(struct vi_info *vi, struct sge_txq *txq) | ||||
sglist_free(txq->gl); | sglist_free(txq->gl); | ||||
free(txq->sdesc, M_CXGBE); | free(txq->sdesc, M_CXGBE); | ||||
mp_ring_free(txq->r); | mp_ring_free(txq->r); | ||||
bzero(txq, sizeof(*txq)); | bzero(txq, sizeof(*txq)); | ||||
return (0); | return (0); | ||||
} | } | ||||
#if defined(TCP_OFFLOAD) || defined(RATELIMIT) | |||||
static int | |||||
alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx, | |||||
struct sysctl_oid *oid) | |||||
{ | |||||
struct adapter *sc = vi->adapter; | |||||
struct sysctl_oid_list *children; | |||||
char name[16]; | |||||
int rc; | |||||
children = SYSCTL_CHILDREN(oid); | |||||
snprintf(name, sizeof(name), "%d", idx); | |||||
oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, | |||||
CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue"); | |||||
rc = alloc_wrq(sc, vi, &ofld_txq->wrq, oid); | |||||
if (rc != 0) | |||||
return (rc); | |||||
return (rc); | |||||
} | |||||
static int | |||||
free_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq) | |||||
{ | |||||
struct adapter *sc = vi->adapter; | |||||
int rc; | |||||
rc = free_wrq(sc, &ofld_txq->wrq); | |||||
if (rc != 0) | |||||
return (rc); | |||||
bzero(ofld_txq, sizeof(*ofld_txq)); | |||||
return (0); | |||||
} | |||||
#endif | |||||
static void | static void | ||||
oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) | oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) | ||||
{ | { | ||||
bus_addr_t *ba = arg; | bus_addr_t *ba = arg; | ||||
KASSERT(nseg == 1, | KASSERT(nseg == 1, | ||||
("%s meant for single segment mappings only.", __func__)); | ("%s meant for single segment mappings only.", __func__)); | ||||
▲ Show 20 Lines • Show All 1,912 Lines • ▼ Show 20 Lines | if (__predict_false(cst->flags & EO_FLOWC_PENDING)) { | ||||
cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq]; | cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq]; | ||||
if (M_HASHTYPE_ISHASH(m0)) | if (M_HASHTYPE_ISHASH(m0)) | ||||
rss_hash = m0->m_pkthdr.flowid; | rss_hash = m0->m_pkthdr.flowid; | ||||
else | else | ||||
rss_hash = arc4random(); | rss_hash = arc4random(); | ||||
/* We assume RSS hashing */ | /* We assume RSS hashing */ | ||||
cst->iqid = vi->rss[rss_hash & rss_mask]; | cst->iqid = vi->rss[rss_hash & rss_mask]; | ||||
cst->eo_txq += rss_hash % vi->nofldtxq; | cst->eo_txq += rss_hash % vi->nofldtxq; | ||||
jhb: I changed the type of `cst->eo_txq` so that this line would still work (and it also seemed more… | |||||
rc = send_etid_flowc_wr(cst, pi, vi); | rc = send_etid_flowc_wr(cst, pi, vi); | ||||
if (rc != 0) | if (rc != 0) | ||||
goto done; | goto done; | ||||
} | } | ||||
if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) { | if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) { | ||||
rc = ENOBUFS; | rc = ENOBUFS; | ||||
goto done; | goto done; | ||||
▲ Show 20 Lines • Show All 104 Lines • Show Last 20 Lines |
I changed the type of cst->eo_txq so that this line would still work (and it also seemed more natural for eo_txq to point to the actual txq rather than just the embedded wrq).