Differential D29382 Diff 86372 sys/dev/cxgbe/t4_sge.c

Changeset View

Standalone View

sys/dev/cxgbe/t4_sge.c

Show First 20 Lines • Show All 264 Lines • ▼ Show 20 Lines
static int alloc_eq(struct adapter , struct vi_info , struct sge_eq *);		static int alloc_eq(struct adapter , struct vi_info , struct sge_eq *);
static int free_eq(struct adapter , struct sge_eq );		static int free_eq(struct adapter , struct sge_eq );
static int alloc_wrq(struct adapter , struct vi_info , struct sge_wrq *,		static int alloc_wrq(struct adapter , struct vi_info , struct sge_wrq *,
struct sysctl_oid *);		struct sysctl_oid *);
static int free_wrq(struct adapter , struct sge_wrq );		static int free_wrq(struct adapter , struct sge_wrq );
static int alloc_txq(struct vi_info , struct sge_txq , int,		static int alloc_txq(struct vi_info , struct sge_txq , int,
struct sysctl_oid *);		struct sysctl_oid *);
static int free_txq(struct vi_info , struct sge_txq );		static int free_txq(struct vi_info , struct sge_txq );
		#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
		static int alloc_ofld_txq(struct vi_info , struct sge_ofld_txq , int,
		struct sysctl_oid *);
		static int free_ofld_txq(struct vi_info , struct sge_ofld_txq );
		#endif
static void oneseg_dma_callback(void , bus_dma_segment_t , int, int);		static void oneseg_dma_callback(void , bus_dma_segment_t , int, int);
static inline void ring_fl_db(struct adapter , struct sge_fl );		static inline void ring_fl_db(struct adapter , struct sge_fl );
static int refill_fl(struct adapter , struct sge_fl , int);		static int refill_fl(struct adapter , struct sge_fl , int);
static void refill_sfl(void *);		static void refill_sfl(void *);
static int alloc_fl_sdesc(struct sge_fl *);		static int alloc_fl_sdesc(struct sge_fl *);
static void free_fl_sdesc(struct adapter , struct sge_fl );		static void free_fl_sdesc(struct adapter , struct sge_fl );
static int find_refill_source(struct adapter *, int, bool);		static int find_refill_source(struct adapter *, int, bool);
static void add_fl_to_sfl(struct adapter , struct sge_fl );		static void add_fl_to_sfl(struct adapter , struct sge_fl );
▲ Show 20 Lines • Show All 823 Lines • ▼ Show 20 Lines
{		{
int rc = 0, i, intr_idx, iqidx;		int rc = 0, i, intr_idx, iqidx;
struct sge_rxq *rxq;		struct sge_rxq *rxq;
struct sge_txq *txq;		struct sge_txq *txq;
#ifdef TCP_OFFLOAD		#ifdef TCP_OFFLOAD
struct sge_ofld_rxq *ofld_rxq;		struct sge_ofld_rxq *ofld_rxq;
#endif		#endif
#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)		#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
struct sge_wrq *ofld_txq;		struct sge_ofld_txq *ofld_txq;
#endif		#endif
#ifdef DEV_NETMAP		#ifdef DEV_NETMAP
int saved_idx;		int saved_idx;
struct sge_nm_rxq *nm_rxq;		struct sge_nm_rxq *nm_rxq;
struct sge_nm_txq *nm_txq;		struct sge_nm_txq *nm_txq;
#endif		#endif
char name[16];		char name[16];
struct port_info *pi = vi->pi;		struct port_info *pi = vi->pi;
▲ Show 20 Lines • Show All 102 Lines • ▼ Show 20 Lines	for_each_txq(vi, i, txq) {
rc = alloc_txq(vi, txq, i, oid);		rc = alloc_txq(vi, txq, i, oid);
if (rc != 0)		if (rc != 0)
goto done;		goto done;
}		}
#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)		#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq",		oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq",
CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "tx queues for TOE/ETHOFLD");		CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "tx queues for TOE/ETHOFLD");
for_each_ofld_txq(vi, i, ofld_txq) {		for_each_ofld_txq(vi, i, ofld_txq) {
struct sysctl_oid *oid2;

snprintf(name, sizeof(name), "%s ofld_txq%d",		snprintf(name, sizeof(name), "%s ofld_txq%d",
device_get_nameunit(vi->dev), i);		device_get_nameunit(vi->dev), i);
if (vi->nofldrxq > 0) {		if (vi->nofldrxq > 0) {
iqidx = vi->first_ofld_rxq + (i % vi->nofldrxq);		iqidx = vi->first_ofld_rxq + (i % vi->nofldrxq);
init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq,		init_eq(sc, &ofld_txq->wrq.eq, EQ_OFLD, vi->qsize_txq,
pi->tx_chan, sc->sge.ofld_rxq[iqidx].iq.cntxt_id,		pi->tx_chan, sc->sge.ofld_rxq[iqidx].iq.cntxt_id,
name);		name);
} else {		} else {
iqidx = vi->first_rxq + (i % vi->nrxq);		iqidx = vi->first_rxq + (i % vi->nrxq);
init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq,		init_eq(sc, &ofld_txq->wrq.eq, EQ_OFLD, vi->qsize_txq,
pi->tx_chan, sc->sge.rxq[iqidx].iq.cntxt_id, name);		pi->tx_chan, sc->sge.rxq[iqidx].iq.cntxt_id, name);
}		}

snprintf(name, sizeof(name), "%d", i);		rc = alloc_ofld_txq(vi, ofld_txq, i, oid);
oid2 = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
name, CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "offload tx queue");

rc = alloc_wrq(sc, vi, ofld_txq, oid2);
if (rc != 0)		if (rc != 0)
goto done;		goto done;
}		}
#endif		#endif
done:		done:
if (rc)		if (rc)
t4_teardown_vi_queues(vi);		t4_teardown_vi_queues(vi);

return (rc);		return (rc);
}		}

/*		/*
* Idempotent		* Idempotent
*/		*/
int		int
t4_teardown_vi_queues(struct vi_info *vi)		t4_teardown_vi_queues(struct vi_info *vi)
{		{
int i;		int i;
struct sge_rxq *rxq;		struct sge_rxq *rxq;
struct sge_txq *txq;		struct sge_txq *txq;
#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)		#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
struct port_info *pi = vi->pi;		struct sge_ofld_txq *ofld_txq;
struct adapter *sc = pi->adapter;
struct sge_wrq *ofld_txq;
#endif		#endif
#ifdef TCP_OFFLOAD		#ifdef TCP_OFFLOAD
struct sge_ofld_rxq *ofld_rxq;		struct sge_ofld_rxq *ofld_rxq;
#endif		#endif
#ifdef DEV_NETMAP		#ifdef DEV_NETMAP
struct sge_nm_rxq *nm_rxq;		struct sge_nm_rxq *nm_rxq;
struct sge_nm_txq *nm_txq;		struct sge_nm_txq *nm_txq;
#endif		#endif
Show All 21 Lines	#endif
* (for egress updates, etc.).		* (for egress updates, etc.).
*/		*/

for_each_txq(vi, i, txq) {		for_each_txq(vi, i, txq) {
free_txq(vi, txq);		free_txq(vi, txq);
}		}
#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)		#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
for_each_ofld_txq(vi, i, ofld_txq) {		for_each_ofld_txq(vi, i, ofld_txq) {
free_wrq(sc, ofld_txq);		free_ofld_txq(vi, ofld_txq);
}		}
#endif		#endif

/*		/*
* Then take down the rx queues.		* Then take down the rx queues.
*/		*/

for_each_rxq(vi, i, rxq) {		for_each_rxq(vi, i, rxq) {
▲ Show 20 Lines • Show All 3,156 Lines • ▼ Show 20 Lines	free_txq(struct vi_info vi, struct sge_txq txq)
sglist_free(txq->gl);		sglist_free(txq->gl);
free(txq->sdesc, M_CXGBE);		free(txq->sdesc, M_CXGBE);
mp_ring_free(txq->r);		mp_ring_free(txq->r);

bzero(txq, sizeof(*txq));		bzero(txq, sizeof(*txq));
return (0);		return (0);
}		}

		#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
		static int
		alloc_ofld_txq(struct vi_info vi, struct sge_ofld_txq ofld_txq, int idx,
		struct sysctl_oid *oid)
		{
		struct adapter *sc = vi->adapter;
		struct sysctl_oid_list *children;
		char name[16];
		int rc;

		children = SYSCTL_CHILDREN(oid);

		snprintf(name, sizeof(name), "%d", idx);
		oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name,
		CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "offload tx queue");

		rc = alloc_wrq(sc, vi, &ofld_txq->wrq, oid);
		if (rc != 0)
		return (rc);

		return (rc);
		}

		static int
		free_ofld_txq(struct vi_info vi, struct sge_ofld_txq ofld_txq)
		{
		struct adapter *sc = vi->adapter;
		int rc;

		rc = free_wrq(sc, &ofld_txq->wrq);
		if (rc != 0)
		return (rc);

		bzero(ofld_txq, sizeof(*ofld_txq));
		return (0);
		}
		#endif

static void		static void
oneseg_dma_callback(void arg, bus_dma_segment_t segs, int nseg, int error)		oneseg_dma_callback(void arg, bus_dma_segment_t segs, int nseg, int error)
{		{
bus_addr_t *ba = arg;		bus_addr_t *ba = arg;

KASSERT(nseg == 1,		KASSERT(nseg == 1,
("%s meant for single segment mappings only.", __func__));		("%s meant for single segment mappings only.", __func__));

▲ Show 20 Lines • Show All 1,610 Lines • ▼ Show 20 Lines	send_etid_flowc_wr(struct cxgbe_rate_tag cst, struct port_info pi,
struct wrq_cookie cookie;		struct wrq_cookie cookie;
u_int pfvf = pi->adapter->pf << S_FW_VIID_PFN;		u_int pfvf = pi->adapter->pf << S_FW_VIID_PFN;
struct fw_flowc_wr *flowc;		struct fw_flowc_wr *flowc;

mtx_assert(&cst->lock, MA_OWNED);		mtx_assert(&cst->lock, MA_OWNED);
MPASS((cst->flags & (EO_FLOWC_PENDING \| EO_FLOWC_RPL_PENDING)) ==		MPASS((cst->flags & (EO_FLOWC_PENDING \| EO_FLOWC_RPL_PENDING)) ==
EO_FLOWC_PENDING);		EO_FLOWC_PENDING);

flowc = start_wrq_wr(cst->eo_txq, ETID_FLOWC_LEN16, &cookie);		flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLOWC_LEN16, &cookie);
if (__predict_false(flowc == NULL))		if (__predict_false(flowc == NULL))
return (ENOMEM);		return (ENOMEM);

bzero(flowc, ETID_FLOWC_LEN);		bzero(flowc, ETID_FLOWC_LEN);
flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) \|		flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) \|
V_FW_FLOWC_WR_NPARAMS(ETID_FLOWC_NPARAMS) \| V_FW_WR_COMPL(0));		V_FW_FLOWC_WR_NPARAMS(ETID_FLOWC_NPARAMS) \| V_FW_WR_COMPL(0));
flowc->flowid_len16 = htonl(V_FW_WR_LEN16(ETID_FLOWC_LEN16) \|		flowc->flowid_len16 = htonl(V_FW_WR_LEN16(ETID_FLOWC_LEN16) \|
V_FW_WR_FLOWID(cst->etid));		V_FW_WR_FLOWID(cst->etid));
flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;		flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
flowc->mnemval[0].val = htobe32(pfvf);		flowc->mnemval[0].val = htobe32(pfvf);
flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;		flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
flowc->mnemval[1].val = htobe32(pi->tx_chan);		flowc->mnemval[1].val = htobe32(pi->tx_chan);
flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;		flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
flowc->mnemval[2].val = htobe32(pi->tx_chan);		flowc->mnemval[2].val = htobe32(pi->tx_chan);
flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;		flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
flowc->mnemval[3].val = htobe32(cst->iqid);		flowc->mnemval[3].val = htobe32(cst->iqid);
flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE;		flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE;
flowc->mnemval[4].val = htobe32(FW_FLOWC_MNEM_EOSTATE_ESTABLISHED);		flowc->mnemval[4].val = htobe32(FW_FLOWC_MNEM_EOSTATE_ESTABLISHED);
flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;		flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;
flowc->mnemval[5].val = htobe32(cst->schedcl);		flowc->mnemval[5].val = htobe32(cst->schedcl);

commit_wrq_wr(cst->eo_txq, flowc, &cookie);		commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie);

cst->flags &= ~EO_FLOWC_PENDING;		cst->flags &= ~EO_FLOWC_PENDING;
cst->flags \|= EO_FLOWC_RPL_PENDING;		cst->flags \|= EO_FLOWC_RPL_PENDING;
MPASS(cst->tx_credits >= ETID_FLOWC_LEN16); /* flowc is first WR. */		MPASS(cst->tx_credits >= ETID_FLOWC_LEN16); /* flowc is first WR. */
cst->tx_credits -= ETID_FLOWC_LEN16;		cst->tx_credits -= ETID_FLOWC_LEN16;

return (0);		return (0);
}		}

#define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16))		#define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16))

void		void
send_etid_flush_wr(struct cxgbe_rate_tag *cst)		send_etid_flush_wr(struct cxgbe_rate_tag *cst)
{		{
struct fw_flowc_wr *flowc;		struct fw_flowc_wr *flowc;
struct wrq_cookie cookie;		struct wrq_cookie cookie;

mtx_assert(&cst->lock, MA_OWNED);		mtx_assert(&cst->lock, MA_OWNED);

flowc = start_wrq_wr(cst->eo_txq, ETID_FLUSH_LEN16, &cookie);		flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLUSH_LEN16, &cookie);
if (__predict_false(flowc == NULL))		if (__predict_false(flowc == NULL))
CXGBE_UNIMPLEMENTED(__func__);		CXGBE_UNIMPLEMENTED(__func__);

bzero(flowc, ETID_FLUSH_LEN16 * 16);		bzero(flowc, ETID_FLUSH_LEN16 * 16);
flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) \|		flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) \|
V_FW_FLOWC_WR_NPARAMS(0) \| F_FW_WR_COMPL);		V_FW_FLOWC_WR_NPARAMS(0) \| F_FW_WR_COMPL);
flowc->flowid_len16 = htobe32(V_FW_WR_LEN16(ETID_FLUSH_LEN16) \|		flowc->flowid_len16 = htobe32(V_FW_WR_LEN16(ETID_FLUSH_LEN16) \|
V_FW_WR_FLOWID(cst->etid));		V_FW_WR_FLOWID(cst->etid));

commit_wrq_wr(cst->eo_txq, flowc, &cookie);		commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie);

cst->flags \|= EO_FLUSH_RPL_PENDING;		cst->flags \|= EO_FLUSH_RPL_PENDING;
MPASS(cst->tx_credits >= ETID_FLUSH_LEN16);		MPASS(cst->tx_credits >= ETID_FLUSH_LEN16);
cst->tx_credits -= ETID_FLUSH_LEN16;		cst->tx_credits -= ETID_FLUSH_LEN16;
cst->ncompl++;		cst->ncompl++;
}		}

static void		static void
▲ Show 20 Lines • Show All 168 Lines • ▼ Show 20 Lines	if (next_credits > cst->tx_credits) {
/*		/*
* Tx will make progress eventually because there is at		* Tx will make progress eventually because there is at
* least one outstanding fw4_ack that will return		* least one outstanding fw4_ack that will return
* credits and kick the tx.		* credits and kick the tx.
*/		*/
MPASS(cst->ncompl > 0);		MPASS(cst->ncompl > 0);
return;		return;
}		}
wr = start_wrq_wr(cst->eo_txq, next_credits, &cookie);		wr = start_wrq_wr(&cst->eo_txq->wrq, next_credits, &cookie);
if (__predict_false(wr == NULL)) {		if (__predict_false(wr == NULL)) {
/* XXX: wishful thinking, not a real assertion. */		/* XXX: wishful thinking, not a real assertion. */
MPASS(cst->ncompl > 0);		MPASS(cst->ncompl > 0);
return;		return;
}		}
cst->tx_credits -= next_credits;		cst->tx_credits -= next_credits;
cst->tx_nocompl += next_credits;		cst->tx_nocompl += next_credits;
compl = cst->ncompl == 0 \|\| cst->tx_nocompl >= cst->tx_total / 2;		compl = cst->ncompl == 0 \|\| cst->tx_nocompl >= cst->tx_total / 2;
ETHER_BPF_MTAP(cst->com.ifp, m);		ETHER_BPF_MTAP(cst->com.ifp, m);
write_ethofld_wr(cst, wr, m, compl);		write_ethofld_wr(cst, wr, m, compl);
commit_wrq_wr(cst->eo_txq, wr, &cookie);		commit_wrq_wr(&cst->eo_txq->wrq, wr, &cookie);
if (compl) {		if (compl) {
cst->ncompl++;		cst->ncompl++;
cst->tx_nocompl = 0;		cst->tx_nocompl = 0;
}		}
(void) mbufq_dequeue(&cst->pending_tx);		(void) mbufq_dequeue(&cst->pending_tx);

/*		/*
* Drop the mbuf's reference on the tag now rather		* Drop the mbuf's reference on the tag now rather
Show All 36 Lines	if (__predict_false(cst->flags & EO_FLOWC_PENDING)) {

cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq];		cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq];
if (M_HASHTYPE_ISHASH(m0))		if (M_HASHTYPE_ISHASH(m0))
rss_hash = m0->m_pkthdr.flowid;		rss_hash = m0->m_pkthdr.flowid;
else		else
rss_hash = arc4random();		rss_hash = arc4random();
/* We assume RSS hashing */		/* We assume RSS hashing */
cst->iqid = vi->rss[rss_hash & rss_mask];		cst->iqid = vi->rss[rss_hash & rss_mask];
cst->eo_txq += rss_hash % vi->nofldtxq;		cst->eo_txq += rss_hash % vi->nofldtxq;
		jhbAuthorUnsubmitted Done Inline Actions I changed the type of `cst->eo_txq` so that this line would still work (and it also seemed more natural for `eo_txq` to point to the actual txq rather than just the embedded wrq). jhb: I changed the type of `cst->eo_txq` so that this line would still work (and it also seemed more…
rc = send_etid_flowc_wr(cst, pi, vi);		rc = send_etid_flowc_wr(cst, pi, vi);
if (rc != 0)		if (rc != 0)
goto done;		goto done;
}		}

if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) {		if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) {
rc = ENOBUFS;		rc = ENOBUFS;
goto done;		goto done;
▲ Show 20 Lines • Show All 104 Lines • Show Last 20 Lines