Page MenuHomeFreeBSD
Differential D29904 Diff 87909 sys/dev/cxgbe/tom/t4_cpl_io.c

Changeset View

Standalone View

View Options

sys/dev/cxgbe/tom/t4_cpl_io.c

Show First 20 Lines • Show All 921 Lines • ▼ Show 20 Lines while (plen > 0) {
/* Partial credits. */ /* Partial credits. */
MPASS(plen >= m->m_pkthdr.len); MPASS(plen >= m->m_pkthdr.len);
plen -= m->m_pkthdr.len; plen -= m->m_pkthdr.len;
m_freem(m); m_freem(m);
} }
} }
void static struct wrqe *
t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
{ {
struct mbuf *sndptr, *m; struct mbuf *m;
struct fw_ofld_tx_data_wr *txwr; struct fw_ofld_tx_data_wr *txwr;
struct wrqe *wr; struct wrqe *wr;
u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
u_int adjusted_plen, ulp_submode; u_int adjusted_plen, ulp_submode;
struct inpcb *inp = toep->inp; struct inpcb *inp = toep->inp;
struct tcpcb *tp = intotcpcb(inp); struct tcpcb *tp = intotcpcb(inp);
int tx_credits, shove; int tx_credits, shove;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
struct mbufq *pduq = &toep->ulp_pduq;
static const u_int ulp_extra_len[] = {0, 4, 4, 8}; static const u_int ulp_extra_len[] = {0, 4, 4, 8};
INP_WLOCK_ASSERT(inp); M_ASSERTPKTHDR(sndptr);
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
KASSERT(ulp_mode(toep) == ULP_MODE_ISCSI,
("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
return; if (mbuf_raw_wr(sndptr)) {
plen = sndptr->m_pkthdr.len;
KASSERT(plen <= SGE_MAX_WR_LEN,
("raw WR len %u is greater than max WR len", plen));
if (plen > tx_credits * 16)
return (NULL);
/* wr = alloc_wrqe(roundup2(plen, 16), &toep->ofld_txq->wrq);
* This function doesn't resume by itself. Someone else must clear the if (__predict_false(wr == NULL))
* flag and call this function. return (NULL);
*/
if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { m_copydata(sndptr, 0, plen, wrtod(wr));
KASSERT(drop == 0, return (wr);
("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
return;
} }
if (drop)
rqdrop_locked(&toep->ulp_pdu_reclaimq, drop);
while ((sndptr = mbufq_first(pduq)) != NULL) {
M_ASSERTPKTHDR(sndptr);
tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
max_imm = max_imm_payload(tx_credits); max_imm = max_imm_payload(tx_credits);
max_nsegs = max_dsgl_nsegs(tx_credits); max_nsegs = max_dsgl_nsegs(tx_credits);
plen = 0; plen = 0;
nsegs = 0; nsegs = 0;
max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
for (m = sndptr; m != NULL; m = m->m_next) { for (m = sndptr; m != NULL; m = m->m_next) {
int n = sglist_count(mtod(m, void *), m->m_len); int n = sglist_count(mtod(m, void *), m->m_len);
nsegs += n; nsegs += n;
plen += m->m_len; plen += m->m_len;
/* /*
* This mbuf would send us _over_ the nsegs limit. * This mbuf would send us _over_ the nsegs limit.
* Suspend tx because the PDU can't be sent out. * Suspend tx because the PDU can't be sent out.
*/ */
if (plen > max_imm && nsegs > max_nsegs) { if (plen > max_imm && nsegs > max_nsegs)
toep->flags |= TPF_TX_SUSPENDED; return (NULL);
return;
}
if (max_nsegs_1mbuf < n) if (max_nsegs_1mbuf < n)
max_nsegs_1mbuf = n; max_nsegs_1mbuf = n;
} }
if (__predict_false(toep->flags & TPF_FIN_SENT)) if (__predict_false(toep->flags & TPF_FIN_SENT))
panic("%s: excess tx.", __func__); panic("%s: excess tx.", __func__);
/* /*
* We have a PDU to send. All of it goes out in one WR so 'm' * We have a PDU to send. All of it goes out in one WR so 'm'
* is NULL. A PDU's length is always a multiple of 4. * is NULL. A PDU's length is always a multiple of 4.
*/ */
MPASS(m == NULL); MPASS(m == NULL);
MPASS((plen & 3) == 0); MPASS((plen & 3) == 0);
MPASS(sndptr->m_pkthdr.len == plen); MPASS(sndptr->m_pkthdr.len == plen);
shove = !(tp->t_flags & TF_MORETOCOME); shove = !(tp->t_flags & TF_MORETOCOME);
ulp_submode = mbuf_ulp_submode(sndptr); ulp_submode = mbuf_ulp_submode(sndptr);
MPASS(ulp_submode < nitems(ulp_extra_len)); MPASS(ulp_submode < nitems(ulp_extra_len));
/* /*
* plen doesn't include header and data digests, which are * plen doesn't include header and data digests, which are
* generated and inserted in the right places by the TOE, but * generated and inserted in the right places by the TOE, but
* they do occupy TCP sequence space and need to be accounted * they do occupy TCP sequence space and need to be accounted
* for. * for.
*/ */
adjusted_plen = plen + ulp_extra_len[ulp_submode]; adjusted_plen = plen + ulp_extra_len[ulp_submode];
if (plen <= max_imm) { if (plen <= max_imm) {
/* Immediate data tx */ /* Immediate data tx */
wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
&toep->ofld_txq->wrq); &toep->ofld_txq->wrq);
if (wr == NULL) { if (wr == NULL) {
/* XXX: how will we recover from this? */ /* XXX: how will we recover from this? */
toep->flags |= TPF_TX_SUSPENDED; return (NULL);
return;
} }
txwr = wrtod(wr); txwr = wrtod(wr);
credits = howmany(wr->wr_len, 16); credits = howmany(wr->wr_len, 16);
write_tx_wr(txwr, toep, plen, adjusted_plen, credits, write_tx_wr(txwr, toep, plen, adjusted_plen, credits,
shove, ulp_submode); shove, ulp_submode);
m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
nsegs = 0; nsegs = 0;
} else { } else {
int wr_len; int wr_len;
/* DSGL tx */ /* DSGL tx */
wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
wr = alloc_wrqe(roundup2(wr_len, 16), wr = alloc_wrqe(roundup2(wr_len, 16),
&toep->ofld_txq->wrq); &toep->ofld_txq->wrq);
if (wr == NULL) { if (wr == NULL) {
/* XXX: how will we recover from this? */ /* XXX: how will we recover from this? */
toep->flags |= TPF_TX_SUSPENDED; return (NULL);
return;
} }
txwr = wrtod(wr); txwr = wrtod(wr);
credits = howmany(wr_len, 16); credits = howmany(wr_len, 16);
write_tx_wr(txwr, toep, 0, adjusted_plen, credits, write_tx_wr(txwr, toep, 0, adjusted_plen, credits,
shove, ulp_submode); shove, ulp_submode);
write_tx_sgl(txwr + 1, sndptr, m, nsegs, write_tx_sgl(txwr + 1, sndptr, m, nsegs, max_nsegs_1mbuf);
max_nsegs_1mbuf);
if (wr_len & 0xf) { if (wr_len & 0xf) {
uint64_t *pad = (uint64_t *) uint64_t *pad = (uint64_t *)((uintptr_t)txwr + wr_len);
((uintptr_t)txwr + wr_len);
*pad = 0; *pad = 0;
} }
} }
tp->snd_nxt += adjusted_plen;
tp->snd_max += adjusted_plen;
counter_u64_add(toep->ofld_txq->tx_iscsi_pdus, 1);
counter_u64_add(toep->ofld_txq->tx_iscsi_octets, plen);
return (wr);
}
void
t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
{
struct mbuf *sndptr, *m;
struct fw_wr_hdr *wrhdr;
struct wrqe *wr;
u_int plen, credits;
struct inpcb *inp = toep->inp;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
struct mbufq *pduq = &toep->ulp_pduq;
INP_WLOCK_ASSERT(inp);
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
KASSERT(ulp_mode(toep) == ULP_MODE_ISCSI,
("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
return;
/*
* This function doesn't resume by itself. Someone else must clear the
* flag and call this function.
*/
if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
KASSERT(drop == 0,
("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
return;
}
if (drop)
rqdrop_locked(&toep->ulp_pdu_reclaimq, drop);
while ((sndptr = mbufq_first(pduq)) != NULL) {
wr = write_iscsi_mbuf_wr(toep, sndptr);
if (wr == NULL) {
toep->flags |= TPF_TX_SUSPENDED;
return;
}
plen = sndptr->m_pkthdr.len;
credits = howmany(wr->wr_len, 16);
KASSERT(toep->tx_credits >= credits, KASSERT(toep->tx_credits >= credits,
("%s: not enough credits", __func__)); ("%s: not enough credits", __func__));
m = mbufq_dequeue(pduq); m = mbufq_dequeue(pduq);
MPASS(m == sndptr); MPASS(m == sndptr);
mbufq_enqueue(&toep->ulp_pdu_reclaimq, m); mbufq_enqueue(&toep->ulp_pdu_reclaimq, m);
toep->tx_credits -= credits; toep->tx_credits -= credits;
toep->tx_nocompl += credits; toep->tx_nocompl += credits;
toep->plen_nocompl += plen; toep->plen_nocompl += plen;
if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
/*
* Since we are also sending ulp_tx WR through offload
* queue, make sure that this condition sets
* F_FW_WR_COMPL flag when the available credits is <=
* max ulp_tx WR size.
*/
if (toep->tx_credits <= toep->tx_total * 5 / 8 &&
jhbAuthorUnsubmitted
Done
Inline Actions

I'm not 100% certain of the reason for the change from 3/8 to 5/8 here, and the new comment above probably could use some tweaking? It seems like the new comment instead means something like forcing F_FW_WR_COMPL anytime toep->tx_credits < SGE_MAX_WR_LEN via a new condition rather than the 3/8 -> 5/8 change?

jhb: I'm not 100% certain of the reason for the change from 3/8 to 5/8 here, and the new comment…
npUnsubmitted
Not Done
Inline Actions

tx_total is 84 with default fw config so 3/8 of 84 is 31. These are len16 so that's 31 x 16 = 504B, which is less than SGE_MAX_WR_LEN. The updated driver probably sends a lot more full sized work requests for writing page pods so the 3/8 can cause tx flow control issues between driver/firmware.

1/2 would have worked, 5/8 works too. But they depend on tx_total being 84, which is not guaranteed. A more flexible test would be what you suggested, something like this:
if (toep->tx_credits <= SGE_MAX_WR_LEN * 5 / 4)

np: tx_total is 84 with default fw config so 3/8 of 84 is 31. These are len16 so that's 31 x 16 =…
toep->tx_nocompl >= toep->tx_total / 4) { toep->tx_nocompl >= toep->tx_total / 4) {
txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); wrhdr = wrtod(wr);
wrhdr->hi |= htobe32(F_FW_WR_COMPL);
toep->tx_nocompl = 0; toep->tx_nocompl = 0;
toep->plen_nocompl = 0; toep->plen_nocompl = 0;
} }
tp->snd_nxt += adjusted_plen;
tp->snd_max += adjusted_plen;
toep->flags |= TPF_TX_DATA_SENT; toep->flags |= TPF_TX_DATA_SENT;
if (toep->tx_credits < MIN_OFLD_TX_CREDITS) if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
toep->flags |= TPF_TX_SUSPENDED; toep->flags |= TPF_TX_SUSPENDED;
KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
txsd->plen = plen; txsd->plen = plen;
txsd->tx_credits = credits; txsd->tx_credits = credits;
txsd++; txsd++;
if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
toep->txsd_pidx = 0; toep->txsd_pidx = 0;
txsd = &toep->txsd[0]; txsd = &toep->txsd[0];
} }
toep->txsd_avail--; toep->txsd_avail--;
counter_u64_add(toep->ofld_txq->tx_iscsi_pdus, 1);
counter_u64_add(toep->ofld_txq->tx_iscsi_octets, plen);
t4_l2t_send(sc, wr, toep->l2te); t4_l2t_send(sc, wr, toep->l2te);
} }
/* Send a FIN if requested, but only if there are no more PDUs to send */ /* Send a FIN if requested, but only if there are no more PDUs to send */
if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
t4_close_conn(sc, toep); t4_close_conn(sc, toep);
} }
▲ Show 20 Lines • Show All 1,212 Lines • Show Last 20 Lines