Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/cxgbe/tom/t4_cpl_io.c
| Show First 20 Lines • Show All 613 Lines • ▼ Show 20 Lines | #endif | ||||
| sglist_reset(&sg); | sglist_reset(&sg); | ||||
| } | } | ||||
| if (i & 1) | if (i & 1) | ||||
| usgl->sge[i / 2].len[1] = htobe32(0); | usgl->sge[i / 2].len[1] = htobe32(0); | ||||
| KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p", | KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p", | ||||
| __func__, nsegs, start, stop)); | __func__, nsegs, start, stop)); | ||||
| } | } | ||||
| bool | |||||
| t4_push_raw_wr(struct adapter *sc, struct toepcb *toep, struct mbuf *m) | |||||
| { | |||||
| #ifdef INVARIANTS | |||||
| struct inpcb *inp = toep->inp; | |||||
| #endif | |||||
| struct wrqe *wr; | |||||
| struct ofld_tx_sdesc *txsd; | |||||
| u_int credits, plen; | |||||
| INP_WLOCK_ASSERT(inp); | |||||
| MPASS(mbuf_raw_wr(m)); | |||||
| plen = m->m_pkthdr.len; | |||||
| credits = howmany(plen, 16); | |||||
| if (credits > toep->tx_credits) | |||||
| return (false); | |||||
| wr = alloc_wrqe(roundup2(plen, 16), &toep->ofld_txq->wrq); | |||||
| if (wr == NULL) | |||||
| return (false); | |||||
jhb: In the first version of this I enqueued the mbuf into the `ulp_reclaim_pduq` as we do for iSCSI… | |||||
| m_copydata(m, 0, plen, wrtod(wr)); | |||||
| m_freem(m); | |||||
| toep->tx_credits -= credits; | |||||
| if (toep->tx_credits < MIN_OFLD_TX_CREDITS) | |||||
| toep->flags |= TPF_TX_SUSPENDED; | |||||
| KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); | |||||
| KASSERT(credits <= MAX_OFLD_TX_SDESC_CREDITS, | |||||
| ("%s: tx_credits %u too large", __func__, credits)); | |||||
| txsd = &toep->txsd[toep->txsd_pidx]; | |||||
| txsd->plen = 0; | |||||
| txsd->tx_credits = credits; | |||||
| if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) | |||||
| toep->txsd_pidx = 0; | |||||
| toep->txsd_avail--; | |||||
| t4_wrq_tx(sc, wr); | |||||
| return (true); | |||||
| } | |||||
| /* | /* | ||||
| * Max number of SGL entries an offload tx work request can have. This is 41 | * Max number of SGL entries an offload tx work request can have. This is 41 | ||||
| * (1 + 40) for a full 512B work request. | * (1 + 40) for a full 512B work request. | ||||
| * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40) | * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40) | ||||
| */ | */ | ||||
| #define OFLD_SGL_LEN (41) | #define OFLD_SGL_LEN (41) | ||||
| /* | /* | ||||
| Show All 16 Lines | t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) | ||||
| struct mbuf *sndptr, *m, *sb_sndptr; | struct mbuf *sndptr, *m, *sb_sndptr; | ||||
| struct fw_ofld_tx_data_wr *txwr; | struct fw_ofld_tx_data_wr *txwr; | ||||
| struct wrqe *wr; | struct wrqe *wr; | ||||
| u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; | u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; | ||||
| struct inpcb *inp = toep->inp; | struct inpcb *inp = toep->inp; | ||||
| struct tcpcb *tp = intotcpcb(inp); | struct tcpcb *tp = intotcpcb(inp); | ||||
| struct socket *so = inp->inp_socket; | struct socket *so = inp->inp_socket; | ||||
| struct sockbuf *sb = &so->so_snd; | struct sockbuf *sb = &so->so_snd; | ||||
| struct mbufq *pduq = &toep->ulp_pduq; | |||||
| int tx_credits, shove, compl, sowwakeup; | int tx_credits, shove, compl, sowwakeup; | ||||
| struct ofld_tx_sdesc *txsd; | struct ofld_tx_sdesc *txsd; | ||||
| bool nomap_mbuf_seen; | bool nomap_mbuf_seen; | ||||
| INP_WLOCK_ASSERT(inp); | INP_WLOCK_ASSERT(inp); | ||||
| KASSERT(toep->flags & TPF_FLOWC_WR_SENT, | KASSERT(toep->flags & TPF_FLOWC_WR_SENT, | ||||
| ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); | ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); | ||||
| Show All 28 Lines | #endif | ||||
| } | } | ||||
| txsd = &toep->txsd[toep->txsd_pidx]; | txsd = &toep->txsd[toep->txsd_pidx]; | ||||
| do { | do { | ||||
| tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); | tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); | ||||
| max_imm = max_imm_payload(tx_credits, 0); | max_imm = max_imm_payload(tx_credits, 0); | ||||
| max_nsegs = max_dsgl_nsegs(tx_credits, 0); | max_nsegs = max_dsgl_nsegs(tx_credits, 0); | ||||
| if (__predict_false((sndptr = mbufq_first(pduq)) != NULL)) { | |||||
| if (!t4_push_raw_wr(sc, toep, sndptr)) { | |||||
| toep->flags |= TPF_TX_SUSPENDED; | |||||
| return; | |||||
| } | |||||
| m = mbufq_dequeue(pduq); | |||||
| MPASS(m == sndptr); | |||||
| txsd = &toep->txsd[toep->txsd_pidx]; | |||||
| continue; | |||||
| } | |||||
| SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
| sowwakeup = drop; | sowwakeup = drop; | ||||
| if (drop) { | if (drop) { | ||||
| sbdrop_locked(sb, drop); | sbdrop_locked(sb, drop); | ||||
| drop = 0; | drop = 0; | ||||
| } | } | ||||
| sb_sndptr = sb->sb_sndptr; | sb_sndptr = sb->sb_sndptr; | ||||
| sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; | sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; | ||||
| ▲ Show 20 Lines • Show All 540 Lines • ▼ Show 20 Lines | |||||
| { | { | ||||
| if (ulp_mode(toep) == ULP_MODE_ISCSI) | if (ulp_mode(toep) == ULP_MODE_ISCSI) | ||||
| t4_push_pdus(sc, toep, drop); | t4_push_pdus(sc, toep, drop); | ||||
| else if (toep->flags & TPF_KTLS) | else if (toep->flags & TPF_KTLS) | ||||
| t4_push_ktls(sc, toep, drop); | t4_push_ktls(sc, toep, drop); | ||||
| else | else | ||||
| t4_push_frames(sc, toep, drop); | t4_push_frames(sc, toep, drop); | ||||
| } | |||||
| void | |||||
| t4_raw_wr_tx(struct adapter *sc, struct toepcb *toep, struct mbuf *m) | |||||
| { | |||||
| #ifdef INVARIANTS | |||||
| struct inpcb *inp = toep->inp; | |||||
| #endif | |||||
| INP_WLOCK_ASSERT(inp); | |||||
| /* | |||||
| * If there are other raw WRs enqueued, enqueue to preserve | |||||
| * FIFO ordering. | |||||
| */ | |||||
| if (!mbufq_empty(&toep->ulp_pduq)) { | |||||
| mbufq_enqueue(&toep->ulp_pduq, m); | |||||
| return; | |||||
| } | |||||
| /* | |||||
| * Cannot call t4_push_data here as that will lock so_snd and | |||||
| * some callers of this run in rx handlers with so_rcv locked. | |||||
| * Instead, just try to transmit this WR. | |||||
| */ | |||||
| if (!t4_push_raw_wr(sc, toep, m)) { | |||||
| mbufq_enqueue(&toep->ulp_pduq, m); | |||||
| toep->flags |= TPF_TX_SUSPENDED; | |||||
| } | |||||
| } | } | ||||
| int | int | ||||
| t4_tod_output(struct toedev *tod, struct tcpcb *tp) | t4_tod_output(struct toedev *tod, struct tcpcb *tp) | ||||
| { | { | ||||
| struct adapter *sc = tod->tod_softc; | struct adapter *sc = tod->tod_softc; | ||||
| #ifdef INVARIANTS | #ifdef INVARIANTS | ||||
| struct inpcb *inp = tptoinpcb(tp); | struct inpcb *inp = tptoinpcb(tp); | ||||
| ▲ Show 20 Lines • Show All 1,206 Lines • Show Last 20 Lines | |||||
In the first version of this I enqueued the mbuf into the ulp_reclaim_pduq as we do for iSCSI WRs, but then I realized that I didn't need to do that once the data is copied. iSCSI WRs are a bit different as they can represent DMA and might need to be freed until ACKed, but the "control" WRs are always immediate actions and the mbuf is never holding onto other resources.
ULP_MODE_TLS is also kind of special as it abuses ulp_reclaim_pduq to store received CPL_RX_TLS_DATA mbufs that are later attached to the header from CPL_RX_TLS_CMP to build a TLS PDU, so I also had to change that before simplifying the approach here.