diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index ca04cb88b10f..4c558df3f258 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -1,2455 +1,2455 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2015 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include "opt_kern_tls.h" #include "opt_ratelimit.h" #ifdef TCP_OFFLOAD #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define TCPSTATES #include #include #include #include #include #include #include #include #include #include #include #include "common/common.h" #include "common/t4_msg.h" #include "common/t4_regs.h" #include "common/t4_tcb.h" #include "tom/t4_tom_l2t.h" #include "tom/t4_tom.h" static void t4_aiotx_cancel(struct kaiocb *job); static void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep); void send_flowc_wr(struct toepcb *toep, struct tcpcb *tp) { struct wrqe *wr; struct fw_flowc_wr *flowc; unsigned int nparams, flowclen, paramidx; struct vi_info *vi = toep->vi; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; unsigned int pfvf = sc->pf << S_FW_VIID_PFN; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT), ("%s: flowc for tid %u sent already", __func__, toep->tid)); if (tp != NULL) nparams = 8; else nparams = 6; if (ulp_mode(toep) == ULP_MODE_TLS) nparams++; if (toep->tls.fcplenmax != 0) nparams++; if (toep->params.tc_idx != -1) { MPASS(toep->params.tc_idx >= 0 && toep->params.tc_idx < sc->params.nsched_cls); nparams++; } flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } flowc = wrtod(wr); memset(flowc, 0, wr->wr_len); flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | V_FW_FLOWC_WR_NPARAMS(nparams)); flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | V_FW_WR_FLOWID(toep->tid)); #define FLOWC_PARAM(__m, __v) \ do { \ flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \ flowc->mnemval[paramidx].val = htobe32(__v); \ paramidx++; \ } while (0) paramidx = 0; FLOWC_PARAM(PFNVFN, pfvf); FLOWC_PARAM(CH, pi->tx_chan); FLOWC_PARAM(PORT, pi->tx_chan); FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id); FLOWC_PARAM(SNDBUF, toep->params.sndbuf); if (tp) { FLOWC_PARAM(MSS, toep->params.emss); FLOWC_PARAM(SNDNXT, tp->snd_nxt); FLOWC_PARAM(RCVNXT, tp->rcv_nxt); } else FLOWC_PARAM(MSS, 512); CTR6(KTR_CXGBE, "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x", __func__, toep->tid, toep->params.emss, toep->params.sndbuf, tp ? tp->snd_nxt : 0, tp ? tp->rcv_nxt : 0); if (ulp_mode(toep) == ULP_MODE_TLS) FLOWC_PARAM(ULP_MODE, ulp_mode(toep)); if (toep->tls.fcplenmax != 0) FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax); if (toep->params.tc_idx != -1) FLOWC_PARAM(SCHEDCLASS, toep->params.tc_idx); #undef FLOWC_PARAM KASSERT(paramidx == nparams, ("nparams mismatch")); txsd->tx_credits = howmany(flowclen, 16); txsd->plen = 0; KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, ("%s: not enough credits (%d)", __func__, toep->tx_credits)); toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; toep->flags |= TPF_FLOWC_WR_SENT; t4_wrq_tx(sc, wr); } #ifdef RATELIMIT /* * Input is Bytes/second (so_max_pacing_rate), chip counts in Kilobits/second. */ static int update_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps) { int tc_idx, rc; const u_int kbps = (u_int) (uint64_t)Bps * 8ULL / 1000; const int port_id = toep->vi->pi->port_id; CTR3(KTR_CXGBE, "%s: tid %u, rate %uKbps", __func__, toep->tid, kbps); if (kbps == 0) { /* unbind */ tc_idx = -1; } else { rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx); if (rc != 0) return (rc); MPASS(tc_idx >= 0 && tc_idx < sc->params.nsched_cls); } if (toep->params.tc_idx != tc_idx) { struct wrqe *wr; struct fw_flowc_wr *flowc; int nparams = 1, flowclen, flowclen16; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); flowclen16 = howmany(flowclen, 16); if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0 || (wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq)) == NULL) { if (tc_idx >= 0) t4_release_cl_rl(sc, port_id, tc_idx); return (ENOMEM); } flowc = wrtod(wr); memset(flowc, 0, wr->wr_len); flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | V_FW_FLOWC_WR_NPARAMS(nparams)); flowc->flowid_len16 = htonl(V_FW_WR_LEN16(flowclen16) | V_FW_WR_FLOWID(toep->tid)); flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; if (tc_idx == -1) flowc->mnemval[0].val = htobe32(0xff); else flowc->mnemval[0].val = htobe32(tc_idx); txsd->tx_credits = flowclen16; txsd->plen = 0; toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; t4_wrq_tx(sc, wr); } if (toep->params.tc_idx >= 0) t4_release_cl_rl(sc, port_id, toep->params.tc_idx); toep->params.tc_idx = tc_idx; return (0); } #endif void send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt) { struct wrqe *wr; struct cpl_abort_req *req; int tid = toep->tid; struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */ INP_WLOCK_ASSERT(inp); CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s", __func__, toep->tid, inp->inp_flags & INP_DROPPED ? "inp dropped" : tcpstates[tp->t_state], toep->flags, inp->inp_flags, toep->flags & TPF_ABORT_SHUTDOWN ? " (abort already in progress)" : ""); if (toep->flags & TPF_ABORT_SHUTDOWN) return; /* abort already in progress */ toep->flags |= TPF_ABORT_SHUTDOWN; KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %d.", __func__, tid)); wr = alloc_wrqe(sizeof(*req), &toep->ofld_txq->wrq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid); if (inp->inp_flags & INP_DROPPED) req->rsvd0 = htobe32(snd_nxt); else req->rsvd0 = htobe32(tp->snd_nxt); req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT); req->cmd = CPL_ABORT_SEND_RST; /* * XXX: What's the correct way to tell that the inp hasn't been detached * from its socket? Should I even be flushing the snd buffer here? */ if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { struct socket *so = inp->inp_socket; if (so != NULL) /* because I'm not sure. See comment above */ sbflush(&so->so_snd); } t4_l2t_send(sc, wr, toep->l2te); } /* * Called when a connection is established to translate the TCP options * reported by HW to FreeBSD's native format. */ static void assign_rxopt(struct tcpcb *tp, uint16_t opt) { struct toepcb *toep = tp->t_toe; struct inpcb *inp = tp->t_inpcb; struct adapter *sc = td_adapter(toep->td); INP_LOCK_ASSERT(inp); toep->params.mtu_idx = G_TCPOPT_MSS(opt); tp->t_maxseg = sc->params.mtus[toep->params.mtu_idx]; if (inp->inp_inc.inc_flags & INC_ISIPV6) tp->t_maxseg -= sizeof(struct ip6_hdr) + sizeof(struct tcphdr); else tp->t_maxseg -= sizeof(struct ip) + sizeof(struct tcphdr); toep->params.emss = tp->t_maxseg; if (G_TCPOPT_TSTAMP(opt)) { toep->params.tstamp = 1; toep->params.emss -= TCPOLEN_TSTAMP_APPA; tp->t_flags |= TF_RCVD_TSTMP; /* timestamps ok */ tp->ts_recent = 0; /* hmmm */ tp->ts_recent_age = tcp_ts_getticks(); } else toep->params.tstamp = 0; if (G_TCPOPT_SACK(opt)) { toep->params.sack = 1; tp->t_flags |= TF_SACK_PERMIT; /* should already be set */ } else { toep->params.sack = 0; tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */ } if (G_TCPOPT_WSCALE_OK(opt)) tp->t_flags |= TF_RCVD_SCALE; /* Doing window scaling? */ if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == (TF_RCVD_SCALE | TF_REQ_SCALE)) { tp->rcv_scale = tp->request_r_scale; tp->snd_scale = G_TCPOPT_SND_WSCALE(opt); } else toep->params.wscale = 0; CTR6(KTR_CXGBE, "assign_rxopt: tid %d, mtu_idx %u, emss %u, ts %u, sack %u, wscale %u", toep->tid, toep->params.mtu_idx, toep->params.emss, toep->params.tstamp, toep->params.sack, toep->params.wscale); } /* * Completes some final bits of initialization for just established connections * and changes their state to TCPS_ESTABLISHED. * * The ISNs are from the exchange of SYNs. */ void make_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt) { struct inpcb *inp = toep->inp; struct socket *so = inp->inp_socket; struct tcpcb *tp = intotcpcb(inp); uint16_t tcpopt = be16toh(opt); INP_WLOCK_ASSERT(inp); KASSERT(tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_SYN_RECEIVED, ("%s: TCP state %s", __func__, tcpstates[tp->t_state])); CTR6(KTR_CXGBE, "%s: tid %d, so %p, inp %p, tp %p, toep %p", __func__, toep->tid, so, inp, tp, toep); tcp_state_change(tp, TCPS_ESTABLISHED); tp->t_starttime = ticks; TCPSTAT_INC(tcps_connects); tp->irs = irs; tcp_rcvseqinit(tp); tp->rcv_wnd = (u_int)toep->params.opt0_bufsize << 10; tp->rcv_adv += tp->rcv_wnd; tp->last_ack_sent = tp->rcv_nxt; tp->iss = iss; tcp_sendseqinit(tp); tp->snd_una = iss + 1; tp->snd_nxt = iss + 1; tp->snd_max = iss + 1; assign_rxopt(tp, tcpopt); send_flowc_wr(toep, tp); soisconnected(so); if (ulp_mode(toep) == ULP_MODE_TLS) tls_establish(toep); } int send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits) { struct wrqe *wr; struct cpl_rx_data_ack *req; uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); KASSERT(credits >= 0, ("%s: %d credits", __func__, credits)); wr = alloc_wrqe(sizeof(*req), toep->ctrlq); if (wr == NULL) return (0); req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); req->credit_dack = htobe32(dack | V_RX_CREDITS(credits)); t4_wrq_tx(sc, wr); return (credits); } void send_rx_modulate(struct adapter *sc, struct toepcb *toep) { struct wrqe *wr; struct cpl_rx_data_ack *req; wr = alloc_wrqe(sizeof(*req), toep->ctrlq); if (wr == NULL) return; req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); req->credit_dack = htobe32(F_RX_MODULATE_RX); t4_wrq_tx(sc, wr); } void t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp) { struct adapter *sc = tod->tod_softc; struct inpcb *inp = tp->t_inpcb; struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_rcv; struct toepcb *toep = tp->t_toe; int rx_credits; INP_WLOCK_ASSERT(inp); SOCKBUF_LOCK_ASSERT(sb); rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; if (rx_credits > 0 && (tp->rcv_wnd <= 32 * 1024 || rx_credits >= 64 * 1024 || (rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || sbused(sb) + tp->rcv_wnd < sb->sb_lowat)) { rx_credits = send_rx_credits(sc, toep, rx_credits); tp->rcv_wnd += rx_credits; tp->rcv_adv += rx_credits; } else if (toep->flags & TPF_FORCE_CREDITS) send_rx_modulate(sc, toep); } void t4_rcvd(struct toedev *tod, struct tcpcb *tp) { struct inpcb *inp = tp->t_inpcb; struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_rcv; SOCKBUF_LOCK(sb); t4_rcvd_locked(tod, tp); SOCKBUF_UNLOCK(sb); } /* * Close a connection by sending a CPL_CLOSE_CON_REQ message. */ int t4_close_conn(struct adapter *sc, struct toepcb *toep) { struct wrqe *wr; struct cpl_close_con_req *req; unsigned int tid = toep->tid; CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid, toep->flags & TPF_FIN_SENT ? ", IGNORED" : ""); if (toep->flags & TPF_FIN_SENT) return (0); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, tid)); wr = alloc_wrqe(sizeof(*req), &toep->ofld_txq->wrq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } req = wrtod(wr); req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) | V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr))); req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) | V_FW_WR_FLOWID(tid)); req->wr.wr_lo = cpu_to_be64(0); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); req->rsvd = 0; toep->flags |= TPF_FIN_SENT; toep->flags &= ~TPF_SEND_FIN; t4_l2t_send(sc, wr, toep->l2te); return (0); } #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) #define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16)) #define MIN_ISO_TX_CREDITS (howmany(sizeof(struct cpl_tx_data_iso), 16)) #define MIN_TX_CREDITS(iso) \ (MIN_OFLD_TX_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0)) /* Maximum amount of immediate data we could stuff in a WR */ static inline int max_imm_payload(int tx_credits, int iso) { const int iso_cpl_size = iso ? sizeof(struct cpl_tx_data_iso) : 0; const int n = 1; /* Use no more than one desc for imm. data WR */ KASSERT(tx_credits >= 0 && tx_credits <= MAX_OFLD_TX_CREDITS, ("%s: %d credits", __func__, tx_credits)); if (tx_credits < MIN_TX_CREDITS(iso)) return (0); if (tx_credits >= (n * EQ_ESIZE) / 16) return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr) - iso_cpl_size); else return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr) - iso_cpl_size); } /* Maximum number of SGL entries we could stuff in a WR */ static inline int max_dsgl_nsegs(int tx_credits, int iso) { int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */ int sge_pair_credits = tx_credits - MIN_TX_CREDITS(iso); KASSERT(tx_credits >= 0 && tx_credits <= MAX_OFLD_TX_CREDITS, ("%s: %d credits", __func__, tx_credits)); if (tx_credits < MIN_TX_CREDITS(iso)) return (0); nseg += 2 * (sge_pair_credits * 16 / 24); if ((sge_pair_credits * 16) % 24 == 16) nseg++; return (nseg); } static inline void write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode, unsigned int immdlen, unsigned int plen, uint8_t credits, int shove, int ulp_submode) { struct fw_ofld_tx_data_wr *txwr = dst; txwr->op_to_immdlen = htobe32(V_WR_OP(fw_wr_opcode) | V_FW_WR_IMMDLEN(immdlen)); txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | V_FW_WR_LEN16(credits)); txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ulp_mode(toep)) | V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove)); txwr->plen = htobe32(plen); if (toep->params.tx_align > 0) { if (plen < 2 * toep->params.emss) txwr->lsodisable_to_flags |= htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE); else txwr->lsodisable_to_flags |= htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD | (toep->params.nagle == 0 ? 0 : F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE)); } } /* * Generate a DSGL from a starting mbuf. The total number of segments and the * maximum segments in any one mbuf are provided. */ static void write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) { struct mbuf *m; struct ulptx_sgl *usgl = dst; int i, j, rc; struct sglist sg; struct sglist_seg segs[n]; KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); sglist_init(&sg, n, segs); usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(nsegs)); i = -1; for (m = start; m != stop; m = m->m_next) { if (m->m_flags & M_EXTPG) rc = sglist_append_mbuf_epg(&sg, m, mtod(m, vm_offset_t), m->m_len); else rc = sglist_append(&sg, mtod(m, void *), m->m_len); if (__predict_false(rc != 0)) panic("%s: sglist_append %d", __func__, rc); for (j = 0; j < sg.sg_nseg; i++, j++) { if (i < 0) { usgl->len0 = htobe32(segs[j].ss_len); usgl->addr0 = htobe64(segs[j].ss_paddr); } else { usgl->sge[i / 2].len[i & 1] = htobe32(segs[j].ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(segs[j].ss_paddr); } #ifdef INVARIANTS nsegs--; #endif } sglist_reset(&sg); } if (i & 1) usgl->sge[i / 2].len[1] = htobe32(0); KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p", __func__, nsegs, start, stop)); } /* * Max number of SGL entries an offload tx work request can have. This is 41 * (1 + 40) for a full 512B work request. * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40) */ #define OFLD_SGL_LEN (41) /* * Send data and/or a FIN to the peer. * * The socket's so_snd buffer consists of a stream of data starting with sb_mb * and linked together with m_next. sb_sndptr, if set, is the last mbuf that * was transmitted. * * drop indicates the number of bytes that should be dropped from the head of * the send buffer. It is an optimization that lets do_fw4_ack avoid creating * contention on the send buffer lock (before this change it used to do * sowwakeup and then t4_push_frames right after that when recovering from tx * stalls). When drop is set this function MUST drop the bytes and wake up any * writers. */ void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) { struct mbuf *sndptr, *m, *sb_sndptr; struct fw_ofld_tx_data_wr *txwr; struct wrqe *wr; u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_snd; int tx_credits, shove, compl, sowwakeup; struct ofld_tx_sdesc *txsd; bool nomap_mbuf_seen; INP_WLOCK_ASSERT(inp); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); KASSERT(ulp_mode(toep) == ULP_MODE_NONE || ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS || ulp_mode(toep) == ULP_MODE_RDMA, ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", __func__, toep->tid, toep->flags, tp->t_flags, drop); #endif if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) return; #ifdef RATELIMIT if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; } #endif /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. */ if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { KASSERT(drop == 0, ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); return; } txsd = &toep->txsd[toep->txsd_pidx]; do { tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); max_imm = max_imm_payload(tx_credits, 0); max_nsegs = max_dsgl_nsegs(tx_credits, 0); SOCKBUF_LOCK(sb); sowwakeup = drop; if (drop) { sbdrop_locked(sb, drop); drop = 0; } sb_sndptr = sb->sb_sndptr; sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; plen = 0; nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ nomap_mbuf_seen = false; for (m = sndptr; m != NULL; m = m->m_next) { int n; if ((m->m_flags & M_NOTAVAIL) != 0) break; if (m->m_flags & M_EXTPG) { #ifdef KERN_TLS if (m->m_epg_tls != NULL) { toep->flags |= TPF_KTLS; if (plen == 0) { SOCKBUF_UNLOCK(sb); t4_push_ktls(sc, toep, 0); return; } break; } #endif n = sglist_count_mbuf_epg(m, mtod(m, vm_offset_t), m->m_len); } else n = sglist_count(mtod(m, void *), m->m_len); nsegs += n; plen += m->m_len; /* This mbuf sent us _over_ the nsegs limit, back out */ if (plen > max_imm && nsegs > max_nsegs) { nsegs -= n; plen -= m->m_len; if (plen == 0) { /* Too few credits */ toep->flags |= TPF_TX_SUSPENDED; if (sowwakeup) { if (!TAILQ_EMPTY( &toep->aiotx_jobq)) t4_aiotx_queue_toep(so, toep); sowwakeup_locked(so); } else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); return; } break; } if (m->m_flags & M_EXTPG) nomap_mbuf_seen = true; if (max_nsegs_1mbuf < n) max_nsegs_1mbuf = n; sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ /* This mbuf put us right at the max_nsegs limit */ if (plen > max_imm && nsegs == max_nsegs) { m = m->m_next; break; } } if (sbused(sb) > sb->sb_hiwat * 5 / 8 && toep->plen_nocompl + plen >= sb->sb_hiwat / 4) compl = 1; else compl = 0; if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf && sb->sb_hiwat < V_tcp_autosndbuf_max && sbused(sb) >= sb->sb_hiwat * 7 / 8) { int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, V_tcp_autosndbuf_max); if (!sbreserve_locked(sb, newsize, so, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; else sowwakeup = 1; /* room available */ } if (sowwakeup) { if (!TAILQ_EMPTY(&toep->aiotx_jobq)) t4_aiotx_queue_toep(so, toep); sowwakeup_locked(so); } else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); /* nothing to send */ if (plen == 0) { KASSERT(m == NULL || (m->m_flags & M_NOTAVAIL) != 0, ("%s: nothing to send, but m != NULL is ready", __func__)); break; } if (__predict_false(toep->flags & TPF_FIN_SENT)) panic("%s: excess tx.", __func__); shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); if (plen <= max_imm && !nomap_mbuf_seen) { /* Immediate data tx */ wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), &toep->ofld_txq->wrq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; return; } txwr = wrtod(wr); credits = howmany(wr->wr_len, 16); write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, plen, plen, credits, shove, 0); m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); nsegs = 0; } else { int wr_len; /* DSGL tx */ wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; return; } txwr = wrtod(wr); credits = howmany(wr_len, 16); write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, 0, plen, credits, shove, 0); write_tx_sgl(txwr + 1, sndptr, m, nsegs, max_nsegs_1mbuf); if (wr_len & 0xf) { uint64_t *pad = (uint64_t *) ((uintptr_t)txwr + wr_len); *pad = 0; } } KASSERT(toep->tx_credits >= credits, ("%s: not enough credits", __func__)); toep->tx_credits -= credits; toep->tx_nocompl += credits; toep->plen_nocompl += plen; if (toep->tx_credits <= toep->tx_total * 3 / 8 && toep->tx_nocompl >= toep->tx_total / 4) compl = 1; if (compl || ulp_mode(toep) == ULP_MODE_RDMA) { txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); toep->tx_nocompl = 0; toep->plen_nocompl = 0; } tp->snd_nxt += plen; tp->snd_max += plen; SOCKBUF_LOCK(sb); KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__)); sb->sb_sndptr = sb_sndptr; SOCKBUF_UNLOCK(sb); toep->flags |= TPF_TX_DATA_SENT; if (toep->tx_credits < MIN_OFLD_TX_CREDITS) toep->flags |= TPF_TX_SUSPENDED; KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); txsd->plen = plen; txsd->tx_credits = credits; txsd++; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { toep->txsd_pidx = 0; txsd = &toep->txsd[0]; } toep->txsd_avail--; t4_l2t_send(sc, wr, toep->l2te); } while (m != NULL && (m->m_flags & M_NOTAVAIL) == 0); /* Send a FIN if requested, but only if there's no more data to send */ if (m == NULL && toep->flags & TPF_SEND_FIN) t4_close_conn(sc, toep); } static inline void rqdrop_locked(struct mbufq *q, int plen) { struct mbuf *m; while (plen > 0) { m = mbufq_dequeue(q); /* Too many credits. */ MPASS(m != NULL); M_ASSERTPKTHDR(m); /* Partial credits. */ MPASS(plen >= m->m_pkthdr.len); plen -= m->m_pkthdr.len; m_freem(m); } } /* * Not a bit in the TCB, but is a bit in the ulp_submode field of the * CPL_TX_DATA flags field in FW_ISCSI_TX_DATA_WR. */ #define ULP_ISO G_TX_ULP_SUBMODE(F_FW_ISCSI_TX_DATA_WR_ULPSUBMODE_ISO) static void write_tx_data_iso(void *dst, u_int ulp_submode, uint8_t flags, uint16_t mss, int len, int npdu) { struct cpl_tx_data_iso *cpl; unsigned int burst_size; unsigned int last; /* * The firmware will set the 'F' bit on the last PDU when * either condition is true: * * - this large PDU is marked as the "last" slice * * - the amount of data payload bytes equals the burst_size * * The strategy used here is to always set the burst_size * artificially high (len includes the size of the template * BHS) and only set the "last" flag if the original PDU had * 'F' set. */ burst_size = len; last = !!(flags & CXGBE_ISO_F); cpl = (struct cpl_tx_data_iso *)dst; cpl->op_to_scsi = htonl(V_CPL_TX_DATA_ISO_OP(CPL_TX_DATA_ISO) | V_CPL_TX_DATA_ISO_FIRST(1) | V_CPL_TX_DATA_ISO_LAST(last) | V_CPL_TX_DATA_ISO_CPLHDRLEN(0) | V_CPL_TX_DATA_ISO_HDRCRC(!!(ulp_submode & ULP_CRC_HEADER)) | V_CPL_TX_DATA_ISO_PLDCRC(!!(ulp_submode & ULP_CRC_DATA)) | V_CPL_TX_DATA_ISO_IMMEDIATE(0) | V_CPL_TX_DATA_ISO_SCSI(CXGBE_ISO_TYPE(flags))); cpl->ahs_len = 0; cpl->mpdu = htons(DIV_ROUND_UP(mss, 4)); cpl->burst_size = htonl(DIV_ROUND_UP(burst_size, 4)); cpl->len = htonl(len); cpl->reserved2_seglen_offset = htonl(0); cpl->datasn_offset = htonl(0); cpl->buffer_offset = htonl(0); cpl->reserved3 = 0; } static struct wrqe * write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr) { struct mbuf *m; struct fw_ofld_tx_data_wr *txwr; struct cpl_tx_data_iso *cpl_iso; void *p; struct wrqe *wr; u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; u_int adjusted_plen, imm_data, ulp_submode; struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); int tx_credits, shove, npdu, wr_len; uint16_t iso_mss; static const u_int ulp_extra_len[] = {0, 4, 4, 8}; bool iso; M_ASSERTPKTHDR(sndptr); tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); if (mbuf_raw_wr(sndptr)) { plen = sndptr->m_pkthdr.len; KASSERT(plen <= SGE_MAX_WR_LEN, ("raw WR len %u is greater than max WR len", plen)); if (plen > tx_credits * 16) return (NULL); wr = alloc_wrqe(roundup2(plen, 16), &toep->ofld_txq->wrq); if (__predict_false(wr == NULL)) return (NULL); m_copydata(sndptr, 0, plen, wrtod(wr)); return (wr); } iso = mbuf_iscsi_iso(sndptr); max_imm = max_imm_payload(tx_credits, iso); max_nsegs = max_dsgl_nsegs(tx_credits, iso); iso_mss = mbuf_iscsi_iso_mss(sndptr); plen = 0; nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ for (m = sndptr; m != NULL; m = m->m_next) { int n = sglist_count(mtod(m, void *), m->m_len); nsegs += n; plen += m->m_len; /* * This mbuf would send us _over_ the nsegs limit. * Suspend tx because the PDU can't be sent out. */ if (plen > max_imm && nsegs > max_nsegs) return (NULL); if (max_nsegs_1mbuf < n) max_nsegs_1mbuf = n; } if (__predict_false(toep->flags & TPF_FIN_SENT)) panic("%s: excess tx.", __func__); /* * We have a PDU to send. All of it goes out in one WR so 'm' * is NULL. A PDU's length is always a multiple of 4. */ MPASS(m == NULL); MPASS((plen & 3) == 0); MPASS(sndptr->m_pkthdr.len == plen); shove = !(tp->t_flags & TF_MORETOCOME); /* * plen doesn't include header and data digests, which are * generated and inserted in the right places by the TOE, but * they do occupy TCP sequence space and need to be accounted * for. */ ulp_submode = mbuf_ulp_submode(sndptr); MPASS(ulp_submode < nitems(ulp_extra_len)); npdu = iso ? howmany(plen - ISCSI_BHS_SIZE, iso_mss) : 1; adjusted_plen = plen + ulp_extra_len[ulp_submode] * npdu; if (iso) adjusted_plen += ISCSI_BHS_SIZE * (npdu - 1); wr_len = sizeof(*txwr); if (iso) wr_len += sizeof(struct cpl_tx_data_iso); if (plen <= max_imm) { /* Immediate data tx */ imm_data = plen; wr_len += plen; nsegs = 0; } else { /* DSGL tx */ imm_data = 0; wr_len += sizeof(struct ulptx_sgl) + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; } wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq); if (wr == NULL) { /* XXX: how will we recover from this? */ return (NULL); } txwr = wrtod(wr); credits = howmany(wr->wr_len, 16); if (iso) { write_tx_wr(txwr, toep, FW_ISCSI_TX_DATA_WR, imm_data + sizeof(struct cpl_tx_data_iso), adjusted_plen, credits, shove, ulp_submode | ULP_ISO); cpl_iso = (struct cpl_tx_data_iso *)(txwr + 1); MPASS(plen == sndptr->m_pkthdr.len); write_tx_data_iso(cpl_iso, ulp_submode, mbuf_iscsi_iso_flags(sndptr), iso_mss, plen, npdu); p = cpl_iso + 1; } else { write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, imm_data, adjusted_plen, credits, shove, ulp_submode); p = txwr + 1; } if (imm_data != 0) { m_copydata(sndptr, 0, plen, p); } else { write_tx_sgl(p, sndptr, m, nsegs, max_nsegs_1mbuf); if (wr_len & 0xf) { uint64_t *pad = (uint64_t *)((uintptr_t)txwr + wr_len); *pad = 0; } } KASSERT(toep->tx_credits >= credits, ("%s: not enough credits: credits %u " "toep->tx_credits %u tx_credits %u nsegs %u " "max_nsegs %u iso %d", __func__, credits, toep->tx_credits, tx_credits, nsegs, max_nsegs, iso)); tp->snd_nxt += adjusted_plen; tp->snd_max += adjusted_plen; counter_u64_add(toep->ofld_txq->tx_iscsi_pdus, npdu); counter_u64_add(toep->ofld_txq->tx_iscsi_octets, plen); if (iso) counter_u64_add(toep->ofld_txq->tx_iscsi_iso_wrs, 1); return (wr); } void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) { struct mbuf *sndptr, *m; struct fw_wr_hdr *wrhdr; struct wrqe *wr; u_int plen, credits; struct inpcb *inp = toep->inp; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; struct mbufq *pduq = &toep->ulp_pduq; INP_WLOCK_ASSERT(inp); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); KASSERT(ulp_mode(toep) == ULP_MODE_ISCSI, ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) return; /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. */ if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { KASSERT(drop == 0, ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); return; } if (drop) { struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_snd; int sbu; /* * An unlocked read is ok here as the data should only * transition from a non-zero value to either another * non-zero value or zero. Once it is zero it should * stay zero. */ if (__predict_false(sbused(sb)) > 0) { SOCKBUF_LOCK(sb); sbu = sbused(sb); if (sbu > 0) { /* * The data transmitted before the * tid's ULP mode changed to ISCSI is * still in so_snd. Incoming credits * should account for so_snd first. */ sbdrop_locked(sb, min(sbu, drop)); drop -= min(sbu, drop); } sowwakeup_locked(so); /* unlocks so_snd */ } rqdrop_locked(&toep->ulp_pdu_reclaimq, drop); } while ((sndptr = mbufq_first(pduq)) != NULL) { wr = write_iscsi_mbuf_wr(toep, sndptr); if (wr == NULL) { toep->flags |= TPF_TX_SUSPENDED; return; } plen = sndptr->m_pkthdr.len; credits = howmany(wr->wr_len, 16); KASSERT(toep->tx_credits >= credits, ("%s: not enough credits", __func__)); m = mbufq_dequeue(pduq); MPASS(m == sndptr); mbufq_enqueue(&toep->ulp_pdu_reclaimq, m); toep->tx_credits -= credits; toep->tx_nocompl += credits; toep->plen_nocompl += plen; /* * Ensure there are enough credits for a full-sized WR * as page pod WRs can be full-sized. */ if (toep->tx_credits <= SGE_MAX_WR_LEN * 5 / 4 && toep->tx_nocompl >= toep->tx_total / 4) { wrhdr = wrtod(wr); wrhdr->hi |= htobe32(F_FW_WR_COMPL); toep->tx_nocompl = 0; toep->plen_nocompl = 0; } toep->flags |= TPF_TX_DATA_SENT; if (toep->tx_credits < MIN_OFLD_TX_CREDITS) toep->flags |= TPF_TX_SUSPENDED; KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); txsd->plen = plen; txsd->tx_credits = credits; txsd++; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { toep->txsd_pidx = 0; txsd = &toep->txsd[0]; } toep->txsd_avail--; t4_l2t_send(sc, wr, toep->l2te); } /* Send a FIN if requested, but only if there are no more PDUs to send */ if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) t4_close_conn(sc, toep); } static inline void t4_push_data(struct adapter *sc, struct toepcb *toep, int drop) { if (ulp_mode(toep) == ULP_MODE_ISCSI) t4_push_pdus(sc, toep, drop); else if (toep->flags & TPF_KTLS) t4_push_ktls(sc, toep, drop); else t4_push_frames(sc, toep, drop); } int t4_tod_output(struct toedev *tod, struct tcpcb *tp) { struct adapter *sc = tod->tod_softc; #ifdef INVARIANTS struct inpcb *inp = tp->t_inpcb; #endif struct toepcb *toep = tp->t_toe; INP_WLOCK_ASSERT(inp); KASSERT((inp->inp_flags & INP_DROPPED) == 0, ("%s: inp %p dropped.", __func__, inp)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); t4_push_data(sc, toep, 0); return (0); } int t4_send_fin(struct toedev *tod, struct tcpcb *tp) { struct adapter *sc = tod->tod_softc; #ifdef INVARIANTS struct inpcb *inp = tp->t_inpcb; #endif struct toepcb *toep = tp->t_toe; INP_WLOCK_ASSERT(inp); KASSERT((inp->inp_flags & INP_DROPPED) == 0, ("%s: inp %p dropped.", __func__, inp)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); toep->flags |= TPF_SEND_FIN; if (tp->t_state >= TCPS_ESTABLISHED) t4_push_data(sc, toep, 0); return (0); } int t4_send_rst(struct toedev *tod, struct tcpcb *tp) { struct adapter *sc = tod->tod_softc; #if defined(INVARIANTS) struct inpcb *inp = tp->t_inpcb; #endif struct toepcb *toep = tp->t_toe; INP_WLOCK_ASSERT(inp); KASSERT((inp->inp_flags & INP_DROPPED) == 0, ("%s: inp %p dropped.", __func__, inp)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); /* hmmmm */ KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc for tid %u [%s] not sent already", __func__, toep->tid, tcpstates[tp->t_state])); send_reset(sc, toep, 0); return (0); } /* * Peer has sent us a FIN. */ static int do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_peer_close *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; struct tcpcb *tp = NULL; struct socket *so; struct epoch_tracker et; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif KASSERT(opcode == CPL_PEER_CLOSE, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); if (__predict_false(toep->flags & TPF_SYNQE)) { /* * do_pass_establish must have run before do_peer_close and if * this is still a synqe instead of a toepcb then the connection * must be getting aborted. */ MPASS(toep->flags & TPF_ABORT_SHUTDOWN); CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, toep, toep->flags); return (0); } KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); CURVNET_SET(toep->vnet); NET_EPOCH_ENTER(et); INP_WLOCK(inp); tp = intotcpcb(inp); CTR6(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, ddp_flags 0x%x, inp %p", __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, toep->ddp.flags, inp); if (toep->flags & TPF_ABORT_SHUTDOWN) goto done; + so = inp->inp_socket; + socantrcvmore(so); + if (ulp_mode(toep) == ULP_MODE_TCPDDP) { + DDP_LOCK(toep); + if (__predict_false(toep->ddp.flags & + (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) + handle_ddp_close(toep, tp, cpl->rcv_nxt); + DDP_UNLOCK(toep); + } + if (ulp_mode(toep) == ULP_MODE_RDMA || (ulp_mode(toep) == ULP_MODE_ISCSI && chip_id(sc) >= CHELSIO_T6)) { /* * There might be data received via DDP before the FIN * not reported to the driver. Just assume the * sequence number in the CPL is correct as the * sequence number of the FIN. */ } else { KASSERT(tp->rcv_nxt + 1 == be32toh(cpl->rcv_nxt), ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt, be32toh(cpl->rcv_nxt))); } tp->rcv_nxt = be32toh(cpl->rcv_nxt); - so = inp->inp_socket; - socantrcvmore(so); - if (ulp_mode(toep) == ULP_MODE_TCPDDP) { - DDP_LOCK(toep); - if (__predict_false(toep->ddp.flags & - (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) - handle_ddp_close(toep, tp, cpl->rcv_nxt); - DDP_UNLOCK(toep); - } - switch (tp->t_state) { case TCPS_SYN_RECEIVED: tp->t_starttime = ticks; /* FALLTHROUGH */ case TCPS_ESTABLISHED: tcp_state_change(tp, TCPS_CLOSE_WAIT); break; case TCPS_FIN_WAIT_1: tcp_state_change(tp, TCPS_CLOSING); break; case TCPS_FIN_WAIT_2: restore_so_proto(so, inp->inp_vflag & INP_IPV6); tcp_twstart(tp); INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ NET_EPOCH_EXIT(et); CURVNET_RESTORE(); INP_WLOCK(inp); final_cpl_received(toep); return (0); default: log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n", __func__, tid, tp->t_state); } done: INP_WUNLOCK(inp); NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return (0); } /* * Peer has ACK'd our FIN. */ static int do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; struct tcpcb *tp = NULL; struct socket *so = NULL; struct epoch_tracker et; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif KASSERT(opcode == CPL_CLOSE_CON_RPL, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); CURVNET_SET(toep->vnet); NET_EPOCH_ENTER(et); INP_WLOCK(inp); tp = intotcpcb(inp); CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x", __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags); if (toep->flags & TPF_ABORT_SHUTDOWN) goto done; so = inp->inp_socket; tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */ switch (tp->t_state) { case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */ restore_so_proto(so, inp->inp_vflag & INP_IPV6); tcp_twstart(tp); release: INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ NET_EPOCH_EXIT(et); CURVNET_RESTORE(); INP_WLOCK(inp); final_cpl_received(toep); /* no more CPLs expected */ return (0); case TCPS_LAST_ACK: if (tcp_close(tp)) INP_WUNLOCK(inp); goto release; case TCPS_FIN_WAIT_1: if (so->so_rcv.sb_state & SBS_CANTRCVMORE) soisdisconnected(so); tcp_state_change(tp, TCPS_FIN_WAIT_2); break; default: log(LOG_ERR, "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n", __func__, tid, tcpstates[tp->t_state]); } done: INP_WUNLOCK(inp); NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return (0); } void send_abort_rpl(struct adapter *sc, struct sge_ofld_txq *ofld_txq, int tid, int rst_status) { struct wrqe *wr; struct cpl_abort_rpl *cpl; wr = alloc_wrqe(sizeof(*cpl), &ofld_txq->wrq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } cpl = wrtod(wr); INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid); cpl->cmd = rst_status; t4_wrq_tx(sc, wr); } static int abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason) { switch (abort_reason) { case CPL_ERR_BAD_SYN: case CPL_ERR_CONN_RESET: return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET); case CPL_ERR_XMIT_TIMEDOUT: case CPL_ERR_PERSIST_TIMEDOUT: case CPL_ERR_FINWAIT2_TIMEDOUT: case CPL_ERR_KEEPALIVE_TIMEDOUT: return (ETIMEDOUT); default: return (EIO); } } /* * TCP RST from the peer, timeout, or some other such critical error. */ static int do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct sge_ofld_txq *ofld_txq = toep->ofld_txq; struct inpcb *inp; struct tcpcb *tp; struct epoch_tracker et; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif KASSERT(opcode == CPL_ABORT_REQ_RSS, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); if (toep->flags & TPF_SYNQE) return (do_abort_req_synqe(iq, rss, m)); KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); if (negative_advice(cpl->status)) { CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)", __func__, cpl->status, tid, toep->flags); return (0); /* Ignore negative advice */ } inp = toep->inp; CURVNET_SET(toep->vnet); NET_EPOCH_ENTER(et); /* for tcp_close */ INP_WLOCK(inp); tp = intotcpcb(inp); CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d", __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp->inp_flags, cpl->status); /* * If we'd initiated an abort earlier the reply to it is responsible for * cleaning up resources. Otherwise we tear everything down right here * right now. We owe the T4 a CPL_ABORT_RPL no matter what. */ if (toep->flags & TPF_ABORT_SHUTDOWN) { INP_WUNLOCK(inp); goto done; } toep->flags |= TPF_ABORT_SHUTDOWN; if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { struct socket *so = inp->inp_socket; if (so != NULL) so_error_set(so, abort_status_to_errno(tp, cpl->status)); tp = tcp_close(tp); if (tp == NULL) INP_WLOCK(inp); /* re-acquire */ } final_cpl_received(toep); done: NET_EPOCH_EXIT(et); CURVNET_RESTORE(); send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST); return (0); } /* * Reply to the CPL_ABORT_REQ (send_reset) */ static int do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif KASSERT(opcode == CPL_ABORT_RPL_RSS, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); if (toep->flags & TPF_SYNQE) return (do_abort_rpl_synqe(iq, rss, m)); KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d", __func__, tid, toep, inp, cpl->status); KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, ("%s: wasn't expecting abort reply", __func__)); INP_WLOCK(inp); final_cpl_received(toep); return (0); } static int do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_rx_data *cpl = mtod(m, const void *); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; struct tcpcb *tp; struct socket *so; struct sockbuf *sb; struct epoch_tracker et; int len, rx_credits; uint32_t ddp_placed = 0; if (__predict_false(toep->flags & TPF_SYNQE)) { /* * do_pass_establish must have run before do_rx_data and if this * is still a synqe instead of a toepcb then the connection must * be getting aborted. */ MPASS(toep->flags & TPF_ABORT_SHUTDOWN); CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, toep, toep->flags); m_freem(m); return (0); } KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); /* strip off CPL header */ m_adj(m, sizeof(*cpl)); len = m->m_pkthdr.len; INP_WLOCK(inp); if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", __func__, tid, len, inp->inp_flags); INP_WUNLOCK(inp); m_freem(m); return (0); } tp = intotcpcb(inp); if (__predict_false(ulp_mode(toep) == ULP_MODE_TLS && toep->flags & TPF_TLS_RECEIVE)) { /* Received "raw" data on a TLS socket. */ CTR3(KTR_CXGBE, "%s: tid %u, raw TLS data (%d bytes)", __func__, tid, len); do_rx_data_tls(cpl, toep, m); return (0); } if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq))) ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt; tp->rcv_nxt += len; if (tp->rcv_wnd < len) { KASSERT(ulp_mode(toep) == ULP_MODE_RDMA, ("%s: negative window size", __func__)); } tp->rcv_wnd -= len; tp->t_rcvtime = ticks; if (ulp_mode(toep) == ULP_MODE_TCPDDP) DDP_LOCK(toep); so = inp_inpcbtosocket(inp); sb = &so->so_rcv; SOCKBUF_LOCK(sb); if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", __func__, tid, len); m_freem(m); SOCKBUF_UNLOCK(sb); if (ulp_mode(toep) == ULP_MODE_TCPDDP) DDP_UNLOCK(toep); INP_WUNLOCK(inp); CURVNET_SET(toep->vnet); NET_EPOCH_ENTER(et); INP_WLOCK(inp); tp = tcp_drop(tp, ECONNRESET); if (tp) INP_WUNLOCK(inp); NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return (0); } /* receive buffer autosize */ MPASS(toep->vnet == so->so_vnet); CURVNET_SET(toep->vnet); if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autorcvbuf && sb->sb_hiwat < V_tcp_autorcvbuf_max && len > (sbspace(sb) / 8 * 7)) { unsigned int hiwat = sb->sb_hiwat; unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc, V_tcp_autorcvbuf_max); if (!sbreserve_locked(sb, newsize, so, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; } if (ulp_mode(toep) == ULP_MODE_TCPDDP) { int changed = !(toep->ddp.flags & DDP_ON) ^ cpl->ddp_off; if (toep->ddp.waiting_count != 0 || toep->ddp.active_count != 0) CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)", __func__, tid, len); if (changed) { if (toep->ddp.flags & DDP_SC_REQ) toep->ddp.flags ^= DDP_ON | DDP_SC_REQ; else { KASSERT(cpl->ddp_off == 1, ("%s: DDP switched on by itself.", __func__)); /* Fell out of DDP mode */ toep->ddp.flags &= ~DDP_ON; CTR1(KTR_CXGBE, "%s: fell out of DDP mode", __func__); insert_ddp_data(toep, ddp_placed); } } if (toep->ddp.flags & DDP_ON) { /* * CPL_RX_DATA with DDP on can only be an indicate. * Start posting queued AIO requests via DDP. The * payload that arrived in this indicate is appended * to the socket buffer as usual. */ handle_ddp_indicate(toep); } } sbappendstream_locked(sb, m, 0); rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { rx_credits = send_rx_credits(sc, toep, rx_credits); tp->rcv_wnd += rx_credits; tp->rcv_adv += rx_credits; } if (ulp_mode(toep) == ULP_MODE_TCPDDP && toep->ddp.waiting_count > 0 && sbavail(sb) != 0) { CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__, tid); ddp_queue_toep(toep); } sorwakeup_locked(so); SOCKBUF_UNLOCK_ASSERT(sb); if (ulp_mode(toep) == ULP_MODE_TCPDDP) DDP_UNLOCK(toep); INP_WUNLOCK(inp); CURVNET_RESTORE(); return (0); } static int do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp; struct tcpcb *tp; struct socket *so; uint8_t credits = cpl->credits; struct ofld_tx_sdesc *txsd; int plen; #ifdef INVARIANTS unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl))); #endif /* * Very unusual case: we'd sent a flowc + abort_req for a synq entry and * now this comes back carrying the credits for the flowc. */ if (__predict_false(toep->flags & TPF_SYNQE)) { KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, ("%s: credits for a synq entry %p", __func__, toep)); return (0); } inp = toep->inp; KASSERT(opcode == CPL_FW4_ACK, ("%s: unexpected opcode 0x%x", __func__, opcode)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); INP_WLOCK(inp); if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) { INP_WUNLOCK(inp); return (0); } KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0, ("%s: inp_flags 0x%x", __func__, inp->inp_flags)); tp = intotcpcb(inp); if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) { tcp_seq snd_una = be32toh(cpl->snd_una); #ifdef INVARIANTS if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) { log(LOG_ERR, "%s: unexpected seq# %x for TID %u, snd_una %x\n", __func__, snd_una, toep->tid, tp->snd_una); } #endif if (tp->snd_una != snd_una) { tp->snd_una = snd_una; tp->ts_recent_age = tcp_ts_getticks(); } } #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: tid %d credits %u", __func__, tid, credits); #endif so = inp->inp_socket; txsd = &toep->txsd[toep->txsd_cidx]; plen = 0; while (credits) { KASSERT(credits >= txsd->tx_credits, ("%s: too many (or partial) credits", __func__)); credits -= txsd->tx_credits; toep->tx_credits += txsd->tx_credits; plen += txsd->plen; txsd++; toep->txsd_avail++; KASSERT(toep->txsd_avail <= toep->txsd_total, ("%s: txsd avail > total", __func__)); if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) { txsd = &toep->txsd[0]; toep->txsd_cidx = 0; } } if (toep->tx_credits == toep->tx_total) { toep->tx_nocompl = 0; toep->plen_nocompl = 0; } if (toep->flags & TPF_TX_SUSPENDED && toep->tx_credits >= toep->tx_total / 4) { #ifdef VERBOSE_TRACES CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__, tid); #endif toep->flags &= ~TPF_TX_SUSPENDED; CURVNET_SET(toep->vnet); t4_push_data(sc, toep, plen); CURVNET_RESTORE(); } else if (plen > 0) { struct sockbuf *sb = &so->so_snd; int sbu; SOCKBUF_LOCK(sb); sbu = sbused(sb); if (ulp_mode(toep) == ULP_MODE_ISCSI) { if (__predict_false(sbu > 0)) { /* * The data transmitted before the * tid's ULP mode changed to ISCSI is * still in so_snd. Incoming credits * should account for so_snd first. */ sbdrop_locked(sb, min(sbu, plen)); plen -= min(sbu, plen); } sowwakeup_locked(so); /* unlocks so_snd */ rqdrop_locked(&toep->ulp_pdu_reclaimq, plen); } else { #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__, tid, plen); #endif sbdrop_locked(sb, plen); if (!TAILQ_EMPTY(&toep->aiotx_jobq)) t4_aiotx_queue_toep(so, toep); sowwakeup_locked(so); /* unlocks so_snd */ } SOCKBUF_UNLOCK_ASSERT(sb); } INP_WUNLOCK(inp); return (0); } void t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep, uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie) { struct wrqe *wr; struct cpl_set_tcb_field *req; struct ofld_tx_sdesc *txsd; MPASS((cookie & ~M_COOKIE) == 0); if (reply) { MPASS(cookie != CPL_COOKIE_RESERVED); } wr = alloc_wrqe(sizeof(*req), wrq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id)); if (reply == 0) req->reply_ctrl |= htobe16(F_NO_REPLY); req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie)); req->mask = htobe64(mask); req->val = htobe64(val); if (wrq->eq.type == EQ_OFLD) { txsd = &toep->txsd[toep->txsd_pidx]; txsd->tx_credits = howmany(sizeof(*req), 16); txsd->plen = 0; KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, ("%s: not enough credits (%d)", __func__, toep->tx_credits)); toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; } t4_wrq_tx(sc, wr); } void t4_init_cpl_io_handlers(void) { t4_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close); t4_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl); t4_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req); t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl, CPL_COOKIE_TOM); t4_register_cpl_handler(CPL_RX_DATA, do_rx_data); t4_register_shared_cpl_handler(CPL_FW4_ACK, do_fw4_ack, CPL_COOKIE_TOM); } void t4_uninit_cpl_io_handlers(void) { t4_register_cpl_handler(CPL_PEER_CLOSE, NULL); t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL); t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL); t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM); t4_register_cpl_handler(CPL_RX_DATA, NULL); t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM); } /* * Use the 'backend1' field in AIO jobs to hold an error that should * be reported when the job is completed, the 'backend3' field to * store the amount of data sent by the AIO job so far, and the * 'backend4' field to hold a reference count on the job. * * Each unmapped mbuf holds a reference on the job as does the queue * so long as the job is queued. */ #define aio_error backend1 #define aio_sent backend3 #define aio_refs backend4 #define jobtotid(job) \ (((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid) static void aiotx_free_job(struct kaiocb *job) { long status; int error; if (refcount_release(&job->aio_refs) == 0) return; error = (intptr_t)job->aio_error; status = job->aio_sent; #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__, jobtotid(job), job, status, error); #endif if (error != 0 && status != 0) error = 0; if (error == ECANCELED) aio_cancel(job); else if (error) aio_complete(job, -1, error); else { job->msgsnd = 1; aio_complete(job, status, 0); } } static void aiotx_free_pgs(struct mbuf *m) { struct kaiocb *job; vm_page_t pg; M_ASSERTEXTPG(m); job = m->m_ext.ext_arg1; #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__, m->m_len, jobtotid(job)); #endif for (int i = 0; i < m->m_epg_npgs; i++) { pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]); vm_page_unwire(pg, PQ_ACTIVE); } aiotx_free_job(job); } /* * Allocate a chain of unmapped mbufs describing the next 'len' bytes * of an AIO job. */ static struct mbuf * alloc_aiotx_mbuf(struct kaiocb *job, int len) { struct vmspace *vm; vm_page_t pgs[MBUF_PEXT_MAX_PGS]; struct mbuf *m, *top, *last; vm_map_t map; vm_offset_t start; int i, mlen, npages, pgoff; KASSERT(job->aio_sent + len <= job->uaiocb.aio_nbytes, ("%s(%p, %d): request to send beyond end of buffer", __func__, job, len)); /* * The AIO subsystem will cancel and drain all requests before * permitting a process to exit or exec, so p_vmspace should * be stable here. */ vm = job->userproc->p_vmspace; map = &vm->vm_map; start = (uintptr_t)job->uaiocb.aio_buf + job->aio_sent; pgoff = start & PAGE_MASK; top = NULL; last = NULL; while (len > 0) { mlen = imin(len, MBUF_PEXT_MAX_PGS * PAGE_SIZE - pgoff); KASSERT(mlen == len || ((start + mlen) & PAGE_MASK) == 0, ("%s: next start (%#jx + %#x) is not page aligned", __func__, (uintmax_t)start, mlen)); npages = vm_fault_quick_hold_pages(map, start, mlen, VM_PROT_WRITE, pgs, nitems(pgs)); if (npages < 0) break; m = mb_alloc_ext_pgs(M_WAITOK, aiotx_free_pgs); if (m == NULL) { vm_page_unhold_pages(pgs, npages); break; } m->m_epg_1st_off = pgoff; m->m_epg_npgs = npages; if (npages == 1) { KASSERT(mlen + pgoff <= PAGE_SIZE, ("%s: single page is too large (off %d len %d)", __func__, pgoff, mlen)); m->m_epg_last_len = mlen; } else { m->m_epg_last_len = mlen - (PAGE_SIZE - pgoff) - (npages - 2) * PAGE_SIZE; } for (i = 0; i < npages; i++) m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]); m->m_len = mlen; m->m_ext.ext_size = npages * PAGE_SIZE; m->m_ext.ext_arg1 = job; refcount_acquire(&job->aio_refs); #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %d, new mbuf %p for job %p, npages %d", __func__, jobtotid(job), m, job, npages); #endif if (top == NULL) top = m; else last->m_next = m; last = m; len -= mlen; start += mlen; pgoff = 0; } return (top); } static void t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job) { struct sockbuf *sb; struct file *fp; struct inpcb *inp; struct tcpcb *tp; struct mbuf *m; int error, len; bool moretocome, sendmore; sb = &so->so_snd; SOCKBUF_UNLOCK(sb); fp = job->fd_file; m = NULL; #ifdef MAC error = mac_socket_check_send(fp->f_cred, so); if (error != 0) goto out; #endif /* Inline sosend_generic(). */ error = SOCK_IO_SEND_LOCK(so, SBL_WAIT); MPASS(error == 0); sendanother: SOCKBUF_LOCK(sb); if (so->so_snd.sb_state & SBS_CANTSENDMORE) { SOCKBUF_UNLOCK(sb); SOCK_IO_SEND_UNLOCK(so); if ((so->so_options & SO_NOSIGPIPE) == 0) { PROC_LOCK(job->userproc); kern_psignal(job->userproc, SIGPIPE); PROC_UNLOCK(job->userproc); } error = EPIPE; goto out; } if (so->so_error) { error = so->so_error; so->so_error = 0; SOCKBUF_UNLOCK(sb); SOCK_IO_SEND_UNLOCK(so); goto out; } if ((so->so_state & SS_ISCONNECTED) == 0) { SOCKBUF_UNLOCK(sb); SOCK_IO_SEND_UNLOCK(so); error = ENOTCONN; goto out; } if (sbspace(sb) < sb->sb_lowat) { MPASS(job->aio_sent == 0 || !(so->so_state & SS_NBIO)); /* * Don't block if there is too little room in the socket * buffer. Instead, requeue the request. */ if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { SOCKBUF_UNLOCK(sb); SOCK_IO_SEND_UNLOCK(so); error = ECANCELED; goto out; } TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); SOCKBUF_UNLOCK(sb); SOCK_IO_SEND_UNLOCK(so); goto out; } /* * Write as much data as the socket permits, but no more than a * a single sndbuf at a time. */ len = sbspace(sb); if (len > job->uaiocb.aio_nbytes - job->aio_sent) { len = job->uaiocb.aio_nbytes - job->aio_sent; moretocome = false; } else moretocome = true; if (len > toep->params.sndbuf) { len = toep->params.sndbuf; sendmore = true; } else sendmore = false; if (!TAILQ_EMPTY(&toep->aiotx_jobq)) moretocome = true; SOCKBUF_UNLOCK(sb); MPASS(len != 0); m = alloc_aiotx_mbuf(job, len); if (m == NULL) { SOCK_IO_SEND_UNLOCK(so); error = EFAULT; goto out; } /* Inlined tcp_usr_send(). */ inp = toep->inp; INP_WLOCK(inp); if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { INP_WUNLOCK(inp); SOCK_IO_SEND_UNLOCK(so); error = ECONNRESET; goto out; } job->aio_sent += m_length(m, NULL); sbappendstream(sb, m, 0); m = NULL; if (!(inp->inp_flags & INP_DROPPED)) { tp = intotcpcb(inp); if (moretocome) tp->t_flags |= TF_MORETOCOME; error = tp->t_fb->tfb_tcp_output(tp); if (moretocome) tp->t_flags &= ~TF_MORETOCOME; } INP_WUNLOCK(inp); if (sendmore) goto sendanother; SOCK_IO_SEND_UNLOCK(so); if (error) goto out; /* * If this is a blocking socket and the request has not been * fully completed, requeue it until the socket is ready * again. */ if (job->aio_sent < job->uaiocb.aio_nbytes && !(so->so_state & SS_NBIO)) { SOCKBUF_LOCK(sb); if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { SOCKBUF_UNLOCK(sb); error = ECANCELED; goto out; } TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); return; } /* * If the request will not be requeued, drop the queue's * reference to the job. Any mbufs in flight should still * hold a reference, but this drops the reference that the * queue owns while it is waiting to queue mbufs to the * socket. */ aiotx_free_job(job); out: if (error) { job->aio_error = (void *)(intptr_t)error; aiotx_free_job(job); } m_freem(m); SOCKBUF_LOCK(sb); } static void t4_aiotx_task(void *context, int pending) { struct toepcb *toep = context; struct socket *so; struct kaiocb *job; so = toep->aiotx_so; CURVNET_SET(toep->vnet); SOCKBUF_LOCK(&so->so_snd); while (!TAILQ_EMPTY(&toep->aiotx_jobq) && sowriteable(so)) { job = TAILQ_FIRST(&toep->aiotx_jobq); TAILQ_REMOVE(&toep->aiotx_jobq, job, list); if (!aio_clear_cancel_function(job)) continue; t4_aiotx_process_job(toep, so, job); } toep->aiotx_so = NULL; SOCKBUF_UNLOCK(&so->so_snd); CURVNET_RESTORE(); free_toepcb(toep); SOCK_LOCK(so); sorele(so); } static void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep) { SOCKBUF_LOCK_ASSERT(&toep->inp->inp_socket->so_snd); #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: queueing aiotx task for tid %d, active = %s", __func__, toep->tid, toep->aiotx_so != NULL ? "true" : "false"); #endif if (toep->aiotx_so != NULL) return; soref(so); toep->aiotx_so = so; hold_toepcb(toep); soaio_enqueue(&toep->aiotx_task); } static void t4_aiotx_cancel(struct kaiocb *job) { struct socket *so; struct sockbuf *sb; struct tcpcb *tp; struct toepcb *toep; so = job->fd_file->f_data; tp = so_sototcpcb(so); toep = tp->t_toe; MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE); sb = &so->so_snd; SOCKBUF_LOCK(sb); if (!aio_cancel_cleared(job)) TAILQ_REMOVE(&toep->aiotx_jobq, job, list); SOCKBUF_UNLOCK(sb); job->aio_error = (void *)(intptr_t)ECANCELED; aiotx_free_job(job); } int t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job) { struct tcpcb *tp = so_sototcpcb(so); struct toepcb *toep = tp->t_toe; struct adapter *sc = td_adapter(toep->td); /* This only handles writes. */ if (job->uaiocb.aio_lio_opcode != LIO_WRITE) return (EOPNOTSUPP); if (!sc->tt.tx_zcopy) return (EOPNOTSUPP); if (tls_tx_key(toep)) return (EOPNOTSUPP); SOCKBUF_LOCK(&so->so_snd); #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid); #endif if (!aio_set_cancel_function(job, t4_aiotx_cancel)) panic("new job was cancelled"); refcount_init(&job->aio_refs, 1); TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list); if (sowriteable(so)) t4_aiotx_queue_toep(so, toep); SOCKBUF_UNLOCK(&so->so_snd); return (0); } void aiotx_init_toep(struct toepcb *toep) { TAILQ_INIT(&toep->aiotx_jobq); TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep); } #endif diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c index 2b58cb60d4fd..be142ffb9e4f 100644 --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -1,2140 +1,2141 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define TCPSTATES #include #include #include #include #include #include #include #include #include #include #include #ifdef TCP_OFFLOAD #include "common/common.h" #include "common/t4_msg.h" #include "common/t4_regs.h" #include "common/t4_tcb.h" #include "tom/t4_tom.h" /* * Use the 'backend3' field in AIO jobs to store the amount of data * received by the AIO job so far. */ #define aio_received backend3 static void aio_ddp_requeue_task(void *context, int pending); static void ddp_complete_all(struct toepcb *toep, int error); static void t4_aio_cancel_active(struct kaiocb *job); static void t4_aio_cancel_queued(struct kaiocb *job); static TAILQ_HEAD(, pageset) ddp_orphan_pagesets; static struct mtx ddp_orphan_pagesets_lock; static struct task ddp_orphan_task; #define MAX_DDP_BUFFER_SIZE (M_TCB_RX_DDP_BUF0_LEN) /* * A page set holds information about a buffer used for DDP. The page * set holds resources such as the VM pages backing the buffer (either * held or wired) and the page pods associated with the buffer. * Recently used page sets are cached to allow for efficient reuse of * buffers (avoiding the need to re-fault in pages, hold them, etc.). * Note that cached page sets keep the backing pages wired. The * number of wired pages is capped by only allowing for two wired * pagesets per connection. This is not a perfect cap, but is a * trade-off for performance. * * If an application ping-pongs two buffers for a connection via * aio_read(2) then those buffers should remain wired and expensive VM * fault lookups should be avoided after each buffer has been used * once. If an application uses more than two buffers then this will * fall back to doing expensive VM fault lookups for each operation. */ static void free_pageset(struct tom_data *td, struct pageset *ps) { vm_page_t p; int i; if (ps->prsv.prsv_nppods > 0) t4_free_page_pods(&ps->prsv); for (i = 0; i < ps->npages; i++) { p = ps->pages[i]; vm_page_unwire(p, PQ_INACTIVE); } mtx_lock(&ddp_orphan_pagesets_lock); TAILQ_INSERT_TAIL(&ddp_orphan_pagesets, ps, link); taskqueue_enqueue(taskqueue_thread, &ddp_orphan_task); mtx_unlock(&ddp_orphan_pagesets_lock); } static void ddp_free_orphan_pagesets(void *context, int pending) { struct pageset *ps; mtx_lock(&ddp_orphan_pagesets_lock); while (!TAILQ_EMPTY(&ddp_orphan_pagesets)) { ps = TAILQ_FIRST(&ddp_orphan_pagesets); TAILQ_REMOVE(&ddp_orphan_pagesets, ps, link); mtx_unlock(&ddp_orphan_pagesets_lock); if (ps->vm) vmspace_free(ps->vm); free(ps, M_CXGBE); mtx_lock(&ddp_orphan_pagesets_lock); } mtx_unlock(&ddp_orphan_pagesets_lock); } static void recycle_pageset(struct toepcb *toep, struct pageset *ps) { DDP_ASSERT_LOCKED(toep); if (!(toep->ddp.flags & DDP_DEAD)) { KASSERT(toep->ddp.cached_count + toep->ddp.active_count < nitems(toep->ddp.db), ("too many wired pagesets")); TAILQ_INSERT_HEAD(&toep->ddp.cached_pagesets, ps, link); toep->ddp.cached_count++; } else free_pageset(toep->td, ps); } static void ddp_complete_one(struct kaiocb *job, int error) { long copied; /* * If this job had copied data out of the socket buffer before * it was cancelled, report it as a short read rather than an * error. */ copied = job->aio_received; if (copied != 0 || error == 0) aio_complete(job, copied, 0); else aio_complete(job, -1, error); } static void free_ddp_buffer(struct tom_data *td, struct ddp_buffer *db) { if (db->job) { /* * XXX: If we are un-offloading the socket then we * should requeue these on the socket somehow. If we * got a FIN from the remote end, then this completes * any remaining requests with an EOF read. */ if (!aio_clear_cancel_function(db->job)) ddp_complete_one(db->job, 0); } if (db->ps) free_pageset(td, db->ps); } void ddp_init_toep(struct toepcb *toep) { TAILQ_INIT(&toep->ddp.aiojobq); TASK_INIT(&toep->ddp.requeue_task, 0, aio_ddp_requeue_task, toep); toep->ddp.flags = DDP_OK; toep->ddp.active_id = -1; mtx_init(&toep->ddp.lock, "t4 ddp", NULL, MTX_DEF); } void ddp_uninit_toep(struct toepcb *toep) { mtx_destroy(&toep->ddp.lock); } void release_ddp_resources(struct toepcb *toep) { struct pageset *ps; int i; DDP_LOCK(toep); toep->ddp.flags |= DDP_DEAD; for (i = 0; i < nitems(toep->ddp.db); i++) { free_ddp_buffer(toep->td, &toep->ddp.db[i]); } while ((ps = TAILQ_FIRST(&toep->ddp.cached_pagesets)) != NULL) { TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link); free_pageset(toep->td, ps); } ddp_complete_all(toep, 0); DDP_UNLOCK(toep); } #ifdef INVARIANTS void ddp_assert_empty(struct toepcb *toep) { int i; MPASS(!(toep->ddp.flags & DDP_TASK_ACTIVE)); for (i = 0; i < nitems(toep->ddp.db); i++) { MPASS(toep->ddp.db[i].job == NULL); MPASS(toep->ddp.db[i].ps == NULL); } MPASS(TAILQ_EMPTY(&toep->ddp.cached_pagesets)); MPASS(TAILQ_EMPTY(&toep->ddp.aiojobq)); } #endif static void complete_ddp_buffer(struct toepcb *toep, struct ddp_buffer *db, unsigned int db_idx) { unsigned int db_flag; toep->ddp.active_count--; if (toep->ddp.active_id == db_idx) { if (toep->ddp.active_count == 0) { KASSERT(toep->ddp.db[db_idx ^ 1].job == NULL, ("%s: active_count mismatch", __func__)); toep->ddp.active_id = -1; } else toep->ddp.active_id ^= 1; #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: tid %u, ddp_active_id = %d", __func__, toep->tid, toep->ddp.active_id); #endif } else { KASSERT(toep->ddp.active_count != 0 && toep->ddp.active_id != -1, ("%s: active count mismatch", __func__)); } db->cancel_pending = 0; db->job = NULL; recycle_pageset(toep, db->ps); db->ps = NULL; db_flag = db_idx == 1 ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE; KASSERT(toep->ddp.flags & db_flag, ("%s: DDP buffer not active. toep %p, ddp_flags 0x%x", __func__, toep, toep->ddp.flags)); toep->ddp.flags &= ~db_flag; } /* XXX: handle_ddp_data code duplication */ void insert_ddp_data(struct toepcb *toep, uint32_t n) { struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); struct ddp_buffer *db; struct kaiocb *job; size_t placed; long copied; unsigned int db_flag, db_idx; INP_WLOCK_ASSERT(inp); DDP_ASSERT_LOCKED(toep); tp->rcv_nxt += n; #ifndef USE_DDP_RX_FLOW_CONTROL KASSERT(tp->rcv_wnd >= n, ("%s: negative window size", __func__)); tp->rcv_wnd -= n; #endif CTR2(KTR_CXGBE, "%s: placed %u bytes before falling out of DDP", __func__, n); while (toep->ddp.active_count > 0) { MPASS(toep->ddp.active_id != -1); db_idx = toep->ddp.active_id; db_flag = db_idx == 1 ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE; MPASS((toep->ddp.flags & db_flag) != 0); db = &toep->ddp.db[db_idx]; job = db->job; copied = job->aio_received; placed = n; if (placed > job->uaiocb.aio_nbytes - copied) placed = job->uaiocb.aio_nbytes - copied; if (placed > 0) job->msgrcv = 1; if (!aio_clear_cancel_function(job)) { /* * Update the copied length for when * t4_aio_cancel_active() completes this * request. */ job->aio_received += placed; } else if (copied + placed != 0) { CTR4(KTR_CXGBE, "%s: completing %p (copied %ld, placed %lu)", __func__, job, copied, placed); /* XXX: This always completes if there is some data. */ aio_complete(job, copied + placed, 0); } else if (aio_set_cancel_function(job, t4_aio_cancel_queued)) { TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count++; } else aio_cancel(job); n -= placed; complete_ddp_buffer(toep, db, db_idx); } MPASS(n == 0); } /* SET_TCB_FIELD sent as a ULP command looks like this */ #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \ sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core)) /* RX_DATA_ACK sent as a ULP command looks like this */ #define LEN__RX_DATA_ACK_ULP (sizeof(struct ulp_txpkt) + \ sizeof(struct ulptx_idata) + sizeof(struct cpl_rx_data_ack_core)) static inline void * mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, struct toepcb *toep, uint64_t word, uint64_t mask, uint64_t val) { struct ulptx_idata *ulpsc; struct cpl_set_tcb_field_core *req; ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(sizeof(*req)); req = (struct cpl_set_tcb_field_core *)(ulpsc + 1); OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tid)); req->reply_ctrl = htobe16(V_NO_REPLY(1) | V_QUEUENO(toep->ofld_rxq->iq.abs_id)); req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0)); req->mask = htobe64(mask); req->val = htobe64(val); ulpsc = (struct ulptx_idata *)(req + 1); if (LEN__SET_TCB_FIELD_ULP % 16) { ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); ulpsc->len = htobe32(0); return (ulpsc + 1); } return (ulpsc); } static inline void * mk_rx_data_ack_ulp(struct ulp_txpkt *ulpmc, struct toepcb *toep) { struct ulptx_idata *ulpsc; struct cpl_rx_data_ack_core *req; ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); ulpmc->len = htobe32(howmany(LEN__RX_DATA_ACK_ULP, 16)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(sizeof(*req)); req = (struct cpl_rx_data_ack_core *)(ulpsc + 1); OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tid)); req->credit_dack = htobe32(F_RX_MODULATE_RX); ulpsc = (struct ulptx_idata *)(req + 1); if (LEN__RX_DATA_ACK_ULP % 16) { ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); ulpsc->len = htobe32(0); return (ulpsc + 1); } return (ulpsc); } static struct wrqe * mk_update_tcb_for_ddp(struct adapter *sc, struct toepcb *toep, int db_idx, struct pageset *ps, int offset, uint64_t ddp_flags, uint64_t ddp_flags_mask) { struct wrqe *wr; struct work_request_hdr *wrh; struct ulp_txpkt *ulpmc; int len; KASSERT(db_idx == 0 || db_idx == 1, ("%s: bad DDP buffer index %d", __func__, db_idx)); /* * We'll send a compound work request that has 3 SET_TCB_FIELDs and an * RX_DATA_ACK (with RX_MODULATE to speed up delivery). * * The work request header is 16B and always ends at a 16B boundary. * The ULPTX master commands that follow must all end at 16B boundaries * too so we round up the size to 16. */ len = sizeof(*wrh) + 3 * roundup2(LEN__SET_TCB_FIELD_ULP, 16) + roundup2(LEN__RX_DATA_ACK_ULP, 16); wr = alloc_wrqe(len, toep->ctrlq); if (wr == NULL) return (NULL); wrh = wrtod(wr); INIT_ULPTX_WRH(wrh, len, 1, 0); /* atomic */ ulpmc = (struct ulp_txpkt *)(wrh + 1); /* Write the buffer's tag */ ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_BUF0_TAG + db_idx, V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG), V_TCB_RX_DDP_BUF0_TAG(ps->prsv.prsv_tag)); /* Update the current offset in the DDP buffer and its total length */ if (db_idx == 0) ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_BUF0_OFFSET, V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) | V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), V_TCB_RX_DDP_BUF0_OFFSET(offset) | V_TCB_RX_DDP_BUF0_LEN(ps->len)); else ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_BUF1_OFFSET, V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) | V_TCB_RX_DDP_BUF1_LEN((u64)M_TCB_RX_DDP_BUF1_LEN << 32), V_TCB_RX_DDP_BUF1_OFFSET(offset) | V_TCB_RX_DDP_BUF1_LEN((u64)ps->len << 32)); /* Update DDP flags */ ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_FLAGS, ddp_flags_mask, ddp_flags); /* Gratuitous RX_DATA_ACK with RX_MODULATE set to speed up delivery. */ ulpmc = mk_rx_data_ack_ulp(ulpmc, toep); return (wr); } static int handle_ddp_data(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt, int len) { uint32_t report = be32toh(ddp_report); unsigned int db_idx; struct inpcb *inp = toep->inp; struct ddp_buffer *db; struct tcpcb *tp; struct socket *so; struct sockbuf *sb; struct kaiocb *job; long copied; db_idx = report & F_DDP_BUF_IDX ? 1 : 0; if (__predict_false(!(report & F_DDP_INV))) CXGBE_UNIMPLEMENTED("DDP buffer still valid"); INP_WLOCK(inp); so = inp_inpcbtosocket(inp); sb = &so->so_rcv; DDP_LOCK(toep); KASSERT(toep->ddp.active_id == db_idx, ("completed DDP buffer (%d) != active_id (%d) for tid %d", db_idx, toep->ddp.active_id, toep->tid)); db = &toep->ddp.db[db_idx]; job = db->job; if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) { /* * This can happen due to an administrative tcpdrop(8). * Just fail the request with ECONNRESET. */ CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x", __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags); if (aio_clear_cancel_function(job)) ddp_complete_one(job, ECONNRESET); goto completed; } tp = intotcpcb(inp); /* * For RX_DDP_COMPLETE, len will be zero and rcv_nxt is the * sequence number of the next byte to receive. The length of * the data received for this message must be computed by * comparing the new and old values of rcv_nxt. * * For RX_DATA_DDP, len might be non-zero, but it is only the * length of the most recent DMA. It does not include the * total length of the data received since the previous update * for this DDP buffer. rcv_nxt is the sequence number of the * first received byte from the most recent DMA. */ len += be32toh(rcv_nxt) - tp->rcv_nxt; tp->rcv_nxt += len; tp->t_rcvtime = ticks; #ifndef USE_DDP_RX_FLOW_CONTROL KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); tp->rcv_wnd -= len; #endif #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %u, DDP[%d] placed %d bytes (%#x)", __func__, toep->tid, db_idx, len, report); #endif /* receive buffer autosize */ MPASS(toep->vnet == so->so_vnet); CURVNET_SET(toep->vnet); SOCKBUF_LOCK(sb); if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autorcvbuf && sb->sb_hiwat < V_tcp_autorcvbuf_max && len > (sbspace(sb) / 8 * 7)) { struct adapter *sc = td_adapter(toep->td); unsigned int hiwat = sb->sb_hiwat; unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc, V_tcp_autorcvbuf_max); if (!sbreserve_locked(sb, newsize, so, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; } SOCKBUF_UNLOCK(sb); CURVNET_RESTORE(); job->msgrcv = 1; if (db->cancel_pending) { /* * Update the job's length but defer completion to the * TCB_RPL callback. */ job->aio_received += len; goto out; } else if (!aio_clear_cancel_function(job)) { /* * Update the copied length for when * t4_aio_cancel_active() completes this request. */ job->aio_received += len; } else { copied = job->aio_received; #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %u, completing %p (copied %ld, placed %d)", __func__, toep->tid, job, copied, len); #endif aio_complete(job, copied + len, 0); t4_rcvd(&toep->td->tod, tp); } completed: complete_ddp_buffer(toep, db, db_idx); if (toep->ddp.waiting_count > 0) ddp_queue_toep(toep); out: DDP_UNLOCK(toep); INP_WUNLOCK(inp); return (0); } void handle_ddp_indicate(struct toepcb *toep) { DDP_ASSERT_LOCKED(toep); MPASS(toep->ddp.active_count == 0); MPASS((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0); if (toep->ddp.waiting_count == 0) { /* * The pending requests that triggered the request for an * an indicate were cancelled. Those cancels should have * already disabled DDP. Just ignore this as the data is * going into the socket buffer anyway. */ return; } CTR3(KTR_CXGBE, "%s: tid %d indicated (%d waiting)", __func__, toep->tid, toep->ddp.waiting_count); ddp_queue_toep(toep); } CTASSERT(CPL_COOKIE_DDP0 + 1 == CPL_COOKIE_DDP1); static int do_ddp_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); unsigned int db_idx; struct toepcb *toep; struct inpcb *inp; struct ddp_buffer *db; struct kaiocb *job; long copied; if (cpl->status != CPL_ERR_NONE) panic("XXX: tcp_rpl failed: %d", cpl->status); toep = lookup_tid(sc, tid); inp = toep->inp; switch (cpl->cookie) { case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(CPL_COOKIE_DDP0): case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(CPL_COOKIE_DDP1): /* * XXX: This duplicates a lot of code with handle_ddp_data(). */ db_idx = G_COOKIE(cpl->cookie) - CPL_COOKIE_DDP0; MPASS(db_idx < nitems(toep->ddp.db)); INP_WLOCK(inp); DDP_LOCK(toep); db = &toep->ddp.db[db_idx]; /* * handle_ddp_data() should leave the job around until * this callback runs once a cancel is pending. */ MPASS(db != NULL); MPASS(db->job != NULL); MPASS(db->cancel_pending); /* * XXX: It's not clear what happens if there is data * placed when the buffer is invalidated. I suspect we * need to read the TCB to see how much data was placed. * * For now this just pretends like nothing was placed. * * XXX: Note that if we did check the PCB we would need to * also take care of updating the tp, etc. */ job = db->job; copied = job->aio_received; if (copied == 0) { CTR2(KTR_CXGBE, "%s: cancelling %p", __func__, job); aio_cancel(job); } else { CTR3(KTR_CXGBE, "%s: completing %p (copied %ld)", __func__, job, copied); aio_complete(job, copied, 0); t4_rcvd(&toep->td->tod, intotcpcb(inp)); } complete_ddp_buffer(toep, db, db_idx); if (toep->ddp.waiting_count > 0) ddp_queue_toep(toep); DDP_UNLOCK(toep); INP_WUNLOCK(inp); break; default: panic("XXX: unknown tcb_rpl offset %#x, cookie %#x", G_WORD(cpl->cookie), G_COOKIE(cpl->cookie)); } return (0); } void handle_ddp_close(struct toepcb *toep, struct tcpcb *tp, __be32 rcv_nxt) { struct ddp_buffer *db; struct kaiocb *job; long copied; unsigned int db_flag, db_idx; int len, placed; INP_WLOCK_ASSERT(toep->inp); DDP_ASSERT_LOCKED(toep); - len = be32toh(rcv_nxt) - tp->rcv_nxt; + /* - 1 is to ignore the byte for FIN */ + len = be32toh(rcv_nxt) - tp->rcv_nxt - 1; tp->rcv_nxt += len; while (toep->ddp.active_count > 0) { MPASS(toep->ddp.active_id != -1); db_idx = toep->ddp.active_id; db_flag = db_idx == 1 ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE; MPASS((toep->ddp.flags & db_flag) != 0); db = &toep->ddp.db[db_idx]; job = db->job; copied = job->aio_received; placed = len; if (placed > job->uaiocb.aio_nbytes - copied) placed = job->uaiocb.aio_nbytes - copied; if (placed > 0) job->msgrcv = 1; if (!aio_clear_cancel_function(job)) { /* * Update the copied length for when * t4_aio_cancel_active() completes this * request. */ job->aio_received += placed; } else { CTR4(KTR_CXGBE, "%s: tid %d completed buf %d len %d", __func__, toep->tid, db_idx, placed); aio_complete(job, copied + placed, 0); } len -= placed; complete_ddp_buffer(toep, db, db_idx); } MPASS(len == 0); ddp_complete_all(toep, 0); } #define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\ F_DDP_PPOD_PARITY_ERR | F_DDP_PADDING_ERR | F_DDP_OFFSET_ERR |\ F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\ F_DDP_INVALID_PPOD | F_DDP_HDRCRC_ERR | F_DDP_DATACRC_ERR) extern cpl_handler_t t4_cpl_handler[]; static int do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); uint32_t vld; struct toepcb *toep = lookup_tid(sc, tid); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); KASSERT(!(toep->flags & TPF_SYNQE), ("%s: toep %p claims to be a synq entry", __func__, toep)); vld = be32toh(cpl->ddpvld); if (__predict_false(vld & DDP_ERR)) { panic("%s: DDP error 0x%x (tid %d, toep %p)", __func__, vld, tid, toep); } if (ulp_mode(toep) == ULP_MODE_ISCSI) { t4_cpl_handler[CPL_RX_ISCSI_DDP](iq, rss, m); return (0); } handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len)); return (0); } static int do_rx_ddp_complete(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_rx_ddp_complete *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); KASSERT(!(toep->flags & TPF_SYNQE), ("%s: toep %p claims to be a synq entry", __func__, toep)); handle_ddp_data(toep, cpl->ddp_report, cpl->rcv_nxt, 0); return (0); } static void enable_ddp(struct adapter *sc, struct toepcb *toep) { KASSERT((toep->ddp.flags & (DDP_ON | DDP_OK | DDP_SC_REQ)) == DDP_OK, ("%s: toep %p has bad ddp_flags 0x%x", __func__, toep, toep->ddp.flags)); CTR3(KTR_CXGBE, "%s: tid %u (time %u)", __func__, toep->tid, time_uptime); DDP_ASSERT_LOCKED(toep); toep->ddp.flags |= DDP_SC_REQ; t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_RX_DDP_FLAGS, V_TF_DDP_OFF(1) | V_TF_DDP_INDICATE_OUT(1) | V_TF_DDP_BUF0_INDICATE(1) | V_TF_DDP_BUF1_INDICATE(1) | V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_BUF1_VALID(1), V_TF_DDP_BUF0_INDICATE(1) | V_TF_DDP_BUF1_INDICATE(1), 0, 0); t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS, V_TF_RCV_COALESCE_ENABLE(1), 0, 0, 0); } static int calculate_hcf(int n1, int n2) { int a, b, t; if (n1 <= n2) { a = n1; b = n2; } else { a = n2; b = n1; } while (a != 0) { t = a; a = b % a; b = t; } return (b); } static inline int pages_to_nppods(int npages, int ddp_page_shift) { MPASS(ddp_page_shift >= PAGE_SHIFT); return (howmany(npages >> (ddp_page_shift - PAGE_SHIFT), PPOD_PAGES)); } static int alloc_page_pods(struct ppod_region *pr, u_int nppods, u_int pgsz_idx, struct ppod_reservation *prsv) { vmem_addr_t addr; /* relative to start of region */ if (vmem_alloc(pr->pr_arena, PPOD_SZ(nppods), M_NOWAIT | M_FIRSTFIT, &addr) != 0) return (ENOMEM); CTR5(KTR_CXGBE, "%-17s arena %p, addr 0x%08x, nppods %d, pgsz %d", __func__, pr->pr_arena, (uint32_t)addr & pr->pr_tag_mask, nppods, 1 << pr->pr_page_shift[pgsz_idx]); /* * The hardware tagmask includes an extra invalid bit but the arena was * seeded with valid values only. An allocation out of this arena will * fit inside the tagmask but won't have the invalid bit set. */ MPASS((addr & pr->pr_tag_mask) == addr); MPASS((addr & pr->pr_invalid_bit) == 0); prsv->prsv_pr = pr; prsv->prsv_tag = V_PPOD_PGSZ(pgsz_idx) | addr; prsv->prsv_nppods = nppods; return (0); } int t4_alloc_page_pods_for_ps(struct ppod_region *pr, struct pageset *ps) { int i, hcf, seglen, idx, nppods; struct ppod_reservation *prsv = &ps->prsv; KASSERT(prsv->prsv_nppods == 0, ("%s: page pods already allocated", __func__)); /* * The DDP page size is unrelated to the VM page size. We combine * contiguous physical pages into larger segments to get the best DDP * page size possible. This is the largest of the four sizes in * A_ULP_RX_TDDP_PSZ that evenly divides the HCF of the segment sizes in * the page list. */ hcf = 0; for (i = 0; i < ps->npages; i++) { seglen = PAGE_SIZE; while (i < ps->npages - 1 && ps->pages[i]->phys_addr + PAGE_SIZE == ps->pages[i + 1]->phys_addr) { seglen += PAGE_SIZE; i++; } hcf = calculate_hcf(hcf, seglen); if (hcf < (1 << pr->pr_page_shift[1])) { idx = 0; goto have_pgsz; /* give up, short circuit */ } } #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1) MPASS((hcf & PR_PAGE_MASK(0)) == 0); /* PAGE_SIZE is >= 4K everywhere */ for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) { if ((hcf & PR_PAGE_MASK(idx)) == 0) break; } #undef PR_PAGE_MASK have_pgsz: MPASS(idx <= M_PPOD_PGSZ); nppods = pages_to_nppods(ps->npages, pr->pr_page_shift[idx]); if (alloc_page_pods(pr, nppods, idx, prsv) != 0) return (0); MPASS(prsv->prsv_nppods > 0); return (1); } int t4_alloc_page_pods_for_buf(struct ppod_region *pr, vm_offset_t buf, int len, struct ppod_reservation *prsv) { int hcf, seglen, idx, npages, nppods; uintptr_t start_pva, end_pva, pva, p1; MPASS(buf > 0); MPASS(len > 0); /* * The DDP page size is unrelated to the VM page size. We combine * contiguous physical pages into larger segments to get the best DDP * page size possible. This is the largest of the four sizes in * A_ULP_RX_ISCSI_PSZ that evenly divides the HCF of the segment sizes * in the page list. */ hcf = 0; start_pva = trunc_page(buf); end_pva = trunc_page(buf + len - 1); pva = start_pva; while (pva <= end_pva) { seglen = PAGE_SIZE; p1 = pmap_kextract(pva); pva += PAGE_SIZE; while (pva <= end_pva && p1 + seglen == pmap_kextract(pva)) { seglen += PAGE_SIZE; pva += PAGE_SIZE; } hcf = calculate_hcf(hcf, seglen); if (hcf < (1 << pr->pr_page_shift[1])) { idx = 0; goto have_pgsz; /* give up, short circuit */ } } #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1) MPASS((hcf & PR_PAGE_MASK(0)) == 0); /* PAGE_SIZE is >= 4K everywhere */ for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) { if ((hcf & PR_PAGE_MASK(idx)) == 0) break; } #undef PR_PAGE_MASK have_pgsz: MPASS(idx <= M_PPOD_PGSZ); npages = 1; npages += (end_pva - start_pva) >> pr->pr_page_shift[idx]; nppods = howmany(npages, PPOD_PAGES); if (alloc_page_pods(pr, nppods, idx, prsv) != 0) return (ENOMEM); MPASS(prsv->prsv_nppods > 0); return (0); } int t4_alloc_page_pods_for_sgl(struct ppod_region *pr, struct ctl_sg_entry *sgl, int entries, struct ppod_reservation *prsv) { int hcf, seglen, idx = 0, npages, nppods, i, len; uintptr_t start_pva, end_pva, pva, p1 ; vm_offset_t buf; struct ctl_sg_entry *sge; MPASS(entries > 0); MPASS(sgl); /* * The DDP page size is unrelated to the VM page size. We combine * contiguous physical pages into larger segments to get the best DDP * page size possible. This is the largest of the four sizes in * A_ULP_RX_ISCSI_PSZ that evenly divides the HCF of the segment sizes * in the page list. */ hcf = 0; for (i = entries - 1; i >= 0; i--) { sge = sgl + i; buf = (vm_offset_t)sge->addr; len = sge->len; start_pva = trunc_page(buf); end_pva = trunc_page(buf + len - 1); pva = start_pva; while (pva <= end_pva) { seglen = PAGE_SIZE; p1 = pmap_kextract(pva); pva += PAGE_SIZE; while (pva <= end_pva && p1 + seglen == pmap_kextract(pva)) { seglen += PAGE_SIZE; pva += PAGE_SIZE; } hcf = calculate_hcf(hcf, seglen); if (hcf < (1 << pr->pr_page_shift[1])) { idx = 0; goto have_pgsz; /* give up, short circuit */ } } } #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1) MPASS((hcf & PR_PAGE_MASK(0)) == 0); /* PAGE_SIZE is >= 4K everywhere */ for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) { if ((hcf & PR_PAGE_MASK(idx)) == 0) break; } #undef PR_PAGE_MASK have_pgsz: MPASS(idx <= M_PPOD_PGSZ); npages = 0; while (entries--) { npages++; start_pva = trunc_page((vm_offset_t)sgl->addr); end_pva = trunc_page((vm_offset_t)sgl->addr + sgl->len - 1); npages += (end_pva - start_pva) >> pr->pr_page_shift[idx]; sgl = sgl + 1; } nppods = howmany(npages, PPOD_PAGES); if (alloc_page_pods(pr, nppods, idx, prsv) != 0) return (ENOMEM); MPASS(prsv->prsv_nppods > 0); return (0); } void t4_free_page_pods(struct ppod_reservation *prsv) { struct ppod_region *pr = prsv->prsv_pr; vmem_addr_t addr; MPASS(prsv != NULL); MPASS(prsv->prsv_nppods != 0); addr = prsv->prsv_tag & pr->pr_tag_mask; MPASS((addr & pr->pr_invalid_bit) == 0); CTR4(KTR_CXGBE, "%-17s arena %p, addr 0x%08x, nppods %d", __func__, pr->pr_arena, addr, prsv->prsv_nppods); vmem_free(pr->pr_arena, addr, PPOD_SZ(prsv->prsv_nppods)); prsv->prsv_nppods = 0; } #define NUM_ULP_TX_SC_IMM_PPODS (256 / PPOD_SIZE) int t4_write_page_pods_for_ps(struct adapter *sc, struct sge_wrq *wrq, int tid, struct pageset *ps) { struct wrqe *wr; struct ulp_mem_io *ulpmc; struct ulptx_idata *ulpsc; struct pagepod *ppod; int i, j, k, n, chunk, len, ddp_pgsz, idx; u_int ppod_addr; uint32_t cmd; struct ppod_reservation *prsv = &ps->prsv; struct ppod_region *pr = prsv->prsv_pr; KASSERT(!(ps->flags & PS_PPODS_WRITTEN), ("%s: page pods already written", __func__)); MPASS(prsv->prsv_nppods > 0); cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); if (is_t4(sc)) cmd |= htobe32(F_ULP_MEMIO_ORDER); else cmd |= htobe32(F_T5_ULP_MEMIO_IMM); ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)]; ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask); for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) { /* How many page pods are we writing in this cycle */ n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS); chunk = PPOD_SZ(n); len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16); wr = alloc_wrqe(len, wrq); if (wr == NULL) return (ENOMEM); /* ok to just bail out */ ulpmc = wrtod(wr); INIT_ULPTX_WR(ulpmc, len, 0, 0); ulpmc->cmd = cmd; ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(chunk); ppod = (struct pagepod *)(ulpsc + 1); for (j = 0; j < n; i++, j++, ppod++) { ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID | V_PPOD_TID(tid) | prsv->prsv_tag); ppod->len_offset = htobe64(V_PPOD_LEN(ps->len) | V_PPOD_OFST(ps->offset)); ppod->rsvd = 0; idx = i * PPOD_PAGES * (ddp_pgsz / PAGE_SIZE); for (k = 0; k < nitems(ppod->addr); k++) { if (idx < ps->npages) { ppod->addr[k] = htobe64(ps->pages[idx]->phys_addr); idx += ddp_pgsz / PAGE_SIZE; } else ppod->addr[k] = 0; #if 0 CTR5(KTR_CXGBE, "%s: tid %d ppod[%d]->addr[%d] = %p", __func__, toep->tid, i, k, htobe64(ppod->addr[k])); #endif } } t4_wrq_tx(sc, wr); } ps->flags |= PS_PPODS_WRITTEN; return (0); } static struct mbuf * alloc_raw_wr_mbuf(int len) { struct mbuf *m; if (len <= MHLEN) m = m_gethdr(M_NOWAIT, MT_DATA); else if (len <= MCLBYTES) m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); else m = NULL; if (m == NULL) return (NULL); m->m_pkthdr.len = len; m->m_len = len; set_mbuf_raw_wr(m, true); return (m); } int t4_write_page_pods_for_buf(struct adapter *sc, struct toepcb *toep, struct ppod_reservation *prsv, vm_offset_t buf, int buflen, struct mbufq *wrq) { struct ulp_mem_io *ulpmc; struct ulptx_idata *ulpsc; struct pagepod *ppod; int i, j, k, n, chunk, len, ddp_pgsz; u_int ppod_addr, offset; uint32_t cmd; struct ppod_region *pr = prsv->prsv_pr; uintptr_t end_pva, pva, pa; struct mbuf *m; cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); if (is_t4(sc)) cmd |= htobe32(F_ULP_MEMIO_ORDER); else cmd |= htobe32(F_T5_ULP_MEMIO_IMM); ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)]; offset = buf & PAGE_MASK; ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask); pva = trunc_page(buf); end_pva = trunc_page(buf + buflen - 1); for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) { /* How many page pods are we writing in this cycle */ n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS); MPASS(n > 0); chunk = PPOD_SZ(n); len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16); m = alloc_raw_wr_mbuf(len); if (m == NULL) return (ENOMEM); ulpmc = mtod(m, struct ulp_mem_io *); INIT_ULPTX_WR(ulpmc, len, 0, toep->tid); ulpmc->cmd = cmd; ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(chunk); ppod = (struct pagepod *)(ulpsc + 1); for (j = 0; j < n; i++, j++, ppod++) { ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID | V_PPOD_TID(toep->tid) | (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ))); ppod->len_offset = htobe64(V_PPOD_LEN(buflen) | V_PPOD_OFST(offset)); ppod->rsvd = 0; for (k = 0; k < nitems(ppod->addr); k++) { if (pva > end_pva) ppod->addr[k] = 0; else { pa = pmap_kextract(pva); ppod->addr[k] = htobe64(pa); pva += ddp_pgsz; } #if 0 CTR5(KTR_CXGBE, "%s: tid %d ppod[%d]->addr[%d] = %p", __func__, toep->tid, i, k, htobe64(ppod->addr[k])); #endif } /* * Walk back 1 segment so that the first address in the * next pod is the same as the last one in the current * pod. */ pva -= ddp_pgsz; } mbufq_enqueue(wrq, m); } MPASS(pva <= end_pva); return (0); } int t4_write_page_pods_for_sgl(struct adapter *sc, struct toepcb *toep, struct ppod_reservation *prsv, struct ctl_sg_entry *sgl, int entries, int xferlen, struct mbufq *wrq) { struct ulp_mem_io *ulpmc; struct ulptx_idata *ulpsc; struct pagepod *ppod; int i, j, k, n, chunk, len, ddp_pgsz; u_int ppod_addr, offset, sg_offset = 0; uint32_t cmd; struct ppod_region *pr = prsv->prsv_pr; uintptr_t pva, pa; struct mbuf *m; MPASS(sgl != NULL); MPASS(entries > 0); cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); if (is_t4(sc)) cmd |= htobe32(F_ULP_MEMIO_ORDER); else cmd |= htobe32(F_T5_ULP_MEMIO_IMM); ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)]; offset = (vm_offset_t)sgl->addr & PAGE_MASK; ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask); pva = trunc_page((vm_offset_t)sgl->addr); for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) { /* How many page pods are we writing in this cycle */ n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS); MPASS(n > 0); chunk = PPOD_SZ(n); len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16); m = alloc_raw_wr_mbuf(len); if (m == NULL) return (ENOMEM); ulpmc = mtod(m, struct ulp_mem_io *); INIT_ULPTX_WR(ulpmc, len, 0, toep->tid); ulpmc->cmd = cmd; ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(chunk); ppod = (struct pagepod *)(ulpsc + 1); for (j = 0; j < n; i++, j++, ppod++) { ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID | V_PPOD_TID(toep->tid) | (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ))); ppod->len_offset = htobe64(V_PPOD_LEN(xferlen) | V_PPOD_OFST(offset)); ppod->rsvd = 0; for (k = 0; k < nitems(ppod->addr); k++) { if (entries != 0) { pa = pmap_kextract(pva + sg_offset); ppod->addr[k] = htobe64(pa); } else ppod->addr[k] = 0; #if 0 CTR5(KTR_CXGBE, "%s: tid %d ppod[%d]->addr[%d] = %p", __func__, toep->tid, i, k, htobe64(ppod->addr[k])); #endif /* * If this is the last entry in a pod, * reuse the same entry for first address * in the next pod. */ if (k + 1 == nitems(ppod->addr)) break; /* * Don't move to the next DDP page if the * sgl is already finished. */ if (entries == 0) continue; sg_offset += ddp_pgsz; if (sg_offset == sgl->len) { /* * This sgl entry is done. Go * to the next. */ entries--; sgl++; sg_offset = 0; if (entries != 0) pva = trunc_page( (vm_offset_t)sgl->addr); } } } mbufq_enqueue(wrq, m); } return (0); } /* * Prepare a pageset for DDP. This sets up page pods. */ static int prep_pageset(struct adapter *sc, struct toepcb *toep, struct pageset *ps) { struct tom_data *td = sc->tom_softc; if (ps->prsv.prsv_nppods == 0 && !t4_alloc_page_pods_for_ps(&td->pr, ps)) { return (0); } if (!(ps->flags & PS_PPODS_WRITTEN) && t4_write_page_pods_for_ps(sc, toep->ctrlq, toep->tid, ps) != 0) { return (0); } return (1); } int t4_init_ppod_region(struct ppod_region *pr, struct t4_range *r, u_int psz, const char *name) { int i; MPASS(pr != NULL); MPASS(r->size > 0); pr->pr_start = r->start; pr->pr_len = r->size; pr->pr_page_shift[0] = 12 + G_HPZ0(psz); pr->pr_page_shift[1] = 12 + G_HPZ1(psz); pr->pr_page_shift[2] = 12 + G_HPZ2(psz); pr->pr_page_shift[3] = 12 + G_HPZ3(psz); /* The SGL -> page pod algorithm requires the sizes to be in order. */ for (i = 1; i < nitems(pr->pr_page_shift); i++) { if (pr->pr_page_shift[i] <= pr->pr_page_shift[i - 1]) return (ENXIO); } pr->pr_tag_mask = ((1 << fls(r->size)) - 1) & V_PPOD_TAG(M_PPOD_TAG); pr->pr_alias_mask = V_PPOD_TAG(M_PPOD_TAG) & ~pr->pr_tag_mask; if (pr->pr_tag_mask == 0 || pr->pr_alias_mask == 0) return (ENXIO); pr->pr_alias_shift = fls(pr->pr_tag_mask); pr->pr_invalid_bit = 1 << (pr->pr_alias_shift - 1); pr->pr_arena = vmem_create(name, 0, pr->pr_len, PPOD_SIZE, 0, M_FIRSTFIT | M_NOWAIT); if (pr->pr_arena == NULL) return (ENOMEM); return (0); } void t4_free_ppod_region(struct ppod_region *pr) { MPASS(pr != NULL); if (pr->pr_arena) vmem_destroy(pr->pr_arena); bzero(pr, sizeof(*pr)); } static int pscmp(struct pageset *ps, struct vmspace *vm, vm_offset_t start, int npages, int pgoff, int len) { if (ps->start != start || ps->npages != npages || ps->offset != pgoff || ps->len != len) return (1); return (ps->vm != vm || ps->vm_timestamp != vm->vm_map.timestamp); } static int hold_aio(struct toepcb *toep, struct kaiocb *job, struct pageset **pps) { struct vmspace *vm; vm_map_t map; vm_offset_t start, end, pgoff; struct pageset *ps; int n; DDP_ASSERT_LOCKED(toep); /* * The AIO subsystem will cancel and drain all requests before * permitting a process to exit or exec, so p_vmspace should * be stable here. */ vm = job->userproc->p_vmspace; map = &vm->vm_map; start = (uintptr_t)job->uaiocb.aio_buf; pgoff = start & PAGE_MASK; end = round_page(start + job->uaiocb.aio_nbytes); start = trunc_page(start); if (end - start > MAX_DDP_BUFFER_SIZE) { /* * Truncate the request to a short read. * Alternatively, we could DDP in chunks to the larger * buffer, but that would be quite a bit more work. * * When truncating, round the request down to avoid * crossing a cache line on the final transaction. */ end = rounddown2(start + MAX_DDP_BUFFER_SIZE, CACHE_LINE_SIZE); #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, "%s: tid %d, truncating size from %lu to %lu", __func__, toep->tid, (unsigned long)job->uaiocb.aio_nbytes, (unsigned long)(end - (start + pgoff))); job->uaiocb.aio_nbytes = end - (start + pgoff); #endif end = round_page(end); } n = atop(end - start); /* * Try to reuse a cached pageset. */ TAILQ_FOREACH(ps, &toep->ddp.cached_pagesets, link) { if (pscmp(ps, vm, start, n, pgoff, job->uaiocb.aio_nbytes) == 0) { TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link); toep->ddp.cached_count--; *pps = ps; return (0); } } /* * If there are too many cached pagesets to create a new one, * free a pageset before creating a new one. */ KASSERT(toep->ddp.active_count + toep->ddp.cached_count <= nitems(toep->ddp.db), ("%s: too many wired pagesets", __func__)); if (toep->ddp.active_count + toep->ddp.cached_count == nitems(toep->ddp.db)) { KASSERT(toep->ddp.cached_count > 0, ("no cached pageset to free")); ps = TAILQ_LAST(&toep->ddp.cached_pagesets, pagesetq); TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link); toep->ddp.cached_count--; free_pageset(toep->td, ps); } DDP_UNLOCK(toep); /* Create a new pageset. */ ps = malloc(sizeof(*ps) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK | M_ZERO); ps->pages = (vm_page_t *)(ps + 1); ps->vm_timestamp = map->timestamp; ps->npages = vm_fault_quick_hold_pages(map, start, end - start, VM_PROT_WRITE, ps->pages, n); DDP_LOCK(toep); if (ps->npages < 0) { free(ps, M_CXGBE); return (EFAULT); } KASSERT(ps->npages == n, ("hold_aio: page count mismatch: %d vs %d", ps->npages, n)); ps->offset = pgoff; ps->len = job->uaiocb.aio_nbytes; refcount_acquire(&vm->vm_refcnt); ps->vm = vm; ps->start = start; CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d", __func__, toep->tid, ps, job, ps->npages); *pps = ps; return (0); } static void ddp_complete_all(struct toepcb *toep, int error) { struct kaiocb *job; DDP_ASSERT_LOCKED(toep); while (!TAILQ_EMPTY(&toep->ddp.aiojobq)) { job = TAILQ_FIRST(&toep->ddp.aiojobq); TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count--; if (aio_clear_cancel_function(job)) ddp_complete_one(job, error); } } static void aio_ddp_cancel_one(struct kaiocb *job) { long copied; /* * If this job had copied data out of the socket buffer before * it was cancelled, report it as a short read rather than an * error. */ copied = job->aio_received; if (copied != 0) aio_complete(job, copied, 0); else aio_cancel(job); } /* * Called when the main loop wants to requeue a job to retry it later. * Deals with the race of the job being cancelled while it was being * examined. */ static void aio_ddp_requeue_one(struct toepcb *toep, struct kaiocb *job) { DDP_ASSERT_LOCKED(toep); if (!(toep->ddp.flags & DDP_DEAD) && aio_set_cancel_function(job, t4_aio_cancel_queued)) { TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count++; } else aio_ddp_cancel_one(job); } static void aio_ddp_requeue(struct toepcb *toep) { struct adapter *sc = td_adapter(toep->td); struct socket *so; struct sockbuf *sb; struct inpcb *inp; struct kaiocb *job; struct ddp_buffer *db; size_t copied, offset, resid; struct pageset *ps; struct mbuf *m; uint64_t ddp_flags, ddp_flags_mask; struct wrqe *wr; int buf_flag, db_idx, error; DDP_ASSERT_LOCKED(toep); restart: if (toep->ddp.flags & DDP_DEAD) { MPASS(toep->ddp.waiting_count == 0); MPASS(toep->ddp.active_count == 0); return; } if (toep->ddp.waiting_count == 0 || toep->ddp.active_count == nitems(toep->ddp.db)) { return; } job = TAILQ_FIRST(&toep->ddp.aiojobq); so = job->fd_file->f_data; sb = &so->so_rcv; SOCKBUF_LOCK(sb); /* We will never get anything unless we are or were connected. */ if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { SOCKBUF_UNLOCK(sb); ddp_complete_all(toep, ENOTCONN); return; } KASSERT(toep->ddp.active_count == 0 || sbavail(sb) == 0, ("%s: pending sockbuf data and DDP is active", __func__)); /* Abort if socket has reported problems. */ /* XXX: Wait for any queued DDP's to finish and/or flush them? */ if (so->so_error && sbavail(sb) == 0) { toep->ddp.waiting_count--; TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); if (!aio_clear_cancel_function(job)) { SOCKBUF_UNLOCK(sb); goto restart; } /* * If this job has previously copied some data, report * a short read and leave the error to be reported by * a future request. */ copied = job->aio_received; if (copied != 0) { SOCKBUF_UNLOCK(sb); aio_complete(job, copied, 0); goto restart; } error = so->so_error; so->so_error = 0; SOCKBUF_UNLOCK(sb); aio_complete(job, -1, error); goto restart; } /* * Door is closed. If there is pending data in the socket buffer, * deliver it. If there are pending DDP requests, wait for those * to complete. Once they have completed, return EOF reads. */ if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) { SOCKBUF_UNLOCK(sb); if (toep->ddp.active_count != 0) return; ddp_complete_all(toep, 0); return; } /* * If DDP is not enabled and there is no pending socket buffer * data, try to enable DDP. */ if (sbavail(sb) == 0 && (toep->ddp.flags & DDP_ON) == 0) { SOCKBUF_UNLOCK(sb); /* * Wait for the card to ACK that DDP is enabled before * queueing any buffers. Currently this waits for an * indicate to arrive. This could use a TCB_SET_FIELD_RPL * message to know that DDP was enabled instead of waiting * for the indicate which would avoid copying the indicate * if no data is pending. * * XXX: Might want to limit the indicate size to the size * of the first queued request. */ if ((toep->ddp.flags & DDP_SC_REQ) == 0) enable_ddp(sc, toep); return; } SOCKBUF_UNLOCK(sb); /* * If another thread is queueing a buffer for DDP, let it * drain any work and return. */ if (toep->ddp.queueing != NULL) return; /* Take the next job to prep it for DDP. */ toep->ddp.waiting_count--; TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); if (!aio_clear_cancel_function(job)) goto restart; toep->ddp.queueing = job; /* NB: This drops DDP_LOCK while it holds the backing VM pages. */ error = hold_aio(toep, job, &ps); if (error != 0) { ddp_complete_one(job, error); toep->ddp.queueing = NULL; goto restart; } SOCKBUF_LOCK(sb); if (so->so_error && sbavail(sb) == 0) { copied = job->aio_received; if (copied != 0) { SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); aio_complete(job, copied, 0); toep->ddp.queueing = NULL; goto restart; } error = so->so_error; so->so_error = 0; SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); aio_complete(job, -1, error); toep->ddp.queueing = NULL; goto restart; } if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) { SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); if (toep->ddp.active_count != 0) { /* * The door is closed, but there are still pending * DDP buffers. Requeue. These jobs will all be * completed once those buffers drain. */ aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; return; } ddp_complete_one(job, 0); ddp_complete_all(toep, 0); toep->ddp.queueing = NULL; return; } sbcopy: /* * If the toep is dead, there shouldn't be any data in the socket * buffer, so the above case should have handled this. */ MPASS(!(toep->ddp.flags & DDP_DEAD)); /* * If there is pending data in the socket buffer (either * from before the requests were queued or a DDP indicate), * copy those mbufs out directly. */ copied = 0; offset = ps->offset + job->aio_received; MPASS(job->aio_received <= job->uaiocb.aio_nbytes); resid = job->uaiocb.aio_nbytes - job->aio_received; m = sb->sb_mb; KASSERT(m == NULL || toep->ddp.active_count == 0, ("%s: sockbuf data with active DDP", __func__)); while (m != NULL && resid > 0) { struct iovec iov[1]; struct uio uio; int error; iov[0].iov_base = mtod(m, void *); iov[0].iov_len = m->m_len; if (iov[0].iov_len > resid) iov[0].iov_len = resid; uio.uio_iov = iov; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_resid = iov[0].iov_len; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_WRITE; error = uiomove_fromphys(ps->pages, offset + copied, uio.uio_resid, &uio); MPASS(error == 0 && uio.uio_resid == 0); copied += uio.uio_offset; resid -= uio.uio_offset; m = m->m_next; } if (copied != 0) { sbdrop_locked(sb, copied); job->aio_received += copied; job->msgrcv = 1; copied = job->aio_received; inp = sotoinpcb(so); if (!INP_TRY_WLOCK(inp)) { /* * The reference on the socket file descriptor in * the AIO job should keep 'sb' and 'inp' stable. * Our caller has a reference on the 'toep' that * keeps it stable. */ SOCKBUF_UNLOCK(sb); DDP_UNLOCK(toep); INP_WLOCK(inp); DDP_LOCK(toep); SOCKBUF_LOCK(sb); /* * If the socket has been closed, we should detect * that and complete this request if needed on * the next trip around the loop. */ } t4_rcvd_locked(&toep->td->tod, intotcpcb(inp)); INP_WUNLOCK(inp); if (resid == 0 || toep->ddp.flags & DDP_DEAD) { /* * We filled the entire buffer with socket * data, DDP is not being used, or the socket * is being shut down, so complete the * request. */ SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); aio_complete(job, copied, 0); toep->ddp.queueing = NULL; goto restart; } /* * If DDP is not enabled, requeue this request and restart. * This will either enable DDP or wait for more data to * arrive on the socket buffer. */ if ((toep->ddp.flags & (DDP_ON | DDP_SC_REQ)) != DDP_ON) { SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; goto restart; } /* * An indicate might have arrived and been added to * the socket buffer while it was unlocked after the * copy to lock the INP. If so, restart the copy. */ if (sbavail(sb) != 0) goto sbcopy; } SOCKBUF_UNLOCK(sb); if (prep_pageset(sc, toep, ps) == 0) { recycle_pageset(toep, ps); aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; /* * XXX: Need to retry this later. Mostly need a trigger * when page pods are freed up. */ printf("%s: prep_pageset failed\n", __func__); return; } /* Determine which DDP buffer to use. */ if (toep->ddp.db[0].job == NULL) { db_idx = 0; } else { MPASS(toep->ddp.db[1].job == NULL); db_idx = 1; } ddp_flags = 0; ddp_flags_mask = 0; if (db_idx == 0) { ddp_flags |= V_TF_DDP_BUF0_VALID(1); if (so->so_state & SS_NBIO) ddp_flags |= V_TF_DDP_BUF0_FLUSH(1); ddp_flags_mask |= V_TF_DDP_PSH_NO_INVALIDATE0(1) | V_TF_DDP_PUSH_DISABLE_0(1) | V_TF_DDP_PSHF_ENABLE_0(1) | V_TF_DDP_BUF0_FLUSH(1) | V_TF_DDP_BUF0_VALID(1); buf_flag = DDP_BUF0_ACTIVE; } else { ddp_flags |= V_TF_DDP_BUF1_VALID(1); if (so->so_state & SS_NBIO) ddp_flags |= V_TF_DDP_BUF1_FLUSH(1); ddp_flags_mask |= V_TF_DDP_PSH_NO_INVALIDATE1(1) | V_TF_DDP_PUSH_DISABLE_1(1) | V_TF_DDP_PSHF_ENABLE_1(1) | V_TF_DDP_BUF1_FLUSH(1) | V_TF_DDP_BUF1_VALID(1); buf_flag = DDP_BUF1_ACTIVE; } MPASS((toep->ddp.flags & buf_flag) == 0); if ((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0) { MPASS(db_idx == 0); MPASS(toep->ddp.active_id == -1); MPASS(toep->ddp.active_count == 0); ddp_flags_mask |= V_TF_DDP_ACTIVE_BUF(1); } /* * The TID for this connection should still be valid. If DDP_DEAD * is set, SBS_CANTRCVMORE should be set, so we shouldn't be * this far anyway. Even if the socket is closing on the other * end, the AIO job holds a reference on this end of the socket * which will keep it open and keep the TCP PCB attached until * after the job is completed. */ wr = mk_update_tcb_for_ddp(sc, toep, db_idx, ps, job->aio_received, ddp_flags, ddp_flags_mask); if (wr == NULL) { recycle_pageset(toep, ps); aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; /* * XXX: Need a way to kick a retry here. * * XXX: We know the fixed size needed and could * preallocate this using a blocking request at the * start of the task to avoid having to handle this * edge case. */ printf("%s: mk_update_tcb_for_ddp failed\n", __func__); return; } if (!aio_set_cancel_function(job, t4_aio_cancel_active)) { free_wrqe(wr); recycle_pageset(toep, ps); aio_ddp_cancel_one(job); toep->ddp.queueing = NULL; goto restart; } #ifdef VERBOSE_TRACES CTR6(KTR_CXGBE, "%s: tid %u, scheduling %p for DDP[%d] (flags %#lx/%#lx)", __func__, toep->tid, job, db_idx, ddp_flags, ddp_flags_mask); #endif /* Give the chip the go-ahead. */ t4_wrq_tx(sc, wr); db = &toep->ddp.db[db_idx]; db->cancel_pending = 0; db->job = job; db->ps = ps; toep->ddp.queueing = NULL; toep->ddp.flags |= buf_flag; toep->ddp.active_count++; if (toep->ddp.active_count == 1) { MPASS(toep->ddp.active_id == -1); toep->ddp.active_id = db_idx; CTR2(KTR_CXGBE, "%s: ddp_active_id = %d", __func__, toep->ddp.active_id); } goto restart; } void ddp_queue_toep(struct toepcb *toep) { DDP_ASSERT_LOCKED(toep); if (toep->ddp.flags & DDP_TASK_ACTIVE) return; toep->ddp.flags |= DDP_TASK_ACTIVE; hold_toepcb(toep); soaio_enqueue(&toep->ddp.requeue_task); } static void aio_ddp_requeue_task(void *context, int pending) { struct toepcb *toep = context; DDP_LOCK(toep); aio_ddp_requeue(toep); toep->ddp.flags &= ~DDP_TASK_ACTIVE; DDP_UNLOCK(toep); free_toepcb(toep); } static void t4_aio_cancel_active(struct kaiocb *job) { struct socket *so = job->fd_file->f_data; struct tcpcb *tp = so_sototcpcb(so); struct toepcb *toep = tp->t_toe; struct adapter *sc = td_adapter(toep->td); uint64_t valid_flag; int i; DDP_LOCK(toep); if (aio_cancel_cleared(job)) { DDP_UNLOCK(toep); aio_ddp_cancel_one(job); return; } for (i = 0; i < nitems(toep->ddp.db); i++) { if (toep->ddp.db[i].job == job) { /* Should only ever get one cancel request for a job. */ MPASS(toep->ddp.db[i].cancel_pending == 0); /* * Invalidate this buffer. It will be * cancelled or partially completed once the * card ACKs the invalidate. */ valid_flag = i == 0 ? V_TF_DDP_BUF0_VALID(1) : V_TF_DDP_BUF1_VALID(1); t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_RX_DDP_FLAGS, valid_flag, 0, 1, CPL_COOKIE_DDP0 + i); toep->ddp.db[i].cancel_pending = 1; CTR2(KTR_CXGBE, "%s: request %p marked pending", __func__, job); break; } } DDP_UNLOCK(toep); } static void t4_aio_cancel_queued(struct kaiocb *job) { struct socket *so = job->fd_file->f_data; struct tcpcb *tp = so_sototcpcb(so); struct toepcb *toep = tp->t_toe; DDP_LOCK(toep); if (!aio_cancel_cleared(job)) { TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count--; if (toep->ddp.waiting_count == 0) ddp_queue_toep(toep); } CTR2(KTR_CXGBE, "%s: request %p cancelled", __func__, job); DDP_UNLOCK(toep); aio_ddp_cancel_one(job); } int t4_aio_queue_ddp(struct socket *so, struct kaiocb *job) { struct tcpcb *tp = so_sototcpcb(so); struct toepcb *toep = tp->t_toe; /* Ignore writes. */ if (job->uaiocb.aio_lio_opcode != LIO_READ) return (EOPNOTSUPP); DDP_LOCK(toep); /* * XXX: Think about possibly returning errors for ENOTCONN, * etc. Perhaps the caller would only queue the request * if it failed with EOPNOTSUPP? */ #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid); #endif if (!aio_set_cancel_function(job, t4_aio_cancel_queued)) panic("new job was cancelled"); TAILQ_INSERT_TAIL(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count++; toep->ddp.flags |= DDP_OK; /* * Try to handle this request synchronously. If this has * to block because the task is running, it will just bail * and let the task handle it instead. */ aio_ddp_requeue(toep); DDP_UNLOCK(toep); return (0); } void t4_ddp_mod_load(void) { t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl, CPL_COOKIE_DDP0); t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl, CPL_COOKIE_DDP1); t4_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp); t4_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete); TAILQ_INIT(&ddp_orphan_pagesets); mtx_init(&ddp_orphan_pagesets_lock, "ddp orphans", NULL, MTX_DEF); TASK_INIT(&ddp_orphan_task, 0, ddp_free_orphan_pagesets, NULL); } void t4_ddp_mod_unload(void) { taskqueue_drain(taskqueue_thread, &ddp_orphan_task); MPASS(TAILQ_EMPTY(&ddp_orphan_pagesets)); mtx_destroy(&ddp_orphan_pagesets_lock); t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, NULL, CPL_COOKIE_DDP0); t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, NULL, CPL_COOKIE_DDP1); t4_register_cpl_handler(CPL_RX_DATA_DDP, NULL); t4_register_cpl_handler(CPL_RX_DDP_COMPLETE, NULL); } #endif